From b1be0d0a1116a7a7ce667737c3cc42fab96cfd98 Mon Sep 17 00:00:00 2001 From: luoxiadesu Date: Mon, 2 Nov 2026 21:02:30 +0900 Subject: [PATCH 1/2] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=20GUI=20=E5=AE=89?= =?UTF-8?q?=E8=A3=85/=E6=89=93=E5=8C=85=E3=80=81rla2json=20=E8=AF=BB?= =?UTF-8?q?=E5=8F=96=20archive=E3=80=81RLA=20=E9=9D=9E=E4=B8=A5=E6=A0=BC?= =?UTF-8?q?=E5=AE=B9=E9=94=99=E4=B8=8E=E5=87=AD=E6=8D=AE=E6=B3=84=E6=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GUI 与打包 - 移除 `gui` extra：GooeyEx>=0.1.1 未发布到 PyPI，0.1.0 在无 wx.App 时导入即崩；改用从上游 git 手动安装的指引。 - 移除未被引用且不可安装的 `criware` extra。 - `sssekai-gui` 缺依赖时打印可执行指引并退出，不再抛 ImportError 堆栈。 - 发布工作流删除必然失败的 GUI 构建步骤，改为本地构建说明。 - 删除指向不存在目录的 .vscode 调试配置。 RLA 解析 - rla2json: RLH 头是 bytes，恢复 json.loads 后再取字段。 - read_rla_frames: 非严格模式下解析失败改为跳过（修复 UnboundLocalError）。 - split 包的长度不符、base64 损坏在非严格模式下跳过，并先清理缓冲组。 - 截断 payload 抛 StreamingDataTruncatedException，由调用方丢弃整帧， 不再静默返回短数组。 - 解码错误收窄为白名单（含 zlib.error/gzip.BadGzipFile）， TypeError 等实现 bug 仍会抛出。 凭据脱敏 - 新增 RedactedStr/RedactedHeaders，repr、asdict(deepcopy) 与日志不再 输出 auth_credential、X-Session-Token。 - load() 后重新包装磁盘读出的明文；兼容旧缓存，pickle 仍写普通 str/dict。 - 旧缓存中 dict 形态的 sekai_user_data 归一化为 SekaiUserData， 修复 SEKAI_CREDENTIAL 的 AttributeError 与 repr 泄漏。 - auth.py 存入 dataclass 而非原始响应 dict。 CLI 与文档 - abserve 默认监听 127.0.0.1。 - apphash 示例改用正确的 row_ab_version 关键字。 - abcache 帮助文本不再指向已下线的 Wiki。 测试 - 新增 22 个离线回归测试，覆盖 RLA 容错全部分支、脱敏路径与旧缓存兼容。 Co-Authored-By: Claude Code --- .github/workflows/python-publish.yml | 197 ++- .vscode/launch.json | 923 +++++++-------- README.md | 85 +- setup.py | 99 +- sssekai/__gui__.py | 148 +-- sssekai/__main__.py | 1078 ++++++++--------- sssekai/abcache/__init__.py | 1646 ++++++++++++++------------ sssekai/abcache/auth.py | 60 +- sssekai/entrypoint/apphash.py | 552 ++++----- sssekai/entrypoint/rla2json.py | 159 +-- sssekai/fmt/rla.py | 86 +- tests/test_rla2json.py | 64 + tests/test_rla_error_handling.py | 186 +++ tests/test_security.py | 181 +++ 14 files changed, 3016 insertions(+), 2448 deletions(-) create mode 100644 tests/test_rla2json.py create mode 100644 tests/test_rla_error_handling.py create mode 100644 tests/test_security.py diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 1975ed1..e4ffd71 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -1,102 +1,95 @@ -name: Publish - -on: - workflow_dispatch: - inputs: - release_pypi: - description: 'Publish to PyPI' - type: boolean - default: true - release_github: - description: 'Build and release Windows executables' - type: boolean - default: false - release_gui: - description: 'Include Windows GUI executable (requires GooeyEx)' - type: boolean - default: false - -permissions: - contents: read - -env: - PYTHONUTF8: '1' - -concurrency: - group: publish - cancel-in-progress: false - -jobs: - build: - if: inputs.release_pypi || inputs.release_github - runs-on: windows-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Install build and test dependencies - run: python -m pip install build twine pytest - - name: Build distributions - run: python -m build - - name: Check distributions - run: python -m twine check --strict dist/* - - name: Install wheel - shell: pwsh - run: python -m pip install (Get-Item dist/*.whl).FullName - - name: Test - run: python -m pytest -v - - name: Check installed command - working-directory: ${{ runner.temp }} - run: sssekai --help - - uses: actions/upload-artifact@v4 - with: - name: python-distributions - path: dist/ - if-no-files-found: error - - publish-pypi: - if: inputs.release_pypi - needs: build - runs-on: ubuntu-latest - environment: - name: pypi - url: https://pypi.org/project/sssekai/ - permissions: - id-token: write - steps: - - uses: actions/download-artifact@v4 - with: - name: python-distributions - path: dist/ - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - - release-windows: - if: inputs.release_github - needs: build - runs-on: windows-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Install dependencies - run: python -m pip install pyinstaller ".[il2cpp]" - - name: Install GUI dependencies - if: inputs.release_gui - run: python -m pip install ".[gui]" - - name: Build Windows executables - run: pyinstaller sssekai.spec - - name: Build Windows GUI executable - if: inputs.release_gui - run: pyinstaller sssekai-gui.spec - - name: Create GitHub release - shell: pwsh - env: - GH_TOKEN: ${{ github.token }} - run: | - $version = python -c "import sssekai; print(sssekai.__version__)" - gh release create $version (Get-Item dist/*.exe).FullName --target "${{ github.sha }}" --title "Version $version" --generate-notes +name: Publish + +on: + workflow_dispatch: + inputs: + release_pypi: + description: 'Publish to PyPI' + type: boolean + default: true + release_github: + description: 'Build and release Windows executables' + type: boolean + default: false + +permissions: + contents: read + +env: + PYTHONUTF8: '1' + +concurrency: + group: publish + cancel-in-progress: false + +jobs: + build: + if: inputs.release_pypi || inputs.release_github + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install build and test dependencies + run: python -m pip install build twine pytest + - name: Build distributions + run: python -m build + - name: Check distributions + run: python -m twine check --strict dist/* + - name: Install wheel + shell: pwsh + run: python -m pip install (Get-Item dist/*.whl).FullName + - name: Test + run: python -m pytest -v + - name: Check installed command + working-directory: ${{ runner.temp }} + run: sssekai --help + - uses: actions/upload-artifact@v4 + with: + name: python-distributions + path: dist/ + if-no-files-found: error + + publish-pypi: + if: inputs.release_pypi + needs: build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/sssekai/ + permissions: + id-token: write + steps: + - uses: actions/download-artifact@v4 + with: + name: python-distributions + path: dist/ + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + release-windows: + if: inputs.release_github + needs: build + runs-on: windows-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install dependencies + run: python -m pip install pyinstaller ".[il2cpp]" + # NOTE: The GUI executable is not built here: its dependency GooeyEx is not + # published on public PyPI. Build it locally with `pyinstaller sssekai-gui.spec` + # from an environment where GooeyEx is installed. + - name: Build Windows executables + run: pyinstaller sssekai.spec + - name: Create GitHub release + shell: pwsh + env: + GH_TOKEN: ${{ github.token }} + run: | + $version = python -c "import sssekai; print(sssekai.__version__)" + gh release create $version (Get-Item dist/*.exe).FullName --target "${{ github.sha }}" --title "Version $version" --generate-notes diff --git a/.vscode/launch.json b/.vscode/launch.json index 8e58e2c..0aa433e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,480 +1,445 @@ -{ - "version": "0.2.0", - "configurations": [ - { - "name": "Animation Plot", - "type": "debugpy", - "request": "launch", - "program": "${workspaceFolder}/misc/animation_plot.py", - "justMyCode": false - }, - { - "name": "AbCache Availability Check", - "type": "debugpy", - "request": "launch", - "program": "${workspaceFolder}/misc/abcache_availability_check.py", - "justMyCode": false - }, - { - "name": "GUI Test", - "type": "debugpy", - "request": "launch", - "program": "${workspaceFolder}/sssekai/__gui__.py", - "justMyCode": false - }, - { - "name": "Python: SpineExtract test", - "type": "debugpy", - "request": "launch", - "module": "tests.test_spine", - "justMyCode": false - }, - { - "name": "Python: Live2DExtract test", - "type": "debugpy", - "request": "launch", - "module": "tests.test_live2d", - "justMyCode": false - }, - { - "name": "Python: AppHash test", - "type": "debugpy", - "request": "launch", - "module": "tests.test_apphash", - "justMyCode": false - }, - { - "name": "Python: AbCache test", - "type": "debugpy", - "request": "launch", - "module": "tests.test_abcache", - "justMyCode": false - }, - { - "name": "Python: Legacy Animation format test", - "type": "debugpy", - "request": "launch", - "module": "tests.test_legacy_anim", - "justMyCode": false - }, - { - "name": "Python: Current File", - "type": "python", - "request": "launch", - "program": "${file}", - "console": "integratedTerminal", - "justMyCode": false, - "args": [] - }, - { - "name": "Python: AbCache (JP Prod)", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "abcache", - "--proxy", - "http://127.0.0.1:8080", - "--app-region", - "jp", - "--app-version", - "5.5.1", - "--app-appHash", - "30829b50-b4b7-d39e-1d55-21ba51e0a8bc", - ], - "justMyCode": true - }, - { - "name": "Python: AbCache (JP Staging)", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "abcache", - "--proxy", - "http://127.0.0.1:8080", - "--app-region", - "jp", - "--app-version", - "5.5.0", - "--app-appHash", - "9b75c207-5fa7-a852-71e3-4097eb9b6027", - ], - "justMyCode": true - }, - { - "name": "Python: AbCache (TW)", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "abcache", - "--app-region", - "tw", - "--app-version", - "4.1.1", - "--app-appHash", - "41fd71f2-f715-bc10-5852-0a9d8542f760", - "--app-abVersion", - "4.1.0", - "--download-filter", - ".*/stage_decoration/.*", - "--download-dir", - "~/.sssekai", - ], - "justMyCode": true - }, - { - "name": "Python: AbCache (KR)", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "abcache", - "--app-region", - "kr", - "--app-version", - "3.4.3", - "--app-appHash", - "a3015fe8-785f-27e1-fb8b-546a23c82c1f", - "--app-abVersion", - "3.4.0" - ], - "justMyCode": true - }, - { - "name": "Python: AbCache (EN)", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "abcache", - "--app-region", - "en", - "--app-version", - "3.8.0", - "--app-appHash", - "3272b0b0-f161-448e-8416-1f8bd2c443b3", - "--app-platform", - "ios" - ], - "justMyCode": true - }, - { - "name": "Python: AbCache (CN)", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "abcache", - "--app-region", - "cn", - "--app-version", - "3.4.0", - "--app-appHash", - "a3015fe8-785f-27e1-fb8b-546a23c82c1f" - ], - "justMyCode": true - }, - { - "name": "Python: AbCache (dump Master data)", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "abcache", - "--app-region", - "en", - "--app-version", - "3.2.0", - "--app-appHash", - "1380836d-de9e-49a8-afe5-c52ba589c8c9", - "--dump-master-data", - "~/.sssekai/master_data" - ], - "justMyCode": true - }, - { - "name": "Python: AbServe (FUSE)", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "abserve", - "--fuse", - "~/ProjectSEKAI" - ], - "justMyCode": false - }, - { - "name": "Python: AbServe", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "abserve", - // "--db", - // "/mnt/c/Users/mos9527/.sssekai/abcache.db", - ], - "justMyCode": false - }, - { - "name": "Python: AbCache (DL)", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "abcache", - "--no-update", - "--download-dir", - "~/.sssekai/bundles/", - "--download-filter", - ".*/stage_decoration/0344.*" - ], - "justMyCode": false - }, - { - "name": "Python: MVData", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "mvdata", - "D:\\Reverse\\proseka_reverse\\assets\\live_pv\\mv_data", - "D:\\Reverse\\proseka_reverse\\mvdata.json" - ], - "justMyCode": true - }, - { - "name": "Python: Tests", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/sssekai.test.py", - "cwd": "${workspaceFolder}", - "console": "integratedTerminal", - "justMyCode": false - }, - { - "name": "Python: Convert Sekai Character Mesh", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/sssekai/scripts/blender_mesh_importer.py", - "cwd": "${workspaceFolder}", - "console": "integratedTerminal", - "args": [ - "C:\\Users\\mos9527\\.sssekai\\abcache\\live_pv\\model\\character\\body\\21\\0001\\ladies_s" - ], - "justMyCode": false - }, - { - "name": "Python: Dump Live2D Model Parameter names", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/sssekai/scripts/dump_moc3_assets.py", - "cwd": "${workspaceFolder}", - "console": "integratedTerminal", - "args": [ - "C:\\Users\\mos9527\\.sssekai\\abcache\\live2d\\model" - ], - "justMyCode": false - }, - { - "name": "Python: apphash", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "apphash", - "--apk-src", - "D:\\Reverse\\proseka_reverse\\game_binary\\v550\\550.xapk", - "--deep", - "--format", - "json" - ], - "justMyCode": true - }, - { - "name": "Python: apphash TW w/ metadata", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "apphash", - "--apk-src", - "/Users/mos9527/sekai-apphash/tw/tw.apk" - ], - "justMyCode": true - }, - { - "name": "Python: usmdemux", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "usmdemux", - "D:\\Sekai\\assets\\8015", - "D:\\Sekai\\assets\\8015.usm", - ], - "justMyCode": true - }, - { - "name": "Python: live2dextract", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "live2dextract", - "C:\\Users\\mos9527\\.sssekai\\abcache\\live2d\\model\\21miku_night", - "C:\\Users\\mos9527\\Desktop\\Cache" - ], - "justMyCode": true - }, - { - "name": "Python: live2dextract Generic", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/sssekai/scripts/live2dextract_generic.py", - "args": [ - "C:\\Users\\Huang\\Desktop\\Data", - "C:\\Users\\Huang\\Desktop\\Data_result" - ], - "justMyCode": false - }, - { - "name": "Python: rla2json", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "rla2json", - "/Volumes/mos9527弄丢的盘/Reverse/proseka_reverse/sse_packet_importer/packets/356_5043", - "/Volumes/mos9527弄丢的盘/Reverse/proseka_reverse/sse_packet_importer/exported/356_5043", - "--version", - "1.6", - "--strict" - ], - "justMyCode": true - }, - { - "name": "Python: Test Animation", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/test.animation.py", - "console": "integratedTerminal", - "justMyCode": false, - "args": [] - }, - { - "name": "Python: RLA Test (Single File)", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/sssekai/fmt/rla.py", - "console": "integratedTerminal", - "justMyCode": false, - "args": [ - // "D:\\proseka_reverse\\sse_packet_importer\\packets\\streaming_vbs_1\\sekai_30_00000006.rla.bytes" - "D:\\proseka_reverse\\sse_packet_importer\\packets\\4268\\1728191806276-0.bin" - ] - }, - { - "name": "Python: AbCache (Override EN Asset Version)", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "abcache", - "--app-region", - "en", - // "--app-version", - // "3.6.0", - // "--app-appHash", - // "a3015fe8-785f-27e1-fb8b-546a23c82c1f", - "--app-asset-version", - "3.7.0.0", - "--app-asset-hash", - "d2e5b1bb-d0d4-4535-99ca-bd84ae291a9d" - ], - "justMyCode": true - }, - { - "name": "Python: AbCache (Override JP Asset Version)", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "abcache", - "--app-region", - "jp", - // "--app-version", - // "5.3.0", - // "--app-appHash", - // "2a9bb18f-c309-1993-5b6e-9f25ddc88be3", - "--app-asset-version", - "5.3.0.90", - "--app-asset-hash", - "7f2ebd91-ce39-6ea2-20b2-e05d2872e9e6", - "--app-asset-host", - "cf2d2388" - ], - "justMyCode": true - }, - { - "name": "Python: IL2CPP v381 (Android)", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "il2cpp", - "D:\\Reverse\\proseka_reverse\\game_binary\\v381\\com.sega.pjsekai\\assets\\bin\\Data\\Managed\\Metadata\\global-metadata.dat", - "D:\\Reverse\\proseka_reverse\\game_binary\\v381\\config.arm64_v8a\\lib\\arm64-v8a\\libil2cpp.so", - "D:\\Reverse\\proseka_reverse\\game_binary\\v381\\il2cpp" - ], - "justMyCode": false - }, - { - "name": "Python: IL2CPP v550 (Android)", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "il2cpp", - "D:\\Reverse\\proseka_reverse\\game_binary\\v550\\com.sega.pjsekai\\assets\\bin\\Data\\Managed\\Metadata\\global-metadata.dat", - "D:\\Reverse\\proseka_reverse\\game_binary\\v550\\config.arm64_v8a\\lib\\arm64-v8a\\libil2cpp.so", - "D:\\Reverse\\proseka_reverse\\game_binary\\v550\\il2cpp" - ], - "justMyCode": false - }, - ] +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Animation Plot", + "type": "debugpy", + "request": "launch", + "program": "${workspaceFolder}/misc/animation_plot.py", + "justMyCode": false + }, + { + "name": "AbCache Availability Check", + "type": "debugpy", + "request": "launch", + "program": "${workspaceFolder}/misc/abcache_availability_check.py", + "justMyCode": false + }, + { + "name": "GUI Test", + "type": "debugpy", + "request": "launch", + "program": "${workspaceFolder}/sssekai/__gui__.py", + "justMyCode": false + }, + { + "name": "Python: SpineExtract test", + "type": "debugpy", + "request": "launch", + "module": "tests.test_spine", + "justMyCode": false + }, + { + "name": "Python: Live2DExtract test", + "type": "debugpy", + "request": "launch", + "module": "tests.test_live2d", + "justMyCode": false + }, + { + "name": "Python: AppHash test", + "type": "debugpy", + "request": "launch", + "module": "tests.test_apphash", + "justMyCode": false + }, + { + "name": "Python: AbCache test", + "type": "debugpy", + "request": "launch", + "module": "tests.test_abcache", + "justMyCode": false + }, + { + "name": "Python: Legacy Animation format test", + "type": "debugpy", + "request": "launch", + "module": "tests.test_legacy_anim", + "justMyCode": false + }, + { + "name": "Python: Current File", + "type": "python", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "justMyCode": false, + "args": [] + }, + { + "name": "Python: AbCache (JP Prod)", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "abcache", + "--proxy", + "http://127.0.0.1:8080", + "--app-region", + "jp", + "--app-version", + "5.5.1", + "--app-appHash", + "30829b50-b4b7-d39e-1d55-21ba51e0a8bc", + ], + "justMyCode": true + }, + { + "name": "Python: AbCache (JP Staging)", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "abcache", + "--proxy", + "http://127.0.0.1:8080", + "--app-region", + "jp", + "--app-version", + "5.5.0", + "--app-appHash", + "9b75c207-5fa7-a852-71e3-4097eb9b6027", + ], + "justMyCode": true + }, + { + "name": "Python: AbCache (TW)", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "abcache", + "--app-region", + "tw", + "--app-version", + "4.1.1", + "--app-appHash", + "41fd71f2-f715-bc10-5852-0a9d8542f760", + "--app-abVersion", + "4.1.0", + "--download-filter", + ".*/stage_decoration/.*", + "--download-dir", + "~/.sssekai", + ], + "justMyCode": true + }, + { + "name": "Python: AbCache (KR)", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "abcache", + "--app-region", + "kr", + "--app-version", + "3.4.3", + "--app-appHash", + "a3015fe8-785f-27e1-fb8b-546a23c82c1f", + "--app-abVersion", + "3.4.0" + ], + "justMyCode": true + }, + { + "name": "Python: AbCache (EN)", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "abcache", + "--app-region", + "en", + "--app-version", + "3.8.0", + "--app-appHash", + "3272b0b0-f161-448e-8416-1f8bd2c443b3", + "--app-platform", + "ios" + ], + "justMyCode": true + }, + { + "name": "Python: AbCache (CN)", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "abcache", + "--app-region", + "cn", + "--app-version", + "3.4.0", + "--app-appHash", + "a3015fe8-785f-27e1-fb8b-546a23c82c1f" + ], + "justMyCode": true + }, + { + "name": "Python: AbCache (dump Master data)", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "abcache", + "--app-region", + "en", + "--app-version", + "3.2.0", + "--app-appHash", + "1380836d-de9e-49a8-afe5-c52ba589c8c9", + "--dump-master-data", + "~/.sssekai/master_data" + ], + "justMyCode": true + }, + { + "name": "Python: AbServe (FUSE)", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "abserve", + "--fuse", + "~/ProjectSEKAI" + ], + "justMyCode": false + }, + { + "name": "Python: AbServe", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "abserve", + // "--db", + // "/mnt/c/Users/mos9527/.sssekai/abcache.db", + ], + "justMyCode": false + }, + { + "name": "Python: AbCache (DL)", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "abcache", + "--no-update", + "--download-dir", + "~/.sssekai/bundles/", + "--download-filter", + ".*/stage_decoration/0344.*" + ], + "justMyCode": false + }, + { + "name": "Python: MVData", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "mvdata", + "D:\\Reverse\\proseka_reverse\\assets\\live_pv\\mv_data", + "D:\\Reverse\\proseka_reverse\\mvdata.json" + ], + "justMyCode": true + }, + { + "name": "Python: Tests", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/sssekai.test.py", + "cwd": "${workspaceFolder}", + "console": "integratedTerminal", + "justMyCode": false + }, + { + "name": "Python: apphash", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "apphash", + "--apk-src", + "D:\\Reverse\\proseka_reverse\\game_binary\\v550\\550.xapk", + "--deep", + "--format", + "json" + ], + "justMyCode": true + }, + { + "name": "Python: apphash TW w/ metadata", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "apphash", + "--apk-src", + "/Users/mos9527/sekai-apphash/tw/tw.apk" + ], + "justMyCode": true + }, + { + "name": "Python: usmdemux", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "usmdemux", + "D:\\Sekai\\assets\\8015", + "D:\\Sekai\\assets\\8015.usm", + ], + "justMyCode": true + }, + { + "name": "Python: live2dextract", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "live2dextract", + "C:\\Users\\mos9527\\.sssekai\\abcache\\live2d\\model\\21miku_night", + "C:\\Users\\mos9527\\Desktop\\Cache" + ], + "justMyCode": true + }, + { + "name": "Python: rla2json", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "rla2json", + "/Volumes/mos9527弄丢的盘/Reverse/proseka_reverse/sse_packet_importer/packets/356_5043", + "/Volumes/mos9527弄丢的盘/Reverse/proseka_reverse/sse_packet_importer/exported/356_5043", + "--version", + "1.6", + "--strict" + ], + "justMyCode": true + }, + { + "name": "Python: Test Animation", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/test.animation.py", + "console": "integratedTerminal", + "justMyCode": false, + "args": [] + }, + { + "name": "Python: RLA Test (Single File)", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/sssekai/fmt/rla.py", + "console": "integratedTerminal", + "justMyCode": false, + "args": [ + // "D:\\proseka_reverse\\sse_packet_importer\\packets\\streaming_vbs_1\\sekai_30_00000006.rla.bytes" + "D:\\proseka_reverse\\sse_packet_importer\\packets\\4268\\1728191806276-0.bin" + ] + }, + { + "name": "Python: AbCache (Override EN Asset Version)", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "abcache", + "--app-region", + "en", + // "--app-version", + // "3.6.0", + // "--app-appHash", + // "a3015fe8-785f-27e1-fb8b-546a23c82c1f", + "--app-asset-version", + "3.7.0.0", + "--app-asset-hash", + "d2e5b1bb-d0d4-4535-99ca-bd84ae291a9d" + ], + "justMyCode": true + }, + { + "name": "Python: AbCache (Override JP Asset Version)", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "abcache", + "--app-region", + "jp", + // "--app-version", + // "5.3.0", + // "--app-appHash", + // "2a9bb18f-c309-1993-5b6e-9f25ddc88be3", + "--app-asset-version", + "5.3.0.90", + "--app-asset-hash", + "7f2ebd91-ce39-6ea2-20b2-e05d2872e9e6", + "--app-asset-host", + "cf2d2388" + ], + "justMyCode": true + }, + { + "name": "Python: IL2CPP v381 (Android)", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "il2cpp", + "D:\\Reverse\\proseka_reverse\\game_binary\\v381\\com.sega.pjsekai\\assets\\bin\\Data\\Managed\\Metadata\\global-metadata.dat", + "D:\\Reverse\\proseka_reverse\\game_binary\\v381\\config.arm64_v8a\\lib\\arm64-v8a\\libil2cpp.so", + "D:\\Reverse\\proseka_reverse\\game_binary\\v381\\il2cpp" + ], + "justMyCode": false + }, + { + "name": "Python: IL2CPP v550 (Android)", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "il2cpp", + "D:\\Reverse\\proseka_reverse\\game_binary\\v550\\com.sega.pjsekai\\assets\\bin\\Data\\Managed\\Metadata\\global-metadata.dat", + "D:\\Reverse\\proseka_reverse\\game_binary\\v550\\config.arm64_v8a\\lib\\arm64-v8a\\libil2cpp.so", + "D:\\Reverse\\proseka_reverse\\game_binary\\v550\\il2cpp" + ], + "justMyCode": false + }, + ] } \ No newline at end of file diff --git a/README.md b/README.md index bac07f8..fe10b2b 100644 --- a/README.md +++ b/README.md @@ -1,43 +1,42 @@ -# sssekai - -本项目基于 [mos9527/sssekai](https://github.com/mos9527/sssekai)，由 [MetaSekaiLab](https://github.com/MetaSekaiLab) 继续维护。感谢原作者及历代贡献者。 - -[![Windows Build](https://github.com/MetaSekaiLab/sssekai/actions/workflows/python-publish.yml/badge.svg)](https://github.com/MetaSekaiLab/sssekai/actions/workflows/python-publish.yml) [![Releases](https://img.shields.io/github/downloads/MetaSekaiLab/sssekai/total.svg)](https://github.com/MetaSekaiLab/sssekai/releases/) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) - -Command-line tool (w/Python API support) for Project SEKAI (JP: プロジェクトセカイ カラフルステージ！ feat.初音ミク) game assets. - -# Installing & Updating -**Windows 用户**：请查看本仓库的 [Releases](https://github.com/MetaSekaiLab/sssekai/releases)。 - -Python >=3.10 is required to run this tool. - -通过 PyPI 安装或更新： -```bash -pip install -U sssekai -``` -To use the latest development version, you can install directly from GitHub: -```bash -pip install -U git+https://github.com/MetaSekaiLab/sssekai -``` -## Using the GUI version -Additionally, you can use the GUI version of `sssekai` by installing the `[gui]` extra, which in turn installs [GooeyEx](https://github.com/mos9527/GooeyEx/). - -```bash -pip install -U "sssekai[gui]" -``` -Then you can run the GUI version by: -```bash -sssekai-gui -``` -# 问题反馈 -使用问题和缺陷请提交至本仓库的 [Issues](https://github.com/MetaSekaiLab/sssekai/issues)。 - -# See Also -https://github.com/mos9527/sssekai_blender_io - -# License -MIT - -# References -- https://github.com/K0lb3/UnityPy -- https://mos9527.github.io/tags/project-sekai/ +# sssekai + +本项目基于 [mos9527/sssekai](https://github.com/mos9527/sssekai)，由 [MetaSekaiLab](https://github.com/MetaSekaiLab) 继续维护。感谢原作者及历代贡献者。 + +[![Windows Build](https://github.com/MetaSekaiLab/sssekai/actions/workflows/python-publish.yml/badge.svg)](https://github.com/MetaSekaiLab/sssekai/actions/workflows/python-publish.yml) [![Releases](https://img.shields.io/github/downloads/MetaSekaiLab/sssekai/total.svg)](https://github.com/MetaSekaiLab/sssekai/releases/) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) + +Command-line tool (w/Python API support) for Project SEKAI (JP: プロジェクトセカイ カラフルステージ！ feat.初音ミク) game assets. + +# Installing & Updating +**Windows 用户**：请查看本仓库的 [Releases](https://github.com/MetaSekaiLab/sssekai/releases)。 + +Python >=3.10 is required to run this tool. + +通过 PyPI 安装或更新： +```bash +pip install -U sssekai +``` +To use the latest development version, you can install directly from GitHub: +```bash +pip install -U git+https://github.com/MetaSekaiLab/sssekai +``` +## Using the GUI version +The GUI depends on [GooeyEx](https://github.com/mos9527/GooeyEx/) **>= 0.1.1**, which is not published on PyPI. Install it manually from upstream (a `git` executable and network access are required), then run the GUI: + +```bash +pip install -U "GooeyEx @ git+https://github.com/mos9527/GooeyEx" +sssekai-gui +``` + +`GooeyEx` 0.1.0 (the newest version on PyPI) crashes on import; verify you got 0.1.1 or newer with `python -c "import GooeyEx; print(GooeyEx.__version__)"`. If git is unavailable, use the CLI executable from Releases instead. +# 问题反馈 +使用问题和缺陷请提交至本仓库的 [Issues](https://github.com/MetaSekaiLab/sssekai/issues)。 + +# See Also +https://github.com/mos9527/sssekai_blender_io + +# License +MIT + +# References +- https://github.com/K0lb3/UnityPy +- https://mos9527.github.io/tags/project-sekai/ diff --git a/setup.py b/setup.py index 835a96c..ed14a3b 100644 --- a/setup.py +++ b/setup.py @@ -1,47 +1,52 @@ -import sys, os - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import setuptools, sssekai - -with open("README.md", "r", encoding="utf-8") as fh: - long_description = fh.read() - -setuptools.setup( - name="sssekai", - version=sssekai.__version__, - author="greats3an", - author_email="greats3an@gmail.com", - maintainer="MetaSekaiLab", - description="Project SEKAI Asset Utility / PJSK 资源工具", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/MetaSekaiLab/sssekai", - project_urls={ - "Source": "https://github.com/MetaSekaiLab/sssekai", - "Issues": "https://github.com/MetaSekaiLab/sssekai/issues", - }, - packages=setuptools.find_packages(include=["sssekai", "sssekai.*"]), - classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - ], - install_requires=[ - "msgpack", - "requests", - "pycryptodome", - "unitypy >= 1.25.0, < 2.0.0", - "tqdm", - "coloredlogs", - "fsspec", - ], - extras_require={"gui": ["GooeyEx>=0.1.1"], "il2cpp": ["lief"], "criware": ["PyCriCodecsEx"]}, - entry_points={ - "console_scripts": [ - "sssekai = sssekai.__main__:__main__", - "sssekai-gui = sssekai.__gui__:__main__", - ], - "fsspec.specs": ["abcache = sssekai.abcache.fs.AbCacheFilesystem"], - }, - python_requires=">=3.10", -) +import sys, os + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import setuptools, sssekai + +with open("README.md", "r", encoding="utf-8") as fh: + long_description = fh.read() + +setuptools.setup( + name="sssekai", + version=sssekai.__version__, + author="greats3an", + author_email="greats3an@gmail.com", + maintainer="MetaSekaiLab", + description="Project SEKAI Asset Utility / PJSK 资源工具", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/MetaSekaiLab/sssekai", + project_urls={ + "Source": "https://github.com/MetaSekaiLab/sssekai", + "Issues": "https://github.com/MetaSekaiLab/sssekai/issues", + }, + packages=setuptools.find_packages(include=["sssekai", "sssekai.*"]), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ], + install_requires=[ + "msgpack", + "requests", + "pycryptodome", + "unitypy >= 1.25.0, < 2.0.0", + "tqdm", + "coloredlogs", + "fsspec", + ], + # NOTE: No `gui` extra on purpose. The GUI requires GooeyEx >= 0.1.1, which + # is not published on PyPI (0.1.0 crashes at import without a wx.App), and + # PyPI rejects direct-URL requirements. See README for manual installation. + # The unused `criware` extra was removed with its dependency for the same + # reason: PyCriCodecsEx is not installable and nothing imports it. + extras_require={"il2cpp": ["lief"]}, + entry_points={ + "console_scripts": [ + "sssekai = sssekai.__main__:__main__", + "sssekai-gui = sssekai.__gui__:__main__", + ], + "fsspec.specs": ["abcache = sssekai.abcache.fs.AbCacheFilesystem"], + }, + python_requires=">=3.10", +) diff --git a/sssekai/__gui__.py b/sssekai/__gui__.py index 87e3aca..71a9f15 100644 --- a/sssekai/__gui__.py +++ b/sssekai/__gui__.py @@ -1,72 +1,76 @@ -from dataclasses import dataclass, field -from sssekai.__main__ import create_parser -from typing import Any, List -from argparse import ArgumentParser -from logging import basicConfig, getLogger -import sys - -try: - from GooeyEx import Gooey, GooeyParser -except ImportError as e: - print( - "Please install sssekai[gui] through your Python package manager to use the GUI" - ) - raise e - - -from sssekai.unity import sssekai_set_unity_version - - -@Gooey( - show_preview_warning=False, - program_name="sssekai", - tabbed_groups=True, - advanced=True, - monospace_display=True, - default_size=(800, 600), - menu=[ - { - "name": "Help", - "items": [ - { - "type": "Link", - "menuTitle": "Report an Issue", - "url": "https://github.com/MetaSekaiLab/sssekai/issues", - }, - { - "type": "Link", - "menuTitle": "GitHub", - "url": "https://github.com/MetaSekaiLab/sssekai", - }, - ], - } - ], -) -def __main__(): - from tqdm.std import tqdm as tqdm_c - - parser = create_parser(GooeyParser) - args = parser.parse_args() - - class TqdmMutexStream: - @staticmethod - def write(__s): - # Gooey[Ex] only reads output from stdout so we'd do that here. - with tqdm_c.external_write_mode(file=sys.stdout, nolock=False): - return sys.stdout.write(__s) - - basicConfig( - level="DEBUG", - format="%(asctime)s | %(levelname).1s | %(name)s %(message)s", - datefmt="%H:%M:%S", - stream=TqdmMutexStream, - ) - # override unity version - sssekai_set_unity_version(args.unity_version) - if "func" in args: - args.func(args) - pass - - -if __name__ in {"__main__", "__gui__"}: - __main__() +from dataclasses import dataclass, field +from sssekai.__main__ import create_parser +from typing import Any, List +from argparse import ArgumentParser +from logging import basicConfig, getLogger +import sys + +try: + from GooeyEx import Gooey, GooeyParser +except ImportError as e: + print( + "The GUI requires GooeyEx >= 0.1.1, which is not published on PyPI.\n" + "Install it manually, then retry:\n" + ' pip install -U "GooeyEx @ git+https://github.com/mos9527/GooeyEx"\n' + "Otherwise use the CLI executable from the Releases page.", + file=sys.stderr, + ) + raise SystemExit(1) from e + + +from sssekai.unity import sssekai_set_unity_version + + +@Gooey( + show_preview_warning=False, + program_name="sssekai", + tabbed_groups=True, + advanced=True, + monospace_display=True, + default_size=(800, 600), + menu=[ + { + "name": "Help", + "items": [ + { + "type": "Link", + "menuTitle": "Report an Issue", + "url": "https://github.com/MetaSekaiLab/sssekai/issues", + }, + { + "type": "Link", + "menuTitle": "GitHub", + "url": "https://github.com/MetaSekaiLab/sssekai", + }, + ], + } + ], +) +def __main__(): + from tqdm.std import tqdm as tqdm_c + + parser = create_parser(GooeyParser) + args = parser.parse_args() + + class TqdmMutexStream: + @staticmethod + def write(__s): + # Gooey[Ex] only reads output from stdout so we'd do that here. + with tqdm_c.external_write_mode(file=sys.stdout, nolock=False): + return sys.stdout.write(__s) + + basicConfig( + level="DEBUG", + format="%(asctime)s | %(levelname).1s | %(name)s %(message)s", + datefmt="%H:%M:%S", + stream=TqdmMutexStream, + ) + # override unity version + sssekai_set_unity_version(args.unity_version) + if "func" in args: + args.func(args) + pass + + +if __name__ in {"__main__", "__gui__"}: + __main__() diff --git a/sssekai/__main__.py b/sssekai/__main__.py index cd60571..9cac0c4 100644 --- a/sssekai/__main__.py +++ b/sssekai/__main__.py @@ -1,539 +1,539 @@ -import argparse, sys, logging, traceback - -# region Deferred imports -# This can be done a lot cleaner with importlib. But tools like PyInstaller really -# won't like that. - -def main_apidecrypt(*args, **kwargs): - from sssekai.entrypoint.apidecrypt import main_apidecrypt - return main_apidecrypt(*args, **kwargs) - -def main_abdecrypt(*args, **kwargs): - from sssekai.entrypoint.abdecrypt import main_abdecrypt - return main_abdecrypt(*args, **kwargs) - -def main_rla2json(*args, **kwargs): - from sssekai.entrypoint.rla2json import main_rla2json - return main_rla2json(*args, **kwargs) - -def main_usmdemux(*args, **kwargs): - from sssekai.entrypoint.usmdemux import main_usmdemux - return main_usmdemux(*args, **kwargs) - -def main_abcache(*args, **kwargs): - from sssekai.entrypoint.abcache import main_abcache - return main_abcache(*args, **kwargs) - -def main_abserve(*args, **kwargs): - from sssekai.entrypoint.abserve import main_abserve - return main_abserve(*args, **kwargs) - -def main_live2dextract(*args, **kwargs): - from sssekai.entrypoint.live2dextract import main_live2dextract - return main_live2dextract(*args, **kwargs) - -def main_spineextract(*args, **kwargs): - from sssekai.entrypoint.spineextract import main_spineextract - return main_spineextract(*args, **kwargs) - -def main_apphash(*args, **kwargs): - from sssekai.entrypoint.apphash import main_apphash - return main_apphash(*args, **kwargs) - -def main_il2cpp(*args, **kwargs): - from sssekai.entrypoint.il2cpp import main_il2cpp - return main_il2cpp(*args, **kwargs) - -def main_il2cppextract(*args, **kwargs): - from sssekai.entrypoint.il2cppextract import main_il2cppextract - return main_il2cppextract(*args, **kwargs) - -def main_mvdata(*args, **kwargs): - from sssekai.entrypoint.mvdata import main_mvdata - return main_mvdata(*args, **kwargs) - -def main_moc3paths(*args, **kwargs): - from sssekai.entrypoint.moc3paths import main_moc3paths - return main_moc3paths(*args, **kwargs) - -# endregion - -from sssekai.unity import sssekai_get_unity_version, sssekai_set_unity_version -from sssekai.fmt.rla import RLA_VERSIONS - -logger = logging.getLogger(__name__) - - -def create_parser(clazz=argparse.ArgumentParser): - """ - Create the command line parser for the application. - - Args: - clazz (type): The parser class to use. Defaults to argparse.ArgumentParser. - In GUI mode this should be set to GooeyParser otherwise. - """ - - def gooey_only(**kwargs): - # Silent non-argparse kwargs - if clazz == argparse.ArgumentParser: - return {} - return kwargs - - parser = clazz( - description="""Project SEKAI Asset Utility / PJSK 资源工具""", - formatter_class=argparse.RawTextHelpFormatter, - ) - parser.add_argument( - "--log-level", - type=str, - help="logging level (default: %(default)s)", - default="INFO", - choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], - ) - parser.add_argument( - "--unity-version", - type=str, - help="""Unity version to use (default: %(default)s) -Prior to game version 3.6.0 (JP), this has always been 2020.3.21f1 -This has been changed to 2022.3.21f1 since, which would apply to all the assets from 3.6.0 onwards. -If you encounter any issues, try switching to the old version, or vice versa.""", - default=sssekai_get_unity_version(), - ) - subparsers = parser.add_subparsers( - title="subcommands", description="valid subcommands", help="additional help" - ) - # apidecrypt - apidecrypt_parser = subparsers.add_parser( - "apidecrypt", - usage="""API crypto dumper -This crypto applies to: - - API request/response body dumped by packet sniffer (mitmproxy, wireshark, etc.) - - AssetBundleInfo (see sssekai.abcache)""", - ) - apidecrypt_parser.add_argument( - "infile", type=str, help="input dump file", **gooey_only(widget="FileChooser") - ) - apidecrypt_parser.add_argument( - "outfile", type=str, help="output json file", **gooey_only(widget="FileSaver") - ) - apidecrypt_parser.add_argument( - "--region", - type=str, - help="app region", - default="jp", - choices=["jp", "tw", "en", "kr", "cn"], - ) - apidecrypt_parser.set_defaults(func=main_apidecrypt) - # abdecrypt - abdecrypt_parser = subparsers.add_parser( - "abdecrypt", usage="""Decrypt Sekai AssetBundle""" - ) - abdecrypt_parser.add_argument( - "indir", type=str, help="input directory", **gooey_only(widget="DirChooser") - ) - abdecrypt_parser.add_argument( - "outdir", type=str, help="output directory", **gooey_only(widget="DirChooser") - ) - abdecrypt_parser.set_defaults(func=main_abdecrypt) - # usmdemux - usmdemux_parser = subparsers.add_parser( - "usmdemux", usage="""Demux Sekai USM Video in a AssetBundle""" - ) - usmdemux_parser.add_argument( - "infile", type=str, help="input file", **gooey_only(widget="FileChooser") - ) - usmdemux_parser.add_argument( - "outfile", type=str, help="output file where the .USM file will be saved", **gooey_only(widget="FileSaver") - ) - usmdemux_parser.set_defaults(func=main_usmdemux) - # abcache - abcache_parser = subparsers.add_parser( - "abcache", usage="""Sekai AssetBundle Metadata Cache / Game API Helper""" - ) - abcache_parser.add_argument( - "--proxy", - type=str, - help="HTTP Proxy to use. This overrides the system proxy (environ HTTP_PROXY, HTTPS_PROXY) settings.", - default=None, - ) - group = abcache_parser.add_argument_group("save/load options") - group.add_argument( - "--db", - type=str, - help="""cache database file path (default: %(default)s)""", - default="~/.sssekai/abcache.db", - **gooey_only(widget="FileChooser"), - ) - group.add_argument( - "--no-update", - action="store_true", - help="skip all metadata updates and use cached ones as is. this supercedes 'game version options' and 'authentication options'", - ) - group.add_argument( - "--dump", - type=str, - help="dump the AssetBundle index to the specified JSON file and exit", - default=None, - **gooey_only(widget="FileSaver"), - ) - group = abcache_parser.add_argument_group( - "game version options", - "NOTE: these options are used to *update* the cache database. Use --no-update to skip updating. Also, you may want to read the Wiki to know how to get these values.", - ) - group.add_argument( - "--app-region", - type=str, - help="PJSK app region (default: %(default)s)", - choices=["jp", "tw", "en", "kr", "cn"], - default="jp", - ) - group.add_argument( - "--app-platform", - type=str, - help="PJSK app platform (default: %(default)s", - choices=["ios", "android"], - default="android", - ) # Let's hope this doesn't age well... - group.add_argument( - "--app-version", - type=str, - help="PJSK app version. For assets this is required unless --no-update is specified, or for EN/JP requisite --app-asset..fields are filled. For other functions this is required unless --no-update is specified", - default="", - ) - group.add_argument( - "--app-appHash", - type=str, - help="PJSK app hash. For assets this is required unless --no-update is specified, or for EN/JP requisite --app-asset..fields are filled. For other functions this is required unless --no-update is specified", - default="", - ) - group = abcache_parser.add_argument_group("ROW specific options") - group.add_argument( - "--app-abVersion", - type=str, - help="PJSK AssetBundle URL version. ROW servers only - This is required unless --no-update is specified", - default=None, - ) - group = abcache_parser.add_argument_group("EN/JP specific options") - group.add_argument( - "--app-asset-host", - type=str, - help="Override AssetBundle host URL. EN/JP servers only.", - default=None, - ) - group.add_argument( - "--app-asset-version", - type=str, - help="Override AssetBundle source version. EN/JP servers only.", - default=None, - ) - group.add_argument( - "--app-asset-hash", - type=str, - help="Override AssetBundle hash. EN/JP servers only.", - default=None, - ) - group = abcache_parser.add_argument_group("download options") - group.add_argument( - "--download-no-overwrite", - action="store_true", - help="skip downloading AssetBundles that are already present in the download directory", - ) - group.add_argument( - "--download-filter", - type=str, - help="filter AssetBundles (by bundle names) with this regex pattern", - default=None, - ) - group.add_argument( - "--download-filter-cache-diff", - type=str, - help="filter AssetBundles (by hashes) by *ONLY* downloading ones with a *different* hash from the current hash with this one.", - default=None, - **gooey_only(widget="FileChooser"), - ) - group.add_argument( - "--download-dir", - type=str, - help="asset bundle download directory. leave empty if you don't want to download anything", - default="", - **gooey_only(widget="DirChooser"), - ) - group.add_argument( - "--download-ensure-deps", - action="store_true", - help="ensure dependencies (of the downloaded ones) are downloaded as well", - ) - group.add_argument( - "--download-workers", - type=int, - help="number of download workers (default: %(default)s)", - default=4, - ) - group = abcache_parser.add_argument_group( - "extra options", - "NOTE: when *any* of these options are specified, the cache database *won't* be updated, and no download will be performed either.", - ) - group.add_argument( - "--dump-master-data", - type=str, - help="directory to store the dumped master data in JSON.", - default=None, - **gooey_only(widget="DirChooser"), - ) - group.add_argument( - "--dump-user-data", - type=str, - help="directory to store the dumped master data of the authencicated user in JSON. NOTE: --auth-credential required", - default=None, - **gooey_only(widget="DirChooser"), - ) - group.add_argument( - "--keep-compact", - help="keep compacted datasets as is without expanding them to key-value pairs", - action="store_true", - ) - group = abcache_parser.add_argument_group( - "authentication arguments", - "Only needed for some functionailties (i.e. --dump-user-data)", - ) - group.add_argument( - "--auth-credential", - type=str, - help="Credential for API authentication. Can be omitted on JP/EN servers, in which case a guest account would be automatically registered and used.", - default=None, - ) - abcache_parser.set_defaults(func=main_abcache) - abserve_parser = subparsers.add_parser( - "abserve", usage="""AbCache Filesystem Server / Backend""" - ) - abserve_parser.add_argument( - "--proxy", - type=str, - help="HTTP Proxy to use. This overrides the system proxy (environ HTTP_PROXY, HTTPS_PROXY) settings.", - default=None, - ) - abserve_parser.add_argument( - "--db", - type=str, - help="""cache database file path (default: %(default)s)""", - default="~/.sssekai/abcache.db", - **gooey_only(widget="FileChooser"), - ) - abserve_parser.add_argument( - "--fuse", - type=str, - help="""local mount point for AbCache. Required FUSE on host OS and fusepy (default: %(default)s)""", - default="", - **gooey_only(widget="DirChooser"), - ) - abserve_parser.add_argument( - "--host", - type=str, - help="""address of the interface to listen on (default: %(default)s)""", - default="0.0.0.0", - ) - abserve_parser.add_argument( - "--port", - type=int, - help="""port to listen on (default: %(default)s)""", - default="3939", - ) - abserve_parser.set_defaults(func=main_abserve) - # live2dextract - live2dextract_parser = subparsers.add_parser( - "live2dextract", usage="""Extract Sekai Live2D Models in a AssetBundle""" - ) - live2dextract_parser.add_argument( - "infile", type=str, help="input file", **gooey_only(widget="FileChooser") - ) - live2dextract_parser.add_argument( - "outdir", type=str, help="output directory", **gooey_only(widget="DirChooser") - ) - live2dextract_parser.add_argument( - "--no-anim", action="store_true", help="don't extract animation clips" - ) - live2dextract_parser.set_defaults(func=main_live2dextract) - # spineextract - spineextract_parser = subparsers.add_parser( - "spineextract", - usage="""Extract Sekai Spine (Esoteric Spine2D) Models in a AssetBundle""", - ) - spineextract_parser.add_argument( - "infile", type=str, help="input file", **gooey_only(widget="FileChooser") - ) - spineextract_parser.add_argument( - "outdir", type=str, help="output directory", **gooey_only(widget="DirChooser") - ) - spineextract_parser.set_defaults(func=main_spineextract) - # rla2json - rla2json_parser = subparsers.add_parser( - "rla2json", - usage="""Read streaming_live/archive files and dump their information to JSON or HCA packets""", - ) - rla2json_parser.add_argument( - "input", - type=str, - help="input archive file or directory containing loose packets", - **gooey_only(widget="FileChooser"), - ) - rla2json_parser.add_argument( - "outdir", type=str, help="output directory", **gooey_only(widget="DirChooser") - ) - rla2json_parser.add_argument( - "--version", - type=str, - help="RLA version. Only used for loose packets", - default="1.6", - choices=["%1s.%1s" % x for x in RLA_VERSIONS], - ) - rla2json_parser.add_argument( - "--strict", - action="store_true", - help="strict mode. raise exception on unknown frame type or corrupt frames", - ) - rla2json_parser.set_defaults(func=main_rla2json) - # apphash - apphash_parser = subparsers.add_parser( - "apphash", - usage="""Download/extract game AppHash values -*NOTE*: Only [app version] [app region] [app hash] are written to stdout if successful. Fields are separated by space. Unavailable fields are left as 'unknown'.""", - ) - apphash_parser.add_argument( - "--apk-src", - type=str, - help="APK source file (default: fetch from APKPure)", - default=None, - **gooey_only(widget="FileChooser"), - ) - apphash_parser.add_argument( - "--ab-src", - type=str, - help="AssetBundle (namely, 6350e2ec327334c8a9b7f494f344a761 or c726e51b6fe37463685916a1687158dd) source file (default: fetch from APK)", - default=None, - **gooey_only(widget="FileChooser"), - ) - apphash_parser.add_argument( - "--fetch", action="store_true", help="force fetching the latest APK" - ) - apphash_parser.add_argument( - "--format", - choices=["markdown", "json"], - help="output format", - default="markdown", - ) - apphash_parser.add_argument( - "--proxy", - type=str, - help="HTTP Proxy to use. This overrides the system proxy (environ HTTP_PROXY, HTTPS_PROXY) settings.", - default=None, - ) - apphash_parser.add_argument( - "--deep", - action="store_true", - help="Load all AssetBundles and retrieve all app hashes available.", - ) - apphash_parser.set_defaults(func=main_apphash) - # il2cpp - il2cpp_parser = subparsers.add_parser( - "il2cpp", - description="Extract IL2CPP inputs from JP/EN/CN/TW/KR Android releases; detect metadata format automatically.", - ) - il2cpp_parser.add_argument( - "metadata", - type=str, - help="the global-metadata.dat file at assets/bin/Data/Managed/Metadata", - **gooey_only(widget="FileChooser"), - ) - il2cpp_parser.add_argument( - "binary", - type=str, - help="libil2cpp.so at config.arm64_v8a/lib/arm64-v8a. currently this would be copied as is", - **gooey_only(widget="FileChooser"), - ) - il2cpp_parser.add_argument( - "outdir", - type=str, - help="output directory to dump the extracted metadata (as global-metadata.dat) and binary (as il2cpp.so)", - **gooey_only(widget="DirChooser"), - ) - il2cpp_parser.set_defaults(func=main_il2cpp) - il2cppextract_parser = subparsers.add_parser( - "il2cpp-extract", - help="Prepare IL2CPP analysis inputs from a JP/EN/CN/TW/KR Android ARM64 APK/XAPK", - description="Extract APK/XAPK inputs and run the existing il2cpp command.", - ) - il2cppextract_parser.add_argument( - "infile", help="input APK or XAPK (including ARM64 splits)", - **gooey_only(widget="FileChooser"), - ) - il2cppextract_parser.add_argument( - "outdir", help="output analysis directory", - **gooey_only(widget="DirChooser"), - ) - il2cppextract_parser.set_defaults(func=main_il2cppextract) - # mvdata - mvdata_parser = subparsers.add_parser( - "mvdata", usage="""Extract MV Data from AssetBundle""" - ) - mvdata_parser.add_argument( - "indir", - type=str, - help="cache directory (live_pv/mv_data)", - **gooey_only(widget="DirChooser"), - ) - mvdata_parser.add_argument( - "outfile", - type=str, - help="output JSON file to dump into", - **gooey_only(widget="FileSaver"), - ) - mvdata_parser.set_defaults(func=main_mvdata) - # moc3paths - moc3paths_parser = subparsers.add_parser( - "moc3paths", usage="""Extract animation path CRCs from raw .moc3 binaries""" - ) - moc3paths_parser.add_argument( - "indir", type=str, help="input directory", **gooey_only(widget="DirChooser") - ) - moc3paths_parser.set_defaults(func=main_moc3paths) - return parser - - -def __main__(): - from tqdm.std import tqdm as tqdm_c - - class TqdmMutexStream: - @staticmethod - def write(__s): - with tqdm_c.external_write_mode(file=sys.stderr, nolock=False): - return sys.stderr.write(__s) - - # parse args - parser = create_parser(argparse.ArgumentParser) - args = parser.parse_args() - # set logging level - import coloredlogs - - coloredlogs.install( - level=args.log_level, - fmt="%(asctime)s | %(levelname).1s | %(name)s %(message)s", - datefmt="%H:%M:%S", - isatty=True, - stream=TqdmMutexStream, - ) - # override unity version - sssekai_set_unity_version(args.unity_version) - if "func" in args: - try: - return args.func(args) - except Exception as e: - logger.exception("Error while running command: %s", e) - traceback.print_exc() - return 1 - else: - parser.print_help() - return 1 - - -if __name__ == "__main__": - sys.exit(__main__() or 0) +import argparse, sys, logging, traceback + +# region Deferred imports +# This can be done a lot cleaner with importlib. But tools like PyInstaller really +# won't like that. + +def main_apidecrypt(*args, **kwargs): + from sssekai.entrypoint.apidecrypt import main_apidecrypt + return main_apidecrypt(*args, **kwargs) + +def main_abdecrypt(*args, **kwargs): + from sssekai.entrypoint.abdecrypt import main_abdecrypt + return main_abdecrypt(*args, **kwargs) + +def main_rla2json(*args, **kwargs): + from sssekai.entrypoint.rla2json import main_rla2json + return main_rla2json(*args, **kwargs) + +def main_usmdemux(*args, **kwargs): + from sssekai.entrypoint.usmdemux import main_usmdemux + return main_usmdemux(*args, **kwargs) + +def main_abcache(*args, **kwargs): + from sssekai.entrypoint.abcache import main_abcache + return main_abcache(*args, **kwargs) + +def main_abserve(*args, **kwargs): + from sssekai.entrypoint.abserve import main_abserve + return main_abserve(*args, **kwargs) + +def main_live2dextract(*args, **kwargs): + from sssekai.entrypoint.live2dextract import main_live2dextract + return main_live2dextract(*args, **kwargs) + +def main_spineextract(*args, **kwargs): + from sssekai.entrypoint.spineextract import main_spineextract + return main_spineextract(*args, **kwargs) + +def main_apphash(*args, **kwargs): + from sssekai.entrypoint.apphash import main_apphash + return main_apphash(*args, **kwargs) + +def main_il2cpp(*args, **kwargs): + from sssekai.entrypoint.il2cpp import main_il2cpp + return main_il2cpp(*args, **kwargs) + +def main_il2cppextract(*args, **kwargs): + from sssekai.entrypoint.il2cppextract import main_il2cppextract + return main_il2cppextract(*args, **kwargs) + +def main_mvdata(*args, **kwargs): + from sssekai.entrypoint.mvdata import main_mvdata + return main_mvdata(*args, **kwargs) + +def main_moc3paths(*args, **kwargs): + from sssekai.entrypoint.moc3paths import main_moc3paths + return main_moc3paths(*args, **kwargs) + +# endregion + +from sssekai.unity import sssekai_get_unity_version, sssekai_set_unity_version +from sssekai.fmt.rla import RLA_VERSIONS + +logger = logging.getLogger(__name__) + + +def create_parser(clazz=argparse.ArgumentParser): + """ + Create the command line parser for the application. + + Args: + clazz (type): The parser class to use. Defaults to argparse.ArgumentParser. + In GUI mode this should be set to GooeyParser otherwise. + """ + + def gooey_only(**kwargs): + # Silent non-argparse kwargs + if clazz == argparse.ArgumentParser: + return {} + return kwargs + + parser = clazz( + description="""Project SEKAI Asset Utility / PJSK 资源工具""", + formatter_class=argparse.RawTextHelpFormatter, + ) + parser.add_argument( + "--log-level", + type=str, + help="logging level (default: %(default)s)", + default="INFO", + choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], + ) + parser.add_argument( + "--unity-version", + type=str, + help="""Unity version to use (default: %(default)s) +Prior to game version 3.6.0 (JP), this has always been 2020.3.21f1 +This has been changed to 2022.3.21f1 since, which would apply to all the assets from 3.6.0 onwards. +If you encounter any issues, try switching to the old version, or vice versa.""", + default=sssekai_get_unity_version(), + ) + subparsers = parser.add_subparsers( + title="subcommands", description="valid subcommands", help="additional help" + ) + # apidecrypt + apidecrypt_parser = subparsers.add_parser( + "apidecrypt", + usage="""API crypto dumper +This crypto applies to: + - API request/response body dumped by packet sniffer (mitmproxy, wireshark, etc.) + - AssetBundleInfo (see sssekai.abcache)""", + ) + apidecrypt_parser.add_argument( + "infile", type=str, help="input dump file", **gooey_only(widget="FileChooser") + ) + apidecrypt_parser.add_argument( + "outfile", type=str, help="output json file", **gooey_only(widget="FileSaver") + ) + apidecrypt_parser.add_argument( + "--region", + type=str, + help="app region", + default="jp", + choices=["jp", "tw", "en", "kr", "cn"], + ) + apidecrypt_parser.set_defaults(func=main_apidecrypt) + # abdecrypt + abdecrypt_parser = subparsers.add_parser( + "abdecrypt", usage="""Decrypt Sekai AssetBundle""" + ) + abdecrypt_parser.add_argument( + "indir", type=str, help="input directory", **gooey_only(widget="DirChooser") + ) + abdecrypt_parser.add_argument( + "outdir", type=str, help="output directory", **gooey_only(widget="DirChooser") + ) + abdecrypt_parser.set_defaults(func=main_abdecrypt) + # usmdemux + usmdemux_parser = subparsers.add_parser( + "usmdemux", usage="""Demux Sekai USM Video in a AssetBundle""" + ) + usmdemux_parser.add_argument( + "infile", type=str, help="input file", **gooey_only(widget="FileChooser") + ) + usmdemux_parser.add_argument( + "outfile", type=str, help="output file where the .USM file will be saved", **gooey_only(widget="FileSaver") + ) + usmdemux_parser.set_defaults(func=main_usmdemux) + # abcache + abcache_parser = subparsers.add_parser( + "abcache", usage="""Sekai AssetBundle Metadata Cache / Game API Helper""" + ) + abcache_parser.add_argument( + "--proxy", + type=str, + help="HTTP Proxy to use. This overrides the system proxy (environ HTTP_PROXY, HTTPS_PROXY) settings.", + default=None, + ) + group = abcache_parser.add_argument_group("save/load options") + group.add_argument( + "--db", + type=str, + help="""cache database file path (default: %(default)s)""", + default="~/.sssekai/abcache.db", + **gooey_only(widget="FileChooser"), + ) + group.add_argument( + "--no-update", + action="store_true", + help="skip all metadata updates and use cached ones as is. this supercedes 'game version options' and 'authentication options'", + ) + group.add_argument( + "--dump", + type=str, + help="dump the AssetBundle index to the specified JSON file and exit", + default=None, + **gooey_only(widget="FileSaver"), + ) + group = abcache_parser.add_argument_group( + "game version options", + "NOTE: these options are used to *update* the cache database. Use --no-update to skip updating. You can obtain these values with the `apphash` subcommand.", + ) + group.add_argument( + "--app-region", + type=str, + help="PJSK app region (default: %(default)s)", + choices=["jp", "tw", "en", "kr", "cn"], + default="jp", + ) + group.add_argument( + "--app-platform", + type=str, + help="PJSK app platform (default: %(default)s", + choices=["ios", "android"], + default="android", + ) # Let's hope this doesn't age well... + group.add_argument( + "--app-version", + type=str, + help="PJSK app version. For assets this is required unless --no-update is specified, or for EN/JP requisite --app-asset..fields are filled. For other functions this is required unless --no-update is specified", + default="", + ) + group.add_argument( + "--app-appHash", + type=str, + help="PJSK app hash. For assets this is required unless --no-update is specified, or for EN/JP requisite --app-asset..fields are filled. For other functions this is required unless --no-update is specified", + default="", + ) + group = abcache_parser.add_argument_group("ROW specific options") + group.add_argument( + "--app-abVersion", + type=str, + help="PJSK AssetBundle URL version. ROW servers only - This is required unless --no-update is specified", + default=None, + ) + group = abcache_parser.add_argument_group("EN/JP specific options") + group.add_argument( + "--app-asset-host", + type=str, + help="Override AssetBundle host URL. EN/JP servers only.", + default=None, + ) + group.add_argument( + "--app-asset-version", + type=str, + help="Override AssetBundle source version. EN/JP servers only.", + default=None, + ) + group.add_argument( + "--app-asset-hash", + type=str, + help="Override AssetBundle hash. EN/JP servers only.", + default=None, + ) + group = abcache_parser.add_argument_group("download options") + group.add_argument( + "--download-no-overwrite", + action="store_true", + help="skip downloading AssetBundles that are already present in the download directory", + ) + group.add_argument( + "--download-filter", + type=str, + help="filter AssetBundles (by bundle names) with this regex pattern", + default=None, + ) + group.add_argument( + "--download-filter-cache-diff", + type=str, + help="filter AssetBundles (by hashes) by *ONLY* downloading ones with a *different* hash from the current hash with this one.", + default=None, + **gooey_only(widget="FileChooser"), + ) + group.add_argument( + "--download-dir", + type=str, + help="asset bundle download directory. leave empty if you don't want to download anything", + default="", + **gooey_only(widget="DirChooser"), + ) + group.add_argument( + "--download-ensure-deps", + action="store_true", + help="ensure dependencies (of the downloaded ones) are downloaded as well", + ) + group.add_argument( + "--download-workers", + type=int, + help="number of download workers (default: %(default)s)", + default=4, + ) + group = abcache_parser.add_argument_group( + "extra options", + "NOTE: when *any* of these options are specified, the cache database *won't* be updated, and no download will be performed either.", + ) + group.add_argument( + "--dump-master-data", + type=str, + help="directory to store the dumped master data in JSON.", + default=None, + **gooey_only(widget="DirChooser"), + ) + group.add_argument( + "--dump-user-data", + type=str, + help="directory to store the dumped master data of the authencicated user in JSON. NOTE: --auth-credential required", + default=None, + **gooey_only(widget="DirChooser"), + ) + group.add_argument( + "--keep-compact", + help="keep compacted datasets as is without expanding them to key-value pairs", + action="store_true", + ) + group = abcache_parser.add_argument_group( + "authentication arguments", + "Only needed for some functionailties (i.e. --dump-user-data)", + ) + group.add_argument( + "--auth-credential", + type=str, + help="Credential for API authentication. Can be omitted on JP/EN servers, in which case a guest account would be automatically registered and used.", + default=None, + ) + abcache_parser.set_defaults(func=main_abcache) + abserve_parser = subparsers.add_parser( + "abserve", usage="""AbCache Filesystem Server / Backend""" + ) + abserve_parser.add_argument( + "--proxy", + type=str, + help="HTTP Proxy to use. This overrides the system proxy (environ HTTP_PROXY, HTTPS_PROXY) settings.", + default=None, + ) + abserve_parser.add_argument( + "--db", + type=str, + help="""cache database file path (default: %(default)s)""", + default="~/.sssekai/abcache.db", + **gooey_only(widget="FileChooser"), + ) + abserve_parser.add_argument( + "--fuse", + type=str, + help="""local mount point for AbCache. Required FUSE on host OS and fusepy (default: %(default)s)""", + default="", + **gooey_only(widget="DirChooser"), + ) + abserve_parser.add_argument( + "--host", + type=str, + help="""address of the interface to listen on (default: %(default)s). Use 0.0.0.0 to expose the server to the network.""", + default="127.0.0.1", + ) + abserve_parser.add_argument( + "--port", + type=int, + help="""port to listen on (default: %(default)s)""", + default="3939", + ) + abserve_parser.set_defaults(func=main_abserve) + # live2dextract + live2dextract_parser = subparsers.add_parser( + "live2dextract", usage="""Extract Sekai Live2D Models in a AssetBundle""" + ) + live2dextract_parser.add_argument( + "infile", type=str, help="input file", **gooey_only(widget="FileChooser") + ) + live2dextract_parser.add_argument( + "outdir", type=str, help="output directory", **gooey_only(widget="DirChooser") + ) + live2dextract_parser.add_argument( + "--no-anim", action="store_true", help="don't extract animation clips" + ) + live2dextract_parser.set_defaults(func=main_live2dextract) + # spineextract + spineextract_parser = subparsers.add_parser( + "spineextract", + usage="""Extract Sekai Spine (Esoteric Spine2D) Models in a AssetBundle""", + ) + spineextract_parser.add_argument( + "infile", type=str, help="input file", **gooey_only(widget="FileChooser") + ) + spineextract_parser.add_argument( + "outdir", type=str, help="output directory", **gooey_only(widget="DirChooser") + ) + spineextract_parser.set_defaults(func=main_spineextract) + # rla2json + rla2json_parser = subparsers.add_parser( + "rla2json", + usage="""Read streaming_live/archive files and dump their information to JSON or HCA packets""", + ) + rla2json_parser.add_argument( + "input", + type=str, + help="input archive file or directory containing loose packets", + **gooey_only(widget="FileChooser"), + ) + rla2json_parser.add_argument( + "outdir", type=str, help="output directory", **gooey_only(widget="DirChooser") + ) + rla2json_parser.add_argument( + "--version", + type=str, + help="RLA version. Only used for loose packets", + default="1.6", + choices=["%1s.%1s" % x for x in RLA_VERSIONS], + ) + rla2json_parser.add_argument( + "--strict", + action="store_true", + help="strict mode. raise exception on unknown frame type or corrupt frames", + ) + rla2json_parser.set_defaults(func=main_rla2json) + # apphash + apphash_parser = subparsers.add_parser( + "apphash", + usage="""Download/extract game AppHash values +*NOTE*: Only [app version] [app region] [app hash] are written to stdout if successful. Fields are separated by space. Unavailable fields are left as 'unknown'.""", + ) + apphash_parser.add_argument( + "--apk-src", + type=str, + help="APK source file (default: fetch from APKPure)", + default=None, + **gooey_only(widget="FileChooser"), + ) + apphash_parser.add_argument( + "--ab-src", + type=str, + help="AssetBundle (namely, 6350e2ec327334c8a9b7f494f344a761 or c726e51b6fe37463685916a1687158dd) source file (default: fetch from APK)", + default=None, + **gooey_only(widget="FileChooser"), + ) + apphash_parser.add_argument( + "--fetch", action="store_true", help="force fetching the latest APK" + ) + apphash_parser.add_argument( + "--format", + choices=["markdown", "json"], + help="output format", + default="markdown", + ) + apphash_parser.add_argument( + "--proxy", + type=str, + help="HTTP Proxy to use. This overrides the system proxy (environ HTTP_PROXY, HTTPS_PROXY) settings.", + default=None, + ) + apphash_parser.add_argument( + "--deep", + action="store_true", + help="Load all AssetBundles and retrieve all app hashes available.", + ) + apphash_parser.set_defaults(func=main_apphash) + # il2cpp + il2cpp_parser = subparsers.add_parser( + "il2cpp", + description="Extract IL2CPP inputs from JP/EN/CN/TW/KR Android releases; detect metadata format automatically.", + ) + il2cpp_parser.add_argument( + "metadata", + type=str, + help="the global-metadata.dat file at assets/bin/Data/Managed/Metadata", + **gooey_only(widget="FileChooser"), + ) + il2cpp_parser.add_argument( + "binary", + type=str, + help="libil2cpp.so at config.arm64_v8a/lib/arm64-v8a. currently this would be copied as is", + **gooey_only(widget="FileChooser"), + ) + il2cpp_parser.add_argument( + "outdir", + type=str, + help="output directory to dump the extracted metadata (as global-metadata.dat) and binary (as il2cpp.so)", + **gooey_only(widget="DirChooser"), + ) + il2cpp_parser.set_defaults(func=main_il2cpp) + il2cppextract_parser = subparsers.add_parser( + "il2cpp-extract", + help="Prepare IL2CPP analysis inputs from a JP/EN/CN/TW/KR Android ARM64 APK/XAPK", + description="Extract APK/XAPK inputs and run the existing il2cpp command.", + ) + il2cppextract_parser.add_argument( + "infile", help="input APK or XAPK (including ARM64 splits)", + **gooey_only(widget="FileChooser"), + ) + il2cppextract_parser.add_argument( + "outdir", help="output analysis directory", + **gooey_only(widget="DirChooser"), + ) + il2cppextract_parser.set_defaults(func=main_il2cppextract) + # mvdata + mvdata_parser = subparsers.add_parser( + "mvdata", usage="""Extract MV Data from AssetBundle""" + ) + mvdata_parser.add_argument( + "indir", + type=str, + help="cache directory (live_pv/mv_data)", + **gooey_only(widget="DirChooser"), + ) + mvdata_parser.add_argument( + "outfile", + type=str, + help="output JSON file to dump into", + **gooey_only(widget="FileSaver"), + ) + mvdata_parser.set_defaults(func=main_mvdata) + # moc3paths + moc3paths_parser = subparsers.add_parser( + "moc3paths", usage="""Extract animation path CRCs from raw .moc3 binaries""" + ) + moc3paths_parser.add_argument( + "indir", type=str, help="input directory", **gooey_only(widget="DirChooser") + ) + moc3paths_parser.set_defaults(func=main_moc3paths) + return parser + + +def __main__(): + from tqdm.std import tqdm as tqdm_c + + class TqdmMutexStream: + @staticmethod + def write(__s): + with tqdm_c.external_write_mode(file=sys.stderr, nolock=False): + return sys.stderr.write(__s) + + # parse args + parser = create_parser(argparse.ArgumentParser) + args = parser.parse_args() + # set logging level + import coloredlogs + + coloredlogs.install( + level=args.log_level, + fmt="%(asctime)s | %(levelname).1s | %(name)s %(message)s", + datefmt="%H:%M:%S", + isatty=True, + stream=TqdmMutexStream, + ) + # override unity version + sssekai_set_unity_version(args.unity_version) + if "func" in args: + try: + return args.func(args) + except Exception as e: + logger.exception("Error while running command: %s", e) + traceback.print_exc() + return 1 + else: + parser.print_help() + return 1 + + +if __name__ == "__main__": + sys.exit(__main__() or 0) diff --git a/sssekai/abcache/__init__.py b/sssekai/abcache/__init__.py index 3df59b2..51f39f7 100644 --- a/sssekai/abcache/__init__.py +++ b/sssekai/abcache/__init__.py @@ -1,771 +1,875 @@ -from collections import defaultdict -from pickle import load, dump -from typing import BinaryIO, List, Mapping, Optional, Union, Tuple -from logging import getLogger -from dataclasses import dataclass, fields, is_dataclass, field -from functools import cached_property -from base64 import b64encode, b64decode -import json - -logger = getLogger("sssekai.abcache") - -REGION_JP_EN = {"jp", "en"} -REGION_ROW = {"tw", "kr", "cn"} -REGION_OPTIONS = REGION_JP_EN | REGION_ROW - - -def fromdict(klass: type, d: Union[Mapping, List], warn_missing_fields=True): - """ - Recursively convert a dictionary to a dataclass instance, if applicable. - https://stackoverflow.com/a/54769644 - https://gist.github.com/gatopeich/1efd3e1e4269e1e98fae9983bb914f22 - """ - - while klass.__name__ == "Optional": - klass = klass.__args__[0] # Reduce Optional[T] -> T - - def ensure_iterable(d): - if isinstance(d, Mapping): - return d - if hasattr(d, "__dict__"): - return d.__dict__ - return dict() - - def check_field(key, fields): - if not key in fields: - if warn_missing_fields: - logger.error(f"Field {key} of type {type(d[key])} not found in {klass}") - return False - return True - - if is_dataclass(klass): - fieldtypes = {f.name: f.type for f in fields(klass)} - d = ensure_iterable(d) - return klass( - **( - { - f: fromdict(fieldtypes[f], d[f], warn_missing_fields) - for f in d - if check_field(f, fieldtypes) - } - ) - ) - if isinstance(d, list) and hasattr(klass, "__args__"): - return [fromdict(klass.__args__[0], di, warn_missing_fields) for di in d] - if isinstance(d, dict) and hasattr(klass, "__args__"): - return { - k: fromdict(klass.__args__[1], v, warn_missing_fields) for k, v in d.items() - } - return d - - -from requests import Session, Response, HTTPError -from msgpack import unpackb, packb - -from sssekai import __version__, __version_tuple__ -from sssekai.unity import sssekai_get_unity_version -from sssekai.crypto.APIManager import decrypt, encrypt, SEKAI_APIMANAGER_KEYSETS - - -@dataclass -class AbCacheConfig: - app_region: str - app_version: str - app_platform: str - app_hash: str - - auth_credential: str = None # JWT token for JP/EN, Base64 encoded JWT token for ROW - - row_ab_version: str = None # Override AB version in url for ROW - - asset_host: str = None # Override AssetBundle host for EN/JP - asset_version: str = None # Override AssetBundle version for EN/JP - asset_hash: str = None # Override AssetBundle hash for EN/JP - - version: Tuple[int, int, int] = (0, 0, 0) # Internal versioning - - @property - def auth_jwt(self) -> str: - if self.app_region in REGION_JP_EN: - return self.auth_credential - else: - try: - return b64decode(self.auth_credential).decode() - except: - return None - - @property - def auth_jwt_payload(self) -> dict: - if not self.auth_jwt: - return {} - header, payload, signature = self.auth_jwt.split(".") - return json.loads(b64decode(payload + "==")) - - @property - def auth_userID(self) -> str: - return self.auth_jwt_payload.get("userId", None) - - @property - def auth_available(self): - return self.auth_userID is not None - - @property - def cache_version(self): - return getattr(self, "version", (0, 0, 0)) - - @property - def cache_version_string(self): - return "%s.%s.%s" % self.cache_version if self.cache_version else "pre-0.5.17" - - @property - def is_up_to_date(self): - return self.cache_version == __version_tuple__ - - @property - def need_client_header_update(self): - if self.app_region in REGION_JP_EN: - if self.app_region in {"jp"}: - return not (self.asset_hash and self.asset_host and self.asset_version) - else: - return not (self.asset_hash and self.asset_version) - return False - - -@dataclass -class AbCacheEntry(dict): - bundleName: str - cacheFileName: str - cacheDirectoryName: str - hash: str - category: str - crc: int - fileSize: int - dependencies: List[str] - isBuiltin: bool - isRelocate: Optional[bool] = None # Added in JP 5.4.0 - paths: Optional[List[str]] = None # Removed in ROW - # ROW only - md5Hash: Optional[str] = None - downloadPath: Optional[str] = None - - -@dataclass -class AbCacheIndex(dict): - version: str - os: Optional[str] = None # Undefined in ROW - bundles: Mapping[str, AbCacheEntry] = None - - -# DS for the game's APIs -# There're *plenty* differences across regions. The implementations here -# - Contains all the fields, even if they're not used in all regions -# - Only the fields that are common across regions are non Optional[T] -@dataclass -class SekaiAppVersion: - systemProfile: str - appVersion: str - multiPlayVersion: str - appVersionStatus: str - assetVersion: str - # These are sadly removed from 3.5.0 (JP) - dataVersion: Optional[str] = None - appHash: Optional[str] = None - assetHash: Optional[str] = None - - -@dataclass -class SekaiSystemData: - serverDate: int - # These are all undefined in ROW - timezone: Optional[str] = None - profile: Optional[str] = None - maintenanceStatus: Optional[str] = None - appVersions: Optional[List[SekaiAppVersion]] = None - - @cached_property - def app_version_dict(self): - return {av.appVersion: av for av in self.appVersions or []} - - -@dataclass -class SekaiGameVersionData: - profile: str - assetbundleHostHash: str - domain: str - - -@dataclass -class SekaiUserRegistrationData: - userId: int - # JP/EN only - signature: Optional[str] = None - platform: Optional[str] = None - deviceModel: Optional[str] = None - operatingSystem: Optional[str] = None - registeredAt: Optional[int] = None - - -@dataclass -class SekaiUserData: - userRegistration: SekaiUserRegistrationData - # JP/EN Only - credential: Optional[str] = None - updatedResources: Optional[dict] = None - # ROW - sessionToken: Optional[str] = None - - @property - def user_credentials(self): - return self.credential or self.sessionToken - - -@dataclass -class SekaiUserAuthData: - sessionToken: str - appVersion: str - multiPlayVersion: str - dataVersion: str - assetVersion: str - - # JP/EN only - removeAssetVersion: Optional[str] = None - assetHash: Optional[str] = None - - appVersionStatus: Optional[str] = None - - # JP/EN only - isStreamingVirtualLiveForceOpenUser: Optional[bool] = None - deviceId: Optional[str] = None - updatedResources: Optional[dict] = None - suiteMasterSplitPath: Optional[list] = None - obtainedBondsRewardIds: Optional[list] = None # JP 4.0+ - - # ROW - cdnVersion: Optional[int] = None - configs: Optional[List[dict]] = None - - -@dataclass -class SSSekaiDatabase: - config: AbCacheConfig = None - cached_headers: dict = field(default_factory=dict) - - sekai_user_data: SekaiUserData = None - sekai_user_auth_data: SekaiUserAuthData = None - sekai_abcache_index: AbCacheIndex = None - sekai_system_data: SekaiSystemData = None - sekai_gameversion_data: SekaiGameVersionData = None - - def reset(self): - self.sekai_user_data = None - self.sekai_user_auth_data = None - self.sekai_abcache_index = None - self.sekai_system_data = None - self.sekai_gameversion_data = None - - -class AbCacheBundleNotFoundError(Exception): - bundleName: str - - def __init__(self, bundleName: str): - self.bundleName = bundleName - super().__init__("Bundle not found: %s" % bundleName) - - -class AbCacheUserNotAuthenticatedError(Exception): - pass - - -class AbCache(Session): - database: SSSekaiDatabase - - @property - def SEKAI_API_ENDPOINT(self): - match self.config.app_region: - case "jp": - if self.database.sekai_gameversion_data: - url = self.database.sekai_gameversion_data.domain - url = url.replace( - "{0}", self.database.sekai_gameversion_data.profile - ) - return "https://%s" % url - return "https://production-game-api.sekai.colorfulpalette.org" - case "en": - return f"https://n-{self.SEKAI_API_PROFILE}-game-api.sekai-en.com" - case "tw": - return "https://mkovs-20004-cdn.bytedgame.com" - case "kr": - return "https://mkkorea-obt-prod01-cdn.bytedgame.com" - case "cn": - return "https://mkcn-prod-public-60001-1.dailygn.com" - case _: - raise NotImplementedError - - @property - def SEKAI_API_GAMEVERSION_ENDPOINT(self): - match self.config.app_region: - case "jp": - return "https://game-version.sekai.colorfulpalette.org" - case "en": - return "https://game-version.sekai-en.com" - case _: - raise NotImplementedError - - @property - def SEKAI_ISSUE_ENDPOINT(self): - match self.config.app_region: - case "jp": - return "https://issue.sekai.colorfulpalette.org" - case _: - raise NotImplementedError - - @property - def SEKAI_AB_ROW_CDN(self): - match self.config.app_region: - case "tw": - return "https://lf16-mkovscdn-sg.bytedgame.com/obj/sf-game-alisg/gdl_app_5245" - case "kr": - return ( - "https://lf19-mkkr.bytedgame.com/obj/sf-game-alisg/gdl_app_292248" - ) - case "cn": - return "https://lf3-mkcncdn-tos.dailygn.com/obj/sf-game-lf/gdl_app_5236" - case _: - raise NotImplementedError - - @property - def SEKAI_AB_ROW_PATH(self): - match self.config.app_region: - case "tw": - return "online" - case "kr": - return "kr_online" - case "cn": - return "cn_online" - case _: - raise NotImplementedError - - @cached_property - def SEKAI_AB_ROW_VERSION_NUMBER(self): - url = f"{self.SEKAI_AB_ROW_CDN.replace('sf-game','rt-game')}" - if self.config.app_region in {"cn"}: - url = f"{url}/Mainland" - else: - url = f"{url}/Oversea" - url = f"{url}/{self.SEKAI_AB_ROW_VERSION}/Release/{self.SEKAI_AB_ROW_PATH}/{self.SEKAI_APP_PLATFORM}/version" - resp = self.request("GET", url) - resp.raise_for_status() - return int(resp.text) - - @property - def SEKAI_AB_INFO_ENDPOINT(self): - match self.config.app_region: - case "jp": - return f"https://{self.SEKAI_API_PROFILE}-{self.SEKAI_AB_HOST_HASH}-assetbundle-info.sekai.colorfulpalette.org/api/version/{self.SEKAI_ASSET_VERSION}/{self.SEKAI_AB_HASH}/os/{self.config.app_platform}" - case "en": - # `n` for what, exactly? - return f"https://n-{self.SEKAI_API_PROFILE}-{self.SEKAI_AB_HOST_HASH}-assetbundle-info.sekai-en.com/api/version/{self.SEKAI_ASSET_VERSION}/os/{self.config.app_platform}" - if self.config.app_region in REGION_ROW: - return f"{self.SEKAI_AB_ROW_CDN}/AssetBundle/{self.SEKAI_AB_ROW_VERSION}/Release/{self.SEKAI_AB_ROW_PATH}/{self.SEKAI_APP_PLATFORM}{self.SEKAI_AB_ROW_VERSION_NUMBER}/AssetBundleInfoNew.json" - else: - raise NotImplementedError - - @property - def SEKAI_AB_ENDPOINT(self): - match self.config.app_region: - case "jp": - return f"https://{self.SEKAI_API_PROFILE}-{self.SEKAI_AB_HOST_HASH}-assetbundle.sekai.colorfulpalette.org/" - case "en": - return f"https://n-{self.SEKAI_API_PROFILE}-{self.SEKAI_AB_HOST_HASH}-assetbundle.sekai-en.com/" - if self.config.app_region in REGION_ROW: - return f"{self.SEKAI_AB_ROW_CDN}/AssetBundle/{self.SEKAI_AB_ROW_VERSION}/Release/{self.SEKAI_AB_ROW_PATH}/" - else: - raise NotImplementedError - - def _update_request_headers(self): - app_platform_names = { - 'ios':'iOS', 'android':'Android' - } - self.headers.update( - { - "Accept": "application/octet-stream", - "Content-Type": "application/octet-stream", - "Accept-Encoding": "deflate, gzip", - "User-Agent": "UnityPlayer/%s" % sssekai_get_unity_version(), - "X-Platform": app_platform_names.get(self.config.app_platform,self.config.app_platform), - "X-DeviceModel": "sssekai/%s" % __version__, - "X-OperatingSystem": app_platform_names.get(self.config.app_platform,self.config.app_platform), - "X-Unity-Version": sssekai_get_unity_version(), - "X-App-Version": self.SEKAI_APP_VERSION, - "X-App-Hash": self.SEKAI_APP_HASH, - } - ) - - @property - def config(self): - return self.database.config - - @config.setter - def config(self, v): - self.database.config = v - - @property - def SEKAI_APP_VERSION(self): - return self.config.app_version - - @property - def SEKAI_AB_ROW_VERSION(self): - return self.config.row_ab_version or self.SEKAI_APP_VERSION - - @property - def SEKAI_APP_PLATFORM(self): - return self.config.app_platform - - @property - def SEKAI_APP_HASH(self): - return self.config.app_hash - - @property - def SEKAI_ASSET_VERSION(self): - return ( - self.config.asset_version - or self.database.sekai_system_data.app_version_dict[ - self.SEKAI_APP_VERSION - ].assetVersion - ) - - @property - def SEKAI_AB_HASH(self): - return self.config.asset_hash or self.database.sekai_user_auth_data.assetHash - - @property - def SEKAI_AB_HOST_HASH(self): - return ( - self.config.asset_host - or self.database.sekai_gameversion_data.assetbundleHostHash - ) - - @property - def SEKAI_API_PROFILE(self): - if self.database.sekai_gameversion_data: - return self.database.sekai_gameversion_data.profile - return "production" - - @property - def SEKAI_API_SYSTEM_DATA(self): - return self.SEKAI_API_ENDPOINT + "/api/system" - - @property - def SEKAI_API_USER(self): - return self.SEKAI_API_ENDPOINT + "/api/user" - - @property - def SEKAI_USERID(self): - return self.config.auth_userID or ( - self.database.sekai_user_data.userRegistration.userId - if self.database.sekai_user_data - else None - ) - - @property - def SEKAI_CREDENTIAL(self): - """Keyed in SEKAI_API_USER responses. `accessToken` in ROW servers, `credential` in JP/EN.""" - return self.config.auth_credential or ( - self.database.sekai_user_data.credential - if self.database.sekai_user_data - else None - ) - - @property - def is_authenticated(self): - return self.database.sekai_user_auth_data is not None - - def raise_for_auth(self, *args): - if not self.is_authenticated: - raise AbCacheUserNotAuthenticatedError(*args) - - @property - def SEKAI_API_USER_AUTH(self): - if self.config.app_region in REGION_JP_EN: - assert self.SEKAI_USERID, "User ID must be available" - return f"{self.SEKAI_API_USER}/{self.SEKAI_USERID}/auth" - else: - # For ROW this is the endpoint that retrives SEKAI_API_USER level data as well - return f"{self.SEKAI_API_USER}/auth" - - @property - def SEKAI_API_USER_SUITE(self): - self.raise_for_auth() - return f"{self.SEKAI_API_ENDPOINT}/api/suite/user/{self.SEKAI_USERID}" - - @property - def SEKAI_API_MASTER_SUITE_URLS(self) -> List[str]: - match self.config.app_region: - case "jp": - self.raise_for_auth() - return [ - f"{self.SEKAI_API_ENDPOINT}/api/{path}" - for path in self.database.sekai_user_auth_data.suiteMasterSplitPath - ] - case "en": - self.raise_for_auth() - return [ - f"{self.SEKAI_API_ENDPOINT}/api/{path}" - for path in self.database.sekai_user_auth_data.suiteMasterSplitPath - ] - if self.config.app_region in REGION_ROW: - self.raise_for_auth() - return [f"{self.SEKAI_AB_ROW_CDN}/{self.SEKAI_AB_ROW_PATH}/master-data-{self.database.sekai_user_auth_data.cdnVersion}.info"] - else: - raise NotImplementedError - - @property - def SEKAI_API_INFORMATION(self): - return self.SEKAI_API_ENDPOINT + "/api/information" - - @property - def SEKAI_AB_BASE_PATH(self): - return f"{self.SEKAI_ASSET_VERSION}/{self.SEKAI_AB_HASH}/{self.config.app_platform}/" - - @property - def SEKAI_ISSUE_SIGNATURE_ENDPOINT(self): - return self.SEKAI_ISSUE_ENDPOINT + "/api/signature" - - @property - def abcache_index(self): - return self.database.sekai_abcache_index - - def request_packed(self, method: str, url: str, data: dict = None, **kwargs): - """Send a request with packed data. Data will be packed and encrypted before sending. - - Args: - method (str): HTTP method - url (str): URL - data (dict, optional): Payload data. Defaults to None. - - Returns: - Response: Response object - """ - self._update_request_headers() - if data is not None: - # XXX: packb() does not cover all cases. - # - # Python msgpack implementation doesn't really support variadic float precision within a single pack. - # This generally shouldn't be an issue if the upstream unpacking routine is type-agnostic since - # the type is described in the packed binary as per the spec. - # - # However, if the upstream implementation is not type-agnostic, it WILL lead to issues. - # While most respect the type opcodes (e.g. msgpack-c, msgpack-python, msgpack in Golang), some implementations (e.g. MessagePack-CSharp) disregards - # them altogether when concrete (C#) types are provided. And will raise an exception in such cases when a double is encoded in the place of a float. - # See also: - # https://github.com/mos9527/sssekai/issues/47#issuecomment-3150827990 - # - # Thus `use_single_float=True` is used to cover and only cover single precision cases. - # In the *current* API seen in the game binary, this is fine. And in AbCache's case floats are not at all used. - # But this may change in the future, in which case consult the following issues: - # https://github.com/msgpack/msgpack-python/issues/326 - data = packb(data, use_single_float=True) - data = encrypt(data, SEKAI_APIMANAGER_KEYSETS[self.config.app_region]) - resp = self.request(method=method, url=url, data=data, **kwargs) - if 400 <= resp.status_code < 600: - logger.error(f"{method} {url} {resp.status_code}") - try: # log the error message provided by the API. - content = self.response_to_dict(resp) - logger.error("response=%s" % content) - except: - logger.error("response=%s" % resp.content) - logger.error( - "Unexpected server-side error. Report issues at https://github.com/MetaSekaiLab/sssekai/issues." - ) - resp.raise_for_status() - return resp - - def response_to_dict(self, resp: Response, **kwargs): - """Decrypt and unpack a response content to a dictionary. - - Args: - resp (Response): Response object - **kwargs: Additional arguments for MessagePack unpackb - - Returns: - dict: Decrypted and unpacked data - """ - data = decrypt(resp.content, SEKAI_APIMANAGER_KEYSETS[self.config.app_region]) - data = unpackb(data, **kwargs) - return data - - def _update_user_auth_data(self): - if self.config.auth_available: - logger.debug("Updating user auth data") - if self.config.app_region in REGION_JP_EN: - resp = self.request_packed( - "PUT", - self.SEKAI_API_USER_AUTH, - data={"credential": self.SEKAI_CREDENTIAL, "deviceId": None}, - ) - else: - resp = self.request_packed( - "POST", - self.SEKAI_API_USER_AUTH, - data={ - "accessToken": self.SEKAI_CREDENTIAL, - "deviceId": None, - "userID": 0, - }, - ) - data = self.response_to_dict(resp) - self.database.sekai_user_auth_data = fromdict( - SekaiUserAuthData, data, False - ) - if self.config.app_region in REGION_ROW: - self.database.sekai_user_data = fromdict(SekaiUserData, data, False) - else: - logger.warning( - "No valid auth credential provided. User auth data will not be updated." - ) - - def _update_system_data(self): - logger.debug("Updating system data") - resp = self.request_packed("GET", self.SEKAI_API_SYSTEM_DATA) - data = self.response_to_dict(resp) - self.database.sekai_system_data = fromdict(SekaiSystemData, data) - - def _update_gameversion_data(self): - if self.config.app_region in REGION_JP_EN: - logger.debug("Updating game version data") - resp = self.request_packed( - "GET", - self.SEKAI_API_GAMEVERSION_ENDPOINT - + "/" - + self.SEKAI_APP_VERSION - + "/" - + self.SEKAI_APP_HASH, - ) - data = self.response_to_dict(resp) - self.database.sekai_gameversion_data = fromdict(SekaiGameVersionData, data) - - def __init__(self, config: Optional[AbCacheConfig] = None): - super().__init__() - self.database = SSSekaiDatabase() - self.config = config or AbCacheConfig( - "unknown", "unknown", "unknown", "unknown" - ) - self.config.version = __version_tuple__ - - def update_client_headers(self): - """Authenticate the user and update client headers.""" - logger.debug("Updating metadata") - logger.debug("Set config: %s" % self.config) - try: - self.update_signatures() - self._update_gameversion_data() - self._update_system_data() - except HTTPError as e: - logger.error("Signature update failure: %s" % e) - raise e - if self.config.app_region in REGION_JP_EN: - version_newest = self.database.sekai_system_data.appVersions[-1] - logger.debug("Newest App version: %s" % version_newest) - if version_newest.appVersion != self.SEKAI_APP_VERSION: - logger.warning("App version mismatch. This may cause issues.") - self._update_user_auth_data() - if self.is_authenticated: - self.headers.update( - { - "X-Data-Version": self.database.sekai_user_auth_data.dataVersion, - "X-Asset-Version": self.database.sekai_user_auth_data.assetVersion, - "X-Session-Token": self.database.sekai_user_auth_data.sessionToken, - } - ) - return self.headers - - def update_abcache_index(self) -> AbCacheIndex: - """Update the AssetBundle Index.""" - logger.debug("Updating Assetbundle index") - if self.config.app_region in REGION_JP_EN: - logger.info("Sekai AssetBundle version: %s" % self.SEKAI_ASSET_VERSION) - if self.config.app_region in {"jp"}: - logger.info("Sekai AssetBundle host hash: %s" % self.SEKAI_AB_HOST_HASH) - resp = self.request_packed("GET", self.SEKAI_AB_INFO_ENDPOINT) - data = self.response_to_dict(resp) - self.database.sekai_abcache_index = fromdict(AbCacheIndex, data) - return self.database.sekai_abcache_index - - def update_signatures(self): - """Update client signatures. Only required for JP for the initial setup. - After which the game would cache the signature/cookies.""" - if self.config.app_region in {"jp"}: - logger.debug("Updating signatures") - # XXX: Clear all cookies, it 403s otherwise. Does this trigger a rate limit/expiry check since our session would be tracked? - self.headers["Cookie"] = "" - self.cookies.clear() - resp = self.request_packed("POST", self.SEKAI_ISSUE_SIGNATURE_ENDPOINT) - self.headers["Cookie"] = resp.headers["Set-Cookie"] - # HACK: Per RFC6265, Cookies should not be visible to subdomains since it's not set with Domain attribute (https://github.com/psf/requests/issues/2576) - # But the other endpoints uses it nontheless. So we have to set it manually. - - def update(self): - """Update the cache with the latest AssetBundle Index data from the server. - - NOTE: - AbCache should have been initialized with a valid config before calling this method. - Please refer to `update_client_headers` for more information. - """ - self.update_client_headers() - self.update_abcache_index() - - def save(self, f: BinaryIO): - self._update_request_headers() - self.database.cached_headers = self.headers - self.database.config.version = __version_tuple__ - logger.info("Saving cache: %s" % self) - dump(self.database, f) - - def load(self, f: BinaryIO): - logger.info("Loading cache") - self.database = load(f) - if not self.database.config.is_up_to_date: - logger.warning( - "Cache is outdated (cache: %s, current: %s). It's HIGHLY recommended to update the cache to avoid issues." - % (self.database.config.cache_version_string, __version__) - ) - self.headers.update(self.database.cached_headers) - logger.info("Cache loaded: %s" % self) - - @staticmethod - def from_file(f: BinaryIO): - cache = AbCache() - cache.load(f) - return cache - - def __repr__(self) -> str: - return f"AbCache(config={self.config} bundles={len(self.abcache_index.bundles) if self.abcache_index is not None else 'N/A'})" - - def get_entry_by_bundle_name(self, bundleName: str) -> AbCacheEntry: - return self.abcache_index.bundles.get(bundleName, None) - - def get_entry_download_url(self, entry: AbCacheEntry): - if self.config.app_region in REGION_JP_EN: - return self.SEKAI_AB_ENDPOINT + self.SEKAI_AB_BASE_PATH + entry.bundleName - else: - # https://github.com/mos9527/sssekai/issues/28 - return self.SEKAI_AB_ENDPOINT + entry.downloadPath + "/" + entry.bundleName - - def get_or_update_dependency_tree_flatten(self, bundleName: str, deps: set = None): - """Get a flattened set of asset dependency bundle names (including itself) for a given entry. - - Returned order is topologically sorted. - - if 'deps' is provided, it will be used as the initial set of dependencies. - """ - deps = deps or set() - - def dfs(bundleName): - for dep in self.abcache_index.bundles[bundleName].dependencies: - if dep not in deps: - dfs(dep) - deps.add(dep) - - dfs(bundleName) - deps.add(bundleName) - return deps +from collections import defaultdict +from pickle import load, dump +from typing import BinaryIO, List, Mapping, Optional, Union, Tuple +from logging import getLogger +from dataclasses import dataclass, fields, is_dataclass, field +from functools import cached_property +from base64 import b64encode, b64decode +import json + +logger = getLogger("sssekai.abcache") + +REGION_JP_EN = {"jp", "en"} +REGION_ROW = {"tw", "kr", "cn"} +REGION_OPTIONS = REGION_JP_EN | REGION_ROW + + +class RedactedStr(str): + """A credential that behaves like a str but hides its value in repr. + + `dataclasses.asdict` deep-copies values, so `field(repr=False)` alone does + not keep tokens out of `repr(asdict(...))`. Pickles as a plain str so cache + files stay compatible with older versions. + """ + + def __repr__(self): + return "" + + def __deepcopy__(self, memo): + return self + + def __reduce_ex__(self, protocol): + return str, (str(self),) + + +def fromdict(klass: type, d: Union[Mapping, List], warn_missing_fields=True): + """ + Recursively convert a dictionary to a dataclass instance, if applicable. + https://stackoverflow.com/a/54769644 + https://gist.github.com/gatopeich/1efd3e1e4269e1e98fae9983bb914f22 + """ + + while klass.__name__ == "Optional": + klass = klass.__args__[0] # Reduce Optional[T] -> T + + def ensure_iterable(d): + if isinstance(d, Mapping): + return d + if hasattr(d, "__dict__"): + return d.__dict__ + return dict() + + def check_field(key, fields): + if not key in fields: + if warn_missing_fields: + logger.error(f"Field {key} of type {type(d[key])} not found in {klass}") + return False + return True + + if is_dataclass(klass): + fieldtypes = {f.name: f.type for f in fields(klass)} + d = ensure_iterable(d) + return klass( + **( + { + f: fromdict(fieldtypes[f], d[f], warn_missing_fields) + for f in d + if check_field(f, fieldtypes) + } + ) + ) + if isinstance(d, list) and hasattr(klass, "__args__"): + return [fromdict(klass.__args__[0], di, warn_missing_fields) for di in d] + if isinstance(d, dict) and hasattr(klass, "__args__"): + return { + k: fromdict(klass.__args__[1], v, warn_missing_fields) for k, v in d.items() + } + return d + + +from requests import Session, Response, HTTPError +from msgpack import unpackb, packb + +from sssekai import __version__, __version_tuple__ +from sssekai.unity import sssekai_get_unity_version +from sssekai.crypto.APIManager import decrypt, encrypt, SEKAI_APIMANAGER_KEYSETS + + +@dataclass +class AbCacheConfig: + app_region: str + app_version: str + app_platform: str + app_hash: str + + auth_credential: str = field( + default=None, repr=False + ) # JWT token for JP/EN, Base64 encoded JWT token for ROW + + row_ab_version: str = None # Override AB version in url for ROW + + asset_host: str = None # Override AssetBundle host for EN/JP + asset_version: str = None # Override AssetBundle version for EN/JP + asset_hash: str = None # Override AssetBundle hash for EN/JP + + version: Tuple[int, int, int] = (0, 0, 0) # Internal versioning + + def __setattr__(self, name, value): + # Credentials are also assigned after construction (CLI, auth.py). + if name == "auth_credential" and value is not None: + if not isinstance(value, RedactedStr): + value = RedactedStr(value) + super().__setattr__(name, value) + + @property + def auth_jwt(self) -> str: + if self.app_region in REGION_JP_EN: + return self.auth_credential + else: + try: + return b64decode(self.auth_credential).decode() + except: + return None + + @property + def auth_jwt_payload(self) -> dict: + if not self.auth_jwt: + return {} + header, payload, signature = self.auth_jwt.split(".") + return json.loads(b64decode(payload + "==")) + + @property + def auth_userID(self) -> str: + return self.auth_jwt_payload.get("userId", None) + + @property + def auth_available(self): + return self.auth_userID is not None + + @property + def cache_version(self): + return getattr(self, "version", (0, 0, 0)) + + @property + def cache_version_string(self): + return "%s.%s.%s" % self.cache_version if self.cache_version else "pre-0.5.17" + + @property + def is_up_to_date(self): + return self.cache_version == __version_tuple__ + + @property + def need_client_header_update(self): + if self.app_region in REGION_JP_EN: + if self.app_region in {"jp"}: + return not (self.asset_hash and self.asset_host and self.asset_version) + else: + return not (self.asset_hash and self.asset_version) + return False + + +@dataclass +class AbCacheEntry(dict): + bundleName: str + cacheFileName: str + cacheDirectoryName: str + hash: str + category: str + crc: int + fileSize: int + dependencies: List[str] + isBuiltin: bool + isRelocate: Optional[bool] = None # Added in JP 5.4.0 + paths: Optional[List[str]] = None # Removed in ROW + # ROW only + md5Hash: Optional[str] = None + downloadPath: Optional[str] = None + + +@dataclass +class AbCacheIndex(dict): + version: str + os: Optional[str] = None # Undefined in ROW + bundles: Mapping[str, AbCacheEntry] = None + + +# DS for the game's APIs +# There're *plenty* differences across regions. The implementations here +# - Contains all the fields, even if they're not used in all regions +# - Only the fields that are common across regions are non Optional[T] +@dataclass +class SekaiAppVersion: + systemProfile: str + appVersion: str + multiPlayVersion: str + appVersionStatus: str + assetVersion: str + # These are sadly removed from 3.5.0 (JP) + dataVersion: Optional[str] = None + appHash: Optional[str] = None + assetHash: Optional[str] = None + + +@dataclass +class SekaiSystemData: + serverDate: int + # These are all undefined in ROW + timezone: Optional[str] = None + profile: Optional[str] = None + maintenanceStatus: Optional[str] = None + appVersions: Optional[List[SekaiAppVersion]] = None + + @cached_property + def app_version_dict(self): + return {av.appVersion: av for av in self.appVersions or []} + + +@dataclass +class SekaiGameVersionData: + profile: str + assetbundleHostHash: str + domain: str + + +@dataclass +class SekaiUserRegistrationData: + userId: int + # JP/EN only + signature: Optional[str] = None + platform: Optional[str] = None + deviceModel: Optional[str] = None + operatingSystem: Optional[str] = None + registeredAt: Optional[int] = None + + +@dataclass +class SekaiUserData: + userRegistration: SekaiUserRegistrationData + # JP/EN Only + credential: Optional[str] = field(default=None, repr=False) + updatedResources: Optional[dict] = None + # ROW + sessionToken: Optional[str] = field(default=None, repr=False) + + def __setattr__(self, name, value): + if name in ("credential", "sessionToken") and value is not None: + if not isinstance(value, RedactedStr): + value = RedactedStr(value) + super().__setattr__(name, value) + + @property + def user_credentials(self): + return self.credential or self.sessionToken + + +@dataclass +class SekaiUserAuthData: + sessionToken: str = field(repr=False) + appVersion: str + multiPlayVersion: str + dataVersion: str + assetVersion: str + + # JP/EN only + removeAssetVersion: Optional[str] = None + assetHash: Optional[str] = None + + appVersionStatus: Optional[str] = None + + # JP/EN only + isStreamingVirtualLiveForceOpenUser: Optional[bool] = None + deviceId: Optional[str] = None + updatedResources: Optional[dict] = None + suiteMasterSplitPath: Optional[list] = None + obtainedBondsRewardIds: Optional[list] = None # JP 4.0+ + + # ROW + cdnVersion: Optional[int] = None + configs: Optional[List[dict]] = None + + def __setattr__(self, name, value): + if name == "sessionToken" and value is not None: + if not isinstance(value, RedactedStr): + value = RedactedStr(value) + super().__setattr__(name, value) + + +class RedactedHeaders(dict): + """Request headers kept for cache reuse. Hidden from repr/asdict so tokens + (e.g. X-Session-Token) never reach logs. Pickles as a plain dict so cache + files stay loadable by both old and new versions.""" + + def __repr__(self): + return "" % len(self) + + def __deepcopy__(self, memo): + # dict's default deepcopy returns a plain dict, which would leak through + # dataclasses.asdict(). Header values are str, so a shallow copy suffices. + return RedactedHeaders(self) + + def __reduce_ex__(self, protocol): + return dict, (dict(self),) + + +@dataclass +class SSSekaiDatabase: + config: AbCacheConfig = None + cached_headers: dict = field(default_factory=RedactedHeaders) + + sekai_user_data: SekaiUserData = None + sekai_user_auth_data: SekaiUserAuthData = None + sekai_abcache_index: AbCacheIndex = None + sekai_system_data: SekaiSystemData = None + sekai_gameversion_data: SekaiGameVersionData = None + + def __repr__(self) -> str: + # Two layers guard this: RedactedHeaders hides its own values, and this + # repr also covers a plain dict (databases built by hand, or caches not + # yet normalized by load()) that would otherwise print in full. + values = ", ".join( + "%s=%r" % (f.name, getattr(self, f.name, None)) + for f in fields(self) + if f.name != "cached_headers" + ) + return "SSSekaiDatabase(%s, cached_headers=<%d values>)" % ( + values, + len(self.cached_headers or ()), + ) + + def reset(self): + self.sekai_user_data = None + self.sekai_user_auth_data = None + self.sekai_abcache_index = None + self.sekai_system_data = None + self.sekai_gameversion_data = None + + def redact_secrets(self): + """Re-wrap secrets loaded from disk as RedactedStr. + + RedactedStr pickles as a plain str (so caches stay readable by older + versions), so values read back from a cache file must be wrapped again + before they can reach a repr/asdict. Assignment goes through each + dataclass's __setattr__, which does the wrapping. + """ + if not isinstance(self.cached_headers, RedactedHeaders): + self.cached_headers = RedactedHeaders(self.cached_headers or {}) + # sssekai <= 0.8.2 stored the raw API response here, so old caches may + # hold a plain dict (which also breaks attribute access such as + # SEKAI_CREDENTIAL). Normalize it to the dataclass we expect. + if isinstance(self.sekai_user_data, dict): + self.sekai_user_data = fromdict( + SekaiUserData, self.sekai_user_data, False + ) + secrets = [] + if self.config is not None: + secrets.append((self.config, ("auth_credential",))) + secrets += [ + (self.sekai_user_data, ("credential", "sessionToken")), + (self.sekai_user_auth_data, ("sessionToken",)), + ] + for owner, names in secrets: + if owner is None: + continue + for name in names: + value = getattr(owner, name, None) + if value is not None: + setattr(owner, name, value) + + +class AbCacheBundleNotFoundError(Exception): + bundleName: str + + def __init__(self, bundleName: str): + self.bundleName = bundleName + super().__init__("Bundle not found: %s" % bundleName) + + +class AbCacheUserNotAuthenticatedError(Exception): + pass + + +class AbCache(Session): + database: SSSekaiDatabase + + @property + def SEKAI_API_ENDPOINT(self): + match self.config.app_region: + case "jp": + if self.database.sekai_gameversion_data: + url = self.database.sekai_gameversion_data.domain + url = url.replace( + "{0}", self.database.sekai_gameversion_data.profile + ) + return "https://%s" % url + return "https://production-game-api.sekai.colorfulpalette.org" + case "en": + return f"https://n-{self.SEKAI_API_PROFILE}-game-api.sekai-en.com" + case "tw": + return "https://mkovs-20004-cdn.bytedgame.com" + case "kr": + return "https://mkkorea-obt-prod01-cdn.bytedgame.com" + case "cn": + return "https://mkcn-prod-public-60001-1.dailygn.com" + case _: + raise NotImplementedError + + @property + def SEKAI_API_GAMEVERSION_ENDPOINT(self): + match self.config.app_region: + case "jp": + return "https://game-version.sekai.colorfulpalette.org" + case "en": + return "https://game-version.sekai-en.com" + case _: + raise NotImplementedError + + @property + def SEKAI_ISSUE_ENDPOINT(self): + match self.config.app_region: + case "jp": + return "https://issue.sekai.colorfulpalette.org" + case _: + raise NotImplementedError + + @property + def SEKAI_AB_ROW_CDN(self): + match self.config.app_region: + case "tw": + return "https://lf16-mkovscdn-sg.bytedgame.com/obj/sf-game-alisg/gdl_app_5245" + case "kr": + return ( + "https://lf19-mkkr.bytedgame.com/obj/sf-game-alisg/gdl_app_292248" + ) + case "cn": + return "https://lf3-mkcncdn-tos.dailygn.com/obj/sf-game-lf/gdl_app_5236" + case _: + raise NotImplementedError + + @property + def SEKAI_AB_ROW_PATH(self): + match self.config.app_region: + case "tw": + return "online" + case "kr": + return "kr_online" + case "cn": + return "cn_online" + case _: + raise NotImplementedError + + @cached_property + def SEKAI_AB_ROW_VERSION_NUMBER(self): + url = f"{self.SEKAI_AB_ROW_CDN.replace('sf-game','rt-game')}" + if self.config.app_region in {"cn"}: + url = f"{url}/Mainland" + else: + url = f"{url}/Oversea" + url = f"{url}/{self.SEKAI_AB_ROW_VERSION}/Release/{self.SEKAI_AB_ROW_PATH}/{self.SEKAI_APP_PLATFORM}/version" + resp = self.request("GET", url) + resp.raise_for_status() + return int(resp.text) + + @property + def SEKAI_AB_INFO_ENDPOINT(self): + match self.config.app_region: + case "jp": + return f"https://{self.SEKAI_API_PROFILE}-{self.SEKAI_AB_HOST_HASH}-assetbundle-info.sekai.colorfulpalette.org/api/version/{self.SEKAI_ASSET_VERSION}/{self.SEKAI_AB_HASH}/os/{self.config.app_platform}" + case "en": + # `n` for what, exactly? + return f"https://n-{self.SEKAI_API_PROFILE}-{self.SEKAI_AB_HOST_HASH}-assetbundle-info.sekai-en.com/api/version/{self.SEKAI_ASSET_VERSION}/os/{self.config.app_platform}" + if self.config.app_region in REGION_ROW: + return f"{self.SEKAI_AB_ROW_CDN}/AssetBundle/{self.SEKAI_AB_ROW_VERSION}/Release/{self.SEKAI_AB_ROW_PATH}/{self.SEKAI_APP_PLATFORM}{self.SEKAI_AB_ROW_VERSION_NUMBER}/AssetBundleInfoNew.json" + else: + raise NotImplementedError + + @property + def SEKAI_AB_ENDPOINT(self): + match self.config.app_region: + case "jp": + return f"https://{self.SEKAI_API_PROFILE}-{self.SEKAI_AB_HOST_HASH}-assetbundle.sekai.colorfulpalette.org/" + case "en": + return f"https://n-{self.SEKAI_API_PROFILE}-{self.SEKAI_AB_HOST_HASH}-assetbundle.sekai-en.com/" + if self.config.app_region in REGION_ROW: + return f"{self.SEKAI_AB_ROW_CDN}/AssetBundle/{self.SEKAI_AB_ROW_VERSION}/Release/{self.SEKAI_AB_ROW_PATH}/" + else: + raise NotImplementedError + + def _update_request_headers(self): + app_platform_names = { + 'ios':'iOS', 'android':'Android' + } + self.headers.update( + { + "Accept": "application/octet-stream", + "Content-Type": "application/octet-stream", + "Accept-Encoding": "deflate, gzip", + "User-Agent": "UnityPlayer/%s" % sssekai_get_unity_version(), + "X-Platform": app_platform_names.get(self.config.app_platform,self.config.app_platform), + "X-DeviceModel": "sssekai/%s" % __version__, + "X-OperatingSystem": app_platform_names.get(self.config.app_platform,self.config.app_platform), + "X-Unity-Version": sssekai_get_unity_version(), + "X-App-Version": self.SEKAI_APP_VERSION, + "X-App-Hash": self.SEKAI_APP_HASH, + } + ) + + @property + def config(self): + return self.database.config + + @config.setter + def config(self, v): + self.database.config = v + + @property + def SEKAI_APP_VERSION(self): + return self.config.app_version + + @property + def SEKAI_AB_ROW_VERSION(self): + return self.config.row_ab_version or self.SEKAI_APP_VERSION + + @property + def SEKAI_APP_PLATFORM(self): + return self.config.app_platform + + @property + def SEKAI_APP_HASH(self): + return self.config.app_hash + + @property + def SEKAI_ASSET_VERSION(self): + return ( + self.config.asset_version + or self.database.sekai_system_data.app_version_dict[ + self.SEKAI_APP_VERSION + ].assetVersion + ) + + @property + def SEKAI_AB_HASH(self): + return self.config.asset_hash or self.database.sekai_user_auth_data.assetHash + + @property + def SEKAI_AB_HOST_HASH(self): + return ( + self.config.asset_host + or self.database.sekai_gameversion_data.assetbundleHostHash + ) + + @property + def SEKAI_API_PROFILE(self): + if self.database.sekai_gameversion_data: + return self.database.sekai_gameversion_data.profile + return "production" + + @property + def SEKAI_API_SYSTEM_DATA(self): + return self.SEKAI_API_ENDPOINT + "/api/system" + + @property + def SEKAI_API_USER(self): + return self.SEKAI_API_ENDPOINT + "/api/user" + + @property + def SEKAI_USERID(self): + return self.config.auth_userID or ( + self.database.sekai_user_data.userRegistration.userId + if self.database.sekai_user_data + else None + ) + + @property + def SEKAI_CREDENTIAL(self): + """Keyed in SEKAI_API_USER responses. `accessToken` in ROW servers, `credential` in JP/EN.""" + return self.config.auth_credential or ( + self.database.sekai_user_data.credential + if self.database.sekai_user_data + else None + ) + + @property + def is_authenticated(self): + return self.database.sekai_user_auth_data is not None + + def raise_for_auth(self, *args): + if not self.is_authenticated: + raise AbCacheUserNotAuthenticatedError(*args) + + @property + def SEKAI_API_USER_AUTH(self): + if self.config.app_region in REGION_JP_EN: + assert self.SEKAI_USERID, "User ID must be available" + return f"{self.SEKAI_API_USER}/{self.SEKAI_USERID}/auth" + else: + # For ROW this is the endpoint that retrives SEKAI_API_USER level data as well + return f"{self.SEKAI_API_USER}/auth" + + @property + def SEKAI_API_USER_SUITE(self): + self.raise_for_auth() + return f"{self.SEKAI_API_ENDPOINT}/api/suite/user/{self.SEKAI_USERID}" + + @property + def SEKAI_API_MASTER_SUITE_URLS(self) -> List[str]: + match self.config.app_region: + case "jp": + self.raise_for_auth() + return [ + f"{self.SEKAI_API_ENDPOINT}/api/{path}" + for path in self.database.sekai_user_auth_data.suiteMasterSplitPath + ] + case "en": + self.raise_for_auth() + return [ + f"{self.SEKAI_API_ENDPOINT}/api/{path}" + for path in self.database.sekai_user_auth_data.suiteMasterSplitPath + ] + if self.config.app_region in REGION_ROW: + self.raise_for_auth() + return [f"{self.SEKAI_AB_ROW_CDN}/{self.SEKAI_AB_ROW_PATH}/master-data-{self.database.sekai_user_auth_data.cdnVersion}.info"] + else: + raise NotImplementedError + + @property + def SEKAI_API_INFORMATION(self): + return self.SEKAI_API_ENDPOINT + "/api/information" + + @property + def SEKAI_AB_BASE_PATH(self): + return f"{self.SEKAI_ASSET_VERSION}/{self.SEKAI_AB_HASH}/{self.config.app_platform}/" + + @property + def SEKAI_ISSUE_SIGNATURE_ENDPOINT(self): + return self.SEKAI_ISSUE_ENDPOINT + "/api/signature" + + @property + def abcache_index(self): + return self.database.sekai_abcache_index + + def request_packed(self, method: str, url: str, data: dict = None, **kwargs): + """Send a request with packed data. Data will be packed and encrypted before sending. + + Args: + method (str): HTTP method + url (str): URL + data (dict, optional): Payload data. Defaults to None. + + Returns: + Response: Response object + """ + self._update_request_headers() + if data is not None: + # XXX: packb() does not cover all cases. + # + # Python msgpack implementation doesn't really support variadic float precision within a single pack. + # This generally shouldn't be an issue if the upstream unpacking routine is type-agnostic since + # the type is described in the packed binary as per the spec. + # + # However, if the upstream implementation is not type-agnostic, it WILL lead to issues. + # While most respect the type opcodes (e.g. msgpack-c, msgpack-python, msgpack in Golang), some implementations (e.g. MessagePack-CSharp) disregards + # them altogether when concrete (C#) types are provided. And will raise an exception in such cases when a double is encoded in the place of a float. + # See also: + # https://github.com/mos9527/sssekai/issues/47#issuecomment-3150827990 + # + # Thus `use_single_float=True` is used to cover and only cover single precision cases. + # In the *current* API seen in the game binary, this is fine. And in AbCache's case floats are not at all used. + # But this may change in the future, in which case consult the following issues: + # https://github.com/msgpack/msgpack-python/issues/326 + data = packb(data, use_single_float=True) + data = encrypt(data, SEKAI_APIMANAGER_KEYSETS[self.config.app_region]) + resp = self.request(method=method, url=url, data=data, **kwargs) + if 400 <= resp.status_code < 600: + logger.error(f"{method} {url} {resp.status_code}") + try: # log the error message provided by the API. + content = self.response_to_dict(resp) + logger.error("response=%s" % content) + except: + logger.error("response=%s" % resp.content) + logger.error( + "Unexpected server-side error. Report issues at https://github.com/MetaSekaiLab/sssekai/issues." + ) + resp.raise_for_status() + return resp + + def response_to_dict(self, resp: Response, **kwargs): + """Decrypt and unpack a response content to a dictionary. + + Args: + resp (Response): Response object + **kwargs: Additional arguments for MessagePack unpackb + + Returns: + dict: Decrypted and unpacked data + """ + data = decrypt(resp.content, SEKAI_APIMANAGER_KEYSETS[self.config.app_region]) + data = unpackb(data, **kwargs) + return data + + def _update_user_auth_data(self): + if self.config.auth_available: + logger.debug("Updating user auth data") + if self.config.app_region in REGION_JP_EN: + resp = self.request_packed( + "PUT", + self.SEKAI_API_USER_AUTH, + data={"credential": self.SEKAI_CREDENTIAL, "deviceId": None}, + ) + else: + resp = self.request_packed( + "POST", + self.SEKAI_API_USER_AUTH, + data={ + "accessToken": self.SEKAI_CREDENTIAL, + "deviceId": None, + "userID": 0, + }, + ) + data = self.response_to_dict(resp) + self.database.sekai_user_auth_data = fromdict( + SekaiUserAuthData, data, False + ) + if self.config.app_region in REGION_ROW: + self.database.sekai_user_data = fromdict(SekaiUserData, data, False) + else: + logger.warning( + "No valid auth credential provided. User auth data will not be updated." + ) + + def _update_system_data(self): + logger.debug("Updating system data") + resp = self.request_packed("GET", self.SEKAI_API_SYSTEM_DATA) + data = self.response_to_dict(resp) + self.database.sekai_system_data = fromdict(SekaiSystemData, data) + + def _update_gameversion_data(self): + if self.config.app_region in REGION_JP_EN: + logger.debug("Updating game version data") + resp = self.request_packed( + "GET", + self.SEKAI_API_GAMEVERSION_ENDPOINT + + "/" + + self.SEKAI_APP_VERSION + + "/" + + self.SEKAI_APP_HASH, + ) + data = self.response_to_dict(resp) + self.database.sekai_gameversion_data = fromdict(SekaiGameVersionData, data) + + def __init__(self, config: Optional[AbCacheConfig] = None): + super().__init__() + self.database = SSSekaiDatabase() + self.config = config or AbCacheConfig( + "unknown", "unknown", "unknown", "unknown" + ) + self.config.version = __version_tuple__ + + def update_client_headers(self): + """Authenticate the user and update client headers.""" + logger.debug("Updating metadata") + logger.debug("Set config: %s" % self.config) + try: + self.update_signatures() + self._update_gameversion_data() + self._update_system_data() + except HTTPError as e: + logger.error("Signature update failure: %s" % e) + raise e + if self.config.app_region in REGION_JP_EN: + version_newest = self.database.sekai_system_data.appVersions[-1] + logger.debug("Newest App version: %s" % version_newest) + if version_newest.appVersion != self.SEKAI_APP_VERSION: + logger.warning("App version mismatch. This may cause issues.") + self._update_user_auth_data() + if self.is_authenticated: + self.headers.update( + { + "X-Data-Version": self.database.sekai_user_auth_data.dataVersion, + "X-Asset-Version": self.database.sekai_user_auth_data.assetVersion, + "X-Session-Token": self.database.sekai_user_auth_data.sessionToken, + } + ) + return self.headers + + def update_abcache_index(self) -> AbCacheIndex: + """Update the AssetBundle Index.""" + logger.debug("Updating Assetbundle index") + if self.config.app_region in REGION_JP_EN: + logger.info("Sekai AssetBundle version: %s" % self.SEKAI_ASSET_VERSION) + if self.config.app_region in {"jp"}: + logger.info("Sekai AssetBundle host hash: %s" % self.SEKAI_AB_HOST_HASH) + resp = self.request_packed("GET", self.SEKAI_AB_INFO_ENDPOINT) + data = self.response_to_dict(resp) + self.database.sekai_abcache_index = fromdict(AbCacheIndex, data) + return self.database.sekai_abcache_index + + def update_signatures(self): + """Update client signatures. Only required for JP for the initial setup. + After which the game would cache the signature/cookies.""" + if self.config.app_region in {"jp"}: + logger.debug("Updating signatures") + # XXX: Clear all cookies, it 403s otherwise. Does this trigger a rate limit/expiry check since our session would be tracked? + self.headers["Cookie"] = "" + self.cookies.clear() + resp = self.request_packed("POST", self.SEKAI_ISSUE_SIGNATURE_ENDPOINT) + self.headers["Cookie"] = resp.headers["Set-Cookie"] + # HACK: Per RFC6265, Cookies should not be visible to subdomains since it's not set with Domain attribute (https://github.com/psf/requests/issues/2576) + # But the other endpoints uses it nontheless. So we have to set it manually. + + def update(self): + """Update the cache with the latest AssetBundle Index data from the server. + + NOTE: + AbCache should have been initialized with a valid config before calling this method. + Please refer to `update_client_headers` for more information. + """ + self.update_client_headers() + self.update_abcache_index() + + def save(self, f: BinaryIO): + self._update_request_headers() + self.database.cached_headers = RedactedHeaders(self.headers) + self.database.config.version = __version_tuple__ + logger.info("Saving cache: %s" % self) + dump(self.database, f) + + def load(self, f: BinaryIO): + logger.info("Loading cache") + self.database = load(f) + # Values from disk are plain str/dict; re-wrap so they stay out of logs. + self.database.redact_secrets() + logger.info("Loaded cache: %s" % self) + if not self.database.config.is_up_to_date: + logger.warning( + "Cache is outdated (cache: %s, current: %s). It's HIGHLY recommended to update the cache to avoid issues." + % (self.database.config.cache_version_string, __version__) + ) + self.headers.update(self.database.cached_headers) + + @staticmethod + def from_file(f: BinaryIO): + cache = AbCache() + cache.load(f) + return cache + + def __repr__(self) -> str: + return f"AbCache(config={self.config} bundles={len(self.abcache_index.bundles) if self.abcache_index is not None else 'N/A'})" + + def get_entry_by_bundle_name(self, bundleName: str) -> AbCacheEntry: + return self.abcache_index.bundles.get(bundleName, None) + + def get_entry_download_url(self, entry: AbCacheEntry): + if self.config.app_region in REGION_JP_EN: + return self.SEKAI_AB_ENDPOINT + self.SEKAI_AB_BASE_PATH + entry.bundleName + else: + # https://github.com/mos9527/sssekai/issues/28 + return self.SEKAI_AB_ENDPOINT + entry.downloadPath + "/" + entry.bundleName + + def get_or_update_dependency_tree_flatten(self, bundleName: str, deps: set = None): + """Get a flattened set of asset dependency bundle names (including itself) for a given entry. + + Returned order is topologically sorted. + + if 'deps' is provided, it will be used as the initial set of dependencies. + """ + deps = deps or set() + + def dfs(bundleName): + for dep in self.abcache_index.bundles[bundleName].dependencies: + if dep not in deps: + dfs(dep) + deps.add(dep) + + dfs(bundleName) + deps.add(bundleName) + return deps diff --git a/sssekai/abcache/auth.py b/sssekai/abcache/auth.py index ba96e1d..8896415 100644 --- a/sssekai/abcache/auth.py +++ b/sssekai/abcache/auth.py @@ -1,30 +1,30 @@ -from . import AbCache, SekaiUserData, fromdict -from logging import getLogger - -logger = getLogger(__name__) - - -def register_as_anonymous_user_sega(cache: AbCache) -> SekaiUserData: - """Registers an anonymous account on SEGA servers, and writes relavant data to the AbCacheobject. - - Args: - cache: The AbCache object to write the user data to. - - Returns: - SekaiUserData: The user data object. - """ - logger.info("Registering user data") - # https://github.com/mos9527/sssekai/issues/31 - cache._update_request_headers() - cache.update_signatures() - payload = { - "platform": cache.headers["X-Platform"], - "deviceModel": cache.headers["X-DeviceModel"], - "operatingSystem": cache.headers["X-OperatingSystem"], - } - resp = cache.request_packed("POST", cache.SEKAI_API_USER, data=payload) - data = cache.response_to_dict(resp) - cache.config.auth_credential = data["credential"] - logger.info("Success. User ID=%s" % cache.config.auth_userID) - cache.database.sekai_user_data = data - return fromdict(SekaiUserData, data) +from . import AbCache, SekaiUserData, fromdict +from logging import getLogger + +logger = getLogger(__name__) + + +def register_as_anonymous_user_sega(cache: AbCache) -> SekaiUserData: + """Registers an anonymous account on SEGA servers, and writes relavant data to the AbCacheobject. + + Args: + cache: The AbCache object to write the user data to. + + Returns: + SekaiUserData: The user data object. + """ + logger.info("Registering user data") + # https://github.com/mos9527/sssekai/issues/31 + cache._update_request_headers() + cache.update_signatures() + payload = { + "platform": cache.headers["X-Platform"], + "deviceModel": cache.headers["X-DeviceModel"], + "operatingSystem": cache.headers["X-OperatingSystem"], + } + resp = cache.request_packed("POST", cache.SEKAI_API_USER, data=payload) + data = cache.response_to_dict(resp) + cache.config.auth_credential = data["credential"] + logger.info("Success. User ID=%s" % cache.config.auth_userID) + cache.database.sekai_user_data = fromdict(SekaiUserData, data, False) + return cache.database.sekai_user_data diff --git a/sssekai/entrypoint/apphash.py b/sssekai/entrypoint/apphash.py index da2573f..743b233 100644 --- a/sssekai/entrypoint/apphash.py +++ b/sssekai/entrypoint/apphash.py @@ -1,276 +1,276 @@ -import zipfile -import UnityPy -import logging -import json -import sys - -from io import BytesIO - -from UnityPy.enums import ClassIDType -from UnityPy.helpers.Tpk import get_typetree_node -from sssekai.unity.AssetBundle import load_assetbundle -from tqdm import tqdm - -REGION_MAP = { - # NOTE: Order is used to determine the region - "com.hermes.mk": "cn", - "com.hermes.mk.asia": "tw", - "com.sega.ColorfulStage.en": "en", - "com.sega.pjsekai": "jp", - "com.pjsekai.kr": "kr", -} -ROW_REGIONS = {"cn", "tw", "kr"} -logger = logging.getLogger("apphash") - -PROD_BUNDLES = { - "6350e2ec327334c8a9b7f494f344a761", # PJSK Android - "c726e51b6fe37463685916a1687158dd", # PJSK iOS - "data.unity3d", # TW,KR,CN (ByteDance) -} - - -def enum_candidates(zip_file, filter): - return ( - (f, zip_file.open(f), zip_file) for f in zip_file.filelist if filter(f.filename) - ) - - -def enum_package(zip_file): - yield zip_file - for f in zip_file.filelist: - if f.filename.lower().endswith(".apk"): - yield zipfile.ZipFile(zip_file.open(f)) - - -def dump_axml_stringpool(f: BytesIO): - read_int = lambda nbytes: int.from_bytes(f.read(nbytes), "little") - f.seek(8) - hdr_type, hdr_size, size = read_int(2), read_int(2), read_int(4) - n_strings = read_int(4) - unk = read_int(4) - flags = read_int(4) - is_utf8 = flags & (1 << 8) - string_offset = read_int(4) + 8 - unk = read_int(4) - offsets = [read_int(4) for _ in range(n_strings)] - for offset in offsets: - f.seek(string_offset + offset) - string_len = read_int(2) - if is_utf8: - string = f.read(string_len).decode("utf-8") - else: - string = f.read(string_len * 2).decode("utf-16-le") - yield string - - -def dump_metadata_stringpool(f: BytesIO): - metadata = f.read() - metadata = metadata[metadata.find(b"\xaf\x1b\xb1\xfa") :] - f = BytesIO(metadata) - read_int = lambda nbytes: int.from_bytes(f.read(nbytes), "little") - assert read_int(4) == 0xFAB11BAF, "bad sanity" - logger.info("metadata version: %s" % read_int(4)) - stringLiteralOffset = read_int(4) # string data for managed code - stringLiteralSize = read_int(4) - stringLiteralDataOffset = read_int(4) - stringLiteralDataSize = read_int(4) - f.seek(stringLiteralOffset) - stringLiterals = [ - (read_int(4), read_int(4)) for _ in range(stringLiteralSize // 8) - ] # length, dataIndex - for length, dataIndex in stringLiterals: - f.seek(stringLiteralDataOffset + dataIndex) - string = f.read(length) - yield string.decode("utf-8") - pass - - -def main_apphash(args): - UnityPy.config.SERIALIZED_FILE_PARSE_TYPETREE = False - env = UnityPy.Environment() - app_package = None - app_version = "unknown" - app_hash = "unknown" - app_metadata_strings = None - proxies = None - if args.proxy: - logger.info("Overriding proxy: %s", args.proxy) - proxies = {"http": args.proxy, "https": args.proxy} - if not args.ab_src: - if not args.apk_src or args.fetch: - from requests import get - - src = BytesIO() - logger.debug("Fetching latest game package (JP) from APKPure") - resp = get( - "https://d.apkpure.net/b/XAPK/com.sega.pjsekai?version=latest", - stream=True, - proxies=proxies, - ) - size = resp.headers.get("Content-Length", -1) - with tqdm(total=int(size), unit="B", unit_scale=True) as progress: - for chunk in resp.iter_content(chunk_size=2**20): - src.write(chunk) - progress.update(len(chunk)) - src.seek(0) - else: - src = open(args.apk_src, "rb") - with zipfile.ZipFile(src, "r") as zip_ref: - manifests = [ - manifest - for package in enum_package(zip_ref) - for manifest in enum_candidates( - package, lambda fn: fn == "AndroidManifest.xml" - ) - ] - manifest = manifests[0][1] - manifest_strings = set(dump_axml_stringpool(manifest)) - # Heur: Reverse lookup the package name from the manifest strings - for ky in REGION_MAP: - if ky in manifest_strings: - app_package = ky - break - # if REGION_MAP[app_package] in ROW_REGIONS: - # metadata = [ - # meta - # for package in enum_package(zip_ref) - # for meta in enum_candidates( - # package, - # lambda fn: fn.endswith("global-metadata.dat"), - # ) - # ] - # if metadata: - # metadata = metadata[0][1] - # metadata_strings = list(dump_metadata_stringpool(metadata)) - # app_metadata_strings = [s for s in metadata_strings if "bytedgame" in s] - # pass - candidate_filter = lambda fn: fn.split("/")[-1] in PROD_BUNDLES - candidate_filter_deep = lambda fn: ("_/" + fn).split("/")[-2] == "Data" - - candidates = [ - candidate - for package in enum_package(zip_ref) - for candidate in enum_candidates( - package, - candidate_filter_deep if args.deep else candidate_filter, - ) - ] - for candidate, stream, _ in tqdm(candidates, desc="Loading"): - env.load_file(stream.read()) - else: - logger.info("Loading from AssetBundle %s" % args.ab_src) - with open(args.ab_src, "rb") as f: - env = load_assetbundle(BytesIO(f.read())) - - from sssekai.generated import UTTCGen_AsInstance - from sssekai.generated.Sekai import ( - AndroidPlayerSettingConfig, - IOSPlayerSettingConfig, - ) - - res = dict() - for reader in tqdm(env.objects, desc="Processing"): - if reader.container: - logger.info("Processing %s (%s)", reader.container, reader.type) - if reader.type == UnityPy.enums.ClassIDType.MonoBehaviour: - mono = reader.read(check_read=False) - clazz = None - platform = None - if "_android" in mono.m_Name: - clazz = AndroidPlayerSettingConfig - platform = "android" - elif "_ios" in mono.m_Name: - clazz = IOSPlayerSettingConfig - platform = "ios" - if clazz: - # Works with post 3.4 (JP 3rd Anniversary) builds and downstream regional builds - try: - config = UTTCGen_AsInstance(clazz, reader) - except Exception as e: - logger.error("Failed to parse config for %s: %s", mono.m_Name, e) - continue - config: AndroidPlayerSettingConfig | IOSPlayerSettingConfig - app_version = "%s.%s.%s" % ( - config.clientMajorVersion, - config.clientMinorVersion, - config.clientBuildVersion, - ) - data_version = "%s.%s.%s" % ( - config.clientDataMajorVersion, - config.clientDataMinorVersion, - config.clientDataBuildVersion, - ) - ab_version = "%s.%s.%s" % ( - config.clientMajorVersion, - config.clientMinorVersion, - config.clientDataRevision, - ) - app_hash = config.clientAppHash - # XXX: CN build has the same package name as the JP build?? - # We need another heuristic to determine the package name - # This is previously done by setting app_package - package_heur = app_package or config.bundleIdentifier - region = region = REGION_MAP.get(package_heur, "unknown") - print( - f"Found {config.productName} at {config.m_Name}", - f" Memo: {config.memo}", - f" Package: {config.bundleIdentifier} (actually assumed as {package_heur})", - f" Platform: {platform}", - f" AppHash (app_hash): {config.clientAppHash}", - f" Region (app_region): {region} (determined by {package_heur})", - f" Version (app_version): {app_version}", - f" Version (ab_version): {ab_version}", - f" Bundle Version: {config.bundleVersion}", - f" Data Version: {data_version}", - f" Version Suffix: {config.clientVersionSuffix}", - "", - "", - sep="\n", - file=sys.stderr, - ) - app_package = app_package or "Unknown Package (Failed APK Heuristic)" - # fmt: off - match args.format: - case 'json': - res[mono.m_Name] = { - "package": app_package, - "reported_package": config.bundleIdentifier, - "app_hash": app_hash, - "app_region": region, - "app_version": app_version, - "app_platform": platform, - "ab_version": ab_version, - } - case "markdown": - res[mono.m_Name] = f"""{app_package} ({app_version}, {region}) ---- -Reported Package: {config.bundleIdentifier} - -|{'app_hash'.rjust(48)}| app_region| app_version| ab_version| -|{'-'.rjust(48, '-')}|-------------|-------------|-------------| -|{app_hash.rjust(48)}|{region.rjust(13)}|{app_version.rjust(13)}|{ab_version.rjust(13)}| - -- CLI Usage: - - sssekai abcache --app-platform {platform} --app-region {region} --app-version {app_version} --app-appHash {app_hash} --app-abVersion {ab_version} - -- Python Usage: - - from sssekai.abcache import AbCacheConfig - - AbCacheConfig( - app_region="{region}", - app_version="{app_version}", - ab_version="{ab_version}", - app_hash="{app_hash}", - app_platform="{platform}" - ) -""" - print("###### RESULTS ######", file=sys.stderr) - res = dict(sorted(res.items(), key=lambda x: x[0])) - match args.format: - case "json": - print(json.dumps(res, indent=4, ensure_ascii=False)) - case "markdown": - for name, content in res.items(): - print(f"## {name}\n{content}\n") +import zipfile +import UnityPy +import logging +import json +import sys + +from io import BytesIO + +from UnityPy.enums import ClassIDType +from UnityPy.helpers.Tpk import get_typetree_node +from sssekai.unity.AssetBundle import load_assetbundle +from tqdm import tqdm + +REGION_MAP = { + # NOTE: Order is used to determine the region + "com.hermes.mk": "cn", + "com.hermes.mk.asia": "tw", + "com.sega.ColorfulStage.en": "en", + "com.sega.pjsekai": "jp", + "com.pjsekai.kr": "kr", +} +ROW_REGIONS = {"cn", "tw", "kr"} +logger = logging.getLogger("apphash") + +PROD_BUNDLES = { + "6350e2ec327334c8a9b7f494f344a761", # PJSK Android + "c726e51b6fe37463685916a1687158dd", # PJSK iOS + "data.unity3d", # TW,KR,CN (ByteDance) +} + + +def enum_candidates(zip_file, filter): + return ( + (f, zip_file.open(f), zip_file) for f in zip_file.filelist if filter(f.filename) + ) + + +def enum_package(zip_file): + yield zip_file + for f in zip_file.filelist: + if f.filename.lower().endswith(".apk"): + yield zipfile.ZipFile(zip_file.open(f)) + + +def dump_axml_stringpool(f: BytesIO): + read_int = lambda nbytes: int.from_bytes(f.read(nbytes), "little") + f.seek(8) + hdr_type, hdr_size, size = read_int(2), read_int(2), read_int(4) + n_strings = read_int(4) + unk = read_int(4) + flags = read_int(4) + is_utf8 = flags & (1 << 8) + string_offset = read_int(4) + 8 + unk = read_int(4) + offsets = [read_int(4) for _ in range(n_strings)] + for offset in offsets: + f.seek(string_offset + offset) + string_len = read_int(2) + if is_utf8: + string = f.read(string_len).decode("utf-8") + else: + string = f.read(string_len * 2).decode("utf-16-le") + yield string + + +def dump_metadata_stringpool(f: BytesIO): + metadata = f.read() + metadata = metadata[metadata.find(b"\xaf\x1b\xb1\xfa") :] + f = BytesIO(metadata) + read_int = lambda nbytes: int.from_bytes(f.read(nbytes), "little") + assert read_int(4) == 0xFAB11BAF, "bad sanity" + logger.info("metadata version: %s" % read_int(4)) + stringLiteralOffset = read_int(4) # string data for managed code + stringLiteralSize = read_int(4) + stringLiteralDataOffset = read_int(4) + stringLiteralDataSize = read_int(4) + f.seek(stringLiteralOffset) + stringLiterals = [ + (read_int(4), read_int(4)) for _ in range(stringLiteralSize // 8) + ] # length, dataIndex + for length, dataIndex in stringLiterals: + f.seek(stringLiteralDataOffset + dataIndex) + string = f.read(length) + yield string.decode("utf-8") + pass + + +def main_apphash(args): + UnityPy.config.SERIALIZED_FILE_PARSE_TYPETREE = False + env = UnityPy.Environment() + app_package = None + app_version = "unknown" + app_hash = "unknown" + app_metadata_strings = None + proxies = None + if args.proxy: + logger.info("Overriding proxy: %s", args.proxy) + proxies = {"http": args.proxy, "https": args.proxy} + if not args.ab_src: + if not args.apk_src or args.fetch: + from requests import get + + src = BytesIO() + logger.debug("Fetching latest game package (JP) from APKPure") + resp = get( + "https://d.apkpure.net/b/XAPK/com.sega.pjsekai?version=latest", + stream=True, + proxies=proxies, + ) + size = resp.headers.get("Content-Length", -1) + with tqdm(total=int(size), unit="B", unit_scale=True) as progress: + for chunk in resp.iter_content(chunk_size=2**20): + src.write(chunk) + progress.update(len(chunk)) + src.seek(0) + else: + src = open(args.apk_src, "rb") + with zipfile.ZipFile(src, "r") as zip_ref: + manifests = [ + manifest + for package in enum_package(zip_ref) + for manifest in enum_candidates( + package, lambda fn: fn == "AndroidManifest.xml" + ) + ] + manifest = manifests[0][1] + manifest_strings = set(dump_axml_stringpool(manifest)) + # Heur: Reverse lookup the package name from the manifest strings + for ky in REGION_MAP: + if ky in manifest_strings: + app_package = ky + break + # if REGION_MAP[app_package] in ROW_REGIONS: + # metadata = [ + # meta + # for package in enum_package(zip_ref) + # for meta in enum_candidates( + # package, + # lambda fn: fn.endswith("global-metadata.dat"), + # ) + # ] + # if metadata: + # metadata = metadata[0][1] + # metadata_strings = list(dump_metadata_stringpool(metadata)) + # app_metadata_strings = [s for s in metadata_strings if "bytedgame" in s] + # pass + candidate_filter = lambda fn: fn.split("/")[-1] in PROD_BUNDLES + candidate_filter_deep = lambda fn: ("_/" + fn).split("/")[-2] == "Data" + + candidates = [ + candidate + for package in enum_package(zip_ref) + for candidate in enum_candidates( + package, + candidate_filter_deep if args.deep else candidate_filter, + ) + ] + for candidate, stream, _ in tqdm(candidates, desc="Loading"): + env.load_file(stream.read()) + else: + logger.info("Loading from AssetBundle %s" % args.ab_src) + with open(args.ab_src, "rb") as f: + env = load_assetbundle(BytesIO(f.read())) + + from sssekai.generated import UTTCGen_AsInstance + from sssekai.generated.Sekai import ( + AndroidPlayerSettingConfig, + IOSPlayerSettingConfig, + ) + + res = dict() + for reader in tqdm(env.objects, desc="Processing"): + if reader.container: + logger.info("Processing %s (%s)", reader.container, reader.type) + if reader.type == UnityPy.enums.ClassIDType.MonoBehaviour: + mono = reader.read(check_read=False) + clazz = None + platform = None + if "_android" in mono.m_Name: + clazz = AndroidPlayerSettingConfig + platform = "android" + elif "_ios" in mono.m_Name: + clazz = IOSPlayerSettingConfig + platform = "ios" + if clazz: + # Works with post 3.4 (JP 3rd Anniversary) builds and downstream regional builds + try: + config = UTTCGen_AsInstance(clazz, reader) + except Exception as e: + logger.error("Failed to parse config for %s: %s", mono.m_Name, e) + continue + config: AndroidPlayerSettingConfig | IOSPlayerSettingConfig + app_version = "%s.%s.%s" % ( + config.clientMajorVersion, + config.clientMinorVersion, + config.clientBuildVersion, + ) + data_version = "%s.%s.%s" % ( + config.clientDataMajorVersion, + config.clientDataMinorVersion, + config.clientDataBuildVersion, + ) + ab_version = "%s.%s.%s" % ( + config.clientMajorVersion, + config.clientMinorVersion, + config.clientDataRevision, + ) + app_hash = config.clientAppHash + # XXX: CN build has the same package name as the JP build?? + # We need another heuristic to determine the package name + # This is previously done by setting app_package + package_heur = app_package or config.bundleIdentifier + region = region = REGION_MAP.get(package_heur, "unknown") + print( + f"Found {config.productName} at {config.m_Name}", + f" Memo: {config.memo}", + f" Package: {config.bundleIdentifier} (actually assumed as {package_heur})", + f" Platform: {platform}", + f" AppHash (app_hash): {config.clientAppHash}", + f" Region (app_region): {region} (determined by {package_heur})", + f" Version (app_version): {app_version}", + f" Version (ab_version): {ab_version}", + f" Bundle Version: {config.bundleVersion}", + f" Data Version: {data_version}", + f" Version Suffix: {config.clientVersionSuffix}", + "", + "", + sep="\n", + file=sys.stderr, + ) + app_package = app_package or "Unknown Package (Failed APK Heuristic)" + # fmt: off + match args.format: + case 'json': + res[mono.m_Name] = { + "package": app_package, + "reported_package": config.bundleIdentifier, + "app_hash": app_hash, + "app_region": region, + "app_version": app_version, + "app_platform": platform, + "ab_version": ab_version, + } + case "markdown": + res[mono.m_Name] = f"""{app_package} ({app_version}, {region}) +--- +Reported Package: {config.bundleIdentifier} + +|{'app_hash'.rjust(48)}| app_region| app_version| ab_version| +|{'-'.rjust(48, '-')}|-------------|-------------|-------------| +|{app_hash.rjust(48)}|{region.rjust(13)}|{app_version.rjust(13)}|{ab_version.rjust(13)}| + +- CLI Usage: + + sssekai abcache --app-platform {platform} --app-region {region} --app-version {app_version} --app-appHash {app_hash} --app-abVersion {ab_version} + +- Python Usage: + + from sssekai.abcache import AbCacheConfig + + AbCacheConfig( + app_region="{region}", + app_version="{app_version}", + row_ab_version="{ab_version}", + app_hash="{app_hash}", + app_platform="{platform}" + ) +""" + print("###### RESULTS ######", file=sys.stderr) + res = dict(sorted(res.items(), key=lambda x: x[0])) + match args.format: + case "json": + print(json.dumps(res, indent=4, ensure_ascii=False)) + case "markdown": + for name, content in res.items(): + print(f"## {name}\n{content}\n") diff --git a/sssekai/entrypoint/rla2json.py b/sssekai/entrypoint/rla2json.py index 12b275e..7755695 100644 --- a/sssekai/entrypoint/rla2json.py +++ b/sssekai/entrypoint/rla2json.py @@ -1,79 +1,80 @@ -from logging import getLogger -from sssekai.unity.AssetBundle import load_assetbundle -from UnityPy.enums import ClassIDType -from sssekai.fmt.rla import read_archive_rla_frames, read_rla_frames -import os, io, json, base64 - -logger = getLogger(__name__) - - -def main_rla2json(args): - def __dump_from_frame_gen(frame_gen): - outdir = args.outdir - os.makedirs(outdir, exist_ok=True) - ensure_dir = lambda x: ( - ( - os.makedirs(os.path.dirname(x), exist_ok=True) - if os.path.dirname(x) - else None - ) - or x - ) - for tick, frame in frame_gen: - print("tick: %16s, type: %32s" % (tick, frame["type"]), end="\r") - match frame["type"]: - case "SoundData": - if frame["encoding"] == "hca": - with open( - ensure_dir( - os.path.join(outdir, "SoundData", str(tick) + ".hca"), - ), - "wb", - ) as f: - raw_data = frame["data"] - start = raw_data.find(b"\xff\xff") - f.write(raw_data[start:]) - else: - raise ValueError("unsupported encoding: %s" % frame["encoding"]) - case _: - with open( - ensure_dir( - os.path.join(outdir, frame["type"], str(tick) + ".json") - ), - "w", - encoding="utf-8", - ) as f: - json.dump(frame, f, ensure_ascii=False, indent=4) - - if os.path.isfile(args.input): - with open(args.input, "rb") as f: - datas = dict() - rla_env = load_assetbundle(f) - for obj in rla_env.objects: - if obj.type in {ClassIDType.TextAsset}: - data = obj.read() - datas[data.m_Name] = data.m_Script.encode( - "utf-8", "surrogateescape" - ) - header = datas.get("sekai.rlh", None) - assert header, "RLH Header file not found!" - version = tuple(map(int, header["version"].split("."))) - logger.info("Version: %d.%d" % version) - logger.info("Count: %d" % len(header["splitFileIds"])) - splitSeconds = header["splitSeconds"] - for sid in header["splitFileIds"]: - sname = "sekai_%02d_%08d" % (splitSeconds, sid) - script = datas[sname + ".rla"] - frame_gen = read_archive_rla_frames( - io.BytesIO(script), version, args.strict - ) - __dump_from_frame_gen(frame_gen) - else: - logger.info("Reading from directory") - packet_gen = ( - (file, open(os.path.join(args.input, file), "rb").read()) - for file in os.listdir(args.input) - ) - version = tuple(map(int, args.version.split("."))) - frame_gen = read_rla_frames(packet_gen, version, args.strict) - __dump_from_frame_gen(frame_gen) +from logging import getLogger +from sssekai.unity.AssetBundle import load_assetbundle +from UnityPy.enums import ClassIDType +from sssekai.fmt.rla import read_archive_rla_frames, read_rla_frames +import os, io, json, base64 + +logger = getLogger(__name__) + + +def main_rla2json(args): + def __dump_from_frame_gen(frame_gen): + outdir = args.outdir + os.makedirs(outdir, exist_ok=True) + ensure_dir = lambda x: ( + ( + os.makedirs(os.path.dirname(x), exist_ok=True) + if os.path.dirname(x) + else None + ) + or x + ) + for tick, frame in frame_gen: + print("tick: %16s, type: %32s" % (tick, frame["type"]), end="\r") + match frame["type"]: + case "SoundData": + if frame["encoding"] == "hca": + with open( + ensure_dir( + os.path.join(outdir, "SoundData", str(tick) + ".hca"), + ), + "wb", + ) as f: + raw_data = frame["data"] + start = raw_data.find(b"\xff\xff") + f.write(raw_data[start:]) + else: + raise ValueError("unsupported encoding: %s" % frame["encoding"]) + case _: + with open( + ensure_dir( + os.path.join(outdir, frame["type"], str(tick) + ".json") + ), + "w", + encoding="utf-8", + ) as f: + json.dump(frame, f, ensure_ascii=False, indent=4) + + if os.path.isfile(args.input): + with open(args.input, "rb") as f: + datas = dict() + rla_env = load_assetbundle(f) + for obj in rla_env.objects: + if obj.type in {ClassIDType.TextAsset}: + data = obj.read() + datas[data.m_Name] = data.m_Script.encode( + "utf-8", "surrogateescape" + ) + header = datas.get("sekai.rlh", None) + assert header, "RLH Header file not found!" + header = json.loads(header.decode("utf-8")) + version = tuple(map(int, header["version"].split("."))) + logger.info("Version: %d.%d" % version) + logger.info("Count: %d" % len(header["splitFileIds"])) + splitSeconds = header["splitSeconds"] + for sid in header["splitFileIds"]: + sname = "sekai_%02d_%08d" % (splitSeconds, sid) + script = datas[sname + ".rla"] + frame_gen = read_archive_rla_frames( + io.BytesIO(script), version, args.strict + ) + __dump_from_frame_gen(frame_gen) + else: + logger.info("Reading from directory") + packet_gen = ( + (file, open(os.path.join(args.input, file), "rb").read()) + for file in os.listdir(args.input) + ) + version = tuple(map(int, args.version.split("."))) + frame_gen = read_rla_frames(packet_gen, version, args.strict) + __dump_from_frame_gen(frame_gen) diff --git a/sssekai/fmt/rla.py b/sssekai/fmt/rla.py index d6c7ced..1ebd52d 100644 --- a/sssekai/fmt/rla.py +++ b/sssekai/fmt/rla.py @@ -1,11 +1,15 @@ from base64 import b64decode, b64encode from struct import unpack as s_unpack +import struct, zlib from io import BytesIO from collections import defaultdict from typing import Generator, Tuple, TypeVar, List +from logging import getLogger import math, gzip import msgpack +logger = getLogger(__name__) + RLA_VERSIONS = [(1, 0), (1, 1), (1, 2), (1, 3), (1, 4), (1, 5), (1, 6), (1, 7)] read_int = lambda stream, nbytes, signed=False: int.from_bytes( @@ -32,6 +36,12 @@ def __init__(self, missing): super().__init__(f"Missing %d split packet(s): %s" % (len(missing), missing)) +class StreamingDataTruncatedException(Exception): + """Raised when a payload is incomplete or corrupt and cannot be decoded + in full. In non-strict mode callers drop the whole frame rather than + returning a silently shortened one.""" + + # Sekai_Streaming_StreamingCommon__CheckHeader def decode_buffer_base64(buffer: bytes) -> tuple[int, bytes]: """Decodes the 'RTVL' SSE Message payload into a header signature and data. @@ -124,10 +134,11 @@ def decode_streaming_data( version (tuple): RLA version decoder_signature (int): Decoder signature buffer (bytes): Decoded payload data - strict (bool, optional): If False, incomplete packets will be returned as is. Defaults to True. + strict (bool, optional): If False, truncated payloads raise StreamingDataTruncatedException instead of propagating the underlying error. Defaults to True. Raises: Exception: If the packet is incomplete and strict is True. + StreamingDataTruncatedException: If the packet is incomplete and strict is False. Returns: dict: Parsed streaming data payload @@ -227,16 +238,20 @@ def get_next_mask(): ) # ReadString # HACK: Workaround for incomplete packets. + # In non-strict mode a truncated array aborts the whole frame instead of + # being silently returned as a shorter one: callers must be able to tell a + # complete frame from a truncated one (see StreamingDataTruncatedException). def gen_exception_handler(generator): try: for item in generator: yield item + except StreamingDataTruncatedException: + raise except Exception as e: # XXX: This shouldn't happen unless the packet is corrupted. if strict: raise e - else: - return + raise StreamingDataTruncatedException(str(e)) from e gen_get_next_array = lambda reader: ( reader() for _ in range(get_next_int()) @@ -433,7 +448,7 @@ def read_rla_frames( Args: reader (Generator[Tuple[T, bytes], None, None]): A generator that yields a tuple of timestamp and buffer data version (tuple, optional): RLA version, found in respective RLH (JSON) header files. range: see RLA_VERSIONS. Defaults to (1,0). - strict (bool, optional): If False, incomplete packets will be returned as is. Defaults to True. + strict (bool, optional): If False, malformed or incomplete packets are skipped instead of raising. Defaults to True. Yields: Generator[Tuple[T, dict], None, None]: A generator that yields a tuple of timestamp and parsed frame data. Timestamp format is provided by the reader. @@ -453,13 +468,42 @@ def decode_buffer(header_signature, data): payload = decode_streaming_data(version, decoder_signature, data, strict) return payload + # Errors that indicate a bad packet. Anything else (TypeError, AttributeError, + # ...) is a bug in this decoder and must not be silently swallowed. + DECODE_ERRORS = ( + AssertionError, + IndexError, + KeyError, + ValueError, + UnicodeError, + EOFError, + struct.error, + gzip.BadGzipFile, + zlib.error, + StreamingDataTruncatedException, + ) + + def decode_or_skip(header_signature, data): + """Decode a packet, or return None to skip it when not in strict mode.""" + try: + return decode_buffer(header_signature, data) + except DECODE_ERRORS as e: + if strict: + raise + logger.debug("Skipping undecodable RLA packet: %s", e) + return None + for timestamp, buffer in reader: try: split_info, header_signature, data = decode_buffer_base64(buffer) except Exception as e: if strict: raise e - # Otherwise fail silently + # NOTE: A packet that fails to parse has no trustworthy splitId, so + # it cannot be matched against pending __buffers entries. Pending + # groups are left as is and expire at end of stream. + logger.debug("Skipping malformed RLA packet: %s", e) + continue if split_info: splitId, splitIndex, splitNum, dataLength, totalDataLength, base64 = ( split_info @@ -472,14 +516,36 @@ def decode_buffer(header_signature, data): __buffers[splitId], key=lambda x: x[1][1] ) # splitIndex ) + del __buffers[splitId] if strict: assert len(buffer) == totalDataLength, "incomplete split packet" - del __buffers[splitId] + elif len(buffer) != totalDataLength: + logger.debug( + "Skipping incomplete split packet %s (%d/%d bytes)", + splitId, + len(buffer), + totalDataLength, + ) + continue if base64: - buffer = b64decode(buffer) - yield timestamp, decode_buffer(header_signature, buffer) + try: + buffer = b64decode(buffer) + except Exception as e: + if strict: + raise e + logger.debug( + "Skipping split packet %s with malformed base64: %s", + splitId, + e, + ) + continue + payload = decode_or_skip(header_signature, buffer) + if payload is not None: + yield timestamp, payload else: - yield timestamp, decode_buffer(header_signature, data) + payload = decode_or_skip(header_signature, data) + if payload is not None: + yield timestamp, payload if __buffers and strict: missing = [(payload[0][0], splitId) for splitId, payload in __buffers.items()] @@ -496,7 +562,7 @@ def read_archive_rla_frames( Args: src (BytesIO): Source RLA file stream version (tuple, optional): RLA version, found in respective RLH (JSON) header files. range: see RLA_VERSIONS. Defaults to (1,0). - strict (bool, optional): If False, incomplete packets will be returned as is. Defaults to True. + strict (bool, optional): If False, malformed or incomplete packets are skipped instead of raising. Defaults to True. Yields: Generator[Tuple[int, dict], None, None]: A generator that yields a tuple of timestamp and parsed frame data diff --git a/tests/test_rla2json.py b/tests/test_rla2json.py new file mode 100644 index 0000000..de4f434 --- /dev/null +++ b/tests/test_rla2json.py @@ -0,0 +1,64 @@ +from . import * + +import json +from types import SimpleNamespace + + +class _FakeTextAsset: + def __init__(self, name, script: bytes): + self.m_Name = name + self.m_Script = script.decode("utf-8", "surrogateescape") + + +class _FakeObject: + def __init__(self, name, script: bytes): + from UnityPy.enums import ClassIDType + + self.type = ClassIDType.TextAsset + self._data = _FakeTextAsset(name, script) + + def read(self): + return self._data + + +class _FakeEnv: + def __init__(self, objects): + self.objects = objects + + +def test_rla2json_archive_header_is_parsed(monkeypatch): + """0.7.18 regression: the RLH header is bytes but was indexed as a dict.""" + from sssekai.entrypoint import rla2json + + header = json.dumps( + {"version": "1.0", "splitSeconds": 0, "splitFileIds": []} + ).encode("utf-8") + env = _FakeEnv([_FakeObject("sekai.rlh", header)]) + monkeypatch.setattr(rla2json, "load_assetbundle", lambda f: env) + + args = SimpleNamespace( + input=sample_file_path("rla", "streaming_live_vbs_1-0_0.bin"), + outdir=TEMP_DIR, + strict=False, + ) + # Should not raise; no split files means nothing is dumped. + rla2json.main_rla2json(args) + + +def test_rla2json_missing_header(monkeypatch): + from sssekai.entrypoint import rla2json + + env = _FakeEnv([]) + monkeypatch.setattr(rla2json, "load_assetbundle", lambda f: env) + + args = SimpleNamespace( + input=sample_file_path("rla", "streaming_live_vbs_1-0_0.bin"), + outdir=TEMP_DIR, + strict=False, + ) + try: + rla2json.main_rla2json(args) + except AssertionError: + pass + else: + raise AssertionError("missing RLH header should assert") diff --git a/tests/test_rla_error_handling.py b/tests/test_rla_error_handling.py new file mode 100644 index 0000000..22fed6f --- /dev/null +++ b/tests/test_rla_error_handling.py @@ -0,0 +1,186 @@ +from . import * + +import binascii + +# --- Packet builders ------------------------------------------------------- +# RTVL wire format: "RTVL" + 6 hex total length + base64 flag + split flag + +# 3 digit signature + body. Split bodies start with five zero-padded decimal +# fields: splitId(5) splitIndex(5) splitNum(5) dataLength(10) totalDataLength(10). +# A non-split body is signature + 5 byte payload header + streaming data; the +# 5 byte header's first byte after the signature selects raw vs gzip decoding. + + +def _split_packet(sid, idx, num, declared_total, payload, b64=True, sig=2): + info = b"%05d%05d%05d%010d%010d" % ( + sid, + idx, + num, + len(payload), + declared_total, + ) + body = info + payload + total = len(body) + 15 + flag = b"T" if b64 else b"F" + return b"RTVL" + b"%06X" % total + flag + b"T" + b"%03d" % sig + body + + +def _payload_body(streaming_payload, sig=0): + """Body of a non-split packet: decoder signature + streaming payload.""" + return bytes([sig]) + b"\x01\x00\x00\x00" + streaming_payload + + +def _payload_packet(streaming_payload, sig=0): + body = _payload_body(streaming_payload, sig) + total = len(body) + 15 + return b"RTVL" + b"%06X" % total + b"F" + b"F" + b"%03d" % sig + body + + +VALID_PAYLOAD = b"\x08\x00\x00\x00" + b"\x00\x00\x00\x00" + b"\x02\x00" + b"\x00\x00" +# pose count = 0x7fffffff with no pose data -> array reader hits EOF +TRUNCATED_PAYLOAD = ( + b"\x08\x00\x00\x00" + b"\x7f\xff\xff\xff" + b"\x02\x00" + b"\x00\x0c" +) + + +# --- Non-split packets ----------------------------------------------------- + + +def test_rla_garbage_is_skipped_when_not_strict(): + from sssekai.fmt.rla import read_rla_frames + + frames = list(read_rla_frames([(1, b"GARBAGE")], (1, 6), strict=False)) + assert frames == [] + + +def test_rla_garbage_raises_when_strict(): + from sssekai.fmt.rla import read_rla_frames + + try: + list(read_rla_frames([(1, b"GARBAGE")], (1, 6), strict=True)) + except AssertionError: + pass + else: + raise AssertionError("strict mode should raise on malformed packets") + + +def test_rla_payload_failure_is_skipped_when_not_strict(monkeypatch): + """A valid RTVL header with an undecodable payload must not abort the stream.""" + from sssekai.fmt import rla + + packet = _payload_packet(VALID_PAYLOAD) + calls = {"n": 0} + + def fake_decode_payload(buffer): + calls["n"] += 1 + raise ValueError("cannot decode payload") + + monkeypatch.setattr(rla, "decode_buffer_payload", fake_decode_payload) + frames = list(rla.read_rla_frames([(1, packet)], (1, 6), strict=False)) + assert frames == [] + assert calls["n"] == 1 + + +def test_rla_payload_failure_raises_when_strict(monkeypatch): + from sssekai.fmt import rla + + def fake_decode_payload(buffer): + raise ValueError("cannot decode payload") + + monkeypatch.setattr(rla, "decode_buffer_payload", fake_decode_payload) + packet = _payload_packet(VALID_PAYLOAD) + try: + list(rla.read_rla_frames([(1, packet)], (1, 6), strict=True)) + except ValueError: + pass + else: + raise AssertionError("strict mode should propagate payload errors") + + +def test_rla_truncated_payload_raises_truncated_error(): + """Truncation is reported distinctly so callers can tell it from corruption.""" + from sssekai.fmt.rla import StreamingDataTruncatedException, decode_streaming_data + + try: + decode_streaming_data((1, 6), 0, TRUNCATED_PAYLOAD, strict=False) + except StreamingDataTruncatedException: + pass + else: + raise AssertionError( + "truncated payload should raise StreamingDataTruncatedException" + ) + + +def test_rla_truncated_payload_drops_frame_but_keeps_stream(): + from sssekai.fmt.rla import read_rla_frames + + packets = [ + (1, _payload_packet(VALID_PAYLOAD)), + (2, _payload_packet(TRUNCATED_PAYLOAD)), + ] + frames = list(read_rla_frames(packets, (1, 6), strict=False)) + assert [(tick, f["type"]) for tick, f in frames] == [(1, "MotionData")] + + +# --- Split packets --------------------------------------------------------- + + +def test_rla_split_bad_base64_skipped_when_not_strict(): + from sssekai.fmt.rla import read_rla_frames + + # "abcde" is not valid base64 (length 5 mod 4 == 1) + packet = _split_packet(1, 0, 1, 5, b"abcde", b64=True) + assert list(read_rla_frames([(1, packet)], (1, 6), strict=False)) == [] + + +def test_rla_split_bad_base64_raises_when_strict(): + from sssekai.fmt.rla import read_rla_frames + + packet = _split_packet(1, 0, 1, 5, b"abcde", b64=True) + try: + list(read_rla_frames([(1, packet)], (1, 6), strict=True)) + except binascii.Error: + pass + else: + raise AssertionError("strict mode should raise on malformed split base64") + + +def test_rla_split_length_mismatch_skipped_when_not_strict(): + from sssekai.fmt.rla import read_rla_frames + + packet = _split_packet(1, 0, 1, 999, b"abcdef", b64=False) + assert list(read_rla_frames([(1, packet)], (1, 6), strict=False)) == [] + + +def test_rla_split_length_mismatch_raises_when_strict(): + from sssekai.fmt.rla import read_rla_frames + + packet = _split_packet(1, 0, 1, 999, b"abcdef", b64=False) + try: + list(read_rla_frames([(1, packet)], (1, 6), strict=True)) + except AssertionError as e: + assert "incomplete split packet" in str(e) + else: + raise AssertionError("strict mode should assert on split length mismatch") + + +def test_rla_split_group_is_cleared_after_skip(): + """A discarded group must not bleed into a later packet with the same id.""" + from sssekai.fmt.rla import read_rla_frames + + bad = _split_packet(7, 0, 1, 999, b"abcdef", b64=False) + good_body = _payload_body(VALID_PAYLOAD) + good = _split_packet(7, 0, 1, len(good_body), good_body, b64=False, sig=0) + frames = list(read_rla_frames([(1, bad), (2, good)], (1, 6), strict=False)) + assert [(tick, f["type"]) for tick, f in frames] == [(2, "MotionData")] + + +def test_rla_split_reassembles_in_index_order(): + from sssekai.fmt.rla import read_rla_frames + + body = _payload_body(VALID_PAYLOAD) + half = len(body) // 2 + p0 = _split_packet(3, 0, 2, len(body), body[:half], b64=False, sig=0) + p1 = _split_packet(3, 1, 2, len(body), body[half:], b64=False, sig=0) + # Delivered out of order; reassembly sorts by splitIndex. + frames = list(read_rla_frames([(2, p1), (1, p0)], (1, 6), strict=False)) + assert [f["type"] for _, f in frames] == ["MotionData"] diff --git a/tests/test_security.py b/tests/test_security.py new file mode 100644 index 0000000..9e927e8 --- /dev/null +++ b/tests/test_security.py @@ -0,0 +1,181 @@ +from . import * + +import base64 +import dataclasses +import json + +SECRET = "SUPER_SECRET_JWT_TOKEN_1234567890" + + +def _fake_jwt(user_id=1, signature="sig"): + payload = base64.b64encode(json.dumps({"userId": user_id}).encode()).decode() + return "e30." + payload.rstrip("=") + "." + signature + + +def _make_cache(credential=SECRET): + from sssekai.abcache import AbCache, AbCacheConfig + + return AbCache( + AbCacheConfig("jp", "1.0.0", "android", "hash", auth_credential=credential) + ) + + +def _make_secret_cache(): + """Credential + cached headers both carry the secret.""" + cache = _make_cache(_fake_jwt()) + cache.config.auth_credential = SECRET + cache.database.cached_headers["X-Session-Token"] = SECRET + cache.database.cached_headers["Cookie"] = SECRET + return cache + + +def test_config_repr_hides_credential(): + cache = _make_cache() + assert SECRET not in repr(cache.config) + assert SECRET not in repr(cache) + + +def test_user_data_repr_hides_tokens(): + from sssekai.abcache import ( + SekaiUserAuthData, + SekaiUserData, + SekaiUserRegistrationData, + ) + + user = SekaiUserData( + userRegistration=SekaiUserRegistrationData(userId=1), + credential=SECRET, + sessionToken=SECRET, + ) + auth = SekaiUserAuthData( + sessionToken=SECRET, + appVersion="1.0.0", + multiPlayVersion="1.0.0", + dataVersion="1.0.0", + assetVersion="1.0.0", + ) + assert SECRET not in repr(user) + assert SECRET not in repr(auth) + + +def test_database_repr_and_asdict_hide_cached_headers(): + cache = _make_secret_cache() + assert SECRET not in repr(cache.database) + asdict = dataclasses.asdict(cache.database) + assert SECRET not in repr(asdict) + # asdict's secret-free view must still cover the token-bearing field. + assert "cached_headers" in asdict + + +def test_legacy_plain_dict_headers_are_normalized_on_load(caplog): + """Caches written by older versions store a plain dict in cached_headers, + and save() replaces that field with the live session headers. So the token + must be put on the session headers to actually reach the cache file.""" + import io + import pickle + + caplog.set_level(DEBUG) + cache = _make_secret_cache() + cache.headers["X-Session-Token"] = SECRET + cache.headers["Cookie"] = SECRET + buf = io.BytesIO() + cache.save(buf) + buf.seek(0) + # Round-trip the raw pickle with the header field demoted to a plain dict, + # exactly as an older sssekai wrote it. + database = pickle.load(buf) + database.cached_headers = dict(database.cached_headers) + assert SECRET in repr(database.cached_headers) + buf = io.BytesIO(pickle.dumps(database)) + from sssekai.abcache import AbCache, RedactedHeaders + + loaded = AbCache.from_file(buf) + assert isinstance(loaded.database.cached_headers, RedactedHeaders) + assert dict(loaded.database.cached_headers)["X-Session-Token"] == SECRET + assert SECRET not in repr(loaded.database) + assert SECRET not in repr(dataclasses.asdict(loaded.database)) + assert SECRET not in caplog.text + + +def test_save_and_load_do_not_log_credential(caplog): + import io + + from sssekai.abcache import AbCache + + cache = _make_secret_cache() + caplog.clear() + caplog.set_level(DEBUG) + buf = io.BytesIO() + cache.save(buf) + buf.seek(0) + AbCache.from_file(buf) + assert SECRET not in caplog.text + + +def test_registered_user_data_is_dataclass_and_not_leaked(caplog, monkeypatch): + from sssekai.abcache import SekaiUserData + from sssekai.abcache.auth import register_as_anonymous_user_sega + + cache = _make_cache(credential=None) + cache.headers["X-Platform"] = "Android" + cache.headers["X-DeviceModel"] = "test" + cache.headers["X-OperatingSystem"] = "Android" + + monkeypatch.setattr(cache, "_update_request_headers", lambda: None) + monkeypatch.setattr(cache, "update_signatures", lambda: None) + monkeypatch.setattr(cache, "request_packed", lambda *a, **kw: object()) + monkeypatch.setattr( + cache, + "response_to_dict", + lambda resp: { + "credential": _fake_jwt(signature=SECRET), + "userRegistration": {"userId": 1}, + }, + ) + + caplog.clear() + caplog.set_level(DEBUG) + register_as_anonymous_user_sega(cache) + assert isinstance(cache.database.sekai_user_data, SekaiUserData) + # The credential must really be present, otherwise the assertions below + # would pass vacuously. + assert SECRET in cache.database.sekai_user_data.credential + asdict = dataclasses.asdict(cache.database) + assert SECRET not in repr(cache.database) + assert SECRET not in repr(asdict) + assert SECRET not in caplog.text + + +def test_abserve_host_defaults_to_loopback(): + from sssekai.__main__ import create_parser + + parser = create_parser() + args = parser.parse_args(["abserve"]) + assert args.host == "127.0.0.1" + + +def test_legacy_dict_user_data_is_normalized_and_redacted(caplog): + """sssekai <= 0.8.2 stored the raw API response dict in sekai_user_data.""" + import io + import pickle + + from sssekai.abcache import AbCache, AbCacheConfig, SekaiUserData + + SECRET_LOCAL = "LEGACY_CREDENTIAL_XYZ" + cache = AbCache(AbCacheConfig("jp", "1", "android", "h")) + cache.database.sekai_user_data = { + "credential": SECRET_LOCAL, + "userRegistration": {"userId": 1}, + } + buf = io.BytesIO() + pickle.dump(cache.database, buf) + buf.seek(0) + + caplog.clear() + caplog.set_level(DEBUG) + loaded = AbCache.from_file(buf) + assert isinstance(loaded.database.sekai_user_data, SekaiUserData) + assert loaded.SEKAI_CREDENTIAL == SECRET_LOCAL + assert SECRET_LOCAL not in repr(loaded.database) + assert SECRET_LOCAL not in repr(dataclasses.asdict(loaded.database)) + assert SECRET_LOCAL not in caplog.text From 4426ff6faa568688b25218c37d7588a42ccec1d4 Mon Sep 17 00:00:00 2001 From: MetaMiku Date: Mon, 2 Nov 2026 12:31:39 +0000 Subject: [PATCH 2/2] style: restore LF line endings in PR files --- .github/workflows/python-publish.yml | 190 +-- .vscode/launch.json | 888 ++++++------- README.md | 84 +- setup.py | 104 +- sssekai/__gui__.py | 152 +-- sssekai/__main__.py | 1078 ++++++++-------- sssekai/abcache/__init__.py | 1750 +++++++++++++------------- sssekai/abcache/auth.py | 60 +- sssekai/entrypoint/apphash.py | 552 ++++---- sssekai/entrypoint/rla2json.py | 160 +-- 10 files changed, 2509 insertions(+), 2509 deletions(-) diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index e4ffd71..442976b 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -1,95 +1,95 @@ -name: Publish - -on: - workflow_dispatch: - inputs: - release_pypi: - description: 'Publish to PyPI' - type: boolean - default: true - release_github: - description: 'Build and release Windows executables' - type: boolean - default: false - -permissions: - contents: read - -env: - PYTHONUTF8: '1' - -concurrency: - group: publish - cancel-in-progress: false - -jobs: - build: - if: inputs.release_pypi || inputs.release_github - runs-on: windows-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Install build and test dependencies - run: python -m pip install build twine pytest - - name: Build distributions - run: python -m build - - name: Check distributions - run: python -m twine check --strict dist/* - - name: Install wheel - shell: pwsh - run: python -m pip install (Get-Item dist/*.whl).FullName - - name: Test - run: python -m pytest -v - - name: Check installed command - working-directory: ${{ runner.temp }} - run: sssekai --help - - uses: actions/upload-artifact@v4 - with: - name: python-distributions - path: dist/ - if-no-files-found: error - - publish-pypi: - if: inputs.release_pypi - needs: build - runs-on: ubuntu-latest - environment: - name: pypi - url: https://pypi.org/project/sssekai/ - permissions: - id-token: write - steps: - - uses: actions/download-artifact@v4 - with: - name: python-distributions - path: dist/ - - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - - release-windows: - if: inputs.release_github - needs: build - runs-on: windows-latest - permissions: - contents: write - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Install dependencies - run: python -m pip install pyinstaller ".[il2cpp]" - # NOTE: The GUI executable is not built here: its dependency GooeyEx is not - # published on public PyPI. Build it locally with `pyinstaller sssekai-gui.spec` - # from an environment where GooeyEx is installed. - - name: Build Windows executables - run: pyinstaller sssekai.spec - - name: Create GitHub release - shell: pwsh - env: - GH_TOKEN: ${{ github.token }} - run: | - $version = python -c "import sssekai; print(sssekai.__version__)" - gh release create $version (Get-Item dist/*.exe).FullName --target "${{ github.sha }}" --title "Version $version" --generate-notes +name: Publish + +on: + workflow_dispatch: + inputs: + release_pypi: + description: 'Publish to PyPI' + type: boolean + default: true + release_github: + description: 'Build and release Windows executables' + type: boolean + default: false + +permissions: + contents: read + +env: + PYTHONUTF8: '1' + +concurrency: + group: publish + cancel-in-progress: false + +jobs: + build: + if: inputs.release_pypi || inputs.release_github + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install build and test dependencies + run: python -m pip install build twine pytest + - name: Build distributions + run: python -m build + - name: Check distributions + run: python -m twine check --strict dist/* + - name: Install wheel + shell: pwsh + run: python -m pip install (Get-Item dist/*.whl).FullName + - name: Test + run: python -m pytest -v + - name: Check installed command + working-directory: ${{ runner.temp }} + run: sssekai --help + - uses: actions/upload-artifact@v4 + with: + name: python-distributions + path: dist/ + if-no-files-found: error + + publish-pypi: + if: inputs.release_pypi + needs: build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/sssekai/ + permissions: + id-token: write + steps: + - uses: actions/download-artifact@v4 + with: + name: python-distributions + path: dist/ + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + release-windows: + if: inputs.release_github + needs: build + runs-on: windows-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install dependencies + run: python -m pip install pyinstaller ".[il2cpp]" + # NOTE: The GUI executable is not built here: its dependency GooeyEx is not + # published on public PyPI. Build it locally with `pyinstaller sssekai-gui.spec` + # from an environment where GooeyEx is installed. + - name: Build Windows executables + run: pyinstaller sssekai.spec + - name: Create GitHub release + shell: pwsh + env: + GH_TOKEN: ${{ github.token }} + run: | + $version = python -c "import sssekai; print(sssekai.__version__)" + gh release create $version (Get-Item dist/*.exe).FullName --target "${{ github.sha }}" --title "Version $version" --generate-notes diff --git a/.vscode/launch.json b/.vscode/launch.json index 0aa433e..38db2ac 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,445 +1,445 @@ -{ - "version": "0.2.0", - "configurations": [ - { - "name": "Animation Plot", - "type": "debugpy", - "request": "launch", - "program": "${workspaceFolder}/misc/animation_plot.py", - "justMyCode": false - }, - { - "name": "AbCache Availability Check", - "type": "debugpy", - "request": "launch", - "program": "${workspaceFolder}/misc/abcache_availability_check.py", - "justMyCode": false - }, - { - "name": "GUI Test", - "type": "debugpy", - "request": "launch", - "program": "${workspaceFolder}/sssekai/__gui__.py", - "justMyCode": false - }, - { - "name": "Python: SpineExtract test", - "type": "debugpy", - "request": "launch", - "module": "tests.test_spine", - "justMyCode": false - }, - { - "name": "Python: Live2DExtract test", - "type": "debugpy", - "request": "launch", - "module": "tests.test_live2d", - "justMyCode": false - }, - { - "name": "Python: AppHash test", - "type": "debugpy", - "request": "launch", - "module": "tests.test_apphash", - "justMyCode": false - }, - { - "name": "Python: AbCache test", - "type": "debugpy", - "request": "launch", - "module": "tests.test_abcache", - "justMyCode": false - }, - { - "name": "Python: Legacy Animation format test", - "type": "debugpy", - "request": "launch", - "module": "tests.test_legacy_anim", - "justMyCode": false - }, - { - "name": "Python: Current File", - "type": "python", - "request": "launch", - "program": "${file}", - "console": "integratedTerminal", - "justMyCode": false, - "args": [] - }, - { - "name": "Python: AbCache (JP Prod)", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "abcache", - "--proxy", - "http://127.0.0.1:8080", - "--app-region", - "jp", - "--app-version", - "5.5.1", - "--app-appHash", - "30829b50-b4b7-d39e-1d55-21ba51e0a8bc", - ], - "justMyCode": true - }, - { - "name": "Python: AbCache (JP Staging)", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "abcache", - "--proxy", - "http://127.0.0.1:8080", - "--app-region", - "jp", - "--app-version", - "5.5.0", - "--app-appHash", - "9b75c207-5fa7-a852-71e3-4097eb9b6027", - ], - "justMyCode": true - }, - { - "name": "Python: AbCache (TW)", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "abcache", - "--app-region", - "tw", - "--app-version", - "4.1.1", - "--app-appHash", - "41fd71f2-f715-bc10-5852-0a9d8542f760", - "--app-abVersion", - "4.1.0", - "--download-filter", - ".*/stage_decoration/.*", - "--download-dir", - "~/.sssekai", - ], - "justMyCode": true - }, - { - "name": "Python: AbCache (KR)", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "abcache", - "--app-region", - "kr", - "--app-version", - "3.4.3", - "--app-appHash", - "a3015fe8-785f-27e1-fb8b-546a23c82c1f", - "--app-abVersion", - "3.4.0" - ], - "justMyCode": true - }, - { - "name": "Python: AbCache (EN)", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "abcache", - "--app-region", - "en", - "--app-version", - "3.8.0", - "--app-appHash", - "3272b0b0-f161-448e-8416-1f8bd2c443b3", - "--app-platform", - "ios" - ], - "justMyCode": true - }, - { - "name": "Python: AbCache (CN)", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "abcache", - "--app-region", - "cn", - "--app-version", - "3.4.0", - "--app-appHash", - "a3015fe8-785f-27e1-fb8b-546a23c82c1f" - ], - "justMyCode": true - }, - { - "name": "Python: AbCache (dump Master data)", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "abcache", - "--app-region", - "en", - "--app-version", - "3.2.0", - "--app-appHash", - "1380836d-de9e-49a8-afe5-c52ba589c8c9", - "--dump-master-data", - "~/.sssekai/master_data" - ], - "justMyCode": true - }, - { - "name": "Python: AbServe (FUSE)", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "abserve", - "--fuse", - "~/ProjectSEKAI" - ], - "justMyCode": false - }, - { - "name": "Python: AbServe", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "abserve", - // "--db", - // "/mnt/c/Users/mos9527/.sssekai/abcache.db", - ], - "justMyCode": false - }, - { - "name": "Python: AbCache (DL)", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "abcache", - "--no-update", - "--download-dir", - "~/.sssekai/bundles/", - "--download-filter", - ".*/stage_decoration/0344.*" - ], - "justMyCode": false - }, - { - "name": "Python: MVData", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "mvdata", - "D:\\Reverse\\proseka_reverse\\assets\\live_pv\\mv_data", - "D:\\Reverse\\proseka_reverse\\mvdata.json" - ], - "justMyCode": true - }, - { - "name": "Python: Tests", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/sssekai.test.py", - "cwd": "${workspaceFolder}", - "console": "integratedTerminal", - "justMyCode": false - }, - { - "name": "Python: apphash", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "apphash", - "--apk-src", - "D:\\Reverse\\proseka_reverse\\game_binary\\v550\\550.xapk", - "--deep", - "--format", - "json" - ], - "justMyCode": true - }, - { - "name": "Python: apphash TW w/ metadata", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "apphash", - "--apk-src", - "/Users/mos9527/sekai-apphash/tw/tw.apk" - ], - "justMyCode": true - }, - { - "name": "Python: usmdemux", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "usmdemux", - "D:\\Sekai\\assets\\8015", - "D:\\Sekai\\assets\\8015.usm", - ], - "justMyCode": true - }, - { - "name": "Python: live2dextract", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "live2dextract", - "C:\\Users\\mos9527\\.sssekai\\abcache\\live2d\\model\\21miku_night", - "C:\\Users\\mos9527\\Desktop\\Cache" - ], - "justMyCode": true - }, - { - "name": "Python: rla2json", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "rla2json", - "/Volumes/mos9527弄丢的盘/Reverse/proseka_reverse/sse_packet_importer/packets/356_5043", - "/Volumes/mos9527弄丢的盘/Reverse/proseka_reverse/sse_packet_importer/exported/356_5043", - "--version", - "1.6", - "--strict" - ], - "justMyCode": true - }, - { - "name": "Python: Test Animation", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/test.animation.py", - "console": "integratedTerminal", - "justMyCode": false, - "args": [] - }, - { - "name": "Python: RLA Test (Single File)", - "type": "python", - "request": "launch", - "program": "${workspaceFolder}/sssekai/fmt/rla.py", - "console": "integratedTerminal", - "justMyCode": false, - "args": [ - // "D:\\proseka_reverse\\sse_packet_importer\\packets\\streaming_vbs_1\\sekai_30_00000006.rla.bytes" - "D:\\proseka_reverse\\sse_packet_importer\\packets\\4268\\1728191806276-0.bin" - ] - }, - { - "name": "Python: AbCache (Override EN Asset Version)", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "abcache", - "--app-region", - "en", - // "--app-version", - // "3.6.0", - // "--app-appHash", - // "a3015fe8-785f-27e1-fb8b-546a23c82c1f", - "--app-asset-version", - "3.7.0.0", - "--app-asset-hash", - "d2e5b1bb-d0d4-4535-99ca-bd84ae291a9d" - ], - "justMyCode": true - }, - { - "name": "Python: AbCache (Override JP Asset Version)", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "abcache", - "--app-region", - "jp", - // "--app-version", - // "5.3.0", - // "--app-appHash", - // "2a9bb18f-c309-1993-5b6e-9f25ddc88be3", - "--app-asset-version", - "5.3.0.90", - "--app-asset-hash", - "7f2ebd91-ce39-6ea2-20b2-e05d2872e9e6", - "--app-asset-host", - "cf2d2388" - ], - "justMyCode": true - }, - { - "name": "Python: IL2CPP v381 (Android)", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "il2cpp", - "D:\\Reverse\\proseka_reverse\\game_binary\\v381\\com.sega.pjsekai\\assets\\bin\\Data\\Managed\\Metadata\\global-metadata.dat", - "D:\\Reverse\\proseka_reverse\\game_binary\\v381\\config.arm64_v8a\\lib\\arm64-v8a\\libil2cpp.so", - "D:\\Reverse\\proseka_reverse\\game_binary\\v381\\il2cpp" - ], - "justMyCode": false - }, - { - "name": "Python: IL2CPP v550 (Android)", - "type": "python", - "request": "launch", - "module": "sssekai", - "args": [ - "--log-level", - "DEBUG", - "il2cpp", - "D:\\Reverse\\proseka_reverse\\game_binary\\v550\\com.sega.pjsekai\\assets\\bin\\Data\\Managed\\Metadata\\global-metadata.dat", - "D:\\Reverse\\proseka_reverse\\game_binary\\v550\\config.arm64_v8a\\lib\\arm64-v8a\\libil2cpp.so", - "D:\\Reverse\\proseka_reverse\\game_binary\\v550\\il2cpp" - ], - "justMyCode": false - }, - ] +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Animation Plot", + "type": "debugpy", + "request": "launch", + "program": "${workspaceFolder}/misc/animation_plot.py", + "justMyCode": false + }, + { + "name": "AbCache Availability Check", + "type": "debugpy", + "request": "launch", + "program": "${workspaceFolder}/misc/abcache_availability_check.py", + "justMyCode": false + }, + { + "name": "GUI Test", + "type": "debugpy", + "request": "launch", + "program": "${workspaceFolder}/sssekai/__gui__.py", + "justMyCode": false + }, + { + "name": "Python: SpineExtract test", + "type": "debugpy", + "request": "launch", + "module": "tests.test_spine", + "justMyCode": false + }, + { + "name": "Python: Live2DExtract test", + "type": "debugpy", + "request": "launch", + "module": "tests.test_live2d", + "justMyCode": false + }, + { + "name": "Python: AppHash test", + "type": "debugpy", + "request": "launch", + "module": "tests.test_apphash", + "justMyCode": false + }, + { + "name": "Python: AbCache test", + "type": "debugpy", + "request": "launch", + "module": "tests.test_abcache", + "justMyCode": false + }, + { + "name": "Python: Legacy Animation format test", + "type": "debugpy", + "request": "launch", + "module": "tests.test_legacy_anim", + "justMyCode": false + }, + { + "name": "Python: Current File", + "type": "python", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "justMyCode": false, + "args": [] + }, + { + "name": "Python: AbCache (JP Prod)", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "abcache", + "--proxy", + "http://127.0.0.1:8080", + "--app-region", + "jp", + "--app-version", + "5.5.1", + "--app-appHash", + "30829b50-b4b7-d39e-1d55-21ba51e0a8bc", + ], + "justMyCode": true + }, + { + "name": "Python: AbCache (JP Staging)", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "abcache", + "--proxy", + "http://127.0.0.1:8080", + "--app-region", + "jp", + "--app-version", + "5.5.0", + "--app-appHash", + "9b75c207-5fa7-a852-71e3-4097eb9b6027", + ], + "justMyCode": true + }, + { + "name": "Python: AbCache (TW)", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "abcache", + "--app-region", + "tw", + "--app-version", + "4.1.1", + "--app-appHash", + "41fd71f2-f715-bc10-5852-0a9d8542f760", + "--app-abVersion", + "4.1.0", + "--download-filter", + ".*/stage_decoration/.*", + "--download-dir", + "~/.sssekai", + ], + "justMyCode": true + }, + { + "name": "Python: AbCache (KR)", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "abcache", + "--app-region", + "kr", + "--app-version", + "3.4.3", + "--app-appHash", + "a3015fe8-785f-27e1-fb8b-546a23c82c1f", + "--app-abVersion", + "3.4.0" + ], + "justMyCode": true + }, + { + "name": "Python: AbCache (EN)", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "abcache", + "--app-region", + "en", + "--app-version", + "3.8.0", + "--app-appHash", + "3272b0b0-f161-448e-8416-1f8bd2c443b3", + "--app-platform", + "ios" + ], + "justMyCode": true + }, + { + "name": "Python: AbCache (CN)", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "abcache", + "--app-region", + "cn", + "--app-version", + "3.4.0", + "--app-appHash", + "a3015fe8-785f-27e1-fb8b-546a23c82c1f" + ], + "justMyCode": true + }, + { + "name": "Python: AbCache (dump Master data)", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "abcache", + "--app-region", + "en", + "--app-version", + "3.2.0", + "--app-appHash", + "1380836d-de9e-49a8-afe5-c52ba589c8c9", + "--dump-master-data", + "~/.sssekai/master_data" + ], + "justMyCode": true + }, + { + "name": "Python: AbServe (FUSE)", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "abserve", + "--fuse", + "~/ProjectSEKAI" + ], + "justMyCode": false + }, + { + "name": "Python: AbServe", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "abserve", + // "--db", + // "/mnt/c/Users/mos9527/.sssekai/abcache.db", + ], + "justMyCode": false + }, + { + "name": "Python: AbCache (DL)", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "abcache", + "--no-update", + "--download-dir", + "~/.sssekai/bundles/", + "--download-filter", + ".*/stage_decoration/0344.*" + ], + "justMyCode": false + }, + { + "name": "Python: MVData", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "mvdata", + "D:\\Reverse\\proseka_reverse\\assets\\live_pv\\mv_data", + "D:\\Reverse\\proseka_reverse\\mvdata.json" + ], + "justMyCode": true + }, + { + "name": "Python: Tests", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/sssekai.test.py", + "cwd": "${workspaceFolder}", + "console": "integratedTerminal", + "justMyCode": false + }, + { + "name": "Python: apphash", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "apphash", + "--apk-src", + "D:\\Reverse\\proseka_reverse\\game_binary\\v550\\550.xapk", + "--deep", + "--format", + "json" + ], + "justMyCode": true + }, + { + "name": "Python: apphash TW w/ metadata", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "apphash", + "--apk-src", + "/Users/mos9527/sekai-apphash/tw/tw.apk" + ], + "justMyCode": true + }, + { + "name": "Python: usmdemux", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "usmdemux", + "D:\\Sekai\\assets\\8015", + "D:\\Sekai\\assets\\8015.usm", + ], + "justMyCode": true + }, + { + "name": "Python: live2dextract", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "live2dextract", + "C:\\Users\\mos9527\\.sssekai\\abcache\\live2d\\model\\21miku_night", + "C:\\Users\\mos9527\\Desktop\\Cache" + ], + "justMyCode": true + }, + { + "name": "Python: rla2json", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "rla2json", + "/Volumes/mos9527弄丢的盘/Reverse/proseka_reverse/sse_packet_importer/packets/356_5043", + "/Volumes/mos9527弄丢的盘/Reverse/proseka_reverse/sse_packet_importer/exported/356_5043", + "--version", + "1.6", + "--strict" + ], + "justMyCode": true + }, + { + "name": "Python: Test Animation", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/test.animation.py", + "console": "integratedTerminal", + "justMyCode": false, + "args": [] + }, + { + "name": "Python: RLA Test (Single File)", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/sssekai/fmt/rla.py", + "console": "integratedTerminal", + "justMyCode": false, + "args": [ + // "D:\\proseka_reverse\\sse_packet_importer\\packets\\streaming_vbs_1\\sekai_30_00000006.rla.bytes" + "D:\\proseka_reverse\\sse_packet_importer\\packets\\4268\\1728191806276-0.bin" + ] + }, + { + "name": "Python: AbCache (Override EN Asset Version)", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "abcache", + "--app-region", + "en", + // "--app-version", + // "3.6.0", + // "--app-appHash", + // "a3015fe8-785f-27e1-fb8b-546a23c82c1f", + "--app-asset-version", + "3.7.0.0", + "--app-asset-hash", + "d2e5b1bb-d0d4-4535-99ca-bd84ae291a9d" + ], + "justMyCode": true + }, + { + "name": "Python: AbCache (Override JP Asset Version)", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "abcache", + "--app-region", + "jp", + // "--app-version", + // "5.3.0", + // "--app-appHash", + // "2a9bb18f-c309-1993-5b6e-9f25ddc88be3", + "--app-asset-version", + "5.3.0.90", + "--app-asset-hash", + "7f2ebd91-ce39-6ea2-20b2-e05d2872e9e6", + "--app-asset-host", + "cf2d2388" + ], + "justMyCode": true + }, + { + "name": "Python: IL2CPP v381 (Android)", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "il2cpp", + "D:\\Reverse\\proseka_reverse\\game_binary\\v381\\com.sega.pjsekai\\assets\\bin\\Data\\Managed\\Metadata\\global-metadata.dat", + "D:\\Reverse\\proseka_reverse\\game_binary\\v381\\config.arm64_v8a\\lib\\arm64-v8a\\libil2cpp.so", + "D:\\Reverse\\proseka_reverse\\game_binary\\v381\\il2cpp" + ], + "justMyCode": false + }, + { + "name": "Python: IL2CPP v550 (Android)", + "type": "python", + "request": "launch", + "module": "sssekai", + "args": [ + "--log-level", + "DEBUG", + "il2cpp", + "D:\\Reverse\\proseka_reverse\\game_binary\\v550\\com.sega.pjsekai\\assets\\bin\\Data\\Managed\\Metadata\\global-metadata.dat", + "D:\\Reverse\\proseka_reverse\\game_binary\\v550\\config.arm64_v8a\\lib\\arm64-v8a\\libil2cpp.so", + "D:\\Reverse\\proseka_reverse\\game_binary\\v550\\il2cpp" + ], + "justMyCode": false + }, + ] } \ No newline at end of file diff --git a/README.md b/README.md index fe10b2b..7e6727a 100644 --- a/README.md +++ b/README.md @@ -1,42 +1,42 @@ -# sssekai - -本项目基于 [mos9527/sssekai](https://github.com/mos9527/sssekai)，由 [MetaSekaiLab](https://github.com/MetaSekaiLab) 继续维护。感谢原作者及历代贡献者。 - -[![Windows Build](https://github.com/MetaSekaiLab/sssekai/actions/workflows/python-publish.yml/badge.svg)](https://github.com/MetaSekaiLab/sssekai/actions/workflows/python-publish.yml) [![Releases](https://img.shields.io/github/downloads/MetaSekaiLab/sssekai/total.svg)](https://github.com/MetaSekaiLab/sssekai/releases/) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) - -Command-line tool (w/Python API support) for Project SEKAI (JP: プロジェクトセカイ カラフルステージ！ feat.初音ミク) game assets. - -# Installing & Updating -**Windows 用户**：请查看本仓库的 [Releases](https://github.com/MetaSekaiLab/sssekai/releases)。 - -Python >=3.10 is required to run this tool. - -通过 PyPI 安装或更新： -```bash -pip install -U sssekai -``` -To use the latest development version, you can install directly from GitHub: -```bash -pip install -U git+https://github.com/MetaSekaiLab/sssekai -``` -## Using the GUI version -The GUI depends on [GooeyEx](https://github.com/mos9527/GooeyEx/) **>= 0.1.1**, which is not published on PyPI. Install it manually from upstream (a `git` executable and network access are required), then run the GUI: - -```bash -pip install -U "GooeyEx @ git+https://github.com/mos9527/GooeyEx" -sssekai-gui -``` - -`GooeyEx` 0.1.0 (the newest version on PyPI) crashes on import; verify you got 0.1.1 or newer with `python -c "import GooeyEx; print(GooeyEx.__version__)"`. If git is unavailable, use the CLI executable from Releases instead. -# 问题反馈 -使用问题和缺陷请提交至本仓库的 [Issues](https://github.com/MetaSekaiLab/sssekai/issues)。 - -# See Also -https://github.com/mos9527/sssekai_blender_io - -# License -MIT - -# References -- https://github.com/K0lb3/UnityPy -- https://mos9527.github.io/tags/project-sekai/ +# sssekai + +本项目基于 [mos9527/sssekai](https://github.com/mos9527/sssekai)，由 [MetaSekaiLab](https://github.com/MetaSekaiLab) 继续维护。感谢原作者及历代贡献者。 + +[![Windows Build](https://github.com/MetaSekaiLab/sssekai/actions/workflows/python-publish.yml/badge.svg)](https://github.com/MetaSekaiLab/sssekai/actions/workflows/python-publish.yml) [![Releases](https://img.shields.io/github/downloads/MetaSekaiLab/sssekai/total.svg)](https://github.com/MetaSekaiLab/sssekai/releases/) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) + +Command-line tool (w/Python API support) for Project SEKAI (JP: プロジェクトセカイ カラフルステージ！ feat.初音ミク) game assets. + +# Installing & Updating +**Windows 用户**：请查看本仓库的 [Releases](https://github.com/MetaSekaiLab/sssekai/releases)。 + +Python >=3.10 is required to run this tool. + +通过 PyPI 安装或更新： +```bash +pip install -U sssekai +``` +To use the latest development version, you can install directly from GitHub: +```bash +pip install -U git+https://github.com/MetaSekaiLab/sssekai +``` +## Using the GUI version +The GUI depends on [GooeyEx](https://github.com/mos9527/GooeyEx/) **>= 0.1.1**, which is not published on PyPI. Install it manually from upstream (a `git` executable and network access are required), then run the GUI: + +```bash +pip install -U "GooeyEx @ git+https://github.com/mos9527/GooeyEx" +sssekai-gui +``` + +`GooeyEx` 0.1.0 (the newest version on PyPI) crashes on import; verify you got 0.1.1 or newer with `python -c "import GooeyEx; print(GooeyEx.__version__)"`. If git is unavailable, use the CLI executable from Releases instead. +# 问题反馈 +使用问题和缺陷请提交至本仓库的 [Issues](https://github.com/MetaSekaiLab/sssekai/issues)。 + +# See Also +https://github.com/mos9527/sssekai_blender_io + +# License +MIT + +# References +- https://github.com/K0lb3/UnityPy +- https://mos9527.github.io/tags/project-sekai/ diff --git a/setup.py b/setup.py index ed14a3b..37320f7 100644 --- a/setup.py +++ b/setup.py @@ -1,52 +1,52 @@ -import sys, os - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import setuptools, sssekai - -with open("README.md", "r", encoding="utf-8") as fh: - long_description = fh.read() - -setuptools.setup( - name="sssekai", - version=sssekai.__version__, - author="greats3an", - author_email="greats3an@gmail.com", - maintainer="MetaSekaiLab", - description="Project SEKAI Asset Utility / PJSK 资源工具", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/MetaSekaiLab/sssekai", - project_urls={ - "Source": "https://github.com/MetaSekaiLab/sssekai", - "Issues": "https://github.com/MetaSekaiLab/sssekai/issues", - }, - packages=setuptools.find_packages(include=["sssekai", "sssekai.*"]), - classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - ], - install_requires=[ - "msgpack", - "requests", - "pycryptodome", - "unitypy >= 1.25.0, < 2.0.0", - "tqdm", - "coloredlogs", - "fsspec", - ], - # NOTE: No `gui` extra on purpose. The GUI requires GooeyEx >= 0.1.1, which - # is not published on PyPI (0.1.0 crashes at import without a wx.App), and - # PyPI rejects direct-URL requirements. See README for manual installation. - # The unused `criware` extra was removed with its dependency for the same - # reason: PyCriCodecsEx is not installable and nothing imports it. - extras_require={"il2cpp": ["lief"]}, - entry_points={ - "console_scripts": [ - "sssekai = sssekai.__main__:__main__", - "sssekai-gui = sssekai.__gui__:__main__", - ], - "fsspec.specs": ["abcache = sssekai.abcache.fs.AbCacheFilesystem"], - }, - python_requires=">=3.10", -) +import sys, os + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import setuptools, sssekai + +with open("README.md", "r", encoding="utf-8") as fh: + long_description = fh.read() + +setuptools.setup( + name="sssekai", + version=sssekai.__version__, + author="greats3an", + author_email="greats3an@gmail.com", + maintainer="MetaSekaiLab", + description="Project SEKAI Asset Utility / PJSK 资源工具", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/MetaSekaiLab/sssekai", + project_urls={ + "Source": "https://github.com/MetaSekaiLab/sssekai", + "Issues": "https://github.com/MetaSekaiLab/sssekai/issues", + }, + packages=setuptools.find_packages(include=["sssekai", "sssekai.*"]), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ], + install_requires=[ + "msgpack", + "requests", + "pycryptodome", + "unitypy >= 1.25.0, < 2.0.0", + "tqdm", + "coloredlogs", + "fsspec", + ], + # NOTE: No `gui` extra on purpose. The GUI requires GooeyEx >= 0.1.1, which + # is not published on PyPI (0.1.0 crashes at import without a wx.App), and + # PyPI rejects direct-URL requirements. See README for manual installation. + # The unused `criware` extra was removed with its dependency for the same + # reason: PyCriCodecsEx is not installable and nothing imports it. + extras_require={"il2cpp": ["lief"]}, + entry_points={ + "console_scripts": [ + "sssekai = sssekai.__main__:__main__", + "sssekai-gui = sssekai.__gui__:__main__", + ], + "fsspec.specs": ["abcache = sssekai.abcache.fs.AbCacheFilesystem"], + }, + python_requires=">=3.10", +) diff --git a/sssekai/__gui__.py b/sssekai/__gui__.py index 71a9f15..293fd1e 100644 --- a/sssekai/__gui__.py +++ b/sssekai/__gui__.py @@ -1,76 +1,76 @@ -from dataclasses import dataclass, field -from sssekai.__main__ import create_parser -from typing import Any, List -from argparse import ArgumentParser -from logging import basicConfig, getLogger -import sys - -try: - from GooeyEx import Gooey, GooeyParser -except ImportError as e: - print( - "The GUI requires GooeyEx >= 0.1.1, which is not published on PyPI.\n" - "Install it manually, then retry:\n" - ' pip install -U "GooeyEx @ git+https://github.com/mos9527/GooeyEx"\n' - "Otherwise use the CLI executable from the Releases page.", - file=sys.stderr, - ) - raise SystemExit(1) from e - - -from sssekai.unity import sssekai_set_unity_version - - -@Gooey( - show_preview_warning=False, - program_name="sssekai", - tabbed_groups=True, - advanced=True, - monospace_display=True, - default_size=(800, 600), - menu=[ - { - "name": "Help", - "items": [ - { - "type": "Link", - "menuTitle": "Report an Issue", - "url": "https://github.com/MetaSekaiLab/sssekai/issues", - }, - { - "type": "Link", - "menuTitle": "GitHub", - "url": "https://github.com/MetaSekaiLab/sssekai", - }, - ], - } - ], -) -def __main__(): - from tqdm.std import tqdm as tqdm_c - - parser = create_parser(GooeyParser) - args = parser.parse_args() - - class TqdmMutexStream: - @staticmethod - def write(__s): - # Gooey[Ex] only reads output from stdout so we'd do that here. - with tqdm_c.external_write_mode(file=sys.stdout, nolock=False): - return sys.stdout.write(__s) - - basicConfig( - level="DEBUG", - format="%(asctime)s | %(levelname).1s | %(name)s %(message)s", - datefmt="%H:%M:%S", - stream=TqdmMutexStream, - ) - # override unity version - sssekai_set_unity_version(args.unity_version) - if "func" in args: - args.func(args) - pass - - -if __name__ in {"__main__", "__gui__"}: - __main__() +from dataclasses import dataclass, field +from sssekai.__main__ import create_parser +from typing import Any, List +from argparse import ArgumentParser +from logging import basicConfig, getLogger +import sys + +try: + from GooeyEx import Gooey, GooeyParser +except ImportError as e: + print( + "The GUI requires GooeyEx >= 0.1.1, which is not published on PyPI.\n" + "Install it manually, then retry:\n" + ' pip install -U "GooeyEx @ git+https://github.com/mos9527/GooeyEx"\n' + "Otherwise use the CLI executable from the Releases page.", + file=sys.stderr, + ) + raise SystemExit(1) from e + + +from sssekai.unity import sssekai_set_unity_version + + +@Gooey( + show_preview_warning=False, + program_name="sssekai", + tabbed_groups=True, + advanced=True, + monospace_display=True, + default_size=(800, 600), + menu=[ + { + "name": "Help", + "items": [ + { + "type": "Link", + "menuTitle": "Report an Issue", + "url": "https://github.com/MetaSekaiLab/sssekai/issues", + }, + { + "type": "Link", + "menuTitle": "GitHub", + "url": "https://github.com/MetaSekaiLab/sssekai", + }, + ], + } + ], +) +def __main__(): + from tqdm.std import tqdm as tqdm_c + + parser = create_parser(GooeyParser) + args = parser.parse_args() + + class TqdmMutexStream: + @staticmethod + def write(__s): + # Gooey[Ex] only reads output from stdout so we'd do that here. + with tqdm_c.external_write_mode(file=sys.stdout, nolock=False): + return sys.stdout.write(__s) + + basicConfig( + level="DEBUG", + format="%(asctime)s | %(levelname).1s | %(name)s %(message)s", + datefmt="%H:%M:%S", + stream=TqdmMutexStream, + ) + # override unity version + sssekai_set_unity_version(args.unity_version) + if "func" in args: + args.func(args) + pass + + +if __name__ in {"__main__", "__gui__"}: + __main__() diff --git a/sssekai/__main__.py b/sssekai/__main__.py index 9cac0c4..d643dcf 100644 --- a/sssekai/__main__.py +++ b/sssekai/__main__.py @@ -1,539 +1,539 @@ -import argparse, sys, logging, traceback - -# region Deferred imports -# This can be done a lot cleaner with importlib. But tools like PyInstaller really -# won't like that. - -def main_apidecrypt(*args, **kwargs): - from sssekai.entrypoint.apidecrypt import main_apidecrypt - return main_apidecrypt(*args, **kwargs) - -def main_abdecrypt(*args, **kwargs): - from sssekai.entrypoint.abdecrypt import main_abdecrypt - return main_abdecrypt(*args, **kwargs) - -def main_rla2json(*args, **kwargs): - from sssekai.entrypoint.rla2json import main_rla2json - return main_rla2json(*args, **kwargs) - -def main_usmdemux(*args, **kwargs): - from sssekai.entrypoint.usmdemux import main_usmdemux - return main_usmdemux(*args, **kwargs) - -def main_abcache(*args, **kwargs): - from sssekai.entrypoint.abcache import main_abcache - return main_abcache(*args, **kwargs) - -def main_abserve(*args, **kwargs): - from sssekai.entrypoint.abserve import main_abserve - return main_abserve(*args, **kwargs) - -def main_live2dextract(*args, **kwargs): - from sssekai.entrypoint.live2dextract import main_live2dextract - return main_live2dextract(*args, **kwargs) - -def main_spineextract(*args, **kwargs): - from sssekai.entrypoint.spineextract import main_spineextract - return main_spineextract(*args, **kwargs) - -def main_apphash(*args, **kwargs): - from sssekai.entrypoint.apphash import main_apphash - return main_apphash(*args, **kwargs) - -def main_il2cpp(*args, **kwargs): - from sssekai.entrypoint.il2cpp import main_il2cpp - return main_il2cpp(*args, **kwargs) - -def main_il2cppextract(*args, **kwargs): - from sssekai.entrypoint.il2cppextract import main_il2cppextract - return main_il2cppextract(*args, **kwargs) - -def main_mvdata(*args, **kwargs): - from sssekai.entrypoint.mvdata import main_mvdata - return main_mvdata(*args, **kwargs) - -def main_moc3paths(*args, **kwargs): - from sssekai.entrypoint.moc3paths import main_moc3paths - return main_moc3paths(*args, **kwargs) - -# endregion - -from sssekai.unity import sssekai_get_unity_version, sssekai_set_unity_version -from sssekai.fmt.rla import RLA_VERSIONS - -logger = logging.getLogger(__name__) - - -def create_parser(clazz=argparse.ArgumentParser): - """ - Create the command line parser for the application. - - Args: - clazz (type): The parser class to use. Defaults to argparse.ArgumentParser. - In GUI mode this should be set to GooeyParser otherwise. - """ - - def gooey_only(**kwargs): - # Silent non-argparse kwargs - if clazz == argparse.ArgumentParser: - return {} - return kwargs - - parser = clazz( - description="""Project SEKAI Asset Utility / PJSK 资源工具""", - formatter_class=argparse.RawTextHelpFormatter, - ) - parser.add_argument( - "--log-level", - type=str, - help="logging level (default: %(default)s)", - default="INFO", - choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], - ) - parser.add_argument( - "--unity-version", - type=str, - help="""Unity version to use (default: %(default)s) -Prior to game version 3.6.0 (JP), this has always been 2020.3.21f1 -This has been changed to 2022.3.21f1 since, which would apply to all the assets from 3.6.0 onwards. -If you encounter any issues, try switching to the old version, or vice versa.""", - default=sssekai_get_unity_version(), - ) - subparsers = parser.add_subparsers( - title="subcommands", description="valid subcommands", help="additional help" - ) - # apidecrypt - apidecrypt_parser = subparsers.add_parser( - "apidecrypt", - usage="""API crypto dumper -This crypto applies to: - - API request/response body dumped by packet sniffer (mitmproxy, wireshark, etc.) - - AssetBundleInfo (see sssekai.abcache)""", - ) - apidecrypt_parser.add_argument( - "infile", type=str, help="input dump file", **gooey_only(widget="FileChooser") - ) - apidecrypt_parser.add_argument( - "outfile", type=str, help="output json file", **gooey_only(widget="FileSaver") - ) - apidecrypt_parser.add_argument( - "--region", - type=str, - help="app region", - default="jp", - choices=["jp", "tw", "en", "kr", "cn"], - ) - apidecrypt_parser.set_defaults(func=main_apidecrypt) - # abdecrypt - abdecrypt_parser = subparsers.add_parser( - "abdecrypt", usage="""Decrypt Sekai AssetBundle""" - ) - abdecrypt_parser.add_argument( - "indir", type=str, help="input directory", **gooey_only(widget="DirChooser") - ) - abdecrypt_parser.add_argument( - "outdir", type=str, help="output directory", **gooey_only(widget="DirChooser") - ) - abdecrypt_parser.set_defaults(func=main_abdecrypt) - # usmdemux - usmdemux_parser = subparsers.add_parser( - "usmdemux", usage="""Demux Sekai USM Video in a AssetBundle""" - ) - usmdemux_parser.add_argument( - "infile", type=str, help="input file", **gooey_only(widget="FileChooser") - ) - usmdemux_parser.add_argument( - "outfile", type=str, help="output file where the .USM file will be saved", **gooey_only(widget="FileSaver") - ) - usmdemux_parser.set_defaults(func=main_usmdemux) - # abcache - abcache_parser = subparsers.add_parser( - "abcache", usage="""Sekai AssetBundle Metadata Cache / Game API Helper""" - ) - abcache_parser.add_argument( - "--proxy", - type=str, - help="HTTP Proxy to use. This overrides the system proxy (environ HTTP_PROXY, HTTPS_PROXY) settings.", - default=None, - ) - group = abcache_parser.add_argument_group("save/load options") - group.add_argument( - "--db", - type=str, - help="""cache database file path (default: %(default)s)""", - default="~/.sssekai/abcache.db", - **gooey_only(widget="FileChooser"), - ) - group.add_argument( - "--no-update", - action="store_true", - help="skip all metadata updates and use cached ones as is. this supercedes 'game version options' and 'authentication options'", - ) - group.add_argument( - "--dump", - type=str, - help="dump the AssetBundle index to the specified JSON file and exit", - default=None, - **gooey_only(widget="FileSaver"), - ) - group = abcache_parser.add_argument_group( - "game version options", - "NOTE: these options are used to *update* the cache database. Use --no-update to skip updating. You can obtain these values with the `apphash` subcommand.", - ) - group.add_argument( - "--app-region", - type=str, - help="PJSK app region (default: %(default)s)", - choices=["jp", "tw", "en", "kr", "cn"], - default="jp", - ) - group.add_argument( - "--app-platform", - type=str, - help="PJSK app platform (default: %(default)s", - choices=["ios", "android"], - default="android", - ) # Let's hope this doesn't age well... - group.add_argument( - "--app-version", - type=str, - help="PJSK app version. For assets this is required unless --no-update is specified, or for EN/JP requisite --app-asset..fields are filled. For other functions this is required unless --no-update is specified", - default="", - ) - group.add_argument( - "--app-appHash", - type=str, - help="PJSK app hash. For assets this is required unless --no-update is specified, or for EN/JP requisite --app-asset..fields are filled. For other functions this is required unless --no-update is specified", - default="", - ) - group = abcache_parser.add_argument_group("ROW specific options") - group.add_argument( - "--app-abVersion", - type=str, - help="PJSK AssetBundle URL version. ROW servers only - This is required unless --no-update is specified", - default=None, - ) - group = abcache_parser.add_argument_group("EN/JP specific options") - group.add_argument( - "--app-asset-host", - type=str, - help="Override AssetBundle host URL. EN/JP servers only.", - default=None, - ) - group.add_argument( - "--app-asset-version", - type=str, - help="Override AssetBundle source version. EN/JP servers only.", - default=None, - ) - group.add_argument( - "--app-asset-hash", - type=str, - help="Override AssetBundle hash. EN/JP servers only.", - default=None, - ) - group = abcache_parser.add_argument_group("download options") - group.add_argument( - "--download-no-overwrite", - action="store_true", - help="skip downloading AssetBundles that are already present in the download directory", - ) - group.add_argument( - "--download-filter", - type=str, - help="filter AssetBundles (by bundle names) with this regex pattern", - default=None, - ) - group.add_argument( - "--download-filter-cache-diff", - type=str, - help="filter AssetBundles (by hashes) by *ONLY* downloading ones with a *different* hash from the current hash with this one.", - default=None, - **gooey_only(widget="FileChooser"), - ) - group.add_argument( - "--download-dir", - type=str, - help="asset bundle download directory. leave empty if you don't want to download anything", - default="", - **gooey_only(widget="DirChooser"), - ) - group.add_argument( - "--download-ensure-deps", - action="store_true", - help="ensure dependencies (of the downloaded ones) are downloaded as well", - ) - group.add_argument( - "--download-workers", - type=int, - help="number of download workers (default: %(default)s)", - default=4, - ) - group = abcache_parser.add_argument_group( - "extra options", - "NOTE: when *any* of these options are specified, the cache database *won't* be updated, and no download will be performed either.", - ) - group.add_argument( - "--dump-master-data", - type=str, - help="directory to store the dumped master data in JSON.", - default=None, - **gooey_only(widget="DirChooser"), - ) - group.add_argument( - "--dump-user-data", - type=str, - help="directory to store the dumped master data of the authencicated user in JSON. NOTE: --auth-credential required", - default=None, - **gooey_only(widget="DirChooser"), - ) - group.add_argument( - "--keep-compact", - help="keep compacted datasets as is without expanding them to key-value pairs", - action="store_true", - ) - group = abcache_parser.add_argument_group( - "authentication arguments", - "Only needed for some functionailties (i.e. --dump-user-data)", - ) - group.add_argument( - "--auth-credential", - type=str, - help="Credential for API authentication. Can be omitted on JP/EN servers, in which case a guest account would be automatically registered and used.", - default=None, - ) - abcache_parser.set_defaults(func=main_abcache) - abserve_parser = subparsers.add_parser( - "abserve", usage="""AbCache Filesystem Server / Backend""" - ) - abserve_parser.add_argument( - "--proxy", - type=str, - help="HTTP Proxy to use. This overrides the system proxy (environ HTTP_PROXY, HTTPS_PROXY) settings.", - default=None, - ) - abserve_parser.add_argument( - "--db", - type=str, - help="""cache database file path (default: %(default)s)""", - default="~/.sssekai/abcache.db", - **gooey_only(widget="FileChooser"), - ) - abserve_parser.add_argument( - "--fuse", - type=str, - help="""local mount point for AbCache. Required FUSE on host OS and fusepy (default: %(default)s)""", - default="", - **gooey_only(widget="DirChooser"), - ) - abserve_parser.add_argument( - "--host", - type=str, - help="""address of the interface to listen on (default: %(default)s). Use 0.0.0.0 to expose the server to the network.""", - default="127.0.0.1", - ) - abserve_parser.add_argument( - "--port", - type=int, - help="""port to listen on (default: %(default)s)""", - default="3939", - ) - abserve_parser.set_defaults(func=main_abserve) - # live2dextract - live2dextract_parser = subparsers.add_parser( - "live2dextract", usage="""Extract Sekai Live2D Models in a AssetBundle""" - ) - live2dextract_parser.add_argument( - "infile", type=str, help="input file", **gooey_only(widget="FileChooser") - ) - live2dextract_parser.add_argument( - "outdir", type=str, help="output directory", **gooey_only(widget="DirChooser") - ) - live2dextract_parser.add_argument( - "--no-anim", action="store_true", help="don't extract animation clips" - ) - live2dextract_parser.set_defaults(func=main_live2dextract) - # spineextract - spineextract_parser = subparsers.add_parser( - "spineextract", - usage="""Extract Sekai Spine (Esoteric Spine2D) Models in a AssetBundle""", - ) - spineextract_parser.add_argument( - "infile", type=str, help="input file", **gooey_only(widget="FileChooser") - ) - spineextract_parser.add_argument( - "outdir", type=str, help="output directory", **gooey_only(widget="DirChooser") - ) - spineextract_parser.set_defaults(func=main_spineextract) - # rla2json - rla2json_parser = subparsers.add_parser( - "rla2json", - usage="""Read streaming_live/archive files and dump their information to JSON or HCA packets""", - ) - rla2json_parser.add_argument( - "input", - type=str, - help="input archive file or directory containing loose packets", - **gooey_only(widget="FileChooser"), - ) - rla2json_parser.add_argument( - "outdir", type=str, help="output directory", **gooey_only(widget="DirChooser") - ) - rla2json_parser.add_argument( - "--version", - type=str, - help="RLA version. Only used for loose packets", - default="1.6", - choices=["%1s.%1s" % x for x in RLA_VERSIONS], - ) - rla2json_parser.add_argument( - "--strict", - action="store_true", - help="strict mode. raise exception on unknown frame type or corrupt frames", - ) - rla2json_parser.set_defaults(func=main_rla2json) - # apphash - apphash_parser = subparsers.add_parser( - "apphash", - usage="""Download/extract game AppHash values -*NOTE*: Only [app version] [app region] [app hash] are written to stdout if successful. Fields are separated by space. Unavailable fields are left as 'unknown'.""", - ) - apphash_parser.add_argument( - "--apk-src", - type=str, - help="APK source file (default: fetch from APKPure)", - default=None, - **gooey_only(widget="FileChooser"), - ) - apphash_parser.add_argument( - "--ab-src", - type=str, - help="AssetBundle (namely, 6350e2ec327334c8a9b7f494f344a761 or c726e51b6fe37463685916a1687158dd) source file (default: fetch from APK)", - default=None, - **gooey_only(widget="FileChooser"), - ) - apphash_parser.add_argument( - "--fetch", action="store_true", help="force fetching the latest APK" - ) - apphash_parser.add_argument( - "--format", - choices=["markdown", "json"], - help="output format", - default="markdown", - ) - apphash_parser.add_argument( - "--proxy", - type=str, - help="HTTP Proxy to use. This overrides the system proxy (environ HTTP_PROXY, HTTPS_PROXY) settings.", - default=None, - ) - apphash_parser.add_argument( - "--deep", - action="store_true", - help="Load all AssetBundles and retrieve all app hashes available.", - ) - apphash_parser.set_defaults(func=main_apphash) - # il2cpp - il2cpp_parser = subparsers.add_parser( - "il2cpp", - description="Extract IL2CPP inputs from JP/EN/CN/TW/KR Android releases; detect metadata format automatically.", - ) - il2cpp_parser.add_argument( - "metadata", - type=str, - help="the global-metadata.dat file at assets/bin/Data/Managed/Metadata", - **gooey_only(widget="FileChooser"), - ) - il2cpp_parser.add_argument( - "binary", - type=str, - help="libil2cpp.so at config.arm64_v8a/lib/arm64-v8a. currently this would be copied as is", - **gooey_only(widget="FileChooser"), - ) - il2cpp_parser.add_argument( - "outdir", - type=str, - help="output directory to dump the extracted metadata (as global-metadata.dat) and binary (as il2cpp.so)", - **gooey_only(widget="DirChooser"), - ) - il2cpp_parser.set_defaults(func=main_il2cpp) - il2cppextract_parser = subparsers.add_parser( - "il2cpp-extract", - help="Prepare IL2CPP analysis inputs from a JP/EN/CN/TW/KR Android ARM64 APK/XAPK", - description="Extract APK/XAPK inputs and run the existing il2cpp command.", - ) - il2cppextract_parser.add_argument( - "infile", help="input APK or XAPK (including ARM64 splits)", - **gooey_only(widget="FileChooser"), - ) - il2cppextract_parser.add_argument( - "outdir", help="output analysis directory", - **gooey_only(widget="DirChooser"), - ) - il2cppextract_parser.set_defaults(func=main_il2cppextract) - # mvdata - mvdata_parser = subparsers.add_parser( - "mvdata", usage="""Extract MV Data from AssetBundle""" - ) - mvdata_parser.add_argument( - "indir", - type=str, - help="cache directory (live_pv/mv_data)", - **gooey_only(widget="DirChooser"), - ) - mvdata_parser.add_argument( - "outfile", - type=str, - help="output JSON file to dump into", - **gooey_only(widget="FileSaver"), - ) - mvdata_parser.set_defaults(func=main_mvdata) - # moc3paths - moc3paths_parser = subparsers.add_parser( - "moc3paths", usage="""Extract animation path CRCs from raw .moc3 binaries""" - ) - moc3paths_parser.add_argument( - "indir", type=str, help="input directory", **gooey_only(widget="DirChooser") - ) - moc3paths_parser.set_defaults(func=main_moc3paths) - return parser - - -def __main__(): - from tqdm.std import tqdm as tqdm_c - - class TqdmMutexStream: - @staticmethod - def write(__s): - with tqdm_c.external_write_mode(file=sys.stderr, nolock=False): - return sys.stderr.write(__s) - - # parse args - parser = create_parser(argparse.ArgumentParser) - args = parser.parse_args() - # set logging level - import coloredlogs - - coloredlogs.install( - level=args.log_level, - fmt="%(asctime)s | %(levelname).1s | %(name)s %(message)s", - datefmt="%H:%M:%S", - isatty=True, - stream=TqdmMutexStream, - ) - # override unity version - sssekai_set_unity_version(args.unity_version) - if "func" in args: - try: - return args.func(args) - except Exception as e: - logger.exception("Error while running command: %s", e) - traceback.print_exc() - return 1 - else: - parser.print_help() - return 1 - - -if __name__ == "__main__": - sys.exit(__main__() or 0) +import argparse, sys, logging, traceback + +# region Deferred imports +# This can be done a lot cleaner with importlib. But tools like PyInstaller really +# won't like that. + +def main_apidecrypt(*args, **kwargs): + from sssekai.entrypoint.apidecrypt import main_apidecrypt + return main_apidecrypt(*args, **kwargs) + +def main_abdecrypt(*args, **kwargs): + from sssekai.entrypoint.abdecrypt import main_abdecrypt + return main_abdecrypt(*args, **kwargs) + +def main_rla2json(*args, **kwargs): + from sssekai.entrypoint.rla2json import main_rla2json + return main_rla2json(*args, **kwargs) + +def main_usmdemux(*args, **kwargs): + from sssekai.entrypoint.usmdemux import main_usmdemux + return main_usmdemux(*args, **kwargs) + +def main_abcache(*args, **kwargs): + from sssekai.entrypoint.abcache import main_abcache + return main_abcache(*args, **kwargs) + +def main_abserve(*args, **kwargs): + from sssekai.entrypoint.abserve import main_abserve + return main_abserve(*args, **kwargs) + +def main_live2dextract(*args, **kwargs): + from sssekai.entrypoint.live2dextract import main_live2dextract + return main_live2dextract(*args, **kwargs) + +def main_spineextract(*args, **kwargs): + from sssekai.entrypoint.spineextract import main_spineextract + return main_spineextract(*args, **kwargs) + +def main_apphash(*args, **kwargs): + from sssekai.entrypoint.apphash import main_apphash + return main_apphash(*args, **kwargs) + +def main_il2cpp(*args, **kwargs): + from sssekai.entrypoint.il2cpp import main_il2cpp + return main_il2cpp(*args, **kwargs) + +def main_il2cppextract(*args, **kwargs): + from sssekai.entrypoint.il2cppextract import main_il2cppextract + return main_il2cppextract(*args, **kwargs) + +def main_mvdata(*args, **kwargs): + from sssekai.entrypoint.mvdata import main_mvdata + return main_mvdata(*args, **kwargs) + +def main_moc3paths(*args, **kwargs): + from sssekai.entrypoint.moc3paths import main_moc3paths + return main_moc3paths(*args, **kwargs) + +# endregion + +from sssekai.unity import sssekai_get_unity_version, sssekai_set_unity_version +from sssekai.fmt.rla import RLA_VERSIONS + +logger = logging.getLogger(__name__) + + +def create_parser(clazz=argparse.ArgumentParser): + """ + Create the command line parser for the application. + + Args: + clazz (type): The parser class to use. Defaults to argparse.ArgumentParser. + In GUI mode this should be set to GooeyParser otherwise. + """ + + def gooey_only(**kwargs): + # Silent non-argparse kwargs + if clazz == argparse.ArgumentParser: + return {} + return kwargs + + parser = clazz( + description="""Project SEKAI Asset Utility / PJSK 资源工具""", + formatter_class=argparse.RawTextHelpFormatter, + ) + parser.add_argument( + "--log-level", + type=str, + help="logging level (default: %(default)s)", + default="INFO", + choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], + ) + parser.add_argument( + "--unity-version", + type=str, + help="""Unity version to use (default: %(default)s) +Prior to game version 3.6.0 (JP), this has always been 2020.3.21f1 +This has been changed to 2022.3.21f1 since, which would apply to all the assets from 3.6.0 onwards. +If you encounter any issues, try switching to the old version, or vice versa.""", + default=sssekai_get_unity_version(), + ) + subparsers = parser.add_subparsers( + title="subcommands", description="valid subcommands", help="additional help" + ) + # apidecrypt + apidecrypt_parser = subparsers.add_parser( + "apidecrypt", + usage="""API crypto dumper +This crypto applies to: + - API request/response body dumped by packet sniffer (mitmproxy, wireshark, etc.) + - AssetBundleInfo (see sssekai.abcache)""", + ) + apidecrypt_parser.add_argument( + "infile", type=str, help="input dump file", **gooey_only(widget="FileChooser") + ) + apidecrypt_parser.add_argument( + "outfile", type=str, help="output json file", **gooey_only(widget="FileSaver") + ) + apidecrypt_parser.add_argument( + "--region", + type=str, + help="app region", + default="jp", + choices=["jp", "tw", "en", "kr", "cn"], + ) + apidecrypt_parser.set_defaults(func=main_apidecrypt) + # abdecrypt + abdecrypt_parser = subparsers.add_parser( + "abdecrypt", usage="""Decrypt Sekai AssetBundle""" + ) + abdecrypt_parser.add_argument( + "indir", type=str, help="input directory", **gooey_only(widget="DirChooser") + ) + abdecrypt_parser.add_argument( + "outdir", type=str, help="output directory", **gooey_only(widget="DirChooser") + ) + abdecrypt_parser.set_defaults(func=main_abdecrypt) + # usmdemux + usmdemux_parser = subparsers.add_parser( + "usmdemux", usage="""Demux Sekai USM Video in a AssetBundle""" + ) + usmdemux_parser.add_argument( + "infile", type=str, help="input file", **gooey_only(widget="FileChooser") + ) + usmdemux_parser.add_argument( + "outfile", type=str, help="output file where the .USM file will be saved", **gooey_only(widget="FileSaver") + ) + usmdemux_parser.set_defaults(func=main_usmdemux) + # abcache + abcache_parser = subparsers.add_parser( + "abcache", usage="""Sekai AssetBundle Metadata Cache / Game API Helper""" + ) + abcache_parser.add_argument( + "--proxy", + type=str, + help="HTTP Proxy to use. This overrides the system proxy (environ HTTP_PROXY, HTTPS_PROXY) settings.", + default=None, + ) + group = abcache_parser.add_argument_group("save/load options") + group.add_argument( + "--db", + type=str, + help="""cache database file path (default: %(default)s)""", + default="~/.sssekai/abcache.db", + **gooey_only(widget="FileChooser"), + ) + group.add_argument( + "--no-update", + action="store_true", + help="skip all metadata updates and use cached ones as is. this supercedes 'game version options' and 'authentication options'", + ) + group.add_argument( + "--dump", + type=str, + help="dump the AssetBundle index to the specified JSON file and exit", + default=None, + **gooey_only(widget="FileSaver"), + ) + group = abcache_parser.add_argument_group( + "game version options", + "NOTE: these options are used to *update* the cache database. Use --no-update to skip updating. You can obtain these values with the `apphash` subcommand.", + ) + group.add_argument( + "--app-region", + type=str, + help="PJSK app region (default: %(default)s)", + choices=["jp", "tw", "en", "kr", "cn"], + default="jp", + ) + group.add_argument( + "--app-platform", + type=str, + help="PJSK app platform (default: %(default)s", + choices=["ios", "android"], + default="android", + ) # Let's hope this doesn't age well... + group.add_argument( + "--app-version", + type=str, + help="PJSK app version. For assets this is required unless --no-update is specified, or for EN/JP requisite --app-asset..fields are filled. For other functions this is required unless --no-update is specified", + default="", + ) + group.add_argument( + "--app-appHash", + type=str, + help="PJSK app hash. For assets this is required unless --no-update is specified, or for EN/JP requisite --app-asset..fields are filled. For other functions this is required unless --no-update is specified", + default="", + ) + group = abcache_parser.add_argument_group("ROW specific options") + group.add_argument( + "--app-abVersion", + type=str, + help="PJSK AssetBundle URL version. ROW servers only - This is required unless --no-update is specified", + default=None, + ) + group = abcache_parser.add_argument_group("EN/JP specific options") + group.add_argument( + "--app-asset-host", + type=str, + help="Override AssetBundle host URL. EN/JP servers only.", + default=None, + ) + group.add_argument( + "--app-asset-version", + type=str, + help="Override AssetBundle source version. EN/JP servers only.", + default=None, + ) + group.add_argument( + "--app-asset-hash", + type=str, + help="Override AssetBundle hash. EN/JP servers only.", + default=None, + ) + group = abcache_parser.add_argument_group("download options") + group.add_argument( + "--download-no-overwrite", + action="store_true", + help="skip downloading AssetBundles that are already present in the download directory", + ) + group.add_argument( + "--download-filter", + type=str, + help="filter AssetBundles (by bundle names) with this regex pattern", + default=None, + ) + group.add_argument( + "--download-filter-cache-diff", + type=str, + help="filter AssetBundles (by hashes) by *ONLY* downloading ones with a *different* hash from the current hash with this one.", + default=None, + **gooey_only(widget="FileChooser"), + ) + group.add_argument( + "--download-dir", + type=str, + help="asset bundle download directory. leave empty if you don't want to download anything", + default="", + **gooey_only(widget="DirChooser"), + ) + group.add_argument( + "--download-ensure-deps", + action="store_true", + help="ensure dependencies (of the downloaded ones) are downloaded as well", + ) + group.add_argument( + "--download-workers", + type=int, + help="number of download workers (default: %(default)s)", + default=4, + ) + group = abcache_parser.add_argument_group( + "extra options", + "NOTE: when *any* of these options are specified, the cache database *won't* be updated, and no download will be performed either.", + ) + group.add_argument( + "--dump-master-data", + type=str, + help="directory to store the dumped master data in JSON.", + default=None, + **gooey_only(widget="DirChooser"), + ) + group.add_argument( + "--dump-user-data", + type=str, + help="directory to store the dumped master data of the authencicated user in JSON. NOTE: --auth-credential required", + default=None, + **gooey_only(widget="DirChooser"), + ) + group.add_argument( + "--keep-compact", + help="keep compacted datasets as is without expanding them to key-value pairs", + action="store_true", + ) + group = abcache_parser.add_argument_group( + "authentication arguments", + "Only needed for some functionailties (i.e. --dump-user-data)", + ) + group.add_argument( + "--auth-credential", + type=str, + help="Credential for API authentication. Can be omitted on JP/EN servers, in which case a guest account would be automatically registered and used.", + default=None, + ) + abcache_parser.set_defaults(func=main_abcache) + abserve_parser = subparsers.add_parser( + "abserve", usage="""AbCache Filesystem Server / Backend""" + ) + abserve_parser.add_argument( + "--proxy", + type=str, + help="HTTP Proxy to use. This overrides the system proxy (environ HTTP_PROXY, HTTPS_PROXY) settings.", + default=None, + ) + abserve_parser.add_argument( + "--db", + type=str, + help="""cache database file path (default: %(default)s)""", + default="~/.sssekai/abcache.db", + **gooey_only(widget="FileChooser"), + ) + abserve_parser.add_argument( + "--fuse", + type=str, + help="""local mount point for AbCache. Required FUSE on host OS and fusepy (default: %(default)s)""", + default="", + **gooey_only(widget="DirChooser"), + ) + abserve_parser.add_argument( + "--host", + type=str, + help="""address of the interface to listen on (default: %(default)s). Use 0.0.0.0 to expose the server to the network.""", + default="127.0.0.1", + ) + abserve_parser.add_argument( + "--port", + type=int, + help="""port to listen on (default: %(default)s)""", + default="3939", + ) + abserve_parser.set_defaults(func=main_abserve) + # live2dextract + live2dextract_parser = subparsers.add_parser( + "live2dextract", usage="""Extract Sekai Live2D Models in a AssetBundle""" + ) + live2dextract_parser.add_argument( + "infile", type=str, help="input file", **gooey_only(widget="FileChooser") + ) + live2dextract_parser.add_argument( + "outdir", type=str, help="output directory", **gooey_only(widget="DirChooser") + ) + live2dextract_parser.add_argument( + "--no-anim", action="store_true", help="don't extract animation clips" + ) + live2dextract_parser.set_defaults(func=main_live2dextract) + # spineextract + spineextract_parser = subparsers.add_parser( + "spineextract", + usage="""Extract Sekai Spine (Esoteric Spine2D) Models in a AssetBundle""", + ) + spineextract_parser.add_argument( + "infile", type=str, help="input file", **gooey_only(widget="FileChooser") + ) + spineextract_parser.add_argument( + "outdir", type=str, help="output directory", **gooey_only(widget="DirChooser") + ) + spineextract_parser.set_defaults(func=main_spineextract) + # rla2json + rla2json_parser = subparsers.add_parser( + "rla2json", + usage="""Read streaming_live/archive files and dump their information to JSON or HCA packets""", + ) + rla2json_parser.add_argument( + "input", + type=str, + help="input archive file or directory containing loose packets", + **gooey_only(widget="FileChooser"), + ) + rla2json_parser.add_argument( + "outdir", type=str, help="output directory", **gooey_only(widget="DirChooser") + ) + rla2json_parser.add_argument( + "--version", + type=str, + help="RLA version. Only used for loose packets", + default="1.6", + choices=["%1s.%1s" % x for x in RLA_VERSIONS], + ) + rla2json_parser.add_argument( + "--strict", + action="store_true", + help="strict mode. raise exception on unknown frame type or corrupt frames", + ) + rla2json_parser.set_defaults(func=main_rla2json) + # apphash + apphash_parser = subparsers.add_parser( + "apphash", + usage="""Download/extract game AppHash values +*NOTE*: Only [app version] [app region] [app hash] are written to stdout if successful. Fields are separated by space. Unavailable fields are left as 'unknown'.""", + ) + apphash_parser.add_argument( + "--apk-src", + type=str, + help="APK source file (default: fetch from APKPure)", + default=None, + **gooey_only(widget="FileChooser"), + ) + apphash_parser.add_argument( + "--ab-src", + type=str, + help="AssetBundle (namely, 6350e2ec327334c8a9b7f494f344a761 or c726e51b6fe37463685916a1687158dd) source file (default: fetch from APK)", + default=None, + **gooey_only(widget="FileChooser"), + ) + apphash_parser.add_argument( + "--fetch", action="store_true", help="force fetching the latest APK" + ) + apphash_parser.add_argument( + "--format", + choices=["markdown", "json"], + help="output format", + default="markdown", + ) + apphash_parser.add_argument( + "--proxy", + type=str, + help="HTTP Proxy to use. This overrides the system proxy (environ HTTP_PROXY, HTTPS_PROXY) settings.", + default=None, + ) + apphash_parser.add_argument( + "--deep", + action="store_true", + help="Load all AssetBundles and retrieve all app hashes available.", + ) + apphash_parser.set_defaults(func=main_apphash) + # il2cpp + il2cpp_parser = subparsers.add_parser( + "il2cpp", + description="Extract IL2CPP inputs from JP/EN/CN/TW/KR Android releases; detect metadata format automatically.", + ) + il2cpp_parser.add_argument( + "metadata", + type=str, + help="the global-metadata.dat file at assets/bin/Data/Managed/Metadata", + **gooey_only(widget="FileChooser"), + ) + il2cpp_parser.add_argument( + "binary", + type=str, + help="libil2cpp.so at config.arm64_v8a/lib/arm64-v8a. currently this would be copied as is", + **gooey_only(widget="FileChooser"), + ) + il2cpp_parser.add_argument( + "outdir", + type=str, + help="output directory to dump the extracted metadata (as global-metadata.dat) and binary (as il2cpp.so)", + **gooey_only(widget="DirChooser"), + ) + il2cpp_parser.set_defaults(func=main_il2cpp) + il2cppextract_parser = subparsers.add_parser( + "il2cpp-extract", + help="Prepare IL2CPP analysis inputs from a JP/EN/CN/TW/KR Android ARM64 APK/XAPK", + description="Extract APK/XAPK inputs and run the existing il2cpp command.", + ) + il2cppextract_parser.add_argument( + "infile", help="input APK or XAPK (including ARM64 splits)", + **gooey_only(widget="FileChooser"), + ) + il2cppextract_parser.add_argument( + "outdir", help="output analysis directory", + **gooey_only(widget="DirChooser"), + ) + il2cppextract_parser.set_defaults(func=main_il2cppextract) + # mvdata + mvdata_parser = subparsers.add_parser( + "mvdata", usage="""Extract MV Data from AssetBundle""" + ) + mvdata_parser.add_argument( + "indir", + type=str, + help="cache directory (live_pv/mv_data)", + **gooey_only(widget="DirChooser"), + ) + mvdata_parser.add_argument( + "outfile", + type=str, + help="output JSON file to dump into", + **gooey_only(widget="FileSaver"), + ) + mvdata_parser.set_defaults(func=main_mvdata) + # moc3paths + moc3paths_parser = subparsers.add_parser( + "moc3paths", usage="""Extract animation path CRCs from raw .moc3 binaries""" + ) + moc3paths_parser.add_argument( + "indir", type=str, help="input directory", **gooey_only(widget="DirChooser") + ) + moc3paths_parser.set_defaults(func=main_moc3paths) + return parser + + +def __main__(): + from tqdm.std import tqdm as tqdm_c + + class TqdmMutexStream: + @staticmethod + def write(__s): + with tqdm_c.external_write_mode(file=sys.stderr, nolock=False): + return sys.stderr.write(__s) + + # parse args + parser = create_parser(argparse.ArgumentParser) + args = parser.parse_args() + # set logging level + import coloredlogs + + coloredlogs.install( + level=args.log_level, + fmt="%(asctime)s | %(levelname).1s | %(name)s %(message)s", + datefmt="%H:%M:%S", + isatty=True, + stream=TqdmMutexStream, + ) + # override unity version + sssekai_set_unity_version(args.unity_version) + if "func" in args: + try: + return args.func(args) + except Exception as e: + logger.exception("Error while running command: %s", e) + traceback.print_exc() + return 1 + else: + parser.print_help() + return 1 + + +if __name__ == "__main__": + sys.exit(__main__() or 0) diff --git a/sssekai/abcache/__init__.py b/sssekai/abcache/__init__.py index 51f39f7..6173510 100644 --- a/sssekai/abcache/__init__.py +++ b/sssekai/abcache/__init__.py @@ -1,875 +1,875 @@ -from collections import defaultdict -from pickle import load, dump -from typing import BinaryIO, List, Mapping, Optional, Union, Tuple -from logging import getLogger -from dataclasses import dataclass, fields, is_dataclass, field -from functools import cached_property -from base64 import b64encode, b64decode -import json - -logger = getLogger("sssekai.abcache") - -REGION_JP_EN = {"jp", "en"} -REGION_ROW = {"tw", "kr", "cn"} -REGION_OPTIONS = REGION_JP_EN | REGION_ROW - - -class RedactedStr(str): - """A credential that behaves like a str but hides its value in repr. - - `dataclasses.asdict` deep-copies values, so `field(repr=False)` alone does - not keep tokens out of `repr(asdict(...))`. Pickles as a plain str so cache - files stay compatible with older versions. - """ - - def __repr__(self): - return "" - - def __deepcopy__(self, memo): - return self - - def __reduce_ex__(self, protocol): - return str, (str(self),) - - -def fromdict(klass: type, d: Union[Mapping, List], warn_missing_fields=True): - """ - Recursively convert a dictionary to a dataclass instance, if applicable. - https://stackoverflow.com/a/54769644 - https://gist.github.com/gatopeich/1efd3e1e4269e1e98fae9983bb914f22 - """ - - while klass.__name__ == "Optional": - klass = klass.__args__[0] # Reduce Optional[T] -> T - - def ensure_iterable(d): - if isinstance(d, Mapping): - return d - if hasattr(d, "__dict__"): - return d.__dict__ - return dict() - - def check_field(key, fields): - if not key in fields: - if warn_missing_fields: - logger.error(f"Field {key} of type {type(d[key])} not found in {klass}") - return False - return True - - if is_dataclass(klass): - fieldtypes = {f.name: f.type for f in fields(klass)} - d = ensure_iterable(d) - return klass( - **( - { - f: fromdict(fieldtypes[f], d[f], warn_missing_fields) - for f in d - if check_field(f, fieldtypes) - } - ) - ) - if isinstance(d, list) and hasattr(klass, "__args__"): - return [fromdict(klass.__args__[0], di, warn_missing_fields) for di in d] - if isinstance(d, dict) and hasattr(klass, "__args__"): - return { - k: fromdict(klass.__args__[1], v, warn_missing_fields) for k, v in d.items() - } - return d - - -from requests import Session, Response, HTTPError -from msgpack import unpackb, packb - -from sssekai import __version__, __version_tuple__ -from sssekai.unity import sssekai_get_unity_version -from sssekai.crypto.APIManager import decrypt, encrypt, SEKAI_APIMANAGER_KEYSETS - - -@dataclass -class AbCacheConfig: - app_region: str - app_version: str - app_platform: str - app_hash: str - - auth_credential: str = field( - default=None, repr=False - ) # JWT token for JP/EN, Base64 encoded JWT token for ROW - - row_ab_version: str = None # Override AB version in url for ROW - - asset_host: str = None # Override AssetBundle host for EN/JP - asset_version: str = None # Override AssetBundle version for EN/JP - asset_hash: str = None # Override AssetBundle hash for EN/JP - - version: Tuple[int, int, int] = (0, 0, 0) # Internal versioning - - def __setattr__(self, name, value): - # Credentials are also assigned after construction (CLI, auth.py). - if name == "auth_credential" and value is not None: - if not isinstance(value, RedactedStr): - value = RedactedStr(value) - super().__setattr__(name, value) - - @property - def auth_jwt(self) -> str: - if self.app_region in REGION_JP_EN: - return self.auth_credential - else: - try: - return b64decode(self.auth_credential).decode() - except: - return None - - @property - def auth_jwt_payload(self) -> dict: - if not self.auth_jwt: - return {} - header, payload, signature = self.auth_jwt.split(".") - return json.loads(b64decode(payload + "==")) - - @property - def auth_userID(self) -> str: - return self.auth_jwt_payload.get("userId", None) - - @property - def auth_available(self): - return self.auth_userID is not None - - @property - def cache_version(self): - return getattr(self, "version", (0, 0, 0)) - - @property - def cache_version_string(self): - return "%s.%s.%s" % self.cache_version if self.cache_version else "pre-0.5.17" - - @property - def is_up_to_date(self): - return self.cache_version == __version_tuple__ - - @property - def need_client_header_update(self): - if self.app_region in REGION_JP_EN: - if self.app_region in {"jp"}: - return not (self.asset_hash and self.asset_host and self.asset_version) - else: - return not (self.asset_hash and self.asset_version) - return False - - -@dataclass -class AbCacheEntry(dict): - bundleName: str - cacheFileName: str - cacheDirectoryName: str - hash: str - category: str - crc: int - fileSize: int - dependencies: List[str] - isBuiltin: bool - isRelocate: Optional[bool] = None # Added in JP 5.4.0 - paths: Optional[List[str]] = None # Removed in ROW - # ROW only - md5Hash: Optional[str] = None - downloadPath: Optional[str] = None - - -@dataclass -class AbCacheIndex(dict): - version: str - os: Optional[str] = None # Undefined in ROW - bundles: Mapping[str, AbCacheEntry] = None - - -# DS for the game's APIs -# There're *plenty* differences across regions. The implementations here -# - Contains all the fields, even if they're not used in all regions -# - Only the fields that are common across regions are non Optional[T] -@dataclass -class SekaiAppVersion: - systemProfile: str - appVersion: str - multiPlayVersion: str - appVersionStatus: str - assetVersion: str - # These are sadly removed from 3.5.0 (JP) - dataVersion: Optional[str] = None - appHash: Optional[str] = None - assetHash: Optional[str] = None - - -@dataclass -class SekaiSystemData: - serverDate: int - # These are all undefined in ROW - timezone: Optional[str] = None - profile: Optional[str] = None - maintenanceStatus: Optional[str] = None - appVersions: Optional[List[SekaiAppVersion]] = None - - @cached_property - def app_version_dict(self): - return {av.appVersion: av for av in self.appVersions or []} - - -@dataclass -class SekaiGameVersionData: - profile: str - assetbundleHostHash: str - domain: str - - -@dataclass -class SekaiUserRegistrationData: - userId: int - # JP/EN only - signature: Optional[str] = None - platform: Optional[str] = None - deviceModel: Optional[str] = None - operatingSystem: Optional[str] = None - registeredAt: Optional[int] = None - - -@dataclass -class SekaiUserData: - userRegistration: SekaiUserRegistrationData - # JP/EN Only - credential: Optional[str] = field(default=None, repr=False) - updatedResources: Optional[dict] = None - # ROW - sessionToken: Optional[str] = field(default=None, repr=False) - - def __setattr__(self, name, value): - if name in ("credential", "sessionToken") and value is not None: - if not isinstance(value, RedactedStr): - value = RedactedStr(value) - super().__setattr__(name, value) - - @property - def user_credentials(self): - return self.credential or self.sessionToken - - -@dataclass -class SekaiUserAuthData: - sessionToken: str = field(repr=False) - appVersion: str - multiPlayVersion: str - dataVersion: str - assetVersion: str - - # JP/EN only - removeAssetVersion: Optional[str] = None - assetHash: Optional[str] = None - - appVersionStatus: Optional[str] = None - - # JP/EN only - isStreamingVirtualLiveForceOpenUser: Optional[bool] = None - deviceId: Optional[str] = None - updatedResources: Optional[dict] = None - suiteMasterSplitPath: Optional[list] = None - obtainedBondsRewardIds: Optional[list] = None # JP 4.0+ - - # ROW - cdnVersion: Optional[int] = None - configs: Optional[List[dict]] = None - - def __setattr__(self, name, value): - if name == "sessionToken" and value is not None: - if not isinstance(value, RedactedStr): - value = RedactedStr(value) - super().__setattr__(name, value) - - -class RedactedHeaders(dict): - """Request headers kept for cache reuse. Hidden from repr/asdict so tokens - (e.g. X-Session-Token) never reach logs. Pickles as a plain dict so cache - files stay loadable by both old and new versions.""" - - def __repr__(self): - return "" % len(self) - - def __deepcopy__(self, memo): - # dict's default deepcopy returns a plain dict, which would leak through - # dataclasses.asdict(). Header values are str, so a shallow copy suffices. - return RedactedHeaders(self) - - def __reduce_ex__(self, protocol): - return dict, (dict(self),) - - -@dataclass -class SSSekaiDatabase: - config: AbCacheConfig = None - cached_headers: dict = field(default_factory=RedactedHeaders) - - sekai_user_data: SekaiUserData = None - sekai_user_auth_data: SekaiUserAuthData = None - sekai_abcache_index: AbCacheIndex = None - sekai_system_data: SekaiSystemData = None - sekai_gameversion_data: SekaiGameVersionData = None - - def __repr__(self) -> str: - # Two layers guard this: RedactedHeaders hides its own values, and this - # repr also covers a plain dict (databases built by hand, or caches not - # yet normalized by load()) that would otherwise print in full. - values = ", ".join( - "%s=%r" % (f.name, getattr(self, f.name, None)) - for f in fields(self) - if f.name != "cached_headers" - ) - return "SSSekaiDatabase(%s, cached_headers=<%d values>)" % ( - values, - len(self.cached_headers or ()), - ) - - def reset(self): - self.sekai_user_data = None - self.sekai_user_auth_data = None - self.sekai_abcache_index = None - self.sekai_system_data = None - self.sekai_gameversion_data = None - - def redact_secrets(self): - """Re-wrap secrets loaded from disk as RedactedStr. - - RedactedStr pickles as a plain str (so caches stay readable by older - versions), so values read back from a cache file must be wrapped again - before they can reach a repr/asdict. Assignment goes through each - dataclass's __setattr__, which does the wrapping. - """ - if not isinstance(self.cached_headers, RedactedHeaders): - self.cached_headers = RedactedHeaders(self.cached_headers or {}) - # sssekai <= 0.8.2 stored the raw API response here, so old caches may - # hold a plain dict (which also breaks attribute access such as - # SEKAI_CREDENTIAL). Normalize it to the dataclass we expect. - if isinstance(self.sekai_user_data, dict): - self.sekai_user_data = fromdict( - SekaiUserData, self.sekai_user_data, False - ) - secrets = [] - if self.config is not None: - secrets.append((self.config, ("auth_credential",))) - secrets += [ - (self.sekai_user_data, ("credential", "sessionToken")), - (self.sekai_user_auth_data, ("sessionToken",)), - ] - for owner, names in secrets: - if owner is None: - continue - for name in names: - value = getattr(owner, name, None) - if value is not None: - setattr(owner, name, value) - - -class AbCacheBundleNotFoundError(Exception): - bundleName: str - - def __init__(self, bundleName: str): - self.bundleName = bundleName - super().__init__("Bundle not found: %s" % bundleName) - - -class AbCacheUserNotAuthenticatedError(Exception): - pass - - -class AbCache(Session): - database: SSSekaiDatabase - - @property - def SEKAI_API_ENDPOINT(self): - match self.config.app_region: - case "jp": - if self.database.sekai_gameversion_data: - url = self.database.sekai_gameversion_data.domain - url = url.replace( - "{0}", self.database.sekai_gameversion_data.profile - ) - return "https://%s" % url - return "https://production-game-api.sekai.colorfulpalette.org" - case "en": - return f"https://n-{self.SEKAI_API_PROFILE}-game-api.sekai-en.com" - case "tw": - return "https://mkovs-20004-cdn.bytedgame.com" - case "kr": - return "https://mkkorea-obt-prod01-cdn.bytedgame.com" - case "cn": - return "https://mkcn-prod-public-60001-1.dailygn.com" - case _: - raise NotImplementedError - - @property - def SEKAI_API_GAMEVERSION_ENDPOINT(self): - match self.config.app_region: - case "jp": - return "https://game-version.sekai.colorfulpalette.org" - case "en": - return "https://game-version.sekai-en.com" - case _: - raise NotImplementedError - - @property - def SEKAI_ISSUE_ENDPOINT(self): - match self.config.app_region: - case "jp": - return "https://issue.sekai.colorfulpalette.org" - case _: - raise NotImplementedError - - @property - def SEKAI_AB_ROW_CDN(self): - match self.config.app_region: - case "tw": - return "https://lf16-mkovscdn-sg.bytedgame.com/obj/sf-game-alisg/gdl_app_5245" - case "kr": - return ( - "https://lf19-mkkr.bytedgame.com/obj/sf-game-alisg/gdl_app_292248" - ) - case "cn": - return "https://lf3-mkcncdn-tos.dailygn.com/obj/sf-game-lf/gdl_app_5236" - case _: - raise NotImplementedError - - @property - def SEKAI_AB_ROW_PATH(self): - match self.config.app_region: - case "tw": - return "online" - case "kr": - return "kr_online" - case "cn": - return "cn_online" - case _: - raise NotImplementedError - - @cached_property - def SEKAI_AB_ROW_VERSION_NUMBER(self): - url = f"{self.SEKAI_AB_ROW_CDN.replace('sf-game','rt-game')}" - if self.config.app_region in {"cn"}: - url = f"{url}/Mainland" - else: - url = f"{url}/Oversea" - url = f"{url}/{self.SEKAI_AB_ROW_VERSION}/Release/{self.SEKAI_AB_ROW_PATH}/{self.SEKAI_APP_PLATFORM}/version" - resp = self.request("GET", url) - resp.raise_for_status() - return int(resp.text) - - @property - def SEKAI_AB_INFO_ENDPOINT(self): - match self.config.app_region: - case "jp": - return f"https://{self.SEKAI_API_PROFILE}-{self.SEKAI_AB_HOST_HASH}-assetbundle-info.sekai.colorfulpalette.org/api/version/{self.SEKAI_ASSET_VERSION}/{self.SEKAI_AB_HASH}/os/{self.config.app_platform}" - case "en": - # `n` for what, exactly? - return f"https://n-{self.SEKAI_API_PROFILE}-{self.SEKAI_AB_HOST_HASH}-assetbundle-info.sekai-en.com/api/version/{self.SEKAI_ASSET_VERSION}/os/{self.config.app_platform}" - if self.config.app_region in REGION_ROW: - return f"{self.SEKAI_AB_ROW_CDN}/AssetBundle/{self.SEKAI_AB_ROW_VERSION}/Release/{self.SEKAI_AB_ROW_PATH}/{self.SEKAI_APP_PLATFORM}{self.SEKAI_AB_ROW_VERSION_NUMBER}/AssetBundleInfoNew.json" - else: - raise NotImplementedError - - @property - def SEKAI_AB_ENDPOINT(self): - match self.config.app_region: - case "jp": - return f"https://{self.SEKAI_API_PROFILE}-{self.SEKAI_AB_HOST_HASH}-assetbundle.sekai.colorfulpalette.org/" - case "en": - return f"https://n-{self.SEKAI_API_PROFILE}-{self.SEKAI_AB_HOST_HASH}-assetbundle.sekai-en.com/" - if self.config.app_region in REGION_ROW: - return f"{self.SEKAI_AB_ROW_CDN}/AssetBundle/{self.SEKAI_AB_ROW_VERSION}/Release/{self.SEKAI_AB_ROW_PATH}/" - else: - raise NotImplementedError - - def _update_request_headers(self): - app_platform_names = { - 'ios':'iOS', 'android':'Android' - } - self.headers.update( - { - "Accept": "application/octet-stream", - "Content-Type": "application/octet-stream", - "Accept-Encoding": "deflate, gzip", - "User-Agent": "UnityPlayer/%s" % sssekai_get_unity_version(), - "X-Platform": app_platform_names.get(self.config.app_platform,self.config.app_platform), - "X-DeviceModel": "sssekai/%s" % __version__, - "X-OperatingSystem": app_platform_names.get(self.config.app_platform,self.config.app_platform), - "X-Unity-Version": sssekai_get_unity_version(), - "X-App-Version": self.SEKAI_APP_VERSION, - "X-App-Hash": self.SEKAI_APP_HASH, - } - ) - - @property - def config(self): - return self.database.config - - @config.setter - def config(self, v): - self.database.config = v - - @property - def SEKAI_APP_VERSION(self): - return self.config.app_version - - @property - def SEKAI_AB_ROW_VERSION(self): - return self.config.row_ab_version or self.SEKAI_APP_VERSION - - @property - def SEKAI_APP_PLATFORM(self): - return self.config.app_platform - - @property - def SEKAI_APP_HASH(self): - return self.config.app_hash - - @property - def SEKAI_ASSET_VERSION(self): - return ( - self.config.asset_version - or self.database.sekai_system_data.app_version_dict[ - self.SEKAI_APP_VERSION - ].assetVersion - ) - - @property - def SEKAI_AB_HASH(self): - return self.config.asset_hash or self.database.sekai_user_auth_data.assetHash - - @property - def SEKAI_AB_HOST_HASH(self): - return ( - self.config.asset_host - or self.database.sekai_gameversion_data.assetbundleHostHash - ) - - @property - def SEKAI_API_PROFILE(self): - if self.database.sekai_gameversion_data: - return self.database.sekai_gameversion_data.profile - return "production" - - @property - def SEKAI_API_SYSTEM_DATA(self): - return self.SEKAI_API_ENDPOINT + "/api/system" - - @property - def SEKAI_API_USER(self): - return self.SEKAI_API_ENDPOINT + "/api/user" - - @property - def SEKAI_USERID(self): - return self.config.auth_userID or ( - self.database.sekai_user_data.userRegistration.userId - if self.database.sekai_user_data - else None - ) - - @property - def SEKAI_CREDENTIAL(self): - """Keyed in SEKAI_API_USER responses. `accessToken` in ROW servers, `credential` in JP/EN.""" - return self.config.auth_credential or ( - self.database.sekai_user_data.credential - if self.database.sekai_user_data - else None - ) - - @property - def is_authenticated(self): - return self.database.sekai_user_auth_data is not None - - def raise_for_auth(self, *args): - if not self.is_authenticated: - raise AbCacheUserNotAuthenticatedError(*args) - - @property - def SEKAI_API_USER_AUTH(self): - if self.config.app_region in REGION_JP_EN: - assert self.SEKAI_USERID, "User ID must be available" - return f"{self.SEKAI_API_USER}/{self.SEKAI_USERID}/auth" - else: - # For ROW this is the endpoint that retrives SEKAI_API_USER level data as well - return f"{self.SEKAI_API_USER}/auth" - - @property - def SEKAI_API_USER_SUITE(self): - self.raise_for_auth() - return f"{self.SEKAI_API_ENDPOINT}/api/suite/user/{self.SEKAI_USERID}" - - @property - def SEKAI_API_MASTER_SUITE_URLS(self) -> List[str]: - match self.config.app_region: - case "jp": - self.raise_for_auth() - return [ - f"{self.SEKAI_API_ENDPOINT}/api/{path}" - for path in self.database.sekai_user_auth_data.suiteMasterSplitPath - ] - case "en": - self.raise_for_auth() - return [ - f"{self.SEKAI_API_ENDPOINT}/api/{path}" - for path in self.database.sekai_user_auth_data.suiteMasterSplitPath - ] - if self.config.app_region in REGION_ROW: - self.raise_for_auth() - return [f"{self.SEKAI_AB_ROW_CDN}/{self.SEKAI_AB_ROW_PATH}/master-data-{self.database.sekai_user_auth_data.cdnVersion}.info"] - else: - raise NotImplementedError - - @property - def SEKAI_API_INFORMATION(self): - return self.SEKAI_API_ENDPOINT + "/api/information" - - @property - def SEKAI_AB_BASE_PATH(self): - return f"{self.SEKAI_ASSET_VERSION}/{self.SEKAI_AB_HASH}/{self.config.app_platform}/" - - @property - def SEKAI_ISSUE_SIGNATURE_ENDPOINT(self): - return self.SEKAI_ISSUE_ENDPOINT + "/api/signature" - - @property - def abcache_index(self): - return self.database.sekai_abcache_index - - def request_packed(self, method: str, url: str, data: dict = None, **kwargs): - """Send a request with packed data. Data will be packed and encrypted before sending. - - Args: - method (str): HTTP method - url (str): URL - data (dict, optional): Payload data. Defaults to None. - - Returns: - Response: Response object - """ - self._update_request_headers() - if data is not None: - # XXX: packb() does not cover all cases. - # - # Python msgpack implementation doesn't really support variadic float precision within a single pack. - # This generally shouldn't be an issue if the upstream unpacking routine is type-agnostic since - # the type is described in the packed binary as per the spec. - # - # However, if the upstream implementation is not type-agnostic, it WILL lead to issues. - # While most respect the type opcodes (e.g. msgpack-c, msgpack-python, msgpack in Golang), some implementations (e.g. MessagePack-CSharp) disregards - # them altogether when concrete (C#) types are provided. And will raise an exception in such cases when a double is encoded in the place of a float. - # See also: - # https://github.com/mos9527/sssekai/issues/47#issuecomment-3150827990 - # - # Thus `use_single_float=True` is used to cover and only cover single precision cases. - # In the *current* API seen in the game binary, this is fine. And in AbCache's case floats are not at all used. - # But this may change in the future, in which case consult the following issues: - # https://github.com/msgpack/msgpack-python/issues/326 - data = packb(data, use_single_float=True) - data = encrypt(data, SEKAI_APIMANAGER_KEYSETS[self.config.app_region]) - resp = self.request(method=method, url=url, data=data, **kwargs) - if 400 <= resp.status_code < 600: - logger.error(f"{method} {url} {resp.status_code}") - try: # log the error message provided by the API. - content = self.response_to_dict(resp) - logger.error("response=%s" % content) - except: - logger.error("response=%s" % resp.content) - logger.error( - "Unexpected server-side error. Report issues at https://github.com/MetaSekaiLab/sssekai/issues." - ) - resp.raise_for_status() - return resp - - def response_to_dict(self, resp: Response, **kwargs): - """Decrypt and unpack a response content to a dictionary. - - Args: - resp (Response): Response object - **kwargs: Additional arguments for MessagePack unpackb - - Returns: - dict: Decrypted and unpacked data - """ - data = decrypt(resp.content, SEKAI_APIMANAGER_KEYSETS[self.config.app_region]) - data = unpackb(data, **kwargs) - return data - - def _update_user_auth_data(self): - if self.config.auth_available: - logger.debug("Updating user auth data") - if self.config.app_region in REGION_JP_EN: - resp = self.request_packed( - "PUT", - self.SEKAI_API_USER_AUTH, - data={"credential": self.SEKAI_CREDENTIAL, "deviceId": None}, - ) - else: - resp = self.request_packed( - "POST", - self.SEKAI_API_USER_AUTH, - data={ - "accessToken": self.SEKAI_CREDENTIAL, - "deviceId": None, - "userID": 0, - }, - ) - data = self.response_to_dict(resp) - self.database.sekai_user_auth_data = fromdict( - SekaiUserAuthData, data, False - ) - if self.config.app_region in REGION_ROW: - self.database.sekai_user_data = fromdict(SekaiUserData, data, False) - else: - logger.warning( - "No valid auth credential provided. User auth data will not be updated." - ) - - def _update_system_data(self): - logger.debug("Updating system data") - resp = self.request_packed("GET", self.SEKAI_API_SYSTEM_DATA) - data = self.response_to_dict(resp) - self.database.sekai_system_data = fromdict(SekaiSystemData, data) - - def _update_gameversion_data(self): - if self.config.app_region in REGION_JP_EN: - logger.debug("Updating game version data") - resp = self.request_packed( - "GET", - self.SEKAI_API_GAMEVERSION_ENDPOINT - + "/" - + self.SEKAI_APP_VERSION - + "/" - + self.SEKAI_APP_HASH, - ) - data = self.response_to_dict(resp) - self.database.sekai_gameversion_data = fromdict(SekaiGameVersionData, data) - - def __init__(self, config: Optional[AbCacheConfig] = None): - super().__init__() - self.database = SSSekaiDatabase() - self.config = config or AbCacheConfig( - "unknown", "unknown", "unknown", "unknown" - ) - self.config.version = __version_tuple__ - - def update_client_headers(self): - """Authenticate the user and update client headers.""" - logger.debug("Updating metadata") - logger.debug("Set config: %s" % self.config) - try: - self.update_signatures() - self._update_gameversion_data() - self._update_system_data() - except HTTPError as e: - logger.error("Signature update failure: %s" % e) - raise e - if self.config.app_region in REGION_JP_EN: - version_newest = self.database.sekai_system_data.appVersions[-1] - logger.debug("Newest App version: %s" % version_newest) - if version_newest.appVersion != self.SEKAI_APP_VERSION: - logger.warning("App version mismatch. This may cause issues.") - self._update_user_auth_data() - if self.is_authenticated: - self.headers.update( - { - "X-Data-Version": self.database.sekai_user_auth_data.dataVersion, - "X-Asset-Version": self.database.sekai_user_auth_data.assetVersion, - "X-Session-Token": self.database.sekai_user_auth_data.sessionToken, - } - ) - return self.headers - - def update_abcache_index(self) -> AbCacheIndex: - """Update the AssetBundle Index.""" - logger.debug("Updating Assetbundle index") - if self.config.app_region in REGION_JP_EN: - logger.info("Sekai AssetBundle version: %s" % self.SEKAI_ASSET_VERSION) - if self.config.app_region in {"jp"}: - logger.info("Sekai AssetBundle host hash: %s" % self.SEKAI_AB_HOST_HASH) - resp = self.request_packed("GET", self.SEKAI_AB_INFO_ENDPOINT) - data = self.response_to_dict(resp) - self.database.sekai_abcache_index = fromdict(AbCacheIndex, data) - return self.database.sekai_abcache_index - - def update_signatures(self): - """Update client signatures. Only required for JP for the initial setup. - After which the game would cache the signature/cookies.""" - if self.config.app_region in {"jp"}: - logger.debug("Updating signatures") - # XXX: Clear all cookies, it 403s otherwise. Does this trigger a rate limit/expiry check since our session would be tracked? - self.headers["Cookie"] = "" - self.cookies.clear() - resp = self.request_packed("POST", self.SEKAI_ISSUE_SIGNATURE_ENDPOINT) - self.headers["Cookie"] = resp.headers["Set-Cookie"] - # HACK: Per RFC6265, Cookies should not be visible to subdomains since it's not set with Domain attribute (https://github.com/psf/requests/issues/2576) - # But the other endpoints uses it nontheless. So we have to set it manually. - - def update(self): - """Update the cache with the latest AssetBundle Index data from the server. - - NOTE: - AbCache should have been initialized with a valid config before calling this method. - Please refer to `update_client_headers` for more information. - """ - self.update_client_headers() - self.update_abcache_index() - - def save(self, f: BinaryIO): - self._update_request_headers() - self.database.cached_headers = RedactedHeaders(self.headers) - self.database.config.version = __version_tuple__ - logger.info("Saving cache: %s" % self) - dump(self.database, f) - - def load(self, f: BinaryIO): - logger.info("Loading cache") - self.database = load(f) - # Values from disk are plain str/dict; re-wrap so they stay out of logs. - self.database.redact_secrets() - logger.info("Loaded cache: %s" % self) - if not self.database.config.is_up_to_date: - logger.warning( - "Cache is outdated (cache: %s, current: %s). It's HIGHLY recommended to update the cache to avoid issues." - % (self.database.config.cache_version_string, __version__) - ) - self.headers.update(self.database.cached_headers) - - @staticmethod - def from_file(f: BinaryIO): - cache = AbCache() - cache.load(f) - return cache - - def __repr__(self) -> str: - return f"AbCache(config={self.config} bundles={len(self.abcache_index.bundles) if self.abcache_index is not None else 'N/A'})" - - def get_entry_by_bundle_name(self, bundleName: str) -> AbCacheEntry: - return self.abcache_index.bundles.get(bundleName, None) - - def get_entry_download_url(self, entry: AbCacheEntry): - if self.config.app_region in REGION_JP_EN: - return self.SEKAI_AB_ENDPOINT + self.SEKAI_AB_BASE_PATH + entry.bundleName - else: - # https://github.com/mos9527/sssekai/issues/28 - return self.SEKAI_AB_ENDPOINT + entry.downloadPath + "/" + entry.bundleName - - def get_or_update_dependency_tree_flatten(self, bundleName: str, deps: set = None): - """Get a flattened set of asset dependency bundle names (including itself) for a given entry. - - Returned order is topologically sorted. - - if 'deps' is provided, it will be used as the initial set of dependencies. - """ - deps = deps or set() - - def dfs(bundleName): - for dep in self.abcache_index.bundles[bundleName].dependencies: - if dep not in deps: - dfs(dep) - deps.add(dep) - - dfs(bundleName) - deps.add(bundleName) - return deps +from collections import defaultdict +from pickle import load, dump +from typing import BinaryIO, List, Mapping, Optional, Union, Tuple +from logging import getLogger +from dataclasses import dataclass, fields, is_dataclass, field +from functools import cached_property +from base64 import b64encode, b64decode +import json + +logger = getLogger("sssekai.abcache") + +REGION_JP_EN = {"jp", "en"} +REGION_ROW = {"tw", "kr", "cn"} +REGION_OPTIONS = REGION_JP_EN | REGION_ROW + + +class RedactedStr(str): + """A credential that behaves like a str but hides its value in repr. + + `dataclasses.asdict` deep-copies values, so `field(repr=False)` alone does + not keep tokens out of `repr(asdict(...))`. Pickles as a plain str so cache + files stay compatible with older versions. + """ + + def __repr__(self): + return "" + + def __deepcopy__(self, memo): + return self + + def __reduce_ex__(self, protocol): + return str, (str(self),) + + +def fromdict(klass: type, d: Union[Mapping, List], warn_missing_fields=True): + """ + Recursively convert a dictionary to a dataclass instance, if applicable. + https://stackoverflow.com/a/54769644 + https://gist.github.com/gatopeich/1efd3e1e4269e1e98fae9983bb914f22 + """ + + while klass.__name__ == "Optional": + klass = klass.__args__[0] # Reduce Optional[T] -> T + + def ensure_iterable(d): + if isinstance(d, Mapping): + return d + if hasattr(d, "__dict__"): + return d.__dict__ + return dict() + + def check_field(key, fields): + if not key in fields: + if warn_missing_fields: + logger.error(f"Field {key} of type {type(d[key])} not found in {klass}") + return False + return True + + if is_dataclass(klass): + fieldtypes = {f.name: f.type for f in fields(klass)} + d = ensure_iterable(d) + return klass( + **( + { + f: fromdict(fieldtypes[f], d[f], warn_missing_fields) + for f in d + if check_field(f, fieldtypes) + } + ) + ) + if isinstance(d, list) and hasattr(klass, "__args__"): + return [fromdict(klass.__args__[0], di, warn_missing_fields) for di in d] + if isinstance(d, dict) and hasattr(klass, "__args__"): + return { + k: fromdict(klass.__args__[1], v, warn_missing_fields) for k, v in d.items() + } + return d + + +from requests import Session, Response, HTTPError +from msgpack import unpackb, packb + +from sssekai import __version__, __version_tuple__ +from sssekai.unity import sssekai_get_unity_version +from sssekai.crypto.APIManager import decrypt, encrypt, SEKAI_APIMANAGER_KEYSETS + + +@dataclass +class AbCacheConfig: + app_region: str + app_version: str + app_platform: str + app_hash: str + + auth_credential: str = field( + default=None, repr=False + ) # JWT token for JP/EN, Base64 encoded JWT token for ROW + + row_ab_version: str = None # Override AB version in url for ROW + + asset_host: str = None # Override AssetBundle host for EN/JP + asset_version: str = None # Override AssetBundle version for EN/JP + asset_hash: str = None # Override AssetBundle hash for EN/JP + + version: Tuple[int, int, int] = (0, 0, 0) # Internal versioning + + def __setattr__(self, name, value): + # Credentials are also assigned after construction (CLI, auth.py). + if name == "auth_credential" and value is not None: + if not isinstance(value, RedactedStr): + value = RedactedStr(value) + super().__setattr__(name, value) + + @property + def auth_jwt(self) -> str: + if self.app_region in REGION_JP_EN: + return self.auth_credential + else: + try: + return b64decode(self.auth_credential).decode() + except: + return None + + @property + def auth_jwt_payload(self) -> dict: + if not self.auth_jwt: + return {} + header, payload, signature = self.auth_jwt.split(".") + return json.loads(b64decode(payload + "==")) + + @property + def auth_userID(self) -> str: + return self.auth_jwt_payload.get("userId", None) + + @property + def auth_available(self): + return self.auth_userID is not None + + @property + def cache_version(self): + return getattr(self, "version", (0, 0, 0)) + + @property + def cache_version_string(self): + return "%s.%s.%s" % self.cache_version if self.cache_version else "pre-0.5.17" + + @property + def is_up_to_date(self): + return self.cache_version == __version_tuple__ + + @property + def need_client_header_update(self): + if self.app_region in REGION_JP_EN: + if self.app_region in {"jp"}: + return not (self.asset_hash and self.asset_host and self.asset_version) + else: + return not (self.asset_hash and self.asset_version) + return False + + +@dataclass +class AbCacheEntry(dict): + bundleName: str + cacheFileName: str + cacheDirectoryName: str + hash: str + category: str + crc: int + fileSize: int + dependencies: List[str] + isBuiltin: bool + isRelocate: Optional[bool] = None # Added in JP 5.4.0 + paths: Optional[List[str]] = None # Removed in ROW + # ROW only + md5Hash: Optional[str] = None + downloadPath: Optional[str] = None + + +@dataclass +class AbCacheIndex(dict): + version: str + os: Optional[str] = None # Undefined in ROW + bundles: Mapping[str, AbCacheEntry] = None + + +# DS for the game's APIs +# There're *plenty* differences across regions. The implementations here +# - Contains all the fields, even if they're not used in all regions +# - Only the fields that are common across regions are non Optional[T] +@dataclass +class SekaiAppVersion: + systemProfile: str + appVersion: str + multiPlayVersion: str + appVersionStatus: str + assetVersion: str + # These are sadly removed from 3.5.0 (JP) + dataVersion: Optional[str] = None + appHash: Optional[str] = None + assetHash: Optional[str] = None + + +@dataclass +class SekaiSystemData: + serverDate: int + # These are all undefined in ROW + timezone: Optional[str] = None + profile: Optional[str] = None + maintenanceStatus: Optional[str] = None + appVersions: Optional[List[SekaiAppVersion]] = None + + @cached_property + def app_version_dict(self): + return {av.appVersion: av for av in self.appVersions or []} + + +@dataclass +class SekaiGameVersionData: + profile: str + assetbundleHostHash: str + domain: str + + +@dataclass +class SekaiUserRegistrationData: + userId: int + # JP/EN only + signature: Optional[str] = None + platform: Optional[str] = None + deviceModel: Optional[str] = None + operatingSystem: Optional[str] = None + registeredAt: Optional[int] = None + + +@dataclass +class SekaiUserData: + userRegistration: SekaiUserRegistrationData + # JP/EN Only + credential: Optional[str] = field(default=None, repr=False) + updatedResources: Optional[dict] = None + # ROW + sessionToken: Optional[str] = field(default=None, repr=False) + + def __setattr__(self, name, value): + if name in ("credential", "sessionToken") and value is not None: + if not isinstance(value, RedactedStr): + value = RedactedStr(value) + super().__setattr__(name, value) + + @property + def user_credentials(self): + return self.credential or self.sessionToken + + +@dataclass +class SekaiUserAuthData: + sessionToken: str = field(repr=False) + appVersion: str + multiPlayVersion: str + dataVersion: str + assetVersion: str + + # JP/EN only + removeAssetVersion: Optional[str] = None + assetHash: Optional[str] = None + + appVersionStatus: Optional[str] = None + + # JP/EN only + isStreamingVirtualLiveForceOpenUser: Optional[bool] = None + deviceId: Optional[str] = None + updatedResources: Optional[dict] = None + suiteMasterSplitPath: Optional[list] = None + obtainedBondsRewardIds: Optional[list] = None # JP 4.0+ + + # ROW + cdnVersion: Optional[int] = None + configs: Optional[List[dict]] = None + + def __setattr__(self, name, value): + if name == "sessionToken" and value is not None: + if not isinstance(value, RedactedStr): + value = RedactedStr(value) + super().__setattr__(name, value) + + +class RedactedHeaders(dict): + """Request headers kept for cache reuse. Hidden from repr/asdict so tokens + (e.g. X-Session-Token) never reach logs. Pickles as a plain dict so cache + files stay loadable by both old and new versions.""" + + def __repr__(self): + return "" % len(self) + + def __deepcopy__(self, memo): + # dict's default deepcopy returns a plain dict, which would leak through + # dataclasses.asdict(). Header values are str, so a shallow copy suffices. + return RedactedHeaders(self) + + def __reduce_ex__(self, protocol): + return dict, (dict(self),) + + +@dataclass +class SSSekaiDatabase: + config: AbCacheConfig = None + cached_headers: dict = field(default_factory=RedactedHeaders) + + sekai_user_data: SekaiUserData = None + sekai_user_auth_data: SekaiUserAuthData = None + sekai_abcache_index: AbCacheIndex = None + sekai_system_data: SekaiSystemData = None + sekai_gameversion_data: SekaiGameVersionData = None + + def __repr__(self) -> str: + # Two layers guard this: RedactedHeaders hides its own values, and this + # repr also covers a plain dict (databases built by hand, or caches not + # yet normalized by load()) that would otherwise print in full. + values = ", ".join( + "%s=%r" % (f.name, getattr(self, f.name, None)) + for f in fields(self) + if f.name != "cached_headers" + ) + return "SSSekaiDatabase(%s, cached_headers=<%d values>)" % ( + values, + len(self.cached_headers or ()), + ) + + def reset(self): + self.sekai_user_data = None + self.sekai_user_auth_data = None + self.sekai_abcache_index = None + self.sekai_system_data = None + self.sekai_gameversion_data = None + + def redact_secrets(self): + """Re-wrap secrets loaded from disk as RedactedStr. + + RedactedStr pickles as a plain str (so caches stay readable by older + versions), so values read back from a cache file must be wrapped again + before they can reach a repr/asdict. Assignment goes through each + dataclass's __setattr__, which does the wrapping. + """ + if not isinstance(self.cached_headers, RedactedHeaders): + self.cached_headers = RedactedHeaders(self.cached_headers or {}) + # sssekai <= 0.8.2 stored the raw API response here, so old caches may + # hold a plain dict (which also breaks attribute access such as + # SEKAI_CREDENTIAL). Normalize it to the dataclass we expect. + if isinstance(self.sekai_user_data, dict): + self.sekai_user_data = fromdict( + SekaiUserData, self.sekai_user_data, False + ) + secrets = [] + if self.config is not None: + secrets.append((self.config, ("auth_credential",))) + secrets += [ + (self.sekai_user_data, ("credential", "sessionToken")), + (self.sekai_user_auth_data, ("sessionToken",)), + ] + for owner, names in secrets: + if owner is None: + continue + for name in names: + value = getattr(owner, name, None) + if value is not None: + setattr(owner, name, value) + + +class AbCacheBundleNotFoundError(Exception): + bundleName: str + + def __init__(self, bundleName: str): + self.bundleName = bundleName + super().__init__("Bundle not found: %s" % bundleName) + + +class AbCacheUserNotAuthenticatedError(Exception): + pass + + +class AbCache(Session): + database: SSSekaiDatabase + + @property + def SEKAI_API_ENDPOINT(self): + match self.config.app_region: + case "jp": + if self.database.sekai_gameversion_data: + url = self.database.sekai_gameversion_data.domain + url = url.replace( + "{0}", self.database.sekai_gameversion_data.profile + ) + return "https://%s" % url + return "https://production-game-api.sekai.colorfulpalette.org" + case "en": + return f"https://n-{self.SEKAI_API_PROFILE}-game-api.sekai-en.com" + case "tw": + return "https://mkovs-20004-cdn.bytedgame.com" + case "kr": + return "https://mkkorea-obt-prod01-cdn.bytedgame.com" + case "cn": + return "https://mkcn-prod-public-60001-1.dailygn.com" + case _: + raise NotImplementedError + + @property + def SEKAI_API_GAMEVERSION_ENDPOINT(self): + match self.config.app_region: + case "jp": + return "https://game-version.sekai.colorfulpalette.org" + case "en": + return "https://game-version.sekai-en.com" + case _: + raise NotImplementedError + + @property + def SEKAI_ISSUE_ENDPOINT(self): + match self.config.app_region: + case "jp": + return "https://issue.sekai.colorfulpalette.org" + case _: + raise NotImplementedError + + @property + def SEKAI_AB_ROW_CDN(self): + match self.config.app_region: + case "tw": + return "https://lf16-mkovscdn-sg.bytedgame.com/obj/sf-game-alisg/gdl_app_5245" + case "kr": + return ( + "https://lf19-mkkr.bytedgame.com/obj/sf-game-alisg/gdl_app_292248" + ) + case "cn": + return "https://lf3-mkcncdn-tos.dailygn.com/obj/sf-game-lf/gdl_app_5236" + case _: + raise NotImplementedError + + @property + def SEKAI_AB_ROW_PATH(self): + match self.config.app_region: + case "tw": + return "online" + case "kr": + return "kr_online" + case "cn": + return "cn_online" + case _: + raise NotImplementedError + + @cached_property + def SEKAI_AB_ROW_VERSION_NUMBER(self): + url = f"{self.SEKAI_AB_ROW_CDN.replace('sf-game','rt-game')}" + if self.config.app_region in {"cn"}: + url = f"{url}/Mainland" + else: + url = f"{url}/Oversea" + url = f"{url}/{self.SEKAI_AB_ROW_VERSION}/Release/{self.SEKAI_AB_ROW_PATH}/{self.SEKAI_APP_PLATFORM}/version" + resp = self.request("GET", url) + resp.raise_for_status() + return int(resp.text) + + @property + def SEKAI_AB_INFO_ENDPOINT(self): + match self.config.app_region: + case "jp": + return f"https://{self.SEKAI_API_PROFILE}-{self.SEKAI_AB_HOST_HASH}-assetbundle-info.sekai.colorfulpalette.org/api/version/{self.SEKAI_ASSET_VERSION}/{self.SEKAI_AB_HASH}/os/{self.config.app_platform}" + case "en": + # `n` for what, exactly? + return f"https://n-{self.SEKAI_API_PROFILE}-{self.SEKAI_AB_HOST_HASH}-assetbundle-info.sekai-en.com/api/version/{self.SEKAI_ASSET_VERSION}/os/{self.config.app_platform}" + if self.config.app_region in REGION_ROW: + return f"{self.SEKAI_AB_ROW_CDN}/AssetBundle/{self.SEKAI_AB_ROW_VERSION}/Release/{self.SEKAI_AB_ROW_PATH}/{self.SEKAI_APP_PLATFORM}{self.SEKAI_AB_ROW_VERSION_NUMBER}/AssetBundleInfoNew.json" + else: + raise NotImplementedError + + @property + def SEKAI_AB_ENDPOINT(self): + match self.config.app_region: + case "jp": + return f"https://{self.SEKAI_API_PROFILE}-{self.SEKAI_AB_HOST_HASH}-assetbundle.sekai.colorfulpalette.org/" + case "en": + return f"https://n-{self.SEKAI_API_PROFILE}-{self.SEKAI_AB_HOST_HASH}-assetbundle.sekai-en.com/" + if self.config.app_region in REGION_ROW: + return f"{self.SEKAI_AB_ROW_CDN}/AssetBundle/{self.SEKAI_AB_ROW_VERSION}/Release/{self.SEKAI_AB_ROW_PATH}/" + else: + raise NotImplementedError + + def _update_request_headers(self): + app_platform_names = { + 'ios':'iOS', 'android':'Android' + } + self.headers.update( + { + "Accept": "application/octet-stream", + "Content-Type": "application/octet-stream", + "Accept-Encoding": "deflate, gzip", + "User-Agent": "UnityPlayer/%s" % sssekai_get_unity_version(), + "X-Platform": app_platform_names.get(self.config.app_platform,self.config.app_platform), + "X-DeviceModel": "sssekai/%s" % __version__, + "X-OperatingSystem": app_platform_names.get(self.config.app_platform,self.config.app_platform), + "X-Unity-Version": sssekai_get_unity_version(), + "X-App-Version": self.SEKAI_APP_VERSION, + "X-App-Hash": self.SEKAI_APP_HASH, + } + ) + + @property + def config(self): + return self.database.config + + @config.setter + def config(self, v): + self.database.config = v + + @property + def SEKAI_APP_VERSION(self): + return self.config.app_version + + @property + def SEKAI_AB_ROW_VERSION(self): + return self.config.row_ab_version or self.SEKAI_APP_VERSION + + @property + def SEKAI_APP_PLATFORM(self): + return self.config.app_platform + + @property + def SEKAI_APP_HASH(self): + return self.config.app_hash + + @property + def SEKAI_ASSET_VERSION(self): + return ( + self.config.asset_version + or self.database.sekai_system_data.app_version_dict[ + self.SEKAI_APP_VERSION + ].assetVersion + ) + + @property + def SEKAI_AB_HASH(self): + return self.config.asset_hash or self.database.sekai_user_auth_data.assetHash + + @property + def SEKAI_AB_HOST_HASH(self): + return ( + self.config.asset_host + or self.database.sekai_gameversion_data.assetbundleHostHash + ) + + @property + def SEKAI_API_PROFILE(self): + if self.database.sekai_gameversion_data: + return self.database.sekai_gameversion_data.profile + return "production" + + @property + def SEKAI_API_SYSTEM_DATA(self): + return self.SEKAI_API_ENDPOINT + "/api/system" + + @property + def SEKAI_API_USER(self): + return self.SEKAI_API_ENDPOINT + "/api/user" + + @property + def SEKAI_USERID(self): + return self.config.auth_userID or ( + self.database.sekai_user_data.userRegistration.userId + if self.database.sekai_user_data + else None + ) + + @property + def SEKAI_CREDENTIAL(self): + """Keyed in SEKAI_API_USER responses. `accessToken` in ROW servers, `credential` in JP/EN.""" + return self.config.auth_credential or ( + self.database.sekai_user_data.credential + if self.database.sekai_user_data + else None + ) + + @property + def is_authenticated(self): + return self.database.sekai_user_auth_data is not None + + def raise_for_auth(self, *args): + if not self.is_authenticated: + raise AbCacheUserNotAuthenticatedError(*args) + + @property + def SEKAI_API_USER_AUTH(self): + if self.config.app_region in REGION_JP_EN: + assert self.SEKAI_USERID, "User ID must be available" + return f"{self.SEKAI_API_USER}/{self.SEKAI_USERID}/auth" + else: + # For ROW this is the endpoint that retrives SEKAI_API_USER level data as well + return f"{self.SEKAI_API_USER}/auth" + + @property + def SEKAI_API_USER_SUITE(self): + self.raise_for_auth() + return f"{self.SEKAI_API_ENDPOINT}/api/suite/user/{self.SEKAI_USERID}" + + @property + def SEKAI_API_MASTER_SUITE_URLS(self) -> List[str]: + match self.config.app_region: + case "jp": + self.raise_for_auth() + return [ + f"{self.SEKAI_API_ENDPOINT}/api/{path}" + for path in self.database.sekai_user_auth_data.suiteMasterSplitPath + ] + case "en": + self.raise_for_auth() + return [ + f"{self.SEKAI_API_ENDPOINT}/api/{path}" + for path in self.database.sekai_user_auth_data.suiteMasterSplitPath + ] + if self.config.app_region in REGION_ROW: + self.raise_for_auth() + return [f"{self.SEKAI_AB_ROW_CDN}/{self.SEKAI_AB_ROW_PATH}/master-data-{self.database.sekai_user_auth_data.cdnVersion}.info"] + else: + raise NotImplementedError + + @property + def SEKAI_API_INFORMATION(self): + return self.SEKAI_API_ENDPOINT + "/api/information" + + @property + def SEKAI_AB_BASE_PATH(self): + return f"{self.SEKAI_ASSET_VERSION}/{self.SEKAI_AB_HASH}/{self.config.app_platform}/" + + @property + def SEKAI_ISSUE_SIGNATURE_ENDPOINT(self): + return self.SEKAI_ISSUE_ENDPOINT + "/api/signature" + + @property + def abcache_index(self): + return self.database.sekai_abcache_index + + def request_packed(self, method: str, url: str, data: dict = None, **kwargs): + """Send a request with packed data. Data will be packed and encrypted before sending. + + Args: + method (str): HTTP method + url (str): URL + data (dict, optional): Payload data. Defaults to None. + + Returns: + Response: Response object + """ + self._update_request_headers() + if data is not None: + # XXX: packb() does not cover all cases. + # + # Python msgpack implementation doesn't really support variadic float precision within a single pack. + # This generally shouldn't be an issue if the upstream unpacking routine is type-agnostic since + # the type is described in the packed binary as per the spec. + # + # However, if the upstream implementation is not type-agnostic, it WILL lead to issues. + # While most respect the type opcodes (e.g. msgpack-c, msgpack-python, msgpack in Golang), some implementations (e.g. MessagePack-CSharp) disregards + # them altogether when concrete (C#) types are provided. And will raise an exception in such cases when a double is encoded in the place of a float. + # See also: + # https://github.com/mos9527/sssekai/issues/47#issuecomment-3150827990 + # + # Thus `use_single_float=True` is used to cover and only cover single precision cases. + # In the *current* API seen in the game binary, this is fine. And in AbCache's case floats are not at all used. + # But this may change in the future, in which case consult the following issues: + # https://github.com/msgpack/msgpack-python/issues/326 + data = packb(data, use_single_float=True) + data = encrypt(data, SEKAI_APIMANAGER_KEYSETS[self.config.app_region]) + resp = self.request(method=method, url=url, data=data, **kwargs) + if 400 <= resp.status_code < 600: + logger.error(f"{method} {url} {resp.status_code}") + try: # log the error message provided by the API. + content = self.response_to_dict(resp) + logger.error("response=%s" % content) + except: + logger.error("response=%s" % resp.content) + logger.error( + "Unexpected server-side error. Report issues at https://github.com/MetaSekaiLab/sssekai/issues." + ) + resp.raise_for_status() + return resp + + def response_to_dict(self, resp: Response, **kwargs): + """Decrypt and unpack a response content to a dictionary. + + Args: + resp (Response): Response object + **kwargs: Additional arguments for MessagePack unpackb + + Returns: + dict: Decrypted and unpacked data + """ + data = decrypt(resp.content, SEKAI_APIMANAGER_KEYSETS[self.config.app_region]) + data = unpackb(data, **kwargs) + return data + + def _update_user_auth_data(self): + if self.config.auth_available: + logger.debug("Updating user auth data") + if self.config.app_region in REGION_JP_EN: + resp = self.request_packed( + "PUT", + self.SEKAI_API_USER_AUTH, + data={"credential": self.SEKAI_CREDENTIAL, "deviceId": None}, + ) + else: + resp = self.request_packed( + "POST", + self.SEKAI_API_USER_AUTH, + data={ + "accessToken": self.SEKAI_CREDENTIAL, + "deviceId": None, + "userID": 0, + }, + ) + data = self.response_to_dict(resp) + self.database.sekai_user_auth_data = fromdict( + SekaiUserAuthData, data, False + ) + if self.config.app_region in REGION_ROW: + self.database.sekai_user_data = fromdict(SekaiUserData, data, False) + else: + logger.warning( + "No valid auth credential provided. User auth data will not be updated." + ) + + def _update_system_data(self): + logger.debug("Updating system data") + resp = self.request_packed("GET", self.SEKAI_API_SYSTEM_DATA) + data = self.response_to_dict(resp) + self.database.sekai_system_data = fromdict(SekaiSystemData, data) + + def _update_gameversion_data(self): + if self.config.app_region in REGION_JP_EN: + logger.debug("Updating game version data") + resp = self.request_packed( + "GET", + self.SEKAI_API_GAMEVERSION_ENDPOINT + + "/" + + self.SEKAI_APP_VERSION + + "/" + + self.SEKAI_APP_HASH, + ) + data = self.response_to_dict(resp) + self.database.sekai_gameversion_data = fromdict(SekaiGameVersionData, data) + + def __init__(self, config: Optional[AbCacheConfig] = None): + super().__init__() + self.database = SSSekaiDatabase() + self.config = config or AbCacheConfig( + "unknown", "unknown", "unknown", "unknown" + ) + self.config.version = __version_tuple__ + + def update_client_headers(self): + """Authenticate the user and update client headers.""" + logger.debug("Updating metadata") + logger.debug("Set config: %s" % self.config) + try: + self.update_signatures() + self._update_gameversion_data() + self._update_system_data() + except HTTPError as e: + logger.error("Signature update failure: %s" % e) + raise e + if self.config.app_region in REGION_JP_EN: + version_newest = self.database.sekai_system_data.appVersions[-1] + logger.debug("Newest App version: %s" % version_newest) + if version_newest.appVersion != self.SEKAI_APP_VERSION: + logger.warning("App version mismatch. This may cause issues.") + self._update_user_auth_data() + if self.is_authenticated: + self.headers.update( + { + "X-Data-Version": self.database.sekai_user_auth_data.dataVersion, + "X-Asset-Version": self.database.sekai_user_auth_data.assetVersion, + "X-Session-Token": self.database.sekai_user_auth_data.sessionToken, + } + ) + return self.headers + + def update_abcache_index(self) -> AbCacheIndex: + """Update the AssetBundle Index.""" + logger.debug("Updating Assetbundle index") + if self.config.app_region in REGION_JP_EN: + logger.info("Sekai AssetBundle version: %s" % self.SEKAI_ASSET_VERSION) + if self.config.app_region in {"jp"}: + logger.info("Sekai AssetBundle host hash: %s" % self.SEKAI_AB_HOST_HASH) + resp = self.request_packed("GET", self.SEKAI_AB_INFO_ENDPOINT) + data = self.response_to_dict(resp) + self.database.sekai_abcache_index = fromdict(AbCacheIndex, data) + return self.database.sekai_abcache_index + + def update_signatures(self): + """Update client signatures. Only required for JP for the initial setup. + After which the game would cache the signature/cookies.""" + if self.config.app_region in {"jp"}: + logger.debug("Updating signatures") + # XXX: Clear all cookies, it 403s otherwise. Does this trigger a rate limit/expiry check since our session would be tracked? + self.headers["Cookie"] = "" + self.cookies.clear() + resp = self.request_packed("POST", self.SEKAI_ISSUE_SIGNATURE_ENDPOINT) + self.headers["Cookie"] = resp.headers["Set-Cookie"] + # HACK: Per RFC6265, Cookies should not be visible to subdomains since it's not set with Domain attribute (https://github.com/psf/requests/issues/2576) + # But the other endpoints uses it nontheless. So we have to set it manually. + + def update(self): + """Update the cache with the latest AssetBundle Index data from the server. + + NOTE: + AbCache should have been initialized with a valid config before calling this method. + Please refer to `update_client_headers` for more information. + """ + self.update_client_headers() + self.update_abcache_index() + + def save(self, f: BinaryIO): + self._update_request_headers() + self.database.cached_headers = RedactedHeaders(self.headers) + self.database.config.version = __version_tuple__ + logger.info("Saving cache: %s" % self) + dump(self.database, f) + + def load(self, f: BinaryIO): + logger.info("Loading cache") + self.database = load(f) + # Values from disk are plain str/dict; re-wrap so they stay out of logs. + self.database.redact_secrets() + logger.info("Loaded cache: %s" % self) + if not self.database.config.is_up_to_date: + logger.warning( + "Cache is outdated (cache: %s, current: %s). It's HIGHLY recommended to update the cache to avoid issues." + % (self.database.config.cache_version_string, __version__) + ) + self.headers.update(self.database.cached_headers) + + @staticmethod + def from_file(f: BinaryIO): + cache = AbCache() + cache.load(f) + return cache + + def __repr__(self) -> str: + return f"AbCache(config={self.config} bundles={len(self.abcache_index.bundles) if self.abcache_index is not None else 'N/A'})" + + def get_entry_by_bundle_name(self, bundleName: str) -> AbCacheEntry: + return self.abcache_index.bundles.get(bundleName, None) + + def get_entry_download_url(self, entry: AbCacheEntry): + if self.config.app_region in REGION_JP_EN: + return self.SEKAI_AB_ENDPOINT + self.SEKAI_AB_BASE_PATH + entry.bundleName + else: + # https://github.com/mos9527/sssekai/issues/28 + return self.SEKAI_AB_ENDPOINT + entry.downloadPath + "/" + entry.bundleName + + def get_or_update_dependency_tree_flatten(self, bundleName: str, deps: set = None): + """Get a flattened set of asset dependency bundle names (including itself) for a given entry. + + Returned order is topologically sorted. + + if 'deps' is provided, it will be used as the initial set of dependencies. + """ + deps = deps or set() + + def dfs(bundleName): + for dep in self.abcache_index.bundles[bundleName].dependencies: + if dep not in deps: + dfs(dep) + deps.add(dep) + + dfs(bundleName) + deps.add(bundleName) + return deps diff --git a/sssekai/abcache/auth.py b/sssekai/abcache/auth.py index 8896415..37dbc7e 100644 --- a/sssekai/abcache/auth.py +++ b/sssekai/abcache/auth.py @@ -1,30 +1,30 @@ -from . import AbCache, SekaiUserData, fromdict -from logging import getLogger - -logger = getLogger(__name__) - - -def register_as_anonymous_user_sega(cache: AbCache) -> SekaiUserData: - """Registers an anonymous account on SEGA servers, and writes relavant data to the AbCacheobject. - - Args: - cache: The AbCache object to write the user data to. - - Returns: - SekaiUserData: The user data object. - """ - logger.info("Registering user data") - # https://github.com/mos9527/sssekai/issues/31 - cache._update_request_headers() - cache.update_signatures() - payload = { - "platform": cache.headers["X-Platform"], - "deviceModel": cache.headers["X-DeviceModel"], - "operatingSystem": cache.headers["X-OperatingSystem"], - } - resp = cache.request_packed("POST", cache.SEKAI_API_USER, data=payload) - data = cache.response_to_dict(resp) - cache.config.auth_credential = data["credential"] - logger.info("Success. User ID=%s" % cache.config.auth_userID) - cache.database.sekai_user_data = fromdict(SekaiUserData, data, False) - return cache.database.sekai_user_data +from . import AbCache, SekaiUserData, fromdict +from logging import getLogger + +logger = getLogger(__name__) + + +def register_as_anonymous_user_sega(cache: AbCache) -> SekaiUserData: + """Registers an anonymous account on SEGA servers, and writes relavant data to the AbCacheobject. + + Args: + cache: The AbCache object to write the user data to. + + Returns: + SekaiUserData: The user data object. + """ + logger.info("Registering user data") + # https://github.com/mos9527/sssekai/issues/31 + cache._update_request_headers() + cache.update_signatures() + payload = { + "platform": cache.headers["X-Platform"], + "deviceModel": cache.headers["X-DeviceModel"], + "operatingSystem": cache.headers["X-OperatingSystem"], + } + resp = cache.request_packed("POST", cache.SEKAI_API_USER, data=payload) + data = cache.response_to_dict(resp) + cache.config.auth_credential = data["credential"] + logger.info("Success. User ID=%s" % cache.config.auth_userID) + cache.database.sekai_user_data = fromdict(SekaiUserData, data, False) + return cache.database.sekai_user_data diff --git a/sssekai/entrypoint/apphash.py b/sssekai/entrypoint/apphash.py index 743b233..95423b3 100644 --- a/sssekai/entrypoint/apphash.py +++ b/sssekai/entrypoint/apphash.py @@ -1,276 +1,276 @@ -import zipfile -import UnityPy -import logging -import json -import sys - -from io import BytesIO - -from UnityPy.enums import ClassIDType -from UnityPy.helpers.Tpk import get_typetree_node -from sssekai.unity.AssetBundle import load_assetbundle -from tqdm import tqdm - -REGION_MAP = { - # NOTE: Order is used to determine the region - "com.hermes.mk": "cn", - "com.hermes.mk.asia": "tw", - "com.sega.ColorfulStage.en": "en", - "com.sega.pjsekai": "jp", - "com.pjsekai.kr": "kr", -} -ROW_REGIONS = {"cn", "tw", "kr"} -logger = logging.getLogger("apphash") - -PROD_BUNDLES = { - "6350e2ec327334c8a9b7f494f344a761", # PJSK Android - "c726e51b6fe37463685916a1687158dd", # PJSK iOS - "data.unity3d", # TW,KR,CN (ByteDance) -} - - -def enum_candidates(zip_file, filter): - return ( - (f, zip_file.open(f), zip_file) for f in zip_file.filelist if filter(f.filename) - ) - - -def enum_package(zip_file): - yield zip_file - for f in zip_file.filelist: - if f.filename.lower().endswith(".apk"): - yield zipfile.ZipFile(zip_file.open(f)) - - -def dump_axml_stringpool(f: BytesIO): - read_int = lambda nbytes: int.from_bytes(f.read(nbytes), "little") - f.seek(8) - hdr_type, hdr_size, size = read_int(2), read_int(2), read_int(4) - n_strings = read_int(4) - unk = read_int(4) - flags = read_int(4) - is_utf8 = flags & (1 << 8) - string_offset = read_int(4) + 8 - unk = read_int(4) - offsets = [read_int(4) for _ in range(n_strings)] - for offset in offsets: - f.seek(string_offset + offset) - string_len = read_int(2) - if is_utf8: - string = f.read(string_len).decode("utf-8") - else: - string = f.read(string_len * 2).decode("utf-16-le") - yield string - - -def dump_metadata_stringpool(f: BytesIO): - metadata = f.read() - metadata = metadata[metadata.find(b"\xaf\x1b\xb1\xfa") :] - f = BytesIO(metadata) - read_int = lambda nbytes: int.from_bytes(f.read(nbytes), "little") - assert read_int(4) == 0xFAB11BAF, "bad sanity" - logger.info("metadata version: %s" % read_int(4)) - stringLiteralOffset = read_int(4) # string data for managed code - stringLiteralSize = read_int(4) - stringLiteralDataOffset = read_int(4) - stringLiteralDataSize = read_int(4) - f.seek(stringLiteralOffset) - stringLiterals = [ - (read_int(4), read_int(4)) for _ in range(stringLiteralSize // 8) - ] # length, dataIndex - for length, dataIndex in stringLiterals: - f.seek(stringLiteralDataOffset + dataIndex) - string = f.read(length) - yield string.decode("utf-8") - pass - - -def main_apphash(args): - UnityPy.config.SERIALIZED_FILE_PARSE_TYPETREE = False - env = UnityPy.Environment() - app_package = None - app_version = "unknown" - app_hash = "unknown" - app_metadata_strings = None - proxies = None - if args.proxy: - logger.info("Overriding proxy: %s", args.proxy) - proxies = {"http": args.proxy, "https": args.proxy} - if not args.ab_src: - if not args.apk_src or args.fetch: - from requests import get - - src = BytesIO() - logger.debug("Fetching latest game package (JP) from APKPure") - resp = get( - "https://d.apkpure.net/b/XAPK/com.sega.pjsekai?version=latest", - stream=True, - proxies=proxies, - ) - size = resp.headers.get("Content-Length", -1) - with tqdm(total=int(size), unit="B", unit_scale=True) as progress: - for chunk in resp.iter_content(chunk_size=2**20): - src.write(chunk) - progress.update(len(chunk)) - src.seek(0) - else: - src = open(args.apk_src, "rb") - with zipfile.ZipFile(src, "r") as zip_ref: - manifests = [ - manifest - for package in enum_package(zip_ref) - for manifest in enum_candidates( - package, lambda fn: fn == "AndroidManifest.xml" - ) - ] - manifest = manifests[0][1] - manifest_strings = set(dump_axml_stringpool(manifest)) - # Heur: Reverse lookup the package name from the manifest strings - for ky in REGION_MAP: - if ky in manifest_strings: - app_package = ky - break - # if REGION_MAP[app_package] in ROW_REGIONS: - # metadata = [ - # meta - # for package in enum_package(zip_ref) - # for meta in enum_candidates( - # package, - # lambda fn: fn.endswith("global-metadata.dat"), - # ) - # ] - # if metadata: - # metadata = metadata[0][1] - # metadata_strings = list(dump_metadata_stringpool(metadata)) - # app_metadata_strings = [s for s in metadata_strings if "bytedgame" in s] - # pass - candidate_filter = lambda fn: fn.split("/")[-1] in PROD_BUNDLES - candidate_filter_deep = lambda fn: ("_/" + fn).split("/")[-2] == "Data" - - candidates = [ - candidate - for package in enum_package(zip_ref) - for candidate in enum_candidates( - package, - candidate_filter_deep if args.deep else candidate_filter, - ) - ] - for candidate, stream, _ in tqdm(candidates, desc="Loading"): - env.load_file(stream.read()) - else: - logger.info("Loading from AssetBundle %s" % args.ab_src) - with open(args.ab_src, "rb") as f: - env = load_assetbundle(BytesIO(f.read())) - - from sssekai.generated import UTTCGen_AsInstance - from sssekai.generated.Sekai import ( - AndroidPlayerSettingConfig, - IOSPlayerSettingConfig, - ) - - res = dict() - for reader in tqdm(env.objects, desc="Processing"): - if reader.container: - logger.info("Processing %s (%s)", reader.container, reader.type) - if reader.type == UnityPy.enums.ClassIDType.MonoBehaviour: - mono = reader.read(check_read=False) - clazz = None - platform = None - if "_android" in mono.m_Name: - clazz = AndroidPlayerSettingConfig - platform = "android" - elif "_ios" in mono.m_Name: - clazz = IOSPlayerSettingConfig - platform = "ios" - if clazz: - # Works with post 3.4 (JP 3rd Anniversary) builds and downstream regional builds - try: - config = UTTCGen_AsInstance(clazz, reader) - except Exception as e: - logger.error("Failed to parse config for %s: %s", mono.m_Name, e) - continue - config: AndroidPlayerSettingConfig | IOSPlayerSettingConfig - app_version = "%s.%s.%s" % ( - config.clientMajorVersion, - config.clientMinorVersion, - config.clientBuildVersion, - ) - data_version = "%s.%s.%s" % ( - config.clientDataMajorVersion, - config.clientDataMinorVersion, - config.clientDataBuildVersion, - ) - ab_version = "%s.%s.%s" % ( - config.clientMajorVersion, - config.clientMinorVersion, - config.clientDataRevision, - ) - app_hash = config.clientAppHash - # XXX: CN build has the same package name as the JP build?? - # We need another heuristic to determine the package name - # This is previously done by setting app_package - package_heur = app_package or config.bundleIdentifier - region = region = REGION_MAP.get(package_heur, "unknown") - print( - f"Found {config.productName} at {config.m_Name}", - f" Memo: {config.memo}", - f" Package: {config.bundleIdentifier} (actually assumed as {package_heur})", - f" Platform: {platform}", - f" AppHash (app_hash): {config.clientAppHash}", - f" Region (app_region): {region} (determined by {package_heur})", - f" Version (app_version): {app_version}", - f" Version (ab_version): {ab_version}", - f" Bundle Version: {config.bundleVersion}", - f" Data Version: {data_version}", - f" Version Suffix: {config.clientVersionSuffix}", - "", - "", - sep="\n", - file=sys.stderr, - ) - app_package = app_package or "Unknown Package (Failed APK Heuristic)" - # fmt: off - match args.format: - case 'json': - res[mono.m_Name] = { - "package": app_package, - "reported_package": config.bundleIdentifier, - "app_hash": app_hash, - "app_region": region, - "app_version": app_version, - "app_platform": platform, - "ab_version": ab_version, - } - case "markdown": - res[mono.m_Name] = f"""{app_package} ({app_version}, {region}) ---- -Reported Package: {config.bundleIdentifier} - -|{'app_hash'.rjust(48)}| app_region| app_version| ab_version| -|{'-'.rjust(48, '-')}|-------------|-------------|-------------| -|{app_hash.rjust(48)}|{region.rjust(13)}|{app_version.rjust(13)}|{ab_version.rjust(13)}| - -- CLI Usage: - - sssekai abcache --app-platform {platform} --app-region {region} --app-version {app_version} --app-appHash {app_hash} --app-abVersion {ab_version} - -- Python Usage: - - from sssekai.abcache import AbCacheConfig - - AbCacheConfig( - app_region="{region}", - app_version="{app_version}", - row_ab_version="{ab_version}", - app_hash="{app_hash}", - app_platform="{platform}" - ) -""" - print("###### RESULTS ######", file=sys.stderr) - res = dict(sorted(res.items(), key=lambda x: x[0])) - match args.format: - case "json": - print(json.dumps(res, indent=4, ensure_ascii=False)) - case "markdown": - for name, content in res.items(): - print(f"## {name}\n{content}\n") +import zipfile +import UnityPy +import logging +import json +import sys + +from io import BytesIO + +from UnityPy.enums import ClassIDType +from UnityPy.helpers.Tpk import get_typetree_node +from sssekai.unity.AssetBundle import load_assetbundle +from tqdm import tqdm + +REGION_MAP = { + # NOTE: Order is used to determine the region + "com.hermes.mk": "cn", + "com.hermes.mk.asia": "tw", + "com.sega.ColorfulStage.en": "en", + "com.sega.pjsekai": "jp", + "com.pjsekai.kr": "kr", +} +ROW_REGIONS = {"cn", "tw", "kr"} +logger = logging.getLogger("apphash") + +PROD_BUNDLES = { + "6350e2ec327334c8a9b7f494f344a761", # PJSK Android + "c726e51b6fe37463685916a1687158dd", # PJSK iOS + "data.unity3d", # TW,KR,CN (ByteDance) +} + + +def enum_candidates(zip_file, filter): + return ( + (f, zip_file.open(f), zip_file) for f in zip_file.filelist if filter(f.filename) + ) + + +def enum_package(zip_file): + yield zip_file + for f in zip_file.filelist: + if f.filename.lower().endswith(".apk"): + yield zipfile.ZipFile(zip_file.open(f)) + + +def dump_axml_stringpool(f: BytesIO): + read_int = lambda nbytes: int.from_bytes(f.read(nbytes), "little") + f.seek(8) + hdr_type, hdr_size, size = read_int(2), read_int(2), read_int(4) + n_strings = read_int(4) + unk = read_int(4) + flags = read_int(4) + is_utf8 = flags & (1 << 8) + string_offset = read_int(4) + 8 + unk = read_int(4) + offsets = [read_int(4) for _ in range(n_strings)] + for offset in offsets: + f.seek(string_offset + offset) + string_len = read_int(2) + if is_utf8: + string = f.read(string_len).decode("utf-8") + else: + string = f.read(string_len * 2).decode("utf-16-le") + yield string + + +def dump_metadata_stringpool(f: BytesIO): + metadata = f.read() + metadata = metadata[metadata.find(b"\xaf\x1b\xb1\xfa") :] + f = BytesIO(metadata) + read_int = lambda nbytes: int.from_bytes(f.read(nbytes), "little") + assert read_int(4) == 0xFAB11BAF, "bad sanity" + logger.info("metadata version: %s" % read_int(4)) + stringLiteralOffset = read_int(4) # string data for managed code + stringLiteralSize = read_int(4) + stringLiteralDataOffset = read_int(4) + stringLiteralDataSize = read_int(4) + f.seek(stringLiteralOffset) + stringLiterals = [ + (read_int(4), read_int(4)) for _ in range(stringLiteralSize // 8) + ] # length, dataIndex + for length, dataIndex in stringLiterals: + f.seek(stringLiteralDataOffset + dataIndex) + string = f.read(length) + yield string.decode("utf-8") + pass + + +def main_apphash(args): + UnityPy.config.SERIALIZED_FILE_PARSE_TYPETREE = False + env = UnityPy.Environment() + app_package = None + app_version = "unknown" + app_hash = "unknown" + app_metadata_strings = None + proxies = None + if args.proxy: + logger.info("Overriding proxy: %s", args.proxy) + proxies = {"http": args.proxy, "https": args.proxy} + if not args.ab_src: + if not args.apk_src or args.fetch: + from requests import get + + src = BytesIO() + logger.debug("Fetching latest game package (JP) from APKPure") + resp = get( + "https://d.apkpure.net/b/XAPK/com.sega.pjsekai?version=latest", + stream=True, + proxies=proxies, + ) + size = resp.headers.get("Content-Length", -1) + with tqdm(total=int(size), unit="B", unit_scale=True) as progress: + for chunk in resp.iter_content(chunk_size=2**20): + src.write(chunk) + progress.update(len(chunk)) + src.seek(0) + else: + src = open(args.apk_src, "rb") + with zipfile.ZipFile(src, "r") as zip_ref: + manifests = [ + manifest + for package in enum_package(zip_ref) + for manifest in enum_candidates( + package, lambda fn: fn == "AndroidManifest.xml" + ) + ] + manifest = manifests[0][1] + manifest_strings = set(dump_axml_stringpool(manifest)) + # Heur: Reverse lookup the package name from the manifest strings + for ky in REGION_MAP: + if ky in manifest_strings: + app_package = ky + break + # if REGION_MAP[app_package] in ROW_REGIONS: + # metadata = [ + # meta + # for package in enum_package(zip_ref) + # for meta in enum_candidates( + # package, + # lambda fn: fn.endswith("global-metadata.dat"), + # ) + # ] + # if metadata: + # metadata = metadata[0][1] + # metadata_strings = list(dump_metadata_stringpool(metadata)) + # app_metadata_strings = [s for s in metadata_strings if "bytedgame" in s] + # pass + candidate_filter = lambda fn: fn.split("/")[-1] in PROD_BUNDLES + candidate_filter_deep = lambda fn: ("_/" + fn).split("/")[-2] == "Data" + + candidates = [ + candidate + for package in enum_package(zip_ref) + for candidate in enum_candidates( + package, + candidate_filter_deep if args.deep else candidate_filter, + ) + ] + for candidate, stream, _ in tqdm(candidates, desc="Loading"): + env.load_file(stream.read()) + else: + logger.info("Loading from AssetBundle %s" % args.ab_src) + with open(args.ab_src, "rb") as f: + env = load_assetbundle(BytesIO(f.read())) + + from sssekai.generated import UTTCGen_AsInstance + from sssekai.generated.Sekai import ( + AndroidPlayerSettingConfig, + IOSPlayerSettingConfig, + ) + + res = dict() + for reader in tqdm(env.objects, desc="Processing"): + if reader.container: + logger.info("Processing %s (%s)", reader.container, reader.type) + if reader.type == UnityPy.enums.ClassIDType.MonoBehaviour: + mono = reader.read(check_read=False) + clazz = None + platform = None + if "_android" in mono.m_Name: + clazz = AndroidPlayerSettingConfig + platform = "android" + elif "_ios" in mono.m_Name: + clazz = IOSPlayerSettingConfig + platform = "ios" + if clazz: + # Works with post 3.4 (JP 3rd Anniversary) builds and downstream regional builds + try: + config = UTTCGen_AsInstance(clazz, reader) + except Exception as e: + logger.error("Failed to parse config for %s: %s", mono.m_Name, e) + continue + config: AndroidPlayerSettingConfig | IOSPlayerSettingConfig + app_version = "%s.%s.%s" % ( + config.clientMajorVersion, + config.clientMinorVersion, + config.clientBuildVersion, + ) + data_version = "%s.%s.%s" % ( + config.clientDataMajorVersion, + config.clientDataMinorVersion, + config.clientDataBuildVersion, + ) + ab_version = "%s.%s.%s" % ( + config.clientMajorVersion, + config.clientMinorVersion, + config.clientDataRevision, + ) + app_hash = config.clientAppHash + # XXX: CN build has the same package name as the JP build?? + # We need another heuristic to determine the package name + # This is previously done by setting app_package + package_heur = app_package or config.bundleIdentifier + region = region = REGION_MAP.get(package_heur, "unknown") + print( + f"Found {config.productName} at {config.m_Name}", + f" Memo: {config.memo}", + f" Package: {config.bundleIdentifier} (actually assumed as {package_heur})", + f" Platform: {platform}", + f" AppHash (app_hash): {config.clientAppHash}", + f" Region (app_region): {region} (determined by {package_heur})", + f" Version (app_version): {app_version}", + f" Version (ab_version): {ab_version}", + f" Bundle Version: {config.bundleVersion}", + f" Data Version: {data_version}", + f" Version Suffix: {config.clientVersionSuffix}", + "", + "", + sep="\n", + file=sys.stderr, + ) + app_package = app_package or "Unknown Package (Failed APK Heuristic)" + # fmt: off + match args.format: + case 'json': + res[mono.m_Name] = { + "package": app_package, + "reported_package": config.bundleIdentifier, + "app_hash": app_hash, + "app_region": region, + "app_version": app_version, + "app_platform": platform, + "ab_version": ab_version, + } + case "markdown": + res[mono.m_Name] = f"""{app_package} ({app_version}, {region}) +--- +Reported Package: {config.bundleIdentifier} + +|{'app_hash'.rjust(48)}| app_region| app_version| ab_version| +|{'-'.rjust(48, '-')}|-------------|-------------|-------------| +|{app_hash.rjust(48)}|{region.rjust(13)}|{app_version.rjust(13)}|{ab_version.rjust(13)}| + +- CLI Usage: + + sssekai abcache --app-platform {platform} --app-region {region} --app-version {app_version} --app-appHash {app_hash} --app-abVersion {ab_version} + +- Python Usage: + + from sssekai.abcache import AbCacheConfig + + AbCacheConfig( + app_region="{region}", + app_version="{app_version}", + row_ab_version="{ab_version}", + app_hash="{app_hash}", + app_platform="{platform}" + ) +""" + print("###### RESULTS ######", file=sys.stderr) + res = dict(sorted(res.items(), key=lambda x: x[0])) + match args.format: + case "json": + print(json.dumps(res, indent=4, ensure_ascii=False)) + case "markdown": + for name, content in res.items(): + print(f"## {name}\n{content}\n") diff --git a/sssekai/entrypoint/rla2json.py b/sssekai/entrypoint/rla2json.py index 7755695..78f7fe3 100644 --- a/sssekai/entrypoint/rla2json.py +++ b/sssekai/entrypoint/rla2json.py @@ -1,80 +1,80 @@ -from logging import getLogger -from sssekai.unity.AssetBundle import load_assetbundle -from UnityPy.enums import ClassIDType -from sssekai.fmt.rla import read_archive_rla_frames, read_rla_frames -import os, io, json, base64 - -logger = getLogger(__name__) - - -def main_rla2json(args): - def __dump_from_frame_gen(frame_gen): - outdir = args.outdir - os.makedirs(outdir, exist_ok=True) - ensure_dir = lambda x: ( - ( - os.makedirs(os.path.dirname(x), exist_ok=True) - if os.path.dirname(x) - else None - ) - or x - ) - for tick, frame in frame_gen: - print("tick: %16s, type: %32s" % (tick, frame["type"]), end="\r") - match frame["type"]: - case "SoundData": - if frame["encoding"] == "hca": - with open( - ensure_dir( - os.path.join(outdir, "SoundData", str(tick) + ".hca"), - ), - "wb", - ) as f: - raw_data = frame["data"] - start = raw_data.find(b"\xff\xff") - f.write(raw_data[start:]) - else: - raise ValueError("unsupported encoding: %s" % frame["encoding"]) - case _: - with open( - ensure_dir( - os.path.join(outdir, frame["type"], str(tick) + ".json") - ), - "w", - encoding="utf-8", - ) as f: - json.dump(frame, f, ensure_ascii=False, indent=4) - - if os.path.isfile(args.input): - with open(args.input, "rb") as f: - datas = dict() - rla_env = load_assetbundle(f) - for obj in rla_env.objects: - if obj.type in {ClassIDType.TextAsset}: - data = obj.read() - datas[data.m_Name] = data.m_Script.encode( - "utf-8", "surrogateescape" - ) - header = datas.get("sekai.rlh", None) - assert header, "RLH Header file not found!" - header = json.loads(header.decode("utf-8")) - version = tuple(map(int, header["version"].split("."))) - logger.info("Version: %d.%d" % version) - logger.info("Count: %d" % len(header["splitFileIds"])) - splitSeconds = header["splitSeconds"] - for sid in header["splitFileIds"]: - sname = "sekai_%02d_%08d" % (splitSeconds, sid) - script = datas[sname + ".rla"] - frame_gen = read_archive_rla_frames( - io.BytesIO(script), version, args.strict - ) - __dump_from_frame_gen(frame_gen) - else: - logger.info("Reading from directory") - packet_gen = ( - (file, open(os.path.join(args.input, file), "rb").read()) - for file in os.listdir(args.input) - ) - version = tuple(map(int, args.version.split("."))) - frame_gen = read_rla_frames(packet_gen, version, args.strict) - __dump_from_frame_gen(frame_gen) +from logging import getLogger +from sssekai.unity.AssetBundle import load_assetbundle +from UnityPy.enums import ClassIDType +from sssekai.fmt.rla import read_archive_rla_frames, read_rla_frames +import os, io, json, base64 + +logger = getLogger(__name__) + + +def main_rla2json(args): + def __dump_from_frame_gen(frame_gen): + outdir = args.outdir + os.makedirs(outdir, exist_ok=True) + ensure_dir = lambda x: ( + ( + os.makedirs(os.path.dirname(x), exist_ok=True) + if os.path.dirname(x) + else None + ) + or x + ) + for tick, frame in frame_gen: + print("tick: %16s, type: %32s" % (tick, frame["type"]), end="\r") + match frame["type"]: + case "SoundData": + if frame["encoding"] == "hca": + with open( + ensure_dir( + os.path.join(outdir, "SoundData", str(tick) + ".hca"), + ), + "wb", + ) as f: + raw_data = frame["data"] + start = raw_data.find(b"\xff\xff") + f.write(raw_data[start:]) + else: + raise ValueError("unsupported encoding: %s" % frame["encoding"]) + case _: + with open( + ensure_dir( + os.path.join(outdir, frame["type"], str(tick) + ".json") + ), + "w", + encoding="utf-8", + ) as f: + json.dump(frame, f, ensure_ascii=False, indent=4) + + if os.path.isfile(args.input): + with open(args.input, "rb") as f: + datas = dict() + rla_env = load_assetbundle(f) + for obj in rla_env.objects: + if obj.type in {ClassIDType.TextAsset}: + data = obj.read() + datas[data.m_Name] = data.m_Script.encode( + "utf-8", "surrogateescape" + ) + header = datas.get("sekai.rlh", None) + assert header, "RLH Header file not found!" + header = json.loads(header.decode("utf-8")) + version = tuple(map(int, header["version"].split("."))) + logger.info("Version: %d.%d" % version) + logger.info("Count: %d" % len(header["splitFileIds"])) + splitSeconds = header["splitSeconds"] + for sid in header["splitFileIds"]: + sname = "sekai_%02d_%08d" % (splitSeconds, sid) + script = datas[sname + ".rla"] + frame_gen = read_archive_rla_frames( + io.BytesIO(script), version, args.strict + ) + __dump_from_frame_gen(frame_gen) + else: + logger.info("Reading from directory") + packet_gen = ( + (file, open(os.path.join(args.input, file), "rb").read()) + for file in os.listdir(args.input) + ) + version = tuple(map(int, args.version.split("."))) + frame_gen = read_rla_frames(packet_gen, version, args.strict) + __dump_from_frame_gen(frame_gen)