From 44d5289326d5a3e93c6173908dfc36b3239f0805 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Wed, 14 Oct 2026 13:58:23 +0800 Subject: [PATCH] feat(cli)!: arrange the commands as groups of commands Every command is now `nnnotes `: - config init|set|unset|check|path, servers - catalog keys|browse|versions|import|fetch|diff, bundle pull - master version|download|decode - export assets|episode|story|live|live2d|spot|room|shader|audio|ui - apk settings|hca-key - store plan|run|verify - voices list|search|summary|get - site build -o SITE, music-data -o FILE The global options are accepted before or after the command name, and include --store and --json. Every command prints a short summary, or with --json its report as JSON; notes and errors go to standard error. Commands that write files take -o/--out. Option names: - --catalog-version pins the international resource version (the [catalog] version setting); --stored-catalog selects a catalog version of the store for export assets and store plan. - site build serves the regions of --site-region; --region is the global [catalog] region. - voices search limits the search to --search-language; voices list shows the text in the global --language. - master download takes --fetch-workers; voices get takes --flac-level. - --workers of master decode and store verify default to the CPUs the process may use; --workers 0 runs without parallel workers (one thread for music-data). Each group is a module of nnnotes.cli. docs/cli.md lists every command and argument; python -m nnnotes.cli.reference writes it from the parser, and a test keeps it current. BREAKING CHANGE: the command names and the options above changed; the previous names are not accepted. Co-Authored-By: Claude Opus 5.5 --- README.en.md | 80 +- README.md | 77 +- docs/assets.md | 59 +- docs/cli.md | 465 ++++++ docs/commands.md | 155 +- docs/configuration.md | 101 +- docs/contracts.md | 2 +- docs/jp.md | 15 +- docs/music-data.md | 4 +- docs/stages.md | 8 +- docs/views.md | 4 +- scripts/atom_eval.py | 2 +- src/nnnotes/cli/__init__.py | 1246 +---------------- src/nnnotes/cli/apk.py | 57 + src/nnnotes/cli/{cli_assets.py => assets.py} | 440 +----- src/nnnotes/cli/browse.py | 4 +- src/nnnotes/cli/bundle.py | 21 + src/nnnotes/cli/catalog.py | 239 ++++ src/nnnotes/cli/common.py | 321 +++++ src/nnnotes/cli/config.py | 236 ++++ src/nnnotes/cli/export.py | 324 +++++ src/nnnotes/cli/master.py | 103 ++ src/nnnotes/cli/musicdata.py | 133 ++ src/nnnotes/cli/reference.py | 140 ++ src/nnnotes/cli/servers.py | 35 + src/nnnotes/cli/site.py | 256 ++++ src/nnnotes/cli/store.py | 48 + src/nnnotes/cli/voices.py | 247 ++++ src/nnnotes/content/ui.py | 35 +- src/nnnotes/musicdata/__init__.py | 6 +- src/nnnotes/pipeline/catalogdb.py | 9 + src/nnnotes/pipeline/modelstages.py | 22 +- src/nnnotes/pipeline/objexport.py | 9 +- src/nnnotes/pipeline/plan.py | 4 +- src/nnnotes/pipeline/voices.py | 259 +--- src/nnnotes/site/compress.py | 2 +- src/nnnotes/site/web.py | 2 +- src/nnnotes/site/webaudio.py | 6 +- src/nnnotes/site/webmodel.py | 6 +- src/nnnotes/source/config.py | 2 +- src/nnnotes/source/jp.py | 2 +- src/nnnotes/source/nnnotes.example.toml | 12 +- src/nnnotes/source/session.py | 17 +- .../{test_cli_assets.py => test_assets.py} | 324 +++-- tests/cli/test_cli.py | 151 +- tests/cli/test_reference.py | 28 + tests/cli/test_voices.py | 189 +++ tests/content/test_story_kinds.py | 5 +- tests/content/test_ui.py | 8 +- tests/musicdata/test_deckdata.py | 2 +- tests/musicdata/test_statscache.py | 10 +- tests/pipeline/test_cristages.py | 2 +- tests/pipeline/test_voices.py | 95 -- tests/pipeline/test_voices_story.py | 83 +- tests/site/test_live_options_web.py | 5 +- tests/site/test_readsets.py | 23 +- tests/site/test_story_site.py | 17 +- tests/site/test_web.py | 47 +- tests/site/test_webmodel.py | 20 +- tests/source/test_catalog_lazy.py | 7 +- tests/source/test_catalog_versions.py | 17 +- tests/source/test_configfile.py | 3 +- tests/source/test_gameapi.py | 2 +- tests/source/test_jp.py | 10 +- tests/unity/test_typetrees.py | 4 +- 65 files changed, 3707 insertions(+), 2560 deletions(-) create mode 100644 docs/cli.md create mode 100644 src/nnnotes/cli/apk.py rename src/nnnotes/cli/{cli_assets.py => assets.py} (80%) create mode 100644 src/nnnotes/cli/bundle.py create mode 100644 src/nnnotes/cli/catalog.py create mode 100644 src/nnnotes/cli/common.py create mode 100644 src/nnnotes/cli/config.py create mode 100644 src/nnnotes/cli/export.py create mode 100644 src/nnnotes/cli/master.py create mode 100644 src/nnnotes/cli/musicdata.py create mode 100644 src/nnnotes/cli/reference.py create mode 100644 src/nnnotes/cli/servers.py create mode 100644 src/nnnotes/cli/site.py create mode 100644 src/nnnotes/cli/store.py create mode 100644 src/nnnotes/cli/voices.py rename tests/cli/{test_cli_assets.py => test_assets.py} (81%) create mode 100644 tests/cli/test_reference.py create mode 100644 tests/cli/test_voices.py diff --git a/README.en.md b/README.en.md index 787ee4c..43eb131 100644 --- a/README.en.md +++ b/README.en.md @@ -8,7 +8,7 @@ for configuration and validation limits. nnnotes is an offline data toolkit for the game files of BanG Dream! Our Notes: it reads Addressables catalogs, downloads and decrypts asset bundles, downloads and decodes master data, and exports stories, Live2D models, spots, -shaders, CRI audio and live charts as structured JSON and common file formats. The charts, Live2D models and stories `web` +shaders, CRI audio and live charts as structured JSON and common file formats. The charts, Live2D models and stories `site build` exports are the data that [ournotes-player](https://github.com/empty-sekai/ournotes-player) reads. The naming was inspired by sssekai. This is an unofficial fan project, not affiliated with the game's developer or operator. The repository contains no @@ -36,31 +36,31 @@ nnnotes config check # each setting's origin and whether it is valid | Command | Input | Output | |---|---|---| -| `catalog` | the region's catalog | addressable keys (optionally by prefix) | -| `browse` | the configured regions' catalogs | a local web page to browse catalogs and bundles (`--host` / `--port`) | -| `pull` | addressable keys | every bundle of each key's dependency closure, decrypted into the local cache | +| `catalog keys` | the region's catalog | addressable keys (optionally by prefix) | +| `catalog browse` | the configured regions' catalogs | a local web page to browse catalogs and bundles (`--host` / `--port`) | +| `bundle pull` | addressable keys | every bundle of each key's dependency closure, decrypted into the local cache | | `servers` | the bootstrap API root `[bootstrap] api` | the game API's server list: each region's name, area id and CDN / API roots (the roots only with `--show-hosts`) | | `master version` | region | the master data version and resource version the region serves now (anonymous game API call) | | `master download` | master data version, or `--latest` (the region's current one) | that version's `MasterManifest.json` and every `.bin` file (SHA-256 checked) | | `master decode` | master data `.bin` files or directories | one JSON per table (Rijndael-256 CBC decryption + gzip) | -| `adv` | episode ID | `episode.json`: command list, lines in five languages, voice / sound / video index | -| `story` | episode ID | a story directory: episode, audio, stage scene and shaders, story UI, frames / particle effects / post effects / stills / talk windows / chat assets, videos (WebM); the Live2D models it uses are exported into a models directory (`--models`, default `live2d/` next to the story directory), where other stories reuse them | -| `live2d` | model key or model id | a Live2D (Cubism) runtime directory: moc3, textures, motion3, physics, expressions, prefab parameters | -| `spot` | spot ID | `spot.json` + Spine characters + the room as `room.glb` + shaders | -| `room` | background prefab key | the room model (binary glTF) | -| `shader` | key or APK bundles | every platform variant of the shaders (GLSL ES and others) with an index | -| `audio` | CRI cue sheet | one audio file per cue (FLAC / Ogg / WAV) + cue metadata | -| `crikey` | APK | the CRI HCA keycode from the game's boot data (shows whether it was found; can write a `.hcakey`) | -| `player` | APK | render-related global settings (color space, quality levels, renderers) as JSON | +| `export episode` | episode ID | `episode.json`: command list, lines in five languages, voice / sound / video index | +| `export story` | episode ID | a story directory: episode, audio, stage scene and shaders, story UI, frames / particle effects / post effects / stills / talk windows / chat assets, videos (WebM); the Live2D models it uses are exported into a models directory (`--models`, default `live2d/` next to the story directory), where other stories reuse them | +| `export live2d` | model key or model id | a Live2D (Cubism) runtime directory: moc3, textures, motion3, physics, expressions, prefab parameters | +| `export spot` | spot ID | `spot.json` + Spine characters + the room as `room.glb` + shaders | +| `export room` | background prefab key | the room model (binary glTF) | +| `export shader` | key or APK bundles | every platform variant of the shaders (GLSL ES and others) with an index | +| `export audio` | CRI cue sheet | one audio file per cue (FLAC / Ogg / WAV) + cue metadata | +| `apk hca-key` | APK | the CRI HCA keycode from the game's boot data (shows whether it was found; can write a `.hcakey`) | +| `apk settings` | APK | render-related global settings (color space, quality levels, renderers) as JSON | | `ui` (experimental) | APK set and its embedded catalog | an offline serialized prefab library with dependency roots, AnimatorControllers, stable references and PNG/font resources for the optional ournotes-player UI preview ([format and limits](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/ui.md)) | -| `live` | music ID + difficulty | a full chart directory: chart and runtime notes, 3D scene, note and effect assets, BGM and sounds, sound routing | -| `web` | `--pair music:difficulty` (repeatable) or `--all`; `--live2d model` (repeatable) or `--all-live2d`; `--story episode` (repeatable) or `--all-stories`; `--region region` (repeatable) or `--all-regions` | an ournotes-player static site: shared player + per-chart / per-model / per-episode manifests + content-addressed assets; the Live2D models of the stories are built first as models, listed in `models.json`, and the story manifests reference them; one site can serve several regions, with listing texts in five languages; a story's interface texts are grouped by language, with TextMeshPro font assets generated from open fonts (the game's fonts with `--fonts game`); compressible assets (JSON, shaders, moc3, ...) are stored gzip encoded by default (`--compress br` / `none`) | +| `export live` | music ID + difficulty | a full chart directory: chart and runtime notes, 3D scene, note and effect assets, BGM and sounds, sound routing | +| `site build` | `--pair music:difficulty` (repeatable) or `--all`; `--live2d model` (repeatable) or `--all-live2d`; `--story episode` (repeatable) or `--all-stories`; `--site-region region` (repeatable) or `--all-regions` | an ournotes-player static site: shared player + per-chart / per-model / per-episode manifests + content-addressed assets; the Live2D models of the stories are built first as models, listed in `models.json`, and the story manifests reference them; one site can serve several regions, with listing texts in five languages; a story's interface texts are grouped by language, with TextMeshPro font assets generated from open fonts (the game's fonts with `--fonts game`); compressible assets (JSON, shaders, moc3, ...) are stored gzip encoded by default (`--compress br` / `none`) | | `music-data` | master data files (`--master-files` directory or `--apk-master`), or decoded master data with its manifest (`--decoded-master`) | one JSON file with every song and chart: titles and credits in five languages, bands, vocal characters, category, tags, release time, score ranks, BGM length; the Gekisou catalog (member cards, snaps and their Gekisou skills and support skills); per difficulty the level, note counts, BPM, chart times, skill events and fever ranges; and the chart statistics the deck model ournotes-deck (installed with `pip install 'nnnotes[deck]'`) measures on its whole-live simulation (the no-skill score, the weight of every score-up skill kind at every position), plus single-shape Gekisou aptitude (increments and standard errors, not an optimal deck; omit with `--no-gekisou-aptitude`). `--full` adds the deck model's input: every chart's runtime notes and the master data tables about cards, skills, bonuses, scores and events ([format](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/music-data.md)) | Export conventions: -- Commands that write files take the output path with `-o` (required); `web` takes the site directory as a - positional argument. +- Commands that write files take the output path with `-o` (required). +- A command prints a short summary; with `--json` it prints its report as JSON. - JSON is always UTF-8 with LF line endings; infinity is written as `1e999`. - Exports are deterministic for a given installation: the same input with the same versions of nnnotes, its Python dependencies (UnityPy, Pillow, numpy, ...) and the external tools (vgmstream, FFmpeg) gives byte-identical files. @@ -79,27 +79,27 @@ Based on all data of the Taiwan server, version 1.0.1 (zh-Hant): |---|---| | catalog / bundle decryption / dependency closure | working | | master data decoding | working | -| stories, `adv` | 946 / 946 episodes | -| stories, `story` (full directory) | 946 / 946 episodes have all their resources in the catalog and of supported kinds (the resource closure equals the game's own per-episode download list); 946 / 946 exported and checked one by one: the files, models and audio the output refers to exist, every JSON parses and every path it refers to resolves; the 235 episodes with frames, effects, post effects, stills, chats, talk windows or videos are byte-identical across two exports except for the models directory path | +| stories, `export episode` | 946 / 946 episodes | +| stories, `export story` (full directory) | 946 / 946 episodes have all their resources in the catalog and of supported kinds (the resource closure equals the game's own per-episode download list); 946 / 946 exported and checked one by one: the files, models and audio the output refers to exist, every JSON parses and every path it refers to resolves; the 235 episodes with frames, effects, post effects, stills, chats, talk windows or videos are byte-identical across two exports except for the models directory path | | Live2D models | 239 / 239 (every model of the catalog; episodes use 185 of them) | | CRI audio | 681 / 681 cue sheets | -| charts, `live` | 336 / 336 (music, difficulty) pairs | -| web site, `web` | 336 / 336 charts, 239 / 239 Live2D models, 946 / 946 story episodes (two full builds in English without audio are byte-identical and 946 / 946 pass the data validation); two full builds of the three regions in five languages with AAC audio (`--all-regions`, with the live master data: 340 charts of 85 musics) are likewise byte-identical and pass the data validation in full | -| spots, `spot` / `room` | 39 / 39 spots, every reference resolves; `room` exports 20 / 20 room backgrounds of the catalog. Every glb passes the Khronos glTF-Validator with 0 errors | +| charts, `export live` | 336 / 336 (music, difficulty) pairs | +| web site, `site build` | 336 / 336 charts, 239 / 239 Live2D models, 946 / 946 story episodes (two full builds in English without audio are byte-identical and 946 / 946 pass the data validation); two full builds of the three regions in five languages with AAC audio (`--all-regions`, with the live master data: 340 charts of 85 musics) are likewise byte-identical and pass the data validation in full | +| spots, `export spot` / `export room` | 39 / 39 spots, every reference resolves; `export room` exports 20 / 20 room backgrounds of the catalog. Every glb passes the Khronos glTF-Validator with 0 errors | | other regions (en / kr) and languages | checked: the regions serve the same catalog for a language and the same bundles, and the keys are shared; the master tables the charts use are the same in the three regions, and the text tables have all five languages; the chart exports checked match the Taiwan server's. A full multi-region site build is verified (the row above) | ## Requirements -- Python 3.11+, 3.13 recommended (a `web` build reads and writes a lot of JSON, and the standard library encodes +- Python 3.11+, 3.13 recommended (a `site build` reads and writes a lot of JSON, and the standard library encodes JSON faster on 3.13); for development `pip install -e .` in the repository -- the game's `base.apk`: APK-local bundles, the CRI keycode, boot settings. `player`, `story`, `live` and `web` read +- the game's `base.apk`: APK-local bundles, the CRI keycode, boot settings. `apk settings`, `export story`, `export live` and `site build` read MonoBehaviours of its boot data with type trees that ship with nnnotes; supported now: game version 1.0.1 (Unity 6000.3.12f1). With an APK of another version whose classes do not match, these commands stop with an error naming the class, the game version and the Unity version -- a decoded master data directory (from `master download` + `master decode`): needed by `adv`, `story`, `spot`, - `live` and the charts of `web`; the Live2D models of `web` use it only for their character names (optional) +- a decoded master data directory (from `master download` + `master decode`): needed by `export episode`, `export story`, `export spot`, + `export live` and the charts of `site build`; the Live2D models of `site build` use it only for their character names (optional) - external tools: [vgmstream](https://vgmstream.org/) (CRI HCA decoding), [FFmpeg](https://ffmpeg.org/) - (transcoding, including the WebM muxing and Opus audio of story videos); `web` also needs Node.js 20+ and a built + (transcoding, including the WebM muxing and Opus audio of story videos); `site build` also needs Node.js 20+ and a built ournotes-player ## Configuration @@ -112,7 +112,7 @@ overriding earlier ones: `nnnotes config path` lists the lookup order and the file read) 2. environment variables: `NNNOTES_
_` (e.g. `NNNOTES_BUNDLE_KEY`, `NNNOTES_SERVERS_TW_CDN`) 3. command-line flags: `--region`, `--language`, `--catalog`, `--cache`, `--master`, `--apk`, `--ffmpeg`, - `--vgmstream`, `--node` go before the command name; `--player` is an option of `web` + `--vgmstream`, `--node` go before or after the command name; `--player` is an option of `site build` `nnnotes config init` writes the config template (every setting empty; it ships with the package, in the repository [`src/nnnotes/source/nnnotes.example.toml`](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/source/nnnotes.example.toml)), @@ -132,22 +132,22 @@ See [docs/configuration.md](https://github.com/MetaSekaiLab/nnnotes/blob/main/do ## Examples ```bash -nnnotes catalog --prefix Live/MusicScore/ --limit 20 -nnnotes browse --port 8000 -nnnotes pull Live/MusicScore/0001/0001_03 +nnnotes catalog keys --prefix Live/MusicScore/ --limit 20 +nnnotes catalog browse --port 8000 +nnnotes bundle pull Live/MusicScore/0001/0001_03 nnnotes master version nnnotes master download --latest -o work/master-bin nnnotes master decode work/master-bin -o work/master -nnnotes --master work/master adv 10462 -o out/adv_10462.json -nnnotes story 10462 -o out/story_10462 -nnnotes live 100001 --difficulty expert -o out/live_100001 -nnnotes web out/site --all --player --workers 5 -nnnotes web out/site --pair 100001:expert --pair 100001:hard --player -nnnotes web out/site --all --region tw --region en --region kr --player -nnnotes web out/site --live2d adv_live2d_rana_003_casual_spring_01 --player +nnnotes --master work/master export episode 10462 -o out/adv_10462.json +nnnotes export story 10462 -o out/story_10462 +nnnotes export live 100001 --difficulty expert -o out/live_100001 +nnnotes site build -o out/site --all --player --workers 5 +nnnotes site build -o out/site --pair 100001:expert --pair 100001:hard --player +nnnotes site build -o out/site --all --site-region tw --site-region en --site-region kr --player +nnnotes site build -o out/site --live2d adv_live2d_rana_003_casual_spring_01 --player ``` -`web` builds incrementally: charts and models whose manifest exists are skipped (`--force` rebuilds them) and assets +`site build` builds incrementally: charts and models whose manifest exists are skipped (`--force` rebuilds them) and assets no longer referenced are removed. Common options: - `--format aac|opus|vorbis|mp3|flac`: BGM format, default AAC; `--no-audio`: no audio files @@ -158,7 +158,7 @@ no longer referenced are removed. Common options: temporary build directory (default `.tmp`) - `--live2d MODEL` (model id or key, repeatable) / `--all-live2d`: add Live2D models (every model of the catalog); charts and models can be added in the same run -- `--region REGION` (repeatable) / `--all-regions`: the regions the site serves (default: `[catalog] region`); +- `--site-region REGION` (repeatable) / `--all-regions`: the regions the site serves (default: `[catalog] region`); each region's master data is `[servers.] master`. Regions with the same chart data share one manifest; the chart list page switches region and language with `?region=&lang=` - `--player-only`: rewrite the player files, `charts.json` and `models.json` only; `--reingest-json`: store every diff --git a/README.md b/README.md index 0f86f2d..b0af862 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [简体中文](README.md) | [English](README.en.md) -nnnotes 是 BanG Dream! Our Notes 游戏文件的离线数据工具包：读取 Addressables catalog、下载并解密资源包、下载与解码 masterdata，再把剧情、Live2D、据点场景、着色器、CRI 音频和谱面导出成结构化的 JSON 与通用格式文件。`web` 导出的谱面、Live2D 模型与剧情站点就是 [ournotes-player](https://github.com/empty-sekai/ournotes-player) 读取的数据。命名灵感来自 sssekai。 +nnnotes 是 BanG Dream! Our Notes 游戏文件的离线数据工具包：读取 Addressables catalog、下载并解密资源包、下载与解码 masterdata，再把剧情、Live2D、据点场景、着色器、CRI 音频和谱面导出成结构化的 JSON 与通用格式文件。`site build` 导出的谱面、Live2D 模型与剧情站点就是 [ournotes-player](https://github.com/empty-sekai/ournotes-player) 读取的数据。命名灵感来自 sssekai。 本项目为非官方爱好者项目，与游戏的开发和运营方无关。仓库不包含任何游戏资源、密钥或服务器地址：游戏文件、解密所需的密钥和服务器地址都由使用者在配置中自行提供，导出结果只保存在使用者指定的本地目录。 @@ -22,30 +22,31 @@ nnnotes config check # 每项设置的来源和格式是否有效， | 命令 | 输入 | 输出 | |---|---|---| -| `catalog` | 区服 catalog | 列出 Addressables 资源键（可按前缀过滤） | -| `browse` | 已配置的各区服 catalog | 本地网页浏览 catalog 与资源包（`--host` / `--port`） | -| `pull` | 资源键 | 下载并解密该键依赖闭包中的全部资源包到本地缓存 | +| `catalog keys` | 区服 catalog | 列出 Addressables 资源键（可按前缀过滤） | +| `catalog browse` | 已配置的各区服 catalog | 本地网页浏览 catalog 与资源包（`--host` / `--port`） | +| `bundle pull` | 资源键 | 下载并解密该键依赖闭包中的全部资源包到本地缓存 | | `servers` | 引导 API 根地址 `[bootstrap] api` | 游戏 API 的服务器列表：各区服名称、区域 ID 与 CDN / API 根地址（地址仅在 `--show-hosts` 时输出） | | `master version` | 区服 | 该区服当前的 masterdata 版本与资源版本（游戏 API 匿名调用） | | `master download` | masterdata 版本号，或 `--latest`（区服当前版本） | 该版本的 `MasterManifest.json` 与全部 `.bin` 文件（SHA-256 校验） | | `master decode` | masterdata `.bin` 文件或目录 | 每张表一个 JSON（Rijndael-256 CBC 解密 + gzip 解压） | -| `adv` | 剧情 ID | `episode.json`：命令表、五语台词、语音 / 音效 / 视频索引 | -| `story` | 剧情 ID | 剧情目录：episode、音频、舞台场景与着色器、剧情 UI、Frame / 粒子特效 / 后处理 / 静帧 / 对话框 / 聊天资源、视频（WebM）；用到的 Live2D 模型导出到模型目录（`--models`，默认为剧情目录旁的 `live2d/`），已导出的模型供其他剧情复用 | -| `live2d` | 模型资源键或模型 ID | Live2D（Cubism）运行时目录：moc3、贴图、motion3、物理、表情、预制体参数 | -| `spot` | 据点 ID | `spot.json` + Spine 角色 + 房间 `room.glb` + 着色器 | -| `room` | 背景预制体键 | 房间模型（binary glTF） | -| `shader` | 资源键或 APK 内资源包 | 着色器各平台变体（GLSL ES 等）与索引 | -| `audio` | CRI cue sheet | 每个 cue 一个音频文件（FLAC / Ogg / WAV）+ cue 元数据 | -| `crikey` | APK | 读出游戏启动数据中的 CRI HCA 解码密钥（只显示是否找到，可写成 `.hcakey`） | -| `player` | APK | 渲染相关的全局设置（色彩空间、画质等级、渲染器）JSON | +| `export episode` | 剧情 ID | `episode.json`：命令表、五语台词、语音 / 音效 / 视频索引 | +| `export story` | 剧情 ID | 剧情目录：episode、音频、舞台场景与着色器、剧情 UI、Frame / 粒子特效 / 后处理 / 静帧 / 对话框 / 聊天资源、视频（WebM）；用到的 Live2D 模型导出到模型目录（`--models`，默认为剧情目录旁的 `live2d/`），已导出的模型供其他剧情复用 | +| `export live2d` | 模型资源键或模型 ID | Live2D（Cubism）运行时目录：moc3、贴图、motion3、物理、表情、预制体参数 | +| `export spot` | 据点 ID | `spot.json` + Spine 角色 + 房间 `room.glb` + 着色器 | +| `export room` | 背景预制体键 | 房间模型（binary glTF） | +| `export shader` | 资源键或 APK 内资源包 | 着色器各平台变体（GLSL ES 等）与索引 | +| `export audio` | CRI cue sheet | 每个 cue 一个音频文件（FLAC / Ogg / WAV）+ cue 元数据 | +| `apk hca-key` | APK | 读出游戏启动数据中的 CRI HCA 解码密钥（只显示是否找到，可写成 `.hcakey`） | +| `apk settings` | APK | 渲染相关的全局设置（色彩空间、画质等级、渲染器）JSON | | `ui`（实验性） | APK 集合及内嵌 catalog | 离线序列化预制体库，包含依赖根对象、AnimatorController、稳定引用与 PNG／字体资源，供 ournotes-player 可选 UI 预览使用（[格式与边界](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/ui.md)） | -| `live` | 曲目 ID + 难度 | 完整谱面目录：谱面与运行时音符、3D 场景、音符与特效资源、BGM 与音效、声音路由 | -| `web` | `--pair 曲目:难度`（可重复）或 `--all`；`--live2d 模型`（可重复）或 `--all-live2d`；`--story 剧情 ID`（可重复）或 `--all-stories`；`--region 区服`（可重复）或 `--all-regions` | ournotes-player 静态站点：共享播放器 + 每谱 / 每模型 / 每集剧情清单 + 内容寻址资源；剧情用到的 Live2D 模型先按模型构建并列入 `models.json`，剧情清单引用它们；一个站点可服务多个区服，列表文本含五种语言；剧情的界面文字按语言分组，字形由开源字体生成 TextMeshPro 字体资源（`--fonts game` 时用游戏字体）；可压缩的资源（JSON、着色器、moc3 等）默认以 gzip 存储，`--compress br` / `none` 可改 | +| `export live` | 曲目 ID + 难度 | 完整谱面目录：谱面与运行时音符、3D 场景、音符与特效资源、BGM 与音效、声音路由 | +| `site build` | `--pair 曲目:难度`（可重复）或 `--all`；`--live2d 模型`（可重复）或 `--all-live2d`；`--story 剧情 ID`（可重复）或 `--all-stories`；`--site-region 区服`（可重复）或 `--all-regions` | ournotes-player 静态站点：共享播放器 + 每谱 / 每模型 / 每集剧情清单 + 内容寻址资源；剧情用到的 Live2D 模型先按模型构建并列入 `models.json`，剧情清单引用它们；一个站点可服务多个区服，列表文本含五种语言；剧情的界面文字按语言分组，字形由开源字体生成 TextMeshPro 字体资源（`--fonts game` 时用游戏字体）；可压缩的资源（JSON、着色器、moc3 等）默认以 gzip 存储，`--compress br` / `none` 可改 | | `music-data` | masterdata 文件（`--master-files` 目录或 `--apk-master`），或带清单的解码 masterdata（`--decoded-master`） | 全部歌曲与谱面的单个 JSON：五语标题与作词作曲编曲、乐队、演唱角色、分类、标签、上线时间、评级线、BGM 时长；激走目录（成员卡、小卡及其激走技能与激走支援技能）；每个难度的等级、音符数、BPM、谱面时间、技能事件与 fever 区间；以及组卡模型 ournotes-deck（`pip install 'nnnotes[deck]'` 安装）在整场模拟上实测的谱面统计（无技能得分、每种加分技能在每个演出位的权重），以及单个激走技能形状的谱面适性（增量与标准误，不选最佳编成；`--no-gekisou-aptitude` 可跳过）。`--full` 另附组卡模型的输入：每张谱面的运行时音符与卡牌、技能、加成、分数、活动相关的 masterdata 表（[格式](docs/music-data.md)） | 导出约定： -- 写文件的命令都用 `-o` 指定输出路径（必填）；`web` 的站点目录是位置参数。 +- 写文件的命令都用 `-o` 指定输出路径（必填）。 +- 命令默认输出简短摘要，加 `--json` 时输出 JSON 报告。 - JSON 一律 UTF-8、LF 换行；无穷大写作 `1e999`。 - 在同一套环境下导出是确定的：同样的输入，在 nnnotes、Python 依赖库（UnityPy、Pillow、numpy 等）和外部工具（vgmstream、FFmpeg）版本都相同时，导出文件逐字节相同。版本不同时，相同内容可能写成不同的字节：例如不同版本的 Pillow 写出的 PNG 像素相同，字节可能不同。 - 数值保留 Unity 序列化值与字段名（`m_LocalPosition`、`_bandIDs` 等），便于与游戏数据对照。 @@ -62,21 +63,21 @@ nnnotes config check # 每项设置的来源和格式是否有效， |---|---| | catalog / 资源包解密 / 依赖闭包 | 可用 | | masterdata 解码 | 可用 | -| 剧情 `adv` | 946 / 946 集可导出 | -| 剧情 `story`（完整目录） | 946 / 946 集的资源都在 catalog 中且类型均受支持（资源闭包与游戏自带的每集下载清单一致）；946 / 946 集已逐集导出验证：输出中引用的文件、模型与音频均存在，JSON 均可解析，引用的路径均可解析；其中 235 集用到 Frame / Effect / PostEffect / Still / Chat / TalkWindow / 视频资源，两次导出除模型目录路径外逐字节一致 | +| 剧情 `export episode` | 946 / 946 集可导出 | +| 剧情 `export story`（完整目录） | 946 / 946 集的资源都在 catalog 中且类型均受支持（资源闭包与游戏自带的每集下载清单一致）；946 / 946 集已逐集导出验证：输出中引用的文件、模型与音频均存在，JSON 均可解析，引用的路径均可解析；其中 235 集用到 Frame / Effect / PostEffect / Still / Chat / TalkWindow / 视频资源，两次导出除模型目录路径外逐字节一致 | | Live2D 模型 | 239 / 239 个可导出（catalog 中的全部模型，剧情用到其中 185 个） | | CRI 音频 | 681 / 681 个 cue sheet 可解码 | -| 谱面 `live` | 336 / 336 个（曲目, 难度）组合可导出 | -| 网页站点 `web` | 336 / 336 张谱面，239 / 239 个 Live2D 模型，946 / 946 集剧情（英语、无音频的全量构建两次结果逐字节一致，946 / 946 集通过数据校验）；三个区服、五种语言、含 AAC 音频的全量构建（`--all-regions`，用线上 masterdata：85 首曲目的 340 张谱面）两次结果同样逐字节一致，全部通过数据校验 | -| 据点 `spot` / `room` | 39 / 39 个据点可导出，引用均可解析；catalog 中 20 / 20 个房间背景可由 `room` 导出。生成的 glb 均通过 Khronos glTF-Validator，0 个错误 | +| 谱面 `export live` | 336 / 336 个（曲目, 难度）组合可导出 | +| 网页站点 `site build` | 336 / 336 张谱面，239 / 239 个 Live2D 模型，946 / 946 集剧情（英语、无音频的全量构建两次结果逐字节一致，946 / 946 集通过数据校验）；三个区服、五种语言、含 AAC 音频的全量构建（`--all-regions`，用线上 masterdata：85 首曲目的 340 张谱面）两次结果同样逐字节一致，全部通过数据校验 | +| 据点 `export spot` / `export room` | 39 / 39 个据点可导出，引用均可解析；catalog 中 20 / 20 个房间背景可由 `export room` 导出。生成的 glb 均通过 Khronos glTF-Validator，0 个错误 | | 其他区服（en / kr）与其他语言 | 已核对：各区服同一语言的 catalog 相同、资源包相同，密钥通用；谱面用到的 masterdata 表在三个区服间一致，文本表五种语言齐全；抽查的谱面导出与台服一致。多区服整站已全量构建验证（见上一行） | ## 需要准备 -- Python 3.11+，推荐 3.13（`web` 构建读写大量 JSON，3.13 的标准库 JSON 编码更快）；开发时在仓库内 `pip install -e .` -- 游戏安装包 `base.apk`：APK 内置资源包、CRI 解码密钥、启动设置。`player`、`story`、`live`、`web` 用 nnnotes 自带的类型树读取 APK 启动数据中的 MonoBehaviour，目前支持游戏版本 1.0.1（Unity 6000.3.12f1）；其他版本的 APK 若类型不符，这些命令会报错并给出类名、游戏版本和 Unity 版本 -- 解码后的 masterdata 目录（可用 `master download` + `master decode` 生成）：`adv`、`story`、`spot`、`live` 和 `web` 的谱面需要；`web` 的 Live2D 模型只用它取角色名（可选） -- 外部工具：[vgmstream](https://vgmstream.org/)（CRI HCA 解码）、[FFmpeg](https://ffmpeg.org/)（转码，含剧情视频的 WebM 封装与 Opus 音频）；`web` 另需 Node.js 20+ 与构建好的 ournotes-player +- Python 3.11+，推荐 3.13（`site build` 构建读写大量 JSON，3.13 的标准库 JSON 编码更快）；开发时在仓库内 `pip install -e .` +- 游戏安装包 `base.apk`：APK 内置资源包、CRI 解码密钥、启动设置。`apk settings`、`export story`、`export live`、`site build` 用 nnnotes 自带的类型树读取 APK 启动数据中的 MonoBehaviour，目前支持游戏版本 1.0.1（Unity 6000.3.12f1）；其他版本的 APK 若类型不符，这些命令会报错并给出类名、游戏版本和 Unity 版本 +- 解码后的 masterdata 目录（可用 `master download` + `master decode` 生成）：`export episode`、`export story`、`export spot`、`export live` 和 `site build` 的谱面需要；`site build` 的 Live2D 模型只用它取角色名（可选） +- 外部工具：[vgmstream](https://vgmstream.org/)（CRI HCA 解码）、[FFmpeg](https://ffmpeg.org/)（转码，含剧情视频的 WebM 封装与 Opus 音频）；`site build` 另需 Node.js 20+ 与构建好的 ournotes-player ## 配置 @@ -84,7 +85,7 @@ nnnotes config check # 每项设置的来源和格式是否有效， 1. 配置文件：`--config <文件>`，否则 `NNNOTES_CONFIG`，否则当前目录的 `nnnotes.toml`，否则用户配置文件（Windows 为 `%APPDATA%\nnnotes\nnnotes.toml`，其他系统为 `~/.config/nnnotes/nnnotes.toml`；`nnnotes config path` 列出查找顺序与实际读取的文件） 2. 环境变量：`NNNOTES_<节>_<键>`（如 `NNNOTES_BUNDLE_KEY`、`NNNOTES_SERVERS_TW_CDN`） -3. 命令行参数：`--region`、`--language`、`--catalog`、`--cache`、`--master`、`--apk`、`--ffmpeg`、`--vgmstream`、`--node` 写在命令名之前；`--player` 是 `web` 的参数 +3. 命令行参数：`--region`、`--language`、`--catalog`、`--cache`、`--master`、`--apk`、`--ffmpeg`、`--vgmstream`、`--node` 写在命令名之前或之后；`--player` 是 `site build` 的参数 `nnnotes config init` 写出配置模板（每项设置为空，模板随包安装，仓库中为 [`src/nnnotes/source/nnnotes.example.toml`](src/nnnotes/source/nnnotes.example.toml)），在终端中会逐项询问；`config set` / `config unset` 修改单项并保留文件中的注释。需要的设置包括：资源包解密密钥与 nonce 种子、masterdata 的密钥与 IV、使用的区服 `[catalog] region` 与 catalog 语言 `[catalog] language`、每个区服的 CDN 地址与 API 根地址（`[servers.<区服>]` 的 `cdn`、`api`）、客户端版本 `[client] version`（未设置时读取 APK 的 versionName）、可选的引导 API 根地址 `[bootstrap] api`（`servers` 使用），以及缓存目录、APK、masterdata 目录、ournotes-player 和 vgmstream / FFmpeg / Node.js 的路径（三个工具未设置时在 `PATH` 中查找）。这些值都来自使用者自己的游戏客户端。 @@ -95,28 +96,28 @@ nnnotes config check # 每项设置的来源和格式是否有效， ## 使用示例 ```bash -nnnotes catalog --prefix Live/MusicScore/ --limit 20 -nnnotes browse --port 8000 -nnnotes pull Live/MusicScore/0001/0001_03 +nnnotes catalog keys --prefix Live/MusicScore/ --limit 20 +nnnotes catalog browse --port 8000 +nnnotes bundle pull Live/MusicScore/0001/0001_03 nnnotes master version nnnotes master download --latest -o work/master-bin nnnotes master decode work/master-bin -o work/master -nnnotes --master work/master adv 10462 -o out/adv_10462.json -nnnotes story 10462 -o out/story_10462 -nnnotes live 100001 --difficulty expert -o out/live_100001 -nnnotes web out/site --all --player --workers 5 -nnnotes web out/site --pair 100001:expert --pair 100001:hard --player -nnnotes web out/site --all --region tw --region en --region kr --player -nnnotes web out/site --live2d adv_live2d_rana_003_casual_spring_01 --player +nnnotes --master work/master export episode 10462 -o out/adv_10462.json +nnnotes export story 10462 -o out/story_10462 +nnnotes export live 100001 --difficulty expert -o out/live_100001 +nnnotes site build -o out/site --all --player --workers 5 +nnnotes site build -o out/site --pair 100001:expert --pair 100001:hard --player +nnnotes site build -o out/site --all --site-region tw --site-region en --site-region kr --player +nnnotes site build -o out/site --live2d adv_live2d_rana_003_casual_spring_01 --player ``` -`web` 可以增量构建：清单已存在的谱面和模型会跳过（`--force` 重建），不再被引用的资源会被清理。常用参数： +`site build` 可以增量构建：清单已存在的谱面和模型会跳过（`--force` 重建），不再被引用的资源会被清理。常用参数： - `--format aac|opus|vorbis|mp3|flac`：BGM 格式，默认 AAC；`--no-audio`：不导出音频 - `--band` / `--leader-card`：轻量背景与开场时间轴所用乐队；默认取曲目第一位演唱角色的乐队 - `--workers`：并行处理的曲目进程数（默认为 CPU 数的四分之一，最多 8）与模型进程数（默认最多 4）；`--read-workers`：同时运行的谱面读取集（Node.js 进程）数（默认为 CPU 数的一半，最多 16）；`--tmp`：临时构建目录（默认 `<站点>.tmp`） - `--live2d 模型`（模型 ID 或资源键，可重复）/ `--all-live2d`：加入 Live2D 模型（catalog 中全部模型），可与谱面在同一次构建中加入 -- `--region 区服`（可重复）/ `--all-regions`：站点服务的区服（默认 `[catalog] region`）；每个区服的 masterdata 由 `[servers.<区服>] master` 指定。谱面数据相同的区服共用一份清单，列表页用 `?region=&lang=` 切换区服与语言 +- `--site-region 区服`（可重复）/ `--all-regions`：站点服务的区服（默认 `[catalog] region`）；每个区服的 masterdata 由 `[servers.<区服>] master` 指定。谱面数据相同的区服共用一份清单，列表页用 `?region=&lang=` 切换区服与语言 - `--player-only`：只重写播放器文件与 `charts.json`、`models.json`；`--reingest-json`：按当前规则重新存储所有谱面与模型的 JSON 各命令的参数与输出目录结构见 [docs/commands.md](docs/commands.md)。 diff --git a/docs/assets.md b/docs/assets.md index e71c860..4e089bf 100644 --- a/docs/assets.md +++ b/docs/assets.md @@ -1,6 +1,6 @@ # Asset export -`nnnotes export` writes every object of the selected bundles in a common format: textures and sprites as PNG, +`nnnotes export assets` writes every object of the selected bundles in a common format: textures and sprites as PNG, meshes as binary glTF, text assets and fonts as their stored bytes, prefabs, materials, animations and every other class as JSON. It also writes the semantic views of the master data (cards, characters, stamps, …) with the files of the objects they name. The work is split into stages ([stages.md](stages.md)); each stage runs one task per subject @@ -9,28 +9,28 @@ the objects they name. The work is split into stages ([stages.md](stages.md)); e rewrites what changed. ``` -nnnotes export -o out/assets -nnnotes export -o out/cards --select group:membercard --views cards -nnnotes plan --select key:Image/Jacket/ -nnnotes plan --check && echo "up to date" +nnnotes export assets -o out/assets +nnnotes export assets -o out/cards --select group:membercard --views cards +nnnotes store plan --select key:Image/Jacket/ +nnnotes store plan --check && echo "up to date" ``` ## Commands ``` -nnnotes export -o OUT [--layout original[,cas]] [--select group:G|key:PREFIX|bundle:GLOB ...] [--views all|none|a,b] - [--store DIR] [--workers N] [--memory GiB] [--fetch-workers N] [--catalog-version L|SHA] +nnnotes export assets -o OUT [--layout original[,cas]] [--select group:G|key:PREFIX|bundle:GLOB ...] [--views all|none|a,b] + [--store DIR] [--workers N] [--memory GiB] [--fetch-workers N] [--stored-catalog L|SHA] [--png-level N] [--flac-level N] [--only-class C,...] [--link auto|clone|hard|copy] [--strict] [--dry-run] [--explain] -nnnotes plan [the selection and parameters of export] [-o OUT] [--json] [--since RUN] [--check] [--census] +nnnotes store plan [the selection and parameters of export] [-o OUT] [--json] [--since RUN] [--check] [--census] [--emit-tasks DIR] [--why TASK] -nnnotes run-stage TASK.json|DIR [...] [--store DIR] [--fetch] [--force] -nnnotes catalogs list [--json] | import FILE [--label L] [--apk-catalog FILE] [--resource-version V] | fetch [--label L] +nnnotes store run TASK.json|DIR [...] [--store DIR] [--fetch] [--force] +nnnotes catalog versions [--json] | import FILE [--label L] [--apk-catalog FILE] [--resource-version V] | fetch [--label L] | diff OLD NEW [--json] -nnnotes store verify [--quick] +nnnotes store verify [--quick] [--workers N] ``` -### export +### export assets | Flag | Meaning | |---|---| @@ -42,7 +42,7 @@ nnnotes store verify [--quick] | `--workers N` | worker processes (default: the CPUs the process may run on, its affinity mask as `taskset` or a container's cpuset sets it; `0`: every task in this process) | | `--memory GiB` | memory budget (default: 80 % of the physical memory where it can be read, else none) | | `--fetch-workers N` | parallel downloads of bundles not in the cache (default 8) | -| `--catalog-version` | an imported catalog version (`catalogs list`) instead of the current catalog | +| `--stored-catalog` | a catalog version of the store (`catalog versions`) instead of the current catalog | | `--png-level N` | zlib level 0-9 of the PNG files (default 6); part of the task keys | | `--flac-level N` | FLAC compression level 0-12 of the CRI audio and the movies' audio (default 8; every level decodes to the same samples; 12 writes files 0.2-1.3 % smaller in about five times the encoding time); part of the task keys | | `--only-class C,...` | export only objects of these Unity classes; part of the task keys | @@ -73,9 +73,9 @@ A failed object is reported with its task's result and stays failed until someth fails as a whole (an exception, an input that cannot be read, a worker that dies) leaves no result and runs again next time. -### plan +### store plan -What `export` with the same flags would do, without running or fetching anything: every task with its status +What `export assets` with the same flags would do, without running or fetching anything: every task with its status (`hit`: its result is stored, `run`, `unknown`: it reads the result of a task that would run first, or of a bundle not fetched yet), the reasons it runs (compared with the latest run of the same selection, or `--since RUN`), the estimated CPU time and peak memory, and with `-o` the files each layout would write, remove and keep. The text @@ -94,15 +94,15 @@ Reasons: `new-input`, `input-changed`, `stage-version`, `atom-changed`, `params- another run); unknown tasks are `pending` on a task, or on `fetch:` for a bundle whose content is not known yet. See [contracts.md](contracts.md#plan-nnnotesplan1). -### run-stage +### store run Runs task descriptions (`nnnotes.task/1`, as `plan --emit-tasks` writes them) into a store: each is checked (its key is recomputed, its stage must be installed at its version), its inputs are read by content id through their -locators (the store, the cache, a file, or with `--fetch` the catalog), and its result is stored exactly as `export` +locators (the store, the cache, a file, or with `--fetch` the catalog), and its result is stored exactly as `export assets` would store it. Several tasks run one after another in one process; stages that do not read bundles never load UnityPy. `--force` runs tasks whose result is stored again (the new result must be identical). -### catalogs +### catalog versions, import, fetch, diff The catalog versions of a store: `import` a catalog file (with the APK's catalog from `[paths] apk` or `--apk-catalog`), `fetch` the current catalog of the configured region and language, `list` them, and `diff` two of @@ -112,12 +112,13 @@ stable address (an object keeps its stable address while its object id changes w is labelled by `--label`, else by its resource version, else by the first 12 hex digits of its sha256. `fetch` records the resource version it downloads (none for `main`); `import` takes `--resource-version`, else the JP catalog's source version, else the version in a file name `catalog__.bin` as `fetch` and the -cache name them (none for `main`). `export` imports the current catalog when it is not imported yet. +cache name them (none for `main`). `export assets` imports the current catalog when it is not imported yet. ### store verify Checks that every stored content has the sha256 its name says (`--quick`: sizes only) and that every result names -its own key, has the key of its parts and names stored contents of the right size. Exit 1 when a problem is found. +its own key, has the key of its parts and names stored contents of the right size. Exit 1 when a problem is found. `--workers` hashes in +parallel (default: the CPUs the process may use; `0`: one at a time). ## Layouts @@ -183,9 +184,9 @@ and the reports directory. | Command | 0 | 1 | 2 | 3 | 4 | |---|---|---|---|---|---| -| `export` | complete (unsupported objects allowed) | a task, an object or a download failed (reports written); with `--strict` also view gaps | usage or settings | aborted: a setting or tool a task needs is missing, or interrupted; results written so far stay valid | | -| `plan` | | with `--check`: something would run or change | usage or settings | | | -| `run-stage` | every task ran or hit | a task failed or its result is partial | usage or settings | | a task is incompatible (edited, unknown stage or version); nothing of the batch runs | +| `export assets` | complete (unsupported objects allowed) | a task, an object or a download failed (reports written); with `--strict` also view gaps | usage or settings | aborted: a setting or tool a task needs is missing, or interrupted; results written so far stay valid | | +| `store plan` | | with `--check`: something would run or change | usage or settings | | | +| `store run` | every task ran or hit | a task failed or its result is partial | usage or settings | | a task is incompatible (edited, unknown stage or version); nothing of the batch runs | | `store verify` | no problem | problems found | usage or settings | | | ## Running tasks elsewhere @@ -193,22 +194,22 @@ and the reports directory. Task descriptions carry everything a task needs; any machine with the same nnnotes version and access to the store can run them. A task can only be described once the content of its inputs is known, so the bundles are identified first (`plan --census` fetches and hashes them and takes their census). Then plan the tasks that can run now, run -them in batches on as many workers as you like, and plan again until nothing is left; `export` finally writes the +them in batches on as many workers as you like, and plan again until nothing is left; `export assets` finally writes the layouts and reports (every task is a hit by then). ```sh store=/data/nnnotes-store -nnnotes plan --store "$store" --census > /dev/null +nnnotes store plan --store "$store" --census > /dev/null for round in 1 2 3 4 5 6 7 8; do rm -rf tasks - nnnotes plan --store "$store" --emit-tasks tasks > /dev/null + nnnotes store plan --store "$store" --emit-tasks tasks > /dev/null ls tasks/*.json > /dev/null 2>&1 || break # nothing left to run - ls tasks/*.json | xargs -n 50 -P 16 nnnotes run-stage --store "$store" --fetch + ls tasks/*.json | xargs -n 50 -P 16 nnnotes store run --store "$store" --fetch done -nnnotes export --store "$store" -o out/assets +nnnotes export assets --store "$store" -o out/assets ``` Each round runs the tasks whose inputs are known: the catalog index and the censuses, then the script and address tables, then the bundle exports, then the stages that read their results. A task whose result is already stored is a hit, so a batch can be run twice; a task that keeps failing is planned again in every round (the loop above stops -after eight) and reported by `export`. +after eight) and reported by `export assets`. diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 0000000..3683c4b --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,465 @@ +# Command reference + +Every command of `nnnotes`, its arguments and their defaults. The global options go before or after the command +name. A command prints a short summary on standard output, or with `--json` its report as JSON; notes and errors go +to standard error. Exit status 2 is a usage or settings error. + +## Global options + +| Argument | Description | +| --- | --- | +| `--config` `FILE` | TOML config file (else NNNOTES_CONFIG, else ./nnnotes.toml) | +| `--region` `REGION` | region: a [servers.] table ([catalog] region) | +| `--language` `LANG` | catalog and client language: ja, en, zh-Hant, zh-Hans or ko ([catalog] language) | +| `--catalog` `FILE` | catalog .bin file ([paths] catalog; else downloaded into the cache) | +| `--catalog-version` `VERSION` | the resource version of an international catalog ([catalog] version; else the version the game API serves when an API root is configured, otherwise main) | +| `--cache` `DIR` | cache directory ([paths] cache) | +| `--store` `DIR` | store directory ([paths] store; default /store) | +| `--master` `DIR` | decoded master data directory ([paths] master) | +| `--apk` `FILE` | base.apk ([paths] apk) | +| `--ffmpeg` `EXE` | ffmpeg executable ([paths] ffmpeg; else on PATH) | +| `--vgmstream` `EXE` | vgmstream-cli executable ([paths] vgmstream; else on PATH) | +| `--node` `EXE` | Node.js executable ([paths] node; else on PATH) | +| `--json` | print the command's report as JSON on standard output (default: a short summary) | + +## nnnotes config init + +write a config file from the template: the values are asked in a terminal, else taken from --set (the other settings empty) + +| Argument | Description | +| --- | --- | +| `--file` `FILE` | the config file to write (default: the file --config or NNNOTES_CONFIG names, else ./nnnotes.toml) | +| `--user` | write the per-user config file, read from any directory (`config path` shows it) | +| `--set` `SECTION.KEY=VALUE` | a value to write (repeatable; no questions then): VALUE - reads it from standard input; a list comma-separated; a path is written absolute | +| `--no-input` | ask nothing, also in a terminal | +| `--force` | overwrite an existing file | + +At most one of `--file`, `--user`. + +## nnnotes config set + +set one value of the config file the commands read (or of --file / --user; a missing one is made from the template); comments and other lines stay + +| Argument | Description | +| --- | --- | +| `SECTION.KEY` | the setting: bundle.key, servers.tw.cdn, paths.fonts.ja, ... | +| `value` | the value (a list comma-separated); - reads it from standard input, without echo for the keys in a terminal | +| `--file` `FILE` | the config file to edit | +| `--user` | edit the per-user config file, read from any directory (`config path` shows it) | + +At most one of `--file`, `--user`. + +## nnnotes config unset + +empty one value of the config file (an empty value counts as unset) + +| Argument | Description | +| --- | --- | +| `SECTION.KEY` | the setting | +| `--file` `FILE` | the config file to edit | +| `--user` | edit the per-user config file, read from any directory (`config path` shows it) | + +At most one of `--file`, `--user`. + +## nnnotes config check + +every setting: where it comes from and whether it is valid, never its value; exit status 1 when one is invalid, not found or unknown (--json: nnnotes.config-check/1) + +## nnnotes config path + +the config files looked up, in order, and the one the commands read (--json: nnnotes.config-path/1) + +## nnnotes servers + +the server list of the bootstrap API root: regions, their CDN and API roots + +| Argument | Description | +| --- | --- | +| `--show-hosts` | print the CDN and API roots, not only their counts | + +## nnnotes catalog keys + +the addressable keys of the catalog (the total on standard error) + +| Argument | Description | +| --- | --- | +| `--prefix` `PREFIX` | only the keys that start with this prefix (default ``) | +| `--limit` `LIMIT` | at most N keys (default 200; 0: all) (default `200`) | + +## nnnotes catalog browse + +browse the configured regions' catalogs and bundles in a local web page + +| Argument | Description | +| --- | --- | +| `--port` `PORT` | port (default 8000) (default `8000`) | +| `--host` `HOST` | address to listen on (default 127.0.0.1) (default `127.0.0.1`) | + +## nnnotes catalog versions + +the catalog versions imported into the store (--json: nnnotes.catalogs/1) + +## nnnotes catalog import + +import a catalog file (with the APK's catalog when [paths] apk is set) + +| Argument | Description | +| --- | --- | +| `file` | catalog__.bin | +| `--label` `LABEL` | label of the version (default: its resource version, else the sha256 prefix) | +| `--apk-catalog` `APK_CATALOG` | the APK's catalog.bin (default: read from [paths] apk) | +| `--resource-version` `RESOURCE_VERSION` | the game's resource version of this catalog (default: the JP source's version, else the version in the file name catalog__.bin) | + +## nnnotes catalog fetch + +fetch the region's current catalog and import it into the store + +| Argument | Description | +| --- | --- | +| `--label` `LABEL` | label of the version (default: the resource version when an API root is set) | + +## nnnotes catalog diff + +keys, bundles, raw files (and objects) that differ between two catalog versions of the store (--json: nnnotes.catalog-diff/1) + +| Argument | Description | +| --- | --- | +| `old` | label, version id or sha256 prefix | +| `new` | label, version id or sha256 prefix | + +## nnnotes bundle pull + +fetch the bundle closure of keys into the cache (prints the cached bundle files) + +| Argument | Description | +| --- | --- | +| `KEY` ... | addressable keys (`catalog keys` lists them) | + +## nnnotes master version + +the master data and resource version the region serves now (game API) + +## nnnotes master decode + +master data .bin files -> .json + +| Argument | Description | +| --- | --- | +| `inputs` ... | directories (their *.bin files) or files | +| `-o`, `--out` `OUT` | output directory (required) | +| `--workers` `N` | parallel decodes (default the CPUs this process may use; 0: one at a time) | + +## nnnotes master download + +a master data version from the region's CDN (SHA-256 checked) + +| Argument | Description | +| --- | --- | +| `--version` `VERSION` | master data version | +| `--latest` | the version the region serves now (`master version`) | +| `-o`, `--out` `OUT` | directory for MasterManifest.json and the .bin files (required) | +| `--fetch-workers` `N` | parallel downloads (default 16) | + +Exactly one of `--version`, `--latest`. + +## nnnotes export assets + +every object of the selected bundles -> common formats (the store and its layouts) + +| Argument | Description | +| --- | --- | +| `-o`, `--out` `OUT` | output directory (required) | +| `--select` `SEL` | group:, key: or bundle: (repeatable; default: everything) | +| `--views` `all\|none\|a,b` | semantic views to build (default: all when master data is set, else none) | +| `--stored-catalog` `LABEL\|SHA` | a catalog version of the store (`catalog versions`; default: the current catalog) | +| `--png-level` `N` | PNG compression level 0-9 (changes the output) | +| `--flac-level` `N` | FLAC compression level 0-12 of the CRI audio (default 8; every level decodes to the same samples; changes the files) | +| `--only-class` `C,...` | export only objects of these classes | +| `--workers` `N` | worker processes (default: the CPUs this process may use; 0: this process) | +| `--memory` `GiB` | memory budget of the running tasks and workers (default: 80 % of the physical memory) | +| `--fetch-workers` `N` | parallel downloads (default 8) | +| `--layout` `LAYOUT` | original, cas or original,cas (default original) (default `original`) | +| `--explain` | print every setting of the run first | +| `--link` `LINK` | how the layout files are made from the store: auto (default; [export] link): a clone where the file system makes them, else a hard link (read-only), else a copy; or that method only (one of `auto`, `clone`, `hard`, `copy`) | +| `--strict` | exit 1 when a view has gaps | +| `--dry-run` | print the plan, run nothing | + +## nnnotes export episode + +an ADV episode -> JSON + +| Argument | Description | +| --- | --- | +| `adv_id` | MasterAdv id of the episode | +| `-o`, `--out` `OUT` | output .json file (required) | + +## nnnotes export story + +an ADV episode -> story directory (episode, audio, scene, UI, media, videos) + its Live2D models + +| Argument | Description | +| --- | --- | +| `adv_id` | MasterAdv id of the episode | +| `-o`, `--out` `OUT` | output directory (required) | +| `--models` `DIR` | directory of the Live2D models, one / each (default: OUT/../live2d) | +| `--force` | export the models again whose directory exists | +| `--format` `FORMAT` | audio file format (default flac) (one of `flac`, `ogg`, `wav`; default `flac`) | +| `--flac-level` `N` | FLAC compression level 0-12 of --format flac (default 8; every level decodes to the same samples) | +| `--no-audio` | do not decode the cue sheets | +| `--fonts` `FONTS` | open (default): text layout and style only, no font data; game: also the game's TMP fonts (needs the 'fonts' extra) (one of `open`, `game`; default `open`) | + +## nnnotes export live + +a live (music + difficulty) -> self-contained live directory + +| Argument | Description | +| --- | --- | +| `music_id` | MasterLiveMusic id | +| `--difficulty` `DIFFICULTY` | chart difficulty (default expert) (one of `easy`, `normal`, `hard`, `expert`; default `expert`) | +| `--format` `FORMAT` | audio file format (default flac) (one of `flac`, `ogg`, `wav`; default `flac`) | +| `--flac-level` `N` | FLAC compression level 0-12 of --format flac (default 8; every level decodes to the same samples) | +| `--fonts` `FONTS` | start canvas text: open (default): layout and style only, no font data (your fonts are not used); game: also the game's TMP fonts (needs the 'fonts' extra) (one of `open`, `game`; default `open`) | +| `--band` `BAND` | band of the LightWeight background and start timeline (default: band of the music's first vocal character) | +| `--leader-card` `LEADER_CARD` | deck centre MasterMemberCard id (its character's band) | +| `--live-option` `OPTION[=VALUES]` | also export the files of a Live option's variants (repeatable): MirrorChart, MeasureLineDisplay, NoteDesignId, NoteEffectId, LiveQuality, NoteSePatternId (every value of the master data, or =v,v...), defaults (the option defaults and ranges only) or all | +| `-o`, `--out` `OUT` | output directory (required) | + +At most one of `--band`, `--leader-card`. + +## nnnotes export live2d + +a Live2D model -> runtime model directory + +| Argument | Description | +| --- | --- | +| `key` | model key Character/Live2D///model/, or its model id | +| `-o`, `--out` `OUT` | output directory (required) | + +## nnnotes export spot + +a spot -> spot.json + Spine + room.glb + shaders + +| Argument | Description | +| --- | --- | +| `spot_id` | MasterHomeSpot id | +| `-o`, `--out` `OUT` | output directory (required) | + +## nnnotes export room + +a background prefab -> glb + +| Argument | Description | +| --- | --- | +| `key` | addressable key of the prefab | +| `-o`, `--out` `OUT` | output .glb file (required) | + +## nnnotes export shader + +the shaders of a key's bundle closure or of APK bundles + +| Argument | Description | +| --- | --- | +| `--key` `KEY` | addressable key: the shaders of its bundle closure | +| `--apk-bundle` `APK_BUNDLE` ... | substring(s) of APK bundle file names | +| `-o`, `--out` `OUT` | output directory (required) | + +Exactly one of `--key`, `--apk-bundle`. + +## nnnotes export audio + +a CRI cue sheet -> audio files + +| Argument | Description | +| --- | --- | +| `cue_sheet` | cue sheet name: the key Cri/Sound/ of the catalog | +| `-o`, `--out` `OUT` | output directory (required) | +| `--format` `FORMAT` | audio file format (default flac) (one of `flac`, `ogg`, `wav`; default `flac`) | +| `--flac-level` `N` | FLAC compression level 0-12 of --format flac (default 8; every level decodes to the same samples) | + +## nnnotes export ui + +the APK's UI prefabs -> an offline UI prefab library for ournotes-player/ui + +| Argument | Description | +| --- | --- | +| `-o`, `--out` `OUT` | UI data directory outside the source repositories (required) | +| `--key` `KEY` | exact APK catalog key (repeatable) | +| `--prefix` `PREFIX` | APK UI key prefix (default EmbUI/) (default `EmbUI/`) | +| `--limit` `LIMIT` | maximum number of keys | +| `--no-dependencies` | omit additional prefab roots and controllers | +| `--force` | re-export the selected keys | + +## nnnotes apk settings + +the player's graphics settings (color space, quality levels) -> JSON + +| Argument | Description | +| --- | --- | +| `-o`, `--out` `OUT` | output .json file (required) | + +## nnnotes apk hca-key + +find the HCA keycode of the CRI audio (never printed) + +| Argument | Description | +| --- | --- | +| `--write` `DIR` | write it as DIR/.hcakey (the file vgmstream reads) | + +## nnnotes store plan + +what `export assets` would run and why, without running it (--json: nnnotes.plan/1) + +| Argument | Description | +| --- | --- | +| `-o`, `--out` `OUT` | output directory of the export (counts the layout files to write) | +| `--select` `SEL` | group:, key: or bundle: (repeatable; default: everything) | +| `--views` `all\|none\|a,b` | semantic views to build (default: all when master data is set, else none) | +| `--stored-catalog` `LABEL\|SHA` | a catalog version of the store (`catalog versions`; default: the current catalog) | +| `--png-level` `N` | PNG compression level 0-9 (changes the output) | +| `--flac-level` `N` | FLAC compression level 0-12 of the CRI audio (default 8; every level decodes to the same samples; changes the files) | +| `--only-class` `C,...` | export only objects of these classes | +| `--workers` `N` | worker processes (default: the CPUs this process may use; 0: this process) | +| `--memory` `GiB` | memory budget of the running tasks and workers (default: 80 % of the physical memory) | +| `--fetch-workers` `N` | parallel downloads (default 8) | +| `--layout` `LAYOUT` | original, cas or original,cas (default original) (default `original`) | +| `--explain` | print every setting of the run first | +| `--since` `RUN` | explain against this run (default: the latest of the selection) | +| `--check` | exit 1 when anything would run or change | +| `--census` | first fetch and census the bundles without one | +| `--emit-tasks` `DIR` | write the runnable tasks as task descriptions | +| `--why` `TASK` | the reasons and key parts of one task | + +## nnnotes store run + +run task descriptions (nnnotes.task/1) into the store + +| Argument | Description | +| --- | --- | +| `TASK` ... | task description files, or directories of them | +| `--fetch` | fetch inputs through their catalog locators | +| `--force` | run tasks whose result is stored (it must not change) | + +## nnnotes store verify + +check every stored content and result + +| Argument | Description | +| --- | --- | +| `--quick` | check sizes only, do not hash the contents | +| `--workers` `N` | parallel hashing (default the CPUs this process may use; 0: one at a time) | + +## nnnotes voices list + +the voices matching the filters (the text column in the --language language, default [catalog] language) + +| Argument | Description | +| --- | --- | +| `--from` `OUT` | an export directory with views/voices.json (export assets --views voices); else the index is built from the configured master data and catalog (no files) | +| `--character` `CHARACTER` | a MasterCharacter id or a part of a name in any language | +| `--category` `CATEGORY` | a category (CharacterRankUp or character-rank-up), Source.Category or a source | +| `--source` `SOURCE` | a source (Talk, CharacterVoice, ..., HomeSpot, Title, Sound, Story) | +| `--episode` `ADV` | the Story voices of one episode: a MasterAdv id or an episode asset (without --from: read from the catalog) | +| `--status` `STATUS` | only the rows with this status (one of `ok`, `missing-cue`, `missing-key`, `not-exported`, `unsupported`, `no-value`) | +| `--limit` `LIMIT` | at most N rows (0: all) | + +## nnnotes voices search + +the voices whose text (or cue name) contains TEXT + +| Argument | Description | +| --- | --- | +| `text` | text to look for | +| `--from` `OUT` | an export directory with views/voices.json (export assets --views voices); else the index is built from the configured master data and catalog (no files) | +| `--character` `CHARACTER` | a MasterCharacter id or a part of a name in any language | +| `--category` `CATEGORY` | a category (CharacterRankUp or character-rank-up), Source.Category or a source | +| `--source` `SOURCE` | a source (Talk, CharacterVoice, ..., HomeSpot, Title, Sound, Story) | +| `--episode` `ADV` | the Story voices of one episode: a MasterAdv id or an episode asset (without --from: read from the catalog) | +| `--status` `STATUS` | only the rows with this status (one of `ok`, `missing-cue`, `missing-key`, `not-exported`, `unsupported`, `no-value`) | +| `--limit` `LIMIT` | at most N rows (0: all) | +| `--search-language` `SEARCH_LANGUAGE` | search this language only (default: all five) (one of `ja`, `en`, `zh-Hant`, `zh-Hans`, `ko`) | + +## nnnotes voices summary + +rows per status, category and character; gaps both ways + +| Argument | Description | +| --- | --- | +| `--from` `OUT` | an export directory with views/voices.json (export assets --views voices); else the index is built from the configured master data and catalog (no files) | + +## nnnotes voices get + +the files of one voice: a row id (MasterTalk:561, MasterAdv::) or a sound id + +| Argument | Description | +| --- | --- | +| `id` | row id or sound id | +| `-o`, `--out` `OUT` | output directory (required) | +| `--from` `OUT` | an export directory with views/voices.json (export assets --views voices); else the index is built from the configured master data and catalog (no files) | +| `--format` `FORMAT` | flac (default: the exported files when --from has them), ogg or wav (decodes the sheet) (one of `flac`, `ogg`, `wav`; default `flac`) | +| `--flac-level` `N` | FLAC compression level 0-12 of --format flac: decode the sheet at this level, also with --from (default 8; every level decodes to the same samples) | + +## nnnotes site build + +live charts, Live2D models and stories -> static site for ournotes-player (shared player + per-chart / per-model / per-story data) + +| Argument | Description | +| --- | --- | +| `-o`, `--out` `SITE` | site directory (required) | +| `--player` `PLAYER` | ournotes-player checkout (built) or installed package ([paths] player) | +| `--pair` `PAIR` | : (repeatable) | +| `--all` | every MasterLiveMusic x difficulty | +| `--player-only` | rewrite the player files, charts.json and models.json only | +| `--reingest-json` | store every chart's and model's JSON files again | +| `--live2d` `MODEL` | Live2D model id or key Character/Live2D///model/ (repeatable) | +| `--all-live2d` | every Live2D model of the catalog | +| `--format` `FORMAT` | audio format of the charts and stories (default aac) (one of `aac`, `opus`, `vorbis`, `mp3`, `flac`; default `aac`) | +| `--no-audio` | export no audio files | +| `--compress` `COMPRESS` | encoding of the compressible assets (JSON, shaders, moc3, ...) where it makes them smaller: gzip (default), br (brotli) or none (one of `gzip`, `br`, `none`; default `gzip`) | +| `--force` | rebuild charts, models and stories whose manifest exists (stories: also their models) | +| `--tmp` `TMP` | directory for the temporary live and model builds (default .tmp) | +| `--workers` `N` | parallel music / story / model processes (default a quarter of the CPUs, up to 8; models up to 4; 0: this process) | +| `--read-workers` `N` | chart read sets run at a time (default half the CPUs, up to 16) | +| `--story` `ADV_ID` | story episode: its MasterAdv id (repeatable) | +| `--all-stories` | every story episode (MasterAdv) of the master data | +| `--story-languages` `LANGS` | languages of the stories' text, comma-separated (default: ja,en,zh-Hant,zh-Hans,ko) | +| `--font` `LANG=PATH` | font file the story text of a language is drawn with (repeatable; [paths] fonts.); emoji=PATH: the colour emoji font the emoji sprites are drawn from ([paths] fonts.emoji) | +| `--site-region` `REGION` | a region the site serves: a [servers.] table (repeatable; the first is the base; default: [catalog] region) | +| `--all-regions` | every configured region | +| `--fonts` `FONTS` | text of the start canvas and the stories: open (default): the start canvas has layout and style only, the stories glyphs from your font files (--font); game: the game's TMP fonts (needs the 'fonts' extra) (one of `open`, `game`; default `open`) | +| `--band` `BAND` | band of the LightWeight background and start timeline (default: band of the music's first vocal character) | +| `--leader-card` `LEADER_CARD` | deck centre MasterMemberCard id (its character's band) | +| `--live-option` `OPTION[=VALUES]` | also export the files of a Live option's variants (repeatable): MirrorChart, MeasureLineDisplay, NoteDesignId, NoteEffectId, LiveQuality, NoteSePatternId (every value of the master data, or =v,v...), defaults (the option defaults and ranges only) or all | + +At most one of `--pair`, `--all`, `--player-only`, `--reingest-json`. + +At most one of `--live2d`, `--all-live2d`. + +At most one of `--story`, `--all-stories`. + +At most one of `--site-region`, `--all-regions`. + +At most one of `--band`, `--leader-card`. + +## nnnotes music-data + +every live song and chart: metadata in every language, chart facts and the deck model's chart statistics -> one JSON file + +| Argument | Description | +| --- | --- | +| `--master-files` `DIR` | master data files as served: MasterManifest.json and the .bin files it lists (`master download`) | +| `--apk-master` | the master data files of base.apk ([paths] apk) | +| `--decoded-master` | decoded master data ([paths] master or --master) with the MasterManifest.json of the files it was decoded from; no master key | +| `--full` | also write the deck model's input: every chart's runtime notes and the master data tables about cards, skills, bonuses, scores and events | +| `--no-deck` | do not run the deck model (every chart's deck is null) | +| `--seeds` `N` | replay seeds for a chart with a luck range (default 8); expectations do not use sampling (default `8`) | +| `--workers` `N` | threads measuring charts (default: every processor; 0: one thread) | +| `--no-gekisou-aptitude` | leave out the charts' Gekisou aptitude (every gekisouAptitude is null) | +| `--stats-cache` `DIR` | reuse native simulation programs and runs in DIR across builds; completed cache entries survive interrupted builds | +| `--no-bgm` | do not read the BGM cue sheets (every song's bgm.length is null) | +| `--replay-dir` `DIR` | write canonical runtime DeckData, per-chart inputs and replay manifest under the output directory | +| `--replay-engine` `PKG` | the deck model's replay WASM release package (ournotes-replay-wasm-vVERSION.tar.gz or its directory); its web JS/WASM and build-info.json are copied into --replay-dir | +| `--recommend-engine` `PKG` | the deck model's recommendation WASM release package (ournotes-recommend-wasm-vVERSION.tar.gz or its directory); its web JS/WASM and build-info.json are copied into --replay-dir | +| `--jackets` `DIR` | also write every song's jacket as DIR/.webp (at most 320 px on the longer side) | +| `-o`, `--out` `OUT` | output file (.json, or .json.gz for gzip) (required) | + +Exactly one of `--master-files`, `--apk-master`, `--decoded-master`. diff --git a/docs/commands.md b/docs/commands.md index a59a96c..49e2468 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -1,17 +1,18 @@ # Commands ``` -nnnotes [global options] [command options] +nnnotes [options] ``` -Global options (before the command): `--config`, `--region`, `--language`, `--catalog`, `--catalog-release`, `--cache`, `--master`, -`--apk`, `--ffmpeg`, `--vgmstream`, `--node`, plus `--version` and `--help`. They set the settings -described in [configuration.md](configuration.md), which also lists the settings each command needs. -`nnnotes --help` prints the options of a command. The asset export commands (`export`, `plan`, -`run-stage`, `catalogs`, `store`) are described in [assets.md](assets.md). +Global options (before or after the command name): `--config`, `--region`, `--language`, `--catalog`, +`--catalog-version`, `--cache`, `--store`, `--master`, `--apk`, `--ffmpeg`, `--vgmstream`, `--node` and `--json`, +plus `--version` and `--help`. They set the settings described in [configuration.md](configuration.md), which also +lists the settings each command needs. `nnnotes --help` prints the options of a command; +[cli.md](cli.md) lists every command with its options. The asset export commands (`export assets`, `store plan`, +`store run`, `store verify` and the catalog versions of the store) are described in [assets.md](assets.md). -Commands that write files take the output path with `-o/--out` (required), except `web`, which takes the site -directory as its positional argument. JSON summaries on standard output are UTF-8. Every JSON file is written by one +Commands that write files take the output path with `-o/--out` (required). A command prints a short summary on +standard output; with `--json` it prints its report as JSON (UTF-8). Every JSON file is written by one writer: UTF-8, LF line endings, non-finite numbers as `1e999` / `-1e999` (a NaN is an error). The output is deterministic for a given installation: the same inputs with the same versions of nnnotes, its Python dependencies and the external tools give byte-identical files. Other versions can encode the same content into other bytes (a @@ -39,13 +40,13 @@ setting's value. Details in [configuration.md](configuration.md#writing-the-conf ## Cache -`pull` and every extractor read bundles through the cache (`[paths] cache`): +`bundle pull` and every extractor read bundles through the cache (`[paths] cache`): ``` /catalog_main_.bin the legacy main catalog (explicit main or no API/version pin) /bundles/ bundles of the dependency closures, decrypted (UnityFS) /raw/ raw CDN files stored as they are (e.g. CRI cue sheet data) -/catalogs// catalogs downloaded by `browse` +/catalogs// catalogs downloaded by `catalog browse` /international/// versioned catalog, bundles and raw files; isolated from old main caches ``` @@ -54,34 +55,34 @@ Files already in the cache are not downloaded again. When `[paths] apk` is set, the region's and bundles that ship inside the APK are read from it. For international regions with a configured API, catalog-based commands discover `resource_version` first and -download `catalog__.bin`. `catalogs fetch` labels the bytes with the version used to -select that file, rather than querying a newer label after downloading main. Use `--catalog-release VERSION` or +download `catalog__.bin`. `catalog fetch` labels the bytes with the version used to +select that file, rather than querying a newer label after downloading main. Use `--catalog-version VERSION` or the [version settings](configuration.md) to pin builds and use cached data offline. An explicit `--catalog FILE` bypasses discovery. JP keeps its existing Version-driven path. -## catalog +## catalog keys ``` -nnnotes catalog [--prefix PREFIX] [--limit LIMIT] +nnnotes catalog keys [--prefix PREFIX] [--limit LIMIT] ``` Prints the addressable keys that start with `PREFIX` (default: all), sorted, at most `LIMIT` (default 200), one per line; the total count goes to standard error. -## browse +## catalog browse ``` -nnnotes browse [--port PORT] [--host HOST] +nnnotes catalog browse [--port PORT] [--host HOST] ``` Serves a local web page on `HOST:PORT` (default `127.0.0.1:8000`) that lists every configured region, its catalog languages, and per catalog the asset paths as directories. A link under an asset path downloads the bundle holding it, decrypted; `bundles/` lists every bundle directly. -## pull +## bundle pull ``` -nnnotes pull KEY [KEY ...] +nnnotes bundle pull KEY [KEY ...] ``` Fetches the bundle closure (the key's location and all its dependencies) of each key into the cache and prints the @@ -94,7 +95,7 @@ nnnotes servers [--show-hosts] ``` Reads the server list from the bootstrap API root (`[bootstrap] api`) with an anonymous call to the game's API and -prints `{servers: [{name, displayName, areaId, region, cdnRoots, apiRoots}]}`: per server its name, its area id, the +reports `{servers: [{name, displayName, areaId, region, cdnRoots, apiRoots}]}`: per server its name, its area id, the configured region (`[servers.]` table) whose `cdn` or `api` is one of the server's roots, or `null`, and how many alternative CDN and API roots the server list gives. `--show-hosts` adds the roots themselves (`cdn`, `api`: lists), ready for the `cdn` and `api` settings of a region. The exit status is 1 when the call fails. @@ -106,7 +107,7 @@ nnnotes master version ``` Asks the region's API root (`[servers.] api`) for the master data version and the resource version the -region serves now (the game's API, anonymous) and prints `{region, masterVersion, resourceVersion}`. The client +region serves now (the game's API, anonymous) and reports `{region, masterVersion, resourceVersion}`. The client version sent with the call is `[client] version`, else the `versionName` of `[paths] apk`. The exit status is 1 when the call fails. @@ -118,25 +119,27 @@ nnnotes master decode INPUT [INPUT ...] -o OUT [--workers N] `INPUT`: master data files, or directories whose `*.bin` files are decoded. Each file (64-byte prefix, then Rijndael-256 CBC with PKCS7 padding over gzip-compressed JSON) is written as `OUT/.json` (UTF-8, one-space -indent, non-finite numbers as `1e999` / `-1e999`; a table with a NaN fails). `--workers` (default 8) decodes in parallel. Prints `{decoded, failed: [{file, error}], out}`; the exit +indent, non-finite numbers as `1e999` / `-1e999`; a table with a NaN fails). `--workers` (default: the CPUs the process may use; `0`: one at a time) decodes in parallel. The report is +`{decoded, failed: [{file, error}], out}`; the exit status is 1 when a file failed. ## master download ``` -nnnotes master download (--version VERSION | --latest) -o OUT [--workers N] +nnnotes master download (--version VERSION | --latest) -o OUT [--fetch-workers N] ``` Downloads master data version `VERSION`, or with `--latest` the version the region serves now (as `master version`), from the region's CDN: `OUT/MasterManifest.json` and every `.bin` file it lists, each checked against the manifest's SHA-256. Files already present with the right hash are kept. -`--workers` (default 16) downloads in parallel. Prints `{version, files, downloaded, kept, failed, out}`; the exit +`--fetch-workers` (default 16) downloads in parallel. The report is `{version, files, downloaded, kept, failed, out}`; +the exit status is 1 when a file failed or, with `--latest`, the version could not be fetched. -## adv +## export episode ``` -nnnotes adv ADV_ID -o OUT.json +nnnotes export episode ADV_ID -o OUT.json ``` One ADV episode as JSON: `advId`, `asset`, `commandCount`, `commands` (the episode's command list with the text, @@ -150,10 +153,10 @@ takes the player settings' default transition, which is read from `[paths] apk`) `chatwindow` and `chaticon` from the `MasterAdvChat` row of the chat rows' chat id; `video` from the episode's video row of a Movie / Clip row's video id. -## story +## export story ``` -nnnotes story ADV_ID -o OUT [--models MODELS] [--force] [--format flac|ogg|wav] [--flac-level N] [--no-audio] +nnnotes export story ADV_ID -o OUT [--models MODELS] [--force] [--format flac|ogg|wav] [--flac-level N] [--no-audio] [--fonts open|game] ``` @@ -161,9 +164,9 @@ One ADV episode as a directory, and its Live2D models in a models directory: ``` MODELS// every Live2D model of the episode (MODELS: `--models`, default OUT/../live2d), the files - of a site's model manifest (see `web`): model.json, moc3, prefab, the atlas pages and the + of a site's model manifest (see `site build`): model.json, moc3, prefab, the atlas pages and the shader programs its drawables use -OUT/episode.json as `adv` +OUT/episode.json as `export episode` OUT/audio// every cue sheet of the episode, one file per cue + cues.json, streams.json OUT/scene.json player graphics, cameras, ADV fields, volumes, settings and stages OUT/frames.json Frame prefabs by asset name: uGUI, Animator controllers and clips, UI particle systems @@ -184,11 +187,11 @@ OUT/story.json index of the above (a file the episode does not need i story.json `models` maps the key of each model to its id (`` of `Character/Live2D///model/`) and `modelsDir` is the path from OUT to MODELS. A model directory that exists is used as it is, so the stories of one models directory export each model once; `--force` exports the episode's models again. The printed summary lists -the models exported (`modelsBuilt`) and those used as they were (`modelsSkipped`). Several `story` processes may +the models exported (`modelsBuilt`) and those used as they were (`modelsSkipped`). Several `export story` processes may share a models directory: a model appears in it in one rename, whole, and a model two processes export at once is kept from the first to finish (the other's export, the same bytes, is dropped and counted in `modelsSkipped`). -`--format` (default `flac`) is the audio format and `--flac-level` its FLAC compression level (as for `audio`); +`--format` (default `flac`) is the audio format and `--flac-level` its FLAC compression level (as for `export audio`); `--no-audio` leaves the cue sheets undecoded (no `audio/`, `audio` in story.json is empty). With audio, `crilips/` holds the weights of the game's CRI Lips mouth analysis, read from the CRI Lips library in `[paths] apk` (or in the arm64-v8a split next to it, `split_config.arm64_v8a.apk`), when a voice @@ -262,14 +265,14 @@ way the TMP shader applies them. The document-level `textStyle.roles` gives, per `descentEm` (line pitch = `lineHeightEm` + `lineSpacing` / 100 em) and the font's `spacingOffset` / `boldSpacing` (1/100 em) for the export language. -## live2d +## export live2d ``` -nnnotes live2d MODEL -o OUT +nnnotes export live2d MODEL -o OUT ``` A Live2D (Cubism) model prefab as a Cubism runtime directory. `MODEL` is the model's key -`Character/Live2D///model/` or its id `` (`nnnotes catalog --prefix Character/Live2D/` lists +`Character/Live2D///model/` or its id `` (`nnnotes catalog keys --prefix Character/Live2D/` lists the keys): ``` @@ -284,29 +287,29 @@ OUT/.model3.json file references, EyeBlink / LipSync groups Needs `[paths] apk` (component classes are resolved through the APK). Prints a JSON summary. -## spot +## export spot ``` -nnnotes spot SPOT_ID -o OUT +nnnotes export spot SPOT_ID -o OUT ``` ``` OUT/spot.json master row, names, tap-talk episodes, situation settings, tap targets, Spine characters OUT/spine/ Spine skeletons (.json or .skel), atlases, atlas page PNGs -OUT/room.glb the spot's background as binary glTF (as `room`), with room.json -OUT/shaders/ shaders of the spot's bundles (as `shader`) +OUT/room.glb the spot's background as binary glTF (as `export room`), with room.json +OUT/shaders/ shaders of the spot's bundles (as `export shader`) ``` Needs `[paths] apk` and master data (`MasterHomeSpot`, `MasterText`, `MasterStoryHomeSpotTapTalkEpisode`). `room.glb` and -`room.json` are the `room` export of the spot's background. +`room.json` are the `export room` export of the spot's background. A reference the situation prefab leaves empty is written as null: a Spine character with no `_animation` has null `skeletonData`, `animation` and `world`, and a tap target with no `_focus` has a null `focusWorld`. -## room +## export room ``` -nnnotes room KEY -o OUT.glb +nnnotes export room KEY -o OUT.glb ``` A spot background prefab as binary glTF: every mesh baked into prefab space, converted to glTF's right-handed space, @@ -330,10 +333,10 @@ left with no submesh is not written. A MeshFilter with no mesh, or a mesh with n nothing and is not written; the summary lists the object path in `nullMeshFilters`, or the mesh and object path in `meshesWithoutTriangles`. -## shader +## export shader ``` -nnnotes shader (--key KEY | --apk-bundle SUBSTRING [SUBSTRING ...]) -o OUT +nnnotes export shader (--key KEY | --apk-bundle SUBSTRING [SUBSTRING ...]) -o OUT ``` Every Shader object in the bundle closure of `KEY`, or in the APK bundles whose file names contain the substrings @@ -345,10 +348,10 @@ OUT///sp

__. every compiled sub-program (G OUT/shaders.json index with the keywords of every variant ``` -## audio +## export audio ``` -nnnotes audio CUE_SHEET -o OUT [--format flac|ogg|wav] [--flac-level N] +nnnotes export audio CUE_SHEET -o OUT [--format flac|ogg|wav] [--flac-level N] ``` Decodes the CRI cue sheet of the key `Cri/Sound/`: one file per stream (a name repeated within the @@ -361,17 +364,17 @@ The HCA keycode is read from `[paths] apk`. ``` nnnotes voices list [--from OUT] [--character C] [--category C] [--source S] [--episode ADV] [--status S] - [--limit N] [--json] [--language L] -nnnotes voices search TEXT [--from OUT] [--language L] [--character C] [--category C] [--source S] - [--episode ADV] [--status S] [--limit N] [--json] -nnnotes voices summary [--from OUT] [--json] -nnnotes voices get ID -o DIR [--from OUT] [--format flac|ogg|wav] + [--limit N] +nnnotes voices search TEXT [--from OUT] [--search-language L] [--character C] [--category C] [--source S] + [--episode ADV] [--status S] [--limit N] +nnnotes voices summary [--from OUT] +nnnotes voices get ID -o DIR [--from OUT] [--format flac|ogg|wav] [--flac-level N] ``` Queries the voices index ([views.md](views.md#voices)): the character voices the master data names and the voices of the story episodes, with their source row, characters (for story voices, inferred from the speaker), category, five-language text, cue sheet and cue. `--from OUT` reads `OUT/views/voices.json`, written by -`nnnotes export --views voices` (or `all`) with the exported audio and episodes (`--select key:Adv/Episode/`): then +`nnnotes export assets --views voices` (or `all`) with the exported audio and episodes (`--select key:Adv/Episode/`): then every voice also has its status and the files of its streams. Without `--from` the index is built from the configured master data and catalog: content and catalog keys only, every present voice `not-exported`; story voices only for the episode of `--episode` (or of a `get` row id), read from the catalog. @@ -383,44 +386,45 @@ voices only for the episode of `--episode` (or of a `get` row id), read from the `LiveDialogueFixedPair`, `LiveStartCharacterVoice`, `HomeSpot`, `Title`, `Sound` or `Story`. `--episode`: the story voices of one episode, by MasterAdv id or episode asset. `--status`: a status of the index. - `list` and `search` print one line per voice: id, characters (story voices: the speaker's name in `--language`), - `Source.Category`, `sheet/cue`, status and the text in `--language` (default `[catalog] language`); `--json` - prints the rows. `search TEXT` matches a part of the text in `--language` (every language when not given) or of - the cue name, case ignored. + `Source.Category`, `sheet/cue`, status and the text in the global `--language` (default `[catalog] language`); + `--json` prints the rows. `search TEXT` matches a part of the text in `--search-language` (every language when not + given) or of the cue name, case ignored. - `summary`: rows per status, per source and category and per character, the gaps (voices whose sheet has no key or whose decoded sheet lacks the cue, episodes without a key) and the unreferenced keys and streams; for the story episodes, their number per status and kind, the rows and inferred characters per speaker name, and the story voices whose cue a row of another source also names. - `get ID`: `ID` is a row id (`MasterTalk:561`, `MasterAdv::`) or a sound id. Writes the files of the voice's streams into `DIR`, named as the export names them, and `voice.json` (its rows and files). - With `--from` and the files exported, they are copied; otherwise the voice's cue sheet alone is decoded as - `nnnotes audio` does (`--format`, FLAC level 8) and the cue's streams are picked by the ACB's cue table. Exit + With `--from` and the files exported, they are copied; otherwise (or with `--flac-level`) the voice's cue sheet + alone is decoded as `nnnotes export audio` does (`--format`, `--flac-level`, default 8) and the cue's streams are + picked by the ACB's cue table. Exit status 1 when the voice has no file (its status says why). -## crikey +## apk hca-key ``` -nnnotes crikey [--write DIR] +nnnotes apk hca-key [--write DIR] ``` Looks up the CRI HCA keycode in the APK's boot data and prints whether one was found (and its number of digits, not its value). `--write` writes it as `DIR/.hcakey` (8 bytes, big-endian), the file vgmstream reads next to its input; `DIR` is created when missing. -## player +## apk settings ``` -nnnotes player -o OUT.json +nnnotes apk settings -o OUT.json ``` The game's render settings from the APK's boot data as JSON: `colorSpace`, `defaultPipeline`, `qualityLevels`, `qualityPerPlatform`, `pipelines`, `renderers`, `postProcessData`. Needs `[paths] apk`. The MonoBehaviours are read with the type trees that ship with nnnotes; an APK whose classes do not match them stops the command with exit -status 2 and a line naming the class, the game version and the Unity version (as `story`, `live` and `web`). +status 2 and a line naming the class, the game version and the Unity version (as `export story`, `export live` and `site build`). -## live +## export live ``` -nnnotes live MUSIC_ID -o OUT [--difficulty easy|normal|hard|expert] [--format flac|ogg|wav] +nnnotes export live MUSIC_ID -o OUT [--difficulty easy|normal|hard|expert] [--format flac|ogg|wav] [--flac-level N] [--fonts open|game] [--band BAND | --leader-card CARD_ID] [--live-option OPTION[=VALUES] ...] ``` @@ -441,7 +445,7 @@ OUT/live.json index of the above ``` `--format` (default `flac`) is the format of every decoded cue sheet and `--flac-level` its FLAC compression level -(as for `audio`). +(as for `export audio`). The start canvas follows the client language `[catalog] language` (`ja`, `en`, `zh-Hant`, `zh-Hans` or `ko`): its text table column, fonts and line spacing. `--fonts open` (default) writes each text's layout and style (size, @@ -473,16 +477,16 @@ An option without values offers every value the master data has; `NAME=v,v,...` value is always included; a value the master data does not have is a usage error). Any option also writes the `defaults` tables. Without `--live-option` the directory holds the files of the default options only. -## web +## site build ``` -nnnotes web SITE [--pair MUSIC_ID:DIFFICULTY [--pair ...] | --all] [--live2d MODEL [--live2d ...] | --all-live2d] +nnnotes site build -o SITE [--pair MUSIC_ID:DIFFICULTY [--pair ...] | --all] [--live2d MODEL [--live2d ...] | --all-live2d] [--story ADV_ID [--story ...] | --all-stories] [--story-languages LANG,...] [--font LANG=PATH ...] -nnnotes web SITE (--player-only | --reingest-json) +nnnotes site build -o SITE (--player-only | --reingest-json) [--player DIR] [--format aac|opus|vorbis|mp3|flac] [--no-audio] [--compress gzip|br|none] [--force] [--tmp DIR] [--workers N] [--read-workers N] [--band BAND | --leader-card CARD_ID] - [--region REGION [--region ...] | --all-regions] [--fonts open|game] + [--site-region REGION [--site-region ...] | --all-regions] [--fonts open|game] [--live-option OPTION[=VALUES] ...] ``` @@ -520,7 +524,7 @@ SITE/assets/.[.gz|.br] content-addressed files shared by all Live2D models; `--all-live2d` adds every model key of the catalog. A model's id is its ``. Its manifest lists the files the player's Live2D viewer and story page read: `model.json` (index: moc3, prefab, textures, shader index, the Cubism mask materials and `motionSync`, whether the prefab's root has the MotionSync controller with its CRI - audio input), the moc3 and prefab as `live2d` writes them, the atlas pages the drawables use, and the GLSL ES 3.00 + audio input), the moc3 and prefab as `export live2d` writes them, the atlas pages the drawables use, and the GLSL ES 3.00 programs of the Live2D shaders that the drawables' materials select, each keyword set also with `_ADDITIONAL_LIGHTS_VERTEX` (the story player adds it at quality 4), and the mask shader's for a model with masked drawables. Charts and models can be added in one run; models are built first. @@ -568,9 +572,9 @@ SITE/assets/.[.gz|.br] content-addressed files shared by all and nnnotes' own code), so a later build with the same `--tmp` reuses what is unchanged and writes the same files faster. Deleting it is safe at any time outside a build; the next build then makes everything again. -- `--band` / `--leader-card`: as for `live`, for every chart. `--fonts`: for charts as for `live` (the site stores +- `--band` / `--leader-card`: as for `export live`, for every chart. `--fonts`: for charts as for `export live` (the site stores no start canvas files); for stories where the glyphs of the story text come from (see Stories). -- `--live-option` (repeatable): as for `live`, for the charts of this run (with `--pair` or `--all`). The live +- `--live-option` (repeatable): as for `export live`, for the charts of this run (with `--pair` or `--all`). The live directories carry the variants' files, and a chart's read set is the union of its read set with the default options and its read sets with the settings of each offered variant (every combination of the offered mirror, note skin, effect set and quality values, the bar lines on, and each other note sound set; the player's read-set @@ -585,13 +589,12 @@ SITE/assets/.[.gz|.br] content-addressed files shared by all `MasterText` row of `_nameTextID`, the languages that have a text) and `label` (the name in `[catalog] language`). A key the rows map to two characters gets none. The manifests of skipped models get this build's fields; without master data the fields are not written and skipped manifests keep theirs. -- `--region` (repeatable) / `--all-regions`: the regions the site serves (see Regions); default: the one - `[catalog] region`. This `--region` follows the command name (`nnnotes web SITE --region tw --region kr`); the - global `--region` before it sets `[catalog] region`. +- `--site-region` (repeatable) / `--all-regions`: the regions the site serves (see Regions); default: the one + `[catalog] region`. The global `--region` sets `[catalog] region`. ### Stories -A story's manifest lists what the player's story page reads, in the layout of `story` (below the site's `assets/`): +A story's manifest lists what the player's story page reads, in the layout of `export story` (below the site's `assets/`): `story.json`, `episode.json`, `scene.json`, the scene shaders (GLSL ES 3.00 programs only), textures, per cue sheet `cues.json` and the file of each cue in the `--format` (an AAC file's encoder delay in `cues.json` as `encoderDelay`), the media files the episode uses and its videos; per language (`--story-languages`, @@ -709,7 +712,7 @@ files of the master data directory (`[paths] master`, `--master`) and the `Maste decoded from, whose version and SHA-256 the file records (region `[catalog] region`). `--no-bgm` skips the cue sheets (every `bgm.length` is null). `--jackets DIR` also writes every song's jacket as `DIR/.webp` (at most 320 px on the longer side). `FILE` ending in `.gz` is written gzip-compressed; the file is canonical: the same inputs and -nnnotes version give the same bytes. Prints `{out, format, region, masterSource, masterVersion, songs, charts, deck, +nnnotes version give the same bytes. Reports `{out, format, region, masterSource, masterVersion, songs, charts, deck, unplayable, full, bgm, jackets, bytes, fileBytes, sha256}`. A missing or unreadable input (a master data file, a column, a text id, a chart asset, a cue sheet or cue, a jacket), a chart the deck model cannot measure, or deck statistics that disagree with the chart facts stop the command with exit status 1 before the file is written. The diff --git a/docs/configuration.md b/docs/configuration.md index 7da2092..fe913ca 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -59,17 +59,16 @@ No `config` command prints a setting's value. The config files these commands wr The global flags go **before** the command name: ```bash -nnnotes --config ~/nnnotes.toml --cache /data/nnnotes-cache pull -NNNOTES_PATHS_MASTER=/data/master nnnotes adv 10462 -o episode.json +nnnotes --config ~/nnnotes.toml --cache /data/nnnotes-cache bundle pull +NNNOTES_PATHS_MASTER=/data/master nnnotes export episode 10462 -o episode.json ``` -`--player` is the exception: it is an option of `web` and follows the command. `web` also has a `--region` of its -own (with `--all-regions`), which follows the command and chooses the regions the site serves; the global `--region` -before the command sets `[catalog] region`: +`--player` is the exception: it is an option of `site build`. `site build` also has `--site-region` (with +`--all-regions`), which chooses the regions the site serves; the global `--region` sets `[catalog] region`: ```bash -nnnotes --region tw web out/site --all # the one region [catalog] region = tw -nnnotes web out/site --all --region tw --region kr # a site for two regions +nnnotes --region tw site build -o out/site --all # the one region [catalog] region = tw +nnnotes site build -o out/site --all --site-region tw --site-region kr # a site for two regions ``` ### Paths @@ -87,38 +86,38 @@ exist when they are given. | `[master] key` | `NNNOTES_MASTER_KEY` | — | Rijndael-256 key of the master data files: 32 bytes as 64 hex digits | | `[master] iv` | `NNNOTES_MASTER_IV` | — | Rijndael-256 CBC initialization vector: 32 bytes as 64 hex digits | | `[catalog] region` | `NNNOTES_CATALOG_REGION` | `--region` | name of one `[servers.]` table | -| `[catalog] language` | `NNNOTES_CATALOG_LANGUAGE` | `--language` | catalog language: the `` of `catalog__.bin`: `ja`, `en`, `zh-Hant`, `zh-Hans` or `ko`; also the client language of `live`, `story` and `web` (the text field, fonts and line spacing of their UI) and of the model labels of `web --live2d` | -| `[catalog] version` | `NNNOTES_CATALOG_VERSION` | `--catalog-release` | optional international resource version; unset: query the configured region API, or use legacy `main` when no API is configured; JP uses its own discovery | -| `[servers.] name` | `NNNOTES_SERVERS__NAME` | — | label of the region in `browse` (default: the region name) | +| `[catalog] language` | `NNNOTES_CATALOG_LANGUAGE` | `--language` | catalog language: the `` of `catalog__.bin`: `ja`, `en`, `zh-Hant`, `zh-Hans` or `ko`; also the client language of `export live`, `export story` and `site build` (the text field, fonts and line spacing of their UI) and of the model labels of `site build --live2d` | +| `[catalog] version` | `NNNOTES_CATALOG_VERSION` | `--catalog-version` | optional international resource version; unset: query the configured region API, or use legacy `main` when no API is configured; JP uses its own discovery | +| `[servers.] name` | `NNNOTES_SERVERS__NAME` | — | label of the region in `catalog browse` (default: the region name) | | `[servers.] provider` | `NNNOTES_SERVERS__PROVIDER` | — | `international` or `jp`; empty selects `jp` for the region named jp, international otherwise | | `[servers.] client_version` | `NNNOTES_SERVERS__CLIENT_VERSION` | — | per-region API client version; overrides `[client] version`, then falls back to the region's APK versionName | | `[servers.] apk` | `NNNOTES_SERVERS__APK` | `--apk` | per-region APK, APKS/XAPK or directory with base.apk; adjacent splits of base.apk are read automatically; the flag overrides it | | `[servers.] catalog` | `NNNOTES_SERVERS__CATALOG` | `--catalog` | per-region catalog file; JP needs the matching `.source.json`; the flag overrides it | -| `[servers.] catalog_version` | `NNNOTES_SERVERS__CATALOG_VERSION` | `--catalog-release` | per-region international version pin; overrides `[catalog] version`, but not the command-line flag | +| `[servers.] catalog_version` | `NNNOTES_SERVERS__CATALOG_VERSION` | `--catalog-version` | per-region international version pin; overrides `[catalog] version`, but not the command-line flag | | `[servers.] cdn` | `NNNOTES_SERVERS__CDN` | — | CDN base URL of the region (a trailing `/` is ignored) | -| `[servers.] languages` | `NNNOTES_SERVERS__LANGUAGES` | — | catalog languages `browse` lists: a TOML array of strings; comma-separated in the environment | +| `[servers.] languages` | `NNNOTES_SERVERS__LANGUAGES` | — | catalog languages `catalog browse` lists: a TOML array of strings; comma-separated in the environment | | `[servers.] api` | `NNNOTES_SERVERS__API` | — | API root of the region: `https://host[:port]` (TLS, port 443 by default), `host[:port]`, or `http://host[:port]` for a plain-text local server; no path | | `[servers.] master` | `NNNOTES_SERVERS__MASTER` | — | decoded master data directory of the region (default: `[paths] master`) | | `[bootstrap] api` | `NNNOTES_BOOTSTRAP_API` | — | API root that serves the server list (`nnnotes servers`), same format | | `[client] version` | `NNNOTES_CLIENT_VERSION` | — | client version sent to the game's API, e.g. `1.0.1`; unset: the `versionName` of `[paths] apk` | | `[paths] catalog` | `NNNOTES_PATHS_CATALOG` | `--catalog` | a catalog `.bin` file to read instead of discovering/downloading a catalog; explicit files stay usable offline | | `[paths] cache` | `NNNOTES_PATHS_CACHE` | `--cache` | cache directory (created when missing) | -| `[paths] store` | `NNNOTES_PATHS_STORE` | `--store` (of `export`, `plan`, `run-stage`, `catalogs`, `store`) | store directory of the asset export ([assets.md](assets.md)); unset: `<[paths] cache>/store` | +| `[paths] store` | `NNNOTES_PATHS_STORE` | `--store` | store directory of the asset export ([assets.md](assets.md)); unset: `<[paths] cache>/store` | | `[paths] master` | `NNNOTES_PATHS_MASTER` | `--master` | decoded master data directory, one `

.json` per table; the flag overrides `[servers.] master` | | `[paths] apk` | `NNNOTES_PATHS_APK` | `--apk` | the game's `base.apk` (the story export reads the CRI Lips library from it, or from `split_config.arm64_v8a.apk` next to it) | | `[paths] ffmpeg` | `NNNOTES_PATHS_FFMPEG` | `--ffmpeg` | `ffmpeg` executable; unset: `ffmpeg` on `PATH` | | `[paths] vgmstream` | `NNNOTES_PATHS_VGMSTREAM` | `--vgmstream` | `vgmstream-cli` executable; unset: `vgmstream-cli` on `PATH` | | `[paths] node` | `NNNOTES_PATHS_NODE` | `--node` | Node.js executable; unset: `node` on `PATH` | -| `[paths] player` | `NNNOTES_PATHS_PLAYER` | `--player` (of `web`) | ournotes-player checkout (built) or installed package | -| `[paths] fonts.` | `NNNOTES_PATHS_FONTS_` (e.g. `NNNOTES_PATHS_FONTS_ZH_HANT`) | `--font =PATH` (of `web`) | font file (OpenType or TrueType) the story text of a language is drawn with (`--fonts open`); also read for the other story languages whose game fonts fall back to that language's font | -| `[paths] fonts.emoji` | `NNNOTES_PATHS_FONTS_EMOJI` | `--font emoji=PATH` (of `web`) | colour emoji font (PNG bitmap glyphs: CBDT or sbix) the stories' emoji sprites are drawn from (`--fonts open`); Noto Color Emoji (SIL Open Font License 1.1) is the tested font. Optional: without it the sprites keep their layout with empty glyphs, listed in `ui/fonts.json` `coverage.sprites.missing` | -| `[export] link` | `NNNOTES_EXPORT_LINK` | `--link` (of `export`) | how `export` makes the layout files from the store: `auto` (unset), `clone`, `hard` or `copy` ([assets.md](assets.md)) | +| `[paths] player` | `NNNOTES_PATHS_PLAYER` | `--player` (of `site build`) | ournotes-player checkout (built) or installed package | +| `[paths] fonts.` | `NNNOTES_PATHS_FONTS_` (e.g. `NNNOTES_PATHS_FONTS_ZH_HANT`) | `--font =PATH` (of `site build`) | font file (OpenType or TrueType) the story text of a language is drawn with (`--fonts open`); also read for the other story languages whose game fonts fall back to that language's font | +| `[paths] fonts.emoji` | `NNNOTES_PATHS_FONTS_EMOJI` | `--font emoji=PATH` (of `site build`) | colour emoji font (PNG bitmap glyphs: CBDT or sbix) the stories' emoji sprites are drawn from (`--fonts open`); Noto Color Emoji (SIL Open Font License 1.1) is the tested font. Optional: without it the sprites keep their layout with empty glyphs, listed in `ui/fonts.json` `coverage.sprites.missing` | +| `[export] link` | `NNNOTES_EXPORT_LINK` | `--link` (of `export assets`) | how `export assets` makes the layout files from the store: `auto` (unset), `clone`, `hard` or `copy` ([assets.md](assets.md)) | Hex values are case-insensitive; a `0x` prefix and surrounding whitespace are ignored. All values are TOML strings except `languages` (array of strings). -Regions: `browse` serves every region that has a `[servers.]` table or an `NNNOTES_SERVERS__CDN` -variable (the region name is then the lower-cased ``), and `web --region` / `--all-regions` builds one site +Regions: `catalog browse` serves every region that has a `[servers.]` table or an `NNNOTES_SERVERS__CDN` +variable (the region name is then the lower-cased ``), and `site build --site-region` / `--all-regions` builds one site for several of them. The other commands use the one region named by `[catalog] region`. Add one `[servers.]` table per region. @@ -132,13 +131,13 @@ is retained. JP's version/hash directory and authentication remain unchanged. For repeatable builds, pin the resource version belonging to your master snapshot: ```sh -nnnotes --region tw --catalog-release 1.0.0.201 web site --story 10948 -nnnotes --region tw --catalog-release 1.0.0.201 live2d MODEL_ID -o out/model +nnnotes --region tw --catalog-version 1.0.0.201 site build -o site --story 10948 +nnnotes --region tw --catalog-version 1.0.0.201 export live2d MODEL_ID -o out/model ``` Alternatively set `NNNOTES_SERVERS_TW_CATALOG_VERSION` in the build environment; workers inherit this pin. -`--catalog-release` selects a remote resource release. The asset commands' existing `--catalog-version LABEL|SHA` -still selects an already imported store record. To inspect the historical main catalog, explicitly pin `main`. +`--catalog-version` selects a remote resource release; `--stored-catalog LABEL|SHA` of `export assets` and `store +plan` selects a catalog version of the store. To inspect the historical main catalog, explicitly pin `main`. Versioned caches are isolated by CDN root and resource version; a cached pin needs the CDN setting to identify its directory but performs no API/catalog download. A missing versioned file never falls back to main. @@ -152,37 +151,37 @@ in, and bundles that ship inside the APK can only be read with it. "Master data" means the decoded master data directory of the region: the `--master` flag, else `[servers.] master` of `[catalog] region`, else `[paths] master`. -`--fonts game` (`story`, `live`, `web`) also needs the optional `fonts` dependencies (`pip install 'nnnotes[fonts]'`), -and so does `web --story` / `--all-stories` with either font source (it generates the TextMesh Pro font assets of the +`--fonts game` (`export story`, `export live`, `site build`) also needs the optional `fonts` dependencies (`pip install 'nnnotes[fonts]'`), +and so does `site build --story` / `--all-stories` with either font source (it generates the TextMesh Pro font assets of the story text); it is not a setting. | Command | Settings | |---|---| -| `catalog` | `[paths] cache`; `[paths] catalog`, or `[catalog] language` (plus region and `cdn` while the catalog is not in the cache); no bundle key | -| `browse` | `[paths] cache`, `[bundle] key` + `nonce_seed`, and per region `cdn` + `languages` (`name` optional) | -| `pull` | catalog | +| `catalog keys` | `[paths] cache`; `[paths] catalog`, or `[catalog] language` (plus region and `cdn` while the catalog is not in the cache); no bundle key | +| `catalog browse` | `[paths] cache`, `[bundle] key` + `nonce_seed`, and per region `cdn` + `languages` (`name` optional) | +| `bundle pull` | catalog | | `master decode` | `[master] key` + `iv` | | `master download` | `[catalog] region` and that region's `cdn`; `--latest` also that region's `api` and the client version | | `master version` | `[catalog] region` and that region's `api`; the client version: `[client] version` or `[paths] apk` | | `servers` | `[bootstrap] api`; the client version: `[client] version` or `[paths] apk` | -| `adv` | catalog, master data | -| `story` | catalog, `[catalog] language`, master data, `[paths] apk`, `ffmpeg` (audio and videos), `vgmstream` (not with `--no-audio`) | -| `live2d` | catalog, `[paths] apk` | -| `spot` | catalog, master data, `[paths] apk` | -| `room` | catalog | -| `shader` | catalog; `--apk-bundle` also `[paths] apk` | -| `audio` | catalog, `[paths] apk`, `vgmstream`, `ffmpeg` (not for `--format wav`) | -| `crikey` | `[paths] apk` | -| `player` | `[paths] apk` | -| `live` | catalog, `[catalog] language`, master data, `[paths] apk`, `vgmstream`, `ffmpeg` | -| `web --pair` / `--all` | as `live`, plus `[paths] player`, `node`; with `--region` / `--all-regions` each region's `[servers.]` table (its `cdn` for what must be downloaded) and master data (`[servers.] master`; `[paths] master` for at most one region) | -| `web --live2d` / `--all-live2d` | catalog (bundles from the CDN of the site's first region), `[paths] apk`, `[paths] player`; not `node`; master data only for the model names (optional: without it `models.json` has no names) | -| `web --player-only` / `--reingest-json` | `[paths] player` | +| `export episode` | catalog, master data | +| `export story` | catalog, `[catalog] language`, master data, `[paths] apk`, `ffmpeg` (audio and videos), `vgmstream` (not with `--no-audio`) | +| `export live2d` | catalog, `[paths] apk` | +| `export spot` | catalog, master data, `[paths] apk` | +| `export room` | catalog | +| `export shader` | catalog; `--apk-bundle` also `[paths] apk` | +| `export audio` | catalog, `[paths] apk`, `vgmstream`, `ffmpeg` (not for `--format wav`) | +| `apk hca-key` | `[paths] apk` | +| `apk settings` | `[paths] apk` | +| `export live` | catalog, `[catalog] language`, master data, `[paths] apk`, `vgmstream`, `ffmpeg` | +| `site build --pair` / `--all` | as `export live`, plus `[paths] player`, `node`; with `--site-region` / `--all-regions` each region's `[servers.]` table (its `cdn` for what must be downloaded) and master data (`[servers.] master`; `[paths] master` for at most one region) | +| `site build --live2d` / `--all-live2d` | catalog (bundles from the CDN of the site's first region), `[paths] apk`, `[paths] player`; not `node`; master data only for the model names (optional: without it `models.json` has no names) | +| `site build --player-only` / `--reingest-json` | `[paths] player` | | `music-data` | catalog, `[master] key` + `iv` (not with `--decoded-master`); `--master-files` also `[catalog] region`; `--apk-master` also `[paths] apk`; `--decoded-master` also `[catalog] region` and master data with its `MasterManifest.json`; `[paths] apk` (optional otherwise) for the client version | -| `export`, `plan` | the store (`[paths] store` or `[paths] cache`); catalog (bundles are fetched into the cache); `[paths] apk` for the bundles inside the APK (without it they are reported as `source.absent`); master data for `--views`; with `--catalog-version` an imported catalog version instead of the current catalog | -| `run-stage` | the store; `[paths] cache` for inputs located in the cache; `--fetch` also what fetching needs (region, `cdn`, bundle key, `[paths] apk`) | -| `catalogs list` / `import` / `diff`, `store verify` | the store; `import` reads the APK's catalog from `[paths] apk` when it is set | -| `catalogs fetch` | the store, `[catalog] region` and `language`, that region's `cdn`; its `api` (optional) for the resource version label | +| `export assets`, `store plan` | the store (`[paths] store` or `[paths] cache`); catalog (bundles are fetched into the cache); `[paths] apk` for the bundles inside the APK (without it they are reported as `source.absent`); master data for `--views`; with `--stored-catalog` a catalog version of the store instead of the current catalog | +| `store run` | the store; `[paths] cache` for inputs located in the cache; `--fetch` also what fetching needs (region, `cdn`, bundle key, `[paths] apk`) | +| `catalog versions` / `import` / `diff`, `store verify` | the store; `import` reads the APK's catalog from `[paths] apk` when it is set | +| `catalog fetch` | the store, `[catalog] region` and `language`, that region's `cdn`; its `api` (optional) for the resource version label | The region, its `cdn`, `[bundle] key` and `nonce_seed` are read only when a file must be downloaded: when the catalog and every bundle a command needs are in the cache, none of them is needed. A download that needs a missing @@ -207,8 +206,8 @@ with status 2: a missing `-o`, or a key or id the data does not have (a key not music id without its master data row, a Live2D model the catalog does not have), which the error line names: ``` -nnnotes adv: error: episode 99999: no MasterAdv row with this id -nnnotes pull: error: not a key of the catalog: Live/MusicScore/9999/9999_03 +nnnotes export episode: error: episode 99999: no MasterAdv row with this id +nnnotes bundle pull: error: not a key of the catalog: Live/MusicScore/9999/9999_03 ``` Setting values are never printed, logged or put into an error message, and the `repr` of the settings and key @@ -231,7 +230,7 @@ nnnotes: game API call Version to [servers.tw] api failed: UNAVAILABLE (server u can hold several alternatives separated by `|`, and any one of them can be used as `cdn` / `api`. - **Client version**: the version of your game client (`versionName` of its `base.apk`). The game's API rejects a version older than the one it accepts (`game error code CLIENT_UPDATE_REQUIRED`). -- **`[paths] apk`**: the `base.apk` of your own installation of the game. `player`, `story`, `live` and `web` +- **`[paths] apk`**: the `base.apk` of your own installation of the game. `apk settings`, `export story`, `export live` and `site build` read MonoBehaviours of its boot data with type trees that ship with nnnotes, one set per Unity version (currently game version 1.0.1, Unity 6000.3.12f1). With an APK whose classes do not match them, these commands stop with exit status 2 and a line naming the class, the game version and the Unity version. @@ -241,8 +240,8 @@ nnnotes: game API call Version to [servers.tw] api failed: UNAVAILABLE (server u then `nnnotes master decode -o ` (the regions serve different master data versions). `music-data --decoded-master` also reads the `MasterManifest.json` of the decoded files there (copy it from the download directory; a published master data snapshot may carry it). -- **CRI HCA keycode**: not a setting. `audio`, `story`, `live` and `web` read it from the APK's boot data; - `nnnotes crikey` shows whether one was found and can write it as a `.hcakey` file for vgmstream. +- **CRI HCA keycode**: not a setting. `export audio`, `export story`, `export live` and `site build` read it from the APK's boot data; + `nnnotes apk hca-key` shows whether one was found and can write it as a `.hcakey` file for vgmstream. - **Tools**: [vgmstream](https://vgmstream.org/) (`vgmstream-cli`), [FFmpeg](https://ffmpeg.org/) and, for - `web`, [Node.js](https://nodejs.org/) 20+ with a built + `site build`, [Node.js](https://nodejs.org/) 20+ with a built [ournotes-player](https://github.com/empty-sekai/ournotes-player). diff --git a/docs/contracts.md b/docs/contracts.md index 5c3f55a..48007f3 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -223,7 +223,7 @@ Documents written by particular stages (artifacts of no object: their ids are `< The catalog versions of a store, `nnnotes.catalogs/1` ([catalogs.schema.json](../src/nnnotes/schemas/catalogs.schema.json), `/catalogs/index.json`), and the difference between two of them, `nnnotes.catalog-diff/1` -([catalog-diff.schema.json](../src/nnnotes/schemas/catalog-diff.schema.json), `nnnotes catalogs diff --json`), are written +([catalog-diff.schema.json](../src/nnnotes/schemas/catalog-diff.schema.json), `nnnotes catalog diff --json`), are written by the commands. The music data file of `nnnotes music-data`, `nnnotes.music-data/2` diff --git a/docs/jp.md b/docs/jp.md index 43a95d5..755496f 100644 --- a/docs/jp.md +++ b/docs/jp.md @@ -64,10 +64,10 @@ models must use a separate site directory from international releases, since the nnnotes --region jp master version nnnotes --region jp master download --latest -o work/jp-master-bin nnnotes master decode work/jp-master-bin -o work/jp-master -nnnotes --region jp catalog --prefix Live/MusicScore/ --limit 10 -nnnotes --region jp pull Live/MusicScore/0001/0001_00 -nnnotes --region jp catalogs fetch -nnnotes --region jp export --select key:Image/Jacket/jacket_temporary --views none -o out/jp-sample +nnnotes --region jp catalog keys --prefix Live/MusicScore/ --limit 10 +nnnotes --region jp bundle pull Live/MusicScore/0001/0001_00 +nnnotes --region jp catalog fetch +nnnotes --region jp export assets --select key:Image/Jacket/jacket_temporary --views none -o out/jp-sample nnnotes --region jp music-data --decoded-master --no-bgm -o out/jp-music-data.json ``` @@ -79,12 +79,13 @@ the JP hash in `provenance.catalog.resourceHash`. JP catalogs are `catalog_main.bin` in a version/hash directory, without a language suffix. The HTTP payload can be gzip; nnnotes keeps its original bytes and SHA-256, and bounds decompression before parsing. Remote bundle and CRI locations use `{Fwk.Resource.RemoteAssetDir}`. These are indexed, resolved and downloaded through the selected -snapshot, including in `browse`, `catalogs`, `export`, `plan` and `run-stage`. +snapshot, including in `catalog browse`, the other `catalog` commands, `export assets`, `store plan` and +`store run`. The cache lives under `/jp////`. It cannot reuse the international catalog cache. Embedded files are further isolated under `apk//`, so an APK update cannot reuse old local bundles when the CDN snapshot stays unchanged. Each cached `catalog_main.bin` has a `catalog_main.bin.source.json` with its SHA-256 -and public source metadata. For an offline `--catalog` or `catalogs import`, copy the pair together. Credentials +and public source metadata. For an offline `--catalog` or `catalog import`, copy the pair together. Credentials are reacquired only when a missing file must be downloaded. Catalog database records include this source metadata, and JP catalog identity includes version/hash even when @@ -104,7 +105,7 @@ Live validation on 2026-09-30 used JP client 1.0.4 and local JP Android 1.0.3 da - 238 master tables downloaded with manifest hashes and decoded successfully; - a 73-note chart and a 512×512 jacket read successfully; - one Live2D runtime model and one complete story exported; the story's 22 FLAC files passed full FFmpeg decoding; -- selected `export` produced six files without failed tasks; +- selected `export assets` produced six files without failed tasks; - `music-data --decoded-master --no-bgm` built 85 songs / 340 charts, including the pinned deck model's statistics, with no unplayable chart; output passed JSON Schema validation. diff --git a/docs/music-data.md b/docs/music-data.md index 64c7e00..9e6fff5 100644 --- a/docs/music-data.md +++ b/docs/music-data.md @@ -27,7 +27,7 @@ nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--ful skills, bonuses, scores and events ([the deck input](#the-deck-input---full)), for tools that run a deck model of their own. - `--no-deck`: do not run the deck model; every chart's `deck` is null. A full statistics build evaluates many - complete lives per chart. `--workers N` sets the native worker count (default: every available processor). + complete lives per chart. `--workers N` sets the native worker count (default: every available processor; `0`: one thread). - `--seeds N`: list this many replay seeds for a chart with a luck range (default 8). Expectations and aptitude use the model's independent nominal probabilities, not a sample of these seeds. Increasing this option changes the replay examples without changing the expected scores or their numerical enclosures. @@ -103,7 +103,7 @@ A **text** is an object with one string per language of `languages` (`{"ja": ... |---|---| | `region` | the region whose master data this is (a configured region name), or `embedded` for the APK's master data | | `client.versionName`, `client.versionCode` | the APK's version name and code (null without `[paths] apk`) | -| `catalog.resourceVersion` | the resource version used to select the downloaded catalog, or recorded for an explicit file in the catalog store (`nnnotes catalogs fetch` / `import`); null when unknown | +| `catalog.resourceVersion` | the resource version used to select the downloaded catalog, or recorded for an explicit file in the catalog store (`nnnotes catalog fetch` / `import`); null when unknown | | `catalog.sha256` | SHA-256 of the remote catalog file the charts were read with | | `master.source` | `api` (`--master-files`, `--decoded-master`: the region's files) or `embedded` (`--apk-master`) | | `master.version` | the `version` of the master data manifest | diff --git a/docs/stages.md b/docs/stages.md index 153f1e9..16d9e9f 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -25,8 +25,8 @@ gzip input and Japanese remote asset placeholders). | `sprite.crop` | a SpriteAtlas | the atlas' JSON, the images of its textures, the metas of the sprites of other bundles it packs | the sprites' images | | `cri.audio` | ACB content | `acb` (a raw file of the catalog, or the `acb` artifact of a cue sheet held in a bundle), `awb` (an AWB the catalog pairs with it), `boot` (the game's boot data: the key's source) | FLAC per stream, `cues.json`, `streams.json` | | `cri.movie` | USM content | `usm` (a raw file of the catalog, or the `usm` artifact of a movie held in a bundle), `boot` | every stream as stored (video, alpha, audio, subtitles), `movie.mkv` (or `movie.webm`) | -| `live2d.model` | a Live2D model key (`Character/Live2D///model/`) whose bundle is selected | `bundle:`: the bundles of the key's dependency closure in the order the extractor loads them (the order is part of the key); context `internalId` (the key's location); atom `live2d.extract_model` (the extractor of `nnnotes live2d`, run unchanged) | `live2d.model:#`: every file `nnnotes live2d` writes for the model, `` relative to the model's directory; placed under `/` | -| `spine.skeleton` | a SkeletonDataAsset of a selected bundle, by its stable address `:` (in a bundle of several serialized files the file's tag follows the bundle name, as in `link.addresses`) | `bundle:`: the bundles holding the serialized files the asset's file reaches through its externals; context `object` (the asset's object id); atom `spot.write_skeleton` (the Spine writer of `nnnotes spot`) | `spine.skeleton:#`: the skeleton (`.json` or `.skel`), its atlases (`.atlas`) and their page images, the bytes `nnnotes spot` writes into `spine/`; one item for the asset | +| `live2d.model` | a Live2D model key (`Character/Live2D///model/`) whose bundle is selected | `bundle:`: the bundles of the key's dependency closure in the order the extractor loads them (the order is part of the key); context `internalId` (the key's location); atom `live2d.extract_model` (the extractor of `nnnotes export live2d`, run unchanged) | `live2d.model:#`: every file `nnnotes export live2d` writes for the model, `` relative to the model's directory; placed under `/` | +| `spine.skeleton` | a SkeletonDataAsset of a selected bundle, by its stable address `:` (in a bundle of several serialized files the file's tag follows the bundle name, as in `link.addresses`) | `bundle:`: the bundles holding the serialized files the asset's file reaches through its externals; context `object` (the asset's object id); atom `spot.write_skeleton` (the Spine writer of `nnnotes export spot`) | `spine.skeleton:#`: the skeleton (`.json` or `.skel`), its atlases (`.atlas`) and their page images, the bytes `nnnotes export spot` writes into `spine/`; one item for the asset | | `link.addresses` | global | catalog index, censuses | key -> objects, serialized file -> bundle, bundle -> keys | | `link.artifacts` | global | the above, the results | object id -> artifact ids | | `view.` | a catalog (`main`) | `master:
` per master table it reads, `rules` (the view's rule and the resolvers it uses), `addresses` (the part of the address table it read: the keys looked up and the prefixes listed); no parameters, atoms or context | `view.:#view` (facts: rows, entries, gaps, unreferenced) | @@ -51,7 +51,7 @@ with its reason code; any other error of a converter gives it with `generic.erro | Mesh | `mesh.glb/1` | `mesh`: binary glTF in the mesh's local space | `mesh.arrays`, `gltf.write` | see `gltf.write` below; limits of the skin and blend shapes as issues of the artifact | | GameObject (a root) | `prefab.json/1` | `prefab`: the hierarchy's nodes (path, path ids, transform) with their components' fields | | the objects below it are `contained`; a root of a scene's file names its scene | | Material | `material.json/1` | `json`: shader, keywords, textures, floats, colours | | | -| Shader | `shader.json/1` | `json`: the parsed form (the bytes of `nnnotes shader`'s `.json`), `program:/sp

__`: each compiled sub-program as stored (`.glsl`, `.metal`, `.vkprog`), `index`: the variants (platform, subshader, pass, stage, GPU program type, keywords, program role), `typetree`: the typetree JSON | | a shader without a parsed form: its typetree with `generic.error` | +| Shader | `shader.json/1` | `json`: the parsed form (the bytes of `nnnotes export shader`'s `.json`), `program:/sp

__`: each compiled sub-program as stored (`.glsl`, `.metal`, `.vkprog`), `index`: the variants (platform, subshader, pass, stage, GPU program type, keywords, program role), `typetree`: the typetree JSON | | a shader without a parsed form: its typetree with `generic.error` | | AnimationClip | `clip.json/1` | `json`: the Mecanim clip data with its bindings resolved in the bundle | | a legacy or compressed clip: `generic.clip.legacy` | | AnimatorController | `controller.json/1` | `json`: layers, state machines, parameters | | an AnimatorOverrideController: `generic.controller.override` | | MonoBehaviour (not in a hierarchy) | `mono.json/2`, `cri.usm/1` | `json`: `$script` (assembly, namespace, class) and the typetree; a cue sheet asset also `acb`, a movie asset holding its USM also `usm` (see below) | `sniff` | the class from the task's script table | @@ -111,7 +111,7 @@ bundle. | Stage | Parameters | Atoms | Artifacts (`#`) | |---|---|---|---| -| `cri.audio` | `flac.level` (8): ffmpeg's FLAC compression level 0-12 (every level decodes to the same samples) | `hca.decode` (vgmstream), `flac.encode` (ffmpeg) | one FLAC per vgmstream stream (a name repeated within the sheet gets `_`; facts: stream, name, sample rate, channels, samples, loop points), `cues.json` (the first stream of each name), `streams.json` (every stream in order): the bytes `nnnotes audio` writes for the sheet (FLAC) | +| `cri.audio` | `flac.level` (8): ffmpeg's FLAC compression level 0-12 (every level decodes to the same samples) | `hca.decode` (vgmstream), `flac.encode` (ffmpeg) | one FLAC per vgmstream stream (a name repeated within the sheet gets `_`; facts: stream, name, sample rate, channels, samples, loop points), `cues.json` (the first stream of each name), `streams.json` (every stream in order): the bytes `nnnotes export audio` writes for the sheet (FLAC) | | `cri.movie` | `format`: `mkv` (default; with `flac.level`) or `webm` | `usm.demux` (nnnotes: `numpy-/2`), `movie.mux` (ffmpeg), `hca.decode` (vgmstream) | per stream as stored and unmasked (a stream stored without its mask is recognised by its structure), named by its kind and, past channel 0, `_`: `video.` and `alpha.` (`ivf`: VP9 in IVF; `m1v`: MPEG-1 elementary stream, CRI's Sofdec.Prime; `h264`: H.264 Annex B; facts: codec, size, display size, frame rate, frames, an alpha stream's alpha type), `audio.` (`adx`; `hca`, still enciphered as stored; facts: channels, sample rate, samples or blocks, cipher), `subtitle.json` (the records as stored: language, time unit, start, duration, the text decoded as UTF-8 or `textHex`) with `subtitle.srt` and `subtitle.vtt`; `movie.mkv`: every video and alpha stream copied (the alpha as a second video track titled `alpha`), every audio stream as FLAC (HCA decoded by vgmstream with the boot data's key), every subtitle channel as a SubRip track (facts: the tracks); with `format: webm` `movie.webm`, the story's form: the first video stream, VP9 copied, or re-encoded as VP9 (libvpx, CRF 20) when it is not VP9 or has an alpha stream, which becomes its alpha channel cropped to the display size; the first audio stream as Opus; no subtitles | `cri.movie` in players: a player shows the first video track of `movie.mkv` and no player composites the `alpha` diff --git a/docs/views.md b/docs/views.md index 8a90ffe..b3cb362 100644 --- a/docs/views.md +++ b/docs/views.md @@ -108,8 +108,8 @@ resolver's name). ## Stages -`nnnotes.pipeline.views:stages` gives one stage per view, `view.` (docs/stages.md), run by `nnnotes export --views` -and planned by `nnnotes plan`: +`nnnotes.pipeline.views:stages` gives one stage per view, `view.` (docs/stages.md), run by `nnnotes export assets +--views` and planned by `nnnotes store plan`: - Subjects: the catalog subjects of `link.addresses` (the export command has one, `main`), when master data is set and the view is selected (the planning facts `master` and `views`). diff --git a/scripts/atom_eval.py b/scripts/atom_eval.py index e3543ce..327520e 100644 --- a/scripts/atom_eval.py +++ b/scripts/atom_eval.py @@ -412,7 +412,7 @@ def merge(total: dict, rec: dict) -> None: # ---------------------------------------------------------------- sample def cmd_sample(a) -> dict: - from nnnotes.cli.cli_assets import bundle_group + from nnnotes.cli.assets import bundle_group from nnnotes.pipeline import contract root = Path(a.dir) diff --git a/src/nnnotes/cli/__init__.py b/src/nnnotes/cli/__init__.py index b8958e8..0020406 100644 --- a/src/nnnotes/cli/__init__.py +++ b/src/nnnotes/cli/__init__.py @@ -1,58 +1,32 @@ -"""nnnotes command line. +"""nnnotes command line: `nnnotes `. Every setting (keys, the CDN base of each region, the paths of your own data and tools) comes from the config file, -the environment or the flags below; see config.py and nnnotes.example.toml (`nnnotes config init` writes it). JSON -summaries are printed as UTF-8 whatever the console encoding. +the environment or the global options; see source/config.py and source/nnnotes.example.toml (`nnnotes config init` +writes it). The global options, --json among them, go before or after the command's name. A command prints a short +summary on standard output, or with --json its report as JSON (UTF-8 whatever the console encoding); notes and +errors go to standard error. Exit status 2 is a usage or settings error. - nnnotes config init [--user | --file F] [--set SECTION.KEY=VALUE ...] - nnnotes config set SECTION.KEY VALUE - nnnotes config check [--json] - nnnotes config path [--json] - nnnotes catalog --prefix Spot/ --limit 40 - nnnotes browse [--port 8000] - nnnotes pull [ ...] + nnnotes config init|set|unset|check|path nnnotes servers [--show-hosts] - nnnotes master version - nnnotes master decode

-o - nnnotes master download --version | --latest -o - nnnotes adv 10462 -o out/adv_10462.json - nnnotes story 10462 -o out/story_10462 [--models out/live2d] [--force] - nnnotes live2d -o out/live2d/x - nnnotes spot 10001 -o out/spot_10001 - nnnotes room -o out/room.glb - nnnotes shader --key | --apk-bundle -o out/shaders - nnnotes audio -o out/audio - nnnotes voices list [--from out/assets] [--character ] [--category ] [--source ] - [--episode ] - nnnotes voices search [--from out/assets] [--language ja] [--character ...] [--category ...] - nnnotes voices summary [--from out/assets] - nnnotes voices get -o out/voice [--from out/assets] [--format flac] - nnnotes crikey [--write ] - nnnotes player -o out/player.json - nnnotes live 100001 --difficulty expert [--band 1 | --leader-card ] [--fonts game] - -o out/live_100001 - nnnotes web out/site --player [--pair 100001:expert [--pair ...] | --all] [--format aac] - [--live2d [--live2d ...] | --all-live2d] - [--story 10462 [--story ...] | --all-stories] [--story-languages en,ja] [--font en=] - [--font emoji=] - [--region [--region ...] | --all-regions] - nnnotes music-data --master-files | --apk-master | --decoded-master [--full] [--no-deck] - [--no-gekisou-aptitude] [--stats-cache DIR] [--no-bgm] [--jackets DIR] - -o out/music-data.json[.gz] - nnnotes export -o out/assets [--select group: | key: | bundle: ...] [--layout original,cas] - nnnotes plan [--select ...] [--json] [--check] [--emit-tasks ] - nnnotes run-stage [...] - nnnotes catalogs list | import | fetch | diff - nnnotes store verify + nnnotes catalog keys|browse|versions|import|fetch|diff + nnnotes bundle pull KEY... + nnnotes master version|download|decode + nnnotes export assets|episode|story|live|live2d|spot|room|shader|audio|ui ... -o OUT + nnnotes apk settings -o FILE | hca-key [--write DIR] + nnnotes store plan|run|verify + nnnotes voices list|search|summary|get + nnnotes site build -o SITE + nnnotes music-data -o FILE + +Each group is a module of this package with a `register(subparsers)` function; a command imports what it runs when +it runs. """ from __future__ import annotations import argparse import os -import re import sys -from pathlib import Path # numpy's OpenBLAS starts one busy thread per CPU in every process that imports it, and no export uses its # parallelism. Set when the command line loads, before a command imports numpy; the worker processes of a build @@ -60,1184 +34,36 @@ os.environ.setdefault("OPENBLAS_NUM_THREADS", "1") from .. import __version__ -from ..core.jsonio import dumps, write_json -from ..pipeline import voices -from ..site.compress import DEFAULT_ENCODING, ENCODINGS -from ..site.webaudio import DEFAULT_AUDIO_FORMAT, WEB_AUDIO -from ..source.catalog import Catalog -from ..source.config import DEFAULT_FILE, ENV_CONFIG, Config, ConfigError, config_files, find_file, use, user_file +from ..source.config import ConfigError from ..source.gameapi import GameApiError -from ..source.session import Session -from . import cli_assets +from . import apk, bundle, catalog, config, export, master, musicdata, servers, site, store, voices +from .common import GLOBALS, Formatter, fill_globals, load_config -DIFFICULTY_CHOICES = ("easy", "normal", "hard", "expert") -AUDIO_CHOICES = ("flac", "ogg", "wav") -# command-line flags of settings: (section, key) -> (argparse dest, flag) -FLAG_SETTINGS = { - ("catalog", "region"): ("region", "--region"), - ("catalog", "language"): ("language", "--language"), - ("catalog", "version"): ("catalog_release", "--catalog-release"), - ("paths", "catalog"): ("catalog", "--catalog"), - ("paths", "cache"): ("cache", "--cache"), - ("paths", "master"): ("master", "--master"), - ("paths", "apk"): ("apk", "--apk"), - ("paths", "ffmpeg"): ("ffmpeg", "--ffmpeg"), - ("paths", "vgmstream"): ("vgmstream", "--vgmstream"), - ("paths", "node"): ("node", "--node"), - ("paths", "player"): ("player", "--player"), - ("paths", "store"): ("store", "--store"), -} - - -# ---------------------------------------------------------------- settings -> data -def load_config(args) -> Config: - overrides = {k: getattr(args, dest, None) for k, (dest, _) in FLAG_SETTINGS.items()} - cfg = Config.load( - getattr(args, "config", None), overrides=overrides, flags={k: flag for k, (_, flag) in FLAG_SETTINGS.items()} - ) - if getattr(args, "func", None) is cmd_config_check: - return use(cfg) # diagnose raw settings before applying runtime defaults - if cfg.provider() == "jp" and not cfg.has("catalog", "language"): - cfg = cfg.for_region(cfg.region()) - return use(cfg) - - -def known_keys(args, cat: Catalog, keys) -> None: - """A usage error (exit 2) naming the keys the catalog does not have.""" - unknown = [k for k in keys if not cat.has(k)] - if unknown: - args.usage(f"not a key of the catalog: {', '.join(unknown)}") - - -def known_row(args, md: Path, table: str, row_id: int, what: str) -> None: - """A usage error (exit 2) naming `row_id` when the master table `table` has no row with that `_id`.""" - from ..source.master import has_row - - try: - found = has_row(md, table, row_id) - except FileNotFoundError: - raise ConfigError( - f"master data {md}: no {table}.json (a directory written by `nnnotes master decode`)" - ) from None - if not found: - args.usage(f"{what} {row_id}: no {table} row with this id") - - -def _print_json(r) -> None: - """Print a JSON summary as UTF-8 bytes (a GBK / cp932 console cannot encode every string).""" - sys.stdout.flush() - sys.stdout.buffer.write(dumps(r, ensure_ascii=False, indent=1).encode("utf-8") + b"\n") - sys.stdout.buffer.flush() - - -# ---------------------------------------------------------------- commands -def cmd_catalog(args, cfg): - cat = Session(cfg).catalog(bundles=False) - ks = cat.keys(args.prefix) - for k in ks[: args.limit]: - print(k) - print(f"# {len(ks)} keys", file=sys.stderr) - - -def cmd_browse(args, cfg): - from .browse import Region, serve - - names = cfg.regions() - if not names: - raise ConfigError( - "no region configured: add a [servers.] table with `cdn` to the config file " - "(or set NNNOTES_SERVERS__CDN)" - ) - regions = [] - for r in names: - langs = cfg.get_list(f"servers.{r}", "languages") - if not langs: - raise cfg.missing(f"servers.{r}", "languages") - regions.append(Region(r, cfg.get(f"servers.{r}", "name") or r, cfg.cdn(r), langs, cfg)) - serve(regions, Session(cfg).bundle_key(), cfg.require_path("paths", "cache"), args.port, args.host) - - -def cmd_pull(args, cfg): - cat = Session(cfg).catalog() - known_keys(args, cat, args.keys) - for key in args.keys: - for p in cat.fetch_key(key): - print(p) - - -def cmd_master_decode(args, cfg): - from ..source import master - - key = Session(cfg).master_key() - files = master.input_files(args.inputs) - if not files: - raise SystemExit("nnnotes: no master data files (*.bin) among the inputs") - r = master.decode_files(files, Path(args.out), key, workers=args.workers) - _print_json(r) - if r["failed"]: - sys.exit(1) - - -def _tty() -> bool: - try: - return sys.stdin.isatty() and sys.stdout.isatty() - except (AttributeError, OSError, ValueError): - return False - - -def _config_target(args, default: Path | None) -> Path: - """The file a config command writes: --file, --user, else `default` (None: the file the commands read).""" - if args.file: - return Path(args.file) - if args.user: - target = user_file() - if target is None: - raise ConfigError( - "no per-user config directory: the environment sets neither APPDATA (Windows) nor " - "XDG_CONFIG_HOME / HOME" - ) - return target - if default is not None: - return default - target = find_file(args.config) - if target is None: - raise ConfigError( - "no config file is read (`nnnotes config path`): name one with --file or --user, or " - "write one with `nnnotes config init`" - ) - return target - - -def _config_value(args, name: str, text: str, stdin_used: list): - """(section, key, value) of `name` = `text` (`-`: read from standard input, once), checked.""" - from ..source import configfile - - setting, section = configfile.resolve(name) - if text == "-": - if stdin_used: - args.usage("only one value can be read from standard input") - stdin_used.append(name) - if setting.secret and _tty(): - import getpass - - text = getpass.getpass(f"[{section}] {setting.key} (hidden): ") - else: - text = sys.stdin.readline().rstrip("\r\n") - try: - return section, setting.key, configfile.parse_value(setting, text) - except ValueError as e: - raise ConfigError(f"setting {section}.{setting.key}: {e}") from None - - -def _config_note(target: Path) -> None: - read = find_file() - if read is None or read.absolute() != target.absolute(): - print( - f"# the commands read {read.absolute() if read else 'no config file'} first (`nnnotes config path`)", - file=sys.stderr, - ) - - -def cmd_config_init(args, cfg): - from ..source import configfile - - named = next((f for n, f in config_files(args.config)[:2] if f is not None), None) - target = _config_target(args, named or Path(DEFAULT_FILE)) - if target.exists() and not args.force: - raise ConfigError(f"config file {target} exists (--force overwrites it; `nnnotes config set` edits one value)") - values, regions, stdin_used = [], [], [] - for item in args.set or []: - name, sep, text = item.partition("=") - if not sep: - args.usage(f"--set {name}: give it as SECTION.KEY=VALUE") - section, key, value = _config_value(args, name.strip(), text, stdin_used) - values.append((section, key, value)) - region = section.split(".", 1)[1] if section.startswith("servers.") else None - if region and region not in regions: - regions.append(region) - if not args.set and not args.no_input and _tty(): - try: - values, regions = configfile.prompt() - except (KeyboardInterrupt, EOFError): - print("\nnnnotes: config init stopped, nothing written", file=sys.stderr) - sys.exit(1) - configfile.write(target, configfile.render(values, regions)) - print(target.absolute()) - print( - f"# {len(values)} settings written, the others empty; `nnnotes config check` shows the status of each", - file=sys.stderr, - ) - _config_note(target) - - -def _config_edit(args, value_text: str | None) -> None: - from ..source import configfile - - target = _config_target(args, None) - text = target.read_text(encoding="utf-8") if target.exists() else configfile.template().decode("utf-8") - if value_text is None: # unset: an empty value - setting, section = configfile.resolve(args.name) - section, key, value = section, setting.key, [] if setting.kind == "list" else "" - else: - section, key, value = _config_value(args, args.name, value_text, []) - configfile.write(target, configfile.edit(text, [(section, key, value)], str(target))) - print(f"{target.absolute()}: {section}.{key} {'set' if value else 'emptied'}") - _config_note(target) - - -def cmd_config_set(args, cfg): - _config_edit(args, args.value) - - -def cmd_config_unset(args, cfg): - _config_edit(args, None) - - -def cmd_config_check(args, cfg): - from ..source import configfile - - r = configfile.check(cfg) - if args.json: - _print_json(r) - else: - print(f"# file: {r['file']}" if r["file"] else "# no config file is read (`nnnotes config path`)") - for s in r["settings"]: - status = s["status"] + (f": {s['reason']}" if s.get("reason") else "") - print(f"{s['name']}\t{s.get('origin') or '-'}\t{status}") - sys.stdout.flush() - print(f"# {r['problems']} problems", file=sys.stderr) - if r["problems"]: - sys.exit(1) - - -def cmd_config_path(args, cfg): - from ..source import configfile - - files, read = [], None - for name, file in config_files(args.config): - if file is None: - state = "not given" - elif not file.is_file(): - state = "not found" - elif read is None: - read, state = file, "read" - else: - state = "found, not read" - files.append({"name": name, "path": str(file.absolute()) if file is not None else None, "state": state}) - if read is None and file is not None and name in ("--config", ENV_CONFIG): - break # a named file that is missing stops every command - if args.json: - _print_json({"schema": configfile.PATHS, "files": files, "read": str(read.absolute()) if read else None}) - return - for f in files: - print(f"{f['name']}\t{f['path'] or '-'}\t{f['state']}") - if read is None: - print("# no config file is read: the settings come from the environment and the flags only", file=sys.stderr) - - -def cmd_servers(args, cfg): - from ..source import gameapi - - try: - servers = gameapi.server_list(cfg) - except gameapi.GameApiError as e: - sys.exit(f"nnnotes: {e}") - configured = gameapi.configured_roots(cfg) - _print_json({"servers": [gameapi.server_summary(s, configured, args.show_hosts) for s in servers]}) - - -def cmd_master_version(args, cfg): - from ..source import gameapi - - region = cfg.region() - try: - v = Session(cfg).master_version(region) - except gameapi.GameApiError as e: - sys.exit(f"nnnotes: {e}") - result = {"region": region, "masterVersion": v.version, "resourceVersion": v.resource_version} - if v.resource_hash is not None: - result["resourceHash"] = v.resource_hash - _print_json(result) - - -def cmd_master_download(args, cfg): - from ..source import gameapi, master - - region = cfg.region() - cdn = cfg.cdn(region) - try: - if cfg.provider(region) == "jp": - from ..source.jp import Client, master_version - - client = Client(cfg, region) - observation = client.observe() - version = observation.version.version if args.latest else master_version(args.version) - r = master.download( - observation.cdn, - version, - Path(args.out), - workers=args.workers, - get=lambda url: client.get(url, master=version), - strict=True, - ) - else: - version = Session(cfg).master_version(region).version if args.latest else args.version - r = master.download(cdn, version, Path(args.out), workers=args.workers) - except (gameapi.GameApiError, master.DownloadError) as e: - sys.exit(f"nnnotes: {e}") - _print_json(r) - if r["failed"]: - sys.exit(1) - - -def cmd_adv(args, cfg): - from ..content import adv - - md = Session(cfg).master_dir() - known_row(args, md, "MasterAdv", args.adv_id, "episode") - cat = Session(cfg).catalog() - doc = adv.to_json(adv.extract(cat, md, args.adv_id)) - out = Path(args.out) - out.parent.mkdir(parents=True, exist_ok=True) - write_json(out, doc) - print(f"{out} ({doc['commandCount']} commands, {len(doc['resources'])} resources)") - - -def cmd_live2d(args, cfg): - from ..site import webmodel - from ..unity import live2d - - cfg.require_path("paths", "apk") # the Cubism component classes are read from the APK - cat = Session(cfg).catalog() - try: # a model id or a model key - (key,) = webmodel.catalog_models(cat, [args.key]).values() - except ValueError as e: - args.usage(str(e)) - r = live2d.extract_model(cat, key, Path(args.out)) - print(dumps(r, ensure_ascii=False)) - - -def cmd_spot(args, cfg): - from ..content import room, spot - from ..formats import shader - - cfg.require_path("paths", "apk") # the Spot / Spine component classes are read from the APK - md = Session(cfg).master_dir() - known_row(args, md, "MasterHomeSpot", args.spot_id, "spot") - cat = Session(cfg).catalog() - out = Path(args.out) - doc = spot.extract(cat, md, args.spot_id, out) - print(f"{out}/spot.json ({len(doc['characters'])} tap targets, {len(doc['skeletons'])} skeletons)") - bg = doc["master"]["_backgroundAssetPath"] - r = room.extract_room(cat, bg, out / "room.glb") - print(f"{out}/room.glb ({r['meshCount']} meshes, {r['inactiveMeshes']} inactive)") - bundles = cat.fetch_key(bg) + cat.fetch_key(doc["master"]["_situationAssetPath"]) - s = shader.dump(bundles, out / "shaders") - print(f"{out}/shaders/ ({s['count']} shaders, {s['variants']} variants)") - - -def cmd_room(args, cfg): - from ..content import room - - cat = Session(cfg).catalog() - known_keys(args, cat, [args.key]) - r = room.extract_room(cat, args.key, Path(args.out)) - print(f"{r['glb']} ({r['meshCount']} meshes, {r['inactiveMeshes']} inactive)") - - -def cmd_shader(args, cfg): - from ..formats import shader - - cat = Session(cfg).catalog() - if args.key: - known_keys(args, cat, [args.key]) - bundles = cat.fetch_key(args.key) - else: - if cat.apk is None: - cfg.require_path("paths", "apk") - bundles = [] - for s in args.apk_bundle: - try: - bundles.append(cat.apk_bundle(s)) - except KeyError as e: # none or several bundles match - args.usage(f"--apk-bundle {e.args[0]}") - r = shader.dump(bundles, Path(args.out)) - print(f"{r['count']} shaders, {r['variants']} variants") - for n in r["names"]: - print(f" {n}") - - -def cmd_audio(args, cfg): - from ..formats import cri - - flac = _flac_options(args) - cfg.require_path("paths", "apk") # the HCA keycode is read from the APK - cat = Session(cfg).catalog() - if not cat.has(f"Cri/Sound/{args.cue_sheet}"): - args.usage(f"cue sheet {args.cue_sheet}: no key Cri/Sound/{args.cue_sheet} in the catalog") - out = Path(args.out) - r = cri.decode(cat, args.cue_sheet, out, fmt=args.format, **flac) - print(f"{out} ({len(r)} cues)") - - -def cmd_crikey(args, cfg): - from ..formats import crikey - - cfg.require_path("paths", "apk") - k = crikey.find_key(Session(cfg).path("paths", "apk")) - print(f"HCA keycode found ({len(str(k))} digits)" if k else "no HCA keycode (decryption disabled)") - if args.write and k: - d = Path(args.write) - d.mkdir(parents=True, exist_ok=True) - print(crikey.write_hcakey(k, d)) - - -def cmd_player(args, cfg): - from ..unity.player import PlayerData - - g = PlayerData(Session(cfg).apk()).graphics() - out = Path(args.out) - out.parent.mkdir(parents=True, exist_ok=True) - write_json(out, g) - print(f"{out} ({g['colorSpace']}, {len(g['qualityLevels'])} quality levels, {len(g['renderers'])} renderers)") - - -def cmd_story(args, cfg): - from ..content import story - from ..site import webmodel - from ..source import languages - from ..unity.player import PlayerData - - if args.fonts == "game": - from ..unity.tmpfont import require_extra - - require_extra("--fonts game") - flac = _flac_options(args) - out = Path(os.path.abspath(args.out)) - models = Path(os.path.abspath(args.models or Path(args.out) / os.pardir / "live2d")) - try: - os.path.relpath(models, out) - except ValueError: - args.usage(f"--models {models}: not on the drive of {out} (story.json names it relative to the story)") - language = languages.check(cfg.require("catalog", "language")) # the story UI's language - md = Session(cfg).master_dir() - known_row(args, md, "MasterAdv", args.adv_id, "episode") - cat = Session(cfg).catalog() - player = PlayerData(Session(cfg).apk()) - source = webmodel.ModelDir(cat, player, models, force=args.force) - r = story.build( - cat, - md, - player, - args.adv_id, - out, - audio_format=args.format, - audio=not args.no_audio, - fonts=args.fonts, - models=source, - language=language, - audio_options=flac, - ) - _print_json({**r, "modelsBuilt": source.built, "modelsSkipped": source.skipped}) - - -def _flac_options(args) -> dict: - """The cri.decode options of --flac-level ({} without it); a level out of range, or the flag where no FLAC is - written, is a usage error.""" - level = args.flac_level - if level is None: - return {} - if not 0 <= level <= 12: - args.usage(f"--flac-level {level}: expected 0 to 12") - if args.format != "flac": - args.usage(f"--flac-level: --format {args.format} writes no FLAC") - if getattr(args, "no_audio", False): - args.usage("--flac-level: --no-audio decodes no audio") - return {"flac_level": level} - - -def _fonts_extra(fonts: str) -> None: - """`--fonts game` needs the optional `fonts` dependencies.""" - if fonts == "game": - from ..unity.tmpfont import require_extra - - require_extra("--fonts game") - - -def _live_options(args) -> dict: - """The --live-option specs as a request (liveoptions.parse_specs); a malformed spec is a usage error.""" - from ..content.liveoptions import OptionSpecError, parse_specs - - try: - return parse_specs(args.live_option) - except OptionSpecError as e: - args.usage(str(e)) - - -def cmd_live(args, cfg): - from ..content import live, liveoptions - from ..site.web import all_pairs - from ..source import languages - from ..unity.player import PlayerData - - _fonts_extra(args.fonts) - request = _live_options(args) - flac = _flac_options(args) - language = languages.check(cfg.require("catalog", "language")) - md = Session(cfg).master_dir() - known_row(args, md, "MasterLiveMusic", args.music_id, "music") - if (args.music_id, args.difficulty) not in all_pairs(md): - args.usage(f"music {args.music_id}: no {args.difficulty} chart (no MasterLiveMusicScore row)") - try: - options = liveoptions.resolve(request, md) - except liveoptions.OptionSpecError as e: - args.usage(str(e)) - cat = Session(cfg).catalog() - try: # an unknown --leader-card, a --band without its scene keys - live.resolve_band(cat, md, args.music_id, band=args.band, leader_card=args.leader_card) - except (KeyError, ValueError) as e: - args.usage(e.args[0] if e.args else type(e).__name__) - r = live.build( - cat, - md, - PlayerData(Session(cfg).apk()), - args.music_id, - args.difficulty, - Path(args.out), - audio_format=args.format, - band=args.band, - leader_card=args.leader_card, - language=language, - fonts=args.fonts, - options=options, - **flac, - ) - _print_json(r) - - -def cmd_web(args, cfg): - from ..content import liveoptions - from ..site import sitefiles, siteindex, storysite, web, webmodel - - live_options = _live_options(args) - charts, models = bool(args.pair or args.all), bool(args.live2d or args.all_live2d) - stories = bool(args.story or args.all_stories) - if not (charts or models or stories or args.player_only or args.reingest_json): - args.usage( - "one of the arguments --pair --all --live2d --all-live2d --story --all-stories --player-only " - "--reingest-json is required" - ) - if models and (args.player_only or args.reingest_json): - args.usage("--player-only and --reingest-json build nothing: leave out --live2d / --all-live2d") - if stories and (args.player_only or args.reingest_json): - args.usage("--player-only and --reingest-json build nothing: leave out --story / --all-stories") - if live_options and not charts: - args.usage("--live-option applies to charts: give --pair or --all") - player = cfg.path("paths", "player") - out = Path(args.out) - if args.player_only or args.reingest_json: - r = { - **(sitefiles.reingest_json(out, args.compress) if args.reingest_json else {}), - **sitefiles.write_player(out, sitefiles.check_player(player)), - **siteindex.write_index(out), - } - else: - sitefiles.check_player(player) - regions = None # the one [catalog] region - if args.web_regions or args.all_regions: - regions = sitefiles.site_regions(cfg, args.web_regions, args.all_regions) - if (stories or models) and regions and len({cfg.provider(r) for r in regions}) > 1: - raise ConfigError("build JP stories/models in a separate site directory from international releases") - if (stories or models) and regions: - cfg = use(cfg.for_region(regions[0])) - base = {"region": regions[0]} if regions else {} - unknown = web.unknown_pairs(cfg, args.pair, regions) if args.pair else [] - if unknown: - args.usage( - f"chart{'s' if len(unknown) > 1 else ''} {', '.join(f'{m}:{d}' for m, d in unknown)}: no " - f"MasterLiveMusicScore row in the master data of the site's region(s)" - ) - if live_options: # a value a region's master data does not have - try: - for md in sitefiles.region_masters(cfg, sitefiles.site_regions(cfg, regions)).values(): - liveoptions.resolve(live_options, md) - except liveoptions.OptionSpecError as e: - args.usage(str(e)) - r = {} - encoding = args.compress - if models: - cfg.require_path("paths", "apk") # the Cubism component classes and mask materials: the APK - try: - selected = webmodel.catalog_models(Session(cfg).catalog(bundles=False, **base), args.live2d) - except ValueError as e: - args.usage(str(e)) - r.update( - webmodel.build( - out, - selected, - cfg, - player, - force=args.force, - tmp_dir=args.tmp, - workers=args.workers, - encoding=encoding, - **base, - ) - ) - if charts: - _fonts_extra(args.fonts) - r.update( - web.build( - out, - None if args.all else args.pair, - cfg, - player, - args.format, - audio=not args.no_audio, - force=args.force, - tmp_dir=args.tmp, - workers=args.workers, - band=args.band, - leader_card=args.leader_card, - regions=regions, - fonts=args.fonts, - read_workers=args.read_workers, - live_options=live_options, - encoding=args.compress, - ) - ) - if stories: - from ..unity.tmpfont import require_extra - - require_extra("--story / --all-stories") # the font assets of the story text (open and game) - cfg.require_path("paths", "apk") # the ADV settings and UI are embedded content of the APK - unknown = storysite.unknown_stories(cfg, args.story, regions) if args.story else [] - if unknown: - noun = "stories" if len(unknown) > 1 else "story" - args.usage( - f"{noun} {', '.join(map(str, unknown))}: no MasterAdv row in the master data of the " - f"site's region(s)" - ) - r.update( - storysite.build( - out, - args.story, - cfg, - player, - args.format, - audio=not args.no_audio, - force=args.force, - tmp_dir=args.tmp, - workers=args.workers, - regions=regions, - story_languages=args.story_languages, - fonts=args.fonts, - fonts_flags=dict(args.font or []), - encoding=encoding, - ) - ) - _print_json(r) - if ( - r.get("failed") - or r.get("modelsFailed") - or r.get("storiesFailed") - or (r.get("storyModels") or {}).get("modelsFailed") - ): - sys.exit(1) - - -def cmd_music_data(args, cfg): - from .. import musicdata - from ..musicdata import deckdata - - if args.apk_master: - cfg.require_path("paths", "apk") # the master data files ship in the APK - apk = Session(cfg).path("paths", "apk") - try: - deck = ( - None - if args.no_deck - else musicdata.Deck( - seeds=args.seeds, workers=args.workers, aptitude=not args.no_gekisou_aptitude, cache=args.stats_cache - ) - ) - if args.apk_master: - src, region = deckdata.apk_master(apk), deckdata.EMBEDDED - elif args.decoded_master: # decoded elsewhere: no master key - src, region = deckdata.decoded_master(Session(cfg).master_dir()), cfg.region() - else: - src, region = deckdata.master_files(Path(args.master_files)), cfg.region() - key = None if src.decoded else Session(cfg).master_key() - cat = Session(cfg).catalog() - r = musicdata.export( - Path(args.out), - src, - key, - deckdata.catalog_fetch(cat), - None if args.no_bgm else musicdata.catalog_bgm(cat), - region=region, - client=deckdata.apk_client(apk) if apk is not None else {}, - catalog=deckdata.catalog_info(cat, cli_assets.store_root(args, cfg)), - deck=deck, - full=args.full, - jacket=musicdata.catalog_jacket(cat) if args.jackets else None, - jackets_dir=args.jackets, - replay_dir=args.replay_dir, - replay_engine=args.replay_engine, - recommend_engine=args.recommend_engine, - ) - except (deckdata.DeckDataError, musicdata.MusicDataError) as e: - sys.exit(f"nnnotes: {e}") - _print_json(r) - - -# ---------------------------------------------------------------- parser -def parse_pair(s: str) -> tuple[int, str]: - m = re.fullmatch(r"(\d+)[:_](easy|normal|hard|expert)", s) - if not m: - raise argparse.ArgumentTypeError(f"{s}: expected :") - return int(m.group(1)), m.group(2) - - -def parse_languages(s: str) -> list[str]: - from ..source.languages import LANGUAGES - - codes = [c.strip() for c in s.split(",") if c.strip()] - if not codes or any(c not in LANGUAGES for c in codes): - raise argparse.ArgumentTypeError(f"{s}: expected languages from {', '.join(LANGUAGES)}, comma-separated") - return codes - - -EMOJI_FONT = "emoji" # --font emoji=, as storysite.EMOJI_FONT - - -def parse_font(s: str) -> tuple[str, str]: - from ..source.languages import LANGUAGES - - lang, sep, path = s.partition("=") - if not sep or (lang not in LANGUAGES and lang != EMOJI_FONT) or not path: - raise argparse.ArgumentTypeError( - f"{s}: expected = or {EMOJI_FONT}=, " - f"language one of {', '.join(LANGUAGES)}" - ) - return lang, path - - -def _live_option_arg(c) -> None: - c.add_argument( - "--live-option", - action="append", - metavar="OPTION[=VALUES]", - help="also export the files of a Live option's variants (repeatable): MirrorChart, " - "MeasureLineDisplay, NoteDesignId, NoteEffectId, LiveQuality, NoteSePatternId (every value " - "of the master data, or =v,v...), defaults (the option defaults and ranges only) or all", - ) - - -def _band_args(c) -> None: - g = c.add_mutually_exclusive_group() - g.add_argument( - "--band", - type=int, - help="band of the LightWeight background and start timeline " - "(default: band of the music's first vocal character)", - ) - g.add_argument("--leader-card", type=int, help="deck centre MasterMemberCard id (its character's band)") - - -def _fonts_arg(c) -> None: - c.add_argument( - "--fonts", - default="open", - choices=("open", "game"), - help="start canvas text: open (default): layout and style only, no font data (your fonts are not " - "used); game: also the game's TMP fonts (needs the 'fonts' extra)", - ) - - -def _audio_args(c) -> None: - """--format and --flac-level of a command that decodes cue sheets.""" - c.add_argument("--format", default="flac", choices=AUDIO_CHOICES, help="audio file format (default flac)") - c.add_argument( - "--flac-level", - type=int, - metavar="N", - help="FLAC compression level 0-12 of --format flac (default 8; every level decodes to the same samples)", - ) - - -def _out(c, what: str, required: bool = True) -> None: - c.add_argument("-o", "--out", required=required, help=what) - - -def cmd_ui(args, cfg): - from ..content import ui - - ui.command(args, cfg) +GROUPS = (config, servers, catalog, bundle, master, export, apk, store, voices, site, musicdata) def build_parser() -> argparse.ArgumentParser: - p = argparse.ArgumentParser(prog="nnnotes", description="BanG Dream! Our Notes data toolkit") - p.add_argument("--version", action="version", version=f"nnnotes {__version__}") - p.add_argument("--config", help="TOML config file (else NNNOTES_CONFIG, else ./nnnotes.toml)") - p.add_argument("--region", help="region: a [servers.] table ([catalog] region)") - p.add_argument( - "--language", help="catalog and client language: ja, en, zh-Hant, zh-Hans or ko ([catalog] language)" - ) - p.add_argument("--catalog", help="catalog .bin file ([paths] catalog; else downloaded into the cache)") - p.add_argument( - "--catalog-release", - help="pin an international resource version ([catalog] version; else API discovery when configured, " - "otherwise main)", - ) - p.add_argument("--cache", help="cache directory ([paths] cache)") - p.add_argument("--master", help="decoded master data directory ([paths] master)") - p.add_argument("--apk", help="base.apk ([paths] apk)") - p.add_argument("--ffmpeg", help="ffmpeg executable ([paths] ffmpeg; else on PATH)") - p.add_argument("--vgmstream", help="vgmstream-cli executable ([paths] vgmstream; else on PATH)") - p.add_argument("--node", help="Node.js executable ([paths] node; else on PATH)") - sub = p.add_subparsers(dest="cmd", required=True, metavar="") - - c = sub.add_parser("config", help="the config file: write it, set one value, check the settings, show where it is") - csub = c.add_subparsers(dest="config_cmd", required=True, metavar="") - - def target(m, what): - g = m.add_mutually_exclusive_group() - g.add_argument("--file", help=f"the config file to {what}") - g.add_argument( - "--user", - action="store_true", - help=f"{what} the per-user config file, read from any directory (`config path` shows it)", - ) - - m = csub.add_parser( - "init", - help="write a config file from the template: the values are asked in a terminal, " - "else taken from --set (the other settings empty)", - ) - target(m, f"write (default: the file --config or NNNOTES_CONFIG names, else ./{DEFAULT_FILE})") - m.add_argument( - "--set", - action="append", - metavar="SECTION.KEY=VALUE", - help="a value to write (repeatable; no questions then): VALUE - reads it from standard input; a " - "list comma-separated; a path is written absolute", - ) - m.add_argument("--no-input", action="store_true", help="ask nothing, also in a terminal") - m.add_argument("--force", action="store_true", help="overwrite an existing file") - m.set_defaults(func=cmd_config_init, reads_config=False, usage=m.error) - m = csub.add_parser( - "set", - help="set one value of the config file the commands read (or of --file / --user; " - "a missing one is made from the template); comments and other lines stay", + p = argparse.ArgumentParser( + prog="nnnotes", description="BanG Dream! Our Notes data toolkit", parents=[GLOBALS], formatter_class=Formatter ) - m.add_argument("name", metavar="SECTION.KEY", help="the setting: bundle.key, servers.tw.cdn, paths.fonts.ja, ...") - m.add_argument( - "value", - help="the value (a list comma-separated); - reads it from standard input, without echo " - "for the keys in a terminal", - ) - target(m, "edit") - m.set_defaults(func=cmd_config_set, reads_config=False, usage=m.error) - m = csub.add_parser("unset", help="empty one value of the config file (an empty value counts as unset)") - m.add_argument("name", metavar="SECTION.KEY", help="the setting") - target(m, "edit") - m.set_defaults(func=cmd_config_unset, reads_config=False, usage=m.error) - m = csub.add_parser( - "check", - help="every setting: where it comes from and whether it is valid, never its value; " - "exit status 1 when one is invalid, not found or unknown", - ) - m.add_argument("--json", action="store_true", help="the report as JSON (nnnotes.config-check/1)") - m.set_defaults(func=cmd_config_check) - m = csub.add_parser("path", help="the config files looked up, in order, and the one the commands read") - m.add_argument("--json", action="store_true", help="the list as JSON (nnnotes.config-path/1)") - m.set_defaults(func=cmd_config_path, reads_config=False) - - c = sub.add_parser("catalog", help="list addressable keys") - c.add_argument("--prefix", default="", help="only the keys that start with this prefix") - c.add_argument( - "--limit", type=int, default=200, help="print at most N keys (default 200; the total goes to stderr)" - ) - c.set_defaults(func=cmd_catalog) - - c = sub.add_parser("browse", help="browse the configured regions' catalogs and bundles in a local web page") - c.add_argument("--port", type=int, default=8000, help="port (default 8000)") - c.add_argument("--host", default="127.0.0.1", help="address to listen on (default 127.0.0.1)") - c.set_defaults(func=cmd_browse) - - c = sub.add_parser("pull", help="fetch the bundle closure of keys into the cache") - c.add_argument("keys", nargs="+", metavar="KEY", help="addressable keys (`catalog` lists them)") - c.set_defaults(func=cmd_pull, usage=c.error) - - c = sub.add_parser("servers", help="the server list of the bootstrap API root: regions, their CDN and API roots") - c.add_argument("--show-hosts", action="store_true", help="print the CDN and API roots, not only their counts") - c.set_defaults(func=cmd_servers) - - c = sub.add_parser("master", help="master data files") - msub = c.add_subparsers(dest="master_cmd", required=True, metavar="") - m = msub.add_parser("version", help="the master data and resource version the region serves now (game API)") - m.set_defaults(func=cmd_master_version) - m = msub.add_parser("decode", help="master data .bin files ->
.json") - m.add_argument("inputs", nargs="+", type=Path, help="directories (their *.bin files) or files") - _out(m, "output directory") - m.add_argument("--workers", type=int, default=8, help="parallel decodes") - m.set_defaults(func=cmd_master_decode) - m = msub.add_parser("download", help="a master data version from the region's CDN (SHA-256 checked)") - g = m.add_mutually_exclusive_group(required=True) - g.add_argument("--version", help="master data version") - g.add_argument("--latest", action="store_true", help="the version the region serves now (`master version`)") - _out(m, "directory for MasterManifest.json and the .bin files") - m.add_argument("--workers", type=int, default=16, help="parallel downloads") - m.set_defaults(func=cmd_master_download) - - c = sub.add_parser("adv", help="ADV episode -> JSON") - c.add_argument("adv_id", type=int, help="MasterAdv id of the episode") - _out(c, "output .json file") - c.set_defaults(func=cmd_adv, usage=c.error) - - c = sub.add_parser( - "story", help="ADV episode -> story dir (episode, audio, scene, UI, media, videos) + its Live2D models" - ) - c.add_argument("adv_id", type=int, help="MasterAdv id of the episode") - _out(c, "output directory") - c.add_argument( - "--models", metavar="DIR", help="directory of the Live2D models, one / each (default: OUT/../live2d)" - ) - c.add_argument("--force", action="store_true", help="export the models again whose directory exists") - _audio_args(c) - c.add_argument("--no-audio", action="store_true", help="do not decode the cue sheets") - c.add_argument( - "--fonts", - default="open", - choices=("open", "game"), - help="open (default): text layout and style only, no font data; game: also the game's TMP fonts " - "(needs the 'fonts' extra)", - ) - c.set_defaults(func=cmd_story, usage=c.error) - - c = sub.add_parser("live2d", help="Live2D model -> runtime model dir") - c.add_argument("key", help="model key Character/Live2D///model/, or its model id ") - _out(c, "output directory") - c.set_defaults(func=cmd_live2d, usage=c.error) - - c = sub.add_parser("spot", help="spot -> spot.json + Spine + room.glb + shaders") - c.add_argument("spot_id", type=int, help="MasterSpot id") - _out(c, "output directory") - c.set_defaults(func=cmd_spot, usage=c.error) - - c = sub.add_parser("room", help="background prefab -> glb") - c.add_argument("key", help="addressable key of the prefab") - _out(c, "output .glb file") - c.set_defaults(func=cmd_room, usage=c.error) - - c = sub.add_parser("shader", help="dump shaders of a key's closure or of APK bundles") - g = c.add_mutually_exclusive_group(required=True) - g.add_argument("--key", help="addressable key: the shaders of its bundle closure") - g.add_argument("--apk-bundle", nargs="+", help="substring(s) of APK bundle file names") - _out(c, "output directory") - c.set_defaults(func=cmd_shader, usage=c.error) - - c = sub.add_parser("audio", help="decode a CRI cue sheet") - c.add_argument("cue_sheet", help="cue sheet name: the key Cri/Sound/ of the catalog") - _out(c, "output directory") - _audio_args(c) - c.set_defaults(func=cmd_audio, usage=c.error) - - c = sub.add_parser("crikey", help="find the HCA keycode in base.apk") - c.add_argument("--write", help="directory to write .hcakey into") - c.set_defaults(func=cmd_crikey) - - c = sub.add_parser("player", help="player graphics settings -> JSON") - _out(c, "output .json file") - c.set_defaults(func=cmd_player) - - c = sub.add_parser("live", help="live (music + difficulty) -> self-contained live dir") - c.add_argument("music_id", type=int, help="MasterLiveMusic id") - c.add_argument( - "--difficulty", default="expert", choices=DIFFICULTY_CHOICES, help="chart difficulty (default expert)" - ) - _audio_args(c) - _fonts_arg(c) - _band_args(c) - _live_option_arg(c) - _out(c, "output directory") - c.set_defaults(func=cmd_live, usage=c.error) - - c = sub.add_parser( - "web", - help="live charts, Live2D models and stories -> static site for ournotes-player " - "(shared player + per-chart / per-model / per-story data)", - ) - c.add_argument("out", help="site directory") - c.add_argument("--player", help="ournotes-player checkout (built) or installed package ([paths] player)") - g = c.add_mutually_exclusive_group() - g.add_argument("--pair", type=parse_pair, action="append", help=": (repeatable)") - g.add_argument("--all", action="store_true", help="every MasterLiveMusic x difficulty") - g.add_argument( - "--player-only", action="store_true", help="rewrite the player files, charts.json and models.json only" - ) - g.add_argument("--reingest-json", action="store_true", help="store every chart's and model's JSON files again") - m = c.add_mutually_exclusive_group() - m.add_argument( - "--live2d", - action="append", - metavar="MODEL", - help="Live2D model id or key Character/Live2D///model/ (repeatable)", - ) - m.add_argument("--all-live2d", action="store_true", help="every Live2D model of the catalog") - c.add_argument("--format", default=DEFAULT_AUDIO_FORMAT, choices=tuple(WEB_AUDIO), help="BGM format") - c.add_argument("--no-audio", action="store_true", help="export no audio files") - c.add_argument( - "--compress", - default=DEFAULT_ENCODING, - choices=ENCODINGS, - help="encoding of the compressible assets (JSON, shaders, moc3, ...) where it makes them " - "smaller: gzip (default), br (brotli) or none", - ) - c.add_argument( - "--force", - action="store_true", - help="rebuild charts, models and stories whose manifest exists (stories: also their models)", - ) - c.add_argument("--tmp", help="directory for the temporary live and model builds (default .tmp)") - c.add_argument( - "--workers", - type=int, - help="parallel music / model processes (default a quarter of the CPUs, up to 8 / up to 4; 1 = this process)", - ) - c.add_argument("--read-workers", type=int, help="chart read sets run at a time (default half the CPUs, up to 16)") - t = c.add_mutually_exclusive_group() - t.add_argument( - "--story", type=int, action="append", metavar="ADV_ID", help="story episode: its MasterAdv id (repeatable)" - ) - t.add_argument("--all-stories", action="store_true", help="every story episode (MasterAdv) of the master data") - c.add_argument( - "--story-languages", - type=parse_languages, - metavar="LANGS", - help="languages of the stories' text, comma-separated (default: ja,en,zh-Hant,zh-Hans,ko)", - ) - c.add_argument( - "--font", - type=parse_font, - action="append", - metavar="LANG=PATH", - help="font file the story text of a language is drawn with (repeatable; [paths] fonts.); " - "emoji=PATH: the colour emoji font the emoji sprites are drawn from ([paths] fonts.emoji)", - ) - r = c.add_mutually_exclusive_group() - r.add_argument( - "--region", - dest="web_regions", - action="append", - metavar="REGION", - help="a region the site serves: a [servers.] table (repeatable; the first is the base; " - "default: [catalog] region)", - ) - r.add_argument("--all-regions", action="store_true", help="every configured region") - c.add_argument( - "--fonts", - default="open", - choices=("open", "game"), - help="text of the start canvas and the stories: open (default): the start canvas has layout and " - "style only, the stories glyphs from your font files (--font); game: the game's TMP fonts " - "(needs the 'fonts' extra)", - ) - _band_args(c) - _live_option_arg(c) - c.set_defaults(func=cmd_web, usage=c.error) + p.add_argument("--version", action="version", version=f"nnnotes {__version__}") + sub = p.add_subparsers(dest="command", required=True, metavar="") + for g in GROUPS: + g.register(sub) + return p - c = sub.add_parser( - "music-data", - help="every live song and chart: metadata in every language, chart facts and " - "the deck model's chart statistics -> one JSON file", - ) - g = c.add_mutually_exclusive_group(required=True) - g.add_argument( - "--master-files", - metavar="DIR", - help="master data files as served: MasterManifest.json and the .bin files it lists (`master download`)", - ) - g.add_argument("--apk-master", action="store_true", help="the master data files of base.apk ([paths] apk)") - g.add_argument( - "--decoded-master", - action="store_true", - help="decoded master data ([paths] master or --master) with the MasterManifest.json of the files " - "it was decoded from; no master key", - ) - c.add_argument( - "--full", - action="store_true", - help="also write the deck model's input: every chart's runtime notes and the master data tables " - "about cards, skills, bonuses, scores and events", - ) - c.add_argument("--no-deck", action="store_true", help="do not run the deck model (every chart's deck is null)") - c.add_argument( - "--seeds", - type=int, - default=8, - metavar="N", - help="replay seeds for a chart with a luck range (default 8); expectations do not use sampling", - ) - c.add_argument("--workers", type=int, metavar="N", help="threads measuring charts (default: every processor)") - c.add_argument( - "--no-gekisou-aptitude", - action="store_true", - help="leave out the charts' Gekisou aptitude (every gekisouAptitude is null)", - ) - c.add_argument( - "--stats-cache", - type=Path, - metavar="DIR", - help="reuse native simulation programs and runs in DIR across builds; " - "completed cache entries survive interrupted builds", - ) - c.add_argument( - "--no-bgm", action="store_true", help="do not read the BGM cue sheets (every song's bgm.length is null)" - ) - c.add_argument( - "--replay-dir", - metavar="DIR", - help="write canonical runtime DeckData, per-chart inputs and replay manifest under the output directory", - ) - c.add_argument( - "--replay-engine", - metavar="PKG", - help="the deck model's replay WASM release package (ournotes-replay-wasm-vVERSION.tar.gz or its " - "directory); its web JS/WASM and build-info.json are copied into --replay-dir", - ) - c.add_argument( - "--recommend-engine", - metavar="PKG", - help="the deck model's recommendation WASM release package (ournotes-recommend-wasm-vVERSION.tar.gz " - "or its directory); its web JS/WASM and build-info.json are copied into --replay-dir", - ) - c.add_argument( - "--jackets", - metavar="DIR", - help="also write every song's jacket as DIR/.webp (at most 320 px on the longer side)", - ) - _out(c, "output file (.json, or .json.gz for gzip)") - c.set_defaults(func=cmd_music_data, usage=c.error) - cli_assets.register(sub, argparse.Namespace(session=Session, print_json=_print_json)) - voices.register(sub, argparse.Namespace(session=Session)) - c = sub.add_parser("ui", help="export an offline UI prefab library for ournotes-player/ui") - _out(c, "UI data directory outside the source repositories") - c.add_argument("--key", action="append", help="exact APK catalog key (repeatable)") - c.add_argument("--prefix", default="EmbUI/", help="APK UI key prefix (default EmbUI/)") - c.add_argument("--limit", type=int, help="maximum number of keys") - c.add_argument("--no-dependencies", action="store_true", help="omit additional prefab roots and controllers") - c.add_argument("--force", action="store_true", help="re-export the selected keys") - c.set_defaults(func=cmd_ui, usage=c.error) - return p +def parse_args(argv=None) -> argparse.Namespace: + """The arguments of a command line, every global option present (None, or False for --json, when not given).""" + args = build_parser().parse_args(argv) + fill_globals(args) + return args def main(argv=None): - args = build_parser().parse_args(argv) + args = parse_args(argv) try: - cfg = load_config(args) if getattr(args, "reads_config", True) else None + cfg = load_config(args) if args.reads_config else None args.func(args, cfg) except ConfigError as e: print(f"nnnotes: {e}", file=sys.stderr) diff --git a/src/nnnotes/cli/apk.py b/src/nnnotes/cli/apk.py new file mode 100644 index 0000000..25b2f72 --- /dev/null +++ b/src/nnnotes/cli/apk.py @@ -0,0 +1,57 @@ +"""`nnnotes apk`: facts of base.apk: the player's graphics settings and the HCA keycode.""" + +from __future__ import annotations + +from pathlib import Path + +from ..core.jsonio import write_json +from ..source.session import Session +from .common import command, group, out_arg, report + + +def cmd_settings(args, cfg): + from ..unity.player import PlayerData + + g = PlayerData(Session(cfg).apk()).graphics() + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + write_json(out, g) + r = { + "out": str(out), + "colorSpace": g["colorSpace"], + "qualityLevels": len(g["qualityLevels"]), + "renderers": len(g["renderers"]), + } + report( + args, + r, + lambda r: [f"{r['out']} ({r['colorSpace']}, {r['qualityLevels']} quality levels, {r['renderers']} renderers)"], + ) + + +def cmd_hca_key(args, cfg): + from ..formats import crikey + + cfg.require_path("paths", "apk") + k = crikey.find_key(Session(cfg).path("paths", "apk")) + r = {"found": bool(k), "digits": len(str(k)) if k else None, "written": None} + if args.write and k: + d = Path(args.write) + d.mkdir(parents=True, exist_ok=True) + r["written"] = str(crikey.write_hcakey(k, d)) + report( + args, + r, + lambda r: [ + f"HCA keycode found ({r['digits']} digits)" if r["found"] else "no HCA keycode (decryption disabled)", + *([r["written"]] if r["written"] else []), + ], + ) + + +def register(sub) -> None: + gs = group(sub, "apk", "facts of base.apk ([paths] apk)") + c = command(gs, "settings", cmd_settings, "the player's graphics settings (color space, quality levels) -> JSON") + out_arg(c, "output .json file") + c = command(gs, "hca-key", cmd_hca_key, "find the HCA keycode of the CRI audio (never printed)") + c.add_argument("--write", metavar="DIR", help="write it as DIR/.hcakey (the file vgmstream reads)") diff --git a/src/nnnotes/cli/cli_assets.py b/src/nnnotes/cli/assets.py similarity index 80% rename from src/nnnotes/cli/cli_assets.py rename to src/nnnotes/cli/assets.py index 121570b..181304e 100644 --- a/src/nnnotes/cli/cli_assets.py +++ b/src/nnnotes/cli/assets.py @@ -1,21 +1,18 @@ -"""The asset commands: the default orchestration of the stages (export), its plan, single tasks, catalog versions and -the store. - - nnnotes export -o OUT [--layout original[,cas]] [--select group:G|key:PREFIX|bundle:GLOB ...] - [--views all|none|a,b] [--store DIR] [--workers N] [--memory GiB] [--fetch-workers N] - [--catalog-version L|SHA] [--png-level N] [--flac-level N] [--only-class C,...] - [--link auto|clone|hard|copy] [--strict] [--dry-run] [--explain] - nnnotes plan [the selection and parameters of export] [-o OUT] [--json] [--since RUN] [--check] [--census] - [--emit-tasks DIR] [--why TASK] - nnnotes run-stage TASK.json|DIR [...] [--store DIR] [--fetch] [--force] - nnnotes catalogs list [--json] | import FILE [--label L] [--resource-version V] | fetch [--label L] - | diff A B [--json] - nnnotes store verify [--quick] +"""The asset commands: the default orchestration of the stages (`export assets`), its plan and single tasks (`store +plan`, `store run`). + + nnnotes export assets -o OUT [--layout original[,cas]] [--select group:G|key:PREFIX|bundle:GLOB ...] + [--views all|none|a,b] [--workers N] [--memory GiB] [--fetch-workers N] + [--stored-catalog L|SHA] [--png-level N] [--flac-level N] [--only-class C,...] + [--link auto|clone|hard|copy] [--strict] [--dry-run] [--explain] + nnnotes store plan [the selection and parameters of export assets] [-o OUT] [--since RUN] [--check] [--census] + [--emit-tasks DIR] [--why TASK] + nnnotes store run TASK.json|DIR [...] [--fetch] [--force] The pipeline is the registry STAGE_SOURCES in order: each entry names the module and attribute of a stage (a Stage class) or of a family of stages (a function returning them, `view.`). An entry whose module or attribute is not installed is left out of the pipeline and named by --explain. Stage modules import heavy libraries (UnityPy) only -when a task runs, so `plan`, and `run-stage` of stages that do not read bundles, never load them. +when a task runs, so `store plan`, and `store run` of stages that do not read bundles, never load them. Facts the stages read while their tasks are described (stages.Env): @@ -39,9 +36,10 @@ Task cost estimates (scheduling only: never part of a key) are each stage's estimate scaled per stage by the costs measured in earlier runs (/costs/calibration.json, updated after every export). -Exit codes: export 0 complete (unsupported objects allowed), 1 item or task failures (reports written), 2 usage or -settings, 3 aborted (results written so far stay valid); run-stage 0, 1 (a task failed or a result is partial), 2, -4 an incompatible task (nothing of the batch runs); plan --check 1 when anything would run or change. +Exit codes: export assets 0 complete (unsupported objects allowed), 1 item or task failures (reports written), 2 +usage or settings, 3 aborted (results written so far stay valid); store run 0, 1 (a task failed or a result is +partial), 2, 4 an incompatible task (nothing of the batch runs); store plan --check 1 when anything would run or +change. """ from __future__ import annotations @@ -49,7 +47,6 @@ import argparse import fnmatch import json -import re import sys import threading import zipfile @@ -59,14 +56,15 @@ from dataclasses import dataclass from importlib import import_module, util from pathlib import Path -from types import SimpleNamespace from ..pipeline import contract from ..pipeline.contract import Cost, IncompatibleTask, Input, Task from ..pipeline.stages import Env, Pending, Stage, describe, execute from ..source.apkset import ApkSet -from ..source.config import ConfigError, check_catalog_version, usable_cpus, use -from ..source.config import describe as describe_setting +from ..source.config import ConfigError, usable_cpus, use +from ..source.session import Session +from . import common +from .common import nonnegative, print_json GIB = 1 << 30 LAYOUT_NAMES = ("original", "cas") @@ -149,7 +147,7 @@ class Installed: def installed(names=None) -> Installed: - """The stages of STAGE_SOURCES (`names`: only the sources of these stage names, for run-stage).""" + """The stages of STAGE_SOURCES (`names`: only the sources of these stage names, for `store run`).""" stages, of, missing = {}, {}, [] for src in STAGE_SOURCES: if names is not None and not any(src.matches(n) for n in names): @@ -230,23 +228,6 @@ def select(index: dict, selection: list[str]) -> Selection: # ---------------------------------------------------------------- settings -def store_root(args, cfg) -> Path: - """--store, else [paths] store, else <[paths] cache>/store.""" - p = getattr(args, "store", None) - if p: - return Path(p).expanduser() - p = cfg.path("paths", "store") - if p is not None: - return p - cache = cfg.path("paths", "cache") - if cache is not None: - return cache / "store" - raise ConfigError( - "setting paths.store is not set (nor paths.cache, whose store/ directory is the default): " - f"give it as {describe_setting('paths', 'store', '--store')}" - ) - - def default_memory() -> int | None: """80 % of the physical memory (Linux), else None (no budget).""" try: @@ -471,12 +452,12 @@ class Workspace: """What export and plan share: the store, the catalog version, its index, the selection, the pipeline and its parameters, the facts.""" - def __init__(self, args, cfg, common, log=_log): + def __init__(self, args, cfg, log=_log): from ..pipeline.catalogdb import CatalogDB from ..pipeline.store import Store - self.args, self.cfg, self.common, self.log = args, cfg, common, log - self.root = store_root(args, cfg) + self.args, self.cfg, self.log = args, cfg, log + self.root = Session(cfg).store_root() self.cache = cfg.path("paths", "cache") self.apk = cfg.path("paths", "apk") self.fetcher = CatalogFetcher(self.root, self.cache, cfg) @@ -506,7 +487,7 @@ def __init__(self, args, cfg, common, log=_log): def _catalog_version(self): from ..pipeline import catalogdb - ref = getattr(self.args, "catalog_version", None) + ref = getattr(self.args, "stored_catalog", None) if ref: try: try: @@ -514,10 +495,10 @@ def _catalog_version(self): except ValueError: v = self.db.get(ref, self.cfg.get("catalog", "region"), self.cfg.get("catalog", "language")) except (KeyError, ValueError) as e: - self.args.usage(f"--catalog-version {ref}: {e.args[0] if e.args else e}") + self.args.usage(f"--stored-catalog {ref}: {e.args[0] if e.args else e}") remote, apk = self.db.catalog_bytes(v) return v, remote, apk - cat = self.common.session(self.cfg).catalog(bundles=False) + cat = Session(self.cfg).catalog(bundles=False) src = cat.sources() remote, apk = src["remote"], src.get("apk") source = cat.source.to_dict() if getattr(cat, "source", None) else None @@ -529,7 +510,7 @@ def _catalog_version(self): apk, region=self.cfg.get("catalog", "region"), language=self.cfg.get("catalog", "language"), - apk_version_name=_apk_version(self.apk), + apk_version_name=catalogdb.apk_version(self.apk), source=source, resource_version=source["version"] if source else getattr(cat, "resource_version", None), ) @@ -1113,14 +1094,6 @@ def parse_layouts(spec: str) -> list[str]: return names -def _apk_version(apk: Path | None) -> str | None: - if apk is None or not apk.is_file(): - return None - from ..source.gameapi import apk_version_name - - return apk_version_name(apk) - - # ---------------------------------------------------------------- export class Totals: """What export reports of the results besides the run's counts, from its one reading of each (Run.scan): the @@ -1160,7 +1133,7 @@ def _write_report(d: Path, name: str, doc) -> Path: return p -def cmd_export(args, cfg, common): +def cmd_export(args, cfg): from ..pipeline import layout from ..pipeline.orchestrate import Orchestrator, failure @@ -1169,7 +1142,7 @@ def cmd_export(args, cfg, common): except ValueError as e: args.usage(str(e)) use(cfg) # the tools the atoms name are part of the keys - ws = Workspace(args, cfg, common) + ws = Workspace(args, cfg) pipeline = ws.pipeline() params = ws.params(pipeline) orch = Orchestrator( @@ -1283,7 +1256,8 @@ def visit(tid, doc): _write_report(reports, "sources.json", {"problems": ws.problems()}) s = manifest["summary"] gaps = view_gaps(totals.views) - common.print_json( + common.report( + args, { "run": manifest["id"], "selection": ws.selection, @@ -1296,7 +1270,7 @@ def visit(tid, doc): "placement": manifest["placement"], "layouts": written, "reports": str(reports), - } + }, ) if aborted: print(f"nnnotes: aborted: {aborted}", file=sys.stderr) @@ -1319,7 +1293,7 @@ def link_mode(args, cfg) -> str: # ---------------------------------------------------------------- plan -def cmd_plan(args, cfg, common): +def cmd_plan(args, cfg): from ..pipeline.orchestrate import Orchestrator layouts = [] @@ -1329,7 +1303,7 @@ def cmd_plan(args, cfg, common): except ValueError as e: args.usage(str(e)) use(cfg) # the tools the atoms name are part of the keys - ws = Workspace(args, cfg, common) + ws = Workspace(args, cfg) pipeline = ws.pipeline() params = ws.params(pipeline) if args.explain: @@ -1354,9 +1328,9 @@ def cmd_plan(args, cfg, common): files = p.emit(args.emit_tasks) ws.log(f"{len(files)} task descriptions in {args.emit_tasks}") if args.why: - common.print_json(why(ws, p, args.why, args.since)) + common.report(args, why(ws, p, args.why, args.since), why_lines) elif args.json: - common.print_json(p.doc) + print_json(p.doc) else: sys.stdout.write(p.text()) if args.check: @@ -1391,7 +1365,27 @@ def why(ws: Workspace, p, tid: str, since=None) -> dict: } -# ---------------------------------------------------------------- run-stage +def why_lines(r: dict) -> list[str]: + from ..pipeline.plan import reason_text + + prev = r["previous"] + return [ + f"{r['task']}: {r['status']} ({', '.join(reason_text(x) for x in r['reasons']) or '-'})", + f"key {r['key'] or '-'}", + f"previous run {prev['run']}: {prev['status']}, key {prev['key'] or '-'}" if prev else "no previous run", + ] + + +# ---------------------------------------------------------------- store run +def run_lines(r: dict) -> list[str]: + lines = [] + for t in r["tasks"]: + tail = t["result"] if t["status"] != "failed" else f"{t['code']}: {t['message']}" + lines.append(f"{t['status']:<6} {t['id']} {tail}") + s = r["summary"] + return [*lines, f"{s['tasks']} tasks: {s['hit']} hit, {s['ran']} ran, {s['failed']} failed"] + + def _task_files(items) -> list[Path]: out = [] for s in items: @@ -1400,7 +1394,7 @@ def _task_files(items) -> list[Path]: return out -def cmd_run_stage(args, cfg, common): +def cmd_run_stage(args, cfg): from ..pipeline.orchestrate import run_one from ..pipeline.store import Store @@ -1426,7 +1420,7 @@ def cmd_run_stage(args, cfg, common): for m in problems: print(f"nnnotes: incompatible task: {m}", file=sys.stderr) sys.exit(EXIT_INCOMPATIBLE) - root = store_root(args, cfg) + root = Session(cfg).store_root() cache = cfg.path("paths", "cache") store = Store(root, cache=cache, fetch=CatalogFetcher(root, cache, cfg) if args.fetch else None) use(cfg) @@ -1442,155 +1436,16 @@ def cmd_run_stage(args, cfg, common): if r["code"] == "config": break n = Counter(o["status"] for o in out) - common.print_json( - {"tasks": out, "summary": {"tasks": len(tasks), "hit": n["hit"], "ran": n["ran"], "failed": n["failed"]}} + common.report( + args, + {"tasks": out, "summary": {"tasks": len(tasks), "hit": n["hit"], "ran": n["ran"], "failed": n["failed"]}}, + run_lines, ) if bad: sys.exit(EXIT_FAILED) -# ---------------------------------------------------------------- catalogs -def _db(args, cfg): - from ..pipeline.catalogdb import CatalogDB - - return CatalogDB(store_root(args, cfg)) - - -def _version_line(v: dict) -> str: - apk = v["apk"]["sha256"][:12] if v.get("apk") else "-" - return ( - f"{v['seq']:>4} {v['id'][:12]} {','.join(v['labels']) or '-':<24} {v.get('region') or '-':<6} " - f"{v.get('language') or '-':<8} remote {v['remote']['sha256'][:12]} apk {apk} " - f"resource {v.get('resourceVersion') or '-'}" - ) - - -def cmd_catalogs_list(args, cfg, common): - from ..pipeline.catalogdb import CATALOGS - - vs = _db(args, cfg).versions() - if args.json: - common.print_json({"schema": CATALOGS, "versions": vs}) - return - for v in vs: - print(_version_line(v)) - print(f"# {len(vs)} catalog versions", file=sys.stderr) - - -def _apk_catalog(args, cfg) -> bytes | None: - from ..pipeline.catalogdb import apk_catalog - - if getattr(args, "apk_catalog", None): - return Path(args.apk_catalog).read_bytes() - apk = cfg.path("paths", "apk") - if apk is None: - return None - if not apk.exists(): - raise ConfigError(f"setting paths.apk: file {apk} not found") - return apk_catalog(apk) - - -CATALOG_FILE = re.compile(r"catalog_([A-Za-z0-9][A-Za-z0-9.-]*)_[A-Za-z0-9-]+\.bin") - - -def _import_resource_version(given: str | None, file: Path, source: dict | None) -> str | None: - """The resource version of an imported catalog: `given` (--resource-version), else the JP source's version, else - the version in an international file name `catalog__.bin` (none for `main`). A JP source is - authoritative: a different `given` is an error.""" - if given is not None: - try: - check_catalog_version(given) - except ValueError as e: - raise ValueError(f"--resource-version {given!r}: {e}") from None - if source is not None: - if given is not None and given != source["version"]: - raise ValueError( - f"--resource-version {given} differs from the catalog's source version {source['version']}" - ) - return source["version"] - if given is not None: - return given - m = CATALOG_FILE.fullmatch(file.name) - return m[1] if m and m[1] != "main" and ".." not in m[1] else None - - -def cmd_catalogs_import(args, cfg, common): - try: - remote = Path(args.file).read_bytes() - except OSError as e: - args.usage(f"{args.file}: {e.strerror or e}") - try: - source = None - if cfg.provider() == "jp": - from ..source.jp import read_source - - source = read_source(args.file, remote).to_dict() - resource = _import_resource_version(args.resource_version, Path(args.file), source) - v = _db(args, cfg).add( - remote, - _apk_catalog(args, cfg), - label=args.label, - region=cfg.get("catalog", "region"), - language=cfg.get("catalog", "language"), - resource_version=resource, - apk_version_name=_apk_version(cfg.path("paths", "apk")), - source=source, - ) - except ValueError as e: - args.usage(str(e)) - print(_version_line(v)) - - -def cmd_catalogs_fetch(args, cfg, common): - from ..pipeline import catalogdb - from ..source.catalog import Catalog - - region = cfg.region() - if cfg.provider(region) == "jp": - from ..source.jp import open_catalog - - cat = open_catalog(cfg, region, apk=cfg.path("paths", "apk")) - src = cat.sources() - v = _db(args, cfg).add( - src["remote"], - src.get("apk"), - label=args.label, - region=region, - language=cfg.get("catalog", "language") or "ja", - source=cat.source.to_dict(), - resource_version=cat.source.version, - apk_version_name=_apk_version(cfg.path("paths", "apk")), - ) - print(_version_line(v)) - return - cdn = cfg.cdn(region) - language = cfg.require("catalog", "language") - version = common.session(cfg).catalog_version(region) - filename = Catalog.cache_file(language, Path("."), version=version).name - try: - data, hash_text = catalogdb.fetch(cdn, language, version=version) - except OSError as e: # URLError, HTTPError, timeouts (the message has no URL) - sys.exit( - f"nnnotes: {filename} could not be fetched: {type(e).__name__}: " - f"{getattr(e, 'reason', None) or getattr(e, 'code', None) or ''}".rstrip(": ") - ) - resource = version if version != "main" else None - try: - v = _db(args, cfg).add( - data, - _apk_catalog(args, cfg), - label=args.label, - region=region, - language=language, - hash_text=hash_text, - resource_version=resource, - apk_version_name=_apk_version(cfg.path("paths", "apk")), - ) - except ValueError as e: - args.usage(str(e)) - print(_version_line(v)) - - +# ---------------------------------------------------------------- catalog versions def addresses_of(store, vid: str) -> dict | None: """The address table (link.addresses) of the latest run over a catalog version, None when no run has one.""" runs = sorted((store.root / "runs").glob("*.json"), key=lambda p: p.stat().st_mtime, reverse=True) @@ -1616,46 +1471,7 @@ def addresses_of(store, vid: str) -> dict | None: return None -def cmd_catalogs_diff(args, cfg, common): - from ..pipeline import catalogdb - from ..pipeline.store import Store - - db = _db(args, cfg) - vs = [] - for ref in (args.old, args.new): - try: - vs.append(db.get(ref)) - except (KeyError, ValueError) as e: - args.usage(f"{ref}: {e.args[0] if e.args else e}") - store = Store(store_root(args, cfg)) - a, b = vs - d = catalogdb.diff(db.index(a), db.index(b), addresses_of(store, a["id"]), addresses_of(store, b["id"])) - if args.json: - common.print_json(d) - else: - sys.stdout.buffer.write(catalogdb.format_diff(d).encode("utf-8")) - sys.stdout.buffer.flush() - - -# ---------------------------------------------------------------- store -def cmd_store_verify(args, cfg, common): - from ..pipeline.store import Store - - r = Store(store_root(args, cfg)).verify(quick=args.quick, workers=args.workers or FETCH_WORKERS) - common.print_json(r) - if r["problems"]: - sys.exit(EXIT_FAILED) - - # ---------------------------------------------------------------- parser -def default_common(): - """The helpers of the command line the handlers use (session: the data sources of a Config; print_json).""" - from .. import cli - from ..source.session import Session - - return SimpleNamespace(session=Session, print_json=cli._print_json) - - def _gib(s: str) -> float: try: v = float(s) @@ -1666,12 +1482,8 @@ def _gib(s: str) -> float: return v -def _store_arg(c) -> None: - c.add_argument("--store", help="store directory ([paths] store; default /store)") - - -def _run_args(c) -> None: - """The selection and parameters shared by export and plan.""" +def run_args(c) -> None: + """The selection and parameters shared by `export assets` and `store plan`.""" c.add_argument( "--select", action="append", @@ -1684,9 +1496,9 @@ def _run_args(c) -> None: help="semantic views to build (default: all when master data is set, else none)", ) c.add_argument( - "--catalog-version", + "--stored-catalog", metavar="LABEL|SHA", - help="an imported catalog version (`catalogs list`; default: the current catalog)", + help="a catalog version of the store (`catalog versions`; default: the current catalog)", ) c.add_argument("--png-level", type=int, metavar="N", help="PNG compression level 0-9 (changes the output)") c.add_argument( @@ -1698,7 +1510,10 @@ def _run_args(c) -> None: ) c.add_argument("--only-class", metavar="C,...", help="export only objects of these classes") c.add_argument( - "--workers", type=int, help="worker processes (default: the CPUs this process may use; 0: this process)" + "--workers", + type=nonnegative, + metavar="N", + help="worker processes (default: the CPUs this process may use; 0: this process)", ) c.add_argument( "--memory", @@ -1709,114 +1524,3 @@ def _run_args(c) -> None: c.add_argument("--fetch-workers", type=int, metavar="N", help=f"parallel downloads (default {FETCH_WORKERS})") c.add_argument("--layout", default="original", help="original, cas or original,cas (default original)") c.add_argument("--explain", action="store_true", help="print every setting of the run first") - _store_arg(c) - - -def register(sub, common=None) -> None: - """Add the asset commands to the command line's subparsers. `common`: the helpers of cli (open_catalog, - print_json; default_common()).""" - common = common or default_common() - - def bind(fn): - return lambda args, cfg: fn(args, cfg, common) - - c = sub.add_parser("export", help="every object of the selected bundles -> common formats (store + layouts)") - c.add_argument("-o", "--out", required=True, help="output directory") - _run_args(c) - c.add_argument( - "--link", - choices=("auto", "clone", "hard", "copy"), - help="how the layout files are made from the store: auto (default; [export] link): a clone where " - "the file system makes them, else a hard link (read-only), else a copy; or that method only", - ) - c.add_argument("--strict", action="store_true", help="exit 1 when a view has gaps") - c.add_argument("--dry-run", action="store_true", help="print the plan, run nothing") - c.set_defaults(func=bind(cmd_export), usage=c.error) - - c = sub.add_parser("plan", help="what export would run and why, without running it") - c.add_argument("-o", "--out", help="output directory of the export (counts the layout files to write)") - _run_args(c) - c.add_argument("--json", action="store_true", help="the plan document (nnnotes.plan/1)") - c.add_argument("--since", metavar="RUN", help="explain against this run (default: the latest of the selection)") - c.add_argument("--check", action="store_true", help="exit 1 when anything would run or change") - c.add_argument("--census", action="store_true", help="first fetch and census the bundles without one") - c.add_argument("--emit-tasks", metavar="DIR", help="write the runnable tasks as task descriptions") - c.add_argument("--why", metavar="TASK", help="the reasons and key parts of one task") - c.set_defaults(func=bind(cmd_plan), usage=c.error) - - c = sub.add_parser("run-stage", help="run task descriptions (nnnotes.task/1) into the store") - c.add_argument("tasks", nargs="+", metavar="TASK", help="task description files, or directories of them") - _store_arg(c) - c.add_argument("--fetch", action="store_true", help="fetch inputs through their catalog locators") - c.add_argument("--force", action="store_true", help="run tasks whose result is stored (it must not change)") - c.set_defaults(func=bind(cmd_run_stage), usage=c.error) - - c = sub.add_parser("catalogs", help="catalog versions of the store") - csub = c.add_subparsers(dest="catalogs_cmd", required=True, metavar="") - m = csub.add_parser("list", help="the imported catalog versions") - m.add_argument("--json", action="store_true", help="the versions as JSON") - _store_arg(m) - m.set_defaults(func=bind(cmd_catalogs_list), usage=m.error) - m = csub.add_parser("import", help="import a catalog file (with the APK's catalog when [paths] apk is set)") - m.add_argument("file", help="catalog__.bin") - m.add_argument("--label", help="label of the version (default: its resource version, else the sha256 prefix)") - m.add_argument("--apk-catalog", help="the APK's catalog.bin (default: read from [paths] apk)") - m.add_argument( - "--resource-version", - help="the game's resource version of this catalog (default: the JP source's version, else the " - "version in the file name catalog__.bin)", - ) - _store_arg(m) - m.set_defaults(func=bind(cmd_catalogs_import), usage=m.error) - m = csub.add_parser("fetch", help="fetch the region's current catalog and import it") - m.add_argument("--label", help="label of the version (default: the resource version when an API root is set)") - _store_arg(m) - m.set_defaults(func=bind(cmd_catalogs_fetch), usage=m.error) - m = csub.add_parser("diff", help="keys, bundles, raw files (and objects) that differ between two versions") - m.add_argument("old", help="label, version id or sha256 prefix") - m.add_argument("new", help="label, version id or sha256 prefix") - m.add_argument("--json", action="store_true", help="the diff document (nnnotes.catalog-diff/1)") - _store_arg(m) - m.set_defaults(func=bind(cmd_catalogs_diff), usage=m.error) - - c = sub.add_parser("store", help="the store") - ssub = c.add_subparsers(dest="store_cmd", required=True, metavar="") - m = ssub.add_parser("verify", help="check every stored content and result") - m.add_argument("--quick", action="store_true", help="check sizes only, do not hash the contents") - m.add_argument("--workers", type=int, help=f"parallel hashing (default {FETCH_WORKERS})") - _store_arg(m) - m.set_defaults(func=bind(cmd_store_verify), usage=m.error) - - -def main(argv=None) -> None: - """The asset commands alone, with the global flags of the command line (tests; `nnnotes` has them too).""" - from . import load_config - - p = argparse.ArgumentParser(prog="nnnotes") - for flag in ( - "--config", - "--region", - "--language", - "--catalog", - "--catalog-release", - "--cache", - "--master", - "--apk", - "--ffmpeg", - "--vgmstream", - "--node", - ): - p.add_argument(flag) - sub = p.add_subparsers(dest="cmd", required=True, metavar="") - register(sub) - args = p.parse_args(argv) - try: - cfg = load_config(args) - args.func(args, cfg) - except ConfigError as e: - print(f"nnnotes: {e}", file=sys.stderr) - sys.exit(EXIT_USAGE) - - -if __name__ == "__main__": # pragma: no cover - main() diff --git a/src/nnnotes/cli/browse.py b/src/nnnotes/cli/browse.py index ec17ec2..cd6f31b 100644 --- a/src/nnnotes/cli/browse.py +++ b/src/nnnotes/cli/browse.py @@ -1,5 +1,5 @@ """A local catalog browser: the catalogs of the configured regions as directory listings, and their bundles as -decrypted downloads (`nnnotes browse`).""" +decrypted downloads (`nnnotes catalog browse`).""" from __future__ import annotations @@ -166,7 +166,7 @@ def serve(regions: list[Region], bundle_key: BundleKey, cache: Path, port: int, server.cache = Path(cache) server.catalogs = {} server.jp_catalogs = {} - print(f"nnnotes browse: serving on {host}:{port}", flush=True) + print(f"nnnotes catalog browse: serving on {host}:{port}", flush=True) try: server.serve_forever() except KeyboardInterrupt: diff --git a/src/nnnotes/cli/bundle.py b/src/nnnotes/cli/bundle.py new file mode 100644 index 0000000..72a94bb --- /dev/null +++ b/src/nnnotes/cli/bundle.py @@ -0,0 +1,21 @@ +"""`nnnotes bundle`: fetch bundles into the cache.""" + +from __future__ import annotations + +from ..source.session import Session +from .common import command, group, known_keys, report + + +def cmd_pull(args, cfg): + cat = Session(cfg).catalog() + known_keys(args, cat, args.keys) + files = {key: [str(p) for p in cat.fetch_key(key)] for key in args.keys} + report(args, {"keys": files}, lambda r: [p for ps in r["keys"].values() for p in ps]) + + +def register(sub) -> None: + bs = group(sub, "bundle", "the bundles of the catalog") + c = command( + bs, "pull", cmd_pull, "fetch the bundle closure of keys into the cache (prints the cached bundle files)" + ) + c.add_argument("keys", nargs="+", metavar="KEY", help="addressable keys (`catalog keys` lists them)") diff --git a/src/nnnotes/cli/catalog.py b/src/nnnotes/cli/catalog.py new file mode 100644 index 0000000..1144a5a --- /dev/null +++ b/src/nnnotes/cli/catalog.py @@ -0,0 +1,239 @@ +"""`nnnotes catalog`: the region's addressable keys, a local catalog browser, and the catalog versions of the store +(import, fetch, list, diff).""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +from ..source.config import ConfigError, check_catalog_version +from ..source.session import Session +from .common import command, group, note, report, write + +CATALOG_FILE = re.compile(r"catalog_([A-Za-z0-9][A-Za-z0-9.-]*)_[A-Za-z0-9-]+\.bin") + + +def cmd_keys(args, cfg): + cat = Session(cfg).catalog(bundles=False) + ks = cat.keys(args.prefix) + shown = ks[: args.limit] if args.limit else ks + if args.json: + report(args, {"keys": shown, "total": len(ks)}) + return + write("".join(f"{k}\n" for k in shown)) + note(f"# {len(ks)} keys") + + +def cmd_browse(args, cfg): + from .browse import Region, serve + + names = cfg.regions() + if not names: + raise ConfigError( + "no region configured: add a [servers.] table with `cdn` to the config file " + "(or set NNNOTES_SERVERS__CDN)" + ) + regions = [] + for r in names: + langs = cfg.get_list(f"servers.{r}", "languages") + if not langs: + raise cfg.missing(f"servers.{r}", "languages") + regions.append(Region(r, cfg.get(f"servers.{r}", "name") or r, cfg.cdn(r), langs, cfg)) + serve(regions, Session(cfg).bundle_key(), cfg.require_path("paths", "cache"), args.port, args.host) + + +# ---------------------------------------------------------------- catalog versions of the store +def _db(cfg): + from ..pipeline.catalogdb import CatalogDB + + return CatalogDB(Session(cfg).store_root()) + + +def version_line(v: dict) -> str: + apk = v["apk"]["sha256"][:12] if v.get("apk") else "-" + return ( + f"{v['seq']:>4} {v['id'][:12]} {','.join(v['labels']) or '-':<24} {v.get('region') or '-':<6} " + f"{v.get('language') or '-':<8} remote {v['remote']['sha256'][:12]} apk {apk} " + f"resource {v.get('resourceVersion') or '-'}" + ) + + +def cmd_versions(args, cfg): + from ..pipeline.catalogdb import CATALOGS + + vs = _db(cfg).versions() + if args.json: + report(args, {"schema": CATALOGS, "versions": vs}) + return + write("".join(f"{version_line(v)}\n" for v in vs)) + note(f"# {len(vs)} catalog versions") + + +def _apk_catalog(args, cfg) -> bytes | None: + from ..pipeline.catalogdb import apk_catalog + + if getattr(args, "apk_catalog", None): + return Path(args.apk_catalog).read_bytes() + apk = cfg.path("paths", "apk") + if apk is None: + return None + if not apk.exists(): + raise ConfigError(f"setting paths.apk: file {apk} not found") + return apk_catalog(apk) + + +def import_resource_version(given: str | None, file: Path, source: dict | None) -> str | None: + """The resource version of an imported catalog: `given` (--resource-version), else the JP source's version, else + the version in an international file name `catalog__.bin` (none for `main`). A JP source is + authoritative: a different `given` is an error.""" + if given is not None: + try: + check_catalog_version(given) + except ValueError as e: + raise ValueError(f"--resource-version {given!r}: {e}") from None + if source is not None: + if given is not None and given != source["version"]: + raise ValueError( + f"--resource-version {given} differs from the catalog's source version {source['version']}" + ) + return source["version"] + if given is not None: + return given + m = CATALOG_FILE.fullmatch(file.name) + return m[1] if m and m[1] != "main" and ".." not in m[1] else None + + +def _version(args, v: dict) -> None: + report(args, v, lambda v: [version_line(v)]) + + +def cmd_import(args, cfg): + from ..pipeline import catalogdb + + try: + remote = Path(args.file).read_bytes() + except OSError as e: + args.usage(f"{args.file}: {e.strerror or e}") + try: + source = None + if cfg.provider() == "jp": + from ..source.jp import read_source + + source = read_source(args.file, remote).to_dict() + resource = import_resource_version(args.resource_version, Path(args.file), source) + v = _db(cfg).add( + remote, + _apk_catalog(args, cfg), + label=args.label, + region=cfg.get("catalog", "region"), + language=cfg.get("catalog", "language"), + resource_version=resource, + apk_version_name=catalogdb.apk_version(cfg.path("paths", "apk")), + source=source, + ) + except ValueError as e: + args.usage(str(e)) + _version(args, v) + + +def cmd_fetch(args, cfg): + from ..pipeline import catalogdb + from ..source.catalog import Catalog + + region = cfg.region() + if cfg.provider(region) == "jp": + from ..source.jp import open_catalog + + cat = open_catalog(cfg, region, apk=cfg.path("paths", "apk")) + src = cat.sources() + v = _db(cfg).add( + src["remote"], + src.get("apk"), + label=args.label, + region=region, + language=cfg.get("catalog", "language") or "ja", + source=cat.source.to_dict(), + resource_version=cat.source.version, + apk_version_name=catalogdb.apk_version(cfg.path("paths", "apk")), + ) + _version(args, v) + return + cdn = cfg.cdn(region) + language = cfg.require("catalog", "language") + version = Session(cfg).catalog_version(region) + filename = Catalog.cache_file(language, Path("."), version=version).name + try: + data, hash_text = catalogdb.fetch(cdn, language, version=version) + except OSError as e: # URLError, HTTPError, timeouts (the message has no URL) + sys.exit( + f"nnnotes: {filename} could not be fetched: {type(e).__name__}: " + f"{getattr(e, 'reason', None) or getattr(e, 'code', None) or ''}".rstrip(": ") + ) + resource = version if version != "main" else None + try: + v = _db(cfg).add( + data, + _apk_catalog(args, cfg), + label=args.label, + region=region, + language=language, + hash_text=hash_text, + resource_version=resource, + apk_version_name=catalogdb.apk_version(cfg.path("paths", "apk")), + ) + except ValueError as e: + args.usage(str(e)) + _version(args, v) + + +def cmd_diff(args, cfg): + from ..pipeline import catalogdb + from ..pipeline.store import Store + from .assets import addresses_of + + db = _db(cfg) + vs = [] + for ref in (args.old, args.new): + try: + vs.append(db.get(ref)) + except (KeyError, ValueError) as e: + args.usage(f"{ref}: {e.args[0] if e.args else e}") + store = Store(Session(cfg).store_root()) + a, b = vs + d = catalogdb.diff(db.index(a), db.index(b), addresses_of(store, a["id"]), addresses_of(store, b["id"])) + if args.json: + report(args, d) + else: + write(catalogdb.format_diff(d)) + + +def register(sub) -> None: + cs = group(sub, "catalog", "the region's catalog: its keys, a local browser, and the catalog versions of the store") + c = command(cs, "keys", cmd_keys, "the addressable keys of the catalog (the total on standard error)") + c.add_argument("--prefix", default="", help="only the keys that start with this prefix") + c.add_argument("--limit", type=int, default=200, help="at most N keys (default 200; 0: all)") + c = command(cs, "browse", cmd_browse, "browse the configured regions' catalogs and bundles in a local web page") + c.add_argument("--port", type=int, default=8000, help="port (default 8000)") + c.add_argument("--host", default="127.0.0.1", help="address to listen on (default 127.0.0.1)") + command(cs, "versions", cmd_versions, "the catalog versions imported into the store (--json: nnnotes.catalogs/1)") + c = command(cs, "import", cmd_import, "import a catalog file (with the APK's catalog when [paths] apk is set)") + c.add_argument("file", help="catalog__.bin") + c.add_argument("--label", help="label of the version (default: its resource version, else the sha256 prefix)") + c.add_argument("--apk-catalog", help="the APK's catalog.bin (default: read from [paths] apk)") + c.add_argument( + "--resource-version", + help="the game's resource version of this catalog (default: the JP source's version, else the " + "version in the file name catalog__.bin)", + ) + c = command(cs, "fetch", cmd_fetch, "fetch the region's current catalog and import it into the store") + c.add_argument("--label", help="label of the version (default: the resource version when an API root is set)") + c = command( + cs, + "diff", + cmd_diff, + "keys, bundles, raw files (and objects) that differ between two catalog versions of the store " + "(--json: nnnotes.catalog-diff/1)", + ) + c.add_argument("old", help="label, version id or sha256 prefix") + c.add_argument("new", help="label, version id or sha256 prefix") diff --git a/src/nnnotes/cli/common.py b/src/nnnotes/cli/common.py new file mode 100644 index 0000000..cf0894e --- /dev/null +++ b/src/nnnotes/cli/common.py @@ -0,0 +1,321 @@ +"""What the commands share: the global options and the settings they set, the command parsers, usage errors and the +output (a short summary, or with --json the command's report as JSON).""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +from ..core.jsonio import dumps +from ..source.catalog import Catalog +from ..source.config import Config, ConfigError, use + +DIFFICULTY_CHOICES = ("easy", "normal", "hard", "expert") +AUDIO_CHOICES = ("flac", "ogg", "wav") +FLAC_LEVELS = range(13) +EMOJI_FONT = "emoji" # --font emoji=, as storysite.EMOJI_FONT +# command-line flags of settings: (section, key) -> (argparse dest, flag) +FLAG_SETTINGS = { + ("catalog", "region"): ("region", "--region"), + ("catalog", "language"): ("language", "--language"), + ("catalog", "version"): ("catalog_version", "--catalog-version"), + ("paths", "catalog"): ("catalog", "--catalog"), + ("paths", "cache"): ("cache", "--cache"), + ("paths", "master"): ("master", "--master"), + ("paths", "apk"): ("apk", "--apk"), + ("paths", "ffmpeg"): ("ffmpeg", "--ffmpeg"), + ("paths", "vgmstream"): ("vgmstream", "--vgmstream"), + ("paths", "node"): ("node", "--node"), + ("paths", "store"): ("store", "--store"), + ("paths", "player"): ("player", "--player"), +} +# the global options: dest -> (flag, metavar, help); --player is an option of `site build` only +GLOBAL_OPTIONS = { + "config": ("--config", "FILE", "TOML config file (else NNNOTES_CONFIG, else ./nnnotes.toml)"), + "region": ("--region", "REGION", "region: a [servers.] table ([catalog] region)"), + "language": ( + "--language", + "LANG", + "catalog and client language: ja, en, zh-Hant, zh-Hans or ko ([catalog] language)", + ), + "catalog": ("--catalog", "FILE", "catalog .bin file ([paths] catalog; else downloaded into the cache)"), + "catalog_version": ( + "--catalog-version", + "VERSION", + "the resource version of an international catalog ([catalog] version; else the version the game API " + "serves when an API root is configured, otherwise main)", + ), + "cache": ("--cache", "DIR", "cache directory ([paths] cache)"), + "store": ("--store", "DIR", "store directory ([paths] store; default /store)"), + "master": ("--master", "DIR", "decoded master data directory ([paths] master)"), + "apk": ("--apk", "FILE", "base.apk ([paths] apk)"), + "ffmpeg": ("--ffmpeg", "EXE", "ffmpeg executable ([paths] ffmpeg; else on PATH)"), + "vgmstream": ("--vgmstream", "EXE", "vgmstream-cli executable ([paths] vgmstream; else on PATH)"), + "node": ("--node", "EXE", "Node.js executable ([paths] node; else on PATH)"), +} +JSON_HELP = "print the command's report as JSON on standard output (default: a short summary)" + + +class Formatter(argparse.HelpFormatter): + """The help of the command line: a usage line without the global options (the help lists them).""" + + def add_usage(self, usage, actions, groups, prefix=None): + hidden = {*GLOBAL_OPTIONS, "json"} + super().add_usage(usage, [a for a in actions if a.dest not in hidden], groups, prefix) + + +def global_options() -> argparse.ArgumentParser: + """The parent parser of the options every command takes, before or after the command's name. Their defaults are + left out of the namespace (an option given before the command name stays when the command's parser does not see + it again) and filled in by `fill_globals`.""" + p = argparse.ArgumentParser(add_help=False) + g = p.add_argument_group("global options") + for dest, (flag, metavar, text) in GLOBAL_OPTIONS.items(): + g.add_argument(flag, dest=dest, metavar=metavar, default=argparse.SUPPRESS, help=text) + g.add_argument("--json", action="store_true", default=argparse.SUPPRESS, help=JSON_HELP) + return p + + +GLOBALS = global_options() + + +def fill_globals(args) -> None: + for dest in (*GLOBAL_OPTIONS, "player"): + if not hasattr(args, dest): + setattr(args, dest, None) + if not hasattr(args, "json"): + args.json = False + + +def load_config(args) -> Config: + """The settings of the config file, the environment and the global options (config.Config.load), made the + process's settings (config.use).""" + overrides = {k: getattr(args, dest, None) for k, (dest, _) in FLAG_SETTINGS.items()} + flags = {k: flag for k, (_, flag) in FLAG_SETTINGS.items()} + cfg = Config.load(getattr(args, "config", None), overrides=overrides, flags=flags) + if getattr(args, "raw_config", False): + return use(cfg) # diagnose the settings as given, before the runtime defaults + if cfg.provider() == "jp" and not cfg.has("catalog", "language"): + cfg = cfg.for_region(cfg.region()) + return use(cfg) + + +def group(sub, name: str, text: str): + """A group of commands `nnnotes NAME `: its subparsers.""" + p = sub.add_parser(name, help=text, description=text, parents=[GLOBALS], formatter_class=Formatter) + return p.add_subparsers(dest=f"{name.replace('-', '_')}_command", required=True, metavar="") + + +def command(sub, name: str, func, text: str, *, reads_config: bool = True) -> argparse.ArgumentParser: + """The parser of a command that runs `func(args, cfg)` (cfg: the settings, None when `reads_config` is false); + `args.usage(message)` exits with a usage error (status 2) of this command.""" + p = sub.add_parser(name, help=text, description=text, parents=[GLOBALS], formatter_class=Formatter) + p.set_defaults(func=func, usage=p.error, reads_config=reads_config) + return p + + +# ---------------------------------------------------------------- output +def write(text: str) -> None: + """Text on standard output as UTF-8 bytes (a GBK / cp932 console cannot encode every string).""" + sys.stdout.flush() + sys.stdout.buffer.write(text.encode("utf-8")) + sys.stdout.buffer.flush() + + +def print_json(doc) -> None: + write(dumps(doc, ensure_ascii=False, indent=1) + "\n") + + +def note(text: str) -> None: + """A line for the reader on standard error (a count, a hint): never part of a report.""" + print(text, file=sys.stderr) + + +def _scalar(v) -> str: + if v is None: + return "-" + if isinstance(v, str): + return v + return dumps(v) + + +def summarize(doc) -> list[str]: + """The summary lines of a report: per field `name: value`, a list by its length, an object by its number of + entries (a small object of plain values inline).""" + if not isinstance(doc, dict): + return [f"{len(doc)} items"] if isinstance(doc, list) else [_scalar(doc)] + lines = [] + for k, v in doc.items(): + if isinstance(v, dict): + plain = all(not isinstance(x, (dict, list)) for x in v.values()) + if v and plain and len(v) <= 8: + lines.append(f"{k}: " + ", ".join(f"{a} {_scalar(b)}" for a, b in v.items())) + else: + lines.append(f"{k}: {len(v)} entries") + elif isinstance(v, list): + lines.append(f"{k}: {len(v)} items") + else: + lines.append(f"{k}: {_scalar(v)}") + return lines + + +def report(args, doc, lines=None) -> None: + """The command's report: as JSON with --json, else the lines `lines(doc)` (default: summarize).""" + if args.json: + print_json(doc) + return + out = (lines or summarize)(doc) + if out: + write("".join(f"{line}\n" for line in out)) + + +# ---------------------------------------------------------------- usage errors +def known_keys(args, cat: Catalog, keys) -> None: + """A usage error (exit 2) naming the keys the catalog does not have.""" + unknown = [k for k in keys if not cat.has(k)] + if unknown: + args.usage(f"not a key of the catalog: {', '.join(unknown)}") + + +def known_row(args, md: Path, table: str, row_id: int, what: str) -> None: + """A usage error (exit 2) naming `row_id` when the master table `table` has no row with that `_id`.""" + from ..source.master import has_row + + try: + found = has_row(md, table, row_id) + except FileNotFoundError: + raise ConfigError( + f"master data {md}: no {table}.json (a directory written by `nnnotes master decode`)" + ) from None + if not found: + args.usage(f"{what} {row_id}: no {table} row with this id") + + +def flac_options(args) -> dict: + """The cri.decode options of --flac-level ({} without it); a level out of range, or the flag where no FLAC is + written, is a usage error.""" + level = args.flac_level + if level is None: + return {} + if level not in FLAC_LEVELS: + args.usage(f"--flac-level {level}: expected 0 to 12") + if args.format != "flac": + args.usage(f"--flac-level: --format {args.format} writes no FLAC") + if getattr(args, "no_audio", False): + args.usage("--flac-level: --no-audio decodes no audio") + return {"flac_level": level} + + +def fonts_extra(fonts: str) -> None: + """`--fonts game` needs the optional `fonts` dependencies.""" + if fonts == "game": + from ..unity.tmpfont import require_extra + + require_extra("--fonts game") + + +def live_options(args) -> dict: + """The --live-option specs as a request (liveoptions.parse_specs); a malformed spec is a usage error.""" + from ..content.liveoptions import OptionSpecError, parse_specs + + try: + return parse_specs(args.live_option) + except OptionSpecError as e: + args.usage(str(e)) + + +# ---------------------------------------------------------------- argument types and options +def nonnegative(s: str) -> int: + try: + v = int(s) + except ValueError: + raise argparse.ArgumentTypeError(f"{s}: expected a whole number") from None + if v < 0: + raise argparse.ArgumentTypeError(f"{s}: expected 0 or more") + return v + + +def positive(s: str) -> int: + v = nonnegative(s) + if v == 0: + raise argparse.ArgumentTypeError(f"{s}: expected 1 or more") + return v + + +def parse_pair(s: str) -> tuple[int, str]: + m = re.fullmatch(r"(\d+)[:_](easy|normal|hard|expert)", s) + if not m: + raise argparse.ArgumentTypeError(f"{s}: expected :") + return int(m.group(1)), m.group(2) + + +def parse_languages(s: str) -> list[str]: + from ..source.languages import LANGUAGES + + codes = [c.strip() for c in s.split(",") if c.strip()] + if not codes or any(c not in LANGUAGES for c in codes): + raise argparse.ArgumentTypeError(f"{s}: expected languages from {', '.join(LANGUAGES)}, comma-separated") + return codes + + +def parse_font(s: str) -> tuple[str, str]: + from ..source.languages import LANGUAGES + + lang, sep, path = s.partition("=") + if not sep or (lang not in LANGUAGES and lang != EMOJI_FONT) or not path: + raise argparse.ArgumentTypeError( + f"{s}: expected = or {EMOJI_FONT}=, " + f"language one of {', '.join(LANGUAGES)}" + ) + return lang, path + + +def out_arg(c, what: str, required: bool = True) -> None: + c.add_argument("-o", "--out", required=required, help=what) + + +def workers_arg(c, what: str, default: str, zero: str = "this process") -> None: + c.add_argument("--workers", type=nonnegative, metavar="N", help=f"{what} (default {default}; 0: {zero})") + + +def audio_args(c, default: str = "flac") -> None: + """--format and --flac-level of a command that decodes cue sheets.""" + c.add_argument("--format", default=default, choices=AUDIO_CHOICES, help=f"audio file format (default {default})") + flac_level_arg(c) + + +def flac_level_arg(c, what: str = "of --format flac") -> None: + c.add_argument( + "--flac-level", + type=int, + metavar="N", + help=f"FLAC compression level 0-12 {what} (default 8; every level decodes to the same samples)", + ) + + +def fonts_arg(c, text: str) -> None: + c.add_argument("--fonts", default="open", choices=("open", "game"), help=text) + + +def band_args(c) -> None: + g = c.add_mutually_exclusive_group() + g.add_argument( + "--band", + type=int, + help="band of the LightWeight background and start timeline " + "(default: band of the music's first vocal character)", + ) + g.add_argument("--leader-card", type=int, help="deck centre MasterMemberCard id (its character's band)") + + +def live_option_arg(c) -> None: + c.add_argument( + "--live-option", + action="append", + metavar="OPTION[=VALUES]", + help="also export the files of a Live option's variants (repeatable): MirrorChart, " + "MeasureLineDisplay, NoteDesignId, NoteEffectId, LiveQuality, NoteSePatternId (every value " + "of the master data, or =v,v...), defaults (the option defaults and ranges only) or all", + ) diff --git a/src/nnnotes/cli/config.py b/src/nnnotes/cli/config.py new file mode 100644 index 0000000..4be3429 --- /dev/null +++ b/src/nnnotes/cli/config.py @@ -0,0 +1,236 @@ +"""`nnnotes config`: write the config file, set or empty one value, check the settings, show the files looked up.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +from ..source.config import DEFAULT_FILE, ENV_CONFIG, ConfigError, config_files, find_file, user_file +from .common import command, group, note, print_json, report, write + + +def _tty() -> bool: + try: + return sys.stdin.isatty() and sys.stdout.isatty() + except (AttributeError, OSError, ValueError): + return False + + +def _target(args, default: Path | None) -> Path: + """The file a config command writes: --file, --user, else `default` (None: the file the commands read).""" + if args.file: + return Path(args.file) + if args.user: + target = user_file() + if target is None: + raise ConfigError( + "no per-user config directory: the environment sets neither APPDATA (Windows) nor " + "XDG_CONFIG_HOME / HOME" + ) + return target + if default is not None: + return default + target = find_file(args.config) + if target is None: + raise ConfigError( + "no config file is read (`nnnotes config path`): name one with --file or --user, or " + "write one with `nnnotes config init`" + ) + return target + + +def _value(args, name: str, text: str, stdin_used: list): + """(section, key, value) of `name` = `text` (`-`: read from standard input, once), checked.""" + from ..source import configfile + + setting, section = configfile.resolve(name) + if text == "-": + if stdin_used: + args.usage("only one value can be read from standard input") + stdin_used.append(name) + if setting.secret and _tty(): + import getpass + + text = getpass.getpass(f"[{section}] {setting.key} (hidden): ") + else: + text = sys.stdin.readline().rstrip("\r\n") + try: + return section, setting.key, configfile.parse_value(setting, text) + except ValueError as e: + raise ConfigError(f"setting {section}.{setting.key}: {e}") from None + + +def _read_note(target: Path) -> None: + read = find_file() + if read is None or read.absolute() != target.absolute(): + note(f"# the commands read {read.absolute() if read else 'no config file'} first (`nnnotes config path`)") + + +def cmd_init(args, cfg): + from ..source import configfile + + named = next((f for n, f in config_files(args.config)[:2] if f is not None), None) + target = _target(args, named or Path(DEFAULT_FILE)) + if target.exists() and not args.force: + raise ConfigError(f"config file {target} exists (--force overwrites it; `nnnotes config set` edits one value)") + values, regions, stdin_used = [], [], [] + for item in args.set or []: + name, sep, text = item.partition("=") + if not sep: + args.usage(f"--set {name}: give it as SECTION.KEY=VALUE") + section, key, value = _value(args, name.strip(), text, stdin_used) + values.append((section, key, value)) + region = section.split(".", 1)[1] if section.startswith("servers.") else None + if region and region not in regions: + regions.append(region) + if not args.set and not args.no_input and not args.json and _tty(): + try: + values, regions = configfile.prompt() + except (KeyboardInterrupt, EOFError): + note("\nnnnotes: config init stopped, nothing written") + sys.exit(1) + configfile.write(target, configfile.render(values, regions)) + report(args, {"file": str(target.absolute()), "written": [f"{s}.{k}" for s, k, _ in values]}, lambda r: [r["file"]]) + note(f"# {len(values)} settings written, the others empty; `nnnotes config check` shows the status of each") + _read_note(target) + + +def _edit(args, value_text: str | None) -> None: + from ..source import configfile + + target = _target(args, None) + text = target.read_text(encoding="utf-8") if target.exists() else configfile.template().decode("utf-8") + if value_text is None: # unset: an empty value + setting, section = configfile.resolve(args.name) + section, key, value = section, setting.key, [] if setting.kind == "list" else "" + else: + section, key, value = _value(args, args.name, value_text, []) + configfile.write(target, configfile.edit(text, [(section, key, value)], str(target))) + report( + args, + {"file": str(target.absolute()), "setting": f"{section}.{key}", "set": bool(value)}, + lambda r: [f"{r['file']}: {r['setting']} {'set' if r['set'] else 'emptied'}"], + ) + _read_note(target) + + +def cmd_set(args, cfg): + _edit(args, args.value) + + +def cmd_unset(args, cfg): + _edit(args, None) + + +def _check_lines(r: dict) -> list[str]: + lines = [f"# file: {r['file']}" if r["file"] else "# no config file is read (`nnnotes config path`)"] + for s in r["settings"]: + status = s["status"] + (f": {s['reason']}" if s.get("reason") else "") + lines.append(f"{s['name']}\t{s.get('origin') or '-'}\t{status}") + return lines + + +def cmd_check(args, cfg): + from ..source import configfile + + r = configfile.check(cfg) + report(args, r, _check_lines) + if not args.json: + note(f"# {r['problems']} problems") + if r["problems"]: + sys.exit(1) + + +def cmd_path(args, cfg): + from ..source import configfile + + files, read = [], None + for name, file in config_files(args.config): + if file is None: + state = "not given" + elif not file.is_file(): + state = "not found" + elif read is None: + read, state = file, "read" + else: + state = "found, not read" + files.append({"name": name, "path": str(file.absolute()) if file is not None else None, "state": state}) + if read is None and file is not None and name in ("--config", ENV_CONFIG): + break # a named file that is missing stops every command + if args.json: + print_json({"schema": configfile.PATHS, "files": files, "read": str(read.absolute()) if read else None}) + return + write("".join(f"{f['name']}\t{f['path'] or '-'}\t{f['state']}\n" for f in files)) + if read is None: + note("# no config file is read: the settings come from the environment and the flags only") + + +def register(sub) -> None: + cs = group(sub, "config", "the config file: write it, set one value, check the settings, show where it is") + + def target(m, verb, default=""): + g = m.add_mutually_exclusive_group() + g.add_argument("--file", help=f"the config file to {verb}{default}") + g.add_argument( + "--user", + action="store_true", + help=f"{verb} the per-user config file, read from any directory (`config path` shows it)", + ) + + m = command( + cs, + "init", + cmd_init, + "write a config file from the template: the values are asked in a terminal, else taken from --set " + "(the other settings empty)", + reads_config=False, + ) + target(m, "write", f" (default: the file --config or NNNOTES_CONFIG names, else ./{DEFAULT_FILE})") + m.add_argument( + "--set", + action="append", + metavar="SECTION.KEY=VALUE", + help="a value to write (repeatable; no questions then): VALUE - reads it from standard input; a " + "list comma-separated; a path is written absolute", + ) + m.add_argument("--no-input", action="store_true", help="ask nothing, also in a terminal") + m.add_argument("--force", action="store_true", help="overwrite an existing file") + m = command( + cs, + "set", + cmd_set, + "set one value of the config file the commands read (or of --file / --user; a missing one is made " + "from the template); comments and other lines stay", + reads_config=False, + ) + m.add_argument("name", metavar="SECTION.KEY", help="the setting: bundle.key, servers.tw.cdn, paths.fonts.ja, ...") + m.add_argument( + "value", + help="the value (a list comma-separated); - reads it from standard input, without echo " + "for the keys in a terminal", + ) + target(m, "edit") + m = command( + cs, + "unset", + cmd_unset, + "empty one value of the config file (an empty value counts as unset)", + reads_config=False, + ) + m.add_argument("name", metavar="SECTION.KEY", help="the setting") + target(m, "edit") + m = command( + cs, + "check", + cmd_check, + "every setting: where it comes from and whether it is valid, never its value; exit status 1 when one " + "is invalid, not found or unknown (--json: nnnotes.config-check/1)", + ) + m.set_defaults(raw_config=True) + command( + cs, + "path", + cmd_path, + "the config files looked up, in order, and the one the commands read (--json: nnnotes.config-path/1)", + reads_config=False, + ) diff --git a/src/nnnotes/cli/export.py b/src/nnnotes/cli/export.py new file mode 100644 index 0000000..67b782f --- /dev/null +++ b/src/nnnotes/cli/export.py @@ -0,0 +1,324 @@ +"""`nnnotes export`: the selected bundles of the catalog into the store and its layouts (`assets`), and single +subjects into a directory or file: an ADV episode, a story, a live, a Live2D model, a spot, a room, shaders, a cue +sheet, the APK's UI library.""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +from ..core.jsonio import write_json +from ..source.session import Session +from . import assets +from .common import ( + DIFFICULTY_CHOICES, + audio_args, + band_args, + command, + flac_options, + fonts_arg, + fonts_extra, + group, + known_keys, + known_row, + live_option_arg, + live_options, + out_arg, + report, +) + + +def cmd_episode(args, cfg): + from ..content import adv + + md = Session(cfg).master_dir() + known_row(args, md, "MasterAdv", args.adv_id, "episode") + cat = Session(cfg).catalog() + doc = adv.to_json(adv.extract(cat, md, args.adv_id)) + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + write_json(out, doc) + r = {"out": str(out), "commands": doc["commandCount"], "resources": len(doc["resources"])} + report(args, r, lambda r: [f"{r['out']} ({r['commands']} commands, {r['resources']} resources)"]) + + +def cmd_story(args, cfg): + from ..content import story + from ..site import webmodel + from ..source import languages + from ..unity.player import PlayerData + + fonts_extra(args.fonts) + flac = flac_options(args) + out = Path(os.path.abspath(args.out)) + models = Path(os.path.abspath(args.models or Path(args.out) / os.pardir / "live2d")) + try: + os.path.relpath(models, out) + except ValueError: + args.usage(f"--models {models}: not on the drive of {out} (story.json names it relative to the story)") + language = languages.check(cfg.require("catalog", "language")) # the story UI's language + md = Session(cfg).master_dir() + known_row(args, md, "MasterAdv", args.adv_id, "episode") + cat = Session(cfg).catalog() + player = PlayerData(Session(cfg).apk()) + source = webmodel.ModelDir(cat, player, models, force=args.force) + r = story.build( + cat, + md, + player, + args.adv_id, + out, + audio_format=args.format, + audio=not args.no_audio, + fonts=args.fonts, + models=source, + language=language, + audio_options=flac, + ) + report(args, {**r, "modelsBuilt": source.built, "modelsSkipped": source.skipped}) + + +def cmd_live(args, cfg): + from ..content import live, liveoptions + from ..site.web import all_pairs + from ..source import languages + from ..unity.player import PlayerData + + fonts_extra(args.fonts) + request = live_options(args) + flac = flac_options(args) + language = languages.check(cfg.require("catalog", "language")) + md = Session(cfg).master_dir() + known_row(args, md, "MasterLiveMusic", args.music_id, "music") + if (args.music_id, args.difficulty) not in all_pairs(md): + args.usage(f"music {args.music_id}: no {args.difficulty} chart (no MasterLiveMusicScore row)") + try: + options = liveoptions.resolve(request, md) + except liveoptions.OptionSpecError as e: + args.usage(str(e)) + cat = Session(cfg).catalog() + try: # an unknown --leader-card, a --band without its scene keys + live.resolve_band(cat, md, args.music_id, band=args.band, leader_card=args.leader_card) + except (KeyError, ValueError) as e: + args.usage(e.args[0] if e.args else type(e).__name__) + r = live.build( + cat, + md, + PlayerData(Session(cfg).apk()), + args.music_id, + args.difficulty, + Path(args.out), + audio_format=args.format, + band=args.band, + leader_card=args.leader_card, + language=language, + fonts=args.fonts, + options=options, + **flac, + ) + report(args, r) + + +def cmd_live2d(args, cfg): + from ..site import webmodel + from ..unity import live2d + + cfg.require_path("paths", "apk") # the Cubism component classes are read from the APK + cat = Session(cfg).catalog() + try: # a model id or a model key + (key,) = webmodel.catalog_models(cat, [args.key]).values() + except ValueError as e: + args.usage(str(e)) + report(args, live2d.extract_model(cat, key, Path(args.out))) + + +def cmd_spot(args, cfg): + from ..content import room, spot + from ..formats import shader + + cfg.require_path("paths", "apk") # the Spot / Spine component classes are read from the APK + md = Session(cfg).master_dir() + known_row(args, md, "MasterHomeSpot", args.spot_id, "spot") + cat = Session(cfg).catalog() + out = Path(args.out) + doc = spot.extract(cat, md, args.spot_id, out) + bg = doc["master"]["_backgroundAssetPath"] + rm = room.extract_room(cat, bg, out / "room.glb") + bundles = cat.fetch_key(bg) + cat.fetch_key(doc["master"]["_situationAssetPath"]) + s = shader.dump(bundles, out / "shaders") + r = { + "out": str(out), + "spot": {"tapTargets": len(doc["characters"]), "skeletons": len(doc["skeletons"])}, + "room": {"meshes": rm["meshCount"], "inactiveMeshes": rm["inactiveMeshes"]}, + "shaders": {"shaders": s["count"], "variants": s["variants"]}, + } + report( + args, + r, + lambda r: [ + f"{r['out']}/spot.json ({r['spot']['tapTargets']} tap targets, {r['spot']['skeletons']} skeletons)", + f"{r['out']}/room.glb ({r['room']['meshes']} meshes, {r['room']['inactiveMeshes']} inactive)", + f"{r['out']}/shaders/ ({r['shaders']['shaders']} shaders, {r['shaders']['variants']} variants)", + ], + ) + + +def cmd_room(args, cfg): + from ..content import room + + cat = Session(cfg).catalog() + known_keys(args, cat, [args.key]) + r = room.extract_room(cat, args.key, Path(args.out)) + report(args, r, lambda r: [f"{r['glb']} ({r['meshCount']} meshes, {r['inactiveMeshes']} inactive)"]) + + +def cmd_shader(args, cfg): + from ..formats import shader + + cat = Session(cfg).catalog() + if args.key: + known_keys(args, cat, [args.key]) + bundles = cat.fetch_key(args.key) + else: + if cat.apk is None: + cfg.require_path("paths", "apk") + bundles = [] + for s in args.apk_bundle: + try: + bundles.append(cat.apk_bundle(s)) + except KeyError as e: # none or several bundles match + args.usage(f"--apk-bundle {e.args[0]}") + r = shader.dump(bundles, Path(args.out)) + report(args, r, lambda r: [f"{r['count']} shaders, {r['variants']} variants", *(f" {n}" for n in r["names"])]) + + +def cmd_audio(args, cfg): + from ..formats import cri + + flac = flac_options(args) + cfg.require_path("paths", "apk") # the HCA keycode is read from the APK + cat = Session(cfg).catalog() + if not cat.has(f"Cri/Sound/{args.cue_sheet}"): + args.usage(f"cue sheet {args.cue_sheet}: no key Cri/Sound/{args.cue_sheet} in the catalog") + out = Path(args.out) + cues = cri.decode(cat, args.cue_sheet, out, fmt=args.format, **flac) + report(args, {"out": str(out), "cues": len(cues)}, lambda r: [f"{r['out']} ({r['cues']} cues)"]) + + +def cmd_ui(args, cfg): + from ..content import ui + + if args.limit is not None and args.limit < 1: + args.usage("--limit must be positive") + index = ui.export_apk_library( + cfg, + args.out, + keys=args.key, + prefix=args.prefix, + limit=args.limit, + dependencies=not args.no_dependencies, + force=args.force, + ) + report( + args, + { + "index": str(Path(args.out).resolve() / "index.json"), + "counts": index["counts"], + "embedded": len(index["embedded"]), + "controllers": len(index["controllers"]), + }, + ) + if index["counts"].get("failed"): + sys.exit(1) + + +def register(sub) -> None: + es = group(sub, "export", "game data -> common formats: the store's asset export and single subjects") + + c = command( + es, + "assets", + assets.cmd_export, + "every object of the selected bundles -> common formats (the store and its layouts)", + ) + out_arg(c, "output directory") + assets.run_args(c) + c.add_argument( + "--link", + choices=("auto", "clone", "hard", "copy"), + help="how the layout files are made from the store: auto (default; [export] link): a clone where " + "the file system makes them, else a hard link (read-only), else a copy; or that method only", + ) + c.add_argument("--strict", action="store_true", help="exit 1 when a view has gaps") + c.add_argument("--dry-run", action="store_true", help="print the plan, run nothing") + + c = command(es, "episode", cmd_episode, "an ADV episode -> JSON") + c.add_argument("adv_id", type=int, help="MasterAdv id of the episode") + out_arg(c, "output .json file") + + c = command( + es, + "story", + cmd_story, + "an ADV episode -> story directory (episode, audio, scene, UI, media, videos) + its Live2D models", + ) + c.add_argument("adv_id", type=int, help="MasterAdv id of the episode") + out_arg(c, "output directory") + c.add_argument( + "--models", metavar="DIR", help="directory of the Live2D models, one / each (default: OUT/../live2d)" + ) + c.add_argument("--force", action="store_true", help="export the models again whose directory exists") + audio_args(c) + c.add_argument("--no-audio", action="store_true", help="do not decode the cue sheets") + fonts_arg( + c, + "open (default): text layout and style only, no font data; game: also the game's TMP fonts " + "(needs the 'fonts' extra)", + ) + + c = command(es, "live", cmd_live, "a live (music + difficulty) -> self-contained live directory") + c.add_argument("music_id", type=int, help="MasterLiveMusic id") + c.add_argument( + "--difficulty", default="expert", choices=DIFFICULTY_CHOICES, help="chart difficulty (default expert)" + ) + audio_args(c) + fonts_arg( + c, + "start canvas text: open (default): layout and style only, no font data (your fonts are not used); " + "game: also the game's TMP fonts (needs the 'fonts' extra)", + ) + band_args(c) + live_option_arg(c) + out_arg(c, "output directory") + + c = command(es, "live2d", cmd_live2d, "a Live2D model -> runtime model directory") + c.add_argument("key", help="model key Character/Live2D///model/, or its model id ") + out_arg(c, "output directory") + + c = command(es, "spot", cmd_spot, "a spot -> spot.json + Spine + room.glb + shaders") + c.add_argument("spot_id", type=int, help="MasterHomeSpot id") + out_arg(c, "output directory") + + c = command(es, "room", cmd_room, "a background prefab -> glb") + c.add_argument("key", help="addressable key of the prefab") + out_arg(c, "output .glb file") + + c = command(es, "shader", cmd_shader, "the shaders of a key's bundle closure or of APK bundles") + g = c.add_mutually_exclusive_group(required=True) + g.add_argument("--key", help="addressable key: the shaders of its bundle closure") + g.add_argument("--apk-bundle", nargs="+", help="substring(s) of APK bundle file names") + out_arg(c, "output directory") + + c = command(es, "audio", cmd_audio, "a CRI cue sheet -> audio files") + c.add_argument("cue_sheet", help="cue sheet name: the key Cri/Sound/ of the catalog") + out_arg(c, "output directory") + audio_args(c) + + c = command(es, "ui", cmd_ui, "the APK's UI prefabs -> an offline UI prefab library for ournotes-player/ui") + out_arg(c, "UI data directory outside the source repositories") + c.add_argument("--key", action="append", help="exact APK catalog key (repeatable)") + c.add_argument("--prefix", default="EmbUI/", help="APK UI key prefix (default EmbUI/)") + c.add_argument("--limit", type=int, help="maximum number of keys") + c.add_argument("--no-dependencies", action="store_true", help="omit additional prefab roots and controllers") + c.add_argument("--force", action="store_true", help="re-export the selected keys") diff --git a/src/nnnotes/cli/master.py b/src/nnnotes/cli/master.py new file mode 100644 index 0000000..b67d353 --- /dev/null +++ b/src/nnnotes/cli/master.py @@ -0,0 +1,103 @@ +"""`nnnotes master`: the master data version a region serves, its download and its decoding.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +from ..source.config import usable_cpus +from ..source.session import Session +from .common import command, group, out_arg, report, workers_arg + +DOWNLOAD_WORKERS = 16 + + +def _failed(r: dict) -> list[str]: + return [f"failed: {f['file']}: {f['error']}" if "error" in f else f"failed: {f}" for f in r["failed"]] + + +def cmd_version(args, cfg): + from ..source import gameapi + + region = cfg.region() + try: + v = Session(cfg).master_version(region) + except gameapi.GameApiError as e: + sys.exit(f"nnnotes: {e}") + result = {"region": region, "masterVersion": v.version, "resourceVersion": v.resource_version} + if v.resource_hash is not None: + result["resourceHash"] = v.resource_hash + report(args, result) + + +def cmd_decode(args, cfg): + from ..source import master + + key = Session(cfg).master_key() + files = master.input_files(args.inputs) + if not files: + raise SystemExit("nnnotes: no master data files (*.bin) among the inputs") + workers = usable_cpus() if args.workers is None else args.workers + r = master.decode_files(files, Path(args.out), key, workers=workers) + report(args, r, lambda r: [f"{r['out']} ({r['decoded']} tables decoded, {len(r['failed'])} failed)", *_failed(r)]) + if r["failed"]: + sys.exit(1) + + +def cmd_download(args, cfg): + from ..source import gameapi, master + + region = cfg.region() + cdn = cfg.cdn(region) + workers = DOWNLOAD_WORKERS if args.fetch_workers is None else args.fetch_workers + try: + if cfg.provider(region) == "jp": + from ..source.jp import Client, master_version + + client = Client(cfg, region) + observation = client.observe() + version = observation.version.version if args.latest else master_version(args.version) + r = master.download( + observation.cdn, + version, + Path(args.out), + workers=workers, + get=lambda url: client.get(url, master=version), + strict=True, + ) + else: + version = Session(cfg).master_version(region).version if args.latest else args.version + r = master.download(cdn, version, Path(args.out), workers=workers) + except (gameapi.GameApiError, master.DownloadError) as e: + sys.exit(f"nnnotes: {e}") + report( + args, + r, + lambda r: [ + f"{r['out']} (version {r['version']}: {r['downloaded']} downloaded, {r['kept']} kept, " + f"{len(r['failed'])} failed of {r['files']} files)", + *_failed(r), + ], + ) + if r["failed"]: + sys.exit(1) + + +def register(sub) -> None: + ms = group(sub, "master", "master data files: the version a region serves, download, decode") + command(ms, "version", cmd_version, "the master data and resource version the region serves now (game API)") + m = command(ms, "decode", cmd_decode, "master data .bin files ->
.json") + m.add_argument("inputs", nargs="+", type=Path, help="directories (their *.bin files) or files") + out_arg(m, "output directory") + workers_arg(m, "parallel decodes", "the CPUs this process may use", zero="one at a time") + m = command(ms, "download", cmd_download, "a master data version from the region's CDN (SHA-256 checked)") + g = m.add_mutually_exclusive_group(required=True) + g.add_argument("--version", help="master data version") + g.add_argument("--latest", action="store_true", help="the version the region serves now (`master version`)") + out_arg(m, "directory for MasterManifest.json and the .bin files") + m.add_argument( + "--fetch-workers", + type=int, + metavar="N", + help=f"parallel downloads (default {DOWNLOAD_WORKERS})", + ) diff --git a/src/nnnotes/cli/musicdata.py b/src/nnnotes/cli/musicdata.py new file mode 100644 index 0000000..dab1236 --- /dev/null +++ b/src/nnnotes/cli/musicdata.py @@ -0,0 +1,133 @@ +"""`nnnotes music-data`: every live song and chart -> one JSON file (musicdata).""" + +from __future__ import annotations + +import sys +from pathlib import Path + +from ..source.session import Session +from .common import command, out_arg, report + + +def cmd_music_data(args, cfg): + from .. import musicdata + from ..musicdata import deckdata + + if args.apk_master: + cfg.require_path("paths", "apk") # the master data files ship in the APK + session = Session(cfg) + apk = session.path("paths", "apk") + try: + deck = ( + None + if args.no_deck + else musicdata.Deck( + seeds=args.seeds, workers=args.workers, aptitude=not args.no_gekisou_aptitude, cache=args.stats_cache + ) + ) + if args.apk_master: + src, region = deckdata.apk_master(apk), deckdata.EMBEDDED + elif args.decoded_master: # decoded elsewhere: no master key + src, region = deckdata.decoded_master(session.master_dir()), cfg.region() + else: + src, region = deckdata.master_files(Path(args.master_files)), cfg.region() + key = None if src.decoded else session.master_key() + cat = session.catalog() + r = musicdata.export( + Path(args.out), + src, + key, + deckdata.catalog_fetch(cat), + None if args.no_bgm else musicdata.catalog_bgm(cat), + region=region, + client=deckdata.apk_client(apk) if apk is not None else {}, + catalog=deckdata.catalog_info(cat, session.store_root()), + deck=deck, + full=args.full, + jacket=musicdata.catalog_jacket(cat) if args.jackets else None, + jackets_dir=args.jackets, + replay_dir=args.replay_dir, + replay_engine=args.replay_engine, + recommend_engine=args.recommend_engine, + ) + except (deckdata.DeckDataError, musicdata.MusicDataError) as e: + sys.exit(f"nnnotes: {e}") + report(args, r) + + +def register(sub) -> None: + c = command( + sub, + "music-data", + cmd_music_data, + "every live song and chart: metadata in every language, chart facts and the deck model's chart " + "statistics -> one JSON file", + ) + g = c.add_mutually_exclusive_group(required=True) + g.add_argument( + "--master-files", + metavar="DIR", + help="master data files as served: MasterManifest.json and the .bin files it lists (`master download`)", + ) + g.add_argument("--apk-master", action="store_true", help="the master data files of base.apk ([paths] apk)") + g.add_argument( + "--decoded-master", + action="store_true", + help="decoded master data ([paths] master or --master) with the MasterManifest.json of the files " + "it was decoded from; no master key", + ) + c.add_argument( + "--full", + action="store_true", + help="also write the deck model's input: every chart's runtime notes and the master data tables " + "about cards, skills, bonuses, scores and events", + ) + c.add_argument("--no-deck", action="store_true", help="do not run the deck model (every chart's deck is null)") + c.add_argument( + "--seeds", + type=int, + default=8, + metavar="N", + help="replay seeds for a chart with a luck range (default 8); expectations do not use sampling", + ) + c.add_argument( + "--workers", type=int, metavar="N", help="threads measuring charts (default: every processor; 0: one thread)" + ) + c.add_argument( + "--no-gekisou-aptitude", + action="store_true", + help="leave out the charts' Gekisou aptitude (every gekisouAptitude is null)", + ) + c.add_argument( + "--stats-cache", + type=Path, + metavar="DIR", + help="reuse native simulation programs and runs in DIR across builds; " + "completed cache entries survive interrupted builds", + ) + c.add_argument( + "--no-bgm", action="store_true", help="do not read the BGM cue sheets (every song's bgm.length is null)" + ) + c.add_argument( + "--replay-dir", + metavar="DIR", + help="write canonical runtime DeckData, per-chart inputs and replay manifest under the output directory", + ) + c.add_argument( + "--replay-engine", + metavar="PKG", + help="the deck model's replay WASM release package (ournotes-replay-wasm-vVERSION.tar.gz or its " + "directory); its web JS/WASM and build-info.json are copied into --replay-dir", + ) + c.add_argument( + "--recommend-engine", + metavar="PKG", + help="the deck model's recommendation WASM release package (ournotes-recommend-wasm-vVERSION.tar.gz " + "or its directory); its web JS/WASM and build-info.json are copied into --replay-dir", + ) + c.add_argument( + "--jackets", + metavar="DIR", + help="also write every song's jacket as DIR/.webp (at most 320 px on the longer side)", + ) + out_arg(c, "output file (.json, or .json.gz for gzip)") diff --git a/src/nnnotes/cli/reference.py b/src/nnnotes/cli/reference.py new file mode 100644 index 0000000..8410465 --- /dev/null +++ b/src/nnnotes/cli/reference.py @@ -0,0 +1,140 @@ +"""The command reference: every command of the command line with its arguments and their help, read from the parser +(not from argparse's help layout, so the text is the same on every Python version). docs/cli.md is `markdown()`; +`python -m nnnotes.cli.reference FILE` writes it.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +HEADER = """# Command reference + +Every command of `nnnotes`, its arguments and their defaults. The global options go before or after the command +name. A command prints a short summary on standard output, or with `--json` its report as JSON; notes and errors go +to standard error. Exit status 2 is a usage or settings error. +""" + + +def _subparsers(p: argparse.ArgumentParser) -> argparse._SubParsersAction | None: + return next((a for a in p._actions if isinstance(a, argparse._SubParsersAction)), None) + + +def _help(a) -> str: + return " ".join((a.help or "").replace("%%", "%").split()) + + +def _choices(sub: argparse._SubParsersAction) -> dict[str, str]: + return {c.dest: _help(c) for c in sub._choices_actions} + + +def argument(a: argparse.Action) -> dict: + """An argument: its flags (or name), value name, help and the facts the help does not say.""" + d = {"flags": list(a.option_strings) or [a.metavar or a.dest], "help": _help(a)} + takes = a.nargs != 0 + if a.option_strings and takes: + d["value"] = a.metavar or a.dest.upper() + if a.choices is not None: + d["choices"] = [str(c) for c in a.choices] + if a.nargs in ("+", "*"): + d["many"] = True + if isinstance(a, argparse._AppendAction): + d["repeatable"] = True + if a.required and a.option_strings: + d["required"] = True + if takes and a.default not in (None, argparse.SUPPRESS, False) and not isinstance(a.default, list): + d["default"] = str(a.default) + return d + + +def _groups(p: argparse.ArgumentParser) -> list[dict]: + out = [] + for g in p._mutually_exclusive_groups: + names = [a.option_strings[-1] for a in g._group_actions if a.option_strings] + out.append({"of": names, "required": g.required}) + return out + + +def commands(parser: argparse.ArgumentParser, global_dests: set[str]) -> list[dict]: + """[{command, help, arguments, exclusive}] of every command, in the order of the help; `global_dests`: the + arguments every command shares (left out).""" + out = [] + + def walk(p, path, text): + sub = _subparsers(p) + if sub is None: + args = [ + argument(a) + for a in p._actions + if not isinstance(a, argparse._HelpAction) and a.dest not in global_dests + ] + out.append({"command": " ".join(path), "help": text, "arguments": args, "exclusive": _groups(p)}) + return + helps = _choices(sub) + for name, child in sub.choices.items(): + walk(child, [*path, name], helps.get(name, "")) + + walk(parser, [], "") + return out + + +def _cell(s: str) -> str: + return s.replace("|", "\\|") + + +def _usage(d: dict) -> str: + flags = ", ".join(f"`{f}`" for f in d["flags"]) + if "value" in d: + flags += f" `{d['value']}`" + (" ..." if d.get("many") else "") + elif d.get("many"): + flags += " ..." + return flags + + +def _facts(d: dict) -> str: + facts = [] + if "choices" in d: + facts.append("one of " + ", ".join(f"`{c}`" for c in d["choices"])) + if "default" in d: + facts.append(f"default `{d['default']}`") + if d.get("repeatable") and "repeatable" not in d["help"]: + facts.append("repeatable") + if d.get("required"): + facts.append("required") + return f" ({'; '.join(facts)})" if facts else "" + + +def _table(rows: list[dict]) -> list[str]: + lines = ["| Argument | Description |", "| --- | --- |"] + lines += [f"| {_cell(_usage(d))} | {_cell(d['help'] + _facts(d))} |" for d in rows] + return lines + + +def markdown(parser: argparse.ArgumentParser | None = None) -> str: + from . import build_parser + from .common import GLOBALS + + parser = parser or build_parser() + shared = [a for a in GLOBALS._actions if not isinstance(a, argparse._HelpAction)] + lines = [HEADER, "## Global options", "", *_table([argument(a) for a in shared]), ""] + for c in commands(parser, {a.dest for a in shared}): + lines += [f"## nnnotes {c['command']}", "", c["help"], ""] + if c["arguments"]: + lines += [*_table(c["arguments"]), ""] + for g in c["exclusive"]: + names = ", ".join(f"`{n}`" for n in g["of"]) + lines += [f"{'Exactly' if g['required'] else 'At most'} one of {names}.", ""] + return "\n".join(lines).rstrip() + "\n" + + +def main(argv=None) -> None: + argv = sys.argv[1:] if argv is None else argv + text = markdown() + if argv: + Path(argv[0]).write_bytes(text.encode("utf-8")) + else: + sys.stdout.buffer.write(text.encode("utf-8")) + + +if __name__ == "__main__": + main() diff --git a/src/nnnotes/cli/servers.py b/src/nnnotes/cli/servers.py new file mode 100644 index 0000000..915243a --- /dev/null +++ b/src/nnnotes/cli/servers.py @@ -0,0 +1,35 @@ +"""`nnnotes servers`: the server list of the bootstrap API root.""" + +from __future__ import annotations + +import sys + +from .common import command, report + + +def _lines(r: dict) -> list[str]: + out = [] + for s in r["servers"]: + roots = f"{s['cdnRoots']} CDN roots, {s['apiRoots']} API roots" + if "cdn" in s: + roots = f"CDN {', '.join(s['cdn']) or '-'}; API {', '.join(s['api']) or '-'}" + out.append(f"{s['name']}\t{s['displayName']}\tarea {s['areaId']}\t{s['region'] or '-'}\t{roots}") + return out + + +def cmd_servers(args, cfg): + from ..source import gameapi + + try: + servers = gameapi.server_list(cfg) + except gameapi.GameApiError as e: + sys.exit(f"nnnotes: {e}") + configured = gameapi.configured_roots(cfg) + report(args, {"servers": [gameapi.server_summary(s, configured, args.show_hosts) for s in servers]}, _lines) + + +def register(sub) -> None: + c = command( + sub, "servers", cmd_servers, "the server list of the bootstrap API root: regions, their CDN and API roots" + ) + c.add_argument("--show-hosts", action="store_true", help="print the CDN and API roots, not only their counts") diff --git a/src/nnnotes/cli/site.py b/src/nnnotes/cli/site.py new file mode 100644 index 0000000..3ea55d0 --- /dev/null +++ b/src/nnnotes/cli/site.py @@ -0,0 +1,256 @@ +"""`nnnotes site build`: live charts, Live2D models and stories -> a static site for ournotes-player.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +from ..site.compress import DEFAULT_ENCODING, ENCODINGS +from ..site.webaudio import DEFAULT_AUDIO_FORMAT, WEB_AUDIO +from ..source.config import ConfigError, use +from ..source.session import Session +from .common import ( + band_args, + command, + fonts_extra, + group, + live_option_arg, + live_options, + nonnegative, + parse_font, + parse_languages, + parse_pair, + positive, + report, +) + + +def cmd_build(args, cfg): + from ..content import liveoptions + from ..site import sitefiles, siteindex, storysite, web, webmodel + + request = live_options(args) + charts, models = bool(args.pair or args.all), bool(args.live2d or args.all_live2d) + stories = bool(args.story or args.all_stories) + if not (charts or models or stories or args.player_only or args.reingest_json): + args.usage( + "one of the arguments --pair --all --live2d --all-live2d --story --all-stories --player-only " + "--reingest-json is required" + ) + if models and (args.player_only or args.reingest_json): + args.usage("--player-only and --reingest-json build nothing: leave out --live2d / --all-live2d") + if stories and (args.player_only or args.reingest_json): + args.usage("--player-only and --reingest-json build nothing: leave out --story / --all-stories") + if request and not charts: + args.usage("--live-option applies to charts: give --pair or --all") + player = cfg.path("paths", "player") + out = Path(args.out) + if args.player_only or args.reingest_json: + r = { + **(sitefiles.reingest_json(out, args.compress) if args.reingest_json else {}), + **sitefiles.write_player(out, sitefiles.check_player(player)), + **siteindex.write_index(out), + } + else: + sitefiles.check_player(player) + regions = None # the one [catalog] region + if args.site_regions or args.all_regions: + regions = sitefiles.site_regions(cfg, args.site_regions, args.all_regions) + if (stories or models) and regions and len({cfg.provider(r) for r in regions}) > 1: + raise ConfigError("build JP stories/models in a separate site directory from international releases") + if (stories or models) and regions: + cfg = use(cfg.for_region(regions[0])) + base = {"region": regions[0]} if regions else {} + unknown = web.unknown_pairs(cfg, args.pair, regions) if args.pair else [] + if unknown: + args.usage( + f"chart{'s' if len(unknown) > 1 else ''} {', '.join(f'{m}:{d}' for m, d in unknown)}: no " + f"MasterLiveMusicScore row in the master data of the site's region(s)" + ) + if request: # a value a region's master data does not have + try: + for md in sitefiles.region_masters(cfg, sitefiles.site_regions(cfg, regions)).values(): + liveoptions.resolve(request, md) + except liveoptions.OptionSpecError as e: + args.usage(str(e)) + r = {} + encoding = args.compress + if models: + cfg.require_path("paths", "apk") # the Cubism component classes and mask materials: the APK + try: + selected = webmodel.catalog_models(Session(cfg).catalog(bundles=False, **base), args.live2d) + except ValueError as e: + args.usage(str(e)) + r.update( + webmodel.build( + out, + selected, + cfg, + player, + force=args.force, + tmp_dir=args.tmp, + workers=args.workers, + encoding=encoding, + **base, + ) + ) + if charts: + fonts_extra(args.fonts) + r.update( + web.build( + out, + None if args.all else args.pair, + cfg, + player, + args.format, + audio=not args.no_audio, + force=args.force, + tmp_dir=args.tmp, + workers=args.workers, + band=args.band, + leader_card=args.leader_card, + regions=regions, + fonts=args.fonts, + read_workers=args.read_workers, + live_options=request, + encoding=args.compress, + ) + ) + if stories: + from ..unity.tmpfont import require_extra + + require_extra("--story / --all-stories") # the font assets of the story text (open and game) + cfg.require_path("paths", "apk") # the ADV settings and UI are embedded content of the APK + unknown = storysite.unknown_stories(cfg, args.story, regions) if args.story else [] + if unknown: + noun = "stories" if len(unknown) > 1 else "story" + args.usage( + f"{noun} {', '.join(map(str, unknown))}: no MasterAdv row in the master data of the " + f"site's region(s)" + ) + r.update( + storysite.build( + out, + args.story, + cfg, + player, + args.format, + audio=not args.no_audio, + force=args.force, + tmp_dir=args.tmp, + workers=args.workers, + regions=regions, + story_languages=args.story_languages, + fonts=args.fonts, + fonts_flags=dict(args.font or []), + encoding=encoding, + ) + ) + report(args, r) + if ( + r.get("failed") + or r.get("modelsFailed") + or r.get("storiesFailed") + or (r.get("storyModels") or {}).get("modelsFailed") + ): + sys.exit(1) + + +def register(sub) -> None: + ss = group(sub, "site", "the static site of ournotes-player") + c = command( + ss, + "build", + cmd_build, + "live charts, Live2D models and stories -> static site for ournotes-player (shared player + per-chart / " + "per-model / per-story data)", + ) + c.add_argument("-o", "--out", required=True, metavar="SITE", help="site directory") + c.add_argument("--player", help="ournotes-player checkout (built) or installed package ([paths] player)") + g = c.add_mutually_exclusive_group() + g.add_argument("--pair", type=parse_pair, action="append", help=": (repeatable)") + g.add_argument("--all", action="store_true", help="every MasterLiveMusic x difficulty") + g.add_argument( + "--player-only", action="store_true", help="rewrite the player files, charts.json and models.json only" + ) + g.add_argument("--reingest-json", action="store_true", help="store every chart's and model's JSON files again") + m = c.add_mutually_exclusive_group() + m.add_argument( + "--live2d", + action="append", + metavar="MODEL", + help="Live2D model id or key Character/Live2D///model/ (repeatable)", + ) + m.add_argument("--all-live2d", action="store_true", help="every Live2D model of the catalog") + c.add_argument( + "--format", + default=DEFAULT_AUDIO_FORMAT, + choices=tuple(WEB_AUDIO), + help=f"audio format of the charts and stories (default {DEFAULT_AUDIO_FORMAT})", + ) + c.add_argument("--no-audio", action="store_true", help="export no audio files") + c.add_argument( + "--compress", + default=DEFAULT_ENCODING, + choices=ENCODINGS, + help="encoding of the compressible assets (JSON, shaders, moc3, ...) where it makes them " + "smaller: gzip (default), br (brotli) or none", + ) + c.add_argument( + "--force", + action="store_true", + help="rebuild charts, models and stories whose manifest exists (stories: also their models)", + ) + c.add_argument("--tmp", help="directory for the temporary live and model builds (default .tmp)") + c.add_argument( + "--workers", + type=nonnegative, + metavar="N", + help="parallel music / story / model processes (default a quarter of the CPUs, up to 8; models up to 4; " + "0: this process)", + ) + c.add_argument( + "--read-workers", + type=positive, + metavar="N", + help="chart read sets run at a time (default half the CPUs, up to 16)", + ) + t = c.add_mutually_exclusive_group() + t.add_argument( + "--story", type=int, action="append", metavar="ADV_ID", help="story episode: its MasterAdv id (repeatable)" + ) + t.add_argument("--all-stories", action="store_true", help="every story episode (MasterAdv) of the master data") + c.add_argument( + "--story-languages", + type=parse_languages, + metavar="LANGS", + help="languages of the stories' text, comma-separated (default: ja,en,zh-Hant,zh-Hans,ko)", + ) + c.add_argument( + "--font", + type=parse_font, + action="append", + metavar="LANG=PATH", + help="font file the story text of a language is drawn with (repeatable; [paths] fonts.); " + "emoji=PATH: the colour emoji font the emoji sprites are drawn from ([paths] fonts.emoji)", + ) + r = c.add_mutually_exclusive_group() + r.add_argument( + "--site-region", + dest="site_regions", + action="append", + metavar="REGION", + help="a region the site serves: a [servers.] table (repeatable; the first is the base; " + "default: [catalog] region)", + ) + r.add_argument("--all-regions", action="store_true", help="every configured region") + c.add_argument( + "--fonts", + default="open", + choices=("open", "game"), + help="text of the start canvas and the stories: open (default): the start canvas has layout and " + "style only, the stories glyphs from your font files (--font); game: the game's TMP fonts " + "(needs the 'fonts' extra)", + ) + band_args(c) + live_option_arg(c) diff --git a/src/nnnotes/cli/store.py b/src/nnnotes/cli/store.py new file mode 100644 index 0000000..4736a4f --- /dev/null +++ b/src/nnnotes/cli/store.py @@ -0,0 +1,48 @@ +"""`nnnotes store`: what `export assets` would run and why (plan), task descriptions run into the store (run), and the +check of every stored content and result (verify).""" + +from __future__ import annotations + +import sys + +from ..source.config import usable_cpus +from ..source.session import Session +from . import assets +from .common import command, group, report, workers_arg + + +def cmd_verify(args, cfg): + from ..pipeline.store import Store + + workers = usable_cpus() if args.workers is None else args.workers + r = Store(Session(cfg).store_root()).verify(quick=args.quick, workers=workers) + report(args, r) + if r["problems"]: + sys.exit(assets.EXIT_FAILED) + + +def register(sub) -> None: + ss = group(sub, "store", "the store of `export assets`: its plan, single tasks, its check") + + c = command( + ss, + "plan", + assets.cmd_plan, + "what `export assets` would run and why, without running it (--json: nnnotes.plan/1)", + ) + c.add_argument("-o", "--out", help="output directory of the export (counts the layout files to write)") + assets.run_args(c) + c.add_argument("--since", metavar="RUN", help="explain against this run (default: the latest of the selection)") + c.add_argument("--check", action="store_true", help="exit 1 when anything would run or change") + c.add_argument("--census", action="store_true", help="first fetch and census the bundles without one") + c.add_argument("--emit-tasks", metavar="DIR", help="write the runnable tasks as task descriptions") + c.add_argument("--why", metavar="TASK", help="the reasons and key parts of one task") + + c = command(ss, "run", assets.cmd_run_stage, "run task descriptions (nnnotes.task/1) into the store") + c.add_argument("tasks", nargs="+", metavar="TASK", help="task description files, or directories of them") + c.add_argument("--fetch", action="store_true", help="fetch inputs through their catalog locators") + c.add_argument("--force", action="store_true", help="run tasks whose result is stored (it must not change)") + + c = command(ss, "verify", cmd_verify, "check every stored content and result") + c.add_argument("--quick", action="store_true", help="check sizes only, do not hash the contents") + workers_arg(c, "parallel hashing", "the CPUs this process may use", zero="one at a time") diff --git a/src/nnnotes/cli/voices.py b/src/nnnotes/cli/voices.py new file mode 100644 index 0000000..d31e50c --- /dev/null +++ b/src/nnnotes/cli/voices.py @@ -0,0 +1,247 @@ +"""`nnnotes voices`: list, search, summarize and fetch the character voices of the voices view (pipeline.voices): +from an export's views/voices.json (--from), else from the document of the configured master data and catalog +(content only; the episodes named by --episode or a row id are read from the catalog).""" + +from __future__ import annotations + +import shutil +import sys +from pathlib import Path + +from ..pipeline.voices import STATUSES +from ..source.languages import LANGUAGES +from ..source.session import Session +from .common import AUDIO_CHOICES, command, flac_level_arg, flac_options, group, note, out_arg, print_json, write + + +def _label(doc: dict, ids, language: str) -> str: + names = {c["id"]: c for c in doc["characters"]} + out = [] + for i in ids: + c = names.get(i) + n = ((c or {}).get("names", {}).get("name") or {}).get(language) if c else None + out.append(n or (c or {}).get("token") or str(i)) + return ",".join(out) or "-" + + +def _line(doc: dict, r: dict, language: str) -> str: + chars = r["characters"] or r.get("inferred", {}).get("characters", []) + text = ((r.get("text") or {}).get("texts") or {}).get(language) or "" + cat = r["category"].get("name") or r.get("inferred", {}).get("situation") or "-" + where = f"{r['sound'].get('sheet')}/{r['sound'].get('cue')}" + who = _label(doc, chars, language) + if "speaker" in r: + names = [(t.get("texts") or {}).get(language) for t in r["speaker"]["texts"]] + who = ",".join(n for n in names if n) or r["speaker"]["name"] or who + return "\t".join([r["id"], who, f"{r['source']['name']}.{cat}", where, r["status"], text.replace("\n", " ")]) + "\n" + + +def _document(args, cfg, episodes=()) -> tuple[dict, Path | None]: + """The document of --from and the export directory it is in, else the index of the configured master data and + catalog with the story episodes `episodes` (ids or assets) read from the catalog.""" + from ..pipeline import contract, views, voices + + if args.source_dir: + p = Path(args.source_dir) + f = p if p.is_file() else p / views.view_path(voices.VIEW, views.MAIN_SUBJECT) + if not f.is_file(): + args.usage( + f"--from {p}: no {views.view_path(voices.VIEW, views.MAIN_SUBJECT)} " + "(export assets --views voices writes it)" + ) + doc = contract.loads(f.read_bytes()) + if doc.get("schema") != voices.SCHEMA: + args.usage(f"{f}: not a voices document") + return doc, (f.parent.parent if not p.is_file() else None) + return voices.index_document(Session(cfg), episodes), None + + +def _language(args, cfg) -> str: + lang = cfg.get("catalog", "language") or "ja" + if lang not in LANGUAGES: + args.usage(f"--language {lang}: one of {', '.join(LANGUAGES)}") + return lang + + +def _filters(args, doc: dict) -> dict: + from ..pipeline import voices + + chars = None + if args.character: + chars = voices.character_ids(doc, args.character) + if not chars: + args.usage(f"--character {args.character}: no character of the master data has this id or name") + eps = None + if args.episode: + eps = voices.episode_ids(doc, args.episode) + if not eps: + args.usage(f"--episode {args.episode}: no episode of the master data has this id or asset") + return { + "characters": chars, + "category": args.category, + "source": args.source, + "status": args.status, + "episodes": eps, + } + + +def _emit(args, doc: dict, rows: list[dict], language: str) -> None: + rows = rows[: args.limit] if args.limit else rows + if args.json: + print_json(rows) + return + write("".join(_line(doc, r, language) for r in rows)) + + +def _episode_args(args) -> tuple: + return (args.episode,) if args.episode else () + + +def cmd_list(args, cfg) -> None: + from ..pipeline import voices + + doc, _ = _document(args, cfg, _episode_args(args)) + lang = _language(args, cfg) + _emit(args, doc, voices.select(doc, **_filters(args, doc)), lang) + + +def cmd_search(args, cfg) -> None: + from ..pipeline import voices + + doc, _ = _document(args, cfg, _episode_args(args)) + lang = _language(args, cfg) + rows = voices.select(doc, **_filters(args, doc), text=args.text, language=args.search_language) + _emit(args, doc, rows, lang) + + +def _summary_lines(s: dict) -> str: + out = [f"rows {s['rows']} gaps {s['gaps']} " + " ".join(f"{k} {v}" for k, v in s["statuses"].items())] + out += [f" {k}\t{v['rows']}\t{v['ok']} ok" for k, v in s["categories"].items()] + rev = s["reverse"] + out.append( + f"reverse: {rev['keys']} keys, {sum(rev['unreferencedKeys'].values())} named by no row; {rev['streams']} " + f"decoded streams, {sum(rev['unreferencedStreams'].values())} reached by no row" + ) + if "story" in s: + st = s["story"] + out.append( + f"story: {st['episodes']} episodes (" + + ", ".join(f"{k} {v}" for k, v in st["statuses"].items()) + + f"); {st['sharedCues']['rows']} voices share a cue with another source" + ) + out += [f" {k}\t{v['episodes']} episodes\t{v['voices']} voices" for k, v in st["kinds"].items()] + return "".join(f"{line}\n" for line in out) + + +def cmd_summary(args, cfg) -> None: + from ..pipeline import voices + + doc, _ = _document(args, cfg) + s = voices.summary(doc) + if args.json: + print_json(s) + else: + write(_summary_lines(s)) + + +def cmd_get(args, cfg) -> None: + from ..core.jsonio import dumps + from ..formats.encoders import FLAC_LEVEL + from ..pipeline import voices + + flac = flac_options(args) + head = f"{voices.load_rules()[voices.VIEW].get('story', {}).get('table')}:" + story = (args.id[len(head) :].split(":", 1)[0],) if args.id.startswith(head) else () + doc, base = _document(args, cfg, story) + rows = voices.rows_for(doc, args.id) + if not rows: + args.usage(f"{args.id}: no voice row or sound id of that number") + r = next((x for x in rows if x["status"] == "ok"), rows[0]) + sheet, cue = r["sound"].get("sheet"), r["sound"].get("cue") + dst = Path(args.out) + dst.mkdir(parents=True, exist_ok=True) + written = [] + streams = (r.get("audio") or {}).get("streams", []) + exported = base is not None and args.format == "flac" and not flac + if exported and streams and all(s.get("path") for s in streams): + for s in streams: + written.append(dst / s["file"]) + shutil.copyfile(base / s["path"], written[-1]) + elif sheet is None or cue is None or r["status"] in ("no-value", "missing-key", "missing-cue", "unsupported"): + note(f"nnnotes: {r['id']}: {r['status']}" + (f" ({r['detail']})" if r.get("detail") else "")) + sys.exit(1) + else: + cfg.require_path("paths", "apk") # the HCA keycode is read from the APK + cat = Session(cfg).catalog() + try: + written = voices.decode_cue(cat, sheet, cue, args.format, flac.get("flac_level", FLAC_LEVEL), dst) + except LookupError as e: + note(f"nnnotes: {r['id']}: {e}") + sys.exit(1) + (dst / "voice.json").write_text( + dumps({"rows": rows, "files": [p.name for p in written]}, ensure_ascii=False, indent=1) + "\n", + encoding="utf-8", + newline="\n", + ) + if args.json: + print_json({"rows": [x["id"] for x in rows], "files": [str(p) for p in written]}) + else: + write("".join(f"{p}\n" for p in written)) + if not written: + sys.exit(1) + + +def register(sub) -> None: + vs = group( + sub, "voices", "character voices of the master data and the story episodes: list, search, summary, get one" + ) + + def base(p, text=True): + p.add_argument( + "--from", + dest="source_dir", + metavar="OUT", + help="an export directory with views/voices.json (export assets --views voices); else the index is " + "built from the configured master data and catalog (no files)", + ) + if text: + p.add_argument("--character", help="a MasterCharacter id or a part of a name in any language") + p.add_argument( + "--category", help="a category (CharacterRankUp or character-rank-up), Source.Category or a source" + ) + p.add_argument("--source", help="a source (Talk, CharacterVoice, ..., HomeSpot, Title, Sound, Story)") + p.add_argument( + "--episode", + metavar="ADV", + help="the Story voices of one episode: a MasterAdv id or an " + "episode asset (without --from: read from the catalog)", + ) + p.add_argument("--status", choices=STATUSES, help="only the rows with this status") + p.add_argument("--limit", type=int, default=0, help="at most N rows (0: all)") + + p = command( + vs, + "list", + cmd_list, + "the voices matching the filters (the text column in the --language language, default [catalog] language)", + ) + base(p) + p = command(vs, "search", cmd_search, "the voices whose text (or cue name) contains TEXT") + p.add_argument("text", help="text to look for") + base(p) + p.add_argument("--search-language", choices=tuple(LANGUAGES), help="search this language only (default: all five)") + p = command(vs, "summary", cmd_summary, "rows per status, category and character; gaps both ways") + base(p, text=False) + p = command( + vs, "get", cmd_get, "the files of one voice: a row id (MasterTalk:561, MasterAdv::) or a sound id" + ) + p.add_argument("id", help="row id or sound id") + out_arg(p, "output directory") + base(p, text=False) + p.add_argument( + "--format", + choices=AUDIO_CHOICES, + default="flac", + help="flac (default: the exported files when --from has them), ogg or wav (decodes the sheet)", + ) + flac_level_arg(p, "of --format flac: decode the sheet at this level, also with --from") diff --git a/src/nnnotes/content/ui.py b/src/nnnotes/content/ui.py index a732f98..a203011 100644 --- a/src/nnnotes/content/ui.py +++ b/src/nnnotes/content/ui.py @@ -374,15 +374,18 @@ def save_index(): return save_index() -def command(args, cfg): +def export_apk_library( + cfg, out, *, keys=None, prefix: str = "EmbUI/", limit: int | None = None, dependencies=True, force=False +) -> dict: + """The library of the APK catalog's keys `keys` (else the keys under `prefix`, the first `limit`), with the + provenance of the APK it was read from -> its index (export_library). A key the APK catalog does not have is a + ConfigError.""" from ..source.apkset import MANIFEST_IN_APK, ApkSet, manifest_version_code, manifest_version_name from ..source.catalog import APK_CATALOG, Catalog from ..source.config import ConfigError from ..source.session import Session from ..unity.player import PlayerData - if args.limit is not None and args.limit < 1: - args.usage("--limit must be positive") apk = cfg.require_path("paths", "apk") with ApkSet(apk) as source: catalog_data = source.read(APK_CATALOG) @@ -397,33 +400,21 @@ def command(args, cfg): cat = Catalog( catalog_data, cfg.require_path("paths", "cache"), bundle_key=lambda: Session(cfg).bundle_key(), apk=apk ) - keys = args.key or cat.keys(args.prefix) - if args.limit is not None: - keys = keys[: args.limit] + keys = keys or cat.keys(prefix) + if limit is not None: + keys = keys[:limit] unknown = sorted(set(keys) - set(cat.keys(""))) if unknown: raise ConfigError("UI key not in the APK catalog: " + ", ".join(unknown)) try: - index = export_library( + return export_library( cat, - args.out, + out, player=PlayerData(Session(cfg).apk()), keys=keys, - dependencies=not args.no_dependencies, - force=args.force, + dependencies=dependencies, + force=force, provenance=provenance, ) except ValueError as exc: raise ConfigError(str(exc)) from None - print( - json.dumps( - { - "index": str(Path(args.out).resolve() / "index.json"), - "counts": index["counts"], - "embedded": len(index["embedded"]), - "controllers": len(index["controllers"]), - } - ) - ) - if index["counts"].get("failed"): - raise SystemExit(1) diff --git a/src/nnnotes/musicdata/__init__.py b/src/nnnotes/musicdata/__init__.py index 2ed65ff..9b6b007 100644 --- a/src/nnnotes/musicdata/__init__.py +++ b/src/nnnotes/musicdata/__init__.py @@ -384,7 +384,9 @@ def __init__( if not _int(seeds) or not 1 <= seeds <= native_max: raise MusicDataError("--seeds must be a positive integer supported by the native model") if workers is not None and (not _int(workers) or not 0 <= workers <= native_max): - raise MusicDataError("--workers must be a nonnegative integer supported by the native model (0: automatic)") + raise MusicDataError( + "--workers must be a nonnegative integer supported by the native model (0: one thread)" + ) if module is None: try: import ournotes_sim as module @@ -393,7 +395,7 @@ def __init__( "the deck model (the package ournotes-sim) is not installed: install " "'nnnotes[deck]', or pass --no-deck" ) from None - self.module, self.seeds, self.workers = module, seeds, None if workers == 0 else workers + self.module, self.seeds, self.workers = module, seeds, 1 if workers == 0 else workers self.aptitude = aptitude self.cache = None if cache is None else Path(cache) self.counts: dict | None = None # native counters, or None when no cache was requested diff --git a/src/nnnotes/pipeline/catalogdb.py b/src/nnnotes/pipeline/catalogdb.py index 7b5388f..b1545c6 100644 --- a/src/nnnotes/pipeline/catalogdb.py +++ b/src/nnnotes/pipeline/catalogdb.py @@ -388,6 +388,15 @@ def apk_catalog(apk) -> bytes: return z.read(APK_CATALOG) +def apk_version(apk: Path | None) -> str | None: + """The versionName of an APK (None without an APK file, or when its manifest has none).""" + if apk is None or not Path(apk).is_file(): + return None + from ..source.gameapi import apk_version_name + + return apk_version_name(apk) + + def fetch(cdn: str, language: str, timeout: float = 120, *, version: str = "main") -> tuple[bytes, str | None]: """The remote catalog of `language` from a CDN base, and the text of its `.hash` file (None when not served).""" base = cdn.rstrip("/") + "/asset/Android/" + Catalog.cache_file(language, Path("."), version=version).name diff --git a/src/nnnotes/pipeline/modelstages.py b/src/nnnotes/pipeline/modelstages.py index d476618..cb5e35c 100644 --- a/src/nnnotes/pipeline/modelstages.py +++ b/src/nnnotes/pipeline/modelstages.py @@ -1,8 +1,8 @@ -"""The model stages: Live2D models and Spine skeletons, written by the extractors of `nnnotes live2d` and -`nnnotes spot` (unchanged) from the bundles of their tasks. +"""The model stages: Live2D models and Spine skeletons, written by the extractors of `nnnotes export live2d` and +`nnnotes export spot` (unchanged) from the bundles of their tasks. -live2d.model, one task per Live2D model key (Character/Live2D///model/, the keys `nnnotes live2d` -takes) whose own bundle (the first dependency of the key's location) is selected: +live2d.model, one task per Live2D model key (Character/Live2D///model/, the keys `nnnotes export +live2d` takes) whose own bundle (the first dependency of the key's location) is selected: inputs "bundle:": the bundles of the key's dependency closure in the order the catalog gives them (Catalog.resolve; n is the position: the extractor loads them in this order, so the order is part of the key) @@ -21,7 +21,7 @@ census holds (the built-in files) are left out context {"object": the asset's object id} artifacts "spine.skeleton:#": the skeleton (.json or .skel), each of its atlases (.atlas) and - the atlas pages (PNG), as spot.write_skeleton writes them for `nnnotes spot` + the atlas pages (PNG), as spot.write_skeleton writes them for `nnnotes export spot` item the asset: exported (its artifacts), or failed with the writer's error Both tasks name their extractor as their one atom ("live2d.extract_model", "spot.write_skeleton"), whose id carries @@ -45,7 +45,7 @@ CENSUS = "unity.census" SCRIPTS = "link.scripts" SCRIPTS_TASK = contract.task_id(SCRIPTS, "all") -# the model keys of `nnnotes live2d` (webmodel.MODEL_KEY; webmodel imports UnityPy) +# the model keys of `nnnotes export live2d` (webmodel.MODEL_KEY; webmodel imports UnityPy) MODEL_KEY = re.compile(r"Character/Live2D/(?P[^/]+)/(?P[^/]+)/model/(?P=name)") SKELETON_CLASS = "SkeletonDataAsset" # the libraries the extractors' output depends on (export._LIBS, the salt of export's PNG cache) @@ -221,8 +221,8 @@ def _tree(root: Path) -> list[tuple[str, Path]]: class Live2DStage(Stage): - """live2d.model: a Live2D model key -> the files of `nnnotes live2d` (module documentation). Subjects and inputs - come from the facts "index" (the catalog index), "selected" and "bundles" ({stable name: Input "bundle"}).""" + """live2d.model: a Live2D model key -> the files of `nnnotes export live2d` (module documentation). Subjects and + inputs come from the facts "index" (the catalog index), "selected" and "bundles" ({stable name: Input "bundle"}).""" name = LIVE2D version = 1 @@ -348,9 +348,9 @@ def reach(index: dict, file: str) -> list[str]: class SpineStage(Stage): - """spine.skeleton: a SkeletonDataAsset -> its Spine files as `nnnotes spot` writes them (module documentation). - Subjects from the census results, the script table (link.scripts:all) and the fact "selected"; inputs from the - fact "bundles". Waits for census tasks that have not run.""" + """spine.skeleton: a SkeletonDataAsset -> its Spine files as `nnnotes export spot` writes them (module + documentation). Subjects from the census results, the script table (link.scripts:all) and the fact "selected"; + inputs from the fact "bundles". Waits for census tasks that have not run.""" name = SPINE version = 1 diff --git a/src/nnnotes/pipeline/objexport.py b/src/nnnotes/pipeline/objexport.py index 733bb22..bc511d6 100644 --- a/src/nnnotes/pipeline/objexport.py +++ b/src/nnnotes/pipeline/objexport.py @@ -14,8 +14,9 @@ Mesh mesh.glb `mesh`: binary glTF in the mesh's local space GameObject (a root) prefab.json `prefab`: the hierarchy below it with its components Material material.json `json` - Shader shader.json `json`: the parsed form as `nnnotes shader` writes it; `program:/ - sp

__`: each compiled sub-program as stored; `index`: the + Shader shader.json `json`: the parsed form as `nnnotes export shader` writes it; + `program:/sp

__`: each compiled sub-program as + stored; `index`: the variants (platform, pass, stage, GPU program type, keywords); `typetree` AnimationClip clip.json `json` AnimatorController controller.json `json` @@ -1114,8 +1115,8 @@ def conv_material(self, info) -> list: return [self._emit(key, "json", encode_object(doc), "json", "material", "json", {})[0]] def conv_shader(self, info) -> list: - """The parsed form (the bytes `nnnotes shader` writes as .json), every compiled sub-program as stored - (`program:/sp

__`), the variant index and the typetree JSON.""" + """The parsed form (the bytes `nnnotes export shader` writes as .json), every compiled sub-program as + stored (`program:/sp

__`), the variant index and the typetree JSON.""" from ..formats import shader key = (info.file, info.path_id) diff --git a/src/nnnotes/pipeline/plan.py b/src/nnnotes/pipeline/plan.py index b898446..781edba 100644 --- a/src/nnnotes/pipeline/plan.py +++ b/src/nnnotes/pipeline/plan.py @@ -99,7 +99,7 @@ def text(self, limit: int | None = 20) -> str: shown[n["stage"]] = shown.get(n["stage"], 0) + 1 if limit is not None and shown[n["stage"]] > limit: continue - reasons = ", ".join(_reason_text(r) for r in n["reasons"]) + reasons = ", ".join(reason_text(r) for r in n["reasons"]) cpu = f"{n['cost']['cpuSeconds']:.1f}" if n.get("cost") else "-" rows.append((n["status"], n["id"], cpu, reasons)) width = max((len(r[1]) for r in rows), default=4) @@ -123,7 +123,7 @@ def text(self, limit: int | None = 20) -> str: return "\n".join(lines) + "\n" -def _reason_text(r: dict) -> str: +def reason_text(r: dict) -> str: extra = {k: v for k, v in r.items() if k != "code"} if not extra: return r["code"] diff --git a/src/nnnotes/pipeline/voices.py b/src/nnnotes/pipeline/voices.py index 5ac9315..3803ce7 100644 --- a/src/nnnotes/pipeline/voices.py +++ b/src/nnnotes/pipeline/voices.py @@ -43,9 +43,9 @@ (nnnotes.voices/1) is the artifact "view.voices:#view"; the `original` layout gets views/voices.json with the file of every stream. -The command `nnnotes voices` lists, searches and fetches voices from that document or, without an export, from an -index built from the configured master data and catalog (content only; the episodes named by --episode or a row id -are read from the catalog). Nothing here imports UnityPy or numpy before a sheet is decoded or an episode loaded. +Without an export, index_document builds the document from the configured master data and catalog (content only; +the story episodes it is given are read from the catalog), and decode_cue decodes one cue from its sheet. Nothing +here imports UnityPy or numpy before a sheet is decoded or an episode loaded. """ from __future__ import annotations @@ -55,7 +55,6 @@ import json import re import shutil -import sys import tempfile import unicodedata from pathlib import Path @@ -1250,40 +1249,8 @@ def _story_summary(doc: dict) -> dict: } -# ---------------------------------------------------------------- command line -def _out(text: str) -> None: - sys.stdout.buffer.write(text.encode("utf-8")) - - -def _print_json(obj) -> None: - from ..core.jsonio import dumps - - _out(dumps(obj, ensure_ascii=False, indent=1) + "\n") - - -def _label(doc: dict, ids, language: str) -> str: - names = {c["id"]: c for c in doc["characters"]} - out = [] - for i in ids: - c = names.get(i) - n = ((c or {}).get("names", {}).get("name") or {}).get(language) if c else None - out.append(n or (c or {}).get("token") or str(i)) - return ",".join(out) or "-" - - -def _line(doc: dict, r: dict, language: str) -> str: - chars = r["characters"] or r.get("inferred", {}).get("characters", []) - text = ((r.get("text") or {}).get("texts") or {}).get(language) or "" - cat = r["category"].get("name") or r.get("inferred", {}).get("situation") or "-" - where = f"{r['sound'].get('sheet')}/{r['sound'].get('cue')}" - who = _label(doc, chars, language) - if "speaker" in r: - names = [(t.get("texts") or {}).get(language) for t in r["speaker"]["texts"]] - who = ",".join(n for n in names if n) or r["speaker"]["name"] or who - return "\t".join([r["id"], who, f"{r['source']['name']}.{cat}", where, r["status"], text.replace("\n", " ")]) + "\n" - - -def _story_ids(tables, rules: dict, specs) -> dict: +# ---------------------------------------------------------------- without an export +def story_ids(tables, rules: dict, specs) -> dict: """{episode asset: adv id} of the story table's rows that `specs` name (an id or an asset each).""" st = rules[VIEW]["story"] want = {str(s).strip() for s in specs} @@ -1295,126 +1262,39 @@ def _story_ids(tables, rules: dict, specs) -> dict: return out -def _document(args, cfg, common, episodes=()) -> tuple[dict, Path | None]: - """The document of --from, else the index of the configured master data and catalog, with the story episodes - `episodes` (ids or assets) read from the catalog.""" - if args.source_dir: - p = Path(args.source_dir) - f = p if p.is_file() else p / views.view_path(VIEW, views.MAIN_SUBJECT) - if not f.is_file(): - args.usage(f"--from {p}: no {views.view_path(VIEW, views.MAIN_SUBJECT)} (export --views voices writes it)") - doc = contract.loads(f.read_bytes()) - if doc.get("schema") != SCHEMA: - args.usage(f"{f}: not a voices document") - return doc, (f.parent.parent if not p.is_file() else None) +def index_document(session, episodes=()) -> dict: + """The document of the master data and catalog of a session.Session, without an export (content only: no row is + exported), with the Story voices of the episodes `episodes` (MasterAdv ids or episode assets) read from the + catalog.""" rules = load_rules() - tables = views.MasterTables(common.session(cfg).master_dir()) - cat = common.session(cfg).catalog(bundles=False) + tables = views.MasterTables(session.master_dir()) + cat = session.catalog(bundles=False) prefixes = [SHEET_PREFIX] + ([rules[VIEW]["story"]["key"].split("{", 1)[0]] if "story" in rules[VIEW] else []) index = views.AddressIndex({k: [] for p in prefixes for k in cat.keys(p)}) story = {} - assets = _story_ids(tables, rules, episodes) if episodes and "story" in rules[VIEW] else {} + assets = story_ids(tables, rules, episodes) if episodes and "story" in rules[VIEW] else {} if assets: - full = common.session(cfg).catalog() + full = session.catalog() column = rules[VIEW]["sound"]["sheet"] for asset in sorted(assets): got = storyvoices.load_episode(full, story_keys(rules, asset)) story[asset] = storyvoices.episode( got.get("script"), got.get("text"), got.get("sound"), got.get("sheets"), sheet_column=column ) - return build(rules, tables, index, story=story), None - - -def _language(args, cfg) -> str: - lang = getattr(args, "language", None) or cfg.get("catalog", "language") or "ja" - if lang not in LANGUAGES: - args.usage(f"--language {lang}: one of {', '.join(LANGUAGES)}") - return lang - - -def _filters(args, doc: dict) -> dict: - chars = None - if getattr(args, "character", None): - chars = character_ids(doc, args.character) - if not chars: - args.usage(f"--character {args.character}: no character of the master data has this id or name") - eps = None - if getattr(args, "episode", None): - eps = episode_ids(doc, args.episode) - if not eps: - args.usage(f"--episode {args.episode}: no episode of the master data has this id or asset") - return { - "characters": chars, - "category": getattr(args, "category", None), - "source": getattr(args, "source", None), - "status": getattr(args, "status", None), - "episodes": eps, - } - - -def _emit(args, doc: dict, rows: list[dict], language: str) -> None: - rows = rows[: args.limit] if args.limit else rows - if args.json: - _print_json(rows) - return - for r in rows: - _out(_line(doc, r, language)) - - -def _episode_args(args) -> tuple: - return (args.episode,) if getattr(args, "episode", None) else () - - -def cmd_list(args, cfg, common) -> None: - doc, _ = _document(args, cfg, common, _episode_args(args)) - lang = _language(args, cfg) - _emit(args, doc, select(doc, **_filters(args, doc)), lang) - + return build(rules, tables, index, story=story) -def cmd_search(args, cfg, common) -> None: - doc, _ = _document(args, cfg, common, _episode_args(args)) - lang = _language(args, cfg) - only = args.language if args.language else None - _emit(args, doc, select(doc, **_filters(args, doc), text=args.text, language=only), lang) - -def cmd_summary(args, cfg, common) -> None: - doc, _ = _document(args, cfg, common) - s = summary(doc) - if args.json: - _print_json(s) - return - _out(f"rows {s['rows']} gaps {s['gaps']} " + " ".join(f"{k} {v}" for k, v in s["statuses"].items()) + "\n") - for k, v in s["categories"].items(): - _out(f" {k}\t{v['rows']}\t{v['ok']} ok\n") - rev = s["reverse"] - _out( - f"reverse: {rev['keys']} keys, {sum(rev['unreferencedKeys'].values())} named by no row; {rev['streams']} " - f"decoded streams, {sum(rev['unreferencedStreams'].values())} reached by no row\n" - ) - if "story" in s: - st = s["story"] - _out( - f"story: {st['episodes']} episodes (" - + ", ".join(f"{k} {v}" for k, v in st["statuses"].items()) - + f"); {st['sharedCues']['rows']} voices share a cue with another source\n" - ) - for k, v in st["kinds"].items(): - _out(f" {k}\t{v['episodes']} episodes\t{v['voices']} voices\n") - - -def _decode_one(cfg, common, sheet: str, cue: str, fmt: str, level: int, dst: Path) -> list[Path]: - """The files of a cue's streams, decoded from its sheet alone (cri.decode, as the export decodes a sheet).""" +def decode_cue(cat, sheet: str, cue: str, fmt: str, flac_level: int, dst: Path) -> list[Path]: + """The files of a cue's streams in `dst`, decoded from its sheet alone (cri.decode, as the export decodes a + sheet). LookupError when the sheet has no such cue.""" from ..formats import cri - cfg.require_path("paths", "apk") - cat = common.session(cfg).catalog() files, _ = cri.acb_data(cat, sheet) c = acb.cue_streams(files["acb"]).get(cue) if c is None: raise LookupError(f"sheet {sheet} has no cue {cue}") with tempfile.TemporaryDirectory(prefix="nnnotes-voice-") as tmp: - cri.decode(cat, sheet, Path(tmp), fmt=fmt, flac_level=level) + cri.decode(cat, sheet, Path(tmp), fmt=fmt, flac_level=flac_level) streams = {s["stream"]: s for s in json.loads((Path(tmp) / "streams.json").read_text(encoding="utf-8"))} out = [] for w in c["streams"]: @@ -1424,106 +1304,3 @@ def _decode_one(cfg, common, sheet: str, cue: str, fmt: str, level: int, dst: Pa out.append(dst / s["file"]) shutil.copyfile(Path(tmp) / s["file"], out[-1]) return out - - -def cmd_get(args, cfg, common) -> None: - head = f"{load_rules()[VIEW].get('story', {}).get('table')}:" - story = (args.id[len(head) :].split(":", 1)[0],) if args.id.startswith(head) else () - doc, base = _document(args, cfg, common, story) - rows = rows_for(doc, args.id) - if not rows: - args.usage(f"{args.id}: no voice row or sound id of that number") - r = next((x for x in rows if x["status"] == "ok"), rows[0]) - sheet, cue = r["sound"].get("sheet"), r["sound"].get("cue") - dst = Path(args.out) - dst.mkdir(parents=True, exist_ok=True) - written = [] - streams = (r.get("audio") or {}).get("streams", []) - if base is not None and args.format == "flac" and streams and all(s.get("path") for s in streams): - for s in streams: - written.append(dst / s["file"]) - shutil.copyfile(base / s["path"], written[-1]) - elif sheet is None or cue is None or r["status"] in ("no-value", "missing-key", "missing-cue", "unsupported"): - print(f"nnnotes: {r['id']}: {r['status']}" + (f" ({r['detail']})" if r.get("detail") else ""), file=sys.stderr) - sys.exit(1) - else: - from ..formats.encoders import FLAC_LEVEL - - try: - written = _decode_one(cfg, common, sheet, cue, args.format, FLAC_LEVEL, dst) - except LookupError as e: - print(f"nnnotes: {r['id']}: {e}", file=sys.stderr) - sys.exit(1) - from ..core.jsonio import dumps - - (dst / "voice.json").write_text( - dumps({"rows": rows, "files": [p.name for p in written]}, ensure_ascii=False, indent=1) + "\n", - encoding="utf-8", - newline="\n", - ) - for p in written: - _out(f"{p}\n") - if not written: - sys.exit(1) - - -def register(sub, common) -> None: - """The `voices` command (common.session(cfg): the data sources, a session.Session).""" - c = sub.add_parser( - "voices", help="character voices of the master data and the story episodes: list, search, summary, get one" - ) - vs = c.add_subparsers(dest="voices_cmd", required=True, metavar="") - - def base(p, text=True): - p.add_argument( - "--from", - dest="source_dir", - metavar="OUT", - help="an export directory with views/voices.json (export --views voices); else the index is " - "built from the configured master data and catalog (no files)", - ) - if text: - p.add_argument("--character", help="a MasterCharacter id or a part of a name in any language") - p.add_argument( - "--category", help="a category (CharacterRankUp or character-rank-up), Source.Category or a source" - ) - p.add_argument("--source", help="a source (Talk, CharacterVoice, ..., HomeSpot, Title, Sound, Story)") - p.add_argument( - "--episode", - metavar="ADV", - help="the Story voices of one episode: a MasterAdv id or an " - "episode asset (without --from: read from the catalog)", - ) - p.add_argument("--status", choices=STATUSES, help="only the rows with this status") - p.add_argument("--limit", type=int, default=0, help="at most N rows (0: all)") - p.add_argument("--json", action="store_true", help="the rows as JSON") - - def done(p, fn): - p.set_defaults(func=lambda args, cfg: fn(args, cfg, common), usage=p.error) - - p = vs.add_parser("list", help="the voices matching the filters") - base(p) - p.add_argument("--language", choices=LANGUAGES, help="the language of the text column ([catalog] language)") - done(p, cmd_list) - p = vs.add_parser("search", help="the voices whose text (or cue name) contains TEXT") - p.add_argument("text", help="text to look for") - base(p) - p.add_argument("--language", choices=LANGUAGES, help="search this language only (default: all five)") - done(p, cmd_search) - p = vs.add_parser("summary", help="rows per status, category and character; gaps both ways") - base(p, text=False) - p.add_argument("--json", action="store_true", help="the summary as JSON") - done(p, cmd_summary) - p = vs.add_parser( - "get", help="the files of one voice: a row id (MasterTalk:561, MasterAdv::) or a sound id" - ) - p.add_argument("id", help="row id or sound id") - p.add_argument("-o", "--out", required=True, help="output directory") - base(p, text=False) - p.add_argument( - "--format", - choices=("flac", "ogg", "wav"), - default="flac", - help="flac (default: the exported files when --from has them), ogg or wav (decodes the sheet)", - ) - done(p, cmd_get) diff --git a/src/nnnotes/site/compress.py b/src/nnnotes/site/compress.py index e0ac4e4..1e0b343 100644 --- a/src/nnnotes/site/compress.py +++ b/src/nnnotes/site/compress.py @@ -14,7 +14,7 @@ import zlib from pathlib import Path -ENCODINGS = ("gzip", "br", "none") # web --compress +ENCODINGS = ("gzip", "br", "none") # site build --compress DEFAULT_ENCODING = "gzip" SUFFIXES = {"gzip": ".gz", "br": ".br"} # stored name suffix of an encoded asset # extensions of the stored names worth encoding (text, meshes, raw PCM); images and compressed audio are left as diff --git a/src/nnnotes/site/web.py b/src/nnnotes/site/web.py index c580b1f..d31b74a 100644 --- a/src/nnnotes/site/web.py +++ b/src/nnnotes/site/web.py @@ -261,7 +261,7 @@ def rd(rel): # ---------------------------------------------------------------- store -ENCODINGS = compress.ENCODINGS # stored encodings of site assets (web --compress) +ENCODINGS = compress.ENCODINGS # stored encodings of site assets (site build --compress) _TEXTS = cache.bucket("webtext") # (build, store, encoding, path, text) -> manifest entry diff --git a/src/nnnotes/site/webaudio.py b/src/nnnotes/site/webaudio.py index fa9d1ac..cb07add 100644 --- a/src/nnnotes/site/webaudio.py +++ b/src/nnnotes/site/webaudio.py @@ -1,6 +1,6 @@ -"""The audio formats of a web build (`web --format`): per format the file suffix and the ffmpeg encoder flags (None: -the FLAC of the decode itself). Imports nothing, so the command line lists the formats without loading the -exporters.""" +"""The audio formats of a site build (`site build --format`): per format the file suffix and the ffmpeg encoder +flags (None: the FLAC of the decode itself). Imports nothing, so the command line lists the formats without loading +the exporters.""" # BGM encodings for the web (ffmpeg). Chromium's decodeAudioData decodes each sample-aligned with the FLAC (encoder # delay / pre-skip removed). Default AAC-LC 160 kbit/s in MP4: decodes in Safari / iOS as well. diff --git a/src/nnnotes/site/webmodel.py b/src/nnnotes/site/webmodel.py index f5910df..5b927d7 100644 --- a/src/nnnotes/site/webmodel.py +++ b/src/nnnotes/site/webmodel.py @@ -38,7 +38,7 @@ Model sources: story.build takes the models of an episode from a source, `ensure(address) -> {"id", "motionSync"}` plus `story_fields(story_dir)` (what story.json gets besides `models`). ModelDir exports each model into

// -(the files above; `nnnotes story`), SiteModels reads model.json from the model manifests of a site (the site's +(the files above; `nnnotes export story`), SiteModels reads model.json from the model manifests of a site (the site's stories). """ @@ -392,8 +392,8 @@ class ModelDir: exports a model whose directory does not exist, or every model once when `force`; `built` / `skipped`: the ids (a model another process installed meanwhile is skipped). - Processes may share `root` (several `nnnotes story` at once): a model directory appears in one rename, whole, and - the first process to install a model wins; the others drop their export (the same bytes: exports are + Processes may share `root` (several `nnnotes export story` at once): a model directory appears in one rename, + whole, and the first process to install a model wins; the others drop their export (the same bytes: exports are deterministic) and use it. `force` moves a previous directory aside before installing; a process that finds the directory gone in between exports the model itself.""" diff --git a/src/nnnotes/source/config.py b/src/nnnotes/source/config.py index 1ec9694..a31d3bc 100644 --- a/src/nnnotes/source/config.py +++ b/src/nnnotes/source/config.py @@ -295,7 +295,7 @@ def cdn(self, region: str) -> str: def catalog_pin(self, region: str | None = None) -> tuple[str, str, str] | None: """The pinned resource version of an international catalog, (value, section, key) of the setting that pins - it: the --catalog-release flag, else `[servers.] catalog_version`, else `[catalog] version`; None + it: the --catalog-version flag, else `[servers.] catalog_version`, else `[catalog] version`; None when nothing pins it (session.Session.catalog_version asks the API then).""" region = region or self.get("catalog", "region") section, key = "catalog", "version" diff --git a/src/nnnotes/source/jp.py b/src/nnnotes/source/jp.py index d59a146..960c3c3 100644 --- a/src/nnnotes/source/jp.py +++ b/src/nnnotes/source/jp.py @@ -274,7 +274,7 @@ def read_source(path, data): raise ValueError() return Source.from_dict(doc["source"]) except (OSError, ValueError, KeyError, TypeError): - raise ConfigError("JP catalog needs its matching .source.json sidecar (written by catalogs fetch)") from None + raise ConfigError("JP catalog needs its matching .source.json sidecar (written by `catalog fetch`)") from None def write_source(path, data, source): diff --git a/src/nnnotes/source/nnnotes.example.toml b/src/nnnotes/source/nnnotes.example.toml index df5e951..8f97483 100644 --- a/src/nnnotes/source/nnnotes.example.toml +++ b/src/nnnotes/source/nnnotes.example.toml @@ -23,7 +23,7 @@ region = "" # catalog language: catalog__.bin, one of ja, en, zh-Hant, zh-Hans, ko; also the client language of the # live start canvas, the story UI and the web model labels (flag --language) language = "" -# optional international resource version (flag --catalog-release); empty: API resource_version when configured, +# optional international resource version (flag --catalog-version); empty: API resource_version when configured, # otherwise legacy main. Set main explicitly to inspect the old catalog; JP uses its own version/hash discovery. version = "" @@ -35,11 +35,11 @@ provider = "" client_version = "" # optional per-region APK, base.apk with adjacent splits, APKS/XAPK, or directory containing base.apk apk = "" -# optional per-region catalog file; JP files use a .source.json sidecar written by catalogs fetch +# optional per-region catalog file; JP files use a .source.json sidecar written by `catalog fetch` catalog = "" -# optional per-region international catalog version; overrides [catalog] version, but not --catalog-release +# optional per-region international catalog version; overrides [catalog] version, but not --catalog-version catalog_version = "" -# label of the region in the local catalog browser (`nnnotes browse`) +# label of the region in the local catalog browser (`nnnotes catalog browse`) name = "" # CDN base URL of the region: catalogs, bundles, raw audio data and master data are fetched below it cdn = "" @@ -76,10 +76,10 @@ ffmpeg = "" vgmstream = "" # Node.js executable for the web build's read-set script; empty: node on PATH (flag --node) node = "" -# ournotes-player checkout (built) or installed package for `nnnotes web` (flag --player) +# ournotes-player checkout (built) or installed package for `nnnotes site build` (flag --player) player = "" [export] -# how `nnnotes export` makes the layout files from the store: auto (a clone where the file system makes them, else +# how `nnnotes export assets` makes the layout files from the store: auto (a clone where the file system makes them, else # a hard link to the read-only store object, else a copy), clone, hard or copy; empty: auto (flag --link) link = "" diff --git a/src/nnnotes/source/session.py b/src/nnnotes/source/session.py index 71740d9..d652272 100644 --- a/src/nnnotes/source/session.py +++ b/src/nnnotes/source/session.py @@ -12,7 +12,7 @@ from . import gameapi from .addressables import BundleKey from .catalog import Catalog -from .config import Config, ConfigError, check_catalog_version +from .config import Config, ConfigError, check_catalog_version, describe from .master import MasterKey @@ -40,6 +40,19 @@ def path(self, section: str, key: str, kind: str = "file") -> Path | None: raise ConfigError(f"setting {section}.{key}: {kind} {p} not found") return p + def store_root(self) -> Path: + """The store directory: `[paths] store`, else the store/ directory of `[paths] cache`.""" + p = self.cfg.path("paths", "store") + if p is not None: + return p + cache = self.cfg.path("paths", "cache") + if cache is not None: + return cache / "store" + raise ConfigError( + "setting paths.store is not set (nor paths.cache, whose store/ directory is the default): " + f"give it as {describe('paths', 'store', '--store')}" + ) + def apk(self, region: str | None = None) -> Path: """`[paths] apk` of `region`, required.""" cfg = self.cfg.for_region(region) if region else self.cfg @@ -67,7 +80,7 @@ def master_version(self, region: str | None = None, *, timeout: float = gameapi. ) def catalog_version(self, region: str | None = None) -> str: - """The resource version of an international catalog: the --catalog-release flag, the regional or global pin, + """The resource version of an international catalog: the --catalog-version flag, the regional or global pin, the resource version the region's API serves, else `main`.""" region = region or self.cfg.get("catalog", "region") pin = self.cfg.catalog_pin(region) diff --git a/tests/cli/test_cli_assets.py b/tests/cli/test_assets.py similarity index 81% rename from tests/cli/test_cli_assets.py rename to tests/cli/test_assets.py index 96a05d0..0c7089b 100644 --- a/tests/cli/test_cli_assets.py +++ b/tests/cli/test_assets.py @@ -1,4 +1,4 @@ -"""The asset commands on synthetic catalogs: export, plan, run-stage, catalogs, store verify. +"""The asset commands on synthetic catalogs: export assets, store plan, store run, catalog versions, store verify. Bundles here are UnityFS-signed files whose body is the census document a real census would find; the census and export stages of the pipeline are replaced by small stages that read them (the registry is patched), the catalog, @@ -14,7 +14,8 @@ import pytest from nnnotes import cli -from nnnotes.cli import cli_assets +from nnnotes.cli import assets, common +from nnnotes.cli import catalog as catalog_cli from nnnotes.pipeline import contract from nnnotes.pipeline.census import CensusStage from nnnotes.pipeline.contract import Cost @@ -246,28 +247,26 @@ def run(self, task, store): return Output(arts, items) -REAL = {s.name: s for s in cli_assets.STAGE_SOURCES} +REAL = {s.name: s for s in assets.STAGE_SOURCES} FAKE_SOURCES = ( REAL["catalog.index"], - cli_assets.StageSource("unity.census", "tests.cli.test_cli_assets:FakeCensus"), + assets.StageSource("unity.census", "tests.cli.test_assets:FakeCensus"), REAL["link.scripts"], REAL["link.addresses"], - cli_assets.StageSource( - "unity.export", "tests.cli.test_cli_assets:FakeExport", output=True, root="_bundles/{subject}" - ), + assets.StageSource("unity.export", "tests.cli.test_assets:FakeExport", output=True, root="_bundles/{subject}"), REAL["link.artifacts"], ) @pytest.fixture def fake(monkeypatch): - monkeypatch.setattr(cli_assets, "STAGE_SOURCES", FAKE_SOURCES) + monkeypatch.setattr(assets, "STAGE_SOURCES", FAKE_SOURCES) def nn(capsys, *argv): - """Run an asset command; (exit code, stdout, stderr).""" + """Run a command; (exit code, stdout, stderr).""" try: - cli_assets.main([str(a) for a in argv]) + cli.main([str(a) for a in argv]) code = 0 except SystemExit as e: code = e.code if isinstance(e.code, int) else (0 if e.code is None else 1) @@ -289,17 +288,30 @@ def tree(root: Path, skip=()) -> dict: def export(capsys, tmp_path, d, store, out, *more): - return nn(capsys, *flags(d), "export", "-o", tmp_path / out, "--store", tmp_path / store, "--workers", "0", *more) + return nn( + capsys, + *flags(d), + "export", + "assets", + "-o", + tmp_path / out, + "--store", + tmp_path / store, + "--workers", + "0", + "--json", + *more, + ) # ---------------------------------------------------------------- selection def test_selection_items_and_groups(): - assert cli_assets.parse_selection([]) == ["all"] - assert cli_assets.parse_selection(["key:Card/", "group:bg", "key:Card/"]) == ["group:bg", "key:Card/"] + assert assets.parse_selection([]) == ["all"] + assert assets.parse_selection(["key:Card/", "group:bg", "key:Card/"]) == ["group:bg", "key:Card/"] with pytest.raises(ValueError): - cli_assets.parse_selection(["card"]) - assert cli_assets.bundle_group("membercard_assets_membercard_100101") == "membercard" - assert cli_assets.bundle_group("live2d_yomogi_023") == "live2d" + assets.parse_selection(["card"]) + assert assets.bundle_group("membercard_assets_membercard_100101") == "membercard" + assert assets.bundle_group("live2d_yomogi_023") == "live2d" def test_select_adds_the_dependencies_to_the_census(): @@ -307,14 +319,14 @@ def test_select_adds_the_dependencies_to_the_census(): idx = index(catalog_bytes()) stable = {n: contract.stable_bundle_name(NAMES[n]) for n in NAMES} - s = cli_assets.select(idx, ["group:card"]) + s = assets.select(idx, ["group:card"]) assert s.selected == sorted([stable["card_1"], stable["card_2"]]) assert s.census == sorted([stable["card_1"], stable["card_2"], stable["shared"]]) - s = cli_assets.select(idx, ["key:Card/1/"]) + s = assets.select(idx, ["key:Card/1/"]) assert s.selected == s.census == sorted([stable["card_1"], stable["shared"]]) - s = cli_assets.select(idx, ["bundle:bg_*"]) + s = assets.select(idx, ["bundle:bg_*"]) assert s.selected == s.census == [stable["bg_x"]] - assert cli_assets.select(idx, ["all"]).selected == sorted(stable.values()) + assert assets.select(idx, ["all"]).selected == sorted(stable.values()) # ---------------------------------------------------------------- export @@ -331,7 +343,7 @@ def test_export_writes_layouts_reports_and_hits_the_second_time(tmp_path, capsys mb = f"_bundles/{contract.stable_bundle_name(NAMES['card_1'])}/MonoBehaviour/~7.json" body = contract.loads((tmp_path / "out" / mb).read_bytes()) assert body["script"] == "Game|Game|Widget" # resolved through link.scripts - reports = tmp_path / "out" / cli_assets.REPORTS + reports = tmp_path / "out" / assets.REPORTS fails = contract.loads((reports / "failures.json").read_bytes())["failures"] assert [(f["code"], f["object"]) for f in fails] == [("fake.broken", f"{CABS['bg_x']}:4")] cov = contract.loads((reports / "coverage.json").read_bytes())["classes"] @@ -353,9 +365,11 @@ def test_export_writes_layouts_reports_and_hits_the_second_time(tmp_path, capsys again = json.loads(out) assert code == 1 and again["tasks"]["ran"] == 0 and again["run"] == summary["run"] assert again["layouts"]["original"]["write"] == 0 - same = lambda t: {k: v for k, v in t.items() if k != f"{cli_assets.REPORTS}/run.json"} # noqa: E731 + same = lambda t: {k: v for k, v in t.items() if k != f"{assets.REPORTS}/run.json"} # noqa: E731 assert same(tree(tmp_path / "out")) == same(first) # the run's statuses are hits now - code, out, _ = nn(capsys, *flags(d), "plan", "-o", tmp_path / "out", "--store", tmp_path / "store", "--check") + code, out, _ = nn( + capsys, *flags(d), "store", "plan", "-o", tmp_path / "out", "--store", tmp_path / "store", "--check" + ) assert code == 0, out @@ -366,6 +380,7 @@ def test_export_is_the_same_with_worker_processes(tmp_path, capsys, fake): capsys, *flags(d), "export", + "assets", "-o", tmp_path / "o2", "--store", @@ -378,7 +393,7 @@ def test_export_is_the_same_with_worker_processes(tmp_path, capsys, fake): assert code == 1, err assert tree(tmp_path / "s0", ["inputs", "runs", "costs"]) == tree(tmp_path / "s2", ["inputs", "runs", "costs"]) o0, o2 = tree(tmp_path / "o0"), tree(tmp_path / "o2" / "original") - assert {k: v for k, v in o0.items() if not k.startswith(cli_assets.REPORTS)} == o2 + assert {k: v for k, v in o0.items() if not k.startswith(assets.REPORTS)} == o2 cas = tree(tmp_path / "o2" / "cas") assert all(p.startswith("assets/") or p == "manifest.json" for p in cas) @@ -389,12 +404,14 @@ def test_plan_emit_tasks_and_run_stage_give_the_exported_store(tmp_path, capsys, rounds = 0 while True: tasks = tmp_path / f"round{rounds}" - code, out, err = nn(capsys, *flags(d), "plan", "--store", tmp_path / "q", "--emit-tasks", tasks, "--json") + code, out, err = nn( + capsys, *flags(d), "store", "plan", "--store", tmp_path / "q", "--emit-tasks", tasks, "--json" + ) assert code == 0, err doc = json.loads(out) if not any(n["status"] == "run" for n in doc["nodes"]): break - code, out, err = nn(capsys, *flags(d), "run-stage", tasks, "--store", tmp_path / "q") + code, out, err = nn(capsys, *flags(d), "store", "run", tasks, "--store", tmp_path / "q") assert code in (0, 1), err rounds += 1 assert rounds == 4 # index + census; scripts + addresses; export; artifacts @@ -404,7 +421,7 @@ def test_plan_emit_tasks_and_run_stage_give_the_exported_store(tmp_path, capsys, def test_plan_explains_and_waits_for_bundles_not_fetched(tmp_path, capsys, fake): d = setup_data(tmp_path, cached={"card_1", "shared", "bg_x"}) - code, out, _ = nn(capsys, *flags(d), "plan", "--store", tmp_path / "s", "--json", "--check") + code, out, _ = nn(capsys, *flags(d), "store", "plan", "--store", tmp_path / "s", "--json", "--check") assert code == 1 nodes = {n["id"]: n for n in json.loads(out)["nodes"]} card2 = contract.stable_bundle_name(NAMES["card_2"]) @@ -412,20 +429,22 @@ def test_plan_explains_and_waits_for_bundles_not_fetched(tmp_path, capsys, fake) assert nodes[f"unity.census:{card2}"]["reasons"] == [{"code": "pending", "task": f"fetch:{NAMES['card_2']}"}] assert nodes["link.scripts:all"]["status"] == "unknown" assert nodes["catalog.index:main"]["reasons"] == [{"code": "new-input"}] - code, out, _ = nn(capsys, *flags(d), "plan", "--store", tmp_path / "s") + code, out, _ = nn(capsys, *flags(d), "store", "plan", "--store", tmp_path / "s") assert "unknown" in out and "tasks:" in out def test_parameters_change_keys_and_are_explained(tmp_path, capsys, fake): d = setup_data(tmp_path) export(capsys, tmp_path, d, "s", "o") - base = [*flags(d), "plan", "--store", tmp_path / "s", "--json"] + base = [*flags(d), "store", "plan", "--store", tmp_path / "s", "--json"] code, out, _ = nn(capsys, *base, "--png-level", "3") runs = [n for n in json.loads(out)["nodes"] if n["status"] == "run"] assert {n["stage"] for n in runs} == {"unity.export"} assert runs[0]["reasons"] == [{"code": "params-changed", "path": "/png/level", "old": 6, "new": 3}] tid = runs[0]["id"] - code, out, _ = nn(capsys, *flags(d), "plan", "--store", tmp_path / "s", "--png-level", "3", "--why", tid) + code, out, _ = nn( + capsys, *flags(d), "store", "plan", "--store", tmp_path / "s", "--png-level", "3", "--why", tid, "--json" + ) w = json.loads(out) assert w["keyParts"]["old"]["params"]["png"]["level"] == 6 and w["keyParts"]["new"]["params"]["png"]["level"] == 3 code, out, _ = export(capsys, tmp_path, d, "s", "o2", "--only-class", "Texture2D", "--select", "group:card") @@ -439,13 +458,13 @@ def test_usage_errors(tmp_path, capsys, fake, monkeypatch): assert export(capsys, tmp_path, d, "s", "o", "--select", "card")[0] == 2 assert export(capsys, tmp_path, d, "s", "o", "--layout", "flat")[0] == 2 assert export(capsys, tmp_path, d, "s", "o", "--views", "cards")[0] == 2 - assert export(capsys, tmp_path, d, "s", "o", "--catalog-version", "nothing")[0] == 2 - monkeypatch.setattr(cli_assets, "STAGE_SOURCES", tuple(s for s in FAKE_SOURCES if s.name != "unity.export")) + assert export(capsys, tmp_path, d, "s", "o", "--stored-catalog", "nothing")[0] == 2 + monkeypatch.setattr(assets, "STAGE_SOURCES", tuple(s for s in FAKE_SOURCES if s.name != "unity.export")) code, _, err = export(capsys, tmp_path, d, "s", "o", "--png-level", "3") assert code == 2 and "--png-level" in err code, _, err = export(capsys, tmp_path, d, "s", "o", "--flac-level", "5") assert code == 2 and "no stage of the pipeline encodes FLAC" in err - code, _, err = nn(capsys, "export", "-o", tmp_path / "o") + code, _, err = nn(capsys, "export", "assets", "-o", tmp_path / "o") assert code == 2 and "paths.store" in err # no store and no cache @@ -454,10 +473,10 @@ def test_the_flac_level_is_a_parameter_of_the_stages_that_encode_flac(tmp_path, d = setup_data(tmp_path) assert export(capsys, tmp_path, d, "s", "o", "--flac-level", "13")[0] == 2 export(capsys, tmp_path, d, "s", "o") - base = [*flags(d), "plan", "--store", tmp_path / "s", "--flac-level", "5"] + base = [*flags(d), "store", "plan", "--store", tmp_path / "s", "--flac-level", "5"] code, out, _ = nn(capsys, *base, "--json") tid = next(n["id"] for n in json.loads(out)["nodes"] if n["stage"] == "unity.export") - code, out, _ = nn(capsys, *base, "--why", tid) + code, out, _ = nn(capsys, *base, "--why", tid, "--json") assert json.loads(out)["keyParts"]["new"]["params"]["flac"] == {"level": 5} @@ -485,7 +504,7 @@ def unsupported(*a, **k): code, out, err = export(capsys, tmp_path, d, "s", f"o-{how}", *(("--link", how) if how != "auto" else ())) summary = json.loads(out) assert code == 1, err - runs[how] = contract.loads((tmp_path / f"o-{how}" / cli_assets.REPORTS / "run.json").read_bytes()) + runs[how] = contract.loads((tmp_path / f"o-{how}" / assets.REPORTS / "run.json").read_bytes()) trees[how] = {k: v for k, v in tree(tmp_path / f"o-{how}").items() if not k.endswith("/run.json")} n = summary["layouts"]["original"] assert summary["placement"] == runs[how]["placement"] and n["failed"] == 0 @@ -515,7 +534,7 @@ def test_an_aborted_run_keeps_its_results(tmp_path, capsys, fake): d = setup_data(tmp_path, docs) code, out, err = export(capsys, tmp_path, d, "s", "o") assert code == 3 and "aborted" in err - run = contract.loads((tmp_path / "o" / cli_assets.REPORTS / "run.json").read_bytes()) + run = contract.loads((tmp_path / "o" / assets.REPORTS / "run.json").read_bytes()) st = {t["id"]: t["status"] for t in run["tasks"]} assert st[f"unity.census:{contract.stable_bundle_name(NAMES['bg_x'])}"] == "ran" @@ -531,7 +550,7 @@ def test_export_dry_run_and_explain(tmp_path, capsys, fake): # ---------------------------------------------------------------- run-stage def test_run_stage_refuses_incompatible_tasks(tmp_path, capsys, fake): d = setup_data(tmp_path) - nn(capsys, *flags(d), "plan", "--store", tmp_path / "s", "--emit-tasks", tmp_path / "t") + nn(capsys, *flags(d), "store", "plan", "--store", tmp_path / "s", "--emit-tasks", tmp_path / "t") (f,) = [ p for p in sorted((tmp_path / "t").glob("*.json")) @@ -540,7 +559,7 @@ def test_run_stage_refuses_incompatible_tasks(tmp_path, capsys, fake): doc = contract.loads(f.read_bytes()) edited = tmp_path / "edited.json" edited.write_bytes(contract.encode(dict(doc, params={"x": 1}))) - code, _, err = nn(capsys, "run-stage", f, edited, "--store", tmp_path / "s") + code, _, err = nn(capsys, "store", "run", f, edited, "--store", tmp_path / "s") assert code == 4 and "incompatible task" in err assert not (tmp_path / "s" / "ac").exists() # nothing of the batch ran other = tmp_path / "other.json" @@ -548,8 +567,8 @@ def test_run_stage_refuses_incompatible_tasks(tmp_path, capsys, fake): other.write_bytes( contract.encode(contract.Task("catalog.index", 9, t.subject, t.params, t.atoms, t.inputs, t.context).to_json()) ) - assert nn(capsys, "run-stage", other, "--store", tmp_path / "s")[0] == 4 - code, out, _ = nn(capsys, "run-stage", f, "--store", tmp_path / "s") + assert nn(capsys, "store", "run", other, "--store", tmp_path / "s")[0] == 4 + code, out, _ = nn(capsys, "store", "run", f, "--store", tmp_path / "s", "--json") assert code == 0 and json.loads(out)["summary"] == {"tasks": 1, "hit": 0, "ran": 1, "failed": 0} @@ -557,21 +576,21 @@ def test_plan_and_run_stage_do_not_import_unitypy(tmp_path): d = setup_data(tmp_path) script = f""" import sys -from nnnotes.cli import cli_assets +from nnnotes import cli args = {[str(a) for a in flags(d)]!r} from pathlib import Path from nnnotes.pipeline import contract def main(argv): try: - cli_assets.main(argv) + cli.main(argv) except SystemExit as e: assert not e.code, e.code -main(args + ["plan", "--store", {str(tmp_path / "s")!r}, "--emit-tasks", {str(tmp_path / "t")!r}]) +main(args + ["store", "plan", "--store", {str(tmp_path / "s")!r}, "--emit-tasks", {str(tmp_path / "t")!r}]) index = [str(p) for p in Path({str(tmp_path / "t")!r}).glob("*.json") if contract.loads(p.read_bytes())["stage"]["name"] == "catalog.index"] assert len(index) == 1, index -main(["run-stage", *index, "--store", {str(tmp_path / "s")!r}]) -main(args + ["plan", "--store", {str(tmp_path / "s")!r}, "--json"]) +main(["store", "run", *index, "--store", {str(tmp_path / "s")!r}]) +main(args + ["store", "plan", "--store", {str(tmp_path / "s")!r}, "--json"]) assert "UnityPy" not in sys.modules, "UnityPy imported" """ r = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, cwd=tmp_path) @@ -590,19 +609,19 @@ def test_catalog_versions_import_list_and_diff(tmp_path, capsys): b = tmp_path / "b.bin" b.write_bytes(synth.CatalogWriter().build(entries)) s = ["--store", tmp_path / "s"] - assert nn(capsys, "catalogs", "import", a, "--label", "v1", *s)[0] == 0 - code, out, _ = nn(capsys, "catalogs", "import", b, *s) + assert nn(capsys, "catalog", "import", a, "--label", "v1", *s)[0] == 0 + code, out, _ = nn(capsys, "catalog", "import", b, *s) assert code == 0 and contract.sha256(b.read_bytes())[:12] in out - assert nn(capsys, "catalogs", "import", b, "--label", "v1", *s)[0] == 2 # a label names one version - code, out, _ = nn(capsys, "catalogs", "list", "--json", *s) + assert nn(capsys, "catalog", "import", b, "--label", "v1", *s)[0] == 2 # a label names one version + code, out, _ = nn(capsys, "catalog", "versions", "--json", *s) vs = json.loads(out)["versions"] assert [v["labels"] for v in vs] == [["v1"], [contract.sha256(b.read_bytes())[:12]]] - code, out, _ = nn(capsys, "catalogs", "diff", "v1", vs[1]["id"][:12], "--json", *s) + code, out, _ = nn(capsys, "catalog", "diff", "v1", vs[1]["id"][:12], "--json", *s) diff = json.loads(out) assert diff["keys"]["added"] == ["Card/3/full"] and diff["keys"]["removed"] == ["Bg/x", "Card/2/full"] - code, out, _ = nn(capsys, "catalogs", "diff", "v1", "v1", *s) + code, out, _ = nn(capsys, "catalog", "diff", "v1", "v1", *s) assert code == 0 and out.splitlines()[-1].startswith("# keys +0 -0 ~0") - assert nn(capsys, "catalogs", "diff", "v1", "v9", *s)[0] == 2 + assert nn(capsys, "catalog", "diff", "v1", "v9", *s)[0] == 2 def test_catalog_import_takes_the_resource_version_from_the_file_name(tmp_path, capsys): @@ -611,9 +630,9 @@ def test_catalog_import_takes_the_resource_version_from_the_file_name(tmp_path, named.write_bytes(catalog_bytes()) main = tmp_path / "catalog_main_en.bin" main.write_bytes(synth.CatalogWriter().build([(NAMES["shared"], synth.remote(NAMES["shared"]), [])])) - assert nn(capsys, "catalogs", "import", named, *s)[0] == 0 - assert nn(capsys, "catalogs", "import", main, *s)[0] == 0 - vs = json.loads(nn(capsys, "catalogs", "list", "--json", *s)[1])["versions"] + assert nn(capsys, "catalog", "import", named, *s)[0] == 0 + assert nn(capsys, "catalog", "import", main, *s)[0] == 0 + vs = json.loads(nn(capsys, "catalog", "versions", "--json", *s)[1])["versions"] assert [(v["labels"], v["resourceVersion"]) for v in vs] == [ (["1.0.0.201"], "1.0.0.201"), ([contract.sha256(main.read_bytes())[:12]], None), @@ -622,30 +641,30 @@ def test_catalog_import_takes_the_resource_version_from_the_file_name(tmp_path, def test_catalog_import_resource_version_option(): file = Path("catalog_1.0.0.201_en.bin") - assert cli_assets._import_resource_version("1.0.0.300", file, None) == "1.0.0.300" - assert cli_assets._import_resource_version(None, Path("renamed.bin"), None) is None - assert cli_assets._import_resource_version(None, file, {"version": "1.0.0.300"}) == "1.0.0.300" + assert catalog_cli.import_resource_version("1.0.0.300", file, None) == "1.0.0.300" + assert catalog_cli.import_resource_version(None, Path("renamed.bin"), None) is None + assert catalog_cli.import_resource_version(None, file, {"version": "1.0.0.300"}) == "1.0.0.300" with pytest.raises(ValueError, match="differs from the catalog's source version"): - cli_assets._import_resource_version("1.0.0.201", file, {"version": "1.0.0.300"}) + catalog_cli.import_resource_version("1.0.0.201", file, {"version": "1.0.0.300"}) with pytest.raises(ValueError, match="--resource-version"): - cli_assets._import_resource_version("../x", file, None) + catalog_cli.import_resource_version("../x", file, None) def test_store_verify(tmp_path, capsys, fake): d = setup_data(tmp_path) export(capsys, tmp_path, d, "s", "o") - code, out, _ = nn(capsys, "store", "verify", "--store", tmp_path / "s") + code, out, _ = nn(capsys, "store", "verify", "--store", tmp_path / "s", "--json") r = json.loads(out) assert code == 0 and r["problemCount"] == 0 and r["contents"] > 0 and r["results"] > 0 victim = next(p for p in sorted((tmp_path / "s" / "cas").rglob("*")) if p.is_file()) assert os.stat(victim).st_mode & 0o222 == 0 # store objects are read-only os.chmod(victim, 0o666) victim.write_bytes(victim.read_bytes() + b"x") - code, out, _ = nn(capsys, "store", "verify", "--store", tmp_path / "s") + code, out, _ = nn(capsys, "store", "verify", "--store", tmp_path / "s", "--json") problems = {p["problem"] for p in json.loads(out)["problems"]} assert code == 1 and "content differs from its id" in problems assert any(p.endswith("bytes of another size") for p in problems) - code, out, _ = nn(capsys, "store", "verify", "--quick", "--store", tmp_path / "s") + code, out, _ = nn(capsys, "store", "verify", "--quick", "--store", tmp_path / "s", "--json") assert code == 1 and all("another size" in p["problem"] for p in json.loads(out)["problems"]) @@ -655,16 +674,28 @@ def test_the_commands_are_part_of_nnnotes(tmp_path, capsys, fake): cli.build_parser().parse_args(["store", "verify"]) except SystemExit: pytest.skip("the command line does not register the asset commands") - assert ("paths", "store") in cli.FLAG_SETTINGS + assert ("paths", "store") in common.FLAG_SETTINGS d = setup_data(tmp_path) with pytest.raises(SystemExit) as e: cli.main( - [*map(str, flags(d)), "export", "-o", str(tmp_path / "o"), "--store", str(tmp_path / "s"), "--workers", "0"] + [ + *map(str, flags(d)), + "export", + "assets", + "-o", + str(tmp_path / "o"), + "--store", + str(tmp_path / "s"), + "--workers", + "0", + ] ) assert e.value.code == 1 capsys.readouterr() with pytest.raises(SystemExit) as e: - cli.main([*map(str, flags(d)), "plan", "-o", str(tmp_path / "o"), "--store", str(tmp_path / "s"), "--check"]) + cli.main( + [*map(str, flags(d)), "store", "plan", "-o", str(tmp_path / "o"), "--store", str(tmp_path / "s"), "--check"] + ) assert e.value.code == 0 @@ -711,9 +742,9 @@ def derived(self, task_id, result, store, paths): def test_derived_documents_read_the_paths_of_the_objects_they_name(tmp_path, capsys, monkeypatch): monkeypatch.setattr( - cli_assets, + assets, "STAGE_SOURCES", - FAKE_SOURCES + (cli_assets.StageSource("fake.paths", "tests.cli.test_cli_assets:ObjectPaths", output=True),), + FAKE_SOURCES + (assets.StageSource("fake.paths", "tests.cli.test_assets:ObjectPaths", output=True),), ) d = setup_data(tmp_path) code, _, err = export(capsys, tmp_path, d, "s", "o") @@ -724,7 +755,7 @@ def test_derived_documents_read_the_paths_of_the_objects_they_name(tmp_path, cap assert doc[card] == {f"{card}#json": "Assets/Game/Card/1/full.png.json"} assert doc[f"{CABS['card_1']}:1"] == doc[card] # contained in it assert (tmp_path / "o" / "Assets" / "Game" / "Card" / "1" / "full.png.json").is_file() - code, out, _ = nn(capsys, *flags(d), "plan", "-o", tmp_path / "o", "--store", tmp_path / "s", "--check") + code, out, _ = nn(capsys, *flags(d), "store", "plan", "-o", tmp_path / "o", "--store", tmp_path / "s", "--check") assert code == 0, out @@ -733,7 +764,7 @@ def test_views_are_placed_with_paths_and_every_document_validates(tmp_path, caps from nnnotes.pipeline import views from tests.pipeline.test_contract import validators - monkeypatch.setattr(cli_assets, "STAGE_SOURCES", FAKE_SOURCES + (REAL["view."],)) + monkeypatch.setattr(assets, "STAGE_SOURCES", FAKE_SOURCES + (REAL["view."],)) rules = views.load_rules() master = tmp_path / "master" master.mkdir() @@ -748,6 +779,7 @@ def test_views_are_placed_with_paths_and_every_document_validates(tmp_path, caps "--master", master, "export", + "assets", "-o", tmp_path / "o", "--store", @@ -760,7 +792,7 @@ def test_views_are_placed_with_paths_and_every_document_validates(tmp_path, caps "original,cas", ) assert code == 1, err - run = contract.loads((tmp_path / "o" / cli_assets.REPORTS / "run.json").read_bytes()) + run = contract.loads((tmp_path / "o" / assets.REPORTS / "run.json").read_bytes()) sha = contract.sha256((master / "MasterText.json").read_bytes()) assert run["context"]["master"]["version"] == "1.2.3" assert ( @@ -772,7 +804,7 @@ def test_views_are_placed_with_paths_and_every_document_validates(tmp_path, caps doc = contract.loads((tmp_path / "o" / "original" / "views" / "cards.json").read_bytes()) assert doc["layout"] == "original" and doc["view"] == "cards" store = Store(tmp_path / "s") - run = contract.loads((tmp_path / "o" / cli_assets.REPORTS / "run.json").read_bytes()) + run = contract.loads((tmp_path / "o" / assets.REPORTS / "run.json").read_bytes()) v = validators() checked = set() for t in run["tasks"]: @@ -799,6 +831,7 @@ def test_views_are_placed_with_paths_and_every_document_validates(tmp_path, caps *flags(d), "--master", master, + "store", "plan", "--store", tmp_path / "s", @@ -816,14 +849,14 @@ def test_views_are_placed_with_paths_and_every_document_validates(tmp_path, caps docs += [(contract.TASK, contract.loads(p.read_bytes())) for p in (tmp_path / "t").glob("*.json")] for name in ("coverage", "failures"): docs.append( - (f"nnnotes.{name}/1", contract.loads((tmp_path / "o" / cli_assets.REPORTS / f"{name}.json").read_bytes())) + (f"nnnotes.{name}/1", contract.loads((tmp_path / "o" / assets.REPORTS / f"{name}.json").read_bytes())) ) docs.append((contract.LAYOUT, contract.loads((tmp_path / "o" / "cas" / "manifest.json").read_bytes()))) for name, body in docs: errors = [e.message for e in v[name].iter_errors(body)] assert errors == [], (name, errors[:3]) vid = run["context"]["catalog"]["id"] - code, out, _ = nn(capsys, "catalogs", "diff", vid[:12], vid[:12], "--json", "--store", tmp_path / "s") + code, out, _ = nn(capsys, "catalog", "diff", vid[:12], vid[:12], "--json", "--store", tmp_path / "s") diff = json.loads(out) assert diff["objects"]["summary"]["unchanged"] > 0 # the address tables of the runs assert [e.message for e in v["nnnotes.catalog-diff/1"].iter_errors(diff)] == [] @@ -845,9 +878,9 @@ def test_calibration_scales_estimates_from_measured_costs(): ) log.append({"event": "start", "task": "link.scripts:all", "estimate": {"cpuSeconds": 1.0, "peakBytes": 1}}) log.append({"event": "end", "task": "link.scripts:all", "ok": True, "cost": {"cpuSeconds": 9.0}}) - f = cli_assets.calibrate(log, {"unity.export": {"cpu": 1.5, "peak": 1.0, "samples": 3}}, 70 << 20) + f = assets.calibrate(log, {"unity.export": {"cpu": 1.5, "peak": 1.0, "samples": 3}}, 70 << 20) assert f == {"unity.export": {"cpu": 3.0, "peak": 0.5, "samples": 12}} # too few samples: no factor - stage = cli_assets.Calibrated(FakeExport(), 3.0, 0.5) + stage = assets.Calibrated(FakeExport(), 3.0, 0.5) assert stage.estimate("x", None, []) == Cost(0.03, 1 << 19) assert stage.name == "unity.export" and stage.PARAMS is FakeExport.PARAMS import pickle @@ -874,30 +907,32 @@ def test_calibration_counts_the_memory_a_task_adds_to_its_worker(): }, } ) # a worker holding 400 MiB from earlier tasks - assert cli_assets.calibrate(log, {}, 70 << 20)["unity.export"]["peak"] == 0.25 + assert assets.calibrate(log, {}, 70 << 20)["unity.export"]["peak"] == 0.25 def test_the_worker_recycling_threshold_follows_the_budget(): - gib = cli_assets.GIB - assert cli_assets.recycle_rss(None, 8) == cli_assets.RECYCLE_RSS == 3 * gib - assert cli_assets.recycle_rss(100 * gib, 12) == 3 * gib - assert cli_assets.recycle_rss(gib, 1) == int(0.4 * gib) - assert cli_assets.recycle_rss(16 * gib, 12) == int(0.4 * 16 * gib / 12) - assert cli_assets.recycle_rss(gib, 0) == int(0.4 * gib) # tasks in this process - assert cli_assets.recycle_rss(gib, 64) == 256 << 20 + gib = assets.GIB + assert assets.recycle_rss(None, 8) == assets.RECYCLE_RSS == 3 * gib + assert assets.recycle_rss(100 * gib, 12) == 3 * gib + assert assets.recycle_rss(gib, 1) == int(0.4 * gib) + assert assets.recycle_rss(16 * gib, 12) == int(0.4 * 16 * gib / 12) + assert assets.recycle_rss(gib, 0) == int(0.4 * gib) # tasks in this process + assert assets.recycle_rss(gib, 64) == 256 << 20 def test_export_records_the_calibration_and_plan_uses_it(tmp_path, capsys, fake): d = setup_data(tmp_path) export(capsys, tmp_path, d, "s", "o") - cal = tmp_path / "s" / "costs" / cli_assets.CALIBRATION + cal = tmp_path / "s" / "costs" / assets.CALIBRATION assert contract.loads(cal.read_bytes()) == {"stages": {}} # four bundles: too few - cli_assets.write_calibration(tmp_path / "s", {"unity.export": {"cpu": 2.0, "peak": 3.0, "samples": 50}}) - code, out, err = nn(capsys, *flags(d), "plan", "--store", tmp_path / "s", "--json", "--png-level", "1", "--explain") + assets.write_calibration(tmp_path / "s", {"unity.export": {"cpu": 2.0, "peak": 3.0, "samples": 50}}) + code, out, err = nn( + capsys, *flags(d), "store", "plan", "--store", tmp_path / "s", "--json", "--png-level", "1", "--explain" + ) assert "cost calibration unity.export: CPU x2.0, peak memory x3.0 (50 measured tasks)" in err runs = [n for n in json.loads(out)["nodes"] if n["stage"] == "unity.export"] assert runs and all(n["cost"] == {"cpuSeconds": 0.02, "peakBytes": 3 << 20} for n in runs) - code, out, _ = nn(capsys, *flags(d), "plan", "--store", tmp_path / "s", "--check") + code, out, _ = nn(capsys, *flags(d), "store", "plan", "--store", tmp_path / "s", "--check") assert code == 0 # keys do not change @@ -907,7 +942,7 @@ def test_files_that_cannot_be_read_are_reported_not_failed(tmp_path, capsys, fak summary = json.loads(out) assert code == 0, err assert summary["sources"] == {"source.absent": 1} and summary["failures"] == 0 - (p,) = contract.loads((tmp_path / "o" / cli_assets.REPORTS / "sources.json").read_bytes())["problems"] + (p,) = contract.loads((tmp_path / "o" / assets.REPORTS / "sources.json").read_bytes())["problems"] assert p["stable"] == "apkonly" and "[paths] apk is not set" in p["message"] @@ -955,9 +990,9 @@ def test_named_contents_are_placed_under_each_name(tmp_path, capsys, monkeypatch from nnnotes.pipeline import layout monkeypatch.setattr( - cli_assets, + assets, "STAGE_SOURCES", - FAKE_SOURCES + (cli_assets.StageSource("toy.named", "tests.cli.test_cli_assets:Named", output=True),), + FAKE_SOURCES + (assets.StageSource("toy.named", "tests.cli.test_assets:Named", output=True),), ) d = setup_data(tmp_path, censuses(broken=False)) code, out, err = export(capsys, tmp_path, d, "s", "o", "--layout", "original,cas") @@ -971,7 +1006,7 @@ def test_named_contents_are_placed_under_each_name(tmp_path, capsys, monkeypatch "_tasks/toy.named/s2/a.flac", } <= set(orig) assert orig["Cri/Sound/x/a.flac"] == orig["Cri/Sound/y/a.flac"] - report = contract.loads((tmp_path / "o" / cli_assets.REPORTS / "layout-original.json").read_bytes()) + report = contract.loads((tmp_path / "o" / assets.REPORTS / "layout-original.json").read_bytes()) assert report["names"] == [{"task": "toy.named:s1", "names": ["Cri/Sound/x", "Cri/Sound/y"]}] mo = contract.loads((tmp_path / "o" / "original" / "manifest.json").read_bytes()) mc = contract.loads((tmp_path / "o" / "cas" / "manifest.json").read_bytes()) @@ -981,14 +1016,14 @@ def test_named_contents_are_placed_under_each_name(tmp_path, capsys, monkeypatch def test_plan_and_export_describe_with_the_same_settings(tmp_path, capsys, monkeypatch): monkeypatch.setattr( - cli_assets, + assets, "STAGE_SOURCES", - FAKE_SOURCES + (cli_assets.StageSource("toy.tool", "tests.cli.test_cli_assets:Tooled"),), + FAKE_SOURCES + (assets.StageSource("toy.tool", "tests.cli.test_assets:Tooled"),), ) d = setup_data(tmp_path, censuses(broken=False)) tool = tmp_path / "ffmpeg-bin" code, _, err = nn( - capsys, *flags(d), "--ffmpeg", tool, "plan", "--store", tmp_path / "s", "--emit-tasks", tmp_path / "t" + capsys, *flags(d), "--ffmpeg", tool, "store", "plan", "--store", tmp_path / "s", "--emit-tasks", tmp_path / "t" ) assert code == 0, err (planned,) = [ @@ -998,9 +1033,20 @@ def test_plan_and_export_describe_with_the_same_settings(tmp_path, capsys, monke ] assert planned["atoms"] == {"tool": str(tool)} code, _, err = nn( - capsys, *flags(d), "--ffmpeg", tool, "export", "-o", tmp_path / "o", "--store", tmp_path / "s", "--workers", "0" + capsys, + *flags(d), + "--ffmpeg", + tool, + "export", + "assets", + "-o", + tmp_path / "o", + "--store", + tmp_path / "s", + "--workers", + "0", ) - run = contract.loads((tmp_path / "o" / cli_assets.REPORTS / "run.json").read_bytes()) + run = contract.loads((tmp_path / "o" / assets.REPORTS / "run.json").read_bytes()) assert {t["id"]: t["key"] for t in run["tasks"]}["toy.tool:s"] == planned["key"] @@ -1014,22 +1060,24 @@ def test_plan_knows_the_files_the_apk_does_not_have(tmp_path, capsys, fake, monk synth.CatalogWriter().build([("apkonly.bundle", synth.local("apkonly.bundle"), [])]), ) d = setup_data(tmp_path, censuses(broken=False), extra=[("apkonly.bundle", synth.local("apkonly.bundle"), [])]) - monkeypatch.setattr(cli_assets, "_apk_version", lambda apk: None) - code, out, err = nn(capsys, *flags(d), "--apk", apk, "plan", "--store", tmp_path / "s", "--json") + monkeypatch.setattr("nnnotes.pipeline.catalogdb.apk_version", lambda apk: None) + code, out, err = nn(capsys, *flags(d), "--apk", apk, "store", "plan", "--store", tmp_path / "s", "--json") nodes = {n["id"]: n for n in json.loads(out)["nodes"]} assert "unity.census:apkonly" not in nodes # not pending on a fetch that cannot happen code, out, err = nn( - capsys, *flags(d), "--apk", apk, "plan", "--store", tmp_path / "s", "--emit-tasks", tmp_path / "t" + capsys, *flags(d), "--apk", apk, "store", "plan", "--store", tmp_path / "s", "--emit-tasks", tmp_path / "t" ) - assert nn(capsys, *flags(d), "run-stage", tmp_path / "t", "--store", tmp_path / "s")[0] == 0 - code, out, _ = nn(capsys, *flags(d), "--apk", apk, "plan", "--store", tmp_path / "s", "--json") + assert nn(capsys, *flags(d), "store", "run", tmp_path / "t", "--store", tmp_path / "s")[0] == 0 + code, out, _ = nn(capsys, *flags(d), "--apk", apk, "store", "plan", "--store", tmp_path / "s", "--json") nodes = {n["id"]: n for n in json.loads(out)["nodes"]} assert nodes["link.scripts:all"]["status"] == "run" def test_plan_census_takes_the_censuses_first(tmp_path, capsys, fake): d = setup_data(tmp_path, censuses(broken=False)) - code, out, err = nn(capsys, *flags(d), "plan", "--store", tmp_path / "s", "--census", "--workers", "0", "--json") + code, out, err = nn( + capsys, *flags(d), "store", "plan", "--store", tmp_path / "s", "--census", "--workers", "0", "--json" + ) assert code == 0, err nodes = json.loads(out)["nodes"] census = [n for n in nodes if n["stage"] == "unity.census"] @@ -1048,17 +1096,16 @@ def names(self, env): def test_names_see_the_tasks_that_ran(tmp_path, capsys, monkeypatch): monkeypatch.setattr( - cli_assets, + assets, "STAGE_SOURCES", - FAKE_SOURCES - + (cli_assets.StageSource("toy.byexport", "tests.cli.test_cli_assets:NamedByExport", output=True),), + FAKE_SOURCES + (assets.StageSource("toy.byexport", "tests.cli.test_assets:NamedByExport", output=True),), ) d = setup_data(tmp_path, censuses(broken=False)) code, out, err = export(capsys, tmp_path, d, "s", "o") assert code == 0, err placed = tree(tmp_path / "o") assert "Named/4/a.flac" in placed and "Named/0/a.flac" not in placed # four bundles were exported - code, out, _ = nn(capsys, *flags(d), "plan", "-o", tmp_path / "o", "--store", tmp_path / "s", "--check") + code, out, _ = nn(capsys, *flags(d), "store", "plan", "-o", tmp_path / "o", "--store", tmp_path / "s", "--check") assert code == 0, out # plan places them the same way @@ -1075,7 +1122,7 @@ def test_calibration_keeps_the_cpu_factor_without_child_measurements(): "cost": {"cpuSeconds": 0.1, "childCpuSeconds": None, "peakRssBytes": (70 << 20) + (100 << 20)}, } ) - f = cli_assets.calibrate(log, {"cri.audio": {"cpu": 2.0, "peak": 1.0, "samples": 20}}, 70 << 20) + f = assets.calibrate(log, {"cri.audio": {"cpu": 2.0, "peak": 1.0, "samples": 20}}, 70 << 20) assert f == {"cri.audio": {"cpu": 2.0, "peak": 1.0, "samples": 12}} @@ -1121,11 +1168,11 @@ def raw_apk(tmp_path, monkeypatch): import zipfile monkeypatch.setattr( - cli_assets, + assets, "STAGE_SOURCES", - FAKE_SOURCES + (cli_assets.StageSource("cri.audio", "tests.cli.test_cli_assets:RawCopy"),), + FAKE_SOURCES + (assets.StageSource("cri.audio", "tests.cli.test_assets:RawCopy"),), ) - monkeypatch.setattr(cli_assets, "_apk_version", lambda apk: None) + monkeypatch.setattr("nnnotes.pipeline.catalogdb.apk_version", lambda apk: None) name = "cri_assets_embcri/sound/initialse_" + "ab" * 16 apk = tmp_path / "base.apk" with zipfile.ZipFile(apk, "w") as z: @@ -1137,28 +1184,51 @@ def raw_apk(tmp_path, monkeypatch): def raw_export(capsys, tmp_path, d, apk): return nn( - capsys, *flags(d), "--apk", apk, "export", "-o", tmp_path / "o", "--store", tmp_path / "s", "--workers", "0" + capsys, + *flags(d), + "--apk", + apk, + "export", + "assets", + "-o", + tmp_path / "o", + "--store", + tmp_path / "s", + "--workers", + "0", ) def test_raw_files_of_the_apk_are_read_from_it(tmp_path, capsys, monkeypatch): apk, name = raw_apk(tmp_path, monkeypatch) d = setup_data(tmp_path, censuses(broken=False)) - code, out, err = nn(capsys, *flags(d), "--apk", apk, "plan", "--store", tmp_path / "s", "--json") + code, out, err = nn(capsys, *flags(d), "--apk", apk, "store", "plan", "--store", tmp_path / "s", "--json") nodes = {n["id"]: n for n in json.loads(out)["nodes"]} assert nodes["cri.audio:cri_assets_embcri/sound/initialse"]["reasons"] == [ {"code": "pending", "task": f"fetch:{name}"} ] code, out, err = nn( - capsys, *flags(d), "--apk", apk, "export", "-o", tmp_path / "o", "--store", tmp_path / "s", "--workers", "0" + capsys, + *flags(d), + "--apk", + apk, + "export", + "assets", + "-o", + tmp_path / "o", + "--store", + tmp_path / "s", + "--workers", + "0", + "--json", ) assert code == 0, err assert json.loads(out)["sources"] == {} assert (d["cache"] / "raw" / "Android" / name).read_bytes() == b"@UTF sheet" # where a CDN file would be - run = contract.loads((tmp_path / "o" / cli_assets.REPORTS / "run.json").read_bytes()) + run = contract.loads((tmp_path / "o" / assets.REPORTS / "run.json").read_bytes()) assert {t["id"]: t["status"] for t in run["tasks"]}["cri.audio:cri_assets_embcri/sound/initialse"] == "ran" code, out, _ = nn( - capsys, *flags(d), "--apk", apk, "plan", "-o", tmp_path / "o", "--store", tmp_path / "s", "--check" + capsys, *flags(d), "--apk", apk, "store", "plan", "-o", tmp_path / "o", "--store", tmp_path / "s", "--check" ) assert code == 0, out # read from the cache now @@ -1168,7 +1238,7 @@ def test_the_raw_files_are_fetched_while_the_bundle_stages_run(tmp_path, capsys, apk, name = raw_apk(tmp_path, monkeypatch) order, exported = [], threading.Event() - real_inputs, real_run = cli_assets.Workspace._inputs, FakeExport.run + real_inputs, real_run = assets.Workspace._inputs, FakeExport.run def inputs(self, kind, entries, fetch): if kind == "raw": @@ -1182,32 +1252,32 @@ def run(self, task, store): exported.set() return real_run(self, task, store) - monkeypatch.setattr(cli_assets.Workspace, "_inputs", inputs) + monkeypatch.setattr(assets.Workspace, "_inputs", inputs) monkeypatch.setattr(FakeExport, "run", run) d = setup_data(tmp_path, censuses(broken=False)) code, out, err = raw_export(capsys, tmp_path, d, apk) assert code == 0, err assert order == ["unity.export", ("raw fetch", False)] # in the background, while the stages run assert "fetched while the bundle stages run" in err and "0 raw files not readable" in err - run_doc = contract.loads((tmp_path / "o" / cli_assets.REPORTS / "run.json").read_bytes()) + run_doc = contract.loads((tmp_path / "o" / assets.REPORTS / "run.json").read_bytes()) assert {t["id"]: t["status"] for t in run_doc["tasks"]}["cri.audio:cri_assets_embcri/sound/initialse"] == "ran" assert (d["cache"] / "raw" / "Android" / name).read_bytes() == b"@UTF sheet" def test_a_setting_the_raw_fetch_lacks_aborts_the_run(tmp_path, capsys, monkeypatch): apk, _ = raw_apk(tmp_path, monkeypatch) - real = cli_assets.Workspace._inputs + real = assets.Workspace._inputs def inputs(self, kind, entries, fetch): if kind == "raw": raise ConfigError("setting servers.zz.cdn is not set") return real(self, kind, entries, fetch) - monkeypatch.setattr(cli_assets.Workspace, "_inputs", inputs) + monkeypatch.setattr(assets.Workspace, "_inputs", inputs) d = setup_data(tmp_path, censuses(broken=False)) code, out, err = raw_export(capsys, tmp_path, d, apk) assert code == 3 and "aborted: setting servers.zz.cdn is not set" in err, err - run_doc = contract.loads((tmp_path / "o" / cli_assets.REPORTS / "run.json").read_bytes()) + run_doc = contract.loads((tmp_path / "o" / assets.REPORTS / "run.json").read_bytes()) st = {t["id"]: t["status"] for t in run_doc["tasks"]} assert st[f"unity.census:{contract.stable_bundle_name(NAMES['bg_x'])}"] == "ran" # kept @@ -1219,7 +1289,7 @@ def test_an_interrupt_starts_no_more_raw_fetches(tmp_path, capsys, monkeypatch): apk, name = raw_apk(tmp_path, monkeypatch) ended = threading.Event() - real = cli_assets.Workspace._inputs + real = assets.Workspace._inputs def inputs(self, kind, entries, fetch): if kind != "raw": @@ -1233,7 +1303,7 @@ def inputs(self, kind, entries, fetch): def interrupted(*a, **kw): raise KeyboardInterrupt - monkeypatch.setattr(cli_assets.Workspace, "_inputs", inputs) + monkeypatch.setattr(assets.Workspace, "_inputs", inputs) monkeypatch.setattr(orchestrate.Orchestrator, "run", interrupted) d = setup_data(tmp_path, censuses(broken=False)) code, _, err = raw_export(capsys, tmp_path, d, apk) @@ -1260,9 +1330,9 @@ def test_worker_counts_follow_the_cpus_this_process_may_use(monkeypatch): monkeypatch.setattr(os, "cpu_count", lambda: None) assert config.usable_cpus() == 1 monkeypatch.setattr(config, "usable_cpus", lambda: 12) - monkeypatch.setattr(cli_assets, "usable_cpus", lambda: 12) + monkeypatch.setattr(assets, "usable_cpus", lambda: 12) monkeypatch.setattr(cri, "usable_cpus", lambda: 12) - ws = cli_assets.Workspace.__new__(cli_assets.Workspace) + ws = assets.Workspace.__new__(assets.Workspace) ws.args = SimpleNamespace(workers=None) assert ws.workers() == 12 and cri.default_workers() == 3 ws.args.workers = 0 diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 72b28a8..36cb2cf 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -9,6 +9,7 @@ import pytest from nnnotes import cli +from nnnotes.cli import common from tests.helpers import synth KEY_HEX = synth.BUNDLE_KEY.hex() @@ -56,21 +57,17 @@ def test_version_and_help(capsys): code, out, _ = run(["--help"], capsys) assert code == 0 for cmd in ( + "config", + "servers", "catalog", - "browse", - "pull", + "bundle", "master", - "adv", - "story", - "live2d", - "spot", - "room", - "shader", - "audio", - "crikey", - "player", - "live", - "web", + "export", + "apk", + "store", + "voices", + "site", + "music-data", ): assert cmd in out @@ -101,15 +98,15 @@ def test_numpy_blas_threads_set_before_numpy_loads(given, seen): @pytest.mark.parametrize( "argv", [ - ["adv", "1"], - ["story", "1"], - ["live2d", "k"], - ["spot", "1"], - ["room", "k"], - ["shader", "--key", "k"], - ["audio", "s"], - ["player"], - ["live", "1"], + ["export", "episode", "1"], + ["export", "story", "1"], + ["export", "live2d", "k"], + ["export", "spot", "1"], + ["export", "room", "k"], + ["export", "shader", "--key", "k"], + ["export", "audio", "s"], + ["apk", "settings"], + ["export", "live", "1"], ["master", "decode", "x"], ["master", "download", "--version", "1"], ], @@ -120,15 +117,15 @@ def test_output_is_required(argv, capsys): def test_web_needs_a_selection(capsys): - code, _, err = run(["web", "out"], capsys) + code, _, err = run(["site", "build", "-o", "out"], capsys) assert code == 2 and "--pair" in err and "--all" in err def test_parse_pair(): - assert cli.parse_pair("100001:expert") == (100001, "expert") - assert cli.parse_pair("7_easy") == (7, "easy") + assert common.parse_pair("100001:expert") == (100001, "expert") + assert common.parse_pair("7_easy") == (7, "easy") with pytest.raises(argparse.ArgumentTypeError): - cli.parse_pair("7:extreme") + common.parse_pair("7:extreme") def test_catalog_listing(catalog_file, tmp_path, capsys): @@ -139,6 +136,7 @@ def test_catalog_listing(catalog_file, tmp_path, capsys): "--cache", str(tmp_path / "cache"), "catalog", + "keys", "--prefix", "Char/", "--limit", @@ -151,7 +149,7 @@ def test_catalog_listing(catalog_file, tmp_path, capsys): def test_missing_settings_exit_2_and_name_the_setting(catalog_file, tmp_path, cdn, capsys, monkeypatch): base = ["--catalog", str(catalog_file), "--cache", str(tmp_path / "cache")] - code, out, err = run(base + ["pull", "Char/A"], capsys) + code, out, err = run(base + ["bundle", "pull", "Char/A"], capsys) assert code == 2 and out == "" assert err.strip() == ( "nnnotes: a_01.bundle is not in the cache: setting catalog.region is not set: give it " @@ -159,13 +157,13 @@ def test_missing_settings_exit_2_and_name_the_setting(catalog_file, tmp_path, cd "NNNOTES_CATALOG_REGION or --region (no config file was found: `nnnotes config init` " "writes one to fill in)" ) - code, _, err = run(base + ["--region", "zz", "pull", "Char/A"], capsys) + code, _, err = run(base + ["--region", "zz", "bundle", "pull", "Char/A"], capsys) assert code == 2 and "servers.zz.cdn" in err and "NNNOTES_SERVERS_ZZ_CDN" in err monkeypatch.setenv("NNNOTES_SERVERS_ZZ_CDN", cdn) - code, _, err = run(base + ["--region", "zz", "pull", "Char/A"], capsys) + code, _, err = run(base + ["--region", "zz", "bundle", "pull", "Char/A"], capsys) assert code == 2 and "bundle.key" in err and "NNNOTES_BUNDLE_KEY" in err assert cdn not in err and len(err.strip().splitlines()) == 1 - code, _, err = run(["pull", "Char/A"], capsys) + code, _, err = run(["bundle", "pull", "Char/A"], capsys) assert code == 2 and "paths.cache" in err and "NNNOTES_PATHS_CACHE" in err and "--cache" in err @@ -175,7 +173,18 @@ def test_malformed_setting_names_setting_not_value(catalog_file, tmp_path, cdn, monkeypatch.setenv("NNNOTES_BUNDLE_NONCE_SEED", SEED_HEX) monkeypatch.setenv("NNNOTES_SERVERS_ZZ_CDN", cdn) code, _, err = run( - ["--catalog", str(catalog_file), "--cache", str(tmp_path / "cache"), "--region", "zz", "pull", "Char/A"], capsys + [ + "--catalog", + str(catalog_file), + "--cache", + str(tmp_path / "cache"), + "--region", + "zz", + "bundle", + "pull", + "Char/A", + ], + capsys, ) assert code == 2 and err.strip() == ( "nnnotes: a_01.bundle is not in the cache: setting bundle.key: must be 16 bytes (32 hex digits)" @@ -184,7 +193,9 @@ def test_malformed_setting_names_setting_not_value(catalog_file, tmp_path, cdn, def test_missing_path_on_disk(tmp_path, capsys): - code, _, err = run(["--cache", str(tmp_path / "c"), "--catalog", str(tmp_path / "absent.bin"), "catalog"], capsys) + code, _, err = run( + ["--cache", str(tmp_path / "c"), "--catalog", str(tmp_path / "absent.bin"), "catalog", "keys"], capsys + ) assert code == 2 and "setting paths.catalog: file" in err @@ -198,7 +209,7 @@ def test_pull_from_config_file(catalog_file, tmp_path, cdn, capsys): f'[paths]\ncatalog = "../{catalog_file.name}"\ncache = "cache"\n', encoding="utf-8", ) - code, out, err = run(["--config", str(conf), "pull", "Char/A", "Char/B"], capsys) + code, out, err = run(["--config", str(conf), "bundle", "pull", "Char/A", "Char/B"], capsys) assert code == 0, err cached = tmp_path / "conf" / "cache" / "bundles" / "a_01.bundle" assert cached.read_bytes() == synth.fake_bundle("a_01.bundle") @@ -214,11 +225,11 @@ def test_master_decode_command(tmp_path, capsys, monkeypatch): assert code == 2 and "master.key" in err and "NNNOTES_MASTER_KEY" in err monkeypatch.setenv("NNNOTES_MASTER_KEY", MKEY_HEX) monkeypatch.setenv("NNNOTES_MASTER_IV", MIV_HEX) - code, out, _ = run(["master", "decode", str(src), "-o", str(tmp_path / "json")], capsys) + code, out, _ = run(["master", "decode", str(src), "-o", str(tmp_path / "json"), "--json"], capsys) assert code == 0 and json.loads(out)["decoded"] == 1 assert json.loads((tmp_path / "json" / "MasterX.json").read_text(encoding="utf-8")) == {"_allData": [{"_id": 1}]} (src / "MasterY.bin").write_bytes(b"\0" * 96) - code, out, _ = run(["master", "decode", str(src), "-o", str(tmp_path / "json")], capsys) + code, out, _ = run(["master", "decode", str(src), "-o", str(tmp_path / "json"), "--json"], capsys) assert code == 1 and json.loads(out)["failed"][0]["file"] == "MasterY.bin" @@ -230,40 +241,40 @@ def test_master_download_command(tmp_path, capsys, monkeypatch): code, _, err = run(["master", "download", "--version", "9.9.9", "-o", str(tmp_path / "m")], capsys) assert code == 2 and "catalog.region" in err code, out, _ = run( - ["--region", "zz", "master", "download", "--version", "9.9.9", "-o", str(tmp_path / "m")], capsys + ["--region", "zz", "master", "download", "--version", "9.9.9", "-o", str(tmp_path / "m"), "--json"], capsys ) assert code == 0 and json.loads(out)["version"] == "9.9.9" assert (tmp_path / "m" / "MasterManifest.json").is_file() def test_browse_needs_a_region(capsys): - code, _, err = run(["browse"], capsys) + code, _, err = run(["catalog", "browse"], capsys) assert code == 2 and "[servers.]" in err def test_browse_region_needs_languages(capsys, monkeypatch): monkeypatch.setenv("NNNOTES_SERVERS_ZZ_CDN", "file:///nowhere") - code, _, err = run(["browse"], capsys) + code, _, err = run(["catalog", "browse"], capsys) assert code == 2 and "servers.zz.languages" in err and "NNNOTES_SERVERS_ZZ_LANGUAGES" in err def test_web_needs_the_player(tmp_path, capsys): - code, _, err = run(["web", str(tmp_path / "site"), "--player-only"], capsys) + code, _, err = run(["site", "build", "-o", str(tmp_path / "site"), "--player-only"], capsys) assert code == 2 and "--player" in err and "NNNOTES_PATHS_PLAYER" in err def test_catalog_from_the_cache_needs_no_region(catalog_file, tmp_path, capsys): cache = tmp_path / "cache" - code, _, err = run(["--cache", str(cache), "--language", "xx", "catalog"], capsys) + code, _, err = run(["--cache", str(cache), "--language", "xx", "catalog", "keys"], capsys) assert code == 2 and "catalog.region" in err and "NNNOTES_CATALOG_REGION" in err cache.mkdir() (cache / "catalog_main_xx.bin").write_bytes(catalog_file.read_bytes()) - code, out, err = run(["--cache", str(cache), "--language", "xx", "catalog", "--prefix", "Char/"], capsys) + code, out, err = run(["--cache", str(cache), "--language", "xx", "catalog", "keys", "--prefix", "Char/"], capsys) assert code == 0, err assert out.splitlines() == ["Char/A", "Char/B"] -@pytest.mark.parametrize("argv", [["live2d", "Char/A"], ["spot", "1"]]) +@pytest.mark.parametrize("argv", [["export", "live2d", "Char/A"], ["export", "spot", "1"]]) def test_apk_is_a_required_setting(argv, catalog_file, tmp_path, cdn, capsys, monkeypatch): monkeypatch.setenv("NNNOTES_BUNDLE_KEY", KEY_HEX) monkeypatch.setenv("NNNOTES_BUNDLE_NONCE_SEED", SEED_HEX) @@ -295,11 +306,11 @@ def test_crikey_write_creates_the_directory(tmp_path, capsys, monkeypatch): apk.write_bytes(b"not read") monkeypatch.setattr(crikey, "find_key", lambda path: 987654321012) target = tmp_path / "keys" / "sub" - code, out, _ = run(["--apk", str(apk), "crikey", "--write", str(target)], capsys) + code, out, _ = run(["--apk", str(apk), "apk", "hca-key", "--write", str(target)], capsys) assert code == 0 assert (target / ".hcakey").read_bytes() == (987654321012).to_bytes(8, "big") assert "12 digits" in out and "987654321012" not in out - code, _, err = run(["crikey"], capsys) + code, _, err = run(["apk", "hca-key"], capsys) assert code == 2 and "paths.apk" in err @@ -347,30 +358,30 @@ def data(catalog_file, tmp_path, cdn, monkeypatch): @pytest.mark.parametrize( "argv, message", [ - (["pull", "Char/A", "Nope/1", "Nope/2"], "not a key of the catalog: Nope/1, Nope/2"), - (["room", "Nope/1", "-o", "r.glb"], "not a key of the catalog: Nope/1"), - (["shader", "--key", "Nope/1", "-o", "s"], "not a key of the catalog: Nope/1"), - (["shader", "--apk-bundle", "zz", "-o", "s"], "--apk-bundle 'zz': 0 APK bundles match"), - (["audio", "Nope", "-o", "a"], "cue sheet Nope: no key Cri/Sound/Nope in the catalog"), - (["live2d", "nope", "-o", "m"], "not a Live2D model of the catalog: nope"), + (["bundle", "pull", "Char/A", "Nope/1", "Nope/2"], "not a key of the catalog: Nope/1, Nope/2"), + (["export", "room", "Nope/1", "-o", "r.glb"], "not a key of the catalog: Nope/1"), + (["export", "shader", "--key", "Nope/1", "-o", "s"], "not a key of the catalog: Nope/1"), + (["export", "shader", "--apk-bundle", "zz", "-o", "s"], "--apk-bundle 'zz': 0 APK bundles match"), + (["export", "audio", "Nope", "-o", "a"], "cue sheet Nope: no key Cri/Sound/Nope in the catalog"), + (["export", "live2d", "nope", "-o", "m"], "not a Live2D model of the catalog: nope"), ( - ["live2d", "Character/Live2D/g/x/model/x", "-o", "m"], + ["export", "live2d", "Character/Live2D/g/x/model/x", "-o", "m"], "not a Live2D model of the catalog: Character/Live2D/g/x", ), - (["live2d", "Char/A", "-o", "m"], "not a Live2D model of the catalog: Char/A"), - (["adv", "99", "-o", "e.json"], "episode 99: no MasterAdv row with this id"), - (["story", "99", "-o", "s"], "episode 99: no MasterAdv row with this id"), - (["spot", "99", "-o", "s"], "spot 99: no MasterHomeSpot row with this id"), - (["live", "99", "-o", "l"], "music 99: no MasterLiveMusic row with this id"), - (["live", "7", "--difficulty", "easy", "-o", "l"], "music 7: no easy chart"), - (["live", "7", "--leader-card", "9", "-o", "l"], "MasterMemberCard 9: 0 rows"), - (["live", "7", "--band", "4", "-o", "l"], "band 4: Band/4/live_stage/lightweight_background"), + (["export", "live2d", "Char/A", "-o", "m"], "not a Live2D model of the catalog: Char/A"), + (["export", "episode", "99", "-o", "e.json"], "episode 99: no MasterAdv row with this id"), + (["export", "story", "99", "-o", "s"], "episode 99: no MasterAdv row with this id"), + (["export", "spot", "99", "-o", "s"], "spot 99: no MasterHomeSpot row with this id"), + (["export", "live", "99", "-o", "l"], "music 99: no MasterLiveMusic row with this id"), + (["export", "live", "7", "--difficulty", "easy", "-o", "l"], "music 7: no easy chart"), + (["export", "live", "7", "--leader-card", "9", "-o", "l"], "MasterMemberCard 9: 0 rows"), + (["export", "live", "7", "--band", "4", "-o", "l"], "band 4: Band/4/live_stage/lightweight_background"), ], ) def test_unknown_key_or_id_is_a_usage_error(data, argv, message, capsys): code, out, err = run(data + argv, capsys) assert code == 2 and out == "", err - assert f"nnnotes {argv[0]}: error: {message}" in err and "Traceback" not in err + assert f"nnnotes {argv[0]} {argv[1]}: error: {message}" in err and "Traceback" not in err assert KEY_HEX not in err @@ -380,19 +391,19 @@ def test_known_id_reaches_the_extractor(data, tmp_path, capsys, monkeypatch): seen = [] monkeypatch.setattr(adv, "extract", lambda cat, md, adv_id: seen.append(adv_id) or "x") monkeypatch.setattr(adv, "to_json", lambda x: {"commandCount": 0, "resources": []}) - code, out, err = run(data + ["adv", "1", "-o", str(tmp_path / "e.json")], capsys) + code, out, err = run(data + ["export", "episode", "1", "-o", str(tmp_path / "e.json")], capsys) assert code == 0 and seen == [1], err def test_missing_master_table_names_it(data, tmp_path, capsys): (tmp_path / "master" / "MasterHomeSpot.json").unlink() - code, _, err = run(data + ["spot", "1", "-o", "s"], capsys) + code, _, err = run(data + ["export", "spot", "1", "-o", "s"], capsys) assert code == 2 and "no MasterHomeSpot.json" in err and len(err.strip().splitlines()) == 1 def test_story_needs_the_language(data, capsys): i = data.index("--language") - code, _, err = run(data[:i] + data[i + 2 :] + ["story", "1", "-o", "s"], capsys) + code, _, err = run(data[:i] + data[i + 2 :] + ["export", "story", "1", "-o", "s"], capsys) assert code == 2 and "catalog.language" in err and "NNNOTES_CATALOG_LANGUAGE" in err @@ -408,17 +419,17 @@ def test_the_command_line_starts_without_the_exporters(): @pytest.mark.parametrize( "argv, message", [ - (["audio", "s", "--flac-level", "13"], "--flac-level 13: expected 0 to 12"), - (["live", "7", "--flac-level", "-1"], "--flac-level -1: expected 0 to 12"), - (["audio", "s", "--format", "ogg", "--flac-level", "5"], "--flac-level: --format ogg writes no FLAC"), - (["live", "7", "--format", "wav", "--flac-level", "5"], "--flac-level: --format wav writes no FLAC"), - (["story", "1", "--no-audio", "--flac-level", "5"], "--flac-level: --no-audio decodes no audio"), + (["export", "audio", "s", "--flac-level", "13"], "--flac-level 13: expected 0 to 12"), + (["export", "live", "7", "--flac-level", "-1"], "--flac-level -1: expected 0 to 12"), + (["export", "audio", "s", "--format", "ogg", "--flac-level", "5"], "--flac-level: --format ogg writes no FLAC"), + (["export", "live", "7", "--format", "wav", "--flac-level", "5"], "--flac-level: --format wav writes no FLAC"), + (["export", "story", "1", "--no-audio", "--flac-level", "5"], "--flac-level: --no-audio decodes no audio"), ], ) def test_flac_level_is_checked(data, argv, message, capsys): code, out, err = run(data + argv + ["-o", "o"], capsys) assert code == 2 and out == "", err - assert f"nnnotes {argv[0]}: error: {message}" in err + assert f"nnnotes {argv[0]} {argv[1]}: error: {message}" in err def test_flac_level_reaches_the_decode(data, tmp_path, capsys, monkeypatch): @@ -437,7 +448,7 @@ def test_flac_level_reaches_the_decode(data, tmp_path, capsys, monkeypatch): monkeypatch.setattr(story, "build", lambda *a, **kw: seen.append(kw["audio_options"].get("flac_level")) or {}) monkeypatch.setattr(live, "resolve_band", lambda *a, **kw: None) monkeypatch.setattr(live, "build", lambda *a, **kw: seen.append(kw.get("flac_level")) or {}) - for argv in (["audio", "s"], ["story", "1"], ["live", "7"]): + for argv in (["export", "audio", "s"], ["export", "story", "1"], ["export", "live", "7"]): for flag, level in (([], None), (["--flac-level", "5"], 5)): seen.clear() code, _, err = run(data + argv + flag + ["-o", str(tmp_path / "o")], capsys) diff --git a/tests/cli/test_reference.py b/tests/cli/test_reference.py new file mode 100644 index 0000000..df3ce4d --- /dev/null +++ b/tests/cli/test_reference.py @@ -0,0 +1,28 @@ +"""The command reference: docs/cli.md is the text the parser gives, and every command and argument has help.""" + +from pathlib import Path + +from nnnotes import cli +from nnnotes.cli import reference + +ROOT = Path(__file__).resolve().parents[2] + + +def test_the_command_reference_is_current(): + text = (ROOT / "docs" / "cli.md").read_text(encoding="utf-8") + assert reference.markdown() == text, "docs/cli.md: run python -m nnnotes.cli.reference docs/cli.md" + + +def test_every_command_and_argument_has_help(): + found = reference.commands(cli.build_parser(), set()) + assert len(found) > 30 + for c in found: + assert c["help"], c["command"] + for a in c["arguments"]: + assert a["help"], (c["command"], a["flags"]) + + +def test_every_command_takes_the_global_options_and_json(): + for c in reference.commands(cli.build_parser(), set()): + flags = {f for a in c["arguments"] for f in a["flags"]} + assert {"--config", "--region", "--language", "--json"} <= flags, c["command"] diff --git a/tests/cli/test_voices.py b/tests/cli/test_voices.py new file mode 100644 index 0000000..28d9396 --- /dev/null +++ b/tests/cli/test_voices.py @@ -0,0 +1,189 @@ +"""The voices command on synthetic voices documents, master data and catalogs.""" + +import json + +import pytest + +from nnnotes import cli +from nnnotes.content import storyvoices +from nnnotes.formats import cri +from nnnotes.pipeline.store import Store +from nnnotes.source.session import Session +from tests.pipeline import test_voices as tv +from tests.pipeline import test_voices_story as ts +from tests.pipeline.test_views import write_master + + +def command(argv, cfg=None): + args = cli.parse_args(argv) + args.func(args, cfg or tv.Cfg()) + + +@pytest.fixture +def session(monkeypatch): + """session(master_dir, catalog): the data sources of the commands that read no export.""" + + def use(mdir, catalog): + monkeypatch.setattr(Session, "master_dir", lambda self, region=None: mdir) + monkeypatch.setattr(Session, "catalog", lambda self, bundles=True, region=None: catalog) + + return use + + +def fake_decode(sheets=None): + """A cri.decode that writes every stream of a synthetic sheet; its calls in `.calls`.""" + + def decode(cat, sheet, out_dir, key=None, fmt="flac", **kw): + decode.calls.append((sheet, fmt, kw.get("flac_level"))) + out_dir.mkdir(parents=True, exist_ok=True) + streams = [] + for n, name, file, _samples in tv.stream_files(sheet, sheets): + (out_dir / f"{file}.{fmt}").write_bytes(f"decoded {sheet}/{file}".encode()) + streams.append({"stream": n, "name": name, "file": f"{file}.{fmt}"}) + (out_dir / "streams.json").write_text(json.dumps(streams), encoding="utf-8") + + decode.calls = [] + return decode + + +def test_list_search_summary(tmp_path, capsysbinary): + out = tv.export_dir(tmp_path, Store(tmp_path / "s")) + command(["voices", "list", "--from", str(out), "--character", "Alpha", "--category", "Talk.CharacterRankUp"]) + line = capsysbinary.readouterr().out.decode("utf-8") + assert ( + line == "MasterTalk:561\tcharacter_name_alpha-ja\tTalk.CharacterRankUp\tVoiceSystem_01/" + "Growth_Alpha_RankUp_01\tok\tテストです\n" + ) + command(["voices", "search", "HELLO", "--from", str(out), "--search-language", "en", "--json"]) + rows = json.loads(capsysbinary.readouterr().out) + assert [r["id"] for r in rows] == ["MasterTalk:561"] + command(["voices", "list", "--from", str(out), "--source", "Sound", "--character", "gamma"]) + assert capsysbinary.readouterr().out.decode().split("\t")[0] == "MasterSound:114" + command(["voices", "summary", "--from", str(out), "--json"]) + s = json.loads(capsysbinary.readouterr().out) + assert s["gaps"] == 8 and s["reverse"]["unreferencedStreams"] == {"VoiceSystem_N": 2} + with pytest.raises(SystemExit): + command(["voices", "list", "--from", str(out), "--character", "Nobody"]) + with pytest.raises(SystemExit): + command(["voices", "list", "--from", str(tmp_path / "none")]) + + +def test_without_an_export_the_content_is_built(tmp_path, capsysbinary, session): + session(write_master(tmp_path / "m", tv.TABLES), tv.Catalog()) + command(["voices", "list", "--character", "1", "--category", "character-rank-up", "--json"]) + rows = json.loads(capsysbinary.readouterr().out) + assert [(r["id"], r["status"]) for r in rows] == [("MasterTalk:561", "not-exported")] + + +def test_get_copies_the_exported_files(tmp_path, capsysbinary): + store = Store(tmp_path / "s") + out = tv.export_dir(tmp_path, store) + dst = tmp_path / "got" + command(["voices", "get", "MasterHomeSpot:10001", "-o", str(dst), "--from", str(out)]) + assert sorted(p.name for p in dst.iterdir()) == ["IntroRandom.flac", "IntroRandom_2.flac", "voice.json"] + assert (dst / "IntroRandom_2.flac").read_bytes() == b"spot_01_01_01/IntroRandom_2" + meta = json.loads((dst / "voice.json").read_text(encoding="utf-8")) + assert [r["id"] for r in meta["rows"]] == ["MasterHomeSpot:10001"] + assert meta["files"] == ["IntroRandom.flac", "IntroRandom_2.flac"] + command(["voices", "get", "101", "-o", str(tmp_path / "g2"), "--from", str(out)]) + assert (tmp_path / "g2" / "Growth_Alpha_RankUp_01.flac").is_file() + with pytest.raises(SystemExit) as e: + command(["voices", "get", "MasterTalk:563", "-o", str(tmp_path / "g3"), "--from", str(out)]) + assert e.value.code == 1 + with pytest.raises(SystemExit) as e: + command(["voices", "get", "MasterTalk:1", "-o", str(tmp_path / "g4"), "--from", str(out)]) + assert e.value.code == 2 + + +def test_get_without_an_export_decodes_the_one_sheet(tmp_path, capsysbinary, monkeypatch, session): + decode = fake_decode() + monkeypatch.setattr(cri, "acb_data", lambda cat, sheet: ({"acb": tv.sheet_acb(sheet)}, cri.RAW_ACB)) + monkeypatch.setattr(cri, "decode", decode) + session(write_master(tmp_path / "m", tv.TABLES), tv.Catalog()) + dst = tmp_path / "got" + command(["voices", "get", "MasterCharacterVoice:1", "-o", str(dst)]) + assert decode.calls == [("VoiceSystem_02", "flac", 8)] + assert sorted(p.name for p in dst.iterdir()) == ["Shared_01.flac", "voice.json"] + assert (dst / "Shared_01.flac").read_bytes() == b"decoded VoiceSystem_02/Shared_01" + command(["voices", "get", "MasterTalk:561", "-o", str(tmp_path / "ogg"), "--format", "ogg"]) + assert (tmp_path / "ogg" / "Growth_Alpha_RankUp_01.ogg").is_file() + + +def test_get_decodes_at_the_flac_level_given(tmp_path, capsysbinary, monkeypatch): + decode = fake_decode() + monkeypatch.setattr(cri, "acb_data", lambda cat, sheet: ({"acb": tv.sheet_acb(sheet)}, cri.RAW_ACB)) + monkeypatch.setattr(cri, "decode", decode) + monkeypatch.setattr(Session, "catalog", lambda self, bundles=True, region=None: tv.Catalog()) + out = tv.export_dir(tmp_path, Store(tmp_path / "s")) + command(["voices", "get", "101", "-o", str(tmp_path / "g"), "--from", str(out), "--flac-level", "5"]) + assert decode.calls == [("VoiceSystem_01", "flac", 5)] + for argv, message in ( + (["--flac-level", "13"], "--flac-level 13: expected 0 to 12"), + (["--format", "ogg", "--flac-level", "5"], "--flac-level: --format ogg writes no FLAC"), + ): + with pytest.raises(SystemExit) as e: + command(["voices", "get", "101", "-o", str(tmp_path / "x"), "--from", str(out), *argv]) + assert e.value.code == 2 and message in capsysbinary.readouterr().err.decode() + + +def test_story_rows_are_filtered(tmp_path, capsysbinary): + out = ts.export_dir(tmp_path, Store(tmp_path / "s")) + command( + ["voices", "list", "--from", str(out), "--source", "story", "--episode", "900001", "--status", "ok"], + tv.Cfg("en"), + ) + got = capsysbinary.readouterr().out.decode("utf-8").splitlines() + sheet = ts.STORY_SHEET + assert got == [ + f"MasterAdv:900001:3\tadv_alpha-en\tStory.Main\t{sheet}/{sheet}_001\tok\tx01_1-en", + f"MasterAdv:900001:5:1\tadv_gamma-en,adv_alpha-en\tStory.Main\t{sheet}/{sheet}_002\tok\tx01_2-en", + f"MasterAdv:900001:12\tadv_both-en\tStory.Main\t{sheet}/{sheet}_003\tok\tx01_5-en", + ] + command(["voices", "search", "x01_3-ZH-HANS", "--from", str(out), "--search-language", "zh-Hans", "--json"]) + assert [r["id"] for r in json.loads(capsysbinary.readouterr().out)] == ["MasterAdv:900001:7"] + command(["voices", "list", "--from", str(out), "--episode", "adv_script_x_02", "--json"]) + assert [r["id"] for r in json.loads(capsysbinary.readouterr().out)] == ["MasterAdv:900002:1"] + command(["voices", "list", "--from", str(out), "--category", "Story.HomeSpotTapTalk", "--character", "alpha"]) + assert capsysbinary.readouterr().out.decode().split("\t")[0] == "MasterAdv:900003:2" + with pytest.raises(SystemExit): + command(["voices", "list", "--from", str(out), "--episode", "123"]) + command(["voices", "summary", "--from", str(out)]) + assert "story: 8 episodes" in capsysbinary.readouterr().out.decode() + dst = tmp_path / "got" + command(["voices", "get", "MasterAdv:900001:5:1", "-o", str(dst), "--from", str(out)]) + assert (dst / f"{sheet}_002.flac").read_bytes() == f"{sheet}/{sheet}_002.flac".encode() + assert [r["id"] for r in json.loads((dst / "voice.json").read_text(encoding="utf-8"))["rows"]] == [ + "MasterAdv:900001:5:1" + ] + command(["voices", "get", "5201", "-o", str(tmp_path / "g2"), "--from", str(out)]) + assert (tmp_path / "g2" / "Growth_Alpha_RankUp_01.flac").is_file() + + +def test_without_an_export_one_episode_is_read(tmp_path, capsysbinary, monkeypatch, session): + loaded = [] + + def load(cat, keys): + asset = keys["script"].rsplit("/", 1)[-1] + loaded.append(asset) + return { + p: ts.EPISODES[asset][p] if p == "script" else ts.shard(ts.EPISODES[asset][p]) for p in ts.EPISODES[asset] + } + + monkeypatch.setattr(storyvoices, "load_episode", load) + monkeypatch.setattr(cri, "acb_data", lambda cat, sheet: ({"acb": tv.sheet_acb(sheet, ts.SHEETS2)}, cri.RAW_ACB)) + monkeypatch.setattr(cri, "decode", fake_decode(ts.SHEETS2)) + session(write_master(tmp_path / "m", ts.tables()), ts.Catalog()) + command(["voices", "list", "--episode", "900001", "--json"]) + rows = json.loads(capsysbinary.readouterr().out) + assert loaded == ["adv_script_x_01"] + assert [(r["id"], r["status"]) for r in rows][:2] == [ + ("MasterAdv:900001:3", "not-exported"), + ("MasterAdv:900001:5:1", "not-exported"), + ] + command(["voices", "list", "--source", "Story", "--json"]) + assert json.loads(capsysbinary.readouterr().out) == [] # no episode read without --episode + dst = tmp_path / "got" + command(["voices", "get", "MasterAdv:900001:3", "-o", str(dst)]) + assert loaded[-1] == "adv_script_x_01" + sheet = ts.STORY_SHEET + assert (dst / f"{sheet}_001.flac").read_bytes() == f"decoded {sheet}/{sheet}_001".encode() diff --git a/tests/content/test_story_kinds.py b/tests/content/test_story_kinds.py index 0e8023f..ff0a568 100644 --- a/tests/content/test_story_kinds.py +++ b/tests/content/test_story_kinds.py @@ -88,10 +88,9 @@ def test_missing_script_component_is_recorded(tmp_path): def test_story_no_audio_flag(): - p = cli.build_parser() - args = p.parse_args(["story", "10001", "-o", "out", "--no-audio"]) + args = cli.parse_args(["export", "story", "10001", "-o", "out", "--no-audio"]) assert args.no_audio is True and args.format == "flac" - assert p.parse_args(["story", "10001", "-o", "out"]).no_audio is False + assert cli.parse_args(["export", "story", "10001", "-o", "out"]).no_audio is False class _Parsed: diff --git a/tests/content/test_ui.py b/tests/content/test_ui.py index 66fd105..d1380bc 100644 --- a/tests/content/test_ui.py +++ b/tests/content/test_ui.py @@ -180,8 +180,8 @@ def test_cli_selects_the_jp_split_apk_and_keeps_public_provenance(tmp_path, expo }, environ={}, ) - args = cli.build_parser().parse_args(["ui", "--key", KEY, "-o", str(tmp_path / "out")]) - ui.command(args, cfg) + args = cli.parse_args(["export", "ui", "--key", KEY, "-o", str(tmp_path / "out")]) + args.func(args, cfg) index = read(tmp_path / "out/index.json") assert index["provenance"]["region"] == "jp" assert len(index["provenance"]["apk_members_sha256"]) == 64 @@ -189,9 +189,9 @@ def test_cli_selects_the_jp_split_apk_and_keeps_public_provenance(tmp_path, expo def test_limit_is_validated_before_any_apk_read(tmp_path, capsys): - args = cli.build_parser().parse_args(["ui", "--limit", "0", "-o", str(tmp_path)]) + args = cli.parse_args(["export", "ui", "--limit", "0", "-o", str(tmp_path)]) with pytest.raises(SystemExit) as exc: - ui.command(args, Config(environ={})) + args.func(args, Config(environ={})) assert exc.value.code == 2 assert "--limit must be positive" in capsys.readouterr().err diff --git a/tests/musicdata/test_deckdata.py b/tests/musicdata/test_deckdata.py index 807262a..558aba3 100644 --- a/tests/musicdata/test_deckdata.py +++ b/tests/musicdata/test_deckdata.py @@ -613,7 +613,7 @@ def run(argv, capsys): """(exit code, stdout, stderr) of the command line; the message of sys.exit(message) is part of stderr.""" code, message = 0, "" try: - cli.main(argv) + cli.main(["--json", *argv]) except SystemExit as e: code, message = (e.code, "") if isinstance(e.code, int) else (1, str(e.code)) out, err = capsys.readouterr() diff --git a/tests/musicdata/test_statscache.py b/tests/musicdata/test_statscache.py index 903e2cb..446a2ee 100644 --- a/tests/musicdata/test_statscache.py +++ b/tests/musicdata/test_statscache.py @@ -90,11 +90,11 @@ def test_invalid_options_are_user_errors_before_native_or_export_work(tmp_path, assert not output.exists() -def test_zero_workers_selects_native_automatic_parallelism(deck_input): - module = CacheBridge() - deck = musicdata.Deck(module=module, seeds=1, workers=0) - deck.stats(deck_input) - assert module.inputs[0][1:] == (1, None) +def test_workers_select_the_native_threads(deck_input): + for workers, native in ((None, None), (0, 1), (3, 3)): + module = CacheBridge() + musicdata.Deck(module=module, seeds=1, workers=workers).stats(deck_input) + assert module.inputs[0][1:] == (1, native) def test_sampling_format_is_rejected_before_calling_the_model(deck_input): diff --git a/tests/pipeline/test_cristages.py b/tests/pipeline/test_cristages.py index 86d21aa..89d9002 100644 --- a/tests/pipeline/test_cristages.py +++ b/tests/pipeline/test_cristages.py @@ -246,7 +246,7 @@ def test_audio_subjects_by_content(tmp_path, tools): def test_audio_waits_for_raw_files_not_fetched(tmp_path, tools): - from nnnotes.cli.cli_assets import Inputs + from nnnotes.cli.assets import Inputs store = Store(tmp_path / "s") env = planning(store, tmp_path, {"cri_assets_cri/sound/a": ACB}) diff --git a/tests/pipeline/test_voices.py b/tests/pipeline/test_voices.py index 7c78353..be7ab29 100644 --- a/tests/pipeline/test_voices.py +++ b/tests/pipeline/test_voices.py @@ -2,7 +2,6 @@ source, the ACB cue tables, the statuses, content apart from availability, the stage, the layout document and the command.""" -import argparse import copy import json import re @@ -10,7 +9,6 @@ import subprocess import sys from pathlib import Path -from types import SimpleNamespace import pytest @@ -877,13 +875,6 @@ def keys(self, prefix=""): return [k for k in KEYS if k.startswith(prefix)] -def command(argv, common, cfg=None): - p = argparse.ArgumentParser(prog="nnnotes") - voices.register(p.add_subparsers(dest="cmd", required=True), common) - args = p.parse_args(argv) - args.func(args, cfg or Cfg()) - - def export_dir(tmp: Path, store) -> Path: """An export directory: views/voices.json with paths and the stream files at them.""" out = tmp / "out" @@ -902,92 +893,6 @@ def export_dir(tmp: Path, store) -> Path: return out -def test_command_list_search_summary(tmp_path, capsysbinary): - out = export_dir(tmp_path, Store(tmp_path / "s")) - common = argparse.Namespace() - command( - ["voices", "list", "--from", str(out), "--character", "Alpha", "--category", "Talk.CharacterRankUp"], common - ) - line = capsysbinary.readouterr().out.decode("utf-8") - assert ( - line == "MasterTalk:561\tcharacter_name_alpha-ja\tTalk.CharacterRankUp\tVoiceSystem_01/" - "Growth_Alpha_RankUp_01\tok\tテストです\n" - ) - command(["voices", "search", "HELLO", "--from", str(out), "--language", "en", "--json"], common) - rows = json.loads(capsysbinary.readouterr().out) - assert [r["id"] for r in rows] == ["MasterTalk:561"] - command(["voices", "list", "--from", str(out), "--source", "Sound", "--character", "gamma"], common) - assert capsysbinary.readouterr().out.decode().split("\t")[0] == "MasterSound:114" - command(["voices", "summary", "--from", str(out), "--json"], common) - s = json.loads(capsysbinary.readouterr().out) - assert s["gaps"] == 8 and s["reverse"]["unreferencedStreams"] == {"VoiceSystem_N": 2} - with pytest.raises(SystemExit): - command(["voices", "list", "--from", str(out), "--character", "Nobody"], common) - with pytest.raises(SystemExit): - command(["voices", "list", "--from", str(tmp_path / "none")], common) - - -def test_command_without_an_export_builds_the_content(tmp_path, capsysbinary): - mdir = write_master(tmp_path / "m", TABLES) - common = argparse.Namespace( - session=lambda cfg: SimpleNamespace(master_dir=lambda: mdir, catalog=lambda bundles=True: Catalog()) - ) - command(["voices", "list", "--character", "1", "--category", "character-rank-up", "--json"], common) - rows = json.loads(capsysbinary.readouterr().out) - assert [(r["id"], r["status"]) for r in rows] == [("MasterTalk:561", "not-exported")] - - -def test_get_copies_the_exported_files(tmp_path, capsysbinary): - store = Store(tmp_path / "s") - out = export_dir(tmp_path, store) - dst = tmp_path / "got" - command(["voices", "get", "MasterHomeSpot:10001", "-o", str(dst), "--from", str(out)], argparse.Namespace()) - assert sorted(p.name for p in dst.iterdir()) == ["IntroRandom.flac", "IntroRandom_2.flac", "voice.json"] - assert (dst / "IntroRandom_2.flac").read_bytes() == b"spot_01_01_01/IntroRandom_2" - meta = json.loads((dst / "voice.json").read_text(encoding="utf-8")) - assert [r["id"] for r in meta["rows"]] == ["MasterHomeSpot:10001"] - assert meta["files"] == ["IntroRandom.flac", "IntroRandom_2.flac"] - command(["voices", "get", "101", "-o", str(tmp_path / "g2"), "--from", str(out)], argparse.Namespace()) - assert (tmp_path / "g2" / "Growth_Alpha_RankUp_01.flac").is_file() - with pytest.raises(SystemExit) as e: - command( - ["voices", "get", "MasterTalk:563", "-o", str(tmp_path / "g3"), "--from", str(out)], argparse.Namespace() - ) - assert e.value.code == 1 - with pytest.raises(SystemExit) as e: - command(["voices", "get", "MasterTalk:1", "-o", str(tmp_path / "g4"), "--from", str(out)], argparse.Namespace()) - assert e.value.code == 2 - - -def test_get_without_an_export_decodes_the_one_sheet(tmp_path, capsysbinary, monkeypatch): - from nnnotes.formats import cri - - mdir = write_master(tmp_path / "m", TABLES) - calls = [] - - def decode(cat, sheet, out_dir, key=None, fmt="flac", **kw): - calls.append((sheet, fmt, kw.get("flac_level"))) - out_dir.mkdir(parents=True, exist_ok=True) - streams = [] - for n, name, file, _samples in stream_files(sheet): - (out_dir / f"{file}.{fmt}").write_bytes(f"decoded {sheet}/{file}".encode()) - streams.append({"stream": n, "name": name, "file": f"{file}.{fmt}"}) - (out_dir / "streams.json").write_text(json.dumps(streams), encoding="utf-8") - - monkeypatch.setattr(cri, "acb_data", lambda cat, sheet: ({"acb": sheet_acb(sheet)}, cri.RAW_ACB)) - monkeypatch.setattr(cri, "decode", decode) - common = argparse.Namespace( - session=lambda cfg: SimpleNamespace(master_dir=lambda: mdir, catalog=lambda bundles=True: Catalog()) - ) - dst = tmp_path / "got" - command(["voices", "get", "MasterCharacterVoice:1", "-o", str(dst)], common) - assert calls == [("VoiceSystem_02", "flac", 8)] - assert sorted(p.name for p in dst.iterdir()) == ["Shared_01.flac", "voice.json"] - assert (dst / "Shared_01.flac").read_bytes() == b"decoded VoiceSystem_02/Shared_01" - command(["voices", "get", "MasterTalk:561", "-o", str(tmp_path / "ogg"), "--format", "ogg"], common) - assert (tmp_path / "ogg" / "Growth_Alpha_RankUp_01.ogg").is_file() - - # ---------------------------------------------------------------- laziness and documentation def test_voices_import_nothing_heavy(): code = ( diff --git a/tests/pipeline/test_voices_story.py b/tests/pipeline/test_voices_story.py index 257c030..5736017 100644 --- a/tests/pipeline/test_voices_story.py +++ b/tests/pipeline/test_voices_story.py @@ -2,11 +2,9 @@ texts, the sounds and cue sheets of the shards, the statuses of episodes and rows, the inferred characters, the stage reading the unity.export artifacts, and the command.""" -import argparse import copy import json import random -from types import SimpleNamespace import pytest @@ -17,8 +15,8 @@ from nnnotes.pipeline.store import Store from nnnotes.pipeline.views import AddressIndex, Obj, ViewError from nnnotes.source import languages -from tests.pipeline.test_views import addresses_doc, write_master -from tests.pipeline.test_voices import KEYS, SHEETS, TABLES, Cfg, Paths, audio, audio_task, command, planning, row, text +from tests.pipeline.test_views import addresses_doc +from tests.pipeline.test_voices import KEYS, SHEETS, TABLES, Paths, audio, audio_task, planning, row, text RULES = voices.load_rules() STORY_SHEET = "adv_voice_x_01" @@ -611,83 +609,6 @@ def export_dir(tmp_path, store): return out -def test_command_filters_story_rows(tmp_path, capsysbinary): - out = export_dir(tmp_path, Store(tmp_path / "s")) - common = argparse.Namespace() - command( - ["voices", "list", "--from", str(out), "--source", "story", "--episode", "900001", "--status", "ok"], - common, - Cfg("en"), - ) - got = capsysbinary.readouterr().out.decode("utf-8").splitlines() - assert got == [ - f"MasterAdv:900001:3\tadv_alpha-en\tStory.Main\t{STORY_SHEET}/{STORY_SHEET}_001\tok\tx01_1-en", - f"MasterAdv:900001:5:1\tadv_gamma-en,adv_alpha-en\tStory.Main\t{STORY_SHEET}/{STORY_SHEET}_002\tok\tx01_2-en", - f"MasterAdv:900001:12\tadv_both-en\tStory.Main\t{STORY_SHEET}/{STORY_SHEET}_003\tok\tx01_5-en", - ] - command(["voices", "search", "x01_3-ZH-HANS", "--from", str(out), "--language", "zh-Hans", "--json"], common) - assert [r["id"] for r in json.loads(capsysbinary.readouterr().out)] == ["MasterAdv:900001:7"] - command(["voices", "list", "--from", str(out), "--episode", "adv_script_x_02", "--json"], common) - assert [r["id"] for r in json.loads(capsysbinary.readouterr().out)] == ["MasterAdv:900002:1"] - command( - ["voices", "list", "--from", str(out), "--category", "Story.HomeSpotTapTalk", "--character", "alpha"], common - ) - assert capsysbinary.readouterr().out.decode().split("\t")[0] == "MasterAdv:900003:2" - with pytest.raises(SystemExit): - command(["voices", "list", "--from", str(out), "--episode", "123"], common) - command(["voices", "summary", "--from", str(out)], common) - assert "story: 8 episodes" in capsysbinary.readouterr().out.decode() - dst = tmp_path / "got" - command(["voices", "get", "MasterAdv:900001:5:1", "-o", str(dst), "--from", str(out)], common) - assert (dst / f"{STORY_SHEET}_002.flac").read_bytes() == f"{STORY_SHEET}/{STORY_SHEET}_002.flac".encode() - assert [r["id"] for r in json.loads((dst / "voice.json").read_text(encoding="utf-8"))["rows"]] == [ - "MasterAdv:900001:5:1" - ] - command(["voices", "get", "5201", "-o", str(tmp_path / "g2"), "--from", str(out)], common) - assert (tmp_path / "g2" / "Growth_Alpha_RankUp_01.flac").is_file() - - class Catalog: def keys(self, prefix=""): return [k for k in catalog_keys() if k.startswith(prefix)] - - -def test_command_without_an_export_reads_one_episode(tmp_path, capsysbinary, monkeypatch): - from nnnotes.formats import cri - from tests.pipeline.test_voices import sheet_acb, stream_files - - mdir = write_master(tmp_path / "m", tables()) - loaded = [] - - def load(cat, keys): - asset = keys["script"].rsplit("/", 1)[-1] - loaded.append(asset) - return {p: EPISODES[asset][p] if p == "script" else shard(EPISODES[asset][p]) for p in EPISODES[asset]} - - def decode(cat, sheet, out_dir, key=None, fmt="flac", **kw): - out_dir.mkdir(parents=True, exist_ok=True) - streams = [] - for n, name, file, _samples in stream_files(sheet, SHEETS2): - (out_dir / f"{file}.{fmt}").write_bytes(f"decoded {sheet}/{file}".encode()) - streams.append({"stream": n, "name": name, "file": f"{file}.{fmt}"}) - (out_dir / "streams.json").write_text(json.dumps(streams), encoding="utf-8") - - monkeypatch.setattr(storyvoices, "load_episode", load) - monkeypatch.setattr(cri, "acb_data", lambda cat, sheet: ({"acb": sheet_acb(sheet, SHEETS2)}, cri.RAW_ACB)) - monkeypatch.setattr(cri, "decode", decode) - common = argparse.Namespace( - session=lambda cfg: SimpleNamespace(master_dir=lambda: mdir, catalog=lambda bundles=True: Catalog()) - ) - command(["voices", "list", "--episode", "900001", "--json"], common) - rows = json.loads(capsysbinary.readouterr().out) - assert loaded == ["adv_script_x_01"] - assert [(r["id"], r["status"]) for r in rows][:2] == [ - ("MasterAdv:900001:3", "not-exported"), - ("MasterAdv:900001:5:1", "not-exported"), - ] - command(["voices", "list", "--source", "Story", "--json"], common) - assert json.loads(capsysbinary.readouterr().out) == [] # no episode read without --episode - dst = tmp_path / "got" - command(["voices", "get", "MasterAdv:900001:3", "-o", str(dst)], common) - assert loaded[-1] == "adv_script_x_01" - assert (dst / f"{STORY_SHEET}_001.flac").read_bytes() == f"decoded {STORY_SHEET}/{STORY_SHEET}_001".encode() diff --git a/tests/site/test_live_options_web.py b/tests/site/test_live_options_web.py index 8b29ece..c0c9db1 100644 --- a/tests/site/test_live_options_web.py +++ b/tests/site/test_live_options_web.py @@ -167,7 +167,10 @@ def test_live_option_flag(tmp_path, capsys, monkeypatch): "zz", "--master", str(master), - "web", + "--json", + "site", + "build", + "-o", str(tmp_path / "s"), "--player", str(fake_player(tmp_path / "p")), diff --git a/tests/site/test_readsets.py b/tests/site/test_readsets.py index 89dd8c2..3ba5b39 100644 --- a/tests/site/test_readsets.py +++ b/tests/site/test_readsets.py @@ -428,8 +428,22 @@ def test_web_read_workers_flag_reaches_the_build(tmp_path, capsys, monkeypatch): for rel in (sitefiles.READ_SET_SCRIPT, *sitefiles.PLAYER_BUNDLES, f"{sitefiles.PLAYER_PAGE_DIR}/index.html"): (player / rel).parent.mkdir(parents=True, exist_ok=True) (player / rel).write_text("x", encoding="utf-8") - cli.main(["web", str(tmp_path / "s"), "--player", str(player), "--pair", "7:hard", "--read-workers", "24"]) - cli.main(["web", str(tmp_path / "s"), "--player", str(player), "--pair", "7:hard"]) + cli.main( + [ + "--json", + "site", + "build", + "-o", + str(tmp_path / "s"), + "--player", + str(player), + "--pair", + "7:hard", + "--read-workers", + "24", + ] + ) + cli.main(["--json", "site", "build", "-o", str(tmp_path / "s"), "--player", str(player), "--pair", "7:hard"]) capsys.readouterr() assert [kw["read_workers"] for kw in calls] == [24, None] @@ -448,7 +462,10 @@ def test_web_pair_that_no_region_has_is_a_usage_error(tmp_path, capsys, monkeypa "zz", "--master", str(master), - "web", + "--json", + "site", + "build", + "-o", str(tmp_path / "s"), "--player", str(fake_player(tmp_path / "p")), diff --git a/tests/site/test_story_site.py b/tests/site/test_story_site.py index c1640a7..9c10617 100644 --- a/tests/site/test_story_site.py +++ b/tests/site/test_story_site.py @@ -806,6 +806,7 @@ def test_a_story_without_its_model_fails(story_build, monkeypatch): def test_cli_web_stories(tmp_path, capsys, monkeypatch): from nnnotes import cli + from nnnotes.cli import common calls = [] @@ -820,7 +821,10 @@ def fake_build(out, ids, cfg, player, fmt, **kw): [ "--apk", str(tmp_path / "base.apk"), - "web", + "--json", + "site", + "build", + "-o", str(tmp_path / "s"), "--player", str(fake_player(tmp_path / "p")), @@ -845,7 +849,7 @@ def fake_build(out, ids, cfg, player, fmt, **kw): assert ids == [10462, 7] and fmt == "aac" and kw["audio"] is False and kw["workers"] == 2 assert kw["story_languages"] == ["en", "ja"] assert kw["fonts_flags"] == {"en": "a.otf", "ja": "b.otf", "emoji": "e.ttf"} - assert cli.EMOJI_FONT == storysite.EMOJI_FONT + assert common.EMOJI_FONT == storysite.EMOJI_FONT assert kw["fonts"] == "open" and kw["encoding"] == "gzip" assert json.loads(capsys.readouterr().out)["storiesFailed"] == [] @@ -859,7 +863,10 @@ def test_cli_web_stories_need_the_fonts_extra(tmp_path, capsys, monkeypatch): [ "--apk", str(tmp_path / "base.apk"), - "web", + "--json", + "site", + "build", + "-o", str(tmp_path / "s"), "--player", str(fake_player(tmp_path / "p")), @@ -885,5 +892,7 @@ def test_cli_web_story_usage(tmp_path, capsys, argv, message): from nnnotes import cli with pytest.raises(SystemExit) as e: - cli.main(["web", str(tmp_path / "s"), "--player", str(fake_player(tmp_path / "p")), *argv]) + cli.main( + ["--json", "site", "build", "-o", str(tmp_path / "s"), "--player", str(fake_player(tmp_path / "p")), *argv] + ) assert e.value.code == 2 and message in capsys.readouterr().err diff --git a/tests/site/test_web.py b/tests/site/test_web.py index f718453..ac12eb9 100644 --- a/tests/site/test_web.py +++ b/tests/site/test_web.py @@ -237,7 +237,7 @@ def test_player_only_command(tmp_path, capsys): out = tmp_path / "site" (out / "assets").mkdir(parents=True) (out / "assets" / "unreferenced.json").write_bytes(b"{}") - cli.main(["web", str(out), "--player", str(fake_player(tmp_path / "p")), "--player-only"]) + cli.main(["--json", "site", "build", "-o", str(out), "--player", str(fake_player(tmp_path / "p")), "--player-only"]) r = json.loads(capsys.readouterr().out) assert (r["charts"], r["removedAssets"], r["pageFiles"]) == (0, 1, 2) assert json.loads((out / "charts.json").read_text(encoding="utf-8")) == { @@ -252,14 +252,14 @@ def test_player_only_and_reingest_over_an_encoded_site(tmp_path, capsys): site, player = tmp_path / "site", str(fake_player(tmp_path / "p")) encoded_site(site) before = (site / "charts" / "1_easy.json").read_bytes() - cli.main(["web", str(site), "--player", player, "--player-only"]) + cli.main(["--json", "site", "build", "-o", str(site), "--player", player, "--player-only"]) r = json.loads(capsys.readouterr().out) assert (r["charts"], r["assets"], r["removedAssets"]) == (2, 3, 1) - cli.main(["web", str(site), "--player", player, "--reingest-json"]) + cli.main(["--json", "site", "build", "-o", str(site), "--player", player, "--reingest-json"]) r = json.loads(capsys.readouterr().out) assert (r["changedEntries"], r["assets"], r["removedAssets"]) == (0, 3, 0) assert json.loads((site / "charts" / "1_easy.json").read_bytes()) == json.loads(before) - cli.main(["web", str(site), "--player", player, "--reingest-json", "--compress", "none"]) + cli.main(["--json", "site", "build", "-o", str(site), "--player", player, "--reingest-json", "--compress", "none"]) r = json.loads(capsys.readouterr().out) assert (r["changedEntries"], r["assets"], r["removedAssets"]) == (2, 3, 1) man = json.loads((site / "charts" / "1_easy.json").read_text(encoding="utf-8")) @@ -271,7 +271,7 @@ def test_player_only_on_a_new_directory(tmp_path, capsys): from nnnotes import cli out = tmp_path / "new-site" - cli.main(["web", str(out), "--player", str(fake_player(tmp_path / "p")), "--player-only"]) + cli.main(["--json", "site", "build", "-o", str(out), "--player", str(fake_player(tmp_path / "p")), "--player-only"]) r = json.loads(capsys.readouterr().out) assert (r["charts"], r["assets"], r["removedAssets"]) == (0, 0, 0) assert (out / "index.html").is_file() and (out / "charts.json").is_file() @@ -293,7 +293,10 @@ def fake_build(out, pairs, cfg, player, fmt, **kw): try: cli.main( [ - "web", + "--json", + "site", + "build", + "-o", str(tmp_path / "s"), "--player", str(fake_player(tmp_path / "p")), @@ -330,8 +333,36 @@ def fake_build(out, pairs, cfg, player, fmt, **kw): monkeypatch.setattr(web, "unknown_pairs", lambda cfg, pairs, regions=None: []) player = str(fake_player(tmp_path / "p")) for flag in ("br", "none"): - cli.main(["web", str(tmp_path / "s"), "--player", player, "--pair", "7:hard", "--compress", flag]) + cli.main( + [ + "--json", + "site", + "build", + "-o", + str(tmp_path / "s"), + "--player", + player, + "--pair", + "7:hard", + "--compress", + flag, + ] + ) assert calls == ["br", "none"] with pytest.raises(SystemExit): - cli.main(["web", str(tmp_path / "s"), "--player", player, "--pair", "7:hard", "--compress", "zstd"]) + cli.main( + [ + "--json", + "site", + "build", + "-o", + str(tmp_path / "s"), + "--player", + player, + "--pair", + "7:hard", + "--compress", + "zstd", + ] + ) capsys.readouterr() diff --git a/tests/site/test_webmodel.py b/tests/site/test_webmodel.py index e037c59..814231f 100644 --- a/tests/site/test_webmodel.py +++ b/tests/site/test_webmodel.py @@ -453,13 +453,13 @@ def run(argv, capsys): def test_web_argument_shape(): p = cli.build_parser() - a = p.parse_args(["web", "s", "--pair", "1:easy", "--live2d", "m1", "--live2d", KEY_B]) + a = p.parse_args(["--json", "site", "build", "-o", "s", "--pair", "1:easy", "--live2d", "m1", "--live2d", KEY_B]) assert a.pair == [(1, "easy")] and a.live2d == ["m1", KEY_B] and not a.all_live2d - a = p.parse_args(["web", "s", "--all", "--all-live2d", "--force"]) + a = p.parse_args(["--json", "site", "build", "-o", "s", "--all", "--all-live2d", "--force"]) assert a.all and a.all_live2d and a.live2d is None and a.force for bad in (["--live2d", "m", "--all-live2d"], ["--pair", "1:easy", "--all"], ["--all", "--player-only"]): with pytest.raises(SystemExit): - p.parse_args(["web", "s", *bad]) + p.parse_args(["--json", "site", "build", "-o", "s", *bad]) @pytest.mark.parametrize( @@ -471,7 +471,7 @@ def test_web_argument_shape(): ], ) def test_web_selection_errors(tmp_path, capsys, argv, message): - code, _, err = run(["web", str(tmp_path / "s"), *argv], capsys) + code, _, err = run(["--json", "site", "build", "-o", str(tmp_path / "s"), *argv], capsys) assert code == 2 and message in err @@ -501,7 +501,10 @@ def fake_charts(out, pairs, cfg, player, fmt, **kw): [ "--apk", str(tmp_path / "base.apk"), - "web", + "--json", + "site", + "build", + "-o", str(tmp_path / "s"), "--player", str(fake_player(tmp_path / "p")), @@ -531,7 +534,10 @@ def test_web_unknown_model_is_a_usage_error(tmp_path, capsys, monkeypatch): [ "--apk", str(tmp_path / "base.apk"), - "web", + "--json", + "site", + "build", + "-o", str(tmp_path / "s"), "--player", str(fake_player(tmp_path / "p")), @@ -916,7 +922,7 @@ def fake_build(cat, md, pl, adv_id, out_dir, *, models, **kw): md = tmp_path / "master" md.mkdir() (md / "MasterAdv.json").write_text(json.dumps({"_allData": [{"_id": 1}]}), encoding="utf-8") - base = ["--master", str(md), "--language", "ja", "story", "1"] + base = ["--json", "--master", str(md), "--language", "ja", "export", "story", "1"] out = tmp_path / "out" code, o, err = run([*base, "-o", str(out / "s1")], capsys) assert code == 0, err diff --git a/tests/source/test_catalog_lazy.py b/tests/source/test_catalog_lazy.py index 36a3ba9..60aec10 100644 --- a/tests/source/test_catalog_lazy.py +++ b/tests/source/test_catalog_lazy.py @@ -193,11 +193,11 @@ def test_pull_of_cached_bundles_needs_no_region_cdn_or_key(tmp_path, capsys, mon monkeypatch.setenv("NNNOTES_BUNDLE_KEY", synth.BUNDLE_KEY.hex()) monkeypatch.setenv("NNNOTES_BUNDLE_NONCE_SEED", synth.BUNDLE_SEED.hex()) monkeypatch.setenv("NNNOTES_SERVERS_ZZ_CDN", cdn_tree(tmp_path / "cdn")) - code, out, err = run(base + ["--region", "zz", "pull", "Char/A"], capsys) + code, out, err = run(base + ["--region", "zz", "bundle", "pull", "Char/A"], capsys) assert code == 0, err for name in ("NNNOTES_BUNDLE_KEY", "NNNOTES_BUNDLE_NONCE_SEED", "NNNOTES_SERVERS_ZZ_CDN"): monkeypatch.delenv(name) - code, again, err = run(base + ["pull", "Char/A", "Char/B"], capsys) + code, again, err = run(base + ["bundle", "pull", "Char/A", "Char/B"], capsys) assert code == 0, err assert again.splitlines() == out.splitlines() * 2 @@ -207,7 +207,8 @@ def test_pull_that_downloads_names_the_missing_setting_in_one_line(tmp_path, cap catbin.write_bytes(synth.CatalogWriter().build(ENTRIES)) monkeypatch.setenv("NNNOTES_SERVERS_ZZ_CDN", cdn_tree(tmp_path / "cdn")) code, out, err = run( - ["--catalog", str(catbin), "--cache", str(tmp_path / "cache"), "--region", "zz", "pull", "Char/A"], capsys + ["--catalog", str(catbin), "--cache", str(tmp_path / "cache"), "--region", "zz", "bundle", "pull", "Char/A"], + capsys, ) assert code == 2 and out == "" and len(err.strip().splitlines()) == 1 assert err.startswith("nnnotes: a_01.bundle is not in the cache: setting bundle.key") diff --git a/tests/source/test_catalog_versions.py b/tests/source/test_catalog_versions.py index 8ce0b59..aebfcc4 100644 --- a/tests/source/test_catalog_versions.py +++ b/tests/source/test_catalog_versions.py @@ -10,7 +10,7 @@ import pytest from nnnotes import cli -from nnnotes.cli import browse +from nnnotes.cli import browse, common from nnnotes.musicdata import deckdata from nnnotes.pipeline import catalogdb from nnnotes.source import catalog, configfile, gameapi @@ -211,7 +211,7 @@ def fetch(cdn, language, *, version): monkeypatch.setattr(Session, "master_version", observe) monkeypatch.setattr(catalogdb, "fetch", fetch) - cli.main(["--region", "tw", "--language", "en", "--cache", str(tmp_path / "cache"), "catalogs", "fetch"]) + cli.main(["--region", "tw", "--language", "en", "--cache", str(tmp_path / "cache"), "catalog", "fetch"]) assert events == ["version", "1.0.0.201"] version = catalogdb.CatalogDB(tmp_path / "cache" / "store").versions()[0] assert version["resourceVersion"] == "1.0.0.201" @@ -240,13 +240,14 @@ def test_browser_uses_the_same_selector(tmp_path, monkeypatch): assert len(calls) == 1 and calls[0].endswith("catalog_1.0.0.201_en.bin") -def test_catalog_release_flag_is_distinct_from_stored_catalog_version(tmp_path): - args = cli.build_parser().parse_args( +def test_catalog_version_flag_is_distinct_from_the_stored_catalog(tmp_path): + args = cli.parse_args( [ - "--catalog-release", + "--catalog-version", "1.0.0.201", "export", - "--catalog-version", + "assets", + "--stored-catalog", "stored-label", "--select", "key:Example", @@ -254,5 +255,5 @@ def test_catalog_release_flag_is_distinct_from_stored_catalog_version(tmp_path): str(tmp_path / "out"), ] ) - assert args.catalog_release == "1.0.0.201" and args.catalog_version == "stored-label" - assert Session(cli.load_config(args)).catalog_version() == "1.0.0.201" + assert args.catalog_version == "1.0.0.201" and args.stored_catalog == "stored-label" + assert Session(common.load_config(args)).catalog_version() == "1.0.0.201" diff --git a/tests/source/test_configfile.py b/tests/source/test_configfile.py index 79c5b12..b77900e 100644 --- a/tests/source/test_configfile.py +++ b/tests/source/test_configfile.py @@ -8,6 +8,7 @@ import pytest from nnnotes import cli +from nnnotes.cli import common from nnnotes.source import configfile from nnnotes.source.config import ConfigError, template @@ -232,7 +233,7 @@ def test_check_does_not_turn_jp_language_defaults_into_flags(tmp_path, capsys): settings = {s["name"]: s for s in json.loads(out)["settings"]} assert settings["catalog.region"]["origin"] == "file" assert settings["catalog.language"]["status"] == "unset" - cfg = cli.load_config(cli.build_parser().parse_args(["--config", str(conf), "master", "version"])) + cfg = common.load_config(cli.parse_args(["--config", str(conf), "master", "version"])) assert cfg.require("catalog", "language") == "ja" # normal commands still apply the JP default diff --git a/tests/source/test_gameapi.py b/tests/source/test_gameapi.py index de0349a..823058d 100644 --- a/tests/source/test_gameapi.py +++ b/tests/source/test_gameapi.py @@ -54,7 +54,7 @@ def server_info(name: str, cdn: str, api: str, area: str = "", display: str = "" def run(argv, capsys): code = 0 try: - cli.main(argv) + cli.main(["--json", *argv]) except SystemExit as e: code = e.code if isinstance(e.code, int) else 1 out, err = capsys.readouterr() diff --git a/tests/source/test_jp.py b/tests/source/test_jp.py index 0d96c92..f318fb2 100644 --- a/tests/source/test_jp.py +++ b/tests/source/test_jp.py @@ -12,7 +12,7 @@ import pytest from nnnotes import cli -from nnnotes.cli.cli_assets import CatalogFetcher +from nnnotes.cli.assets import CatalogFetcher from nnnotes.musicdata import deckdata from nnnotes.pipeline import catalogdb from nnnotes.source import addressables, configfile, gameapi, jp @@ -289,7 +289,7 @@ def test_apk_update_with_same_asset_snapshot_does_not_reuse_embedded_cache(tmp_p cache = tmp_path / "cache" db = catalogdb.CatalogDB(tmp_path / "store") cfg = Config({"catalog": {"region": "jp"}, "paths": {"apk": str(apk)}}, environ={}) - from nnnotes.cli.cli_assets import Workspace + from nnnotes.cli.assets import Workspace for revision in (1, 2): local = synth.CatalogWriter().build( @@ -459,7 +459,9 @@ def build(out, selection, cfg, *args, **kwargs): monkeypatch.setattr(webmodel, "build", build) monkeypatch.setattr(storysite, "build", build) - cli.main(["--config", str(conf), "web", str(tmp_path / "site"), kind, "--region", region]) + cli.main( + ["--config", str(conf), "--json", "site", "build", "-o", str(tmp_path / "site"), kind, "--site-region", region] + ) assert seen == [(region, tmp_path / "target.apk")] @@ -521,7 +523,7 @@ def test_jp_master_rejects_missing_or_malformed_file_lists(tmp_path, files): def test_export_cache_locator_uses_source_namespace(tmp_path): - from nnnotes.cli.cli_assets import Workspace + from nnnotes.cli.assets import Workspace source = jp.Source("jp", "1.0.0.300", HASH, "https://cdn.invalid") workspace = Workspace.__new__(Workspace) diff --git a/tests/unity/test_typetrees.py b/tests/unity/test_typetrees.py index 7a84d71..e62edc1 100644 --- a/tests/unity/test_typetrees.py +++ b/tests/unity/test_typetrees.py @@ -356,7 +356,7 @@ def run(argv, capsys): def test_dummy_dll_flag_is_gone(capsys, tmp_path): - code, _, _ = run(["--dummy-dll", str(tmp_path), "player", "-o", str(tmp_path / "p.json")], capsys) + code, _, _ = run(["--dummy-dll", str(tmp_path), "apk", "settings", "-o", str(tmp_path / "p.json")], capsys) assert code == 2 code, out, _ = run(["--help"], capsys) assert code == 0 and "dummy" not in out.lower() @@ -365,7 +365,7 @@ def test_dummy_dll_flag_is_gone(capsys, tmp_path): def test_leftover_dummy_dll_key_is_ignored(capsys, tmp_path): conf = tmp_path / "c.toml" conf.write_text('[paths]\ndummy_dll = "somewhere"\n', encoding="utf-8") - code, _, err = run(["--config", str(conf), "player", "-o", str(tmp_path / "p.json")], capsys) + code, _, err = run(["--config", str(conf), "apk", "settings", "-o", str(tmp_path / "p.json")], capsys) assert code == 2 and "paths.apk" in err and "dummy" not in err