diff --git a/README.en.md b/README.en.md index 0a723ac..951b73b 100644 --- a/README.en.md +++ b/README.en.md @@ -4,13 +4,29 @@ nnnotes is an offline data toolkit for the game files of BanG Dream! Our Notes: it reads Addressables catalogs, downloads and decrypts asset bundles, downloads and decodes master data, and exports stories, Live2D models, spots, -shaders, CRI audio and live charts as structured JSON and common file formats. The chart export is the data that -[ournotes-player](https://github.com/empty-sekai/ournotes-player) reads. The naming was inspired by [mos9527/sssekai](https://github.com/mos9527/sssekai). +shaders, CRI audio and live charts as structured JSON and common file formats. The charts, Live2D models and stories `web` +exports are the data that [ournotes-player](https://github.com/empty-sekai/ournotes-player) reads. The naming was inspired by [mos9527/sssekai](https://github.com/mos9527/sssekai). This is an unofficial fan project, not affiliated with the game's developer or operator. The repository contains no game assets, keys or server addresses: users supply the game files, the keys needed for decryption and the server addresses in their own configuration, and the exports stay in local directories the user chooses. +## Install and set up + +```sh +pip install nnnotes # Python 3.11+, 3.13 recommended +nnnotes config init --user # asks for each setting in the terminal (keys without echo), writes the per-user config file read from any directory +nnnotes config check # each setting's origin and whether it is valid, without its value +``` + +- The values (keys, the CDN and API roots of the regions, the paths of your own data and tools) come from the game + client you own; see Configuration below. +- Scripts and AI agents get no questions: `nnnotes config init --user --no-input --set SECTION.KEY=VALUE ...` writes + the file at once, `nnnotes config set SECTION.KEY VALUE` changes one value (VALUE `-` reads it from standard input, + so a key need not appear on the command line), and `nnnotes config check --json` reports the states for machines. +- `--fonts game` also needs `pip install 'nnnotes[fonts]'`; the APK, the master data and the external tools are listed + under Requirements below. + ## Features | Command | Input | Output | @@ -23,7 +39,7 @@ addresses in their own configuration, and the exports stay in local directories | `master download` | master data version, or `--latest` (the region's current one) | that version's `MasterManifest.json` and every `.bin` file (SHA-256 checked) | | `master decode` | master data `.bin` files or directories | one JSON per table (Rijndael-256 CBC decryption + gzip) | | `adv` | episode ID | `episode.json`: command list, lines in five languages, voice / sound / video index | -| `story` | episode ID | a full story directory: episode, every Live2D model, audio, stage scene and shaders, story UI, frames / particle effects / post effects / stills / talk windows / chat assets, videos (WebM) | +| `story` | episode ID | a story directory: episode, audio, stage scene and shaders, story UI, frames / particle effects / post effects / stills / talk windows / chat assets, videos (WebM); the Live2D models it uses are exported into a models directory (`--models`, default `live2d/` next to the story directory), where other stories reuse them | | `live2d` | model key or model id | a Live2D (Cubism) runtime directory: moc3, textures, motion3, physics, expressions, prefab parameters | | `spot` | spot ID | `spot.json` + Spine characters + the room as `room.glb` + shaders | | `room` | background prefab key | the room model (binary glTF) | @@ -32,7 +48,7 @@ addresses in their own configuration, and the exports stay in local directories | `crikey` | APK | the CRI HCA keycode from the game's boot data (shows whether it was found; can write a `.hcakey`) | | `player` | APK | render-related global settings (color space, quality levels, renderers) as JSON | | `live` | music ID + difficulty | a full chart directory: chart and runtime notes, 3D scene, note and effect assets, BGM and sounds, sound routing | -| `web` | `--pair music:difficulty` (repeatable) or `--all`; `--live2d model` (repeatable) or `--all-live2d`; `--story episode` (repeatable) or `--all-stories`; `--region region` (repeatable) or `--all-regions` | an ournotes-player static site: shared player + per-chart / per-model / per-episode manifests + content-addressed assets; one site can serve several regions, with listing texts in five languages; a story's interface texts are grouped by language, with TextMeshPro font assets generated from open fonts (the game's fonts with `--fonts game`) | +| `web` | `--pair music:difficulty` (repeatable) or `--all`; `--live2d model` (repeatable) or `--all-live2d`; `--story episode` (repeatable) or `--all-stories`; `--region region` (repeatable) or `--all-regions` | an ournotes-player static site: shared player + per-chart / per-model / per-episode manifests + content-addressed assets; the Live2D models of the stories are built first as models, listed in `models.json`, and the story manifests reference them; one site can serve several regions, with listing texts in five languages; a story's interface texts are grouped by language, with TextMeshPro font assets generated from open fonts (the game's fonts with `--fonts game`); compressible assets (JSON, shaders, moc3, ...) are stored gzip encoded by default (`--compress br` / `none`) | | `deck-data` | master data files (`--master-files` directory or `--apk-master`) | one JSON file for deck-building tools: every chart's runtime notes, skill events and fever ranges, and the master data tables about cards, skills, bonuses, scores and events ([format](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/deck-data.md)) | Export conventions: @@ -69,7 +85,7 @@ Based on all data of the Taiwan server, version 1.0.1 (zh-Hant): ## Requirements - Python 3.11+, 3.13 recommended (a `web` build reads and writes a lot of JSON, and the standard library encodes - JSON faster on 3.13); `pip install nnnotes` (for development `pip install -e .` in the repository) + JSON faster on 3.13); for development `pip install -e .` in the repository - the game's `base.apk`: APK-local bundles, the CRI keycode, boot settings. `player`, `story`, `live` and `web` read MonoBehaviours of its boot data with type trees that ship with nnnotes; supported now: game version 1.0.1 (Unity 6000.3.12f1). With an APK of another version whose classes do not match, these commands stop with an @@ -85,12 +101,17 @@ Based on all data of the Taiwan server, version 1.0.1 (zh-Hant): The code contains no keys, server addresses or default paths. Settings are read in this order, later sources overriding earlier ones: -1. config file: `--config `, else `NNNOTES_CONFIG`, else `nnnotes.toml` in the working directory +1. config file: `--config `, else `NNNOTES_CONFIG`, else `nnnotes.toml` in the working directory, else the + per-user file (`%APPDATA%\nnnotes\nnnotes.toml` on Windows, `~/.config/nnnotes/nnnotes.toml` elsewhere; + `nnnotes config path` lists the lookup order and the file read) 2. environment variables: `NNNOTES_
_` (e.g. `NNNOTES_BUNDLE_KEY`, `NNNOTES_SERVERS_TW_CDN`) 3. command-line flags: `--region`, `--language`, `--catalog`, `--cache`, `--master`, `--apk`, `--ffmpeg`, `--vgmstream`, `--node` go before the command name; `--player` is an option of `web` -Copy [`nnnotes.example.toml`](https://github.com/MetaSekaiLab/nnnotes/blob/main/nnnotes.example.toml) to `nnnotes.toml` and fill it in. The settings are: the bundle +`nnnotes config init` writes the config template (every setting empty; it ships with the package, in the repository +[`src/nnnotes/nnnotes.example.toml`](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/nnnotes.example.toml)), +asking for each value in a terminal; `config set` / `config unset` change one value and keep the file's comments. +The settings are: the bundle key and nonce seed, the master data key and IV, the region in use (`[catalog] region`) and the catalog language (`[catalog] language`), the CDN base and API root of each region (`cdn`, `api` of `[servers.]`), the client version (`[client] version`; unset: the APK's versionName), optionally the bootstrap API root (`[bootstrap] api`, for diff --git a/README.md b/README.md index 47165b2..35beea1 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,22 @@ [简体中文](README.md) | [English](README.en.md) -nnnotes 是 BanG Dream! Our Notes 游戏文件的离线数据工具包：读取 Addressables catalog、下载并解密资源包、下载与解码 masterdata，再把剧情、Live2D、据点场景、着色器、CRI 音频和谱面导出成结构化的 JSON 与通用格式文件。谱面导出结果就是 [ournotes-player](https://github.com/empty-sekai/ournotes-player) 读取的数据。命名灵感来自 [mos9527/sssekai](https://github.com/mos9527/sssekai)。 +nnnotes 是 BanG Dream! Our Notes 游戏文件的离线数据工具包：读取 Addressables catalog、下载并解密资源包、下载与解码 masterdata，再把剧情、Live2D、据点场景、着色器、CRI 音频和谱面导出成结构化的 JSON 与通用格式文件。`web` 导出的谱面、Live2D 模型与剧情站点就是 [ournotes-player](https://github.com/empty-sekai/ournotes-player) 读取的数据。命名灵感来自 [mos9527/sssekai](https://github.com/mos9527/sssekai)。 本项目为非官方爱好者项目，与游戏的开发和运营方无关。仓库不包含任何游戏资源、密钥或服务器地址：游戏文件、解密所需的密钥和服务器地址都由使用者在配置中自行提供，导出结果只保存在使用者指定的本地目录。 +## 安装与初始化 + +```sh +pip install nnnotes # Python 3.11+，推荐 3.13 +nnnotes config init --user # 在终端里逐项填写设置（密钥不回显），写入用户配置文件，在任何目录下都会读取 +nnnotes config check # 每项设置的来源和格式是否有效，不显示设置值 +``` + +- 设置值（密钥、区服的 CDN 与 API 地址、本地数据与工具的路径）都来自使用者自己的游戏客户端，见[配置](#配置)。 +- 在脚本或 AI 代理中不会有交互提问：`nnnotes config init --user --no-input --set 段.键=值 ...` 一次写好，之后用 `nnnotes config set 段.键 值` 修改单项（值写 `-` 时从标准输入读取，密钥不必出现在命令行里），`nnnotes config check --json` 给出机器可读的状态。 +- `--fonts game` 另需 `pip install 'nnnotes[fonts]'`；APK、masterdata 与外部工具见[需要准备](#需要准备)。 + ## 功能 | 命令 | 输入 | 输出 | @@ -18,7 +30,7 @@ nnnotes 是 BanG Dream! Our Notes 游戏文件的离线数据工具包：读取 | `master download` | masterdata 版本号，或 `--latest`（区服当前版本） | 该版本的 `MasterManifest.json` 与全部 `.bin` 文件（SHA-256 校验） | | `master decode` | masterdata `.bin` 文件或目录 | 每张表一个 JSON（Rijndael-256 CBC 解密 + gzip 解压） | | `adv` | 剧情 ID | `episode.json`：命令表、五语台词、语音 / 音效 / 视频索引 | -| `story` | 剧情 ID | 完整剧情目录：episode、全部 Live2D 模型、音频、舞台场景与着色器、剧情 UI、Frame / 粒子特效 / 后处理 / 静帧 / 对话框 / 聊天资源、视频（WebM） | +| `story` | 剧情 ID | 剧情目录：episode、音频、舞台场景与着色器、剧情 UI、Frame / 粒子特效 / 后处理 / 静帧 / 对话框 / 聊天资源、视频（WebM）；用到的 Live2D 模型导出到模型目录（`--models`，默认为剧情目录旁的 `live2d/`），已导出的模型供其他剧情复用 | | `live2d` | 模型资源键或模型 ID | Live2D（Cubism）运行时目录：moc3、贴图、motion3、物理、表情、预制体参数 | | `spot` | 据点 ID | `spot.json` + Spine 角色 + 房间 `room.glb` + 着色器 | | `room` | 背景预制体键 | 房间模型（binary glTF） | @@ -27,7 +39,7 @@ nnnotes 是 BanG Dream! Our Notes 游戏文件的离线数据工具包：读取 | `crikey` | APK | 读出游戏启动数据中的 CRI HCA 解码密钥（只显示是否找到，可写成 `.hcakey`） | | `player` | APK | 渲染相关的全局设置（色彩空间、画质等级、渲染器）JSON | | `live` | 曲目 ID + 难度 | 完整谱面目录：谱面与运行时音符、3D 场景、音符与特效资源、BGM 与音效、声音路由 | -| `web` | `--pair 曲目:难度`（可重复）或 `--all`；`--live2d 模型`（可重复）或 `--all-live2d`；`--story 剧情 ID`（可重复）或 `--all-stories`；`--region 区服`（可重复）或 `--all-regions` | ournotes-player 静态站点：共享播放器 + 每谱 / 每模型 / 每集剧情清单 + 内容寻址资源；一个站点可服务多个区服，列表文本含五种语言；剧情的界面文字按语言分组，字形由开源字体生成 TextMeshPro 字体资源（`--fonts game` 时用游戏字体） | +| `web` | `--pair 曲目:难度`（可重复）或 `--all`；`--live2d 模型`（可重复）或 `--all-live2d`；`--story 剧情 ID`（可重复）或 `--all-stories`；`--region 区服`（可重复）或 `--all-regions` | ournotes-player 静态站点：共享播放器 + 每谱 / 每模型 / 每集剧情清单 + 内容寻址资源；剧情用到的 Live2D 模型先按模型构建并列入 `models.json`，剧情清单引用它们；一个站点可服务多个区服，列表文本含五种语言；剧情的界面文字按语言分组，字形由开源字体生成 TextMeshPro 字体资源（`--fonts game` 时用游戏字体）；可压缩的资源（JSON、着色器、moc3 等）默认以 gzip 存储，`--compress br` / `none` 可改 | | `deck-data` | masterdata 文件（`--master-files` 目录或 `--apk-master`） | 供组卡工具读取的单个 JSON：每张谱面的运行时音符、技能事件与 fever 区间，以及卡牌、技能、加成、分数与活动相关的 masterdata 表（[格式](docs/deck-data.md)） | 导出约定： @@ -57,7 +69,7 @@ nnnotes 是 BanG Dream! Our Notes 游戏文件的离线数据工具包：读取 ## 需要准备 -- Python 3.11+，推荐 3.13（`web` 构建读写大量 JSON，3.13 的标准库 JSON 编码更快），`pip install nnnotes`（开发时在仓库内 `pip install -e .`） +- Python 3.11+，推荐 3.13（`web` 构建读写大量 JSON，3.13 的标准库 JSON 编码更快）；开发时在仓库内 `pip install -e .` - 游戏安装包 `base.apk`：APK 内置资源包、CRI 解码密钥、启动设置。`player`、`story`、`live`、`web` 用 nnnotes 自带的类型树读取 APK 启动数据中的 MonoBehaviour，目前支持游戏版本 1.0.1（Unity 6000.3.12f1）；其他版本的 APK 若类型不符，这些命令会报错并给出类名、游戏版本和 Unity 版本 - 解码后的 masterdata 目录（可用 `master download` + `master decode` 生成）：`adv`、`story`、`spot`、`live` 和 `web` 的谱面需要；`web` 的 Live2D 模型只用它取角色名（可选） - 外部工具：[vgmstream](https://vgmstream.org/)（CRI HCA 解码）、[FFmpeg](https://ffmpeg.org/)（转码，含剧情视频的 WebM 封装与 Opus 音频）；`web` 另需 Node.js 20+ 与构建好的 ournotes-player @@ -66,11 +78,11 @@ nnnotes 是 BanG Dream! Our Notes 游戏文件的离线数据工具包：读取 代码中没有任何密钥、服务器地址和默认路径。所有设置按以下顺序读取，后者覆盖前者： -1. 配置文件：`--config <文件>`，否则 `NNNOTES_CONFIG`，否则当前目录的 `nnnotes.toml` +1. 配置文件：`--config <文件>`，否则 `NNNOTES_CONFIG`，否则当前目录的 `nnnotes.toml`，否则用户配置文件（Windows 为 `%APPDATA%\nnnotes\nnnotes.toml`，其他系统为 `~/.config/nnnotes/nnnotes.toml`；`nnnotes config path` 列出查找顺序与实际读取的文件） 2. 环境变量：`NNNOTES_<节>_<键>`（如 `NNNOTES_BUNDLE_KEY`、`NNNOTES_SERVERS_TW_CDN`） 3. 命令行参数：`--region`、`--language`、`--catalog`、`--cache`、`--master`、`--apk`、`--ffmpeg`、`--vgmstream`、`--node` 写在命令名之前；`--player` 是 `web` 的参数 -复制 [`nnnotes.example.toml`](nnnotes.example.toml) 为 `nnnotes.toml` 后填写。需要的设置包括：资源包解密密钥与 nonce 种子、masterdata 的密钥与 IV、使用的区服 `[catalog] region` 与 catalog 语言 `[catalog] language`、每个区服的 CDN 地址与 API 根地址（`[servers.<区服>]` 的 `cdn`、`api`）、客户端版本 `[client] version`（未设置时读取 APK 的 versionName）、可选的引导 API 根地址 `[bootstrap] api`（`servers` 使用），以及缓存目录、APK、masterdata 目录、ournotes-player 和 vgmstream / FFmpeg / Node.js 的路径（三个工具未设置时在 `PATH` 中查找）。这些值都来自使用者自己的游戏客户端。 +`nnnotes config init` 写出配置模板（每项设置为空，模板随包安装，仓库中为 [`src/nnnotes/nnnotes.example.toml`](src/nnnotes/nnnotes.example.toml)），在终端中会逐项询问；`config set` / `config unset` 修改单项并保留文件中的注释。需要的设置包括：资源包解密密钥与 nonce 种子、masterdata 的密钥与 IV、使用的区服 `[catalog] region` 与 catalog 语言 `[catalog] language`、每个区服的 CDN 地址与 API 根地址（`[servers.<区服>]` 的 `cdn`、`api`）、客户端版本 `[client] version`（未设置时读取 APK 的 versionName）、可选的引导 API 根地址 `[bootstrap] api`（`servers` 使用），以及缓存目录、APK、masterdata 目录、ournotes-player 和 vgmstream / FFmpeg / Node.js 的路径（三个工具未设置时在 `PATH` 中查找）。这些值都来自使用者自己的游戏客户端。 缺少或格式错误的设置会让命令以退出码 2 结束，并用一行说明对应的配置键、环境变量和命令行参数，不会输出任何设置值。`nnnotes.toml` 已在 `.gitignore` 中，请勿提交。 diff --git a/docs/commands.md b/docs/commands.md index 7c28fdd..c489b58 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -23,6 +23,20 @@ thread per CPU in every process, and no command uses its parallelism. A key, id or model the catalog or the master data does not have is a usage error: the command stops with exit status 2 and a line naming it (`nnnotes : error: ...`), before any bundle is fetched or file written. +## config + +``` +nnnotes config init [--user | --file F] [--set SECTION.KEY=VALUE ...] [--no-input] [--force] +nnnotes config set SECTION.KEY VALUE [--user | --file F] +nnnotes config unset SECTION.KEY [--user | --file F] +nnnotes config check [--json] +nnnotes config path [--json] +``` + +Write the config file from the template (asking in a terminal, else from `--set`), set or empty one value in place, +check every setting's origin and format, and list where the config file is looked up. No `config` command prints a +setting's value. Details in [configuration.md](configuration.md#writing-the-config-file). + ## Cache `pull` and every extractor read bundles through the cache (`[paths] cache`): @@ -132,14 +146,17 @@ row of a Movie / Clip row's video id. ## story ``` -nnnotes story ADV_ID -o OUT [--format flac|ogg|wav] [--flac-level N] [--no-audio] [--fonts open|game] +nnnotes story ADV_ID -o OUT [--models MODELS] [--force] [--format flac|ogg|wav] [--flac-level N] [--no-audio] + [--fonts open|game] ``` -One ADV episode as a self-contained directory: +One ADV episode as a directory, and its Live2D models in a models directory: ``` +MODELS// every Live2D model of the episode (MODELS: `--models`, default OUT/../live2d), the files + of a site's model manifest (see `web`): model.json, moc3, prefab, the atlas pages and the + shader programs its drawables use OUT/episode.json as `adv` -OUT/live2d// every Live2D model of the episode: moc3, atlas pages, full prefab OUT/audio// every cue sheet of the episode, one file per cue + cues.json, streams.json OUT/scene.json player graphics, cameras, ADV fields, volumes, settings and stages OUT/frames.json Frame prefabs by asset name: uGUI, Animator controllers and clips, UI particle systems @@ -157,12 +174,18 @@ OUT/crilips/ crilips.json + crilips.bin: the CRI Lips analysis data OUT/story.json index of the above (a file the episode does not need is null and not written) ``` +story.json `models` maps the key of each model to its id (`` of `Character/Live2D///model/`) +and `modelsDir` is the path from OUT to MODELS. A model directory that exists is used as it is, so the stories of one +models directory export each model once; `--force` exports the episode's models again. The printed summary lists +the models exported (`modelsBuilt`) and those used as they were (`modelsSkipped`). + `--format` (default `flac`) is the audio format and `--flac-level` its FLAC compression level (as for `audio`); `--no-audio` leaves the cue sheets undecoded (no `audio/`, `audio` in story.json is empty). With audio, `crilips/` holds the weights of the game's CRI Lips mouth analysis, read from the CRI Lips library in `[paths] apk` (or in the arm64-v8a split next to it, `split_config.arm64_v8a.apk`), when a voice -reaches that analysis: a lip-synced voice while a model has no MotionSync controller, or a voice that other speakers -follow (story.json `crilips`; ournotes-player docs/crilips.md). Videos keep the VP9 stream of the game's USM file and carry its ADX audio as Opus (FFmpeg, +reaches that analysis: a lip-synced voice while a model has no MotionSync controller (model.json `motionSync`), or a +voice that other speakers follow (story.json `crilips`; ournotes-player docs/crilips.md). Videos keep the VP9 stream +of the game's USM file and carry its ADX audio as Opus (FFmpeg, `[paths] ffmpeg`); the USM streams are unmasked with the CRI key read from `[paths] apk`. TextMesh Pro text in frames, talk windows and chat keeps its layout and style; with `--fonts open` its font and sprite assets and text materials are only named (no atlas is written), with `--fonts game` they are exported. The command stops before @@ -282,6 +305,11 @@ textures embedded, materials translated from the shader's render state. Objects `extras.unityActive = false`. A summary (`meshCount`, materials, textures, samplers) is written next to it as `OUT.json`. +Materials and textures are told apart by serialized file and path id (the prefab's bundle and its dependencies can +reuse a path id). Each texture is one glTF texture with its own sampler, and textures whose PNG bytes are equal share +one glTF image. The summary's `textures` has one entry per glTF texture, in glTF order (the index a material's +`baseColorTexture` names): the Unity texture's name. + The render state is that of the shader's first pass for the material: blend factors (colour and alpha), blend operations, colour mask, culling, depth write, depth test and alpha to mask. A state the shader takes from a property (`Blend [_SrcBlend] [_DstBlend]`, `Cull [_Cull]`, `ZWrite [_ZWrite]`, ...) is the material's value of that @@ -442,7 +470,8 @@ value is always included; a value the master data does not have is a usage error nnnotes web SITE [--pair MUSIC_ID:DIFFICULTY [--pair ...] | --all] [--live2d MODEL [--live2d ...] | --all-live2d] [--story ADV_ID [--story ...] | --all-stories] [--story-languages LANG,...] [--font LANG=PATH ...] nnnotes web SITE (--player-only | --reingest-json) - [--player DIR] [--format aac|opus|vorbis|mp3|flac] [--no-audio] [--force] + [--player DIR] [--format aac|opus|vorbis|mp3|flac] [--no-audio] [--compress gzip|br|none] + [--force] [--tmp DIR] [--workers N] [--read-workers N] [--band BAND | --leader-card CARD_ID] [--region REGION [--region ...] | --all-regions] [--fonts open|game] [--live-option OPTION[=VALUES] ...] @@ -456,8 +485,9 @@ SITE/index.html, chart-list.js ... the player's chart list page; ?music=< SITE/ournotes-player.element.min.js the player's built bundle (and its source map) SITE/charts.json chart index: listing facts (texts in every language), regions, manifest path, sizes -SITE/charts/_.json chart manifest: every path the player reads -> {asset, size}, or - {parts: [[key, asset, size], ...], size} for a large JSON object +SITE/charts/_.json chart manifest: every path the player reads -> {asset, size[, stored]}, + or {parts: [[key, asset, size[, stored]], ...], size} for a large JSON + object SITE/charts//.json the manifest of a region whose chart files differ (see Regions) SITE/models.json Live2D model index: id, key, group, canvas, manifest path, sizes; with master data the character and its names @@ -466,10 +496,12 @@ SITE/live2d/ the player's Live2D model page and its SITE/stories.json story index: titles and story groups in every language, commands, languages, manifest path, sizes, regions SITE/stories/.json story manifest: the common files and one file group per language, the - same entry forms as a chart manifest + same entry forms as a chart manifest; the model manifests of its models + and the path to the site root SITE/stories//.json the manifest of a region whose story files differ (see Regions) SITE/story/ the player's story page and its bundle (when the player has them) -SITE/assets/. content-addressed files shared by all charts, models and stories +SITE/assets/.[.gz|.br] content-addressed files shared by all charts, models and stories + (see --compress) ``` - `--pair` (repeatable; `:` or `_`) adds the given charts (a chart that @@ -477,15 +509,20 @@ SITE/assets/. content-addressed files shared by all difficulty that has a `MasterLiveMusicScore` row (in the master data of any of the site's regions). - `--live2d` (repeatable; a model id `` or key `Character/Live2D///model/`) adds the given Live2D models; `--all-live2d` adds every model key of the catalog. A model's id is its ``. Its manifest lists - the files the player's Live2D viewer reads: `model.json` (index: moc3, prefab, textures, shader index and the Cubism - mask materials), the moc3 and prefab as `live2d` writes them, the atlas pages the drawables use, and the GLSL ES 3.00 - programs of the Live2D shaders that the drawables' materials select (the mask shader only for a model with masked - drawables). Charts and models can be added in one run; models are built first. + the files the player's Live2D viewer and story page read: `model.json` (index: moc3, prefab, textures, shader index, + the Cubism mask materials and `motionSync`, whether the prefab's root has the MotionSync controller with its CRI + audio input), the moc3 and prefab as `live2d` writes them, the atlas pages the drawables use, and the GLSL ES 3.00 + programs of the Live2D shaders that the drawables' materials select, each keyword set also with + `_ADDITIONAL_LIGHTS_VERTEX` (the story player adds it at quality 4), and the mask shader's for a model with masked + drawables. Charts and models can be added in one run; models are built first. - `--story` (repeatable; a `MasterAdv` id) adds the given story episodes (an id no region of the site has is a usage error); `--all-stories` adds every `MasterAdv` episode. See [Stories](#stories) below. Stories are built after - models and charts. -- A chart, model or story whose manifest exists is skipped unless `--force`. Assets no chart, no model and no story - (common or language file) references are removed. + models and charts; the Live2D models the stories to build use are built before them, as with `--live2d` (listed in + `models.json`, names from the master data as for `--live2d`). +- A chart, model or story whose manifest exists is skipped unless `--force` (with stories, their models too). A + model manifest whose `model.json` is of an older format (without `motionSync`) is outdated and built again; the + summary lists such models in `modelsRebuilt` (`storyModels.modelsRebuilt` for the models of stories). Assets no + chart, no model and no story (common or language file, or model manifest it names) references are removed. - `--player-only` rewrites the player files, `charts.json`, `models.json` and `stories.json` only; `--reingest-json` stores every chart's and model's JSON files again from the site's own assets (then rewrites the player files and the indexes; story manifests are left as they are). @@ -495,6 +532,17 @@ SITE/assets/. content-addressed files shared by all `examples/live2d/index.html`, that page and `dist/ournotes-player.live2d.element.min.js` go to `SITE/live2d/`; when it has `examples/story-list/index.html` and `dist/ournotes-player.story.element.min.js`, they go to `SITE/story/` (a build that adds stories stops when the page is there without its bundle). +- `--compress` (default `gzip`): how the compressible assets are stored: JSON files (split parts included), GLSL, + moc3, atlas, skel, bin, wav and glb files. With `gzip` or `br` (brotli, quality 11) such a file is stored encoded + when that makes it smaller: `assets/..gz` (or `.br`), named by the SHA-256 of the decoded bytes; its + entry's `size` is the decoded size and `stored` the size of the stored file (a part: `[key, asset, size, stored]`). + Other files, and every file with `none`, are stored as they are, without `stored`. `--reingest-json` stores the + JSON files again with the given `--compress`. +- Hosting: the player decodes a `.gz` or `.br` asset when the bytes it receives have the entry's `stored` length and + uses them as they are when they have its `size`. A gzip site can be served as plain files; `Content-Encoding: + gzip` on its `.gz` files is optional (with it the browser decodes them, without it the player does). A brotli site + needs `Content-Encoding: br` on its `.br` files for Chromium-based browsers, which cannot decode brotli in the + page. - `--format` (default `aac`) is the BGM format of the charts and the format of every story sound (music, sound effects, voices); the charts' note SE, cheers and voices stay FLAC. `--no-audio` stores no audio (the player then runs the chart silent on its own clock, and plays stories without their sounds; story videos keep their sound @@ -535,13 +583,16 @@ SITE/assets/. content-addressed files shared by all ### Stories A story's manifest lists what the player's story page reads, in the layout of `story` (below the site's `assets/`): -`story.json`, `episode.json`, `scene.json`, the scene and model shaders (GLSL ES 3.00 programs only), textures, the -Live2D models, per cue sheet `cues.json` and the file of each cue in the `--format` (an AAC file's encoder delay in +`story.json`, `episode.json`, `scene.json`, the scene shaders (GLSL ES 3.00 programs only), textures, per cue sheet +`cues.json` and the file of each cue in the `--format` (an AAC file's encoder delay in `cues.json` as `encoderDelay`), the media files the episode uses and its videos; per language (`--story-languages`, default `ja,en,zh-Hant,zh-Hans,ko`) the story UI in that language (`ui/ui.json`), its font data (`ui/fonts.json` with glyph pages under `ui/fonts/`) and `ui/languages.json`. UI files that are equal in every language are common files. `scene.json` and `ui/ui.json` are stored split per top-level key, so the parts stories and languages share are stored -once. The format is described in the player's `docs/story-data-format.md`. +once. The Live2D models are the site's: the manifest names the model manifest of each model the story uses +(`models`, `{id: "models/.json"}`, relative to the site root; `root` is the path from the manifest to the site +root), and `stories.json` counts their files in `size.models`. The format is described in the player's +`docs/story-data-format.md`. - Story text is laid out with TextMeshPro's rules from TextMeshPro font assets holding exactly the characters the episode shows in the language (its text table, title and UI labels). `--fonts open` (default) generates them from @@ -610,9 +661,9 @@ One site serves several regions and every language: chart list page switches with `?region=&lang=`. The same inputs with the same versions of nnnotes, its libraries and tools (and the same font files) give -byte-identical outputs. Charts that fail are listed in the printed summary and in `SITE.failures.json`, models that -fail in the summary and in `SITE.model-failures.json`, stories in `SITE.story-failures.json`; the exit status is then -1. +byte-identical outputs; the encoded assets also need the same zlib and brotli versions. Charts that fail are listed +in the printed summary and in `SITE.failures.json`, models that fail in the summary and in +`SITE.model-failures.json`, stories in `SITE.story-failures.json`; the exit status is then 1. ## deck-data diff --git a/docs/configuration.md b/docs/configuration.md index 2305bfd..4b21ee1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -8,15 +8,51 @@ command that needs a setting nobody gave stops before it does any work. Lowest to highest: 1. **TOML file.** `--config `; else the file named by the environment variable `NNNOTES_CONFIG`; else - `nnnotes.toml` in the working directory, when present. A file named by `--config` or `NNNOTES_CONFIG` must exist. + `nnnotes.toml` in the working directory, when present; else the per-user file, when present: + `%APPDATA%\nnnotes\nnnotes.toml` on Windows, `$XDG_CONFIG_HOME/nnnotes/nnnotes.toml` elsewhere (`~/.config` when + `XDG_CONFIG_HOME` is unset). A file named by `--config` or `NNNOTES_CONFIG` must exist. `nnnotes config path` + lists the files in this order and the one that is read. 2. **Environment variables** `NNNOTES_
_`: the setting's dotted name upper-cased, with dots, dashes and other non-alphanumeric characters as underscores (`bundle.key` → `NNNOTES_BUNDLE_KEY`, `servers.tw.cdn` → `NNNOTES_SERVERS_TW_CDN`, `bundle.nonce_seed` → `NNNOTES_BUNDLE_NONCE_SEED`). 3. **Command-line flags**, for the settings that have one (table below). An empty value (`""`, `[]`, an empty environment variable, an empty flag) counts as unset, so a lower source still -applies. Start from [`nnnotes.example.toml`](../nnnotes.example.toml), which lists every setting with an empty value. -`nnnotes.toml` and `*.local.toml` are in `.gitignore`; do not commit a filled-in copy. +applies. The template [`nnnotes.example.toml`](../src/nnnotes/nnnotes.example.toml) ships with the package and lists +every setting with an empty value; `nnnotes config init` writes it (below). `nnnotes.toml` and `*.local.toml` are in +`.gitignore`; do not commit a filled-in copy. + +## Writing the config file + +```bash +nnnotes config init [--user | --file F] [--set SECTION.KEY=VALUE ...] [--no-input] [--force] +nnnotes config set SECTION.KEY VALUE [--user | --file F] +nnnotes config unset SECTION.KEY [--user | --file F] +nnnotes config check [--json] +nnnotes config path [--json] +``` + +- `config init` writes the template to `./nnnotes.toml` (the file `--config` or `NNNOTES_CONFIG` names, when one + does; `--user`: the per-user file; `--file`: that file). An existing file is kept unless `--force`. + - In a terminal, with no `--set` and no `--no-input`, it asks for each setting in turn. Enter leaves a setting + empty, the four keys are read without echo, and a malformed value is asked again. It asks for the region names + first, and writes one `[servers.]` table per region. + - Otherwise it asks nothing: the `--set` values are written (a list comma-separated, a path made absolute) and the + other settings stay empty. A `--set` value `-` is read from standard input, once. +- `config set` sets one value in the file the commands read (`--user` / `--file`: that file, made from the template + when it does not exist). The other lines and the comments stay as they are; a new region gets the template's region + table. VALUE `-` reads the value from standard input, without echo for the keys in a terminal. `config unset` + empties one value. Both check the value's format first and name only the setting when it is malformed. +- `config check` lists every setting: where it comes from (`file`, `env`, `flag` or `-`) and its status: `ok`, `unset`, + `invalid` (with the reason), `not found` (a path that does not exist), or `unknown` (a key of the file or an + `NNNOTES_` variable that names no setting). It exits with status 1 when a setting is invalid, not found or unknown. + `--json` prints the report as `nnnotes.config-check/1`: `file`, `settings` (name, environment variable, flag, + kind, whether it is a key, description, origin, status, reason) and `problems`. +- `config path` lists the config file candidates in lookup order and the one that is read; `--json` prints + `nnnotes.config-path/1`. + +No `config` command prints a setting's value. The config files these commands write are readable by their owner only +(mode 600 where the file system has modes). The global flags go **before** the command name: @@ -142,7 +178,8 @@ nnnotes: setting paths.apk: file not found nnnotes: ffmpeg not found on PATH: give its path as `ffmpeg` in the [paths] table of the config file, the environment variable NNNOTES_PATHS_FFMPEG or --ffmpeg ``` -An unreadable config file (not found, invalid TOML) is reported the same way. Command-line usage errors also exit +When no config file is read, the line of an unset setting ends with `(no config file was found: \`nnnotes config +init\` writes one to fill in)`. An unreadable config file (not found, invalid TOML) is reported the same way. Command-line usage errors also exit with status 2: a missing `-o`, or a key or id the data does not have (a key not in the catalog, an episode, spot or music id without its master data row, a Live2D model the catalog does not have), which the error line names: diff --git a/pyproject.toml b/pyproject.toml index 2abbf0a..001a9d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ "Pillow>=10.0", "pycryptodome>=3.20", "grpcio>=1.49.1", + "brotli>=1.1", ] classifiers = [ "Programming Language :: Python :: 3", @@ -43,7 +44,7 @@ version = { attr = "nnnotes.__version__" } where = ["src"] [tool.setuptools.package-data] -nnnotes = ["typetrees/*.json", "viewrules.json"] +nnnotes = ["typetrees/*.json", "viewrules.json", "nnnotes.example.toml"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/src/nnnotes/__init__.py b/src/nnnotes/__init__.py index cdeccee..60485bb 100644 --- a/src/nnnotes/__init__.py +++ b/src/nnnotes/__init__.py @@ -4,4 +4,4 @@ models, spots, shaders, CRI audio and live charts as JSON and common file formats. Game files, keys and server addresses are supplied by the user through the configuration; nothing of the game is included. """ -__version__ = "0.1.0" +__version__ = "0.1.1" diff --git a/src/nnnotes/advscene.py b/src/nnnotes/advscene.py index c4ddf07..79c61c1 100644 --- a/src/nnnotes/advscene.py +++ b/src/nnnotes/advscene.py @@ -1,7 +1,8 @@ """ADV presentation data -> scene.json + textures + shaders. Everything the game uses to present an ADV episode besides the episode script, -the Live2D models and the audio: +the Live2D models (with the materials of their mask pass: webmodel.py) and the +audio: player colour space, quality levels, URP pipelines, renderer data and features, post-process data (APK boot data, see player.py) @@ -13,8 +14,6 @@ settings AdvPlayerSettings (+ focus data), AdvMasterIdSettings stages every stage prefab the episode uses (AdvStage, lights, background sprite, volume profiles) - resources Resources.Load materials the runtime draws with (Cubism mask - pass: Mask / MaskCulling) postTextures per camera renderer: post-process data textures used by the URP chain (film grain textures, indexed by FilmGrain.type) @@ -47,13 +46,6 @@ STAGE_PREFIX = "Adv/Stage/" -# Resources.Load paths used by the ADV runtime (Cubism mask pass materials) -RESOURCES = { - "cubismMask": "Live2D/Cubism/Materials/Mask", - "cubismMaskCulling": "Live2D/Cubism/Materials/MaskCulling", -} - - def camera_renderer(player_graphics: dict, pipeline: str, renderer_index: int) -> str: rl = player_graphics["pipelines"][pipeline]["m_RendererDataList"] i = player_graphics["pipelines"][pipeline]["m_DefaultRendererIndex"] if renderer_index < 0 else renderer_index @@ -71,7 +63,6 @@ def extract(cat: Catalog, player: PlayerData, episode: dict, out_dir: Path, shad for name, key in ADV_KEYS.items(): doc[name] = ex.prefab(key) doc["settings"] = {name: ex.asset(key) for name, key in SETTINGS_KEYS.items()} - doc["resources"] = {name: ex.material(player.resource(path)) for name, path in RESOURCES.items()} stages = {} for c in episode["commands"]: @@ -82,10 +73,7 @@ def extract(cat: Catalog, player: PlayerData, episode: dict, out_dir: Path, shad stages[c["TargetAssetName"]] = ex.prefab(key) doc["stages"] = stages - # shaders: model closures, everything referenced above, the ADV camera renderer - for r in episode["resources"]: - if r["kind"] == "live2d": - ex.closure_shaders(r["address"]) + # shaders: everything referenced above, the ADV camera renderer renderers = set() for node in doc["cameraManager"]["nodes"]: for comp in node["components"]: diff --git a/src/nnnotes/cli.py b/src/nnnotes/cli.py index f54b9e9..4056856 100644 --- a/src/nnnotes/cli.py +++ b/src/nnnotes/cli.py @@ -1,9 +1,13 @@ """nnnotes command line. Every setting (keys, the CDN base of each region, the paths of your own data and tools) comes from the config file, -the environment or the flags below; see config.py and nnnotes.example.toml. JSON summaries are printed as UTF-8 -whatever the console encoding. +the environment or the flags below; see config.py and nnnotes.example.toml (`nnnotes config init` writes it). JSON +summaries are printed as UTF-8 whatever the console encoding. + nnnotes config init [--user | --file F] [--set SECTION.KEY=VALUE ...] + nnnotes config set SECTION.KEY VALUE + nnnotes config check [--json] + nnnotes config path [--json] nnnotes catalog --prefix Spot/ --limit 40 nnnotes browse [--port 8000] nnnotes pull [ ...] @@ -12,7 +16,7 @@ nnnotes master decode -o nnnotes master download --version | --latest -o nnnotes adv 10462 -o out/adv_10462.json - nnnotes story 10462 -o out/story_10462 + nnnotes story 10462 -o out/story_10462 [--models out/live2d] [--force] nnnotes live2d -o out/live2d/x nnnotes spot 10001 -o out/spot_10001 nnnotes room -o out/room.glb @@ -55,7 +59,8 @@ from . import __version__, cli_assets, voices from .addressables import BundleKey from .catalog import Catalog -from .config import Config, ConfigError, use +from .compress import DEFAULT_ENCODING, ENCODINGS +from .config import DEFAULT_FILE, ENV_CONFIG, Config, ConfigError, config_files, find_file, use, user_file from .jsonio import dumps, write_json from .webaudio import DEFAULT_AUDIO_FORMAT, WEB_AUDIO @@ -201,6 +206,150 @@ def cmd_master_decode(args, cfg): sys.exit(1) +def _tty() -> bool: + try: + return sys.stdin.isatty() and sys.stdout.isatty() + except (AttributeError, OSError, ValueError): + return False + + +def _config_target(args, default: Path | None) -> Path: + """The file a config command writes: --file, --user, else `default` (None: the file the commands read).""" + if args.file: + return Path(args.file) + if args.user: + target = user_file() + if target is None: + raise ConfigError("no per-user config directory: the environment sets neither APPDATA (Windows) nor " + "XDG_CONFIG_HOME / HOME") + return target + if default is not None: + return default + target = find_file(args.config) + if target is None: + raise ConfigError("no config file is read (`nnnotes config path`): name one with --file or --user, or " + "write one with `nnnotes config init`") + return target + + +def _config_value(args, name: str, text: str, stdin_used: list): + """(section, key, value) of `name` = `text` (`-`: read from standard input, once), checked.""" + from . import configfile + setting, section = configfile.resolve(name) + if text == "-": + if stdin_used: + args.usage("only one value can be read from standard input") + stdin_used.append(name) + if setting.secret and _tty(): + import getpass + text = getpass.getpass(f"[{section}] {setting.key} (hidden): ") + else: + text = sys.stdin.readline().rstrip("\r\n") + try: + return section, setting.key, configfile.parse_value(setting, text) + except ValueError as e: + raise ConfigError(f"setting {section}.{setting.key}: {e}") from None + + +def _config_note(target: Path) -> None: + read = find_file() + if read is None or read.absolute() != target.absolute(): + print(f"# the commands read {read.absolute() if read else 'no config file'} first (`nnnotes config path`)", + file=sys.stderr) + + +def cmd_config_init(args, cfg): + from . import configfile + named = next((f for n, f in config_files(args.config)[:2] if f is not None), None) + target = _config_target(args, named or Path(DEFAULT_FILE)) + if target.exists() and not args.force: + raise ConfigError(f"config file {target} exists (--force overwrites it; `nnnotes config set` edits one " + f"value)") + values, regions, stdin_used = [], [], [] + for item in args.set or []: + name, sep, text = item.partition("=") + if not sep: + args.usage(f"--set {name}: give it as SECTION.KEY=VALUE") + section, key, value = _config_value(args, name.strip(), text, stdin_used) + values.append((section, key, value)) + region = section.split(".", 1)[1] if section.startswith("servers.") else None + if region and region not in regions: + regions.append(region) + if not args.set and not args.no_input and _tty(): + try: + values, regions = configfile.prompt() + except (KeyboardInterrupt, EOFError): + print("\nnnnotes: config init stopped, nothing written", file=sys.stderr) + sys.exit(1) + configfile.write(target, configfile.render(values, regions)) + print(target.absolute()) + print(f"# {len(values)} settings written, the others empty; `nnnotes config check` shows the status of each", + file=sys.stderr) + _config_note(target) + + +def _config_edit(args, value_text: str | None) -> None: + from . import configfile + target = _config_target(args, None) + text = target.read_text(encoding="utf-8") if target.exists() else configfile.template().decode("utf-8") + if value_text is None: # unset: an empty value + setting, section = configfile.resolve(args.name) + section, key, value = section, setting.key, [] if setting.kind == "list" else "" + else: + section, key, value = _config_value(args, args.name, value_text, []) + configfile.write(target, configfile.edit(text, [(section, key, value)], str(target))) + print(f"{target.absolute()}: {section}.{key} {'set' if value else 'emptied'}") + _config_note(target) + + +def cmd_config_set(args, cfg): + _config_edit(args, args.value) + + +def cmd_config_unset(args, cfg): + _config_edit(args, None) + + +def cmd_config_check(args, cfg): + from . import configfile + r = configfile.check(cfg) + if args.json: + _print_json(r) + else: + print(f"# file: {r['file']}" if r["file"] else "# no config file is read (`nnnotes config path`)") + for s in r["settings"]: + status = s["status"] + (f": {s['reason']}" if s.get("reason") else "") + print(f"{s['name']}\t{s.get('origin') or '-'}\t{status}") + sys.stdout.flush() + print(f"# {r['problems']} problems", file=sys.stderr) + if r["problems"]: + sys.exit(1) + + +def cmd_config_path(args, cfg): + from . import configfile + files, read = [], None + for name, file in config_files(args.config): + if file is None: + state = "not given" + elif not file.is_file(): + state = "not found" + elif read is None: + read, state = file, "read" + else: + state = "found, not read" + files.append({"name": name, "path": str(file.absolute()) if file is not None else None, "state": state}) + if read is None and file is not None and name in ("--config", ENV_CONFIG): + break # a named file that is missing stops every command + if args.json: + _print_json({"schema": configfile.PATHS, "files": files, "read": str(read.absolute()) if read else None}) + return + for f in files: + print(f"{f['name']}\t{f['path'] or '-'}\t{f['state']}") + if read is None: + print("# no config file is read: the settings come from the environment and the flags only", file=sys.stderr) + + def cmd_servers(args, cfg): from . import gameapi try: @@ -338,18 +487,26 @@ def cmd_player(args, cfg): def cmd_story(args, cfg): - from . import languages, story + from . import languages, story, webmodel if args.fonts == "game": from .tmpfont import require_extra require_extra("--fonts game") flac = _flac_options(args) + out = Path(os.path.abspath(args.out)) + models = Path(os.path.abspath(args.models or Path(args.out) / os.pardir / "live2d")) + try: + os.path.relpath(models, out) + except ValueError: + args.usage(f"--models {models}: not on the drive of {out} (story.json names it relative to the story)") languages.check(cfg.require("catalog", "language")) # the story UI's language (advui reads the setting) md = master_dir(cfg) known_row(args, md, "MasterAdv", args.adv_id, "episode") cat = open_catalog(cfg) - r = story.build(cat, md, player_data(cfg), args.adv_id, Path(args.out), - audio_format=args.format, audio=not args.no_audio, fonts=args.fonts, audio_options=flac) - _print_json(r) + player = player_data(cfg) + source = webmodel.ModelDir(cat, player, models, force=args.force) + r = story.build(cat, md, player, args.adv_id, out, audio_format=args.format, audio=not args.no_audio, + fonts=args.fonts, models=source, audio_options=flac) + _print_json({**r, "modelsBuilt": source.built, "modelsSkipped": source.skipped}) def _flac_options(args) -> dict: @@ -426,7 +583,7 @@ def cmd_web(args, cfg): player = cfg.path("paths", "player") out = Path(args.out) if args.player_only or args.reingest_json: - r = {**(web.reingest_json(out) if args.reingest_json else {}), + r = {**(web.reingest_json(out, args.compress) if args.reingest_json else {}), **web.write_player(out, web.check_player(player)), **web.write_index(out)} else: web.check_player(player) @@ -444,6 +601,7 @@ def cmd_web(args, cfg): except liveoptions.OptionSpecError as e: args.usage(str(e)) r = {} + encoding = args.compress if models: cfg.require_path("paths", "apk") # the Cubism component classes and mask materials: the APK try: @@ -451,13 +609,14 @@ def cmd_web(args, cfg): except ValueError as e: args.usage(str(e)) r.update(webmodel.build(out, selected, cfg, player, force=args.force, tmp_dir=args.tmp, - workers=args.workers, **base)) + workers=args.workers, encoding=encoding, **base)) if charts: _fonts_extra(args.fonts) r.update(web.build(out, None if args.all else args.pair, cfg, player, args.format, audio=not args.no_audio, force=args.force, tmp_dir=args.tmp, workers=args.workers, band=args.band, leader_card=args.leader_card, regions=regions, fonts=args.fonts, - read_workers=args.read_workers, live_options=live_options)) + read_workers=args.read_workers, live_options=live_options, + encoding=args.compress)) if stories: from .tmpfont import require_extra require_extra("--story / --all-stories") # the font assets of the story text (open and game) @@ -470,9 +629,10 @@ def cmd_web(args, cfg): r.update(storysite.build(out, args.story, cfg, player, args.format, audio=not args.no_audio, force=args.force, tmp_dir=args.tmp, workers=args.workers, regions=regions, story_languages=args.story_languages, fonts=args.fonts, - fonts_flags=dict(args.font or []))) + fonts_flags=dict(args.font or []), encoding=encoding)) _print_json(r) - if r.get("failed") or r.get("modelsFailed") or r.get("storiesFailed"): + if (r.get("failed") or r.get("modelsFailed") or r.get("storiesFailed") + or (r.get("storyModels") or {}).get("modelsFailed")): sys.exit(1) @@ -572,6 +732,43 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument("--node", help="Node.js executable ([paths] node; else on PATH)") sub = p.add_subparsers(dest="cmd", required=True, metavar="") + c = sub.add_parser("config", help="the config file: write it, set one value, check the settings, show where it is") + csub = c.add_subparsers(dest="config_cmd", required=True, metavar="") + + def target(m, what): + g = m.add_mutually_exclusive_group() + g.add_argument("--file", help=f"the config file to {what}") + g.add_argument("--user", action="store_true", + help=f"{what} the per-user config file, read from any directory (`config path` shows it)") + + m = csub.add_parser("init", help="write a config file from the template: the values are asked in a terminal, " + "else taken from --set (the other settings empty)") + target(m, f"write (default: the file --config or NNNOTES_CONFIG names, else ./{DEFAULT_FILE})") + m.add_argument("--set", action="append", metavar="SECTION.KEY=VALUE", + help="a value to write (repeatable; no questions then): VALUE - reads it from standard input; a " + "list comma-separated; a path is written absolute") + m.add_argument("--no-input", action="store_true", help="ask nothing, also in a terminal") + m.add_argument("--force", action="store_true", help="overwrite an existing file") + m.set_defaults(func=cmd_config_init, reads_config=False, usage=m.error) + m = csub.add_parser("set", help="set one value of the config file the commands read (or of --file / --user; " + "a missing one is made from the template); comments and other lines stay") + m.add_argument("name", metavar="SECTION.KEY", help="the setting: bundle.key, servers.tw.cdn, paths.fonts.ja, ...") + m.add_argument("value", help="the value (a list comma-separated); - reads it from standard input, without echo " + "for the keys in a terminal") + target(m, "edit") + m.set_defaults(func=cmd_config_set, reads_config=False, usage=m.error) + m = csub.add_parser("unset", help="empty one value of the config file (an empty value counts as unset)") + m.add_argument("name", metavar="SECTION.KEY", help="the setting") + target(m, "edit") + m.set_defaults(func=cmd_config_unset, reads_config=False, usage=m.error) + m = csub.add_parser("check", help="every setting: where it comes from and whether it is valid, never its value; " + "exit status 1 when one is invalid, not found or unknown") + m.add_argument("--json", action="store_true", help="the report as JSON (nnnotes.config-check/1)") + m.set_defaults(func=cmd_config_check) + m = csub.add_parser("path", help="the config files looked up, in order, and the one the commands read") + m.add_argument("--json", action="store_true", help="the list as JSON (nnnotes.config-path/1)") + m.set_defaults(func=cmd_config_path, reads_config=False) + c = sub.add_parser("catalog", help="list addressable keys") c.add_argument("--prefix", default="", help="only the keys that start with this prefix") c.add_argument("--limit", type=int, default=200, help="print at most N keys (default 200; the total goes " @@ -613,9 +810,13 @@ def build_parser() -> argparse.ArgumentParser: _out(c, "output .json file") c.set_defaults(func=cmd_adv, usage=c.error) - c = sub.add_parser("story", help="ADV episode -> story dir (episode, models, audio, scene, UI, media, videos)") + c = sub.add_parser("story", help="ADV episode -> story dir (episode, audio, scene, UI, media, videos) + its Live2D " + "models") c.add_argument("adv_id", type=int, help="MasterAdv id of the episode") _out(c, "output directory") + c.add_argument("--models", metavar="DIR", + help="directory of the Live2D models, one / each (default: OUT/../live2d)") + c.add_argument("--force", action="store_true", help="export the models again whose directory exists") _audio_args(c) c.add_argument("--no-audio", action="store_true", help="do not decode the cue sheets") c.add_argument("--fonts", default="open", choices=("open", "game"), @@ -686,7 +887,11 @@ def build_parser() -> argparse.ArgumentParser: m.add_argument("--all-live2d", action="store_true", help="every Live2D model of the catalog") c.add_argument("--format", default=DEFAULT_AUDIO_FORMAT, choices=tuple(WEB_AUDIO), help="BGM format") c.add_argument("--no-audio", action="store_true", help="export no audio files") - c.add_argument("--force", action="store_true", help="rebuild charts and models whose manifest exists") + c.add_argument("--compress", default=DEFAULT_ENCODING, choices=ENCODINGS, + help="encoding of the compressible assets (JSON, shaders, moc3, ...) where it makes them " + "smaller: gzip (default), br (brotli) or none") + c.add_argument("--force", action="store_true", + help="rebuild charts, models and stories whose manifest exists (stories: also their models)") c.add_argument("--tmp", help="directory for the temporary live and model builds (default .tmp)") c.add_argument("--workers", type=int, help="parallel music / model processes (default a quarter of the CPUs, up to 8 / up to 4; " @@ -733,7 +938,7 @@ def build_parser() -> argparse.ArgumentParser: def main(argv=None): args = build_parser().parse_args(argv) try: - cfg = load_config(args) + cfg = load_config(args) if getattr(args, "reads_config", True) else None args.func(args, cfg) except ConfigError as e: print(f"nnnotes: {e}", file=sys.stderr) diff --git a/src/nnnotes/compress.py b/src/nnnotes/compress.py new file mode 100644 index 0000000..1e8cdbc --- /dev/null +++ b/src/nnnotes/compress.py @@ -0,0 +1,91 @@ +"""Stored encodings of site assets: deterministic gzip and brotli bytes, and their decoding. + +gzip: the 10-byte header 1f 8b 08 00 00 00 00 00 02 ff (no name, no mtime, XFL 2, OS unknown), the raw deflate +stream of zlib at level 9 (window 15, default memory level), the CRC-32 and the length mod 2^32, little-endian; the +header is written here, the same on every platform (zlib's own gzip wrapper writes the platform's OS byte). +brotli: `brotli.compress(data, quality=11)` (the `brotli` module, imported when first used). The bytes are the same +for the same zlib / brotli versions. +""" +from __future__ import annotations + +import gzip +import struct +import zlib +from pathlib import Path + +ENCODINGS = ("gzip", "br", "none") # web --compress +DEFAULT_ENCODING = "gzip" +SUFFIXES = {"gzip": ".gz", "br": ".br"} # stored name suffix of an encoded asset +# extensions of the stored names worth encoding (text, meshes, raw PCM); images and compressed audio are left as +# they are +COMPRESSIBLE = frozenset({"json", "glsl", "moc3", "atlas", "skel", "bin", "wav", "glb"}) +GZIP_HEADER = bytes.fromhex("1f 8b 08 00 00 00 00 00 02 ff") + + +def gzip_bytes(data: bytes) -> bytes: + """`data` as a gzip member (module docstring).""" + return (GZIP_HEADER + zlib.compress(data, level=9, wbits=-15) + + struct.pack(" bytes: + """`data` as a brotli stream at quality 11.""" + return _brotli().compress(data, quality=11) + + +def check(encoding: str) -> None: + """ValueError for an unknown encoding; ImportError when "br" is asked for without the brotli module.""" + if encoding not in ENCODINGS: + raise ValueError(f"asset encoding {encoding!r}: one of {', '.join(ENCODINGS)}") + if encoding == "br": + _brotli() + + +def encode(encoding: str, data: bytes) -> bytes: + """`data` in `encoding` ("gzip" or "br").""" + if encoding == "gzip": + return gzip_bytes(data) + if encoding == "br": + return brotli_bytes(data) + raise ValueError(f"asset encoding {encoding!r}: gzip or br") + + +def decode(name: str, data: bytes) -> bytes: + """The decoded bytes of a stored file named `name` (or given its suffix): .gz and .br decoded, other names as + they are.""" + if name.endswith(".gz"): + return gzip.decompress(data) + if name.endswith(".br"): + return _brotli().decompress(data) + return data + + +def holds(path: Path, data: bytes) -> bool: + """True when the stored file at `path` decodes to `data` (False for a damaged or truncated one).""" + path = Path(path) + errors = (OSError, EOFError, zlib.error) + ((_brotli().error,) if path.name.endswith(".br") else ()) + try: + return decode(path.name, path.read_bytes()) == data + except errors: + return False + + +def decoded_size(path: Path) -> int: + """The decoded size of a stored file: its size, the length field of a .gz member (below 4 GiB), the length of a + .br stream once decoded.""" + path = Path(path) + if path.name.endswith(".gz"): + with open(path, "rb") as f: + f.seek(-4, 2) + return struct.unpack("`, else the file named by `NNNOTES_CONFIG`, else `./nnnotes.toml` when present; +1. the TOML file: `--config `, else the file named by `NNNOTES_CONFIG`, else `./nnnotes.toml` when present, + else the per-user file (`user_file`) when present; 2. environment variables `NNNOTES_
_`: the setting's dotted name upper-cased, dots and dashes as underscores (`servers.tw.cdn` -> `NNNOTES_SERVERS_TW_CDN`, `bundle.nonce_seed` -> `NNNOTES_BUNDLE_NONCE_SEED`); 3. command-line flags. @@ -23,12 +24,57 @@ ENV_PREFIX = "NNNOTES_" ENV_CONFIG = "NNNOTES_CONFIG" DEFAULT_FILE = "nnnotes.toml" +TEMPLATE = "nnnotes.example.toml" # package data: every setting with an empty value class ConfigError(Exception): """A setting is missing or malformed (the message names the setting, never its value).""" +def user_file(environ: dict[str, str] | None = None) -> Path | None: + """The per-user config file: `%APPDATA%\\nnnotes\\nnnotes.toml` on Windows, else + `$XDG_CONFIG_HOME/nnnotes/nnnotes.toml` (`~/.config` when XDG_CONFIG_HOME is unset); None when the environment + names no such directory.""" + env = os.environ if environ is None else environ + if os.name == "nt": + base = env.get("APPDATA") + else: + base = env.get("XDG_CONFIG_HOME") or (str(Path(env["HOME"]) / ".config") if env.get("HOME") else None) + return Path(base) / "nnnotes" / DEFAULT_FILE if base else None + + +def config_files(path: str | Path | None = None, + environ: dict[str, str] | None = None) -> list[tuple[str, Path | None]]: + """The config file candidates in lookup order, (how it is named, path or None): `--config`, NNNOTES_CONFIG, the + working directory's nnnotes.toml, the per-user file. The first two are used when they are given (and must + exist), the last two when they exist.""" + env = os.environ if environ is None else environ + return [("--config", Path(path) if path else None), + (ENV_CONFIG, Path(env[ENV_CONFIG]) if env.get(ENV_CONFIG) else None), + ("working directory", Path(DEFAULT_FILE)), + ("user", user_file(env))] + + +def find_file(path: str | Path | None = None, environ: dict[str, str] | None = None) -> Path | None: + """The config file that is read (config_files), or None when there is none.""" + for name, file in config_files(path, environ): + if file is None: + continue + if name in ("--config", ENV_CONFIG): + if not file.is_file(): + raise ConfigError(f"config file {file} ({name}) not found") + return file + if file.is_file(): + return file + return None + + +def template() -> bytes: + """The config template shipped with the package (`nnnotes config init` writes it).""" + from importlib import resources + return resources.files(__package__).joinpath(TEMPLATE).read_bytes() + + def env_name(section: str, key: str) -> str: return ENV_PREFIX + re.sub(r"[^A-Za-z0-9]", "_", f"{section}.{key}").upper() @@ -65,18 +111,9 @@ def __repr__(self) -> str: # never the values def load(cls, path: str | Path | None = None, *, environ: dict[str, str] | None = None, overrides: dict | None = None, flags: dict | None = None) -> "Config": env = os.environ if environ is None else environ - if path: - file = Path(path) - if not file.is_file(): - raise ConfigError(f"config file {file} (--config) not found") - elif env.get(ENV_CONFIG): - file = Path(env[ENV_CONFIG]) - if not file.is_file(): - raise ConfigError(f"config file {file} ({ENV_CONFIG}) not found") - else: - file = Path(DEFAULT_FILE) - if not file.is_file(): - return cls(environ=env, overrides=overrides, flags=flags) + file = find_file(path, env) + if file is None: + return cls(environ=env, overrides=overrides, flags=flags) try: data = tomllib.loads(file.read_text(encoding="utf-8")) except tomllib.TOMLDecodeError as e: @@ -110,9 +147,31 @@ def origin(self, section: str, key: str) -> str | None: """Where a setting comes from: 'flag', 'env', 'file', or None when it is unset.""" return self._raw(section, key)[1] + def flag(self, section: str, key: str) -> str | None: + """The command-line flag of a setting, when it has one.""" + return self._flags.get((section, key)) + + def file_keys(self) -> list[tuple[str, str]]: + """(section, key) of every value of the TOML file, a nested table as a dotted section ('' at the top).""" + out = [] + + def walk(table: dict, section: str) -> None: + for k, v in table.items(): + if isinstance(v, dict): + walk(v, f"{section}.{k}" if section else k) + else: + out.append((section, k)) + walk(self._data, "") + return out + + def env_names(self) -> list[str]: + """The names of the NNNOTES_ environment variables that set something (never their values).""" + return sorted(k for k, v in self._env.items() if v) + def missing(self, section: str, key: str) -> ConfigError: + hint = "" if self.source else " (no config file was found: `nnnotes config init` writes one to fill in)" return ConfigError(f"setting {section}.{key} is not set: give it as " - f"{describe(section, key, self._flags.get((section, key)))}") + f"{describe(section, key, self._flags.get((section, key)))}{hint}") def _bad(self, section: str, key: str, what: str) -> ConfigError: return ConfigError(f"setting {section}.{key}: {what}") diff --git a/src/nnnotes/configfile.py b/src/nnnotes/configfile.py new file mode 100644 index 0000000..a879f55 --- /dev/null +++ b/src/nnnotes/configfile.py @@ -0,0 +1,344 @@ +"""The config file: the settings it holds, writing and editing it, and checking the settings. + +The settings are those of the template (config.TEMPLATE, every value empty), plus one `[servers.]` table per +region and the `[paths.fonts]` table. `write_new` renders the template with values, `set_value` edits one value of +an existing file in place (comments and the other lines stay), `check` reports each setting's origin and status. +Values are never printed: the prompts for the keys do not echo, and the reports hold states, not values. +""" +from __future__ import annotations + +import getpass +import json +import os +import re +import tomllib +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import urlsplit + +from . import languages +from .config import Config, ConfigError, env_name, template + +SECRETS = {("bundle", "key"), ("bundle", "nonce_seed"), ("master", "key"), ("master", "iv")} +HEX_BYTES = {("bundle", "key"): 16, ("bundle", "nonce_seed"): None, ("master", "key"): 32, ("master", "iv"): 32} +REGION = "servers." # the section of the per-region settings +TEMPLATE_REGION = "tw" # the example region table of the template +FONTS = "paths.fonts" +FONT_KEYS = (*languages.LANGUAGES, "emoji") +LINKS = ("auto", "clone", "hard", "copy") +FILES = {("paths", "catalog"), ("paths", "apk"), ("paths", "ffmpeg"), ("paths", "vgmstream"), ("paths", "node")} +DIRS = {("paths", "master"), ("paths", "player"), (REGION, "master")} +CHECK = "nnnotes.config-check/1" +PATHS = "nnnotes.config-path/1" +PROBLEMS = ("invalid", "not found", "unknown") + +_HEADER = re.compile(r"^\s*\[(?!\[)\s*([A-Za-z0-9_.-]+)\s*\]\s*(?:#.*)?$") +_KEYVAL = re.compile(r"^(\s*)([A-Za-z0-9_-]+)\s*=\s*(.*)$") +_REGION_NAME = re.compile(r"[A-Za-z0-9_-]+") + + +@dataclass(frozen=True) +class Setting: + section: str # "bundle", REGION, FONTS, ... + key: str + kind: str # "string", "list" or "path" + secret: bool + description: str + + +def _template_settings() -> list[Setting]: + out, section, comment = [], None, [] + for line in template().decode("utf-8").splitlines(): + s = line.strip() + if not s: + comment = [] + elif s.startswith("#"): + comment.append(s.lstrip("#").strip()) + elif m := _HEADER.match(line): + section, comment = m.group(1), [] + elif (m := _KEYVAL.match(line)) and section: + sec = REGION if section == f"servers.{TEMPLATE_REGION}" else section + key, value = m.group(2), m.group(3).strip() + kind = "list" if value.startswith("[") else "path" if sec == "paths" or (sec, key) in DIRS else "string" + out.append(Setting(sec, key, kind, (sec, key) in SECRETS, " ".join(comment))) + comment = [] + for k in FONT_KEYS: + what = "colour emoji font (CBDT or sbix) of the stories' emoji sprites" if k == "emoji" else \ + f"font file (OpenType or TrueType) the story text of {k} is drawn with (--fonts open)" + out.append(Setting(FONTS, k, "path", False, what)) + return out + + +SETTINGS = _template_settings() + + +def resolve(name: str) -> tuple[Setting, str]: + """(setting, concrete section) of a dotted name: `bundle.key`, `servers.tw.cdn`, `paths.fonts.zh-Hant`.""" + section, _, key = name.rpartition(".") + for s in SETTINGS: + if s.section == section and s.key == key and s.section != REGION: + return s, section + head, _, region = section.rpartition(".") + if head == "servers" and _REGION_NAME.fullmatch(region): + for s in SETTINGS: + if s.section == REGION and s.key == key: + return s, section + raise ConfigError(f"unknown setting {name}: `nnnotes config check` lists the settings") + + +def parse_value(setting: Setting, text: str): + """The value of a setting from its text form (a list: comma-separated; a path: made absolute), checked.""" + text = text.strip() + if setting.kind == "list": + value = [s.strip() for s in text.split(",") if s.strip()] + elif setting.kind == "path" and text: + value = str(Path(text).expanduser().absolute()) + else: + value = text + if value: + validate(setting, value) + return value + + +def validate(setting: Setting, value) -> None: + """Raise ValueError (a message without the value) when `value` is not a valid value of `setting`.""" + k = (setting.section, setting.key) + if k in HEX_BYTES: + s = value[2:] if value[:2].lower() == "0x" else value + try: + b = bytes.fromhex(s) + except ValueError: + raise ValueError("must be a hex string") from None + size = HEX_BYTES[k] + if size is not None and len(b) != size: + raise ValueError(f"must be {size} bytes ({2 * size} hex digits)") + elif k == ("catalog", "language") or k == (REGION, "languages"): + for v in value if isinstance(value, list) else [value]: + if v not in languages.LANGUAGES: + raise ValueError(f"not one of the game's languages ({', '.join(languages.LANGUAGES)})") + elif k == (REGION, "cdn"): + u = urlsplit(value) + if u.scheme not in ("https", "http") or not u.netloc: + raise ValueError("must be an http(s):// URL") + elif k in ((REGION, "api"), ("bootstrap", "api")): + from .gameapi import channel_target + channel_target(value) + elif k == ("export", "link") and value not in LINKS: + raise ValueError(f"must be one of {', '.join(LINKS)}") + + +# ---------------------------------------------------------------- TOML text +def _toml(value) -> str: + if isinstance(value, list): + return "[" + ", ".join(_toml(v) for v in value) + "]" + if "'" not in value and not re.search(r"[\x00-\x1f\x7f]", value): + return f"'{value}'" # a literal string: paths keep their backslashes + return json.dumps(value, ensure_ascii=False).replace("\x7f", "\\u007f") + + +def _headers(lines: list[str]) -> list[tuple[int, str]]: + return [(i, m.group(1)) for i, line in enumerate(lines) if (m := _HEADER.match(line))] + + +def _region_block(region: str) -> list[str]: + """The template's example region table (comment lines, header, keys) for `region`.""" + lines = template().decode("utf-8").splitlines(keepends=True) + heads = _headers(lines) + for n, (i, name) in enumerate(heads): + if name == f"servers.{TEMPLATE_REGION}": + end = heads[n + 1][0] if n + 1 < len(heads) else len(lines) + block = lines[i:end] + while block and not block[-1].strip(): + block.pop() + return [f"[servers.{region}]\n"] + block[1:] + return [f"[servers.{region}]\n"] + + +def _set_line(text: str, section: str, key: str, value) -> str: + lines = text.splitlines(keepends=True) + if lines and not lines[-1].endswith("\n"): + lines[-1] += "\n" + heads = _headers(lines) + for n, (i, name) in enumerate(heads): + if name != section: + continue + end = heads[n + 1][0] if n + 1 < len(heads) else len(lines) + last = i + for j in range(i + 1, end): + m = _KEYVAL.match(lines[j]) + if not m: + continue + last = j + if m.group(2) == key: + v = m.group(3).strip() + if v.startswith(('"""', "'''")) or (v.startswith("[") and v.count("[") > v.count("]")): + raise ConfigError(f"setting {section}.{key}: its value spans several lines of the file; edit " + f"the file itself") + lines[j] = f"{m.group(1)}{key} = {_toml(value)}\n" + return "".join(lines) + lines.insert(last + 1, f"{key} = {_toml(value)}\n") + return "".join(lines) + block = _region_block(section.split(".", 1)[1]) if section.startswith("servers.") else [f"[{section}]\n"] + text = "".join(lines) + ("\n" if "".join(lines).strip() else "") + "".join(block) + return _set_line(text, section, key, value) + + +def _lookup(data: dict, section: str, key: str): + t = data + for part in section.split("."): + t = t.get(part) if isinstance(t, dict) else None + return t.get(key) if isinstance(t, dict) else None + + +def edit(text: str, values: list[tuple[str, str, object]], where: str) -> str: + """`text` with each (section, key, value) set in place; the result is checked to read back as those values.""" + for section, key, value in values: + text = _set_line(text, section, key, value) + try: + data = tomllib.loads(text) + except tomllib.TOMLDecodeError: + raise ConfigError(f"{where}: the edited file is not valid TOML (a table in a form this command does not " + f"edit); edit the file itself") from None + for section, key, value in values: + if _lookup(data, section, key) != value: + raise ConfigError(f"setting {section}.{key}: {where} defines it in a form this command does not edit; " + f"edit the file itself") + return text + + +def render(values: list[tuple[str, str, object]], regions: list[str]) -> str: + """The template with one region table per region (the example table renamed, repeated) and the values set.""" + text = template().decode("utf-8") + if regions: + lines = text.splitlines(keepends=True) + heads = _headers(lines) + for n, (i, name) in enumerate(heads): + if name == f"servers.{TEMPLATE_REGION}": + end = heads[n + 1][0] if n + 1 < len(heads) else len(lines) + blocks = ["".join(_region_block(r)) for r in regions] + text = "".join(lines[:i]) + "\n".join(blocks) + "\n" + "".join(lines[end:]) + break + return edit(text, values, "the template") + + +def write(path: Path, text: str) -> None: + """Write the config file atomically; readable by its owner only (it holds the keys).""" + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(path.name + ".tmp") + tmp.write_text(text, encoding="utf-8", newline="\n") + if os.name != "nt": + tmp.chmod(0o600) + os.replace(tmp, path) + + +# ---------------------------------------------------------------- prompts +def prompt(regions_hint: str = TEMPLATE_REGION, ask=input, ask_secret=getpass.getpass, say=print) \ + -> tuple[list[tuple[str, str, object]], list[str]]: + """Ask for every setting in the terminal (Enter leaves one empty; the keys are read without echo) -> (values, + regions).""" + values: list[tuple[str, str, object]] = [] + + def one(s: Setting, section: str, default: str = ""): + name = f"[{section}] {s.key}" + hint = " (hidden)" if s.secret else " (comma-separated)" if s.kind == "list" else "" + say(f"# {s.description}" if s.description else "#") + while True: + text = (ask_secret if s.secret else ask)(f"{name}{hint}{f' [{default}]' if default else ''}: ") + try: + v = parse_value(s, text or default) + except (ValueError, ConfigError) as e: + say(f" {e}; again, or Enter to leave it empty") + continue + if v: + values.append((section, s.key, v)) + return + + fixed = [s for s in SETTINGS if s.section not in (REGION, FONTS)] + for s in fixed: + if s.section in ("bundle", "master"): + one(s, s.section) + say("# regions to configure: names of your choice, one [servers.] table each") + regions = [] + while not regions: + text = ask(f"regions (comma-separated) [{regions_hint}]: ") or regions_hint + regions = [r.strip() for r in text.split(",") if r.strip()] + if not all(_REGION_NAME.fullmatch(r) for r in regions): + say(" a region name holds letters, digits, '-' and '_' only") + regions = [] + for s in fixed: + if s.section == "catalog": + one(s, s.section, regions[0] if s.key == "region" else "") + for r in regions: + for s in SETTINGS: + if s.section == REGION: + one(s, f"servers.{r}") + for s in fixed: + if s.section not in ("bundle", "master", "catalog"): + one(s, s.section) + return values, regions + + +# ---------------------------------------------------------------- check +def _status(cfg: Config, s: Setting, section: str) -> tuple[str, str | None]: + if cfg.origin(section, s.key) is None: + return "unset", None + try: + if s.kind == "list": + validate(s, cfg.get_list(section, s.key)) + elif s.kind == "path": + p = cfg.path(section, s.key) + if (s.section, s.key) in FILES or s.section == FONTS: + if not p.is_file(): + return "not found", "no such file" + elif (s.section, s.key) in DIRS and not p.is_dir(): + return "not found", "no such directory" + else: + v = cfg.get(section, s.key) + validate(s, v) + if (section, s.key) == ("catalog", "region") and v not in cfg.regions(): + return "invalid", f"names no [servers.{v}] table and no {env_name('servers.' + v, 'cdn')}" + except ConfigError as e: + return "invalid", str(e).split(": ", 1)[-1] + except ValueError as e: + return "invalid", str(e) + return "ok", None + + +def check(cfg: Config) -> dict: + """Every setting with its origin (flag, env, file or None) and status (ok, unset, invalid, not found), plus the + keys of the file and the NNNOTES_ environment variables that name no setting (unknown). No values.""" + out = [] + known_env = set() + + def record(s: Setting, section: str, status: str, reason: str | None, origin: str | None): + env = env_name(section, s.key) + known_env.add(env) + out.append({"name": f"{section}.{s.key}", "env": env, "flag": cfg.flag(section, s.key), "kind": s.kind, + "secret": s.secret, "description": s.description, "origin": origin, "status": status, + **({"reason": reason} if reason else {})}) + + region_settings = [s for s in SETTINGS if s.section == REGION] + for s in SETTINGS: + if s.section == REGION: + if s is region_settings[0]: # every region's table where the template has its example + for r in cfg.regions(): + for rs in region_settings: + status, reason = _status(cfg, rs, f"servers.{r}") + record(rs, f"servers.{r}", status, reason, cfg.origin(f"servers.{r}", rs.key)) + continue + if s.section == FONTS and cfg.origin(s.section, s.key) is None: + known_env.add(env_name(s.section, s.key)) + continue + status, reason = _status(cfg, s, s.section) + record(s, s.section, status, reason, cfg.origin(s.section, s.key)) + for section, key in cfg.file_keys(): + try: + resolve(f"{section}.{key}") + except ConfigError: + out.append({"name": f"{section}.{key}", "origin": "file", "status": "unknown", + "reason": "not a setting of nnnotes"}) + for env in cfg.env_names(): + if env not in known_env: + out.append({"name": env, "origin": "env", "status": "unknown", "reason": "names no setting of nnnotes"}) + return {"schema": CHECK, "file": cfg.source and str(Path(cfg.source).absolute()), "settings": out, + "problems": sum(r["status"] in PROBLEMS for r in out)} diff --git a/src/nnnotes/export.py b/src/nnnotes/export.py index a98b39b..9a6af9f 100644 --- a/src/nnnotes/export.py +++ b/src/nnnotes/export.py @@ -1455,8 +1455,3 @@ def dump_shaders(self, out_dir: Path, objects: dict | None = None) -> tuple[list shader_mod.dump_objects([objects[name]], out_dir, objects[name].assets_file.name, index) return index, shader_mod.write_index(index, out_dir) - def closure_shaders(self, key: str): - env = load_closure(self.cat.fetch_key(key)) - for o in env.objects: - if o.type.name == "Shader": - self._shader(o) diff --git a/nnnotes.example.toml b/src/nnnotes/nnnotes.example.toml similarity index 93% rename from nnnotes.example.toml rename to src/nnnotes/nnnotes.example.toml index 1a1b0e8..90fd280 100644 --- a/nnnotes.example.toml +++ b/src/nnnotes/nnnotes.example.toml @@ -1,4 +1,5 @@ -# nnnotes configuration. Copy to nnnotes.toml (or point --config / NNNOTES_CONFIG at your copy) and fill in. +# nnnotes configuration. `nnnotes config init` writes this template to ./nnnotes.toml (`--user`: the per-user file, +# read from any directory; `nnnotes config path` shows where the commands look). Fill in the values. # Every value is empty here: nnnotes has no built-in keys, server addresses or data paths, and an empty value # counts as unset. Each setting can also be given as an environment variable NNNOTES_
_ # (e.g. NNNOTES_BUNDLE_KEY, NNNOTES_SERVERS_TW_CDN); command-line flags override both. diff --git a/src/nnnotes/room.py b/src/nnnotes/room.py index 2d863ee..c0f4421 100644 --- a/src/nnnotes/room.py +++ b/src/nnnotes/room.py @@ -7,6 +7,11 @@ embedded. Geometry is read from the mesh's vertex/index buffers; each submesh becomes one primitive with the renderer's material of the same index. +Objects are told apart by (serialized file name, path id) (object_key): a path +id is unique only within its file, and a bundle's closure loads several files. +Each Unity texture is one glTF texture with its own sampler; textures whose PNG +bytes are equal share one glTF image. + Materials are translated from the Unity material's shader and that shader's own pass render state (culling, blend factors and operations, depth write and test, colour mask, alpha to mask, queue tags) -- not from names. A render state value @@ -26,6 +31,7 @@ """ from __future__ import annotations +import hashlib import io import struct from pathlib import Path @@ -35,7 +41,7 @@ from .catalog import Catalog from .jsonio import dumps, write_json -from .unity import SceneGraph, mesh_arrays, texture_image +from .unity import SceneGraph, deref, mesh_arrays, texture_image # UnityEngine.Rendering.BlendMode BLEND_ONE, BLEND_ZERO, BLEND_SRC_ALPHA, BLEND_ONE_MINUS_SRC_ALPHA = 1, 0, 5, 10 @@ -155,6 +161,19 @@ def _main_tex_slot(shader_name: str) -> str: return "_BaseMap" if shader_name.startswith("Universal Render Pipeline/") else "_MainTex" +def object_key(obj) -> tuple[str, int]: + """(serialized file name, path id) of an object: a path id is unique only within its file.""" + return obj.assets_file.name, obj.path_id + + +def main_textures(owner, mat_tt: dict, shader_name: str) -> list: + """[(texture object, texture env)] of a material's main texture slot (_main_tex_slot) when it holds a texture; + `owner` is the material's object, whose file the env's PPtr is read from.""" + slot = _main_tex_slot(shader_name) + return [(deref(owner, env["m_Texture"]), env) for key, env in mat_tt["m_SavedProperties"]["m_TexEnvs"] + if key == slot and env["m_Texture"]["m_PathID"]] + + def _sampler(tex_tt: dict) -> dict: s = tex_tt.get("m_TextureSettings", {}) mips = tex_tt.get("m_MipCount", 1) > 1 @@ -175,6 +194,7 @@ def __init__(self): self.bin = bytearray() self.bufferViews, self.accessors, self.meshes, self.nodes = [], [], [], [] self.materials, self.textures, self.images, self.samplers = [], [], [], [] + self._image_of: dict[str, int] = {} # sha256 of the PNG bytes -> image index self.ext_used: set[str] = set() def _view(self, data: bytes, target=None) -> int: @@ -204,10 +224,15 @@ def accessor_idx(self, idx: np.ndarray) -> int: return len(self.accessors) - 1 def texture(self, png: bytes, name: str, sampler: dict) -> int: - self.images.append({"name": name, "bufferView": self._view(png), "mimeType": "image/png"}) + """A new texture of `png` with `sampler`. Its image is shared with the textures of the same PNG bytes and + keeps the name of the first.""" + digest = hashlib.sha256(png).hexdigest() + if digest not in self._image_of: + self.images.append({"name": name, "bufferView": self._view(png), "mimeType": "image/png"}) + self._image_of[digest] = len(self.images) - 1 if sampler not in self.samplers: self.samplers.append(sampler) - self.textures.append({"source": len(self.images) - 1, + self.textures.append({"name": name, "source": self._image_of[digest], "sampler": self.samplers.index(sampler)}) return len(self.textures) - 1 @@ -268,13 +293,25 @@ def submesh_primitives(tris: list, mats: list, material_for, on_null) -> list: return prims +def mesh_components(env) -> tuple[dict, dict]: + """({GameObject key: MeshFilter}, {GameObject key: MeshRenderer}) of an environment. The key is (file name, + GameObject path id), as object_key: a component is in its GameObject's file.""" + mf_by_go, mr_by_go = {}, {} + for o in env.objects: + if o.type.name == "MeshFilter": + mf_by_go[(o.assets_file.name, o.read_typetree()["m_GameObject"]["m_PathID"])] = o + elif o.type.name == "MeshRenderer": + mr_by_go[(o.assets_file.name, o.read_typetree()["m_GameObject"]["m_PathID"])] = o + return mf_by_go, mr_by_go + + def drawn_meshes(mf_by_go: dict, mr_by_go: dict, graph, on_null_mesh): - """(transform path id, Mesh, material PPtrs) of each GameObject with a MeshFilter and an enabled MeshRenderer. - A MeshFilter whose mesh reference is empty (path id 0) draws nothing: its object path goes to - `on_null_mesh(path)`.""" - for go_pid, mf in mf_by_go.items(): - tf_pid = graph.tf_of_go.get(go_pid) - mr = mr_by_go.get(go_pid) + """(transform path id, Mesh, material PPtrs) of each GameObject with a MeshFilter and an enabled MeshRenderer + (mesh_components maps). A MeshFilter whose mesh reference is empty (path id 0) draws nothing: its object path + goes to `on_null_mesh(path)`.""" + for key, mf in mf_by_go.items(): + tf_pid = graph.tf_of_go.get(key[1]) + mr = mr_by_go.get(key) if tf_pid is None or mr is None: continue if not mr.read_typetree().get("m_Enabled", 1): @@ -289,37 +326,29 @@ def drawn_meshes(mf_by_go: dict, mr_by_go: dict, graph, on_null_mesh): def extract_room(cat: Catalog, bg_key: str, out_path: Path) -> dict: env = UnityPy.load(*[str(p) for p in cat.fetch_key(bg_key)]) graph = SceneGraph(env) - mf_by_go, mr_by_go = {}, {} - for o in env.objects: - if o.type.name == "MeshFilter": - mf_by_go[o.read_typetree()["m_GameObject"]["m_PathID"]] = o - elif o.type.name == "MeshRenderer": - mr_by_go[o.read_typetree()["m_GameObject"]["m_PathID"]] = o + mf_by_go, mr_by_go = mesh_components(env) glb = _Glb() - tex_cache: dict[int, int] = {} - mat_cache: dict[int, int] = {} + tex_cache: dict[tuple[str, int], int] = {} # texture object_key -> glTF texture + mat_cache: dict[tuple[str, int], int] = {} # material object_key -> glTF material def material_for(pptr) -> int: - if pptr.path_id in mat_cache: - return mat_cache[pptr.path_id] - mat = pptr.read() - mat_tt = mat.object_reader.read_typetree() - shader_tt = mat.m_Shader.read().object_reader.read_typetree() + mat = pptr.deref() + mkey = object_key(mat) + if mkey in mat_cache: + return mat_cache[mkey] + mat_tt = mat.read_typetree() + shader_tt = deref(mat, mat_tt["m_Shader"]).read_typetree() gm = translate_material(mat_tt, shader_tt) - slot = _main_tex_slot(shader_tt["m_ParsedForm"]["m_Name"]) - for key, env_ in mat.m_SavedProperties.m_TexEnvs: - if key != slot or not env_.m_Texture.path_id: - continue - tid = env_.m_Texture.path_id + for tex_obj, env_ in main_textures(mat, mat_tt, shader_tt["m_ParsedForm"]["m_Name"]): + tid = object_key(tex_obj) if tid not in tex_cache: - tex = env_.m_Texture.read() - buf = io.BytesIO(); texture_image(tex).save(buf, format="PNG") - tex_cache[tid] = glb.texture(buf.getvalue(), tex.m_Name, - _sampler(tex.object_reader.read_typetree())) + tex_tt = tex_obj.read_typetree() + buf = io.BytesIO(); texture_image(tex_obj.read()).save(buf, format="PNG") + tex_cache[tid] = glb.texture(buf.getvalue(), tex_tt["m_Name"], _sampler(tex_tt)) info = {"index": tex_cache[tid]} - sx, sy = env_.m_Scale.x, env_.m_Scale.y - ox, oy = env_.m_Offset.x, env_.m_Offset.y + sx, sy = env_["m_Scale"]["x"], env_["m_Scale"]["y"] + ox, oy = env_["m_Offset"]["x"], env_["m_Offset"]["y"] if (sx, sy, ox, oy) != (1.0, 1.0, 0.0, 0.0): # Unity samples (u*sx+ox, v*sy+oy) with a bottom-left origin; # in glTF's top-left texture space that is this transform. @@ -327,8 +356,8 @@ def material_for(pptr) -> int: {"scale": [sx, sy], "offset": [ox, 1.0 - sy - oy]}} glb.ext_used.add("KHR_texture_transform") gm["pbrMetallicRoughness"]["baseColorTexture"] = info - mat_cache[pptr.path_id] = glb.material(gm) - return mat_cache[pptr.path_id] + mat_cache[mkey] = glb.material(gm) + return mat_cache[mkey] n_mesh = 0 n_inactive = 0 @@ -384,7 +413,7 @@ def material_for(pptr) -> int: "unlit": "KHR_materials_unlit" in m.get("extensions", {}), "renderState": m["extras"]["unityRenderState"]} for m in glb.materials], - "textures": [im["name"] for im in glb.images], + "textures": [t["name"] for t in glb.textures], "samplers": glb.samplers, } write_json(out_path.with_suffix(".json"), doc) diff --git a/src/nnnotes/story.py b/src/nnnotes/story.py index 83a2d2f..3755097 100644 --- a/src/nnnotes/story.py +++ b/src/nnnotes/story.py @@ -1,8 +1,6 @@ """One ADV episode -> a self-contained story directory that a player can load without game code. /episode.json command list + text/sound/video shards + resource closure (adv.py) - /live2d// every Live2D model the episode uses: moc3, atlas - pages, full prefab (live2d.extract_runtime) /audio// every cue sheet in the episode's -SoundCueSheet shard, decoded per cue + cues.json (cri.py) /scene.json, textures/, shaders/ @@ -20,8 +18,11 @@ APK (crilips.py), when a voice reaches the CRI Lips path (needs_cri_lips) /story.json index of the above (a kind file the episode does not use is null) -Every resource kind of the closure has an exporter (KIND_OUTPUT); the build stops before writing anything when a -resource is missing from the catalog or of a kind without one. +The Live2D models the episode uses come from a model source (webmodel: ModelDir, SiteModels), `ensure(address) -> +{"id", "motionSync"}`: story.json `models` maps each model's address to its id, plus the source's `story_fields` +(ModelDir: `modelsDir`, the directory that holds /). Every resource kind of the closure has an exporter +(KIND_OUTPUT); the build stops before writing anything when a resource is missing from the catalog or of a kind +without one. """ from __future__ import annotations @@ -31,11 +32,11 @@ from .catalog import Catalog from .jsonio import write_json from .player import PlayerData -from . import adv, advmedia, advscene, advui, advvideo, cri, crilips, live2d +from . import adv, advmedia, advscene, advui, advvideo, cri, crilips # resource kind -> where the story directory holds it KIND_OUTPUT = { - "live2d": "live2d//", + "live2d": "story.json models (the model source)", "stage": "scene.json stages", "transition": "ui/ui.json transitions", "video": advvideo.INDEX, @@ -45,24 +46,15 @@ CRILIPS_DIR = "crilips" -# a model's MotionSync path (the player's CubismMotionSyncController.fromPrefab): both components on the prefab's root -MOTION_SYNC_COMPONENTS = ("CubismMotionSyncController", "Live2DMotionSyncCriAudioInput") AIR_LIP_SYNC = ("airlipsync", "airlipsync_holdopen") # Talk Parameter1 modes without a voice analysis -def has_motion_sync(prefab: dict) -> bool: - """The exported prefab (live2d.extract_runtime) has a MotionSync controller with its CRI audio input on its root - node; a model without one takes its mouth from the voice's CRI Lips analysis.""" - classes = {c.get("class") or c.get("type") for n in prefab["nodes"] if "/" not in n["path"] - for c in n["components"]} - return all(k in classes for k in MOTION_SYNC_COMPONENTS) - - def needs_cri_lips(episode: dict, split_key: str, motion_sync: list[bool]) -> bool: """A voice of the episode reaches the CRI Lips analysis: a lip-synced voice row (Talk or Voice with TargetName and VoiceIDs; IgnoreLipSync and IgnoreData not set) when a model of the episode has no MotionSync controller - (`motion_sync`: per model), or a Talk row whose voice also drives other speakers, which use the analysis whatever - their model (StartSharedAnalyzerLipSync: the EveryoneLipSync mode, or more speakers than voices).""" + (`motion_sync`: per model, model.json `motionSync`), or a Talk row whose voice also drives other speakers, which + use the analysis whatever their model (StartSharedAnalyzerLipSync: the EveryoneLipSync mode, or more speakers than + voices).""" for c in episode["commands"]: if c["cmd"] not in ("Talk", "Voice") or c.get("IgnoreData") or c.get("IgnoreLipSync"): continue @@ -90,26 +82,21 @@ def check_kinds(resources: list[dict]) -> None: def build(cat: Catalog, master: Path, player: PlayerData, adv_id: int, out_dir: Path, - audio_format: str = "flac", audio: bool = True, fonts: str = "open", *, ui: bool = True, + audio_format: str = "flac", audio: bool = True, fonts: str = "open", *, models, ui: bool = True, audio_options: dict | None = None) -> dict: - """Build the story directory. `audio=False` leaves the cue sheets undecoded (story.json `audio` is empty); - `fonts`: "open" exports no font data of the game, "game" exports it (advmedia, advui). `ui=False` writes no - ui/ (story.json still names ui/ui.json: the caller writes it, e.g. per language as storysite does); - `audio_options`: further cri.decode options of every cue sheet (flac_level, also). With audio, the CRI Lips data - is read from the catalog's APK when a voice reaches it (needs_cri_lips; story.json `crilips`, else null).""" + """Build the story directory with the models of the model source `models` (module docstring). `audio=False` + leaves the cue sheets undecoded (story.json `audio` is empty); `fonts`: "open" exports no font data of the game, + "game" exports it (advmedia, advui). `ui=False` writes no ui/ (story.json still names ui/ui.json: the caller + writes it, e.g. per language as storysite does); `audio_options`: further cri.decode options of every cue sheet + (flac_level, also). With audio, the CRI Lips data is read from the catalog's APK when a voice reaches it + (needs_cri_lips; story.json `crilips`, else null).""" out_dir = Path(out_dir) out_dir.mkdir(parents=True, exist_ok=True) episode = adv.to_json(adv.extract(cat, master, adv_id)) check_kinds(episode["resources"]) write_json(out_dir / "episode.json", episode) - models = {} - for r in episode["resources"]: - if r["kind"] == "live2d": - name = r["address"].rsplit("/", 1)[-1] - res = live2d.extract_runtime(cat, r["address"], out_dir / "live2d" / name) - models[r["address"]] = {"dir": f"live2d/{name}", "moc3": res["moc3"], - "prefab": res["prefab"]} + used = {r["address"]: models.ensure(r["address"]) for r in episode["resources"] if r["kind"] == "live2d"} sheets = {} if audio: @@ -125,16 +112,14 @@ def build(cat: Catalog, master: Path, player: PlayerData, adv_id: int, out_dir: if audio: split = json.loads((out_dir / "scene.json").read_text(encoding="utf-8"))["settings"]["playerSettings"][ "_targetNameSplitKey"] - ms = [has_motion_sync(json.loads((out_dir / m["dir"] / m["prefab"]).read_text(encoding="utf-8"))) - for m in models.values()] - if needs_cri_lips(episode, split, ms): + if needs_cri_lips(episode, split, [m["motionSync"] for m in used.values()]): if cat.apk is None: raise crilips.CriLipsError("the CRI Lips data is read from the APK: set [paths] apk") crilips.write_data(cat.apk, out_dir / CRILIPS_DIR) lips = {"descriptor": f"{CRILIPS_DIR}/crilips.json", "data": f"{CRILIPS_DIR}/crilips.bin"} index = {"advId": adv_id, "episode": "episode.json", "scene": "scene.json", "ui": "ui/ui.json", - "models": models, "audio": sheets, **media["files"], - "videos": advvideo.INDEX if videos else None, "crilips": lips} + "models": {a: m["id"] for a, m in used.items()}, **models.story_fields(out_dir), + "audio": sheets, **media["files"], "videos": advvideo.INDEX if videos else None, "crilips": lips} write_json(out_dir / "story.json", index) return {**index, "stages": scene["stages"], "shaders": scene["count"], "textures": scene["textures"], "uiSummary": ui, diff --git a/src/nnnotes/storyhost.py b/src/nnnotes/storyhost.py index 59804cc..377f3ad 100644 --- a/src/nnnotes/storyhost.py +++ b/src/nnnotes/storyhost.py @@ -511,12 +511,8 @@ def background(cat, key: str, situation_name: str, ex: Exporter) -> dict: not placed: inactive.""" env = UnityPy.load(*[str(p) for p in cat.fetch_key(key)]) graph = SceneGraph(env) - mf_by_go, mr_by_go = {}, {} - for o in env.objects: - if o.type.name == "MeshFilter": - mf_by_go[o.read_typetree()["m_GameObject"]["m_PathID"]] = o - elif o.type.name == "MeshRenderer": - mr_by_go[o.read_typetree()["m_GameObject"]["m_PathID"]] = o + mf_by_go, mr_by_go = room.mesh_components(env) + rendered = {go for _, go in mr_by_go} # SceneGraph: a GameObject path id is in one file only root_go = container_go(env, cat._entry(key)["internal_id"]).path_id root = graph.tf_of_go[root_go] spot_bg, volumes = [], [] @@ -542,15 +538,16 @@ def background(cat, key: str, situation_name: str, ex: Exporter) -> dict: overrides = prep["overrides"] floor = find_first_child_with_name(graph, root, FLOOR_NAME, FLOOR_EXCLUDE) renderer = None if floor is None else component_in_children( - graph, floor, lambda t: _go(graph, t) in mr_by_go, lambda t: active_in_hierarchy(graph, t, overrides)) + graph, floor, lambda t: _go(graph, t) in rendered, lambda t: active_in_hierarchy(graph, t, overrides)) shown = {**overrides, root_go: True} nodes, mats = glb_order(room.drawn_meshes(mf_by_go, mr_by_go, graph, lambda p: None)) rows = [] for p in mats: - mat = p.read() - sh = mat.m_Shader.read().object_reader - rows.append((mat.object_reader.read_typetree(), sh.read_typetree()["m_ParsedForm"]["m_Name"], sh)) - tex_index = glb_textures([(tt, name) for tt, name, _ in rows]) + mat = p.deref() + tt = mat.read_typetree() + sh = deref(mat, tt["m_Shader"]) + rows.append((mat, tt, sh.read_typetree()["m_ParsedForm"]["m_Name"], sh)) + tex_index = glb_textures([(mat, tt, name) for mat, tt, name, _ in rows]) if len(volumes) > 1: raise NotImplementedError(f"{key}: {len(volumes)} Volume components") volume = None @@ -565,7 +562,7 @@ def background(cat, key: str, situation_name: str, ex: Exporter) -> dict: "roomNodes": [{"path": graph.path(tf), "placed": under(graph, tf, root), "active": under(graph, tf, root) and active_in_hierarchy(graph, tf, shown), "renderQueue": FLOOR_QUEUE if tf == renderer else None} for tf, _ in nodes], - "roomMaterials": [room_material(tt, name, tex_index) for tt, name, _ in rows], + "roomMaterials": [room_material(mat, tt, name, tex_index) for mat, tt, name, _ in rows], "situation": {"name": prep["name"], "matched": prep["matched"], "activate": [graph.path(x) for x in prep["activate"]], "hide": [graph.path(graph.tf_of_go[g]) if g in graph.tf_of_go else None for g in prep["hide"]]}, @@ -574,7 +571,7 @@ def background(cat, key: str, situation_name: str, ex: Exporter) -> dict: "volume": volume, "lights": [{"prefab": "background", **r} for r in lighting_components( env, graph, root, lambda tf: active_in_hierarchy(graph, tf, shown))], - "shaders": {name: sh for _, name, sh in reversed(rows)}, # the first material's object + "shaders": {name: sh for _, _, name, sh in reversed(rows)}, # the first material's object "check": {"nodes": [n for _, n in nodes], "textures": len(tex_index), "mirrored": sum(1 for tf, _ in nodes if under(graph, tf, root) and np.linalg.det(graph.world(tf)[:3, :3]) < 0)}, @@ -728,11 +725,7 @@ def home_host(cat, master_dir, player, host: dict, hdir: Path, keys: dict) -> di sit_env, sit_graph, sit_graph.tf_of_go[sit_go.path_id], lambda tf: active_in_hierarchy(sit_graph, tf, {sit_go.path_id: True}))] bg = background(cat, bg_key, sit_name + CLONE, ex) - glb = glb_json(sdir / "room.glb") - if ([n["name"] for n in glb["nodes"]] != bg["check"]["nodes"] or room_doc["meshCount"] != len(bg["roomNodes"]) - or [m["name"] for m in glb.get("materials", [])] != [m["name"] for m in bg["roomMaterials"]] - or len(glb.get("images", [])) != bg["check"]["textures"]): - raise RuntimeError(f"{bg_key}: room.glb nodes / materials / textures differ from the background's order") + check_room(bg_key, glb_json(sdir / "room.glb"), room_doc, bg) spine_mats, spine_chars, spine_shaders = spine_records(sit_env, ex) if [c["path"] for c in spine_chars] != [c["path"] for c in doc["spineCharacters"]]: raise RuntimeError(f"{sit_key}: Spine characters differ from spot.json's") @@ -898,16 +891,17 @@ def component_in_children(graph, tf: int, has, active) -> int | None: def glb_order(drawn, arrays_of=mesh_arrays) -> tuple[list, list]: """The nodes and materials room.extract_room writes, in its order, from its room.drawn_meshes items [(transform id, Mesh, material PPtrs)]: a mesh without vertex data or triangles, or whose submeshes all lack - a material, is no node; a material is numbered at its first use (room.submesh_primitives order, by path id). - -> ([(transform id, mesh name)], [material PPtr]).""" - index: dict[int, int] = {} + a material, is no node; a material is numbered at its first use (room.submesh_primitives order, by + room.object_key). -> ([(transform id, mesh name)], [material PPtr]).""" + index: dict[tuple[str, int], int] = {} mats, nodes = [], [] def material_for(p) -> int: - if p.path_id not in index: - index[p.path_id] = len(mats) + key = room.object_key(p.deref()) + if key not in index: + index[key] = len(mats) mats.append(p) - return index[p.path_id] + return index[key] for tf, mesh, pptrs in drawn: arrays = arrays_of(mesh) if arrays is None: @@ -922,23 +916,32 @@ def material_for(p) -> int: return nodes, mats -def glb_textures(materials: list) -> dict[int, int]: - """{texture path id: glb texture index} of room.extract_room for its materials [(typetree, shader name)] in - glb order: the main texture slot of each (room._main_tex_slot), numbered at first use.""" - index: dict[int, int] = {} - for tt, shader_name in materials: - slot = room._main_tex_slot(shader_name) - for key, env in tt["m_SavedProperties"]["m_TexEnvs"]: - tid = env["m_Texture"]["m_PathID"] - if key == slot and tid: - index.setdefault(tid, len(index)) +def glb_textures(materials: list) -> dict[tuple[str, int], int]: + """{texture room.object_key: glb texture index} of room.extract_room for its materials [(material object, + typetree, shader name)] in glb order: the main texture slot of each (room.main_textures), numbered at first + use.""" + index: dict[tuple[str, int], int] = {} + for owner, tt, shader_name in materials: + for tex, _ in room.main_textures(owner, tt, shader_name): + index.setdefault(room.object_key(tex), len(index)) return index -def room_material(tt: dict, shader_name: str, tex_index: dict) -> dict: - """A room material as saved (its shader runs as the game's): name, shader, keywords, float / int / colour - properties, custom render queue, and per texture slot the glb texture index (null when the texture is not in - the glb), scale and offset.""" +def _texture_key(owner, ref: dict) -> tuple[str, int] | None: + """room.object_key of the texture a material's PPtr `ref` names (read from `owner`'s typetree); None for an + empty slot or a texture outside the loaded files (so not in the glb).""" + if not ref["m_PathID"]: + return None + try: + return room.object_key(deref(owner, ref)) + except FileNotFoundError: + return None + + +def room_material(owner, tt: dict, shader_name: str, tex_index: dict) -> dict: + """A room material (`owner` its object, `tt` its typetree) as saved (its shader runs as the game's): name, + shader, keywords, float / int / colour properties, custom render queue, and per texture slot the glb texture + index (null when the texture is not in the glb), scale and offset.""" sp = tt["m_SavedProperties"] keywords = tt.get("m_ValidKeywords") if keywords is None: @@ -946,8 +949,7 @@ def room_material(tt: dict, shader_name: str, tex_index: dict) -> dict: return {"name": tt["m_Name"], "shader": shader_name, "keywords": list(keywords), "floats": dict(sp.get("m_Floats", [])), "ints": dict(sp.get("m_Ints", [])), "colors": dict(sp.get("m_Colors", [])), "renderQueue": tt.get("m_CustomRenderQueue"), - "texEnvs": {k: {"texture": tex_index.get(v["m_Texture"]["m_PathID"]) if v["m_Texture"]["m_PathID"] - else None, + "texEnvs": {k: {"texture": tex_index.get(_texture_key(owner, v["m_Texture"])), "scale": [v["m_Scale"]["x"], v["m_Scale"]["y"]], "offset": [v["m_Offset"]["x"], v["m_Offset"]["y"]]} for k, v in sp["m_TexEnvs"]}} @@ -961,6 +963,15 @@ def glb_json(path) -> dict: return json.loads(data[20:20 + n]) +def check_room(key: str, glb: dict, room_doc: dict, bg: dict) -> None: + """room.extract_room's room.glb (its JSON chunk `glb`) and room.json (`room_doc`) have the nodes, materials and + glTF textures of the background record `bg` (background), in its order.""" + if ([n["name"] for n in glb["nodes"]] != bg["check"]["nodes"] or room_doc["meshCount"] != len(bg["roomNodes"]) + or [m["name"] for m in glb.get("materials", [])] != [m["name"] for m in bg["roomMaterials"]] + or len(glb.get("textures", [])) != bg["check"]["textures"]): + raise RuntimeError(f"{key}: room.glb nodes / materials / textures differ from the background's order") + + def volume_record(comp: dict, layer: int) -> dict: """A Volume component (Exporter record: sharedProfile and its components inlined) -> enabled, isGlobal, weight, priority, blendDistance, layer, profile name and per profile component its class, active flag and the diff --git a/src/nnnotes/storysite.py b/src/nnnotes/storysite.py index e353708..7251578 100644 --- a/src/nnnotes/storysite.py +++ b/src/nnnotes/storysite.py @@ -3,7 +3,9 @@ /stories.json story index: per manifest the story facts (titles and story groups in every language, commands, languages), manifest path, sizes, regions - /stories/.json story manifest: `files` (the common files) and `languages` (per language + /stories/.json story manifest (MANIFEST_FORMAT): `root` (the path from the manifest to the + site root), `models` ({model id: its manifest models/.json, relative to + the site root}), `files` (the common files) and `languages` (per language its own files), every path -> {asset, size} or, for a split JSON object, {parts: [[key, asset, size], ...], size} (as a chart manifest) /stories//.json @@ -12,6 +14,11 @@ /story/ the player's story page and bundle (web.write_player) /assets/. content-addressed files, shared with the charts and models +The Live2D models of the stories are the site's models (webmodel.py): a pre-pass reads the episodes of the stories to +build and collects the models they load, webmodel.build adds those to the site (a model whose manifest exists and is +current is skipped unless `force`, an outdated one built again), and each story takes them from the site +(webmodel.SiteModels), so its manifest names them and its files hold none. + Per story: the story directory of story.build without its UI (audio: FLAC, and with a web format other than FLAC the same samples encoded into it by the decode, web.WEB_AUDIO; the site stores the web format and cues.json names it with its encoder delay), then per language the story UI (advui.extract in that language, fonts open) and its fonts @@ -41,14 +48,14 @@ import uuid from pathlib import Path -from . import adv, advui, jsonio, languages, master, story, storyfonts, storyhost +from . import adv, advui, jsonio, languages, master, story, storyfonts, storyhost, webmodel from .config import Config, ConfigError, usable_cpus, use -from .web import (SPLIT_KEY, STORIES_DIR, STORIES_INDEX, WEB_AUDIO, Store, _dump, _lock_fetches, _log, _minify, - check_player, collect, configure_caches, entry_assets, join_parts, merge_regions, mp4_priming, - open_data, region_masters, region_meta, site_regions, text_asset) +from .web import (MODELS_DIR, SPLIT_KEY, STORIES_DIR, STORIES_INDEX, WEB_AUDIO, Store, _dump, _lock_fetches, _log, + _minify, check_player, collect, configure_caches, entry_assets, join_parts, merge_regions, + mp4_priming, open_data, region_masters, region_meta, site_regions, site_store, text_asset) STORIES_FORMAT = "ournotes.stories/1" -MANIFEST_FORMAT = "ournotes.story-manifest/1" +MANIFEST_FORMAT = "ournotes.story-manifest/2" FONT_SOURCES = storyfonts.SOURCES EMOJI_FONT = "emoji" # the emoji font's key in --font / [paths] fonts # master tables a story build reads (adv, advmedia, the story groups): a region's story inputs @@ -62,6 +69,9 @@ f"ui/{storyhost.SIMPLE_DIR}/{storyfonts.FONTS_DOC}") LANGUAGE_DIRS = (f"ui/{storyfonts.PAGES_DIR}/", f"ui/{storyhost.SIMPLE_DIR}/{storyfonts.PAGES_DIR}/") STREAMS_DOC = "streams.json" # the cue sheets' every-stream list: not read by the player +# the keys of the stories' model build (webmodel.build) in the summary's storyModels +STORY_MODEL_KEYS = ("modelsBuilt", "modelsFailed", "modelsSkipped", "modelsRebuilt", "modelSeconds", "modelWorkers", + "modelNames") # ---------------------------------------------------------------- master data @@ -166,6 +176,12 @@ def run_commands(episode: dict, scene: dict) -> list[str]: return sorted(names) +def episode_models(cat, master_dir: Path, adv_id: int) -> list[str]: + """The keys of the Live2D models the episode loads that the catalog has (its live2d resources, adv.closure).""" + return sorted(r["address"] for r in adv.extract(cat, master_dir, adv_id).resources + if r["kind"] == "live2d" and r["present"]) + + def needs_motion_sync(episode: dict) -> bool: """A Talk row (without IgnoreData) that maps a voice onto a character's lip sync: TargetName and VoiceIDs, IgnoreLipSync not set.""" @@ -218,7 +234,7 @@ def build_dirs(cat, master_dir: Path, player, adv_id: int, work: Path, job: dict if job["audio"] and WEB_AUDIO[fmt][1] is not None: opts = {"flac_level": 0, "also": (WEB_AUDIO[fmt][0], WEB_AUDIO[fmt][1])} story.build(cat, master_dir, player, adv_id, sdir, audio_format="flac", audio=job["audio"], fonts="open", - ui=False, audio_options=opts) + models=webmodel.SiteModels(job["site"]), ui=False, audio_options=opts) episode = json.loads((sdir / "episode.json").read_text(encoding="utf-8")) scene = json.loads((sdir / "scene.json").read_text(encoding="utf-8")) dirs = {} @@ -346,7 +362,7 @@ def put_file(store: Store, path: str, data) -> dict: parts = split_always(b) if path in SPLIT_ALWAYS else None if parts is None: return store.put_file(path, b) - return {"size": len(b), "parts": [[k, store.put(f"{path}#{k}.json", t)["asset"], len(t)] for k, t in parts]} + return store.put_parts(path, parts, len(b)) def manifest_file(site: Path, adv_id: int, prefix: str = "") -> Path: @@ -354,6 +370,11 @@ def manifest_file(site: Path, adv_id: int, prefix: str = "") -> Path: return Path(site) / STORIES_DIR / f"{prefix}{adv_id}.json" +def manifest_root(prefix: str = "") -> str: + """The relative path from stories/.json to the site root.""" + return "../" * (1 + prefix.count("/")) + + def story_manifests(site: Path) -> list[Path]: """Every story manifest: stories/*.json, then the region manifests stories//*.json.""" d = Path(site) / STORIES_DIR @@ -371,14 +392,18 @@ def manifest_assets(man: dict) -> set[str]: def ingest(store: Store, site: Path, adv_id: int, dirs: dict, job: dict, groups: list[dict]) -> dict: """One story's files into the store + its manifest stories/.json, serving job `regions` (added - to the regions of the manifest it replaces).""" + to the regions of the manifest it replaces); the manifest's `models` are those of story.json.""" common, per = collect_story(dirs, job["audio"], job["audioFormat"]) episode, scene = dirs["episode"], dirs["scene"] facts = story_facts(episode, scene, groups, job["languages"], job["language"]) files = {p: put_file(store, p, d) for p, d in sorted(common.items())} langs = {lang: {"files": {p: put_file(store, p, d) for p, d in sorted(per[lang].items())}} for lang in job["languages"]} - manifest = {"format": MANIFEST_FORMAT, "advId": adv_id, "story": facts, "language": job["language"], + ids = json.loads((Path(dirs["story"]) / "story.json").read_text(encoding="utf-8"))["models"].values() + prefix = job.get("prefix", "") + manifest = {"format": MANIFEST_FORMAT, "root": manifest_root(prefix), + "models": {i: f"{MODELS_DIR}/{i}.json" for i in sorted(set(ids))}, + "advId": adv_id, "story": facts, "language": job["language"], "audio": bool(job["audio"]), "audioFormat": job["audioFormat"] if job["audio"] else None, "fonts": job["fonts"], "requires": {"commands": facts["commands"], "cubismCore": True, @@ -386,7 +411,7 @@ def ingest(store: Store, site: Path, adv_id: int, dirs: dict, job: dict, groups: "files": files, "languages": langs} if dirs.get("host"): # Overlay stories: storyhost.build's entry (host/host.json, ui/simple/) manifest["host"] = dirs["host"] - path = manifest_file(site, adv_id, job.get("prefix", "")) + path = manifest_file(site, adv_id, prefix) old = json.loads(path.read_text(encoding="utf-8")).get("regions") if path.is_file() else None if old or job.get("regions"): manifest["regions"] = merge_regions(old, job.get("regions") or []) @@ -429,7 +454,7 @@ def story_task(adv_id: int, job: dict | None = None, data=None, groups: dict | N fonts="open", font_files={lang: font_file(job, lang) for lang in job["languages"]}, font_of=lambda code: font_file(job, code)) stage = "ingest" - r = ingest(Store(site), site, adv_id, dirs, job, groups.get(adv_id, [])) + r = ingest(site_store(site, job["encoding"]), site, adv_id, dirs, job, groups.get(adv_id, [])) return {**r, "id": job.get("prefix", "") + r["id"], "seconds": round(time.time() - t0, 1)} except ConfigError: raise @@ -442,8 +467,21 @@ def story_task(adv_id: int, job: dict | None = None, data=None, groups: dict | N shutil.rmtree(work, ignore_errors=True) -def _pool_task(adv_id: int) -> dict: - return story_task(adv_id) +def models_task(adv_id: int, data=None) -> dict: + """The pre-pass of one story: {"id", "models": episode_models}; an episode that cannot be read gives no models + and its `error` (the story build then reports the failure). `data`: as for story_task.""" + cat, master_dir, _ = data or (_W["cat"], _W["master"], _W["player"]) + try: + return {"id": adv_id, "models": episode_models(cat, master_dir, adv_id)} + except ConfigError: + raise + except Exception as e: + return {"id": adv_id, "models": [], "error": f"{type(e).__name__}: {e}"[:300]} + + +def _pool_task(args: tuple[str, int]) -> dict: + kind, adv_id = args + return models_task(adv_id) if kind == "models" else story_task(adv_id) def fold_variant(site: Path, adv_id: int, prefix: str) -> bool: @@ -478,8 +516,10 @@ def add_regions(path: Path, regions: list[str]) -> bool: # ---------------------------------------------------------------- index def write_stories_index(site: Path, language: str | None = None, regions: list[dict] | None = None) -> tuple[int, set]: """site/stories.json from every story manifest present (no stories directory: no stories.json): entries in - advId, manifest order; `language` (default listing language) and `regions` of this build, else those of the - existing index. Returns the number of stories and the assets their manifests reference.""" + advId, manifest order, `size` the decoded bytes of the common files, of each language group and of the files of + the referenced models (`models`; a model manifest not present counts none); `language` (default listing + language) and `regions` of this build, else those of the existing index. Returns the number of stories and the + assets they reference (in their manifests and in the model manifests they name).""" site = Path(site) index = site / STORIES_INDEX if not (site / STORIES_DIR).is_dir(): @@ -491,13 +531,25 @@ def write_stories_index(site: Path, language: str | None = None, regions: list[d old = json.loads(index.read_text(encoding="utf-8")) except ValueError: old = {} + models: dict[str, tuple[int, set]] = {} # model manifest path -> (decoded bytes, assets) + + def model(rel: str) -> tuple[int, set]: + if rel not in models: + f = site / rel + files = json.loads(f.read_text(encoding="utf-8"))["files"].values() if f.is_file() else [] + models[rel] = (sum(e["size"] for e in files), {a for e in files for a in entry_assets(e)}) + return models[rel] entries, used = [], set() for p in story_manifests(site): man = json.loads(p.read_text(encoding="utf-8")) used |= manifest_assets(man) + refs = [model(rel) for rel in (man.get("models") or {}).values()] + for _, assets in refs: + used |= assets size = {"common": sum(e["size"] for e in man["files"].values()), "languages": {lang: sum(e["size"] for e in g["files"].values()) - for lang, g in man["languages"].items()}} + for lang, g in man["languages"].items()}, + "models": sum(n for n, _ in refs)} e = {"id": str(man["advId"]), "manifest": p.relative_to(site).as_posix(), "size": size, "audio": man["audio"], "audioFormat": man["audioFormat"], "fonts": man["fonts"], **man["story"]} if man.get("regions"): @@ -560,29 +612,33 @@ def check_languages(codes) -> list[str]: return [c for c in languages.LANGUAGES if c in set(codes)] -def _run_group(site: Path, ids: list[int], cfg: Config, job: dict, workers: int, log) -> list[dict]: - """The stories `ids` of one region group (job region / prefix / regions) -> one result per story.""" +def _run_group(site: Path, ids: list[int], cfg: Config, job: dict, workers: int, log, + kind: str = "story") -> list[dict]: + """The stories `ids` of one region group (job region / prefix / regions) -> one result per story: its build, or + with `kind` "models" its pre-pass (models_task).""" results = [] if workers <= 1 or len(ids) <= 1: data = open_data(cfg, job["region"]) - groups = story_groups(data[1]) + groups = story_groups(data[1]) if kind == "story" else None _W.update(job=job) for i in ids: - results.append(story_task(i, job, data=data, groups=groups)) + results.append(story_task(i, job, data=data, groups=groups) if kind == "story" else + models_task(i, data=data)) return results ctx = mp.get_context("spawn") with ctx.Manager() as mgr: with ctx.Pool(workers, initializer=_worker_init, initargs=(cfg, mgr.Lock(), job)) as pool: - for r in pool.imap_unordered(_pool_task, ids): + for r in pool.imap_unordered(_pool_task, [(kind, i) for i in ids]): results.append(r) if len(results) % 20 == 0: - log(f"{len(results)}/{len(ids)} stories") + log(f"{len(results)}/{len(ids)} {'stories' if kind == 'story' else 'episodes read'}") return results def build(out_dir, adv_ids, cfg: Config, player_dir, audio_format: str = "aac", audio: bool = True, force: bool = False, *, tmp_dir=None, log=None, workers: int | None = None, regions: list[str] | None = None, - story_languages=None, fonts: str = "open", fonts_flags: dict[str, str] | None = None) -> dict: + story_languages=None, fonts: str = "open", fonts_flags: dict[str, str] | None = None, + encoding: str = "gzip") -> dict: """Add the stories `adv_ids` (MasterAdv ids; None: every story of every region's master data) to the site at `out_dir`, with the player of the ournotes-player checkout or package at `player_dir` (its page files are written, the indexes rebuilt). The data comes from the settings `cfg` (each worker process opens its own). @@ -590,7 +646,10 @@ def build(out_dir, adv_ids, cfg: Config, player_dir, audio_format: str = "aac", groups (default every language); `fonts`: "open" (glyphs from the font file of each language: `fonts_flags` {language: path}, else `[paths] fonts.`; the emoji sprites from `fonts_flags` ["emoji"], else `[paths] fonts.emoji`, when given) or "game"; `regions`: as for web.build (module docstring); - `workers`: parallel story processes (default a quarter of the CPUs, up to 8).""" + `workers`: parallel story processes (default a quarter of the CPUs, up to 8) and model processes (webmodel.build); + `encoding`: the stored encoding of the assets (web.site_store). The models the stories to build load are built + first (module docstring; bundles from the CDN of the first region, names as webmodel.build gives them); the + summary's `storyModels` holds that build's model lists.""" from .web import write_index, write_player from .tmpfont import require_extra player_dir = check_player(player_dir) @@ -609,17 +668,18 @@ def build(out_dir, adv_ids, cfg: Config, player_dir, audio_format: str = "aac", groups = region_groups(masters) site = Path(out_dir).resolve() (site / STORIES_DIR).mkdir(parents=True, exist_ok=True) - Store(site) + site_store(site, encoding) tmp_root = Path(tmp_dir).resolve() if tmp_dir else site.parent / f"{site.name}.tmp" tmp_root.mkdir(parents=True, exist_ok=True) log = log or _log t0 = time.time() job0 = {"site": str(site), "tmp": str(tmp_root), "audioFormat": audio_format, "audio": bool(audio), "languages": langs, "language": default, "fonts": fonts, "fontFiles": files, "emojiFile": emoji, - "cache": str(tmp_root / "cache"), "build": uuid.uuid4().hex} + "cache": str(tmp_root / "cache"), "build": uuid.uuid4().hex, "encoding": encoding} configure_caches(job0) ids = None if adv_ids is None else list(dict.fromkeys(int(i) for i in adv_ids)) results, skipped, folded, offered, used_workers = [], [], [], set(), 1 + runs = [] # (job, todo, workers) per region group with stories to build for gi, group in enumerate(groups): rep = group[0] prefix = "" if gi == 0 else f"{rep}/" @@ -633,11 +693,21 @@ def build(out_dir, adv_ids, cfg: Config, player_dir, audio_format: str = "aac", skipped.append(prefix + str(i)) else: todo.append(i) - if not todo: - continue - w = workers if workers is not None else max(1, min(8, len(todo), usable_cpus() // 4)) + if todo: + w = workers if workers is not None else max(1, min(8, len(todo), usable_cpus() // 4)) + runs.append(({**job0, "region": rep, "prefix": prefix, "regions": group}, todo, w)) + keys: set[str] = set() + for job, todo, w in runs: + where = f" for {job['region']}" if job["prefix"] else "" + log(f"{len(todo)} episodes{where}: the Live2D models they load") + keys.update(k for r in _run_group(site, todo, cfg, job, w, log, kind="models") for k in r["models"]) + model_build = None + if keys: + model_build = webmodel.build(site, webmodel.model_keys(keys), cfg, player_dir, force, tmp_dir=tmp_root, + log=log, workers=workers, region=regions[0], encoding=encoding) + for job, todo, w in runs: + rep, prefix = job["region"], job["prefix"] used_workers = max(used_workers, w) - job = {**job0, "region": rep, "prefix": prefix, "regions": group} log(f"{len(todo)} stories{f' for {rep}' if prefix else ''}, {w} worker(s), languages {', '.join(langs)}, " f"fonts {fonts}, audio {audio_format if audio else 'none'}") results += _run_group(site, todo, cfg, job, w, log) @@ -658,5 +728,6 @@ def build(out_dir, adv_ids, cfg: Config, player_dir, audio_format: str = "aac", "storiesBuilt": [{k: r[k] for k in ("id", "files", "bytes", "size")} for r in results if r["ok"]], "storiesFailed": [{k: r.get(k) for k in ("id", "stage", "error")} for r in failed], "storiesSkipped": skipped, "storyRegionGroups": groups, "foldedStoryManifests": folded, + "storyModels": None if model_build is None else {k: model_build[k] for k in STORY_MODEL_KEYS}, "storyLanguages": langs, "storySeconds": round(time.time() - t0, 1), "storyWorkers": used_workers, **v, **idx} diff --git a/src/nnnotes/web.py b/src/nnnotes/web.py index f9492d5..29bc010 100644 --- a/src/nnnotes/web.py +++ b/src/nnnotes/web.py @@ -8,12 +8,16 @@ manifest path, sizes) /charts/_.json chart manifest: every path the player reads -> {asset, size}, or, for a large JSON object, {parts: [[key, asset, size], ...], size}; - `regions`: the regions it serves + an encoded asset adds its stored length (`stored`, a part's fourth + item); `regions`: the regions it serves /charts//.json the manifest of a region whose chart inputs differ from the first region's (only where its files differ) - /assets/. content-addressed files (text assets as UTF-8, JSON without - whitespace); charts share the note skins, effects, SE sheets, band - stages and the common parts of the scene, each stored once + /assets/.[.gz|.br] content-addressed files, named by the SHA-256 of their decoded bytes + (text assets as UTF-8, JSON without whitespace; JSON, GLSL, moc3 and + the other compress.COMPRESSIBLE files gzip- or brotli-encoded where + that makes them smaller, see Store); charts share the note skins, + effects, SE sheets, band stages and the common parts of the scene, + each stored once /models.json, models/.json Live2D model index and manifests (webmodel.py), same entry forms and the same assets/; write_index keeps what either kind references /live2d/ the player's Live2D model page and its bundle, when the player has @@ -32,8 +36,8 @@ filtered shaders.json), the BGM transcoded to the web format (`audio_format`, once per music; note SE, cheers and voices stay FLAC: small and shared), ingest. Musics run in parallel worker processes (catalog cache writes serialized by a lock); temporary directories are deleted after use. A chart whose manifest exists is skipped unless `force`. -Same inputs give byte-identical outputs. `audio=False` exports no audio file (the player then runs the chart on its -own clock, silent). +Same inputs give byte-identical outputs (the encoded assets: with the same zlib / brotli versions). `audio=False` +exports no audio file (the player then runs the chart on its own clock, silent). Regions: one build can serve several regions. The regions serve the same catalog for a language, so the chart files depend on a region's master data only: regions whose chart tables (LIVE_TABLES) are byte-identical share one manifest @@ -77,12 +81,14 @@ from pathlib import Path from types import SimpleNamespace -from . import cache, cri, jsonio, languages, live, liveoptions, livenotes, livescene +from . import cache, compress, cri, jsonio, languages, live, liveoptions, livenotes, livescene +from .compress import DEFAULT_ENCODING from .config import Config, ConfigError, tool, usable_cpus, use from .score import DIFFICULTIES, master_table from .webaudio import DEFAULT_AUDIO_FORMAT, WEB_AUDIO -SITE_FORMAT = 2 +SITE_FORMAT = 2 # charts.json "format" +MANIFEST_FORMAT = 3 # chart manifest "format" (3: entries may be stored encoded) AUDIO_EXT = {".flac", ".ogg", ".opus", ".wav", ".m4a", ".mp3"} SPLIT_MIN_BYTES = 512 * 1024 # the live directories of a web build: no liveui/ (the player reads none of it), the BGM decoded once for the read @@ -306,56 +312,110 @@ def chart_facts(live_dir: Path, summary: dict, difficulty: str, language: str) - # ---------------------------------------------------------------- store -class Store: - """site/assets/.: write-once content-addressed files (safe for concurrent writers).""" +ENCODINGS = compress.ENCODINGS # stored encodings of site assets (web --compress) + - def __init__(self, site: Path): +class Store: + """site/assets/.[.gz|.br]: write-once content-addressed files named by the SHA-256 of their decoded + bytes (safe for concurrent writers). `encoding` (compress.ENCODINGS): a file whose extension is + compress.COMPRESSIBLE is stored gzip- or brotli-encoded when that makes it smaller, under its name plus .gz / .br, + and its entry gains `stored` (the encoded length); "none" stores every file as it is.""" + + def __init__(self, site: Path, encoding: str = DEFAULT_ENCODING): + compress.check(encoding) + self.encoding = encoding self.dir = Path(site) / "assets" self.dir.mkdir(parents=True, exist_ok=True) self.written = 0 self.reused = 0 - self._have: set[str] = set() # names this store has written or found + self._have: dict[str, dict] = {} # . this store has written or found -> its entry def put(self, path: str, data: bytes) -> dict: ext = Path(path).suffix.lower().lstrip(".") or "bin" - name = f"{hashlib.sha256(data).hexdigest()}.{ext}" + key = f"{hashlib.sha256(data).hexdigest()}.{ext}" + e = self._have.get(key) + if e is None: + e = self._have[key] = self._put_encoded(key, ext, data) or self._put_raw(key, data) + else: + self.reused += 1 + return dict(e) + + def _put_raw(self, name: str, data: bytes) -> dict: dst = self.dir / name - if name in self._have: + if dst.exists() and dst.stat().st_size == len(data): self.reused += 1 - elif dst.exists() and dst.stat().st_size == len(data): + else: + self._write(dst, data) + return {"asset": f"assets/{name}", "size": len(data)} + + def _put_encoded(self, key: str, ext: str, data: bytes) -> dict | None: + """`data` stored encoded as `key` + .gz / .br, None when this store does not encode it (encoding "none", an + extension not compressible, an encoded form not smaller). A file of that name that holds `data` is kept + whatever encoder wrote it, so the entries naming it keep their `stored`.""" + if self.encoding == "none" or ext not in compress.COMPRESSIBLE: + return None + name = key + compress.SUFFIXES[self.encoding] + dst = self.dir / name + stored = dst.stat().st_size if dst.is_file() else None + if stored is not None and stored < len(data) and compress.holds(dst, data): self.reused += 1 - self._have.add(name) else: - tmp = cache.temp_path(dst) - tmp.write_bytes(data) - for i in range(20): - try: - os.replace(tmp, dst) + enc = compress.encode(self.encoding, data) + if len(enc) >= len(data): + return None + self._write(dst, enc) + stored = len(enc) + return {"asset": f"assets/{name}", "size": len(data), "stored": stored} + + def _write(self, dst: Path, stored: bytes) -> None: + tmp = cache.temp_path(dst) + tmp.write_bytes(stored) + for i in range(20): + try: + os.replace(tmp, dst) + break + except PermissionError: + if dst.exists() and dst.stat().st_size == len(stored): + tmp.unlink() break - except PermissionError: - if dst.exists() and dst.stat().st_size == len(data): - tmp.unlink() - break - time.sleep(0.1 * (i + 1)) - else: - raise RuntimeError(f"could not store {dst}") - self.written += 1 - self._have.add(name) - return {"asset": f"assets/{name}", "size": len(data)} + time.sleep(0.1 * (i + 1)) + else: + raise RuntimeError(f"could not store {dst}") + self.written += 1 def put_file(self, path: str, data: bytes) -> dict: """One file of a chart: whole, or per top-level key for a large JSON object (see split_json).""" parts = split_json(data) if path.endswith(".json") else None if parts is None: return self.put(path, data) - return {"size": len(data), "parts": [[k, self.put(f"{path}#{k}.json", t)["asset"], len(t)] for k, t in parts]} + return self.put_parts(path, parts, len(data)) + + def put_parts(self, path: str, parts: list[tuple[str, bytes]], size: int) -> dict: + """A JSON file of `size` bytes stored per top-level key (`parts`: [(key, value text)], join_parts rebuilds + it): {size, parts: [[key, asset, size], ...]}, a part [key, asset, size, stored] when encoded.""" + return {"size": size, "parts": [part_entry(k, self.put(f"{path}#{k}.json", t)) for k, t in parts]} + + +def part_entry(key: str, e: dict) -> list: + """The split-JSON part [key, asset, size(, stored)] of the Store entry `e`.""" + return [key, e["asset"], e["size"], *([e["stored"]] if "stored" in e else [])] + + +def site_store(site: Path, encoding: str = DEFAULT_ENCODING) -> Store: + """The asset store of a site, writing its assets in `encoding`.""" + return Store(site, encoding) + + +def read_asset(site: Path, asset: str) -> bytes: + """The decoded bytes of a site asset (`assets/...`; by the name's suffix, compress.decode).""" + return compress.decode(asset, (Path(site) / asset).read_bytes()) def entry_assets(e: dict) -> list[str]: return [p[1] for p in e["parts"]] if "parts" in e else [e["asset"]] -_TEXTS = cache.bucket("webtext") # (build, store, path, text) -> manifest entry +_TEXTS = cache.bucket("webtext") # (build, store, encoding, path, text) -> manifest entry def stored_text(store: Store, path: str, s: str, memo: str | None = None) -> dict: @@ -363,7 +423,7 @@ def stored_text(store: Store, path: str, s: str, memo: str | None = None) -> dic charts of a music share most files, every chart the note assets).""" if memo is None: return store.put_file(path, text_asset(path, s)) - k = _TEXTS.key(memo, str(store.dir), path, s) + k = _TEXTS.key(memo, str(store.dir), store.encoding, path, s) hit = _TEXTS.get(k) if hit is None: e = store.put_file(path, text_asset(path, s)) @@ -645,7 +705,7 @@ def ingest(store: Store, site: Path, music_id: int, difficulty: str, live_dir: P entries = {p: stored_text(store, p, s, memo) for p, s in text.items()} entries.update({p: store.put(p, b) for p, b in binary.items()}) offered = options.manifest_options() if options else None - manifest = {"format": SITE_FORMAT, "musicId": music_id, "difficulty": difficulty, "audio": bool(audio), + manifest = {"format": MANIFEST_FORMAT, "musicId": music_id, "difficulty": difficulty, "audio": bool(audio), "audioFormat": audio_format if audio else None, "flows": flows, "quality": TRACE_QUALITY, **({"options": offered} if offered else {}), "chart": facts, "files": dict(sorted(entries.items()))} path = manifest_file(site, f"{music_id}_{difficulty}", prefix) @@ -730,7 +790,7 @@ def ingest_task(music_id: int, charts: list, root: str, cfg: dict | None = None) """Ingest the charts [(difficulty, dir, summary, files)] of one music; one result per chart.""" cfg = cfg or _W["cfg"] site = Path(cfg["site"]) - store, bgm, out = Store(site), {}, [] + store, bgm, out = site_store(site, cfg.get("encoding", DEFAULT_ENCODING)), {}, [] for d, pdir, summary, files in charts: try: r = ingest(store, site, music_id, d, Path(pdir), summary, files, list(FLOWS), cfg["audioFormat"], @@ -1341,14 +1401,14 @@ def write_player(site: Path, player_dir: Path, stories: bool = False) -> dict: def entry_text(site: Path, e: dict) -> bytes: """The file text of a manifest entry (a split JSON file rejoined).""" if "parts" in e: - return join_parts([(k, (site / a).read_bytes()) for k, a, _ in e["parts"]]) - return (site / e["asset"]).read_bytes() + return join_parts([(q[0], read_asset(site, q[1])) for q in e["parts"]]) + return read_asset(site, e["asset"]) -def reingest_json(site: Path) -> dict: +def reingest_json(site: Path, encoding: str = DEFAULT_ENCODING) -> dict: """Every JSON file of every chart and model stored again through text_asset (after a change of the stored-text - rules); the manifests follow, unreferenced assets go with write_index.""" - store, changed, memo = Store(site), 0, {} + rules), in `encoding`; the manifests follow, unreferenced assets go with write_index.""" + store, changed, memo = site_store(site, encoding), 0, {} for mf in [*chart_manifests(site), *sorted((site / MODELS_DIR).glob("*.json"))]: man = json.loads(mf.read_text(encoding="utf-8")) files = man["files"] @@ -1391,7 +1451,8 @@ def write_index(site: Path, language: str | None = None, regions: list[dict] | N """site/charts.json from every chart manifest present (index_meta: `language`, `regions` of this build), site/models.json from every model manifest (webmodel.write_models_index) and site/stories.json from every story manifest (storysite.write_stories_index; its `language` and `regions` are those storysite.build wrote); assets - no chart, no model and no story (common files and every language group) references removed.""" + no chart, no model and no story (common files and every language group) references removed. The summary's + `assetBytes` is the size of the asset files, `bytes` their decoded size (compress.decoded_size).""" from .storysite import write_stories_index from .webmodel import write_models_index order = {d: i for i, d in enumerate(DIFFICULTIES)} @@ -1419,8 +1480,9 @@ def write_index(site: Path, language: str | None = None, regions: list[dict] | N if f"assets/{f.name}" not in used: f.unlink() removed += 1 - sizes = [f.stat().st_size for f in (site / "assets").iterdir()] - out = {"charts": len(charts), "models": models, "assets": len(sizes), "assetBytes": sum(sizes), + files = list((site / "assets").iterdir()) + out = {"charts": len(charts), "models": models, "assets": len(files), + "assetBytes": sum(f.stat().st_size for f in files), "bytes": sum(map(compress.decoded_size, files)), "removedAssets": removed} if stories: out["stories"] = stories @@ -1549,7 +1611,8 @@ def read(live_dir, chart_id): def build(out_dir, pairs, cfg: Config, player_dir: Path, audio_format: str = DEFAULT_AUDIO_FORMAT, audio: bool = True, force: bool = False, *, tmp_dir=None, log=None, workers: int | None = None, band: int | None = None, leader_card: int | None = None, regions: list[str] | None = None, - fonts: str = "open", read_workers: int | None = None, live_options: dict | None = None) -> dict: + fonts: str = "open", read_workers: int | None = None, live_options: dict | None = None, + encoding: str = DEFAULT_ENCODING) -> dict: """Add the charts `pairs` ([(musicId, difficulty)]; None: every chart of every region's master data) to the site at `out_dir`, with the player of the ournotes-player checkout or package at `player_dir`. The data comes from the settings `cfg` (each worker process opens its own). `regions`: the regions the charts serve (default: the one @@ -1559,7 +1622,8 @@ def build(out_dir, pairs, cfg: Config, player_dir: Path, audio_format: str = DEF up to 16). `band` / `leader_card`: the band of every chart's stage, as for live.build (default: the band of the music's first vocal character); `fonts`: the start canvas fonts (liveui.extract, with WEB_LIVEUI); `live_options`: the Live option variants the charts offer (a liveoptions.parse_specs request; checked against - every region's master data before anything is built, liveoptions.OptionSpecError).""" + every region's master data before anything is built, liveoptions.OptionSpecError); `encoding`: how the assets + are stored (Store).""" player_dir = check_player(player_dir) if audio_format not in WEB_AUDIO: raise ValueError(f"audio format {audio_format}: one of {', '.join(WEB_AUDIO)}") @@ -1576,14 +1640,15 @@ def build(out_dir, pairs, cfg: Config, player_dir: Path, audio_format: str = DEF groups = region_groups(masters) site = Path(out_dir).resolve() (site / "charts").mkdir(parents=True, exist_ok=True) - Store(site) + site_store(site, encoding) tmp_root = Path(tmp_dir).resolve() if tmp_dir else site.parent / f"{site.name}.tmp" tmp_root.mkdir(parents=True, exist_ok=True) log = log or _log t0 = time.time() job = {"site": str(site), "tmp": str(tmp_root), "audioFormat": audio_format, "audio": bool(audio), "player": str(player_dir), "language": language, "fonts": fonts, "band": band, "leaderCard": leader_card, - "cache": str(tmp_root / "cache"), "build": uuid.uuid4().hex, "liveOptions": live_options or None} + "cache": str(tmp_root / "cache"), "build": uuid.uuid4().hex, "liveOptions": live_options or None, + "encoding": encoding} configure_caches(job) results, skipped, folded, used_workers = [], [], [], 1 offered = set() diff --git a/src/nnnotes/webmodel.py b/src/nnnotes/webmodel.py index b98bbf6..6bd83e6 100644 --- a/src/nnnotes/webmodel.py +++ b/src/nnnotes/webmodel.py @@ -1,40 +1,50 @@ """Live2D models -> the model part of an ournotes-player site (web.py builds the charts of the same site). /models.json model index: id, manifest path, key, listing facts, sizes - /models/.json model manifest: every path of the model's files -> {asset, size}, or, for a large - JSON object, {parts: [[key, asset, size], ...], size} (as a chart manifest) + /models/.json model manifest (format web.MANIFEST_FORMAT): every path of the model's files -> + {asset, size}, or, for a large JSON object, {parts: [[key, asset, size], ...], size} + (as a chart manifest) /assets/. content-addressed files, shared with the charts (the models share their shaders) The files of one model (the paths in its manifest), only those the player reads (read_files): model.json index: format (MODEL_FORMAT), name, key, moc3, prefab, textures, canvas, shaders, - resources + resources, motionSync .moc3 the moc3 (as live2d.extract_runtime writes it) .prefab.json the whole prefab, clips, expressions and controllers inlined (the same) textures/*.png the atlas pages the drawables draw with (the same) shaders/shaders.json the shader index (shader.py layout, filtered as in the chart sites: web.collect), shaders/.json the parsed forms and the GLES3 programs (GLSL ES 3.00) the drawables' materials - shaders//gles3/*.glsl select by their keywords; the mask shader's only when a drawable is masked + shaders//gles3/*.glsl select by their keywords, each also with STORY_KEYWORDS (the keywords the story + renderer adds to a character draw); the mask shader's only when a drawable is masked -`resources` in model.json: the materials the Cubism mask pass draws with (the Resources.Load paths of -advscene.RESOURCES, read from the APK's boot data), exported as the story scene exports them. +`resources` in model.json: the materials the Cubism mask pass draws with (the Resources.Load paths of RESOURCES, read +from the APK's boot data). `motionSync`: the prefab's root has the MotionSync controller with its CRI audio input +(has_motion_sync); a model without it takes its mouth in a story from the voice's CRI Lips analysis. Models are the catalog keys Character/Live2D///model/; a model's id is (unique in the catalog, URL-safe). Each model is exported into a temporary directory, stored, and the directory deleted; models run in -parallel worker processes (catalog cache writes serialized by a lock). A model whose manifest exists is skipped unless -`force`. Same inputs and library versions give byte-identical outputs. A model's files read no master data and the -regions serve the same catalog, so one build serves every region of a site (the bundles are fetched from the CDN of -`region`). +parallel worker processes (catalog cache writes serialized by a lock). A model whose manifest exists and is current +is skipped unless `force`; an outdated one (`outdated`: its model.json older than MODEL_FORMAT or without +motionSync, or unreadable) is built again. Same inputs and library versions give byte-identical outputs. A model's +files read no master data and the regions serve the same catalog, so one build serves every region of a site (the +bundles are fetched from the CDN of `region`). Names (optional): when master data is configured for `region`, the listing of each model the master data maps to a character (model_names: MasterCharacterCostume -> MasterCharacter -> MasterText) gets `character`, `names` and `label`; the manifests of skipped models get the same fields of this build. Without master data the fields are not written (and skipped manifests keep theirs). + +Model sources: story.build takes the models of an episode from a source, `ensure(address) -> {"id", "motionSync"}` +plus `story_fields(story_dir)` (what story.json gets besides `models`). ModelDir exports each model into // +(the files above; `nnnotes story`), SiteModels reads model.json from the model manifests of a site (the site's +stories). """ from __future__ import annotations import json import multiprocessing as mp +import os import re import shutil import tempfile @@ -42,20 +52,31 @@ import traceback from pathlib import Path -from . import advscene, jsonio, languages, live2d, master +from . import jsonio, languages, live2d, master from . import shader as shader_mod from .config import Config, usable_cpus, use from .export import Exporter -from .web import (MODELS_DIR, MODELS_INDEX, SHADER_PLATFORM, SHADER_TYPE, SITE_FORMAT, Store, _dump, _lock_fetches, - _log, check_player, collect, entry_assets, text_asset, write_index, write_player) +from .web import (MANIFEST_FORMAT, MODELS_DIR, MODELS_INDEX, SHADER_PLATFORM, SHADER_TYPE, SITE_FORMAT, Store, _dump, + _lock_fetches, _log, check_player, collect, entry_assets, entry_text, site_store, text_asset, + write_index, write_player) LIVE2D_PREFIX = "Character/Live2D/" MODEL_KEY = re.compile(r"Character/Live2D/(?P[^/]+)/(?P[^/]+)/model/(?P=name)") MODEL_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*") MODEL_INDEX = "model.json" -MODEL_FORMAT = 1 # model.json "format" +MODEL_FORMAT = 2 # model.json "format" SHADER_DIR = "shaders" MASK_KEYWORD = "CUBISM_MASK_ON" # a drawable material's keyword: the drawable is masked +# keywords the story renderer adds to a character draw (_ADDITIONAL_LIGHTS_VERTEX: at quality 4); the mask pass draws +# with none +STORY_KEYWORDS = ("_ADDITIONAL_LIGHTS_VERTEX",) +# Resources.Load paths of the materials the Cubism mask pass draws with +RESOURCES = { + "cubismMask": "Live2D/Cubism/Materials/Mask", + "cubismMaskCulling": "Live2D/Cubism/Materials/MaskCulling", +} +# a model's MotionSync path (the player's CubismMotionSyncController.fromPrefab): both components on the prefab's root +MOTION_SYNC_COMPONENTS = ("CubismMotionSyncController", "Live2DMotionSyncCriAudioInput") COSTUME_TABLE = "MasterCharacterCostume" # _live2dPath (the key after LIVE2D_PREFIX) -> _characterID NAME_FIELDS = ("character", "names", "label") # listing fields from the master data (model_names) @@ -187,20 +208,32 @@ def drawable_materials(prefab: dict) -> list[dict]: if c.get("type") == "MeshRenderer" and c.get("m_Materials") and c["m_Materials"][0]] +def has_motion_sync(prefab: dict) -> bool: + """The exported prefab (live2d.extract_runtime) has a MotionSync controller with its CRI audio input on its root + node; a model without one takes its mouth in a story from the voice's CRI Lips analysis.""" + classes = {c.get("class") or c.get("type") for n in prefab["nodes"] if "/" not in n["path"] + for c in n["components"]} + return all(k in classes for k in MOTION_SYNC_COMPONENTS) + + def drawable_textures(prefab: dict) -> list[str]: """The atlas pages the drawables draw with (every distinct CubismRenderer._mainTexture), sorted.""" return sorted({c["_mainTexture"]["texture"] for n in prefab["nodes"] for c in n["components"] if c.get("class") == "CubismRenderer" and c.get("_mainTexture")}) -def shader_files(index: list, materials: list[dict]) -> list[str]: +def shader_files(index: list, materials: list[dict], extra: tuple[str, ...] = ()) -> list[str]: """The files of a shader directory (paths relative to it) that `materials` draw with: per shader its parsed file and, for each distinct keyword set of its materials, the GLES3 program of subshader 0 pass 0 whose keywords are - exactly the material's keywords among those the shader's programs there use (a global keyword such as - _ADDITIONAL_LIGHTS is never in a material's set, so its variants are never picked).""" + exactly the material's keywords among those the shader's programs there use; with `extra`, also the program of + the set plus `extra` (the same program when the shader's programs use none of them). A global keyword such as + _ADDITIONAL_LIGHTS is never in a material's set, so its variants are picked only through `extra`.""" recs = {r["name"]: r for r in index} + sets = {(m["shader"]["shader"], tuple(m["keywords"])) for m in materials} + if extra: + sets |= {(name, (*kws, *extra)) for name, kws in sets} files = set() - for name, kws in sorted({(m["shader"]["shader"], tuple(m["keywords"])) for m in materials}): + for name, kws in sorted(sets): if name not in recs: raise RuntimeError(f"shader {name}: not in the shader index") variants = [v for v in recs[name]["variants"] if v["platform"] == SHADER_PLATFORM and v["type"] == SHADER_TYPE @@ -215,14 +248,16 @@ def shader_files(index: list, materials: list[dict]) -> list[str]: def read_files(doc: dict, prefab: dict, index: list) -> list[str]: """The files of a model the player reads (model.json `doc`, its prefab, its shader index): model.json, the moc3, - the prefab, the drawables' atlas pages, the shader index and the shader files of the drawables' materials, plus - those of the mask materials when a drawable is masked (MASK_KEYWORD).""" + the prefab, the drawables' atlas pages, the shader index and the shader files of the drawables' materials (each + keyword set alone and with STORY_KEYWORDS), plus those of the mask materials when a drawable is masked + (MASK_KEYWORD).""" materials = drawable_materials(prefab) + files = set(shader_files(index, materials, STORY_KEYWORDS)) if any(MASK_KEYWORD in m["keywords"] for m in materials): - materials += list(doc["resources"].values()) + files |= set(shader_files(index, list(doc["resources"].values()))) sdir = doc["shaders"].rpartition("/")[0] return sorted({MODEL_INDEX, doc["moc3"], doc["prefab"], *doc["textures"], doc["shaders"], - *(f"{sdir}/{f}" if sdir else f for f in shader_files(index, materials))}) + *(f"{sdir}/{f}" if sdir else f for f in files)}) def export_model(cat, player, key: str, out_dir: Path) -> dict: @@ -233,7 +268,7 @@ def export_model(cat, player, key: str, out_dir: Path) -> dict: res, ex = rt.summary, rt.exporter prefab = json.loads((out_dir / res["prefab"]).read_text(encoding="utf-8")) rex = Exporter(cat, out_dir, player=player) - resources = {name: rex.material(player.resource(path)) for name, path in advscene.RESOURCES.items()} + resources = {name: rex.material(player.resource(path)) for name, path in RESOURCES.items()} objs = {**rex.shaders, **ex.shaders} names = sorted(shader_names(prefab) | shader_names(resources)) missing = [n for n in names if n not in objs] @@ -245,7 +280,7 @@ def export_model(cat, player, key: str, out_dir: Path) -> dict: shader_mod.write_index(index, out_dir / SHADER_DIR) doc = {"format": MODEL_FORMAT, "name": res["name"], "key": key, "moc3": res["moc3"], "prefab": res["prefab"], "textures": drawable_textures(prefab), "canvas": prefab["canvas"], "shaders": f"{SHADER_DIR}/shaders.json", - "resources": resources} + "resources": resources, "motionSync": has_motion_sync(prefab)} jsonio.write_json(out_dir / MODEL_INDEX, doc) return {**res, "textures": doc["textures"], "canvas": prefab["canvas"], "shaders": names, "files": read_files(doc, prefab, index)} @@ -266,12 +301,106 @@ def ingest(store: Store, site: Path, mid: str, key: str, model_dir: Path, summar text, binary = collect(Path(model_dir), summary["files"]) entries = {p: store.put_file(p, text_asset(p, s)) for p, s in text.items()} entries.update({p: store.put(p, b) for p, b in binary.items()}) - manifest = {"format": SITE_FORMAT, "id": mid, "key": key, "model": model_facts(key, summary, names), + manifest = {"format": MANIFEST_FORMAT, "id": mid, "key": key, "model": model_facts(key, summary, names), "files": dict(sorted(entries.items()))} (Path(site) / MODELS_DIR / f"{mid}.json").write_bytes(_dump(manifest)) return {"id": mid, "ok": True, "files": len(entries), "bytes": sum(e["size"] for e in entries.values())} +# ---------------------------------------------------------------- model sources of story.build +def source_entry(address: str, doc: dict, where: str) -> dict: + """ensure's answer from the model.json `doc` of the model `address` (read from `where`).""" + if doc.get("key") != address: + raise RuntimeError(f"{where}: model.json of {doc.get('key')}, not of {address}") + if not isinstance(doc.get("motionSync"), bool): + raise RuntimeError(f"{where}: model.json of format {doc.get('format')} has no motionSync; export the model " + f"again (--force)") + return {"id": model_id(address), "motionSync": doc["motionSync"]} + + +class ModelDir: + """The models of `root`: each model in //, the files of a model manifest (read_files; the shader index + reduced to the listed GLES3 programs as web.collect reduces it) in the layout export_model writes. ensure exports + a model whose directory does not exist, or every model once when `force`; `built` / `skipped`: the ids.""" + + def __init__(self, cat, player, root: Path, force: bool = False): + self.cat, self.player, self.root, self.force = cat, player, Path(root), force + self.built: list[str] = [] + self.skipped: list[str] = [] + self._done: dict[str, dict] = {} + + def ensure(self, address: str) -> dict: + mid = model_id(address) + if mid not in self._done: + d = self.root / mid + if self.force or not d.exists(): + self._export(address, d) + self.built.append(mid) + else: + self.skipped.append(mid) + index = d / MODEL_INDEX + if not index.is_file(): + raise RuntimeError(f"{d}: no {MODEL_INDEX} (not a model directory); remove it or export again " + f"(--force)") + self._done[mid] = source_entry(address, json.loads(index.read_text(encoding="utf-8")), str(index)) + return self._done[mid] + + def _export(self, address: str, d: Path) -> None: + """The model into a temporary directory next to `d`, then renamed to `d` (a previous `d` removed).""" + self.root.mkdir(parents=True, exist_ok=True) + work = Path(tempfile.mkdtemp(prefix=f".{d.name}-", dir=self.root)) + try: + summary = export_model(self.cat, self.player, address, work / "export") + text, binary = collect(work / "export", summary["files"]) + files = {**{p: t.encode("utf-8") for p, t in text.items()}, **binary} + for rel, data in sorted(files.items()): + (work / "model" / rel).parent.mkdir(parents=True, exist_ok=True) + (work / "model" / rel).write_bytes(data) + if d.exists(): + shutil.rmtree(d) + (work / "model").rename(d) + finally: + shutil.rmtree(work, ignore_errors=True) + + def story_fields(self, story_dir: Path) -> dict: + """`modelsDir`: the relative path from the story directory to `root` (posix).""" + return {"modelsDir": Path(os.path.relpath(os.path.abspath(self.root), os.path.abspath(story_dir))).as_posix()} + + +class SiteModels: + """The models of the site at `site`: ensure reads model.json from the model manifest models/.json (the models + are built before the stories: storysite.build).""" + + def __init__(self, site: Path): + self.site = Path(site) + + def ensure(self, address: str) -> dict: + where = f"{MODELS_DIR}/{model_id(address)}.json" + p = self.site / where + if not p.is_file(): + raise RuntimeError(f"{address}: no model manifest {where} in the site") + man = json.loads(p.read_text(encoding="utf-8")) + return source_entry(address, json.loads(entry_text(self.site, man["files"][MODEL_INDEX])), where) + + def story_fields(self, story_dir: Path) -> dict: + return {} + + +def outdated(site: Path, mid: str) -> str | None: + """Why the existing model manifest of `mid` is built again: "outdated" (its model.json has a format older than + MODEL_FORMAT or no motionSync), "unreadable" (the manifest or its model.json cannot be read); None when it is + current.""" + try: + man = json.loads((Path(site) / MODELS_DIR / f"{mid}.json").read_text(encoding="utf-8")) + doc = json.loads(entry_text(Path(site), man["files"][MODEL_INDEX])) + fmt, ms = doc.get("format"), doc.get("motionSync") + except Exception: + return "unreadable" + if not isinstance(fmt, int) or fmt < MODEL_FORMAT or not isinstance(ms, bool): + return "outdated" + return None + + # ---------------------------------------------------------------- index def write_models_index(site: Path) -> tuple[int, set[str]]: """site/models.json from every model manifest present (no models directory: no models.json); returns the number @@ -319,7 +448,8 @@ def model_task(mid: str, key: str, job: dict | None = None, data=None) -> dict: try: summary = export_model(cat, player, key, work) stage = "ingest" - r = ingest(Store(site), site, mid, key, work, summary, (job.get("names") or {}).get(key)) + r = ingest(site_store(site, job["encoding"]), site, mid, key, work, summary, + (job.get("names") or {}).get(key)) return {**r, "seconds": round(time.time() - t0, 1)} except Exception as e: cause = f"{type(e).__name__}: {e}" @@ -335,16 +465,18 @@ def _pool_task(args): # ---------------------------------------------------------------- build def build(out_dir, models: dict[str, str] | None, cfg: Config, player_dir, force: bool = False, *, tmp_dir=None, - log=None, workers: int | None = None, region: str | None = None) -> dict: + log=None, workers: int | None = None, region: str | None = None, encoding: str = "gzip") -> dict: """Add the Live2D models `models` ({id: key}, see `catalog_models`; None: every model of the catalog) to the site at `out_dir`, with the player of the ournotes-player checkout or package at `player_dir` (its page files are written, models.json and charts.json rebuilt). The data comes from the settings `cfg` (each worker process opens its own), bundles from the CDN of `region` (default: [catalog] region), names from the master data of `region` - when it is configured (master_names). `workers`: parallel model processes (default up to 4).""" + when it is configured (master_names). `workers`: parallel model processes (default up to 4); `encoding`: the + stored encoding of the assets (web.site_store). Without `force`, an existing manifest that is not current + (`outdated`) is built again and listed in modelsRebuilt with the reason.""" player_dir = check_player(player_dir) site = Path(out_dir).resolve() (site / MODELS_DIR).mkdir(parents=True, exist_ok=True) - Store(site) + site_store(site, encoding) tmp_root = Path(tmp_dir).resolve() if tmp_dir else site.parent / f"{site.name}.tmp" tmp_root.mkdir(parents=True, exist_ok=True) log = log or _log @@ -352,21 +484,24 @@ def build(out_dir, models: dict[str, str] | None, cfg: Config, player_dir, force if models is None: from .cli import open_catalog models = catalog_models(open_catalog(cfg, bundles=False, region=region)) - todo, skipped = [], [] + todo, skipped, rebuilt = [], [], [] for mid, key in sorted(models.items()): if model_id(key) != mid: raise ValueError(f"model {mid}: key {key} has the id {model_id(key)}") if (site / MODELS_DIR / f"{mid}.json").exists() and not force: - skipped.append(mid) - else: - todo.append((mid, key)) + why = outdated(site, mid) + if why is None: + skipped.append(mid) + continue + rebuilt.append({"id": mid, "reason": why}) + todo.append((mid, key)) if workers is None: workers = max(1, min(4, len(todo), usable_cpus() // 2)) names = master_names(cfg, region, log) if names is not None: for mid in skipped: refresh_names(site, mid, names.get(models[mid])) - job = {"site": str(site), "tmp": str(tmp_root), "region": region, "names": names or {}} + job = {"site": str(site), "tmp": str(tmp_root), "region": region, "names": names or {}, "encoding": encoding} results = [] if todo: log(f"{len(todo)} models, {workers} worker(s)") @@ -392,6 +527,7 @@ def build(out_dir, models: dict[str, str] | None, cfg: Config, player_dir, force return {"site": str(site), "modelsBuilt": [{k: r[k] for k in ("id", "files", "bytes")} for r in results if r["ok"]], "modelsFailed": [{k: r.get(k) for k in ("id", "stage", "error")} for r in failed], - "modelsSkipped": skipped, "modelSeconds": round(time.time() - t0, 1), "modelWorkers": workers, + "modelsSkipped": skipped, "modelsRebuilt": rebuilt, "modelSeconds": round(time.time() - t0, 1), + "modelWorkers": workers, "modelNames": None if names is None else sum(1 for k in models.values() if k in names), **v, **idx} diff --git a/tests/conftest.py b/tests/conftest.py index 840df39..d9250cb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,11 +7,13 @@ @pytest.fixture(autouse=True) def isolated(tmp_path, monkeypatch): - """Every test runs in its own directory (no ./nnnotes.toml), without NNNOTES_* variables of the caller and - with no active settings left over.""" + """Every test runs in its own directory (no ./nnnotes.toml), without NNNOTES_* variables of the caller, with an + empty per-user config directory and with no active settings left over.""" for k in list(os.environ): if k.startswith("NNNOTES_"): monkeypatch.delenv(k) + monkeypatch.setenv("APPDATA", str(tmp_path / "user-config")) + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "user-config")) monkeypatch.chdir(tmp_path) monkeypatch.setattr(config, "_active", None) yield tmp_path diff --git a/tests/test_cli.py b/tests/test_cli.py index 71f5228..1558b09 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -111,7 +111,8 @@ def test_missing_settings_exit_2_and_name_the_setting(catalog_file, tmp_path, cd assert code == 2 and out == "" assert err.strip() == ("nnnotes: a_01.bundle is not in the cache: setting catalog.region is not set: give it " "as `region` in the [catalog] table of the config file, the environment variable " - "NNNOTES_CATALOG_REGION or --region") + "NNNOTES_CATALOG_REGION or --region (no config file was found: `nnnotes config init` " + "writes one to fill in)") code, _, err = run(base + ["--region", "zz", "pull", "Char/A"], capsys) assert code == 2 and "servers.zz.cdn" in err and "NNNOTES_SERVERS_ZZ_CDN" in err monkeypatch.setenv("NNNOTES_SERVERS_ZZ_CDN", cdn) diff --git a/tests/test_compress.py b/tests/test_compress.py new file mode 100644 index 0000000..ea20755 --- /dev/null +++ b/tests/test_compress.py @@ -0,0 +1,67 @@ +import gzip +import json +import random +import struct +import zlib + +import pytest + +from nnnotes import compress + +TEXT = json.dumps({str(i): [i, i * i, f"v{i % 7}"] for i in range(500)}).encode() +NOISE = random.Random(7).randbytes(4096) + + +@pytest.mark.parametrize("data", [b"", b"a", TEXT, NOISE]) +def test_gzip_bytes_round_trip(data): + out = compress.gzip_bytes(data) + assert out[:10] == bytes.fromhex("1f 8b 08 00 00 00 00 00 02 ff") + assert out[-8:] == struct.pack("") + else: + known.add(f"{s.section}.{s.key}") + assert documented == known + + +def test_template_values_are_empty(): + data = tomllib.loads(template().decode("utf-8")) + assert all(v in ("", []) for _, v in leaves(data)) + secrets = {f"{s.section}.{s.key}" for s in configfile.SETTINGS if s.secret} + assert secrets == {"bundle.key", "bundle.nonce_seed", "master.key", "master.iv"} + + +def test_resolve(): + assert configfile.resolve("servers.kr.cdn")[1] == "servers.kr" + assert configfile.resolve("paths.fonts.zh-Hant")[0].kind == "path" + for bad in ("bundle.ky", "servers.tw.nope", "servers.a b.cdn", "paths.fonts.xx", "key"): + with pytest.raises(ConfigError, match="unknown setting"): + configfile.resolve(bad) + + +def test_init_with_values(tmp_path, capsys): + code, out, err = run(["config", "init", "--set", f"bundle.key={KEY}", "--set", "servers.en.cdn=https://cdn.test/", + "--set", "servers.en.languages=en, ja", "--set", "catalog.region=en", + "--set", "paths.cache=cache"], capsys) + assert code == 0 and KEY not in out + err + f = tmp_path / "nnnotes.toml" + assert out.strip() == str(f) + data = tomllib.loads(f.read_text(encoding="utf-8")) + assert data["bundle"]["key"] == KEY and data["catalog"]["region"] == "en" + assert data["servers"] == {"en": {"name": "", "cdn": "https://cdn.test/", "api": "", "languages": ["en", "ja"], + "master": ""}} # the example region table renamed + assert data["paths"]["cache"] == str((tmp_path / "cache").absolute()) + assert "# CDN base URL of the region" in f.read_text(encoding="utf-8") # the comments stay + if os.name != "nt": + assert f.stat().st_mode & 0o777 == 0o600 + code, _, err = run(["config", "init"], capsys) + assert code == 2 and "exists" in err + code, _, _ = run(["config", "init", "--force", "--no-input"], capsys) + assert code == 0 and f.read_bytes() == template() + + +def test_init_asks_nothing_outside_a_terminal(tmp_path, capsys): + code, _, _ = run(["config", "init"], capsys) # pytest's stdin is not a terminal + assert code == 0 and (tmp_path / "nnnotes.toml").read_bytes() == template() + + +def test_init_user_file(tmp_path, capsys): + code, out, err = run(["config", "init", "--user", "--no-input"], capsys) + target = tmp_path / "user-config" / "nnnotes" / "nnnotes.toml" + assert code == 0 and out.strip() == str(target) and target.is_file() + code, out, _ = run(["config", "path", "--json"], capsys) + doc = json.loads(out) + assert doc["schema"] == configfile.PATHS and doc["read"] == str(target.absolute()) + assert [f["state"] for f in doc["files"]] == ["not given", "not given", "not found", "read"] + run(["config", "init", "--no-input"], capsys) # ./nnnotes.toml comes first + code, _, err = run(["config", "init", "--user", "--no-input", "--force"], capsys) + assert "the commands read" in err + doc = json.loads(run(["config", "path", "--json"], capsys)[1]) + assert [f["state"] for f in doc["files"]][2:] == ["read", "found, not read"] + + +def test_init_value_errors_name_the_setting_only(capsys): + code, _, err = run(["config", "init", "--set", "master.key=zz" + KEY], capsys) + assert code == 2 and "master.key" in err and KEY not in err + code, _, err = run(["config", "init", "--set", "catalog.language=xx"], capsys) + assert code == 2 and "languages" in err + code, _, err = run(["config", "init", "--set", "servers.tw.api=https://host/path"], capsys) + assert code == 2 and "servers.tw.api" in err + code, _, err = run(["config", "init", "--set", "bundle.key"], capsys) + assert code == 2 and "SECTION.KEY=VALUE" in err + + +def test_prompt_asks_every_setting(): + answers = iter([ + "zz", # bundle.key: invalid, asked again + KEY, "", MKEY, "", # bundle.key, nonce_seed, master.key, master.iv + "tw,kr", # regions + "", "zh-Hant", # catalog.region (default tw), language + ]) + secret_prompts, prompts = [], [] + + def ask(p): + prompts.append(p) + return next(answers, "") + + def ask_secret(p): + secret_prompts.append(p) + return next(answers, "") + + values, regions = configfile.prompt(ask=ask, ask_secret=ask_secret, say=lambda *a: None) + got = {f"{s}.{k}": v for s, k, v in values} + assert regions == ["tw", "kr"] + assert got == {"bundle.key": KEY, "master.key": MKEY, "catalog.region": "tw", "catalog.language": "zh-Hant"} + assert len(secret_prompts) == 5 and all("(hidden)" in p for p in secret_prompts) + assert any(p.startswith("[servers.kr] cdn") for p in prompts) + text = configfile.render(values, regions) + assert set(tomllib.loads(text)["servers"]) == {"tw", "kr"} + + +def test_set_and_unset_edit_in_place(tmp_path, capsys, monkeypatch): + run(["config", "init", "--no-input"], capsys) + f = tmp_path / "nnnotes.toml" + before = f.read_text(encoding="utf-8").splitlines() + code, out, _ = run(["config", "set", "bundle.key", KEY], capsys) + assert code == 0 and KEY not in out + monkeypatch.setattr("sys.stdin", io.StringIO(MKEY + "\n")) + code, out, _ = run(["config", "set", "master.key", "-"], capsys) + assert code == 0 and MKEY not in out + run(["config", "set", "servers.kr.languages", "ko"], capsys) + after = f.read_text(encoding="utf-8").splitlines() + changed = [line for line in before if line not in after] + assert changed == ['key = ""', 'key = ""'] # only the two values' lines + data = tomllib.loads("\n".join(after)) + assert data["bundle"]["key"] == KEY and data["master"]["key"] == MKEY + assert data["servers"]["kr"]["languages"] == ["ko"] and "tw" in data["servers"] + code, _, _ = run(["config", "unset", "servers.kr.languages"], capsys) + assert code == 0 and tomllib.loads(f.read_text(encoding="utf-8"))["servers"]["kr"]["languages"] == [] + code, _, err = run(["config", "set", "bundle.kye", "1"], capsys) + assert code == 2 and "unknown setting" in err + + +def test_set_needs_a_file(tmp_path, capsys): + code, _, err = run(["config", "set", "bundle.key", KEY], capsys) + assert code == 2 and "config init" in err + code, _, _ = run(["config", "set", "--file", str(tmp_path / "x.toml"), "bundle.key", KEY], capsys) + assert code == 0 and tomllib.loads((tmp_path / "x.toml").read_text(encoding="utf-8"))["bundle"]["key"] == KEY + + +def test_edit_refuses_what_it_cannot_edit(): + text = '[servers.tw]\nlanguages = [\n "ja",\n]\n' + with pytest.raises(ConfigError, match="several lines"): + configfile.edit(text, [("servers.tw", "languages", ["en"])], "f") + with pytest.raises(ConfigError, match="edit the file itself"): + configfile.edit('[paths]\nfonts.ja = "a"\n', [("paths.fonts", "ja", "b")], "f") + + +def test_check_reports_states_not_values(tmp_path, capsys, monkeypatch): + (tmp_path / "nnnotes.toml").write_text( + f'[bundle]\nkey = "{KEY}"\n[master]\niv = "00"\n[catalog]\nregion = "xx"\n[servers.tw]\ncdn = "ftp://h"\n' + f'[paths]\napk = "missing.apk"\ncache = "c"\n[typo]\nkey = "1"\n', encoding="utf-8") + monkeypatch.setenv("NNNOTES_BUNDLE_NONCE_SEED", "ab") + monkeypatch.setenv("NNNOTES_BUNDLE_KY", "ab") + code, out, _ = run(["config", "check", "--json"], capsys) + assert code == 1 and KEY not in out and '"ab"' not in out + doc = json.loads(out) + by = {s["name"]: s for s in doc["settings"]} + assert doc["schema"] == configfile.CHECK and doc["file"] == str((tmp_path / "nnnotes.toml").absolute()) + assert by["bundle.key"]["status"] == "ok" and by["bundle.key"]["secret"] + assert (by["bundle.nonce_seed"]["origin"], by["bundle.nonce_seed"]["status"]) == ("env", "ok") + assert by["master.iv"]["status"] == "invalid" and "32 bytes" in by["master.iv"]["reason"] + assert by["master.key"]["status"] == "unset" + assert by["catalog.region"]["status"] == "invalid" + assert by["servers.tw.cdn"]["status"] == "invalid" + assert by["paths.apk"]["status"] == "not found" and by["paths.cache"]["status"] == "ok" + assert by["typo.key"]["status"] == "unknown" and by["NNNOTES_BUNDLE_KY"]["status"] == "unknown" + assert doc["problems"] == 6 + code, out, _ = run(["--region", "tw", "config", "check"], capsys) + assert "catalog.region\tflag\tok" in out and KEY not in out diff --git a/tests/test_crilips.py b/tests/test_crilips.py index e286d59..10a2354 100644 --- a/tests/test_crilips.py +++ b/tests/test_crilips.py @@ -168,10 +168,11 @@ def _prefab(*classes): def test_has_motion_sync_needs_both_components_on_the_root(): - from nnnotes import story - assert story.has_motion_sync(_prefab("CubismModel", "CubismMotionSyncController", "Live2DMotionSyncCriAudioInput")) - assert not story.has_motion_sync(_prefab("CubismModel", "CubismMotionSyncController")) - assert not story.has_motion_sync(_prefab("CubismModel")) # a child's controller does not count + from nnnotes import webmodel + assert webmodel.has_motion_sync(_prefab("CubismModel", "CubismMotionSyncController", + "Live2DMotionSyncCriAudioInput")) + assert not webmodel.has_motion_sync(_prefab("CubismModel", "CubismMotionSyncController")) + assert not webmodel.has_motion_sync(_prefab("CubismModel")) # a child's controller does not count def _talk(names, voices, **kw): diff --git a/tests/test_room.py b/tests/test_room.py index e2a24fa..2a5bc62 100644 --- a/tests/test_room.py +++ b/tests/test_room.py @@ -1,9 +1,19 @@ -"""room: material translation from the pass render state (property-driven values) and empty material slots -(synthetic shader / material typetrees).""" +"""room: material translation from the pass render state (property-driven values), empty material slots, and the +glTF textures, images and materials of a bundle whose serialized files reuse path ids (synthetic shader / material +typetrees and objects).""" +import copy +import json +import struct +from types import SimpleNamespace + import numpy as np import pytest +from PIL import Image +from UnityPy.classes import PPtr as UnityPPtr +from fakeunity import F32, POSITION, UV0, mesh_tt, submesh, unitypy_mesh from nnnotes import room +from nnnotes.unity import DEFAULT_RESOURCES FIXED = "" @@ -149,10 +159,11 @@ def path(self, tf_pid): def test_a_mesh_filter_without_a_mesh_is_skipped_and_reported(): mesh = object() - mfs = {1: Obj({}, m_Mesh=Ref(21, mesh)), 2: Obj({}, m_Mesh=Ref(0)), 3: Obj({}, m_Mesh=Ref(0)), - 4: Obj({}, m_Mesh=Ref(24, mesh))} - mrs = {1: Obj({"m_Enabled": 1}, m_Materials=["m1"]), 2: Obj({"m_Enabled": 1}, m_Materials=[]), - 3: Obj({"m_Enabled": 0}, m_Materials=[])} # 3: disabled renderer, 4: no renderer + mfs = {("f", 1): Obj({}, m_Mesh=Ref(21, mesh)), ("f", 2): Obj({}, m_Mesh=Ref(0)), ("f", 3): Obj({}, m_Mesh=Ref(0)), + ("f", 4): Obj({}, m_Mesh=Ref(24, mesh))} + mrs = {("f", 1): Obj({"m_Enabled": 1}, m_Materials=["m1"]), ("f", 2): Obj({"m_Enabled": 1}, m_Materials=[]), + ("f", 3): Obj({"m_Enabled": 0}, m_Materials=[]), # 3: disabled renderer, 4: no renderer + ("g", 4): Obj({"m_Enabled": 1}, m_Materials=["m4"])} # another file's GameObject 4 nulls = [] drawn = list(room.drawn_meshes(mfs, mrs, Graph(), nulls.append)) assert drawn == [(11, mesh, ["m1"])] @@ -202,3 +213,147 @@ def drawn(mf_by_go, mr_by_go, graph, on_null_mesh): {"mesh": "bare", "path": "Room/obj13"}] assert doc["nullMaterialSubmeshes"] == [{"mesh": "quad", "path": "Room/obj11", "submesh": 0}] assert (tmp_path / "room.glb").is_file() and (tmp_path / "room.json").is_file() + + +# ---------------------------------------------------------------- a bundle whose serialized files reuse path ids +V0, V1 = {"x": 0.0, "y": 0.0, "z": 0.0}, {"x": 1.0, "y": 1.0, "z": 1.0} +Q0 = {"x": 0.0, "y": 0.0, "z": 0.0, "w": 1.0} +ROOM_KEY = "Spot/a/Background/b" +PREFAB = "Assets/Room.prefab" +# the quads under the prefab root: (object name, material path id, its file id in CAB-a: 1 is CAB-b) +ROOM_OBJECTS = [("wall", 5, 0), ("wall_b", 5, 1), ("floor", 6, 0), ("wall_c", 5, 0)] + + +def ref(pid, file_id=0): + return {"m_FileID": file_id, "m_PathID": pid} + + +class File: + """A serialized file as UnityPy's PPtr resolves it: name, objects by path id, externals (paths) and the bundle + holding it (`parent`); a file outside the loaded bundles is not found.""" + def __init__(self, name, bundle, externals=()): + self.name, self.parent, self.objects = name, bundle, {} + self.externals = [SimpleNamespace(path=p) for p in externals] + self.environment = SimpleNamespace(find_file=lambda name: None) + bundle.files[name] = self + + def add(self, kind, pid, tt, obj=None): + o = SimpleNamespace(type=SimpleNamespace(name=kind), path_id=pid, assets_file=self, + read_typetree=lambda: copy.deepcopy(tt), read=lambda: obj) + self.objects[pid] = o + return o + + def pptr(self, pid, file_id=0): + """A PPtr as UnityPy parses it from an object of this file.""" + return UnityPPtr(m_FileID=file_id, m_PathID=pid, assetsfile=self) + + +def tex_env(pid, file_id=0): + return {"m_Texture": ref(pid, file_id), "m_Scale": {"x": 1.0, "y": 1.0}, "m_Offset": {"x": 0.0, "y": 0.0}} + + +def textured(name, **slots): + """A material typetree of shader 3 with the texture envs `slots` {slot: tex_env}.""" + tt = material(name=name) + tt.update(m_Shader=ref(3), m_ValidKeywords=[], m_CustomRenderQueue=-1) + tt["m_SavedProperties"]["m_TexEnvs"] = [[k, v] for k, v in slots.items()] + return tt + + +def quad(name): + return unitypy_mesh(mesh_tt(name, 3, {POSITION: (F32, 3, np.eye(3)), UV0: (F32, 2, np.zeros((3, 2)))}, + [0, 1, 2], [submesh(0, 3)])) + + +def two_file_room(): + """The environment of a bundle of two serialized files that reuse path ids. CAB-a holds the background prefab + (root `room`, GameObject 100, over ROOM_OBJECTS), shader 3, material 5 `a_wall` (main texture its texture 7, + `_Mask` in unity default resources), material 6 `a_floor` (main texture CAB-b's texture 7, `_Detail` its own + texture 7) and texture 7 `wall_a`. CAB-b holds shader 3, material 5 `b_wall` (main texture its texture 7) and + texture 7 `wall_b`: the pixels of `wall_a` with another filter.""" + bundle = SimpleNamespace(files={}) + a = File("CAB-a", bundle, ["archive:/CAB-b/CAB-b", DEFAULT_RESOURCES]) + b = File("CAB-b", bundle) + pixels = Image.new("RGBA", (2, 2), (200, 120, 40, 255)) + for f in (a, b): + f.add("Shader", 3, shader(name="Unlit/Transparent", srcBlend=fv(5.0), destBlend=fv(10.0))) + a.add("Material", 5, textured("a_wall", _MainTex=tex_env(7), _Mask=tex_env(9, 2))) + a.add("Material", 6, textured("a_floor", _MainTex=tex_env(7, 1), _Detail=tex_env(7))) + b.add("Material", 5, textured("b_wall", _MainTex=tex_env(7))) + for f, name, filter_mode in ((a, "wall_a", 1), (b, "wall_b", 0)): + f.add("Texture2D", 7, {"m_Name": name, "m_MipCount": 1, + "m_TextureSettings": {"m_FilterMode": filter_mode, "m_WrapU": 1, "m_WrapV": 1}}, + pixels.copy()) + a.add("AssetBundle", 1, {"m_Container": [[PREFAB.lower(), {"asset": ref(100)}]]}) + a.add("GameObject", 100, {"m_Name": "room", "m_IsActive": 1, "m_Layer": 0}) + a.add("Transform", 200, {"m_GameObject": ref(100), "m_Father": ref(0), "m_LocalPosition": V0, + "m_LocalRotation": Q0, "m_LocalScale": V1, + "m_Children": [ref(201 + i) for i in range(len(ROOM_OBJECTS))]}) + for i, (name, mat, file_id) in enumerate(ROOM_OBJECTS): + go, mesh = 101 + i, quad(f"{name}_mesh") + a.add("GameObject", go, {"m_Name": name, "m_IsActive": 1, "m_Layer": 0}) + a.add("Transform", 201 + i, {"m_GameObject": ref(go), "m_Father": ref(200), "m_Children": [], + "m_LocalPosition": V0, "m_LocalRotation": Q0, "m_LocalScale": V1}) + a.add("MeshFilter", 301 + i, {"m_GameObject": ref(go)}, + SimpleNamespace(m_Mesh=SimpleNamespace(path_id=21 + i, read=lambda mesh=mesh: mesh))) + a.add("MeshRenderer", 401 + i, {"m_GameObject": ref(go), "m_Enabled": 1}, + SimpleNamespace(m_Materials=[a.pptr(mat, file_id)])) + return SimpleNamespace(objects=[o for f in (a, b) for o in f.objects.values()]) + + +class RoomCatalog: + def fetch_key(self, key): + return [] + + def _entry(self, key): + return {"internal_id": PREFAB} + + +def load_room(monkeypatch, env): + """UnityPy.load gives `env`; a texture's image is the object it reads as.""" + monkeypatch.setattr(room.UnityPy, "load", lambda *a: env) + monkeypatch.setattr(room, "texture_image", lambda image: image) + + +def glb_json(path): + data = path.read_bytes() + (n,) = struct.unpack_from(" set: @@ -334,6 +340,22 @@ def test_write_index_keeps_every_kind(tmp_path): assert index["format"] == storysite.STORIES_FORMAT and index["languages"] == ["ja", "en", "ko"] assert (entry["id"], entry["manifest"], entry["advId"], entry["regions"]) == ("5", "stories/5.json", 5, ["r1"]) assert entry["size"]["languages"].keys() == {"ja", "en"} and entry["fonts"] == "open" + assert entry["size"]["models"] == len(b'{"m":1}') + len(b"MOC3-m") # the referenced model's files + + +def test_stories_reference_their_model_manifests(tmp_path): + """The assets of the model manifests a story names count as referenced by the stories (write_index keeps + them whatever models.json lists); a model manifest that is not there counts no bytes.""" + site, _ = ingest(tmp_path) + model_assets = set(chart_and_model(site)[1:]) + n, used = storysite.write_stories_index(site) + assert n == 1 and model_assets <= used + assert model_assets.isdisjoint(storysite.manifest_assets(json.loads((site / "stories/5.json").read_text( + encoding="utf-8")))) + (site / "models/m.json").unlink() + storysite.write_stories_index(site) + (entry,) = json.loads((site / "stories.json").read_text(encoding="utf-8"))["stories"] + assert entry["size"]["models"] == 0 def test_story_only_site_and_sites_without_stories(tmp_path): @@ -365,6 +387,8 @@ def test_fold_variant(tmp_path): dirs = story_dirs(tmp_path / "w3", ui_variant="x") storysite.ingest(web.Store(site), site, 5, dirs, {**JOB, "prefix": "r3/", "regions": ["r3"]}, []) assert not storysite.fold_variant(site, 5, "r3/") and (site / "stories/r3/5.json").exists() + man = json.loads((site / "stories/r3/5.json").read_text(encoding="utf-8")) + assert man["root"] == "../../" and man["models"] == {"m": "models/m.json"} def test_story_groups(tmp_path): @@ -465,6 +489,135 @@ def test_write_player_story_page(tmp_path): web.write_player(site2, fake_player(tmp_path / "q", story_bundle=False), stories=True) +# ---------------------------------------------------------------- the models of the stories +MODEL_A, MODEL_B = "Character/Live2D/g/a/model/a", "Character/Live2D/g/b/model/b" + + +def put_model(site, key, motion_sync=False): + """A model manifest as webmodel.ingest writes it (made-up model.json and moc3).""" + mid = webmodel.model_id(key) + store = web.Store(site) + doc = {"format": webmodel.MODEL_FORMAT, "key": key, "motionSync": motion_sync} + man = {"format": web.MANIFEST_FORMAT, "id": mid, "key": key, "model": {}, + "files": {"model.json": store.put("model.json", json.dumps(doc).encode()), + f"{mid}.moc3": store.put(f"{mid}.moc3", b"MOC3-" + mid.encode())}} + write(site / "models" / f"{mid}.json", web._dump(man)) + return {"id": mid, "ok": True, "files": 2, "bytes": sum(e["size"] for e in man["files"].values())} + + +@pytest.fixture +def story_build(tmp_path, monkeypatch): + """storysite.build over synthetic master data (MasterAdv 5, 6, 7, 8) and episodes: episode_models gives the + models each episode loads, build_dirs writes the story directory with the models it takes from the site + (webmodel.SiteModels, as story.build does), webmodel's model task writes a model manifest. -> run(ids, **kw), + with the calls in run.episodes (pre-pass) and run.models (model builds).""" + uses = {5: [MODEL_A], 6: [MODEL_B, MODEL_A], 7: [], 8: [MODEL_A]} + md = tmp_path / "master" + write(md / "MasterAdv.json", {"_allData": [{"_id": i} for i in uses]}) + font = tmp_path / "f.otf" + font.write_bytes(b"font") + cfg = Config({"catalog": {"region": "zz", "language": "en"}, "servers": {"zz": {"cdn": "https://cdn.invalid"}}, + "paths": {"master": str(md), "fonts": {"ja": str(font), "en": str(font)}}}, environ={}) + site = tmp_path / "site" + episodes, models = [], [] + + def episode_models(cat, master_dir, adv_id): + episodes.append(adv_id) + return sorted(uses[adv_id]) + + def build_dirs(cat, master_dir, player, adv_id, work, job): + src = webmodel.SiteModels(job["site"]) + return story_dirs(work, adv_id, mode=0, models={k: src.ensure(k)["id"] for k in uses[adv_id]}) + + def model_task(mid, key, job, data=None): + models.append(mid) + return put_model(Path(job["site"]), key) + monkeypatch.setattr(tmpfont, "require_extra", lambda what=None: None) + monkeypatch.setattr(storysite, "open_data", lambda cfg, region=None: (None, md, None)) + monkeypatch.setattr(storysite, "episode_models", episode_models) + monkeypatch.setattr(storysite, "build_dirs", build_dirs) + monkeypatch.setattr(storysite.storyhost, "build", lambda *a, **kw: None) + monkeypatch.setattr(storysite, "font_file", lambda job, lang: None) + monkeypatch.setattr(webmodel, "model_task", model_task) + monkeypatch.setattr(webmodel, "_open", lambda cfg, region=None: (None, None)) + player = fake_player(tmp_path / "p") + + def run(ids, **kw): + return storysite.build(site, ids, cfg, player, audio=False, workers=1, story_languages=["ja", "en"], **kw) + run.site, run.episodes, run.models = site, episodes, models + return run + + +def manifest(site, adv_id): + return json.loads((site / "stories" / f"{adv_id}.json").read_text(encoding="utf-8")) + + +def test_story_build_builds_the_models_of_its_stories_first(story_build): + site = story_build.site + r = story_build([5, 6, 7]) + assert r["storiesFailed"] == [] and [s["id"] for s in r["storiesBuilt"]] == ["5", "6", "7"] + assert sorted(story_build.episodes) == [5, 6, 7] + assert story_build.models == ["a", "b"] # each model once, whatever the stories using it + assert [m["id"] for m in r["storyModels"]["modelsBuilt"]] == ["a", "b"] and r["models"] == 2 + index = json.loads((site / "models.json").read_text(encoding="utf-8")) + assert [m["id"] for m in index["models"]] == ["a", "b"] + assert manifest(site, 5)["models"] == {"a": "models/a.json"} + assert manifest(site, 6)["models"] == {"a": "models/a.json", "b": "models/b.json"} + assert manifest(site, 7)["models"] == {} and manifest(site, 7)["root"] == "../" + assert json.loads(web.entry_text(site, manifest(site, 6)["files"]["story.json"]))["models"] == { + MODEL_B: "b", MODEL_A: "a"} + stories = {e["id"]: e for e in json.loads((site / "stories.json").read_text(encoding="utf-8"))["stories"]} + size = {m: sum(e["size"] for e in json.loads((site / f"models/{m}.json").read_text(encoding="utf-8")) + ["files"].values()) for m in ("a", "b")} + assert (stories["5"]["size"]["models"], stories["6"]["size"]["models"], stories["7"]["size"]["models"]) == ( + size["a"], size["a"] + size["b"], 0) + for mf in (site / "models").glob("*.json"): # write_index kept the models' assets + for e in json.loads(mf.read_text(encoding="utf-8"))["files"].values(): + assert all((site / a).is_file() for a in web.entry_assets(e)) + + +def test_story_build_skips_existing_models_unless_forced(story_build): + story_build([5]) + assert story_build.models == ["a"] + # stories whose manifest exists: no pre-pass, no model build + r = story_build([5]) + assert r["storiesSkipped"] == ["5"] and r["storyModels"] is None and story_build.episodes == [5] + # a new story whose model exists: the model manifest is skipped + r = story_build([8]) + assert story_build.models == ["a"] and r["storyModels"]["modelsSkipped"] == ["a"] + assert manifest(story_build.site, 8)["models"] == {"a": "models/a.json"} + # force: the stories and their models built again, each model once + r = story_build([5, 8, 6], force=True) + assert story_build.models == ["a", "a", "b"] and r["storyModels"]["modelsSkipped"] == [] + assert sorted(s["id"] for s in r["storiesBuilt"]) == ["5", "6", "8"] + + +def test_story_build_rebuilds_outdated_models(story_build): + """A model manifest whose model.json is older than MODEL_FORMAT (no motionSync) is built again by the stories' + model build without force; a current one is skipped.""" + site = story_build.site + put_model(site, MODEL_B) # current + put_model(site, MODEL_A) + man = json.loads((site / "models/a.json").read_text(encoding="utf-8")) + man["files"]["model.json"] = web.Store(site).put("model.json", json.dumps({"format": 1, "key": MODEL_A}).encode()) + write(site / "models/a.json", web._dump(man)) + r = story_build([6]) + assert story_build.models == ["a"] + assert r["storyModels"]["modelsRebuilt"] == [{"id": "a", "reason": "outdated"}] + assert r["storyModels"]["modelsSkipped"] == ["b"] and [m["id"] for m in r["storyModels"]["modelsBuilt"]] == ["a"] + assert r["storiesFailed"] == [] and webmodel.outdated(site, "a") is None + assert manifest(site, 6)["models"] == {"a": "models/a.json", "b": "models/b.json"} + +def test_a_story_without_its_model_fails(story_build, monkeypatch): + monkeypatch.setattr(webmodel, "model_task", lambda mid, key, job, data=None: { + "id": mid, "ok": False, "stage": "export", "error": "x"}) + r = story_build([5, 7]) + assert r["storyModels"]["modelsFailed"] == [{"id": "a", "stage": "export", "error": "x"}] + assert [(f["id"], f["stage"]) for f in r["storiesFailed"]] == [("5", "export")] + assert "no model manifest models/a.json" in r["storiesFailed"][0]["error"] + assert [s["id"] for s in r["storiesBuilt"]] == ["7"] + + def test_cli_web_stories(tmp_path, capsys, monkeypatch): from nnnotes import cli calls = [] @@ -483,7 +636,7 @@ def fake_build(out, ids, cfg, player, fmt, **kw): assert kw["story_languages"] == ["en", "ja"] assert kw["fonts_flags"] == {"en": "a.otf", "ja": "b.otf", "emoji": "e.ttf"} assert cli.EMOJI_FONT == storysite.EMOJI_FONT - assert kw["fonts"] == "open" + assert kw["fonts"] == "open" and kw["encoding"] == "gzip" assert json.loads(capsys.readouterr().out)["storiesFailed"] == [] diff --git a/tests/test_web.py b/tests/test_web.py index ce3cffd..a66f3fc 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -1,11 +1,14 @@ +import gzip import hashlib import json import pytest -from nnnotes import web +from nnnotes import compress, web from nnnotes.config import ConfigError +DOC = web._minify({"rows": [{"id": i, "name": f"row {i}", "tags": ["a", "b"]} for i in range(200)]}).encode() + def test_text_asset_minifies_and_drops_doc_fields(): doc = {"spec": "notes for readers", "music": {"soundId": 1}, "voice": {"rule": "r", "x": 1}, "v": 1e999} @@ -34,6 +37,85 @@ def test_store_is_content_addressed(tmp_path): assert store.put("noext", b"z")["asset"].endswith(".bin") +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def test_store_encodes_compressible_files_when_smaller(tmp_path): + store = web.Store(tmp_path) + e = store.put("x/doc.json", DOC) + assert e == {"asset": f"assets/{sha(DOC)}.json.gz", "size": len(DOC), "stored": len(compress.gzip_bytes(DOC))} + assert e["stored"] < e["size"] + assert (tmp_path / e["asset"]).read_bytes() == compress.gzip_bytes(DOC) + assert web.read_asset(tmp_path, e["asset"]) == DOC + shader = b"void main() { gl_FragColor = vec4(1.0); }\n" * 20 + assert store.put("s/a.glsl", shader)["asset"] == f"assets/{sha(shader)}.glsl.gz" + assert store.put("x/tiny.json", b"{}") == {"asset": f"assets/{sha(b'{}')}.json", "size": 2} # gzip not smaller + assert store.put("t/a.png", DOC) == {"asset": f"assets/{sha(DOC)}.png", "size": len(DOC)} # not compressible + with pytest.raises(ValueError, match="asset encoding"): + web.Store(tmp_path, "zstd") + + +def test_store_brotli(tmp_path): + pytest.importorskip("brotli") + e = web.site_store(tmp_path, "br").put("m/model.moc3", DOC) + assert e == {"asset": f"assets/{sha(DOC)}.moc3.br", "size": len(DOC), "stored": len(compress.brotli_bytes(DOC))} + assert web.read_asset(tmp_path, e["asset"]) == DOC + + +def test_store_none_keeps_todays_names_and_bytes(tmp_path): + store = web.site_store(tmp_path, "none") + assert store.put("x/doc.json", DOC) == {"asset": f"assets/{sha(DOC)}.json", "size": len(DOC)} + assert [p.name for p in (tmp_path / "assets").iterdir()] == [f"{sha(DOC)}.json"] + assert (tmp_path / "assets" / f"{sha(DOC)}.json").read_bytes() == DOC + + +def test_store_reuses_encoded_names(tmp_path): + store = web.Store(tmp_path) + e = store.put("a.json", DOC) + assert store.put("b.json", DOC) == e and (store.written, store.reused) == (1, 1) + again = web.Store(tmp_path) + assert again.put("c.json", DOC) == e and (again.written, again.reused) == (0, 1) + # a file of that name that holds the bytes is kept whatever encoder wrote it; its length is the entry's `stored` + other = gzip.compress(DOC, 1, mtime=0) + (tmp_path / e["asset"]).write_bytes(other) + kept = web.Store(tmp_path).put("d.json", DOC) + assert kept == {**e, "stored": len(other)} and (tmp_path / e["asset"]).read_bytes() == other + (tmp_path / e["asset"]).write_bytes(other[:-5]) # damaged: written again + fresh = web.Store(tmp_path) + assert fresh.put("e.json", DOC) == e and fresh.written == 1 + assert (tmp_path / e["asset"]).read_bytes() == compress.gzip_bytes(DOC) + + +def test_split_parts_carry_stored(tmp_path): + big = web._minify({"a": "x" * (web.SPLIT_MIN_BYTES // 2), "b": ["y"] * 200000, "c": {"d": 1}}).encode() + e = web.Store(tmp_path).put_file("scene.json", big) + assert e["size"] == len(big) and [p[0] for p in e["parts"]] == ["a", "b", "c"] + a, b, c = e["parts"] + small = b'{"d":1}' + assert len(a) == len(b) == 4 and a[1].endswith(".json.gz") and a[3] < a[2] + assert c == ["c", f"assets/{sha(small)}.json", len(small)] # a part gzip does not make smaller + assert web.entry_text(tmp_path, e) == big + raw = web.Store(tmp_path / "raw", "none").put_file("scene.json", big) + assert all(len(p) == 3 for p in raw["parts"]) and web.entry_text(tmp_path / "raw", raw) == big + + +def test_stored_text_memo_follows_the_encoding(tmp_path): + doc = json.dumps({"k": list(range(300))}) + a = web.stored_text(web.Store(tmp_path), "x/a.json", doc, memo="m") + b = web.stored_text(web.Store(tmp_path, "none"), "x/a.json", doc, memo="m") + assert a["asset"].endswith(".json.gz") and b == {"asset": a["asset"][:-3], "size": a["size"]} + + +def test_ingest_task_stores_in_the_jobs_encoding(tmp_path, monkeypatch): + seen = [] + monkeypatch.setattr(web, "ingest", lambda store, *a, **k: seen.append(store.encoding) or {"id": "1_easy"}) + cfg = {"site": str(tmp_path), "audioFormat": "aac", "audio": True, "language": "en"} + web.ingest_task(1, [("easy", str(tmp_path), {}, [])], str(tmp_path), {**cfg, "encoding": "none"}) + web.ingest_task(1, [("easy", str(tmp_path), {}, [])], str(tmp_path), cfg) + assert seen == ["none", "gzip"] + + def manifest(site_dir, music_id, difficulty, store, files): entries = {p: store.put(p, data) for p, data in files.items()} doc = {"musicId": music_id, "difficulty": difficulty, "audio": True, "audioFormat": "aac", "flows": ["direct"], @@ -49,7 +131,7 @@ def test_write_index_orders_charts_and_prunes_assets(tmp_path): manifest(tmp_path, 1, "easy", store, {"live.json": b"{}"}) stale = store.put("old.json", b"[0]") r = web.write_index(tmp_path) - assert (r["charts"], r["assets"], r["removedAssets"]) == (3, 2, 1) + assert (r["charts"], r["assets"], r["removedAssets"], r["assetBytes"], r["bytes"]) == (3, 2, 1, 5, 5) assert not (tmp_path / stale["asset"]).exists() index = json.loads((tmp_path / "charts.json").read_text(encoding="utf-8")) assert index["format"] == web.SITE_FORMAT @@ -57,6 +139,28 @@ def test_write_index_orders_charts_and_prunes_assets(tmp_path): assert index["charts"][1]["bytes"] == 5 and index["charts"][1]["title"] == "t1" +def encoded_site(site): + """Two charts sharing an encoded file, a raw file each, and a stale encoded asset.""" + (site / "charts").mkdir(parents=True) + store = web.Store(site) + manifest(site, 1, "easy", store, {"live.json": b"{}", "scene/a.json": DOC}) + manifest(site, 1, "hard", store, {"live.json": b"[1]", "scene/a.json": DOC}) + return store.put("old.json", DOC + b" ") + + +def test_write_index_prunes_encoded_names(tmp_path): + stale = encoded_site(tmp_path) + assert stale["asset"].endswith(".json.gz") + r = web.write_index(tmp_path) + assert (r["charts"], r["assets"], r["removedAssets"]) == (2, 3, 1) + assert not (tmp_path / stale["asset"]).exists() + names = sorted(p.name for p in (tmp_path / "assets").iterdir()) + assert names == sorted([f"{sha(DOC)}.json.gz", f"{sha(b'{}')}.json", f"{sha(b'[1]')}.json"]) + assert (r["assetBytes"], r["bytes"]) == (len(compress.gzip_bytes(DOC)) + 2 + 3, len(DOC) + 2 + 3) + index = json.loads((tmp_path / "charts.json").read_text(encoding="utf-8")) + assert [c["bytes"] for c in index["charts"]] == [len(DOC) + 2, len(DOC) + 3] # decoded sizes + + def fake_player(root): for rel, text in {web.READ_SET_SCRIPT: "// read set\n", web.PLAYER_BUNDLES[0]: "/* bundle */\n", f"{web.PLAYER_PAGE_DIR}/index.html": @@ -101,6 +205,26 @@ def test_player_only_command(tmp_path, capsys): assert json.loads((out / "charts.json").read_text(encoding="utf-8")) == {"charts": [], "format": web.SITE_FORMAT} +def test_player_only_and_reingest_over_an_encoded_site(tmp_path, capsys): + from nnnotes import cli + site, player = tmp_path / "site", str(fake_player(tmp_path / "p")) + encoded_site(site) + before = (site / "charts" / "1_easy.json").read_bytes() + cli.main(["web", str(site), "--player", player, "--player-only"]) + r = json.loads(capsys.readouterr().out) + assert (r["charts"], r["assets"], r["removedAssets"]) == (2, 3, 1) + cli.main(["web", str(site), "--player", player, "--reingest-json"]) + r = json.loads(capsys.readouterr().out) + assert (r["changedEntries"], r["assets"], r["removedAssets"]) == (0, 3, 0) + assert json.loads((site / "charts" / "1_easy.json").read_bytes()) == json.loads(before) + cli.main(["web", str(site), "--player", player, "--reingest-json", "--compress", "none"]) + r = json.loads(capsys.readouterr().out) + assert (r["changedEntries"], r["assets"], r["removedAssets"]) == (2, 3, 1) + man = json.loads((site / "charts" / "1_easy.json").read_text(encoding="utf-8")) + assert man["files"]["scene/a.json"] == {"asset": f"assets/{sha(DOC)}.json", "size": len(DOC)} + assert r["assetBytes"] == r["bytes"] == len(DOC) + 2 + 3 + + def test_player_only_on_a_new_directory(tmp_path, capsys): from nnnotes import cli out = tmp_path / "new-site" @@ -131,4 +255,23 @@ def fake_build(out, pairs, cfg, player, fmt, **kw): assert json.loads(capsys.readouterr().out)["failed"] == failed ((pairs, fmt, kw),) = calls assert pairs == [(7, "hard"), (7, "easy")] and fmt == "opus" - assert kw["audio"] is False and kw["workers"] == 1 and kw["force"] is False + assert kw["audio"] is False and kw["workers"] == 1 and kw["force"] is False and kw["encoding"] == "gzip" + + +def test_web_compress_flag_reaches_the_build(tmp_path, capsys, monkeypatch): + from nnnotes import cli + calls = [] + + def fake_build(out, pairs, cfg, player, fmt, **kw): + calls.append(kw["encoding"]) + return {"site": str(out), "built": [], "failed": [], "skipped": []} + + monkeypatch.setattr(web, "build", fake_build) + monkeypatch.setattr(web, "unknown_pairs", lambda cfg, pairs, regions=None: []) + player = str(fake_player(tmp_path / "p")) + for flag in ("br", "none"): + cli.main(["web", str(tmp_path / "s"), "--player", player, "--pair", "7:hard", "--compress", flag]) + assert calls == ["br", "none"] + with pytest.raises(SystemExit): + cli.main(["web", str(tmp_path / "s"), "--player", player, "--pair", "7:hard", "--compress", "zstd"]) + capsys.readouterr() diff --git a/tests/test_webmodel.py b/tests/test_webmodel.py index a855b22..2d2129d 100644 --- a/tests/test_webmodel.py +++ b/tests/test_webmodel.py @@ -6,7 +6,7 @@ import pytest -from nnnotes import cli, live2d, web, webmodel +from nnnotes import cli, jsonio, live2d, web, webmodel from nnnotes.config import Config, ConfigError KEY_A = "Character/Live2D/001_adv/adv_model_a/model/adv_model_a" @@ -56,11 +56,13 @@ def variant(shader, n, keywords, platform="gles3", kind="GLES3", ext="glsl"): "stage": "vertex", "type": kind, "keywords": keywords} +ALV = "_ADDITIONAL_LIGHTS_VERTEX" SHADER_INDEX = [ {"name": "S/Lit", "source": "x", "parsed": "S_Lit.json", "variants": [ variant("S_Lit", 0, []), variant("S_Lit", 1, ["CUBISM_MASK_ON"]), variant("S_Lit", 2, ["_ADDITIONAL_LIGHTS"]), variant("S_Lit", 3, [], kind="GLES31"), - variant("S_Lit", 4, [], platform="vulkan", kind="SPIRV", ext="vkprog")]}, + variant("S_Lit", 4, [], platform="vulkan", kind="SPIRV", ext="vkprog"), + variant("S_Lit", 5, [ALV]), variant("S_Lit", 6, ["CUBISM_MASK_ON", ALV])]}, {"name": "S/Mask", "source": "y", "parsed": "S_Mask.json", "variants": [variant("S_Mask", 0, [])]}, ] MASKS = {"cubismMask": {"material": "Mask", "shader": {"shader": "S/Mask"}, "keywords": [], "floats": {"_Cull": 0.0}}, @@ -68,14 +70,18 @@ def variant(shader, n, keywords, platform="gles3", kind="GLES3", ext="glsl"): "floats": {"_Cull": 1.0}}} -def fake_model(root, name="m", tex=b"\x89PNG texture a", moc=b"MOC3 bytes", masked=True): +def fake_model(root, name="m", tex=b"\x89PNG texture a", moc=b"MOC3 bytes", masked=True, key=KEY_A, + motion_sync=False): """A model directory in the export layout (made-up contents) and its export summary. Two atlas pages, the - drawables use page 0; drawables masked or not.""" + drawables use page 0; drawables masked or not; the root node with the MotionSync components or not.""" kws = [[], ["CUBISM_MASK_ON"]] if masked else [[], []] - prefab = {"key": name, "nodes": [drawable("textures/p0-00.png", k) for k in kws], "canvas": {"width": 1.0}} - doc = {"format": webmodel.MODEL_FORMAT, "name": name, "key": KEY_A, "moc3": f"{name}.moc3", + root_node = {"path": name, "components": [{"type": "MonoBehaviour", "class": c} for c in + (webmodel.MOTION_SYNC_COMPONENTS if motion_sync else ())]} + prefab = {"key": name, "nodes": [root_node, *(drawable("textures/p0-00.png", k) for k in kws)], + "canvas": {"width": 1.0}} + doc = {"format": webmodel.MODEL_FORMAT, "name": name, "key": key, "moc3": f"{name}.moc3", "prefab": f"{name}.prefab.json", "textures": webmodel.drawable_textures(prefab), "canvas": prefab["canvas"], - "shaders": "shaders/shaders.json", "resources": MASKS} + "shaders": "shaders/shaders.json", "resources": MASKS, "motionSync": webmodel.has_motion_sync(prefab)} files = {f"{name}.moc3": moc, f"{name}.prefab.json": json.dumps(prefab, indent=1).encode(), "textures/p0-00.png": tex, "textures/p1-11.png": b"unused page", "shaders/shaders.json": json.dumps(SHADER_INDEX).encode(), webmodel.MODEL_INDEX: json.dumps(doc).encode(), @@ -92,15 +98,20 @@ def fake_model(root, name="m", tex=b"\x89PNG texture a", moc=b"MOC3 bytes", mask return root, summary +MASKED_FILES = ["m.moc3", "m.prefab.json", "model.json", "shaders/S_Lit.json", + "shaders/S_Lit/gles3/s0p0_vertex_0.glsl", "shaders/S_Lit/gles3/s0p0_vertex_1.glsl", + "shaders/S_Lit/gles3/s0p0_vertex_5.glsl", "shaders/S_Lit/gles3/s0p0_vertex_6.glsl", + "shaders/S_Mask.json", "shaders/S_Mask/gles3/s0p0_vertex_0.glsl", "shaders/shaders.json", + "textures/p0-00.png"] + + def test_read_rule(tmp_path): _, summary = fake_model(tmp_path / "a", masked=True) - assert summary["files"] == ["m.moc3", "m.prefab.json", "model.json", "shaders/S_Lit.json", - "shaders/S_Lit/gles3/s0p0_vertex_0.glsl", "shaders/S_Lit/gles3/s0p0_vertex_1.glsl", - "shaders/S_Mask.json", "shaders/S_Mask/gles3/s0p0_vertex_0.glsl", - "shaders/shaders.json", "textures/p0-00.png"] + assert summary["files"] == MASKED_FILES _, summary = fake_model(tmp_path / "b", masked=False) assert summary["files"] == ["m.moc3", "m.prefab.json", "model.json", "shaders/S_Lit.json", - "shaders/S_Lit/gles3/s0p0_vertex_0.glsl", "shaders/shaders.json", "textures/p0-00.png"] + "shaders/S_Lit/gles3/s0p0_vertex_0.glsl", "shaders/S_Lit/gles3/s0p0_vertex_5.glsl", + "shaders/shaders.json", "textures/p0-00.png"] # keywords the shader's programs do not use are ignored lit = {"shader": {"shader": "S/Lit"}, "keywords": ["CUBISM_INVERT_ON"]} assert webmodel.shader_files(SHADER_INDEX, [lit]) == ["S_Lit.json", "S_Lit/gles3/s0p0_vertex_0.glsl"] @@ -111,23 +122,40 @@ def test_read_rule(tmp_path): webmodel.shader_files(SHADER_INDEX, [{"shader": {"shader": "S/Other"}, "keywords": []}]) +def test_story_keywords_select_the_story_programs_too(): + lit = {"shader": {"shader": "S/Lit"}, "keywords": ["CUBISM_MASK_ON"]} + mask = MASKS["cubismMask"] + assert webmodel.STORY_KEYWORDS == (ALV,) + assert webmodel.shader_files(SHADER_INDEX, [lit], webmodel.STORY_KEYWORDS) == [ + "S_Lit.json", "S_Lit/gles3/s0p0_vertex_1.glsl", "S_Lit/gles3/s0p0_vertex_6.glsl"] + # a shader whose programs do not use the keyword: its one program + assert webmodel.shader_files(SHADER_INDEX, [mask], webmodel.STORY_KEYWORDS) == [ + "S_Mask.json", "S_Mask/gles3/s0p0_vertex_0.glsl"] + # the keyword without a program for a material's set + index = [{**SHADER_INDEX[0], "variants": [v for v in SHADER_INDEX[0]["variants"] if v["keywords"] != [ALV]]}] + with pytest.raises(RuntimeError, match=r"0 GLES3 programs with the keywords \['_ADDITIONAL_LIGHTS_VERTEX'\]"): + webmodel.shader_files(index, [{"shader": {"shader": "S/Lit"}, "keywords": []}], webmodel.STORY_KEYWORDS) + + def test_ingest_writes_a_manifest_of_the_read_files(tmp_path): site = tmp_path / "site" (site / web.MODELS_DIR).mkdir(parents=True) mdir, summary = fake_model(tmp_path / "m") r = webmodel.ingest(web.Store(site), site, "adv_model_a", KEY_A, mdir, summary) man = json.loads((site / "models" / "adv_model_a.json").read_text(encoding="utf-8")) - assert r["ok"] and r["files"] == len(man["files"]) == 10 and sorted(man["files"]) == summary["files"] - assert man["format"] == web.SITE_FORMAT and man["id"] == "adv_model_a" and man["key"] == KEY_A - assert man["model"] == {"group": "001_adv", "canvas": {"width": 1.0}, "textures": 1, "nodes": 2} + assert r["ok"] and r["files"] == len(man["files"]) == 12 and sorted(man["files"]) == summary["files"] + assert man["format"] == web.MANIFEST_FORMAT == 3 and man["id"] == "adv_model_a" and man["key"] == KEY_A + assert man["model"] == {"group": "001_adv", "canvas": {"width": 1.0}, "textures": 1, "nodes": 3} index = json.loads(web.entry_text(site, man["files"]["shaders/shaders.json"])) assert [(r["name"], [v["file"] for v in r["variants"]]) for r in index] == [ - ("S/Lit", ["S_Lit/gles3/s0p0_vertex_0.glsl", "S_Lit/gles3/s0p0_vertex_1.glsl"]), + ("S/Lit", ["S_Lit/gles3/s0p0_vertex_0.glsl", "S_Lit/gles3/s0p0_vertex_1.glsl", + "S_Lit/gles3/s0p0_vertex_5.glsl", "S_Lit/gles3/s0p0_vertex_6.glsl"]), ("S/Mask", ["S_Mask/gles3/s0p0_vertex_0.glsl"])] prefab = web.entry_text(site, man["files"]["m.prefab.json"]) assert json.loads(prefab)["canvas"] == {"width": 1.0} and b" " not in prefab assert web.entry_text(site, man["files"]["m.moc3"]) == b"MOC3 bytes" - assert json.loads(web.entry_text(site, man["files"]["model.json"]))["format"] == 1 + doc = json.loads(web.entry_text(site, man["files"]["model.json"])) + assert doc["format"] == webmodel.MODEL_FORMAT == 2 and doc["motionSync"] is False def test_write_index_keeps_model_assets_and_writes_models_json(tmp_path): @@ -152,7 +180,7 @@ def test_write_index_keeps_model_assets_and_writes_models_json(tmp_path): assert [m["id"] for m in index["models"]] == ["adv_model_a", "adv_model_b"] a = index["models"][0] assert a["manifest"] == "models/adv_model_a.json" and a["key"] == KEY_A and a["group"] == "001_adv" - assert a["files"] == 10 and a["bytes"] > 0 and a["textures"] == 1 + assert a["files"] == 12 and a["bytes"] > 0 and a["textures"] == 1 # the two models share their shader files: stored once shared_glsl = {json.loads((site / "models" / f"{i}.json").read_text(encoding="utf-8"))["files"] ["shaders/S_Lit/gles3/s0p0_vertex_0.glsl"]["asset"] for i in ("adv_model_a", "adv_model_b")} @@ -234,17 +262,23 @@ def test_player_pages_never_overwrite_site_data(tmp_path): web.write_player(tmp_path / "site", player) +def put_manifest(site, mid, key, model=None, doc=None): + """A model manifest with a model.json only (`doc`, default a current one: MODEL_FORMAT, motionSync).""" + doc = {"format": webmodel.MODEL_FORMAT, "key": key, "motionSync": False} if doc is None else doc + entry = web.Store(site).put("model.json", json.dumps(doc).encode()) + (site / "models").mkdir(parents=True, exist_ok=True) + (site / "models" / f"{mid}.json").write_text(json.dumps({"key": key, "model": model or {}, + "files": {"model.json": entry}}), encoding="utf-8") + + def test_build_skips_existing_models_unless_forced(tmp_path, monkeypatch): site = tmp_path / "site" - (site / "models").mkdir(parents=True) - (site / "models" / "adv_model_a.json").write_text(json.dumps({"key": KEY_A, "model": {}, "files": {}}), - encoding="utf-8") + put_manifest(site, "adv_model_a", KEY_A) done = [] def fake_task(mid, key, job, data=None): done.append(mid) - (site / "models" / f"{mid}.json").write_text(json.dumps({"key": key, "model": {}, "files": {}}), - encoding="utf-8") + put_manifest(site, mid, key) return {"id": mid, "ok": True, "files": 0, "bytes": 0} monkeypatch.setattr(webmodel, "model_task", fake_task) @@ -256,10 +290,39 @@ def fake_task(mid, key, job, data=None): assert r["modelNames"] is None # no master data: no names r = webmodel.build(site, models, Config(), fake_player(tmp_path / "p"), force=True, workers=1) assert done == ["adv_model_b", "adv_model_a", "adv_model_b"] and r["modelsSkipped"] == [] + assert r["modelsRebuilt"] == [] # force: every model, none of them for a reason with pytest.raises(ValueError, match="has the id"): webmodel.build(site, {"other": KEY_A}, Config(), fake_player(tmp_path / "p"), workers=1) +def test_build_rebuilds_outdated_manifests(tmp_path, monkeypatch): + """An existing manifest whose model.json is older than MODEL_FORMAT, has no motionSync or cannot be read is built + again without force; a current one is skipped.""" + site = tmp_path / "site" + keys = {f"m{i}": f"Character/Live2D/g/m{i}/model/m{i}" for i in range(5)} + put_manifest(site, "m0", keys["m0"]) # current + put_manifest(site, "m1", keys["m1"], doc={"format": 1, "key": keys["m1"]}) # format 1 + put_manifest(site, "m2", keys["m2"], doc={"format": webmodel.MODEL_FORMAT, "key": keys["m2"]}) # no motionSync + (site / "models" / "m3.json").write_text(json.dumps({"key": keys["m3"], "model": {}, "files": {}}), + encoding="utf-8") # no model.json + done = [] + + def fake_task(mid, key, job, data=None): + done.append(mid) + put_manifest(site, mid, key) + return {"id": mid, "ok": True, "files": 1, "bytes": 1} + monkeypatch.setattr(webmodel, "model_task", fake_task) + monkeypatch.setattr(webmodel, "_open", lambda cfg: (None, None)) + assert webmodel.outdated(site, "m0") is None and webmodel.outdated(site, "m1") == "outdated" + r = webmodel.build(site, keys, Config(), fake_player(tmp_path / "p"), workers=1) + assert done == ["m1", "m2", "m3", "m4"] and r["modelsSkipped"] == ["m0"] + assert r["modelsRebuilt"] == [{"id": "m1", "reason": "outdated"}, {"id": "m2", "reason": "outdated"}, + {"id": "m3", "reason": "unreadable"}] + assert [m["id"] for m in r["modelsBuilt"]] == ["m1", "m2", "m3", "m4"] + r = webmodel.build(site, keys, Config(), fake_player(tmp_path / "p"), workers=1) # all current now + assert r["modelsSkipped"] == list(keys) and r["modelsRebuilt"] == [] and len(done) == 4 + + # ---------------------------------------------------------------- command line def run(argv, capsys): code = 0 @@ -322,6 +385,7 @@ def fake_charts(out, pairs, cfg, player, fmt, **kw): (m, models, mkw), (c, pairs, ckw) = calls assert (m, c) == ("models", "charts") assert models == {"adv_model_a": KEY_A, "adv_model_b": KEY_B} and mkw["force"] is True + assert mkw["encoding"] == "gzip" assert pairs == [(7, "hard")] and ckw["force"] is True @@ -413,14 +477,12 @@ def test_master_names_follow_the_settings(tmp_path): def test_build_adds_names_and_refreshes_skipped_manifests(tmp_path, monkeypatch): site = tmp_path / "site" (site / "models").mkdir(parents=True) - old = {"key": KEY_A, "model": {"group": "001_adv", "character": 9, "label": "old"}, "files": {}} - (site / "models" / "adv_model_a.json").write_text(json.dumps(old), encoding="utf-8") + put_manifest(site, "adv_model_a", KEY_A, model={"group": "001_adv", "character": 9, "label": "old"}) jobs = [] def fake_task(mid, key, job, data=None): jobs.append((mid, job["names"].get(key))) - (site / "models" / f"{mid}.json").write_text(json.dumps({"key": key, "model": {}, "files": {}}), - encoding="utf-8") + put_manifest(site, mid, key) return {"id": mid, "ok": True, "files": 0, "bytes": 0} monkeypatch.setattr(webmodel, "model_task", fake_task) @@ -445,3 +507,169 @@ def fake_task(mid, key, job, data=None): assert webmodel.refresh_names(site, "adv_model_a", None) is True man = json.loads((site / "models" / "adv_model_a.json").read_text(encoding="utf-8")) assert man["model"] == {"group": "001_adv"} + + +# ---------------------------------------------------------------- model.json and the model sources of story.build +class FakeShader: + def __init__(self, name): + self.name = name + self.assets_file = SimpleNamespace(name=f"CAB-{name}") + + +def fake_export(monkeypatch, motion_sync=False): + """export_model's inputs made up: the runtime export writes fake_model's files, the player's mask materials are + MASKS and the shader dump indexes SHADER_INDEX (fake_model wrote its files); -> the player data.""" + def export_runtime(cat, key, out_dir): + name = webmodel.model_id(key) + fake_model(out_dir, name, key=key, motion_sync=motion_sync) + prefab = json.loads((out_dir / f"{name}.prefab.json").read_text(encoding="utf-8")) + summary = {"name": name, "moc3": f"{name}.moc3", "prefab": f"{name}.prefab.json", "moc3Bytes": 10, + "textures": ["textures/p0-00.png", "textures/p1-11.png"], "nodes": len(prefab["nodes"])} + return live2d.RuntimeExport(summary, SimpleNamespace(shaders={"S/Lit": FakeShader("S/Lit")}), None, None, {}) + + class FakeExporter: + def __init__(self, cat, out_dir, player=None): + self.shaders = {"S/Mask": FakeShader("S/Mask")} + + def material(self, path): + return MASKS[{v: k for k, v in webmodel.RESOURCES.items()}[path]] + + def dump_objects(objs, sdir, name, index): + index.extend(r for r in SHADER_INDEX if r["name"] == objs[0].name) + + def write_index(index, sdir): + jsonio.write_json(sdir / "shaders.json", index) + + monkeypatch.setattr(live2d, "export_runtime", export_runtime) + monkeypatch.setattr(webmodel, "Exporter", FakeExporter) + monkeypatch.setattr(webmodel.shader_mod, "dump_objects", dump_objects) + monkeypatch.setattr(webmodel.shader_mod, "write_index", write_index) + return SimpleNamespace(resource=lambda path: path) + + +def model_files(name): + return [f.replace("m.", f"{name}.") for f in MASKED_FILES] + + +@pytest.mark.parametrize("motion_sync", [False, True]) +def test_export_model_writes_model_json(tmp_path, monkeypatch, motion_sync): + player = fake_export(monkeypatch, motion_sync) + r = webmodel.export_model(None, player, KEY_A, tmp_path / "x") + doc = json.loads((tmp_path / "x" / webmodel.MODEL_INDEX).read_text(encoding="utf-8")) + assert doc == {"format": 2, "name": "adv_model_a", "key": KEY_A, "moc3": "adv_model_a.moc3", + "prefab": "adv_model_a.prefab.json", "textures": ["textures/p0-00.png"], "canvas": {"width": 1.0}, + "shaders": "shaders/shaders.json", "resources": MASKS, "motionSync": motion_sync} + assert r["files"] == model_files("adv_model_a") + + +def site_with(tmp_path, *models): + """A site with the model manifests of fake_model for each (key, motion_sync).""" + site = tmp_path / "site" + (site / web.MODELS_DIR).mkdir(parents=True, exist_ok=True) + for key, ms in models: + mid = webmodel.model_id(key) + webmodel.ingest(web.Store(site), site, mid, key, *fake_model(tmp_path / mid, mid, key=key, motion_sync=ms)) + return site + + +def test_site_models_read_model_json(tmp_path): + site = site_with(tmp_path, (KEY_A, True), (KEY_B, False)) + src = webmodel.SiteModels(site) + assert src.ensure(KEY_A) == {"id": "adv_model_a", "motionSync": True} + assert src.ensure(KEY_B) == {"id": "adv_model_b", "motionSync": False} + assert src.story_fields(tmp_path / "story") == {} + with pytest.raises(RuntimeError, match="no model manifest models/c.json in the site"): + src.ensure("Character/Live2D/g/c/model/c") + # a model.json without motionSync (a model built before it had one) + man = json.loads((site / "models" / "adv_model_b.json").read_text(encoding="utf-8")) + doc = {k: v for k, v in json.loads(web.entry_text(site, man["files"]["model.json"])).items() if k != "motionSync"} + man["files"]["model.json"] = web.Store(site).put("model.json", json.dumps({**doc, "format": 1}).encode()) + (site / "models" / "adv_model_b.json").write_text(json.dumps(man), encoding="utf-8") + with pytest.raises(RuntimeError, match="format 1 has no motionSync; export the model again"): + src.ensure(KEY_B) + # the manifest of another key with the same id + with pytest.raises(RuntimeError, match="not of Character/Live2D/other/adv_model_a"): + src.ensure("Character/Live2D/other/adv_model_a/model/adv_model_a") + + +def dir_files(d): + return sorted(p.relative_to(d).as_posix() for p in d.rglob("*") if p.is_file()) + + +def test_model_dir_exports_each_model_once(tmp_path, monkeypatch): + player = fake_export(monkeypatch, motion_sync=True) + calls, real = [], webmodel.export_model + + def counted(*a): + calls.append(a[2]) + return real(*a) + monkeypatch.setattr(webmodel, "export_model", counted) + root = tmp_path / "out" / "live2d" + a = webmodel.ModelDir(None, player, root) + assert a.ensure(KEY_A) == a.ensure(KEY_A) == {"id": "adv_model_a", "motionSync": True} + assert calls == [KEY_A] and (a.built, a.skipped) == (["adv_model_a"], []) + d = root / "adv_model_a" + # the files of the model's manifest: no unused atlas page, no other platform or GLES 3.1 program + assert dir_files(root) == [f"adv_model_a/{f}" for f in model_files("adv_model_a")] + index = json.loads((d / "shaders/shaders.json").read_text(encoding="utf-8")) + assert [v["file"] for r in index for v in r["variants"]] == [ + "S_Lit/gles3/s0p0_vertex_0.glsl", "S_Lit/gles3/s0p0_vertex_1.glsl", "S_Lit/gles3/s0p0_vertex_5.glsl", + "S_Lit/gles3/s0p0_vertex_6.glsl", "S_Mask/gles3/s0p0_vertex_0.glsl"] + assert (d / "adv_model_a.moc3").read_bytes() == b"MOC3 bytes" + assert a.story_fields(tmp_path / "out" / "s1") == {"modelsDir": "../live2d"} + # another story's source: the directory is used as it is + before = {f: (d / f).read_bytes() for f in dir_files(d)} + b = webmodel.ModelDir(None, player, root) + assert b.ensure(KEY_A) == {"id": "adv_model_a", "motionSync": True} + assert calls == [KEY_A] and (b.built, b.skipped) == ([], ["adv_model_a"]) + assert b.story_fields(tmp_path / "out" / "s2") == {"modelsDir": "../live2d"} + # force: exported again, once per source; the same files + c = webmodel.ModelDir(None, player, root, force=True) + c.ensure(KEY_A) + c.ensure(KEY_A) + assert calls == [KEY_A, KEY_A] and c.built == ["adv_model_a"] + assert {f: (d / f).read_bytes() for f in dir_files(d)} == before + + def failing(*a): + raise RuntimeError("export failed") + # a failed export leaves no directory; a directory that is not a model export is refused + monkeypatch.setattr(webmodel, "export_model", failing) + with pytest.raises(RuntimeError, match="export failed"): + webmodel.ModelDir(None, player, root).ensure(KEY_B) + assert sorted(p.name for p in root.iterdir()) == ["adv_model_a"] + (root / "adv_model_b").mkdir() + with pytest.raises(RuntimeError, match="not a model directory"): + webmodel.ModelDir(None, player, root).ensure(KEY_B) + + +def test_story_command_exports_its_models_next_to_the_story(tmp_path, capsys, monkeypatch): + from nnnotes import story + player = fake_export(monkeypatch) + monkeypatch.setattr(cli, "open_catalog", lambda cfg: None) + monkeypatch.setattr(cli, "player_data", lambda cfg: player) + + def fake_build(cat, md, pl, adv_id, out_dir, *, models, **kw): + index = {"models": {KEY_A: models.ensure(KEY_A)["id"]}, **models.story_fields(out_dir)} + out_dir.mkdir(parents=True) + jsonio.write_json(out_dir / "story.json", index) + return index + monkeypatch.setattr(story, "build", fake_build) + md = tmp_path / "master" + md.mkdir() + (md / "MasterAdv.json").write_text(json.dumps({"_allData": [{"_id": 1}]}), encoding="utf-8") + base = ["--master", str(md), "--language", "ja", "story", "1"] + out = tmp_path / "out" + code, o, err = run([*base, "-o", str(out / "s1")], capsys) + assert code == 0, err + r = json.loads(o) + assert r["models"] == {KEY_A: "adv_model_a"} and r["modelsDir"] == "../live2d" + assert (r["modelsBuilt"], r["modelsSkipped"]) == (["adv_model_a"], []) + assert json.loads((out / "s1/story.json").read_text(encoding="utf-8"))["modelsDir"] == "../live2d" + assert (out / "live2d/adv_model_a" / webmodel.MODEL_INDEX).is_file() + code, o, _ = run([*base, "-o", str(out / "s2")], capsys) # a second story: the model is reused + r = json.loads(o) + assert code == 0 and (r["modelsBuilt"], r["modelsSkipped"]) == ([], ["adv_model_a"]) + code, o, _ = run([*base, "-o", str(out / "s3"), "--models", str(tmp_path / "m"), "--force"], capsys) + r = json.loads(o) + assert code == 0 and r["modelsDir"] == "../../m" and r["modelsBuilt"] == ["adv_model_a"] + assert (tmp_path / "m/adv_model_a" / webmodel.MODEL_INDEX).is_file()