From 9b0ed3f7967b43e2a42eb52330709c7f0deb1e9e Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Wed, 14 Oct 2026 14:23:06 +0800 Subject: [PATCH] docs: publish a documentation site and an English readme The documentation is a Material for MkDocs site, built by `mkdocs build --strict` in CI and published to https://metasekailab.github.io/nnnotes/ from main. Its navigation follows the kind of page: - Tutorial: a quick start from an empty config file to an exported story. - How-to guides: configuration, exporting stories and lives, building an ournotes-player site, the asset export, the Japanese release and the music data. - Reference: the generated command reference, the command outputs, the JSON schemas of nnnotes.schemas, the contracts, stages, views, replay data and UI library. - Explanation: the architecture and its layers, what makes the outputs deterministic, and the validation experiments over a whole game version with their reproduction steps. README.md is now the English readme and the package page, with README.zh-CN.md as its translation. The changelog header states which interfaces the version numbers cover. The command reference no longer repeats a default its help already states. pip install 'nnnotes[docs]' installs the site's tools. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 16 +++ .github/workflows/docs.yml | 51 ++++++++ .gitignore | 1 + CHANGELOG.md | 4 +- README.en.md | 205 -------------------------------- README.md | 219 +++++++++++++---------------------- README.zh-CN.md | 90 ++++++++++++++ cliff.toml | 4 +- docs/architecture.md | 56 +++++++++ docs/cli.md | 34 +++--- docs/commands.md | 4 +- docs/configuration.md | 4 +- docs/contracts.md | 42 +++---- docs/determinism.md | 51 ++++++++ docs/export.md | 91 +++++++++++++++ docs/index.md | 56 +++++++++ docs/jp.md | 2 +- docs/music-data.md | 41 ++----- docs/quickstart.md | 78 +++++++++++++ docs/schemas.md | 39 +++++++ docs/site.md | 77 ++++++++++++ docs/validation.md | 51 ++++++++ mkdocs.yml | 68 +++++++++++ pyproject.toml | 6 +- src/nnnotes/cli/reference.py | 5 +- tests/test_repo.py | 63 ++++++++-- 26 files changed, 922 insertions(+), 436 deletions(-) create mode 100644 .github/workflows/docs.yml delete mode 100644 README.en.md create mode 100644 README.zh-CN.md create mode 100644 docs/architecture.md create mode 100644 docs/determinism.md create mode 100644 docs/export.md create mode 100644 docs/index.md create mode 100644 docs/quickstart.md create mode 100644 docs/schemas.md create mode 100644 docs/site.md create mode 100644 docs/validation.md create mode 100644 mkdocs.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c50675..e5ccc8c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,3 +64,19 @@ jobs: env: GOLDENS_STRICT: "1" run: python -m pytest tests/pipeline/test_goldens.py tests/pipeline/test_objexport.py + + docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.13" + cache: pip + cache-dependency-path: pyproject.toml + - name: Install + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[docs]" + - name: Build the documentation site + run: python -m mkdocs build --strict diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..e198791 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,51 @@ +name: Docs + +# Publishes the documentation site (mkdocs.yml, docs/) to GitHub Pages. +on: + push: + branches: [main] + paths: + - "docs/**" + - "mkdocs.yml" + - ".github/workflows/docs.yml" + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.13" + cache: pip + cache-dependency-path: pyproject.toml + - name: Install + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[docs]" + - name: Build + run: python -m mkdocs build --strict + - uses: actions/configure-pages@v6 + - uses: actions/upload-pages-artifact@v5 + with: + path: site + + deploy: + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - id: deployment + uses: actions/deploy-pages@v5 diff --git a/.gitignore b/.gitignore index a179109..607b6a0 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ nnnotes.toml .env out/ cache/ +/site/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 10115f2..3e0f008 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog -All notable changes to this project are listed here. Versions follow [Semantic Versioning](https://semver.org/). +All notable changes to this project are listed here. Versions follow [Semantic Versioning](https://semver.org/) +over the command line (commands, options, exit statuses and `--json` reports), the configuration keys, the JSON +schemas in `nnnotes.schemas` and the documented Python API. Before 1.0 a minor version can change any of them. ## [0.2.1](https://github.com/MetaSekaiLab/nnnotes/compare/v0.2.0...v0.2.1) - 2026-10-13 diff --git a/README.en.md b/README.en.md deleted file mode 100644 index 43eb131..0000000 --- a/README.en.md +++ /dev/null @@ -1,205 +0,0 @@ -# nnnotes？！ - -Japanese-release support includes anonymous version discovery, authenticated CDN downloads, gzip catalogs, -snapshot-isolated caches and split APKs. See the [JP guide](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/jp.md) -for configuration and validation limits. - -[简体中文](https://github.com/MetaSekaiLab/nnnotes/blob/main/README.md) | [English](https://github.com/MetaSekaiLab/nnnotes/blob/main/README.en.md) - -nnnotes is an offline data toolkit for the game files of BanG Dream! Our Notes: it reads Addressables catalogs, -downloads and decrypts asset bundles, downloads and decodes master data, and exports stories, Live2D models, spots, -shaders, CRI audio and live charts as structured JSON and common file formats. The charts, Live2D models and stories `site build` -exports are the data that [ournotes-player](https://github.com/empty-sekai/ournotes-player) reads. The naming was inspired by sssekai. - -This is an unofficial fan project, not affiliated with the game's developer or operator. The repository contains no -game assets, keys or server addresses: users supply the game files, the keys needed for decryption and the server -addresses in their own configuration, and the exports stay in local directories the user chooses. - -## Install and set up - -```sh -pip install nnnotes # Python 3.11+, 3.13 recommended -nnnotes config init --user # asks for each setting in the terminal (keys without echo), writes the per-user config file read from any directory -nnnotes config check # each setting's origin and whether it is valid, without its value -``` - -- The values (keys, the CDN and API roots of the regions, the paths of your own data and tools) come from the game - client you own; see Configuration below. -- Scripts and AI agents get no questions: `nnnotes config init --user --no-input --set SECTION.KEY=VALUE ...` writes - the file at once, `nnnotes config set SECTION.KEY VALUE` changes one value (VALUE `-` reads it from standard input, - so a key need not appear on the command line), and `nnnotes config check --json` reports the states for machines. -- `--fonts game` also needs `pip install 'nnnotes[fonts]'`, and the deck statistics of `music-data` need - `pip install 'nnnotes[deck]'`; the APK, the master data and the external tools are listed - under Requirements below. - -## Features - -| Command | Input | Output | -|---|---|---| -| `catalog keys` | the region's catalog | addressable keys (optionally by prefix) | -| `catalog browse` | the configured regions' catalogs | a local web page to browse catalogs and bundles (`--host` / `--port`) | -| `bundle pull` | addressable keys | every bundle of each key's dependency closure, decrypted into the local cache | -| `servers` | the bootstrap API root `[bootstrap] api` | the game API's server list: each region's name, area id and CDN / API roots (the roots only with `--show-hosts`) | -| `master version` | region | the master data version and resource version the region serves now (anonymous game API call) | -| `master download` | master data version, or `--latest` (the region's current one) | that version's `MasterManifest.json` and every `.bin` file (SHA-256 checked) | -| `master decode` | master data `.bin` files or directories | one JSON per table (Rijndael-256 CBC decryption + gzip) | -| `export episode` | episode ID | `episode.json`: command list, lines in five languages, voice / sound / video index | -| `export story` | episode ID | a story directory: episode, audio, stage scene and shaders, story UI, frames / particle effects / post effects / stills / talk windows / chat assets, videos (WebM); the Live2D models it uses are exported into a models directory (`--models`, default `live2d/` next to the story directory), where other stories reuse them | -| `export live2d` | model key or model id | a Live2D (Cubism) runtime directory: moc3, textures, motion3, physics, expressions, prefab parameters | -| `export spot` | spot ID | `spot.json` + Spine characters + the room as `room.glb` + shaders | -| `export room` | background prefab key | the room model (binary glTF) | -| `export shader` | key or APK bundles | every platform variant of the shaders (GLSL ES and others) with an index | -| `export audio` | CRI cue sheet | one audio file per cue (FLAC / Ogg / WAV) + cue metadata | -| `apk hca-key` | APK | the CRI HCA keycode from the game's boot data (shows whether it was found; can write a `.hcakey`) | -| `apk settings` | APK | render-related global settings (color space, quality levels, renderers) as JSON | -| `ui` (experimental) | APK set and its embedded catalog | an offline serialized prefab library with dependency roots, AnimatorControllers, stable references and PNG/font resources for the optional ournotes-player UI preview ([format and limits](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/ui.md)) | -| `export live` | music ID + difficulty | a full chart directory: chart and runtime notes, 3D scene, note and effect assets, BGM and sounds, sound routing | -| `site build` | `--pair music:difficulty` (repeatable) or `--all`; `--live2d model` (repeatable) or `--all-live2d`; `--story episode` (repeatable) or `--all-stories`; `--site-region region` (repeatable) or `--all-regions` | an ournotes-player static site: shared player + per-chart / per-model / per-episode manifests + content-addressed assets; the Live2D models of the stories are built first as models, listed in `models.json`, and the story manifests reference them; one site can serve several regions, with listing texts in five languages; a story's interface texts are grouped by language, with TextMeshPro font assets generated from open fonts (the game's fonts with `--fonts game`); compressible assets (JSON, shaders, moc3, ...) are stored gzip encoded by default (`--compress br` / `none`) | -| `music-data` | master data files (`--master-files` directory or `--apk-master`), or decoded master data with its manifest (`--decoded-master`) | one JSON file with every song and chart: titles and credits in five languages, bands, vocal characters, category, tags, release time, score ranks, BGM length; the Gekisou catalog (member cards, snaps and their Gekisou skills and support skills); per difficulty the level, note counts, BPM, chart times, skill events and fever ranges; and the chart statistics the deck model ournotes-deck (installed with `pip install 'nnnotes[deck]'`) measures on its whole-live simulation (the no-skill score, the weight of every score-up skill kind at every position), plus single-shape Gekisou aptitude (increments and standard errors, not an optimal deck; omit with `--no-gekisou-aptitude`). `--full` adds the deck model's input: every chart's runtime notes and the master data tables about cards, skills, bonuses, scores and events ([format](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/music-data.md)) | - -Export conventions: - -- Commands that write files take the output path with `-o` (required). -- A command prints a short summary; with `--json` it prints its report as JSON. -- JSON is always UTF-8 with LF line endings; infinity is written as `1e999`. -- Exports are deterministic for a given installation: the same input with the same versions of nnnotes, its Python - dependencies (UnityPy, Pillow, numpy, ...) and the external tools (vgmstream, FFmpeg) gives byte-identical files. - Other versions can encode the same content into other bytes: PNG files written by another Pillow version can - differ in their bytes while their pixels are equal. -- Values keep Unity's serialized values and field names (`m_LocalPosition`, `_bandIDs`, ...), so they can be - compared with the game data. -- Textures are exported as PNG, shaders keep the game's own compiled programs, audio is decoded from the CRI formats - to common formats. - -## Status - -Based on all data of the Taiwan server, version 1.0.1 (zh-Hant): - -| Part | Status | -|---|---| -| catalog / bundle decryption / dependency closure | working | -| master data decoding | working | -| stories, `export episode` | 946 / 946 episodes | -| stories, `export story` (full directory) | 946 / 946 episodes have all their resources in the catalog and of supported kinds (the resource closure equals the game's own per-episode download list); 946 / 946 exported and checked one by one: the files, models and audio the output refers to exist, every JSON parses and every path it refers to resolves; the 235 episodes with frames, effects, post effects, stills, chats, talk windows or videos are byte-identical across two exports except for the models directory path | -| Live2D models | 239 / 239 (every model of the catalog; episodes use 185 of them) | -| CRI audio | 681 / 681 cue sheets | -| charts, `export live` | 336 / 336 (music, difficulty) pairs | -| web site, `site build` | 336 / 336 charts, 239 / 239 Live2D models, 946 / 946 story episodes (two full builds in English without audio are byte-identical and 946 / 946 pass the data validation); two full builds of the three regions in five languages with AAC audio (`--all-regions`, with the live master data: 340 charts of 85 musics) are likewise byte-identical and pass the data validation in full | -| spots, `export spot` / `export room` | 39 / 39 spots, every reference resolves; `export room` exports 20 / 20 room backgrounds of the catalog. Every glb passes the Khronos glTF-Validator with 0 errors | -| other regions (en / kr) and languages | checked: the regions serve the same catalog for a language and the same bundles, and the keys are shared; the master tables the charts use are the same in the three regions, and the text tables have all five languages; the chart exports checked match the Taiwan server's. A full multi-region site build is verified (the row above) | - -## Requirements - -- Python 3.11+, 3.13 recommended (a `site build` reads and writes a lot of JSON, and the standard library encodes - JSON faster on 3.13); for development `pip install -e .` in the repository -- the game's `base.apk`: APK-local bundles, the CRI keycode, boot settings. `apk settings`, `export story`, `export live` and `site build` read - MonoBehaviours of its boot data with type trees that ship with nnnotes; supported now: game version 1.0.1 - (Unity 6000.3.12f1). With an APK of another version whose classes do not match, these commands stop with an - error naming the class, the game version and the Unity version -- a decoded master data directory (from `master download` + `master decode`): needed by `export episode`, `export story`, `export spot`, - `export live` and the charts of `site build`; the Live2D models of `site build` use it only for their character names (optional) -- external tools: [vgmstream](https://vgmstream.org/) (CRI HCA decoding), [FFmpeg](https://ffmpeg.org/) - (transcoding, including the WebM muxing and Opus audio of story videos); `site build` also needs Node.js 20+ and a built - ournotes-player - -## Configuration - -The code contains no keys, server addresses or default paths. Settings are read in this order, later sources -overriding earlier ones: - -1. config file: `--config `, else `NNNOTES_CONFIG`, else `nnnotes.toml` in the working directory, else the - per-user file (`%APPDATA%\nnnotes\nnnotes.toml` on Windows, `~/.config/nnnotes/nnnotes.toml` elsewhere; - `nnnotes config path` lists the lookup order and the file read) -2. environment variables: `NNNOTES_
_` (e.g. `NNNOTES_BUNDLE_KEY`, `NNNOTES_SERVERS_TW_CDN`) -3. command-line flags: `--region`, `--language`, `--catalog`, `--cache`, `--master`, `--apk`, `--ffmpeg`, - `--vgmstream`, `--node` go before or after the command name; `--player` is an option of `site build` - -`nnnotes config init` writes the config template (every setting empty; it ships with the package, in the repository -[`src/nnnotes/source/nnnotes.example.toml`](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/source/nnnotes.example.toml)), -asking for each value in a terminal; `config set` / `config unset` change one value and keep the file's comments. -The settings are: the bundle -key and nonce seed, the master data key and IV, the region in use (`[catalog] region`) and the catalog language -(`[catalog] language`), the CDN base and API root of each region (`cdn`, `api` of `[servers.]`), the client -version (`[client] version`; unset: the APK's versionName), optionally the bootstrap API root (`[bootstrap] api`, for -`servers`), and the paths of the cache, the APK, the master data directory, ournotes-player and vgmstream / FFmpeg / -Node.js (the three tools are looked up on `PATH` when unset). All values come from the game client you own. - -A missing or malformed setting stops the command with exit status 2 and one line naming the TOML key, the -environment variable and the flag; no setting value is printed. `nnnotes.toml` is in `.gitignore`; do not commit it. - -See [docs/configuration.md](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/configuration.md) for the full reference. - -## Examples - -```bash -nnnotes catalog keys --prefix Live/MusicScore/ --limit 20 -nnnotes catalog browse --port 8000 -nnnotes bundle pull Live/MusicScore/0001/0001_03 -nnnotes master version -nnnotes master download --latest -o work/master-bin -nnnotes master decode work/master-bin -o work/master -nnnotes --master work/master export episode 10462 -o out/adv_10462.json -nnnotes export story 10462 -o out/story_10462 -nnnotes export live 100001 --difficulty expert -o out/live_100001 -nnnotes site build -o out/site --all --player --workers 5 -nnnotes site build -o out/site --pair 100001:expert --pair 100001:hard --player -nnnotes site build -o out/site --all --site-region tw --site-region en --site-region kr --player -nnnotes site build -o out/site --live2d adv_live2d_rana_003_casual_spring_01 --player -``` - -`site build` builds incrementally: charts and models whose manifest exists are skipped (`--force` rebuilds them) and assets -no longer referenced are removed. Common options: - -- `--format aac|opus|vorbis|mp3|flac`: BGM format, default AAC; `--no-audio`: no audio files -- `--band` / `--leader-card`: the band of the LightWeight background and the start timeline; default: the band of - the music's first vocal character -- `--workers`: parallel music processes (default a quarter of the CPUs, up to 8) and model processes (default up - to 4); `--read-workers`: chart read sets at a time (Node.js processes; default half the CPUs, up to 16); `--tmp`: - temporary build directory (default `.tmp`) -- `--live2d MODEL` (model id or key, repeatable) / `--all-live2d`: add Live2D models (every model of the catalog); - charts and models can be added in the same run -- `--site-region REGION` (repeatable) / `--all-regions`: the regions the site serves (default: `[catalog] region`); - each region's master data is `[servers.] master`. Regions with the same chart data share one manifest; - the chart list page switches region and language with `?region=&lang=` -- `--player-only`: rewrite the player files, `charts.json` and `models.json` only; `--reingest-json`: store every - chart's and model's JSON files again under the current rules - -See [docs/commands.md](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/commands.md) for every command's options and output layout. - -## Architecture - -``` -settings (TOML / environment / flags) - └─ access addressables (catalog parsing, bundle decryption, local browser) - catalog (dependency closures, remote and APK bundles, local cache) - master (master data download and decoding) - gameapi (anonymous game API calls: current master data version, server list) - └─ reading unity (UnityPy type trees and TextAssets) - export (prefabs / components / references resolved to JSON, textures and shaders alongside) - shader, textstyle (text layout and style), tmpfont (TextMesh Pro fonts), player (boot settings), cri + crikey (CRI audio) - └─ content stories: adv, advscene, advmedia (frames / effects / post effects / stills / chat), - advvideo (USM videos), advui, story - Live2D: live2d, motion - spots: spot, room - charts: score (chart parsing and a reimplementation of the game's chart converter), - livescene, livenotes, liveui, liveaudio, live - site: web, webmodel (ournotes-player data) -``` - -Every JSON file is written by one writer (`jsonio`), so encoding, line endings and number formatting are the same -everywhere. - -## Changelog - -[CHANGELOG.md](https://github.com/MetaSekaiLab/nnnotes/blob/main/CHANGELOG.md) lists the changes in every version, -generated from the commits by [git-cliff](https://git-cliff.org/). The commit that bumps the version regenerates it: - -```bash -git cliff --tag vX.Y.Z -o CHANGELOG.md -``` - -When a `vX.Y.Z` tag is pushed, the release workflow checks the versions and that CHANGELOG.md has a section for it. - -## License - -MIT, see [LICENSE](https://github.com/MetaSekaiLab/nnnotes/blob/main/LICENSE). diff --git a/README.md b/README.md index b0af862..ce4056e 100644 --- a/README.md +++ b/README.md @@ -1,157 +1,94 @@ # nnnotes？！ -[简体中文](README.md) | [English](README.en.md) +[English](https://github.com/MetaSekaiLab/nnnotes/blob/main/README.md) | [简体中文](https://github.com/MetaSekaiLab/nnnotes/blob/main/README.zh-CN.md) -nnnotes 是 BanG Dream! Our Notes 游戏文件的离线数据工具包：读取 Addressables catalog、下载并解密资源包、下载与解码 masterdata，再把剧情、Live2D、据点场景、着色器、CRI 音频和谱面导出成结构化的 JSON 与通用格式文件。`site build` 导出的谱面、Live2D 模型与剧情站点就是 [ournotes-player](https://github.com/empty-sekai/ournotes-player) 读取的数据。命名灵感来自 sssekai。 +nnnotes is an offline data toolkit for the game files of BanG Dream! Our Notes. It reads Addressables catalogs, +downloads and decrypts asset bundles, downloads and decodes master data, and exports stories, Live2D models, spots, +shaders, CRI audio and live charts as structured JSON and common file formats. `nnnotes site build` writes the static +site that [ournotes-player](https://github.com/empty-sekai/ournotes-player) plays, and `nnnotes music-data` one JSON +file with every song and chart. The name was inspired by sssekai. -本项目为非官方爱好者项目，与游戏的开发和运营方无关。仓库不包含任何游戏资源、密钥或服务器地址：游戏文件、解密所需的密钥和服务器地址都由使用者在配置中自行提供，导出结果只保存在使用者指定的本地目录。 +This is an unofficial fan project, not affiliated with the game's developer or operator. The repository contains no +game assets, keys or server addresses: you supply the game files, the keys and the server addresses in your own +configuration, and the exports stay in the local directories you choose. -## 安装与初始化 +**Documentation: ** -```sh -pip install nnnotes # Python 3.11+，推荐 3.13 -nnnotes config init --user # 在终端里逐项填写设置（密钥不回显），写入用户配置文件，在任何目录下都会读取 -nnnotes config check # 每项设置的来源和格式是否有效，不显示设置值 -``` - -- 设置值（密钥、区服的 CDN 与 API 地址、本地数据与工具的路径）都来自使用者自己的游戏客户端，见[配置](#配置)。 -- 在脚本或 AI 代理中不会有交互提问：`nnnotes config init --user --no-input --set 段.键=值 ...` 一次写好，之后用 `nnnotes config set 段.键 值` 修改单项（值写 `-` 时从标准输入读取，密钥不必出现在命令行里），`nnnotes config check --json` 给出机器可读的状态。 -- `--fonts game` 另需 `pip install 'nnnotes[fonts]'`，`music-data` 的组卡统计另需 `pip install 'nnnotes[deck]'`；APK、masterdata 与外部工具见[需要准备](#需要准备)。 - -## 功能 - -| 命令 | 输入 | 输出 | -|---|---|---| -| `catalog keys` | 区服 catalog | 列出 Addressables 资源键（可按前缀过滤） | -| `catalog browse` | 已配置的各区服 catalog | 本地网页浏览 catalog 与资源包（`--host` / `--port`） | -| `bundle pull` | 资源键 | 下载并解密该键依赖闭包中的全部资源包到本地缓存 | -| `servers` | 引导 API 根地址 `[bootstrap] api` | 游戏 API 的服务器列表：各区服名称、区域 ID 与 CDN / API 根地址（地址仅在 `--show-hosts` 时输出） | -| `master version` | 区服 | 该区服当前的 masterdata 版本与资源版本（游戏 API 匿名调用） | -| `master download` | masterdata 版本号，或 `--latest`（区服当前版本） | 该版本的 `MasterManifest.json` 与全部 `.bin` 文件（SHA-256 校验） | -| `master decode` | masterdata `.bin` 文件或目录 | 每张表一个 JSON（Rijndael-256 CBC 解密 + gzip 解压） | -| `export episode` | 剧情 ID | `episode.json`：命令表、五语台词、语音 / 音效 / 视频索引 | -| `export story` | 剧情 ID | 剧情目录：episode、音频、舞台场景与着色器、剧情 UI、Frame / 粒子特效 / 后处理 / 静帧 / 对话框 / 聊天资源、视频（WebM）；用到的 Live2D 模型导出到模型目录（`--models`，默认为剧情目录旁的 `live2d/`），已导出的模型供其他剧情复用 | -| `export live2d` | 模型资源键或模型 ID | Live2D（Cubism）运行时目录：moc3、贴图、motion3、物理、表情、预制体参数 | -| `export spot` | 据点 ID | `spot.json` + Spine 角色 + 房间 `room.glb` + 着色器 | -| `export room` | 背景预制体键 | 房间模型（binary glTF） | -| `export shader` | 资源键或 APK 内资源包 | 着色器各平台变体（GLSL ES 等）与索引 | -| `export audio` | CRI cue sheet | 每个 cue 一个音频文件（FLAC / Ogg / WAV）+ cue 元数据 | -| `apk hca-key` | APK | 读出游戏启动数据中的 CRI HCA 解码密钥（只显示是否找到，可写成 `.hcakey`） | -| `apk settings` | APK | 渲染相关的全局设置（色彩空间、画质等级、渲染器）JSON | -| `ui`（实验性） | APK 集合及内嵌 catalog | 离线序列化预制体库，包含依赖根对象、AnimatorController、稳定引用与 PNG／字体资源，供 ournotes-player 可选 UI 预览使用（[格式与边界](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/ui.md)） | -| `export live` | 曲目 ID + 难度 | 完整谱面目录：谱面与运行时音符、3D 场景、音符与特效资源、BGM 与音效、声音路由 | -| `site build` | `--pair 曲目:难度`（可重复）或 `--all`；`--live2d 模型`（可重复）或 `--all-live2d`；`--story 剧情 ID`（可重复）或 `--all-stories`；`--site-region 区服`（可重复）或 `--all-regions` | ournotes-player 静态站点：共享播放器 + 每谱 / 每模型 / 每集剧情清单 + 内容寻址资源；剧情用到的 Live2D 模型先按模型构建并列入 `models.json`，剧情清单引用它们；一个站点可服务多个区服，列表文本含五种语言；剧情的界面文字按语言分组，字形由开源字体生成 TextMeshPro 字体资源（`--fonts game` 时用游戏字体）；可压缩的资源（JSON、着色器、moc3 等）默认以 gzip 存储，`--compress br` / `none` 可改 | -| `music-data` | masterdata 文件（`--master-files` 目录或 `--apk-master`），或带清单的解码 masterdata（`--decoded-master`） | 全部歌曲与谱面的单个 JSON：五语标题与作词作曲编曲、乐队、演唱角色、分类、标签、上线时间、评级线、BGM 时长；激走目录（成员卡、小卡及其激走技能与激走支援技能）；每个难度的等级、音符数、BPM、谱面时间、技能事件与 fever 区间；以及组卡模型 ournotes-deck（`pip install 'nnnotes[deck]'` 安装）在整场模拟上实测的谱面统计（无技能得分、每种加分技能在每个演出位的权重），以及单个激走技能形状的谱面适性（增量与标准误，不选最佳编成；`--no-gekisou-aptitude` 可跳过）。`--full` 另附组卡模型的输入：每张谱面的运行时音符与卡牌、技能、加成、分数、活动相关的 masterdata 表（[格式](docs/music-data.md)） | - -导出约定： - -- 写文件的命令都用 `-o` 指定输出路径（必填）。 -- 命令默认输出简短摘要，加 `--json` 时输出 JSON 报告。 -- JSON 一律 UTF-8、LF 换行；无穷大写作 `1e999`。 -- 在同一套环境下导出是确定的：同样的输入，在 nnnotes、Python 依赖库（UnityPy、Pillow、numpy 等）和外部工具（vgmstream、FFmpeg）版本都相同时，导出文件逐字节相同。版本不同时，相同内容可能写成不同的字节：例如不同版本的 Pillow 写出的 PNG 像素相同，字节可能不同。 -- 数值保留 Unity 序列化值与字段名（`m_LocalPosition`、`_bandIDs` 等），便于与游戏数据对照。 -- 贴图导出为 PNG，着色器保留游戏自带的编译结果，音频由 CRI 格式解码为通用格式。 - -## 完成度 - -日服现已支持匿名版本查询、CDN 认证下载、gzip catalog、资源路径与缓存隔离，以及 Android 分包读取。 -配置、命令和日服验证范围见 [日服支持](docs/jp.md)。 - -以下结果基于台服 1.0.1（zh-Hant）的全部数据： - -| 部分 | 状态 | -|---|---| -| catalog / 资源包解密 / 依赖闭包 | 可用 | -| masterdata 解码 | 可用 | -| 剧情 `export episode` | 946 / 946 集可导出 | -| 剧情 `export story`（完整目录） | 946 / 946 集的资源都在 catalog 中且类型均受支持（资源闭包与游戏自带的每集下载清单一致）；946 / 946 集已逐集导出验证：输出中引用的文件、模型与音频均存在，JSON 均可解析，引用的路径均可解析；其中 235 集用到 Frame / Effect / PostEffect / Still / Chat / TalkWindow / 视频资源，两次导出除模型目录路径外逐字节一致 | -| Live2D 模型 | 239 / 239 个可导出（catalog 中的全部模型，剧情用到其中 185 个） | -| CRI 音频 | 681 / 681 个 cue sheet 可解码 | -| 谱面 `export live` | 336 / 336 个（曲目, 难度）组合可导出 | -| 网页站点 `site build` | 336 / 336 张谱面，239 / 239 个 Live2D 模型，946 / 946 集剧情（英语、无音频的全量构建两次结果逐字节一致，946 / 946 集通过数据校验）；三个区服、五种语言、含 AAC 音频的全量构建（`--all-regions`，用线上 masterdata：85 首曲目的 340 张谱面）两次结果同样逐字节一致，全部通过数据校验 | -| 据点 `export spot` / `export room` | 39 / 39 个据点可导出，引用均可解析；catalog 中 20 / 20 个房间背景可由 `export room` 导出。生成的 glb 均通过 Khronos glTF-Validator，0 个错误 | -| 其他区服（en / kr）与其他语言 | 已核对：各区服同一语言的 catalog 相同、资源包相同，密钥通用；谱面用到的 masterdata 表在三个区服间一致，文本表五种语言齐全；抽查的谱面导出与台服一致。多区服整站已全量构建验证（见上一行） | - -## 需要准备 - -- Python 3.11+，推荐 3.13（`site build` 构建读写大量 JSON，3.13 的标准库 JSON 编码更快）；开发时在仓库内 `pip install -e .` -- 游戏安装包 `base.apk`：APK 内置资源包、CRI 解码密钥、启动设置。`apk settings`、`export story`、`export live`、`site build` 用 nnnotes 自带的类型树读取 APK 启动数据中的 MonoBehaviour，目前支持游戏版本 1.0.1（Unity 6000.3.12f1）；其他版本的 APK 若类型不符，这些命令会报错并给出类名、游戏版本和 Unity 版本 -- 解码后的 masterdata 目录（可用 `master download` + `master decode` 生成）：`export episode`、`export story`、`export spot`、`export live` 和 `site build` 的谱面需要；`site build` 的 Live2D 模型只用它取角色名（可选） -- 外部工具：[vgmstream](https://vgmstream.org/)（CRI HCA 解码）、[FFmpeg](https://ffmpeg.org/)（转码，含剧情视频的 WebM 封装与 Opus 音频）；`site build` 另需 Node.js 20+ 与构建好的 ournotes-player - -## 配置 - -代码中没有任何密钥、服务器地址和默认路径。所有设置按以下顺序读取，后者覆盖前者： - -1. 配置文件：`--config <文件>`，否则 `NNNOTES_CONFIG`，否则当前目录的 `nnnotes.toml`，否则用户配置文件（Windows 为 `%APPDATA%\nnnotes\nnnotes.toml`，其他系统为 `~/.config/nnnotes/nnnotes.toml`；`nnnotes config path` 列出查找顺序与实际读取的文件） -2. 环境变量：`NNNOTES_<节>_<键>`（如 `NNNOTES_BUNDLE_KEY`、`NNNOTES_SERVERS_TW_CDN`） -3. 命令行参数：`--region`、`--language`、`--catalog`、`--cache`、`--master`、`--apk`、`--ffmpeg`、`--vgmstream`、`--node` 写在命令名之前或之后；`--player` 是 `site build` 的参数 - -`nnnotes config init` 写出配置模板（每项设置为空，模板随包安装，仓库中为 [`src/nnnotes/source/nnnotes.example.toml`](src/nnnotes/source/nnnotes.example.toml)），在终端中会逐项询问；`config set` / `config unset` 修改单项并保留文件中的注释。需要的设置包括：资源包解密密钥与 nonce 种子、masterdata 的密钥与 IV、使用的区服 `[catalog] region` 与 catalog 语言 `[catalog] language`、每个区服的 CDN 地址与 API 根地址（`[servers.<区服>]` 的 `cdn`、`api`）、客户端版本 `[client] version`（未设置时读取 APK 的 versionName）、可选的引导 API 根地址 `[bootstrap] api`（`servers` 使用），以及缓存目录、APK、masterdata 目录、ournotes-player 和 vgmstream / FFmpeg / Node.js 的路径（三个工具未设置时在 `PATH` 中查找）。这些值都来自使用者自己的游戏客户端。 - -缺少或格式错误的设置会让命令以退出码 2 结束，并用一行说明对应的配置键、环境变量和命令行参数，不会输出任何设置值。`nnnotes.toml` 已在 `.gitignore` 中，请勿提交。 - -完整说明见 [docs/configuration.md](docs/configuration.md)。 - -## 使用示例 - -```bash -nnnotes catalog keys --prefix Live/MusicScore/ --limit 20 -nnnotes catalog browse --port 8000 -nnnotes bundle pull Live/MusicScore/0001/0001_03 -nnnotes master version -nnnotes master download --latest -o work/master-bin -nnnotes master decode work/master-bin -o work/master -nnnotes --master work/master export episode 10462 -o out/adv_10462.json -nnnotes export story 10462 -o out/story_10462 -nnnotes export live 100001 --difficulty expert -o out/live_100001 -nnnotes site build -o out/site --all --player --workers 5 -nnnotes site build -o out/site --pair 100001:expert --pair 100001:hard --player -nnnotes site build -o out/site --all --site-region tw --site-region en --site-region kr --player -nnnotes site build -o out/site --live2d adv_live2d_rana_003_casual_spring_01 --player -``` - -`site build` 可以增量构建：清单已存在的谱面和模型会跳过（`--force` 重建），不再被引用的资源会被清理。常用参数： - -- `--format aac|opus|vorbis|mp3|flac`：BGM 格式，默认 AAC；`--no-audio`：不导出音频 -- `--band` / `--leader-card`：轻量背景与开场时间轴所用乐队；默认取曲目第一位演唱角色的乐队 -- `--workers`：并行处理的曲目进程数（默认为 CPU 数的四分之一，最多 8）与模型进程数（默认最多 4）；`--read-workers`：同时运行的谱面读取集（Node.js 进程）数（默认为 CPU 数的一半，最多 16）；`--tmp`：临时构建目录（默认 `<站点>.tmp`） -- `--live2d 模型`（模型 ID 或资源键，可重复）/ `--all-live2d`：加入 Live2D 模型（catalog 中全部模型），可与谱面在同一次构建中加入 -- `--site-region 区服`（可重复）/ `--all-regions`：站点服务的区服（默认 `[catalog] region`）；每个区服的 masterdata 由 `[servers.<区服>] master` 指定。谱面数据相同的区服共用一份清单，列表页用 `?region=&lang=` 切换区服与语言 -- `--player-only`：只重写播放器文件与 `charts.json`、`models.json`；`--reingest-json`：按当前规则重新存储所有谱面与模型的 JSON +## Install -各命令的参数与输出目录结构见 [docs/commands.md](docs/commands.md)。 - -## 架构 - -``` -配置（TOML / 环境变量 / 参数） - └─ 访问层 addressables（catalog 解析、资源包解密、本地浏览） - catalog（依赖闭包、远端与 APK 内资源、本地缓存） - master（masterdata 下载与解码） - gameapi（游戏 API 匿名调用：当前 masterdata 版本、服务器列表） - └─ 读取层 unity（UnityPy 读取 typetree 与 TextAsset） - export（预制体 / 组件 / 引用全部解析为 JSON，贴图与着色器随同导出） - shader、textstyle（文字排版与样式）、tmpfont（TextMesh Pro 字体）、player（启动设置）、cri + crikey（CRI 音频） - └─ 内容层 剧情：adv、advscene、advmedia（Frame / 特效 / 后处理 / 静帧 / 聊天）、advvideo（USM 视频）、advui、story - Live2D：live2d、motion - 据点：spot、room - 谱面：score（谱面解析与游戏谱面转换器的复现）、livescene、livenotes、liveui、liveaudio、live - 站点：web、webmodel（ournotes-player 数据） +```sh +pip install nnnotes # Python 3.11+ +pip install 'nnnotes[fonts]' # the game's fonts (--fonts game) and the story text of site build +pip install 'nnnotes[deck]' # the deck model's chart statistics of music-data ``` -所有 JSON 由同一个写出器生成（`jsonio`），保证编码、换行与数值格式一致。 +nnnotes also needs the game's `base.apk`, decoded master data for the commands that read it, and the external tools +[vgmstream](https://vgmstream.org/) and [FFmpeg](https://ffmpeg.org/) for audio and video (Node.js 20+ and a built +ournotes-player for `site build`). -## 版本记录 +## Quick start -每个版本的改动见 [CHANGELOG.md](CHANGELOG.md)，由 [git-cliff](https://git-cliff.org/) 根据提交生成。发版时，改版本号的那个提交同时重新生成它： - -```bash -git cliff --tag vX.Y.Z -o CHANGELOG.md +```sh +nnnotes config init --user # asks for each setting (keys without echo) and writes the per-user config file +nnnotes config check # where each setting comes from and whether it is valid, never its value +nnnotes master download --latest -o work/master-bin +nnnotes master decode work/master-bin -o work/master +nnnotes --master work/master export story 10462 -o out/story_10462 +nnnotes site build -o out/site --all --player path/to/ournotes-player ``` -推送 `vX.Y.Z` 标签后，发布流程会核对版本号，并确认 CHANGELOG.md 里有这个版本。 +Every command prints a short summary, or with `--json` its report as JSON. Scripts set the config file without +questions: `nnnotes config init --user --no-input --set SECTION.KEY=VALUE ...`. + +## Commands + +| Command | What it does | +| --- | --- | +| `config init\|set\|unset\|check\|path` | write, edit and check the config file | +| `servers` | the game API's server list: regions with their CDN and API roots | +| `catalog keys\|browse` | the addressable keys of a region's catalog; a local web page to browse catalogs and bundles | +| `catalog versions\|import\|fetch\|diff` | catalog versions kept in the store, and what changed between two | +| `bundle pull` | fetch and decrypt the bundle closure of keys into the cache | +| `master version\|download\|decode` | the master data version a region serves, its download and its decoding to JSON | +| `export assets` | every object of the selected bundles in common formats, through the content-addressed store | +| `export episode\|story\|live\|live2d\|spot\|room\|shader\|audio\|ui` | one subject: an ADV episode, a story, a live, a Live2D model, a spot, a room, shaders, a cue sheet, the UI library | +| `apk settings\|hca-key` | the player's graphics settings and the CRI HCA keycode of `base.apk` | +| `store plan\|run\|verify` | what `export assets` would run and why, single tasks, the store's check | +| `voices list\|search\|summary\|get` | the character and story voices of the master data | +| `site build` | the static site of ournotes-player: charts, Live2D models and stories, in several regions and languages | +| `music-data` | every song and chart in one JSON file, with the deck model's chart statistics | + +The [command reference](https://metasekailab.github.io/nnnotes/cli/) lists every option. + +## Documentation + +- [Quick start](https://metasekailab.github.io/nnnotes/quickstart/): from an empty config to an exported story. +- How-to guides: [configure nnnotes](https://metasekailab.github.io/nnnotes/configuration/), + [export stories and lives](https://metasekailab.github.io/nnnotes/export/), + [build an ournotes-player site](https://metasekailab.github.io/nnnotes/site/), + [export every asset](https://metasekailab.github.io/nnnotes/assets/), + [the Japanese release](https://metasekailab.github.io/nnnotes/jp/), + [music data](https://metasekailab.github.io/nnnotes/music-data/). +- Reference: [commands](https://metasekailab.github.io/nnnotes/cli/), + [command outputs](https://metasekailab.github.io/nnnotes/commands/), + [JSON schemas](https://metasekailab.github.io/nnnotes/schemas/), + [contracts](https://metasekailab.github.io/nnnotes/contracts/), [stages](https://metasekailab.github.io/nnnotes/stages/), + [views](https://metasekailab.github.io/nnnotes/views/). +- Explanation: [architecture](https://metasekailab.github.io/nnnotes/architecture/), + [determinism](https://metasekailab.github.io/nnnotes/determinism/), + [validation](https://metasekailab.github.io/nnnotes/validation/). + +Exports are deterministic for a given installation: the same input with the same versions of nnnotes, its Python +dependencies (UnityPy, Pillow, numpy, ...) and the external tools gives byte-identical files; JSON is UTF-8 with LF +line endings and non-finite numbers written as `1e999`. Other versions can encode the same content into other bytes +(PNG files written by another Pillow version can differ in their bytes while their pixels are equal). + +## Changelog and versions + +[CHANGELOG.md](https://github.com/MetaSekaiLab/nnnotes/blob/main/CHANGELOG.md) lists the changes of every version. +Versions follow [Semantic Versioning](https://semver.org/) over the command line, the configuration keys, the JSON +schemas and the documented Python API. ## License -MIT，见 [LICENSE](LICENSE)。 +MIT, see [LICENSE](https://github.com/MetaSekaiLab/nnnotes/blob/main/LICENSE). diff --git a/README.zh-CN.md b/README.zh-CN.md new file mode 100644 index 0000000..425339e --- /dev/null +++ b/README.zh-CN.md @@ -0,0 +1,90 @@ +# nnnotes？！ + +[English](https://github.com/MetaSekaiLab/nnnotes/blob/main/README.md) | [简体中文](https://github.com/MetaSekaiLab/nnnotes/blob/main/README.zh-CN.md) + +nnnotes 是 BanG Dream! Our Notes 游戏文件的离线数据工具包：读取 Addressables catalog，下载并解密资源包，下载并解码 +masterdata，再把剧情、Live2D 模型、据点、着色器、CRI 音频和谱面导出成结构化 JSON 与通用格式文件。 +`nnnotes site build` 写出 [ournotes-player](https://github.com/empty-sekai/ournotes-player) 播放的静态站点， +`nnnotes music-data` 写出包含全部歌曲与谱面的单个 JSON 文件。命名灵感来自 sssekai。 + +本项目为非官方爱好者项目，与游戏的开发和运营方无关。仓库不包含任何游戏资源、密钥或服务器地址：游戏文件、密钥和服务器地址 +都由使用者在自己的配置中提供，导出结果只保存在使用者指定的本地目录。 + +**文档（英文）：** + +## 安装 + +```sh +pip install nnnotes # Python 3.11+ +pip install 'nnnotes[fonts]' # 游戏字体（--fonts game）与 site build 的剧情文字 +pip install 'nnnotes[deck]' # music-data 的组卡模型谱面统计 +``` + +此外还需要游戏的 `base.apk`；读取 masterdata 的命令需要解码后的 masterdata；音频与视频需要外部工具 +[vgmstream](https://vgmstream.org/) 和 [FFmpeg](https://ffmpeg.org/)（`site build` 另需 Node.js 20+ 与构建好的 +ournotes-player）。 + +## 快速上手 + +```sh +nnnotes config init --user # 逐项询问设置（密钥不回显），写入用户配置文件 +nnnotes config check # 每项设置的来源与是否有效，不显示设置值 +nnnotes master download --latest -o work/master-bin +nnnotes master decode work/master-bin -o work/master +nnnotes --master work/master export story 10462 -o out/story_10462 +nnnotes site build -o out/site --all --player path/to/ournotes-player +``` + +每个命令输出简短摘要，加 `--json` 时输出 JSON 报告。脚本中可以不经提问写好配置文件： +`nnnotes config init --user --no-input --set 段.键=值 ...`。 + +## 命令 + +| 命令 | 作用 | +| --- | --- | +| `config init\|set\|unset\|check\|path` | 写入、修改和检查配置文件 | +| `servers` | 游戏 API 的服务器列表：各区服及其 CDN 与 API 根地址 | +| `catalog keys\|browse` | 区服 catalog 的资源键；在本地网页中浏览 catalog 与资源包 | +| `catalog versions\|import\|fetch\|diff` | store 中保存的 catalog 版本，以及两个版本之间的变化 | +| `bundle pull` | 下载并解密资源键的依赖闭包到缓存 | +| `master version\|download\|decode` | 区服当前的 masterdata 版本、下载，以及解码为 JSON | +| `export assets` | 经内容寻址的 store，把所选资源包中的每个对象导出为通用格式 | +| `export episode\|story\|live\|live2d\|spot\|room\|shader\|audio\|ui` | 单个对象：ADV 剧情集、完整剧情、谱面、Live2D 模型、据点、房间、着色器、cue sheet、UI 库 | +| `apk settings\|hca-key` | `base.apk` 中的画面设置与 CRI HCA 密钥 | +| `store plan\|run\|verify` | `export assets` 将运行什么及原因、单独运行任务、检查 store | +| `voices list\|search\|summary\|get` | masterdata 中的角色语音与剧情语音 | +| `site build` | ournotes-player 静态站点：谱面、Live2D 模型与剧情，支持多个区服与语言 | +| `music-data` | 全部歌曲与谱面的单个 JSON 文件，附组卡模型的谱面统计 | + +[命令参考](https://metasekailab.github.io/nnnotes/cli/)列出了所有参数。 + +## 文档 + +- [快速上手](https://metasekailab.github.io/nnnotes/quickstart/)：从空配置到导出一集剧情。 +- 操作指南：[配置](https://metasekailab.github.io/nnnotes/configuration/)、 + [导出剧情与谱面](https://metasekailab.github.io/nnnotes/export/)、 + [构建 ournotes-player 站点](https://metasekailab.github.io/nnnotes/site/)、 + [导出全部资源](https://metasekailab.github.io/nnnotes/assets/)、 + [日服](https://metasekailab.github.io/nnnotes/jp/)、 + [music-data](https://metasekailab.github.io/nnnotes/music-data/)。 +- 参考：[命令](https://metasekailab.github.io/nnnotes/cli/)、 + [命令输出](https://metasekailab.github.io/nnnotes/commands/)、 + [JSON schema](https://metasekailab.github.io/nnnotes/schemas/)、 + [契约](https://metasekailab.github.io/nnnotes/contracts/)、[stage](https://metasekailab.github.io/nnnotes/stages/)、 + [视图](https://metasekailab.github.io/nnnotes/views/)。 +- 原理：[架构](https://metasekailab.github.io/nnnotes/architecture/)、 + [确定性](https://metasekailab.github.io/nnnotes/determinism/)、 + [验证](https://metasekailab.github.io/nnnotes/validation/)。 + +在同一套环境下导出是确定的：同样的输入，在 nnnotes、Python 依赖库（UnityPy、Pillow、numpy 等）和外部工具版本都相同时， +导出文件逐字节相同；JSON 一律 UTF-8、LF 换行，非有限数写作 `1e999`。版本不同时，相同内容可能写成不同的字节（不同版本的 +Pillow 写出的 PNG 像素相同，字节可能不同）。 + +## 版本记录与版本号 + +每个版本的改动见 [CHANGELOG.md](https://github.com/MetaSekaiLab/nnnotes/blob/main/CHANGELOG.md)。版本号遵循 +[语义化版本](https://semver.org/lang/zh-CN/)，约束的接口是命令行、配置键、JSON schema 和文档中的 Python API。 + +## License + +MIT，见 [LICENSE](https://github.com/MetaSekaiLab/nnnotes/blob/main/LICENSE)。 diff --git a/cliff.toml b/cliff.toml index b04045f..67fdb30 100644 --- a/cliff.toml +++ b/cliff.toml @@ -5,7 +5,9 @@ header = """ # Changelog -All notable changes to this project are listed here. Versions follow [Semantic Versioning](https://semver.org/).\n +All notable changes to this project are listed here. Versions follow [Semantic Versioning](https://semver.org/) +over the command line (commands, options, exit statuses and `--json` reports), the configuration keys, the JSON +schemas in `nnnotes.schemas` and the documented Python API. Before 1.0 a minor version can change any of them.\n """ body = """ {%- macro commit_link(id) -%} diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..e259fe3 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,56 @@ +# Architecture + +nnnotes reads the game's files in layers: settings and sources at the bottom, file formats and Unity objects above +them, the game's content above those, then the pipeline, the products and the command line. Each layer is one +subpackage of `nnnotes` and imports only the layers below it. + +## Layers + +| Layer | Subpackage | What it holds | +|---|---|---| +| Command line | `nnnotes.cli` | one module per command group; parses options, reads the configuration, prints summaries and `--json` reports | +| Products | `nnnotes.site`, `nnnotes.musicdata` | the ournotes-player site (charts, models, stories, indexes); the music data file and its deck statistics | +| Pipeline | `nnnotes.pipeline` | stages, task descriptions and results, the content-addressed store, plans, layouts, the catalog versions, the views of the master data | +| Content | `nnnotes.content` | the game's subjects as exports: ADV episodes and stories, lives and scores, spots, rooms, the UI library | +| Unity objects | `nnnotes.unity` | bundle reading, the typetree exporter, the player data, Live2D prefabs, text styles and TextMesh Pro fonts | +| File formats | `nnnotes.formats` | formats read from bytes: CRI ACB/AWB and HCA, USM movies, CRI Lips data, shaders, motions | +| Sources | `nnnotes.source` | the configuration, the APK, Addressables catalogs and bundles, the game API, master data, languages, `Session` | +| Core | `nnnotes.core` | the one JSON writer and the process caches | + +The order is a contract: the `layers` contract of [import-linter](https://import-linter.readthedocs.io/) in +`pyproject.toml` fails the build when a module imports a layer above its own, and `nnnotes.site` and +`nnnotes.musicdata` do not import each other. No library module imports `nnnotes.cli`: the commands parse options +and print, and the work they run is in the library modules below them. + +## Sources and the Session + +Every setting comes from the configuration ([sources and precedence](configuration.md#sources-and-precedence)). +`nnnotes.source.session.Session` turns one configuration into the data sources a command needs: the catalog of a +region, its decoded master data and the master data version it serves, the APK, the bundle and master data keys and +the store root. It reads a setting when it is first needed and raises a `ConfigError` naming the setting when it is missing or wrong; +the command line turns that error into exit status 2. + +The catalog (`nnnotes.source.catalog.Catalog`) resolves an addressable key to its bundle and the bundle's +dependency closure, takes a bundle from the APK when the APK has it and from the CDN otherwise, and keeps fetched and +decrypted bundles in the cache directory. + +## Two ways to export + +- **One subject.** `export episode`, `export story`, `export live`, `export live2d`, `export spot`, `export room`, + `export shader`, `export audio`, `export ui`, `site build` and `music-data` read what one subject needs through the + Session and write their output directory directly ([Export stories and lives](export.md)). +- **Every asset.** `export assets` splits the work into stages ([stages](stages.md)). A task is one stage applied to + one subject (a bundle, a content id, a view); its key covers exactly what its output depends on: the stage and + converter versions, the input contents and the options ([contracts](contracts.md)). Results and their files are + stored once by content hash in the store, and the output directory is laid out from the store. A second run finds + every unchanged result in the store and only lays out what changed; `store plan` says what would run and why + ([asset export](assets.md)). + +Both ways use the same readers below the pipeline layer, so a texture, shader, mesh or audio stream is converted by +one implementation whichever command asks for it. + +## Outputs + +Every JSON file is written by one writer (`nnnotes.core.jsonio`), which fixes the encoding, the line endings and the +number format; [determinism](determinism.md) explains what makes the outputs reproducible. Command reports and the +documents of the asset export have JSON Schemas shipped in `nnnotes.schemas` ([JSON schemas](schemas.md)). diff --git a/docs/cli.md b/docs/cli.md index 3683c4b..1b38fe7 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -84,7 +84,7 @@ the addressable keys of the catalog (the total on standard error) | Argument | Description | | --- | --- | | `--prefix` `PREFIX` | only the keys that start with this prefix (default ``) | -| `--limit` `LIMIT` | at most N keys (default 200; 0: all) (default `200`) | +| `--limit` `LIMIT` | at most N keys (default 200; 0: all) | ## nnnotes catalog browse @@ -92,8 +92,8 @@ browse the configured regions' catalogs and bundles in a local web page | Argument | Description | | --- | --- | -| `--port` `PORT` | port (default 8000) (default `8000`) | -| `--host` `HOST` | address to listen on (default 127.0.0.1) (default `127.0.0.1`) | +| `--port` `PORT` | port (default 8000) | +| `--host` `HOST` | address to listen on (default 127.0.0.1) | ## nnnotes catalog versions @@ -178,7 +178,7 @@ every object of the selected bundles -> common formats (the store and its layout | `--workers` `N` | worker processes (default: the CPUs this process may use; 0: this process) | | `--memory` `GiB` | memory budget of the running tasks and workers (default: 80 % of the physical memory) | | `--fetch-workers` `N` | parallel downloads (default 8) | -| `--layout` `LAYOUT` | original, cas or original,cas (default original) (default `original`) | +| `--layout` `LAYOUT` | original, cas or original,cas (default original) | | `--explain` | print every setting of the run first | | `--link` `LINK` | how the layout files are made from the store: auto (default; [export] link): a clone where the file system makes them, else a hard link (read-only), else a copy; or that method only (one of `auto`, `clone`, `hard`, `copy`) | | `--strict` | exit 1 when a view has gaps | @@ -203,10 +203,10 @@ an ADV episode -> story directory (episode, audio, scene, UI, media, videos) + i | `-o`, `--out` `OUT` | output directory (required) | | `--models` `DIR` | directory of the Live2D models, one / each (default: OUT/../live2d) | | `--force` | export the models again whose directory exists | -| `--format` `FORMAT` | audio file format (default flac) (one of `flac`, `ogg`, `wav`; default `flac`) | +| `--format` `FORMAT` | audio file format (default flac) (one of `flac`, `ogg`, `wav`) | | `--flac-level` `N` | FLAC compression level 0-12 of --format flac (default 8; every level decodes to the same samples) | | `--no-audio` | do not decode the cue sheets | -| `--fonts` `FONTS` | open (default): text layout and style only, no font data; game: also the game's TMP fonts (needs the 'fonts' extra) (one of `open`, `game`; default `open`) | +| `--fonts` `FONTS` | open (default): text layout and style only, no font data; game: also the game's TMP fonts (needs the 'fonts' extra) (one of `open`, `game`) | ## nnnotes export live @@ -215,10 +215,10 @@ a live (music + difficulty) -> self-contained live directory | Argument | Description | | --- | --- | | `music_id` | MasterLiveMusic id | -| `--difficulty` `DIFFICULTY` | chart difficulty (default expert) (one of `easy`, `normal`, `hard`, `expert`; default `expert`) | -| `--format` `FORMAT` | audio file format (default flac) (one of `flac`, `ogg`, `wav`; default `flac`) | +| `--difficulty` `DIFFICULTY` | chart difficulty (default expert) (one of `easy`, `normal`, `hard`, `expert`) | +| `--format` `FORMAT` | audio file format (default flac) (one of `flac`, `ogg`, `wav`) | | `--flac-level` `N` | FLAC compression level 0-12 of --format flac (default 8; every level decodes to the same samples) | -| `--fonts` `FONTS` | start canvas text: open (default): layout and style only, no font data (your fonts are not used); game: also the game's TMP fonts (needs the 'fonts' extra) (one of `open`, `game`; default `open`) | +| `--fonts` `FONTS` | start canvas text: open (default): layout and style only, no font data (your fonts are not used); game: also the game's TMP fonts (needs the 'fonts' extra) (one of `open`, `game`) | | `--band` `BAND` | band of the LightWeight background and start timeline (default: band of the music's first vocal character) | | `--leader-card` `LEADER_CARD` | deck centre MasterMemberCard id (its character's band) | | `--live-option` `OPTION[=VALUES]` | also export the files of a Live option's variants (repeatable): MirrorChart, MeasureLineDisplay, NoteDesignId, NoteEffectId, LiveQuality, NoteSePatternId (every value of the master data, or =v,v...), defaults (the option defaults and ranges only) or all | @@ -273,7 +273,7 @@ a CRI cue sheet -> audio files | --- | --- | | `cue_sheet` | cue sheet name: the key Cri/Sound/ of the catalog | | `-o`, `--out` `OUT` | output directory (required) | -| `--format` `FORMAT` | audio file format (default flac) (one of `flac`, `ogg`, `wav`; default `flac`) | +| `--format` `FORMAT` | audio file format (default flac) (one of `flac`, `ogg`, `wav`) | | `--flac-level` `N` | FLAC compression level 0-12 of --format flac (default 8; every level decodes to the same samples) | ## nnnotes export ui @@ -284,7 +284,7 @@ the APK's UI prefabs -> an offline UI prefab library for ournotes-player/ui | --- | --- | | `-o`, `--out` `OUT` | UI data directory outside the source repositories (required) | | `--key` `KEY` | exact APK catalog key (repeatable) | -| `--prefix` `PREFIX` | APK UI key prefix (default EmbUI/) (default `EmbUI/`) | +| `--prefix` `PREFIX` | APK UI key prefix (default EmbUI/) | | `--limit` `LIMIT` | maximum number of keys | | `--no-dependencies` | omit additional prefab roots and controllers | | `--force` | re-export the selected keys | @@ -321,7 +321,7 @@ what `export assets` would run and why, without running it (--json: nnnotes.plan | `--workers` `N` | worker processes (default: the CPUs this process may use; 0: this process) | | `--memory` `GiB` | memory budget of the running tasks and workers (default: 80 % of the physical memory) | | `--fetch-workers` `N` | parallel downloads (default 8) | -| `--layout` `LAYOUT` | original, cas or original,cas (default original) (default `original`) | +| `--layout` `LAYOUT` | original, cas or original,cas (default original) | | `--explain` | print every setting of the run first | | `--since` `RUN` | explain against this run (default: the latest of the selection) | | `--check` | exit 1 when anything would run or change | @@ -395,7 +395,7 @@ the files of one voice: a row id (MasterTalk:561, MasterAdv::) or a | `id` | row id or sound id | | `-o`, `--out` `OUT` | output directory (required) | | `--from` `OUT` | an export directory with views/voices.json (export assets --views voices); else the index is built from the configured master data and catalog (no files) | -| `--format` `FORMAT` | flac (default: the exported files when --from has them), ogg or wav (decodes the sheet) (one of `flac`, `ogg`, `wav`; default `flac`) | +| `--format` `FORMAT` | flac (default: the exported files when --from has them), ogg or wav (decodes the sheet) (one of `flac`, `ogg`, `wav`) | | `--flac-level` `N` | FLAC compression level 0-12 of --format flac: decode the sheet at this level, also with --from (default 8; every level decodes to the same samples) | ## nnnotes site build @@ -412,9 +412,9 @@ live charts, Live2D models and stories -> static site for ournotes-player (share | `--reingest-json` | store every chart's and model's JSON files again | | `--live2d` `MODEL` | Live2D model id or key Character/Live2D///model/ (repeatable) | | `--all-live2d` | every Live2D model of the catalog | -| `--format` `FORMAT` | audio format of the charts and stories (default aac) (one of `aac`, `opus`, `vorbis`, `mp3`, `flac`; default `aac`) | +| `--format` `FORMAT` | audio format of the charts and stories (default aac) (one of `aac`, `opus`, `vorbis`, `mp3`, `flac`) | | `--no-audio` | export no audio files | -| `--compress` `COMPRESS` | encoding of the compressible assets (JSON, shaders, moc3, ...) where it makes them smaller: gzip (default), br (brotli) or none (one of `gzip`, `br`, `none`; default `gzip`) | +| `--compress` `COMPRESS` | encoding of the compressible assets (JSON, shaders, moc3, ...) where it makes them smaller: gzip (default), br (brotli) or none (one of `gzip`, `br`, `none`) | | `--force` | rebuild charts, models and stories whose manifest exists (stories: also their models) | | `--tmp` `TMP` | directory for the temporary live and model builds (default .tmp) | | `--workers` `N` | parallel music / story / model processes (default a quarter of the CPUs, up to 8; models up to 4; 0: this process) | @@ -425,7 +425,7 @@ live charts, Live2D models and stories -> static site for ournotes-player (share | `--font` `LANG=PATH` | font file the story text of a language is drawn with (repeatable; [paths] fonts.); emoji=PATH: the colour emoji font the emoji sprites are drawn from ([paths] fonts.emoji) | | `--site-region` `REGION` | a region the site serves: a [servers.] table (repeatable; the first is the base; default: [catalog] region) | | `--all-regions` | every configured region | -| `--fonts` `FONTS` | text of the start canvas and the stories: open (default): the start canvas has layout and style only, the stories glyphs from your font files (--font); game: the game's TMP fonts (needs the 'fonts' extra) (one of `open`, `game`; default `open`) | +| `--fonts` `FONTS` | text of the start canvas and the stories: open (default): the start canvas has layout and style only, the stories glyphs from your font files (--font); game: the game's TMP fonts (needs the 'fonts' extra) (one of `open`, `game`) | | `--band` `BAND` | band of the LightWeight background and start timeline (default: band of the music's first vocal character) | | `--leader-card` `LEADER_CARD` | deck centre MasterMemberCard id (its character's band) | | `--live-option` `OPTION[=VALUES]` | also export the files of a Live option's variants (repeatable): MirrorChart, MeasureLineDisplay, NoteDesignId, NoteEffectId, LiveQuality, NoteSePatternId (every value of the master data, or =v,v...), defaults (the option defaults and ranges only) or all | @@ -451,7 +451,7 @@ every live song and chart: metadata in every language, chart facts and the deck | `--decoded-master` | decoded master data ([paths] master or --master) with the MasterManifest.json of the files it was decoded from; no master key | | `--full` | also write the deck model's input: every chart's runtime notes and the master data tables about cards, skills, bonuses, scores and events | | `--no-deck` | do not run the deck model (every chart's deck is null) | -| `--seeds` `N` | replay seeds for a chart with a luck range (default 8); expectations do not use sampling (default `8`) | +| `--seeds` `N` | replay seeds for a chart with a luck range (default 8); expectations do not use sampling | | `--workers` `N` | threads measuring charts (default: every processor; 0: one thread) | | `--no-gekisou-aptitude` | leave out the charts' Gekisou aptitude (every gekisouAptitude is null) | | `--stats-cache` `DIR` | reuse native simulation programs and runs in DIR across builds; completed cache entries survive interrupted builds | diff --git a/docs/commands.md b/docs/commands.md index 49e2468..7dc367c 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -1,4 +1,4 @@ -# Commands +# Command outputs ``` nnnotes [options] @@ -561,7 +561,7 @@ SITE/assets/.[.gz|.br] content-addressed files shared by all runs the chart silent on its own clock, and plays stories without their sounds; story videos keep their sound track). - `--workers`: parallel music processes (default a quarter of the CPUs, up to 8), model processes (default up - to 4) and story processes (default a quarter of the CPUs, up to 8); `1` builds in this process. `--read-workers`: chart read sets run at a time, each a Node.js process + to 4) and story processes (default a quarter of the CPUs, up to 8); `0` or `1` builds in this process. `--read-workers`: chart read sets run at a time, each a Node.js process (default half the CPUs, up to 16). With a player whose read-set script lists a chart's files from its plan (without stepping the chart) and serves many charts from one process, the plans run in one long-lived Node.js process per read-set slot, ended with the build; the full simulations that check a sample of the plans still run diff --git a/docs/configuration.md b/docs/configuration.md index fe913ca..2cfcb5a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -20,7 +20,7 @@ Lowest to highest: 3. **Command-line flags**, for the settings that have one (table below). An empty value (`""`, `[]`, an empty environment variable, an empty flag) counts as unset, so a lower source still -applies. The template [`nnnotes.example.toml`](../src/nnnotes/source/nnnotes.example.toml) ships with the package and lists +applies. The template [`nnnotes.example.toml`](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/source/nnnotes.example.toml) ships with the package and lists every setting with an empty value; `nnnotes config init` writes it (below). `nnnotes.toml` and `*.local.toml` are in `.gitignore`; do not commit a filled-in copy. @@ -56,7 +56,7 @@ nnnotes config path [--json] No `config` command prints a setting's value. The config files these commands write are readable by their owner only (mode 600 where the file system has modes). -The global flags go **before** the command name: +The global flags go before or after the command name: ```bash nnnotes --config ~/nnnotes.toml --cache /data/nnnotes-cache bundle pull diff --git a/docs/contracts.md b/docs/contracts.md index 48007f3..c9e0f2d 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -2,8 +2,8 @@ The asset stages exchange documents in fixed formats. They are the public interface of the extraction: a task description can be run on another machine, a result can be read without nnnotes, and every output layout can be -derived from the records. JSON Schemas of all documents are in [src/nnnotes/schemas](../src/nnnotes/schemas/), the package `nnnotes.schemas` (draft 2020-12, each with -`$id` `urn:nnnotes:schema:`; shared definitions in [common.schema.json](../src/nnnotes/schemas/common.schema.json)). +derived from the records. JSON Schemas of all documents are in [src/nnnotes/schemas](https://github.com/MetaSekaiLab/nnnotes/tree/main/src/nnnotes/schemas), the package `nnnotes.schemas` (draft 2020-12, each with +`$id` `urn:nnnotes:schema:`; shared definitions in [common.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/common.schema.json)). ## Versioning @@ -47,7 +47,7 @@ bundles of a catalog have the same stable name, both keep their full file names. ## Task: nnnotes.task/1 One stage applied to one subject, with everything needed to run it -([task.schema.json](../src/nnnotes/schemas/task.schema.json)): +([task.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/task.schema.json)): ```json {"schema": "nnnotes.task/1", "id": "unity.export:membercard_assets_membercard_100101", @@ -90,7 +90,7 @@ Locator kinds, tried in order to find an input's bytes (each checked against the ## Artifact record: nnnotes.artifact/1 -One output file ([artifact.schema.json](../src/nnnotes/schemas/artifact.schema.json)): +One output file ([artifact.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/artifact.schema.json)): ```json {"id": "CAB-…:-4153…#image", @@ -112,7 +112,7 @@ Roles: an object's primary artifact is the one whose role comes first in `image` ## Result: nnnotes.result/1 The result of a task, written after the bytes of all its artifacts are stored: its presence is the commit -([result.schema.json](../src/nnnotes/schemas/result.schema.json)). +([result.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/result.schema.json)). ```json {"schema": "nnnotes.result/1", "key": "…", "keyParts": {…}, "status": "ok", @@ -139,7 +139,7 @@ stage, an input that cannot be read, a worker process that dies) leaves no resul ## Run manifest: nnnotes.run/1 -What one run covered ([run.schema.json](../src/nnnotes/schemas/run.schema.json)): the context (region, language, the catalog +What one run covered ([run.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/run.schema.json)): the context (region, language, the catalog version, and the master data its views read: the version named by `MasterManifest.json` and the sha256 of each table), the selection, the normalized parameters per stage, the stage versions, every task with its key (`null` when it could not be described) and status (`hit`: the result was in the store, `ran`, `failed`), the @@ -166,7 +166,7 @@ Task failure codes (failed items carry the reason codes of their stages): ## Plan: nnnotes.plan/1 -What a run would do, without running anything ([plan.schema.json](../src/nnnotes/schemas/plan.schema.json)): every task as +What a run would do, without running anything ([plan.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/plan.schema.json)): every task as a node with its key, status (`hit`, `run`, `unknown`), reasons and cost estimate, the edges between tasks, and a summary (counts, estimated CPU seconds and peak memory for the given workers and budget, tasks above the budget, and per layout the files it would write, remove and keep). Reasons compare the key parts with those of the same @@ -191,7 +191,7 @@ task id in the previous run manifest: ## Layout manifest: nnnotes.layout/1 The files an output layout owns in its directory, as `manifest.json` there -([layout.schema.json](../src/nnnotes/schemas/layout.schema.json)): name, parameters, and `entries` `{path, sha256, size, id}` +([layout.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/layout.schema.json)): name, parameters, and `entries` `{path, sha256, size, id}` sorted by path, one per artifact (artifacts of equal content may share a path). A layout changes neither artifact bytes nor keys; the file names of `original` and `cas` both follow from the records, and `cas` also from the `original` manifest alone. Documents derived for `original` from records and paths (the views with the files of @@ -199,14 +199,14 @@ their objects) are the exception: `original` has the derived documents, `cas` th ## Reports -- Coverage, `nnnotes.coverage/1` ([coverage.schema.json](../src/nnnotes/schemas/coverage.schema.json)): per class the census +- Coverage, `nnnotes.coverage/1` ([coverage.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/coverage.schema.json)): per class the census total, the objects without an item (`missing`) and the count of every item status; per reason code its count and the first five object ids. An object with items in several results (a sprite whose image `sprite.crop` makes beside its `unity.export` item) counts once, with the most severe of its statuses (`failed`, `unsupported`, `generic`, `exported`, `contained`, in that order) and that item's reason. -- Failures, `nnnotes.failures/1` ([failures.schema.json](../src/nnnotes/schemas/failures.schema.json)): `{task, stage, input, +- Failures, `nnnotes.failures/1` ([failures.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/failures.schema.json)): `{task, stage, input, object, code, message}` for failed tasks (`object` null) and failed items, sorted by task and object. -- Cost, `nnnotes.cost/1` ([cost.schema.json](../src/nnnotes/schemas/cost.schema.json)): the measured CPU and wall seconds +- Cost, `nnnotes.cost/1` ([cost.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/cost.schema.json)): the measured CPU and wall seconds and peak resident memory of a task's last run, and the resident memory of its process when it started (a worker keeps memory from its earlier tasks: the task's own need is the difference). @@ -214,20 +214,20 @@ Documents written by particular stages (artifacts of no object: their ids are `< | Document | Stage | Schema | |---|---|---| -| `nnnotes.catalog-index/1` | `catalog.index` | [catalog-index.schema.json](../src/nnnotes/schemas/catalog-index.schema.json) | -| `nnnotes.census/1` | `unity.census` | [census.schema.json](../src/nnnotes/schemas/census.schema.json) | -| `nnnotes.scripts/1` | `link.scripts` | [scripts.schema.json](../src/nnnotes/schemas/scripts.schema.json) | -| `nnnotes.addresses/1` | `link.addresses` | [addresses.schema.json](../src/nnnotes/schemas/addresses.schema.json) | -| `nnnotes.artifacts/1` | `link.artifacts` | [artifacts.schema.json](../src/nnnotes/schemas/artifacts.schema.json) | -| `nnnotes.view/1` | `view.` | [view.schema.json](../src/nnnotes/schemas/view.schema.json) | - -The catalog versions of a store, `nnnotes.catalogs/1` ([catalogs.schema.json](../src/nnnotes/schemas/catalogs.schema.json), +| `nnnotes.catalog-index/1` | `catalog.index` | [catalog-index.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/catalog-index.schema.json) | +| `nnnotes.census/1` | `unity.census` | [census.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/census.schema.json) | +| `nnnotes.scripts/1` | `link.scripts` | [scripts.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/scripts.schema.json) | +| `nnnotes.addresses/1` | `link.addresses` | [addresses.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/addresses.schema.json) | +| `nnnotes.artifacts/1` | `link.artifacts` | [artifacts.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/artifacts.schema.json) | +| `nnnotes.view/1` | `view.` | [view.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/view.schema.json) | + +The catalog versions of a store, `nnnotes.catalogs/1` ([catalogs.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/catalogs.schema.json), `/catalogs/index.json`), and the difference between two of them, `nnnotes.catalog-diff/1` -([catalog-diff.schema.json](../src/nnnotes/schemas/catalog-diff.schema.json), `nnnotes catalog diff --json`), are written +([catalog-diff.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/catalog-diff.schema.json), `nnnotes catalog diff --json`), are written by the commands. The music data file of `nnnotes music-data`, `nnnotes.music-data/2` -([music-data.schema.json](../src/nnnotes/schemas/music-data.schema.json)), names its format in a `format` field; it is +([music-data.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/music-data.schema.json)), names its format in a `format` field; it is described in [music-data.md](music-data.md). ## Store diff --git a/docs/determinism.md b/docs/determinism.md new file mode 100644 index 0000000..550828c --- /dev/null +++ b/docs/determinism.md @@ -0,0 +1,51 @@ +# Determinism + +An export is a function of its inputs. The same game files, configuration and options, with the same versions of +nnnotes, its Python dependencies and the external tools, give byte-identical output files. This page says what +fixes the bytes, what can change them, and how the tests hold it. + +## What fixes the bytes + +- **One JSON writer.** Every JSON file goes through `nnnotes.core.jsonio`: UTF-8 without a byte order mark, LF line + endings, non-ASCII characters as they are, and non-finite numbers as `1e999` / `-1e999`. Those literals are valid + JSON and read back as infinity by `JSON.parse` and Python's `json.loads`; a NaN is an error that names where it + occurs. A document's key order is the order it is built in, which is fixed by the code, not by a hash table. +- **canon/1.** The documents of the asset export follow a stricter form, [canon/1](contracts.md#canon1): sorted keys, + fixed indentation, tagged values for NaN and binary data, and content ids that are the SHA-256 of the bytes. A + task's key is the SHA-256 of the compact canonical form of everything its output depends on. +- **Fixed order.** Parallel results are collected in a fixed order and file listings are sorted before they are + written, so `--workers` changes how long a command takes, never its output. +- **Unity values as stored.** Numbers keep the serialized value and field names (`m_LocalPosition`, `_bandIDs`), and + textures, shaders and audio are converted from the stored data with fixed encoder settings. +- **No clocks.** No output holds the time of a run or a path of the machine it ran on. A result of the asset export + holds no input name, locator, timestamp or traceback; the measured cost of a task (CPU and wall seconds, memory) is + kept in the store's cost report, apart from the results. + +## What can change the bytes + +Another version of an encoder can write the same content as other bytes: + +- Pillow writes the PNG files. Another Pillow version can compress the same pixels differently: the bytes differ + while the pixels are equal. +- numpy, UnityPy and the decoders of texture formats shape converted meshes and images. +- vgmstream and FFmpeg decode and encode audio and video; FLAC keeps the decoded samples at every compression level, + while lossy formats depend on the encoder version. +- zlib and brotli encode the compressed assets of a site. + +So the guarantee is per installation: pin the versions to reproduce a result exactly. The store keeps this visible: +a task's key includes the implementation ids of the atoms it uses, with the versions of the libraries they call, so +another installation computes other keys instead of mixing results. + +## How it is tested + +- **Golden records.** `tests/golden/.json` records, for each stage version, the libraries that shape its + bytes and, for every synthetic fixture, the task key and the SHA-256 of the canonical result. A change of a + stage's output without a new stage version fails the test. The CI job that checks the records installs the exact + library versions of `.github/goldens-constraints.txt` and sets `GOLDENS_STRICT=1`; elsewhere a record made with + other library versions is skipped. +- **Two runs, one result.** The tests run exports twice over synthetic inputs and compare the bytes, and compare + one worker with several: a store built with four workers equals one built with one, and parallel audio decoding + equals sequential decoding. +- **Writer tests.** The JSON writer's tests cover the non-finite literals, the NaN error and the line endings. + +[Validation](validation.md) describes the checks over a full game version. diff --git a/docs/export.md b/docs/export.md new file mode 100644 index 0000000..8251fdd --- /dev/null +++ b/docs/export.md @@ -0,0 +1,91 @@ +# Export stories and lives + +The `export` commands each write one subject as a directory or file. They read the catalog, fetch and decrypt the +bundles they need into the cache, and need decoded master data where the subject is defined by it +([what each command needs](configuration.md#what-each-command-needs)). To export every object of many bundles at +once, use [`export assets`](assets.md) instead. + +Every command below prints a short summary; add `--json` for its report as JSON. The layout of each output is in +[Command outputs](commands.md). + +## A story + +```sh +nnnotes export episode 10462 -o out/episode_10462.json # the episode's commands, lines and resources +nnnotes export story 10462 -o out/stories/10462 # everything the episode plays +``` + +`export story` writes the episode, its decoded cue sheets, the stage scene, the story UI, frames, effects, stills, +chats and videos below `-o`, and its Live2D models below the models directory (`--models`, default `live2d/` next to +the story directory). A model directory that exists is used as it is, so exporting the stories of a season into +sibling directories exports each model once. + +- `--format flac|ogg|wav` and `--flac-level N` choose the audio format; `--no-audio` leaves the cue sheets undecoded. +- `--fonts game` also writes the game's fonts reduced to the characters shown (needs `pip install 'nnnotes[fonts]'`). +- `--force` exports the episode's models again. + +`MasterAdv` lists the episode ids; `nnnotes voices list --source story --episode ID` lists the voiced lines of one. +Details: [export episode](commands.md#export-episode), [export story](commands.md#export-story). + +## A live + +```sh +nnnotes export live 100001 --difficulty expert -o out/lives/100001_expert +``` + +One chart as the self-contained directory ournotes-player reads: the chart as shipped and its converted runtime +notes, the 3D scene, the note and effect assets, the BGM and sound effects with their routing, and the start canvas +in the configured language. + +- `--band BAND` or `--leader-card CARD_ID` picks the band of the background and start timeline (default: the band of + the music's first vocal character). +- `--live-option NAME[=VALUES]` (repeatable) adds the files of other values of the live options, such as + `MirrorChart` or `NoteDesignId`; `--live-option all` adds every option with every value. + +Details: [export live](commands.md#export-live). + +## A Live2D model + +```sh +nnnotes catalog keys --prefix Character/Live2D/ --limit 0 # every model key +nnnotes export live2d adv_live2d_rana_003_casual_spring_01 -o out/live2d/rana +``` + +`MODEL` is the model's key or its id, the last part of the key. The output is a Cubism runtime directory: moc3, atlas +pages, motions, expressions, physics, `model3.json` and the whole prefab as JSON. Details: +[export live2d](commands.md#export-live2d). + +## A spot or a room + +```sh +nnnotes export spot SPOT_ID -o out/spots/SPOT_ID +nnnotes export room KEY -o out/rooms/room.glb # a background prefab key +``` + +`export spot` writes the spot's master row with its names and tap-talk episodes, its Spine characters, its +background as binary glTF and the shaders of its bundles. `export room` writes one background prefab as binary glTF +with a summary next to it. Details: [export spot](commands.md#export-spot), [export room](commands.md#export-room). + +## Shaders, audio and the UI library + +```sh +nnnotes export shader --key KEY -o out/shaders # every shader of a key's bundle closure +nnnotes export audio CUE_SHEET -o out/audio/CUE_SHEET # one cue sheet, one file per stream +nnnotes export ui -o out/ui # the prefab library of the APK's embedded catalog +``` + +`export audio` decodes `Cri/Sound/` with the HCA key read from the APK; FLAC keeps the decoded PCM +bit-exact at every `--flac-level`. Details: [export shader](commands.md#export-shader), +[export audio](commands.md#export-audio), [UI library](ui.md). + +## Voices + +```sh +nnnotes voices list --character 1 --category character-rank-up +nnnotes voices search "hello" --search-language en +nnnotes voices get MasterTalk:561 -o out/voices/561 +``` + +The `voices` commands list, search and fetch the character and story voices of the master data. Without an +`export assets` output (`--from`) they read the master data and the catalog directly and decode only the cue sheet a +`voices get` needs. Details: [voices](commands.md#voices). diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..f6df314 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,56 @@ +# nnnotes + +nnnotes is an offline data toolkit for the game files of BanG Dream! Our Notes. It reads Addressables catalogs, +downloads and decrypts asset bundles, downloads and decodes master data, and exports stories, Live2D models, spots, +shaders, CRI audio and live charts as structured JSON and common file formats. `nnnotes site build` writes the static +site that [ournotes-player](https://github.com/empty-sekai/ournotes-player) plays, and `nnnotes music-data` one JSON +file with every song and chart. + +This is an unofficial fan project, not affiliated with the game's developer or operator. nnnotes contains no game +assets, keys or server addresses: you supply the game files, the keys and the server addresses in your own +configuration, and the exports stay in the local directories you choose. + +## Where to start + +
+ +- **[Quick start](quickstart.md)** + + From an empty config file to an exported story, step by step. + +- **How-to guides** + + [Configure nnnotes](configuration.md), [export stories and lives](export.md), + [build an ournotes-player site](site.md), [export every asset](assets.md), + [the Japanese release](jp.md), [music data](music-data.md). + +- **Reference** + + [Commands](cli.md), [command outputs](commands.md), [JSON schemas](schemas.md), [contracts](contracts.md), + [stages](stages.md), [views](views.md), [replay data](replay.md), [UI library](ui.md). + +- **Explanation** + + [Architecture](architecture.md), [determinism](determinism.md), [validation](validation.md). + +
+ +## Install + +```sh +pip install nnnotes # Python 3.11+ +pip install 'nnnotes[fonts]' # the game's fonts (--fonts game) and the story text of site build +pip install 'nnnotes[deck]' # the deck model's chart statistics of music-data +``` + +Besides the package, nnnotes reads the game's `base.apk` and, for the commands that need it, decoded master data; +audio and video need [vgmstream](https://vgmstream.org/) and [FFmpeg](https://ffmpeg.org/), and `site build` also +Node.js 20+ and a built ournotes-player. [Configuration](configuration.md#what-each-command-needs) lists what each +command needs. + +## Versions + +Versions follow [Semantic Versioning](https://semver.org/). The interface a version describes is the command line +(commands, options, exit statuses and the `--json` reports), the configuration keys, the JSON schemas shipped in +`nnnotes.schemas` and the documented Python API. Before 1.0 a minor version can change any of them; the +[changelog](https://github.com/MetaSekaiLab/nnnotes/blob/main/CHANGELOG.md) names every change. diff --git a/docs/jp.md b/docs/jp.md index 755496f..df5067e 100644 --- a/docs/jp.md +++ b/docs/jp.md @@ -100,7 +100,7 @@ The synthetic tests use local gRPC and HTTP servers, including gzip catalogs, sp rotation, redirects, 429, truncated responses, source isolation, APK-only and hash-only updates, offline replay and invalid paths. -Live validation on 2026-09-30 used JP client 1.0.4 and local JP Android 1.0.3 data: +A validation run with JP client 1.0.4 and JP Android 1.0.3 data gave: - 238 master tables downloaded with manifest hashes and decoded successfully; - a 73-note chart and a 512×512 jacket read successfully; diff --git a/docs/music-data.md b/docs/music-data.md index 9e6fff5..88d3a67 100644 --- a/docs/music-data.md +++ b/docs/music-data.md @@ -7,7 +7,7 @@ chart facts (level, note counts, BPM, chart times, skill events, fevers), and pe the chart contributes to the live score whatever the deck, in a solo live (Gekisou off) and in a Gekisou live at every rank, measured by the deck model [ournotes-deck](https://github.com/empty-sekai/ournotes-deck), the package `ournotes-sim` that the extra `deck` installs (`pip install 'nnnotes[deck]'`). The format is `nnnotes.music-data/2`; its JSON Schema is -[music-data.schema.json](../src/nnnotes/schemas/music-data.schema.json). +[music-data.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/music-data.schema.json). ``` nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--full] [--no-deck] [--seeds N] @@ -24,7 +24,7 @@ nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--ful `MasterManifest.json` of the files they were decoded from, as a master data snapshot published with its manifest carries it. The file's `region` is `[catalog] region`; no master key is needed. - `--full`: also write the deck model's input, every chart's runtime notes and the master data tables about cards, - skills, bonuses, scores and events ([the deck input](#the-deck-input---full)), for tools that run a deck model of + skills, bonuses, scores and events ([the deck input](#the-deck-input-full)), for tools that run a deck model of their own. - `--no-deck`: do not run the deck model; every chart's `deck` is null. A full statistics build evaluates many complete lives per chart. `--workers N` sets the native worker count (default: every available processor; `0`: one thread). @@ -36,8 +36,7 @@ nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--ful compiled simulation inputs, so unchanged programs and runs can survive a change to the skill catalog. New shapes need their own measurements; changing a shape's numeric index does not by itself identify a new program. Every invocation still passes the complete current master data and charts to the native model, which constructs - the current header and assembles the requested statistics. Python does not reuse a chart-sized JSON result or - replace the model's dependency analysis with a table allowlist. + the current header and assembles the requested statistics. Cache records are isolated by cache schema and model source SHA-256. Each completed result is saved through a unique temporary file, a file sync and an atomic rename. Invalid or unreadable records are measured again; interrupted work retains its completed records. Concurrent processes may compute the same missing record, but @@ -45,7 +44,7 @@ nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--ful collection, and unused model-source directories can be removed to reclaim space. A record larger than 64 MiB is not persisted. The command's `deckStats` summary reports native requests, hits, completed computations, writes, invalid entries and written bytes; these count cache operations, not charts. Cache counters are not part of the - published music data. This cache does not import the previous Python chart cache. + published music data. Removing the cache changes the amount of work, not the bytes of the music data. - `--no-bgm`: do not read the cue sheets (every `bgm.length` is null). - `--jackets DIR`: also write every song's jacket, the Texture2D `Image/Jacket/`, as `DIR/.webp` (WebP quality 88, scaled down with Lanczos to at most 320 pixels on the longer side, without alpha when opaque); a @@ -107,7 +106,7 @@ A **text** is an object with one string per language of `languages` (`{"ja": ... | `catalog.sha256` | SHA-256 of the remote catalog file the charts were read with | | `master.source` | `api` (`--master-files`, `--decoded-master`: the region's files) or `embedded` (`--apk-master`) | | `master.version` | the `version` of the master data manifest | -| `master.tables..sha256` | SHA-256 of each table's file as served, before decoding: the song tables (`MasterLiveMusic`, `MasterLiveMusicScore`, `MasterText`, `MasterBand`, `MasterCharacter`, `MasterTag`, `MasterLiveMusicCategory`, `MasterSound`, `MasterSoundCueSheet`, `MasterLiveScoreRank`), the tables of the Gekisou catalog (`MasterMemberCard`, `MasterSupportCard`, `MasterSupportCardRank`, `MasterGekisouSkill`, `MasterGekisouSkillEffect`, `MasterGekisouSupportSkill`, `MasterGekisouSupportSkillEffect`) and, when the deck model runs or with `--full`, the tables of [the deck input](#the-deck-input---full) | +| `master.tables.
.sha256` | SHA-256 of each table's file as served, before decoding: the song tables (`MasterLiveMusic`, `MasterLiveMusicScore`, `MasterText`, `MasterBand`, `MasterCharacter`, `MasterTag`, `MasterLiveMusicCategory`, `MasterSound`, `MasterSoundCueSheet`, `MasterLiveScoreRank`), the tables of the Gekisou catalog (`MasterMemberCard`, `MasterSupportCard`, `MasterSupportCardRank`, `MasterGekisouSkill`, `MasterGekisouSkillEffect`, `MasterGekisouSupportSkill`, `MasterGekisouSupportSkillEffect`) and, when the deck model runs or with `--full`, the tables of [the deck input](#the-deck-input-full) | | `exporter.name`, `exporter.version` | `nnnotes` and its version | | `exporter.chartFormat` | the format of the chart converter the notes come from, `nnnotes.live-score/1` | | `deck` | the deck model: `{name, version, source, commit, sourceSha256, format}`, `ournotes-deck`, the package version of its crate ournotes-sim, the repository, the git commit nnnotes is built with and the statistics format (`ournotes-deck.chart-stats/3`); null with `--no-deck` | @@ -458,32 +457,16 @@ the replay and recommendation engines of `--replay-engine` and `--recommend-engi `build-info.json` names the release commit that `provenance.deck.commit` names; engines of another release are rejected. -## Versions - -`format` names the major version. Within `nnnotes.music-data/2`, fields and tables are only added, never renamed, -removed or given another meaning, and readers ignore keys they do not know. Readers must reject an unknown major -version. This version carries `ournotes-deck.chart-stats/3` in `provenance.deck.format` and requires that exact -statistics format from the extension module. - -`nnnotes.music-data/2` replaces the per-seed Gekisou statistics and sampled aptitude of `/1` with nominal -expectations and outward numerical intervals. `seeds` becomes `expectation` plus separate `replaySeeds`; the -aptitude `seedRule` becomes `law`, sampling flags disappear, checks use interval `expected`/`predicted` values, and -Perfect-play range bonuses are explicit. These are changes of meaning as well as shape: an older reader must not -interpret a radius as a standard error. Free Live `offSeeds`, chart facts and the optional `nnnotes.deck-data/1` -input retain their existing semantics. The old aptitude sampling-cap CLI options are no longer accepted. - -The persistent statistics cache also starts a new format. The native cache uses model-source namespaces and -fine-grained program/run identities; no old Python chart-cache entry is migrated or trusted. Removing a cache -changes the amount of work, not the music data bytes. - -Deploy readers supporting `/2` and nominal expectations before publishing the new format. The reference page -is pinned in `.github/workflows/music-data-region.yml`; any configured override must support the same format. -The native release packages, Python producer and deployed consumers must move together as described in -[the publication workflow](../.github/MUSIC_DATA.md#before-the-first-run). - `--replay-dir OUT/replay --replay-engine PKG [--recommend-engine PKG]` writes normalized runtime inputs, the pinned model's WASM release packages (the replay engine and, optionally, the recommendation engine) and same-snapshot Snap label/reference-card metadata as described in [replay.md](replay.md). The music data stays compact and carries the SHA-bound `replay.manifestUrl` pointer; its manifest optionally references `snap-labels.json`. The selected decoded label tables retain names/templates, original per-level effects and artwork IDs. No encrypted master/chart blobs or native binary are included in this artifact bundle. Label metadata does not change the score model. + +## Versions + +`format` names the major version. Within `nnnotes.music-data/2`, fields and tables are only added, never renamed, +removed or given another meaning, and readers ignore keys they do not know. Readers must reject an unknown major +version. This version carries `ournotes-deck.chart-stats/3` in `provenance.deck.format` and requires that exact +statistics format from the extension module. diff --git a/docs/quickstart.md b/docs/quickstart.md new file mode 100644 index 0000000..93d1c43 --- /dev/null +++ b/docs/quickstart.md @@ -0,0 +1,78 @@ +# Quick start + +This walk-through takes you from an empty configuration to an exported ADV episode and a story directory. It needs +your own installation of the game: its `base.apk`, and the keys and server addresses of your client +([where the values come from](configuration.md#where-the-values-come-from)). + +## 1. Install + +```sh +pip install 'nnnotes[fonts]' +``` + +Python 3.11 or later. Install [vgmstream](https://vgmstream.org/) (`vgmstream-cli`) and +[FFmpeg](https://ffmpeg.org/) as well, on `PATH` or named in the config file; stories decode their audio and +videos with them. + +## 2. Write the config file + +```sh +nnnotes config init --user +``` + +In a terminal this asks for each setting in turn, among them the bundle and master data keys, the names of your +regions with each region's CDN base and API root, the catalog language, and the paths of the cache, the APK and the +tools. Enter leaves a setting empty; the keys are read without echo. The file is written to the per-user location, +which every command reads from any directory (`nnnotes config path` shows it). + +Check it: + +```sh +nnnotes config check +``` + +Every setting is listed with where it comes from and whether it is valid, never with its value. A command that +needs an unset setting stops with exit status 2 and names the TOML key, the environment variable and the flag that +set it. + +## 3. Look at the catalog + +```sh +nnnotes catalog keys --prefix Adv/ --limit 5 +``` + +The first command downloads the region's catalog into the cache; the keys it lists are the addressable keys of the +game's assets. `nnnotes catalog browse` serves a local web page to browse the catalogs and bundles of every +configured region. + +## 4. Get the master data + +```sh +nnnotes master version +nnnotes master download --latest -o work/master-bin +nnnotes master decode work/master-bin -o work/master +nnnotes config set paths.master work/master +``` + +`master version` asks the region's game API which master data version it serves now; `master download` fetches +that version and checks every file's SHA-256; `master decode` writes one JSON file per table. The last line makes +the decoded directory the default of every command that reads master data. + +## 5. Export an episode and a story + +```sh +nnnotes export episode 10462 -o out/episode_10462.json +nnnotes export story 10462 -o out/story_10462 +``` + +`export episode` writes the episode's command list with its lines in five languages and the index of its voices, +sounds and videos. `export story` writes the whole story directory: the episode, its audio, the stage scene and +shaders, the story UI, frames, effects, stills, chats and videos; the Live2D models it uses go to `out/live2d/`, +where the next story reuses them. Add `--json` to any command to get its report as JSON. + +## Next + +- [Export stories and lives](export.md): the other single-subject exports and their options. +- [Build an ournotes-player site](site.md): charts, models and stories as one static site. +- [Export every asset](assets.md): every object of the selected bundles through the content-addressed store. +- [Command reference](cli.md): every command and option. diff --git a/docs/schemas.md b/docs/schemas.md new file mode 100644 index 0000000..051f6fe --- /dev/null +++ b/docs/schemas.md @@ -0,0 +1,39 @@ +# JSON schemas + +The documents nnnotes writes in a fixed format have JSON Schemas (draft 2020-12), shipped in the package +`nnnotes.schemas`. Each schema has the `$id` `urn:nnnotes:schema:`, and the shared definitions are in +`common`. Every document names its format and version in a `schema` field, such as `nnnotes.plan/1`; within a major +version fields are only added ([versioning](contracts.md#versioning)). + +```python +from nnnotes import schemas + +schemas.names() # ['addresses', 'artifact', ...] +schemas.load("plan") # the schema as a dict +``` + +| Schema | Format | Document | +|---|---|---| +| `addresses` | `nnnotes.addresses/1` | the address table of a catalog version and the censuses of its bundles | +| `artifact` | `nnnotes.artifact/1` | the content, provenance and semantics of one output file | +| `artifacts` | `nnnotes.artifacts/1` | the artifact table: the artifacts of every object, with content id, size and extension | +| `catalog-diff` | `nnnotes.catalog-diff/1` | the difference between two catalog versions (`catalog diff`) | +| `catalog-index` | `nnnotes.catalog-index/1` | the index of a catalog pair: remote and APK catalogs, every location in catalog order | +| `catalogs` | `nnnotes.catalogs/1` | the catalog versions of a store (`catalog versions`) | +| `census` | `nnnotes.census/1` | the census of one bundle: serialized files, type trees and objects, read without decoding | +| `common` | | definitions shared by the other schemas | +| `cost` | `nnnotes.cost/1` | the measured cost of a task's last run | +| `coverage` | `nnnotes.coverage/1` | the coverage of a run per class and per reason code | +| `failures` | `nnnotes.failures/1` | the failed tasks and items of a run | +| `layout` | `nnnotes.layout/1` | the files a layout owns in its output directory | +| `music-data` | `nnnotes.music-data/2` | every song and chart of one master data version ([music data](music-data.md)) | +| `plan` | `nnnotes.plan/1` | what a run would do and why (`store plan`) | +| `result` | `nnnotes.result/1` | the result of a task, a function of its key alone | +| `run` | `nnnotes.run/1` | a run manifest: context, selection, tasks with keys and statuses, layouts | +| `scripts` | `nnnotes.scripts/1` | every MonoScript of the censuses, by object id | +| `task` | `nnnotes.task/1` | one stage applied to one subject, with everything needed to run it (`store run`) | +| `view` | `nnnotes.view/1` | a semantic view of a master table, each row with its names and the objects it names | + +The documents of the asset export are described in [contracts](contracts.md), the views in [views](views.md). The +schema files are in +[src/nnnotes/schemas](https://github.com/MetaSekaiLab/nnnotes/tree/main/src/nnnotes/schemas). diff --git a/docs/site.md b/docs/site.md new file mode 100644 index 0000000..668bb57 --- /dev/null +++ b/docs/site.md @@ -0,0 +1,77 @@ +# Build an ournotes-player site + +`nnnotes site build` writes a static site of charts, Live2D models and stories that +[ournotes-player](https://github.com/empty-sekai/ournotes-player) plays in the browser. The site is a directory of +plain files: the player's pages and bundles, one manifest per chart, model and story, and the files they read, +stored once by content hash. + +## What it needs + +- The settings of the export commands: keys, the region's CDN, `[paths] apk`, and the decoded master data of every + region the site serves ([configuration](configuration.md)). +- A built ournotes-player: its checkout after `npm run build`, or its installed package, given with `--player` or + `[paths] player`. +- Node.js 20+, which runs the player's read-set script to list the files each chart reads; only those are stored. +- vgmstream and FFmpeg for the audio. +- For stories: `pip install 'nnnotes[fonts]'` and a font file per story language (below). + +## Build a few charts + +```sh +nnnotes site build -o out/site --player path/to/ournotes-player \ + --pair 100001:expert --pair 100001:hard +``` + +`--pair MUSIC_ID:DIFFICULTY` (repeatable) adds charts; `--all` adds every chart of the master data. Serve the +directory with any static file server and open `index.html`: the chart list, with `?music=&difficulty=` +playing one chart. + +The build is incremental. A chart, model or story whose manifest exists is skipped unless `--force`, and files no +manifest references any more are removed, so the same command run again after a game update adds what is new. +`--tmp DIR` (default `SITE.tmp`) keeps a cache of decoded audio, encoded images, shader dumps and read sets keyed by +everything they were made from; deleting it is safe outside a build. + +## Add models and stories + +```sh +nnnotes site build -o out/site --player path/to/ournotes-player \ + --all --all-live2d --all-stories \ + --font ja=fonts/NotoSansJP.otf --font en=fonts/NotoSans.ttf \ + --font zh-Hant=fonts/NotoSansTC.otf --font zh-Hans=fonts/NotoSansSC.otf --font ko=fonts/NotoSansKR.otf +``` + +- `--live2d MODEL` (repeatable) or `--all-live2d` adds Live2D models; the site gets the player's model page. +- `--story ADV_ID` (repeatable) or `--all-stories` adds story episodes, and the models they use; the site gets the + player's story page. +- Story text is laid out with font assets generated from your font files, holding exactly the characters each + episode shows. Give one per language of `--story-languages` (default all five) with `--font LANG=PATH` or + `fonts.` in `[paths]`. `--font emoji=PATH` gives a colour emoji font for the emoji of chat and talk texts. + `--fonts game` uses the game's own font assets instead. + +## Several regions + +```sh +nnnotes site build -o out/site --player path/to/ournotes-player --all --all-regions +``` + +`--site-region REGION` (repeatable) or `--all-regions` chooses the regions the site serves; each region's master +data is `[servers.] master`. Regions whose chart tables are equal share one manifest per chart; a region +whose tables differ gets its own. The list texts are written in all five languages, and the chart list page switches +with `?region=&lang=`. + +## Audio, compression and hosting + +- `--format aac|opus|vorbis|mp3|flac` (default `aac`) is the BGM and story audio format; `--no-audio` stores none. +- `--compress gzip|br|none` (default `gzip`) stores the compressible files encoded when that makes them smaller. A + gzip site works from any static host; a brotli site needs `Content-Encoding: br` on its `.br` files. +- `--workers N` sets the parallel music, model and story processes (`0` builds in this process) and + `--read-workers N` the read sets run at a time. + +## When something fails + +Charts, models and stories that fail are listed in the summary and in `SITE.failures.json`, +`SITE.model-failures.json` and `SITE.story-failures.json`, and the exit status is 1; the others are built. Fix the +cause and run the same command again: what was built is skipped. + +The full option list is in the [command reference](cli.md#nnnotes-site-build), the site layout and every rule in +[Command outputs](commands.md#site-build). diff --git a/docs/validation.md b/docs/validation.md new file mode 100644 index 0000000..63e7ea7 --- /dev/null +++ b/docs/validation.md @@ -0,0 +1,51 @@ +# Validation + +The test suite runs on synthetic inputs, so it needs no game data. The experiments on this page check nnnotes +against a whole game version, which you supply yourself: each one names what it checks, how to run it with your own +configuration, and the result for the version named. Only commands and checks are listed. The outputs stay with +whoever runs them. + +The results below are for the Traditional Chinese release, client 1.0.1, catalog language `zh-Hant`, with its master +data. The [Japanese release](jp.md#validation-and-current-limits) has its own section. + +## Setup + +```sh +nnnotes config check # every setting the commands below need is valid +nnnotes master download --latest -o work/master-bin --json > work/download.json +nnnotes master decode work/master-bin -o work/master --json > work/decode.json +nnnotes config set paths.master work/master +``` + +The lists of subjects come from the master data and the catalog: + +```sh +python -c "from nnnotes.source.master import table; print(*sorted(r['_id'] for r in table('work/master', 'MasterAdv')), sep='\n')" > work/episodes.txt +nnnotes catalog keys --prefix Character/Live2D/ --limit 0 > work/live2d-keys.txt +``` + +## Experiments + +| Experiment | How to run it | Result (client 1.0.1, zh-Hant) | +|---|---|---| +| Master data | `master download` checks every file against the SHA-256 of `MasterManifest.json`; `master decode` reports each table it could not decode under `failed` | every file verified, every table decoded | +| ADV episodes | `export episode ID` for every id of `episodes.txt` | 946 / 946 exported | +| Stories | `export story ID -o out/stories/ID` for every id, all with the same models directory; then read every JSON file and resolve every path `story.json` and the files it indexes name | 946 / 946 exported; every JSON file parses, every named file, model and cue exists | +| Story reproducibility | export the stories that use frames, effects, post effects, stills, chats, talk windows or videos twice into two directories and compare them byte by byte | 235 such episodes; the two exports are equal except the models directory path in `story.json` | +| Live2D models | `export live2d KEY` for every model key of `live2d-keys.txt` (the keys ending in `/model/`) | 239 / 239 exported, 185 of them used by stories | +| CRI audio | `export audio CUE_SHEET` for every cue sheet under `Cri/Sound/` | 681 / 681 decoded | +| Lives | `export live MUSIC_ID --difficulty D` for every music and difficulty with a `MasterLiveMusicScore` row | 336 / 336 exported | +| Spots and rooms | `export spot ID` for every `MasterHomeSpot` row and `export room KEY` for every room background prefab; validate every `.glb` with the [Khronos glTF-Validator](https://github.com/KhronosGroup/glTF-Validator) | 39 / 39 spots, 20 / 20 rooms; every reference resolves; every glb has 0 validator errors | +| Site, one region | `site build --all --all-live2d --all-stories --story-languages en --no-audio` twice into two empty directories, then compare them | 336 charts, 239 models, 946 stories; no failures file; the two sites are byte-identical | +| Site, every region | `site build --all --all-regions --format aac` twice into two empty directories, then compare them | 340 charts of 85 songs from the regions' current master data, five languages; no failures file; the two sites are byte-identical | +| Regions | for each region and language, `catalog keys --limit 0` and the catalog's bundles; compare the master tables a chart build reads across regions | the regions serve the same catalog and bundles for a language with the same keys; the chart tables are equal across the three regions; the text tables have all five languages | + +## Checks of every run + +These hold for every command and are checked by the test suite on synthetic data: + +- A key, id or model that the catalog or the master data does not have stops the command with exit status 2 before + any bundle is fetched or file written. +- The documents of an asset export run (plan, tasks, results, artifacts, run manifest, layout, coverage, failures, + cost) and the music data file validate against their [JSON schemas](schemas.md). +- The same inputs give byte-identical outputs ([determinism](determinism.md)). diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..2d0e209 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,68 @@ +site_name: nnnotes +site_description: Offline data toolkit for BanG Dream! Our Notes game files +site_url: https://metasekailab.github.io/nnnotes/ +repo_url: https://github.com/MetaSekaiLab/nnnotes +repo_name: MetaSekaiLab/nnnotes +edit_uri: edit/main/docs/ +copyright: MIT License + +theme: + name: material + features: + - content.code.copy + - navigation.sections + - navigation.top + - search.highlight + - toc.follow + palette: + - media: "(prefers-color-scheme: light)" + scheme: default + toggle: + icon: material/brightness-7 + name: Dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + toggle: + icon: material/brightness-4 + name: Light mode + +nav: + - Home: index.md + - Tutorial: + - Quick start: quickstart.md + - How-to guides: + - Configure nnnotes: configuration.md + - Export stories and lives: export.md + - Build an ournotes-player site: site.md + - Export every asset: assets.md + - Japanese release: jp.md + - Music data: music-data.md + - Reference: + - Commands: cli.md + - Command outputs: commands.md + - JSON schemas: schemas.md + - Contracts: contracts.md + - Stages: stages.md + - Views: views.md + - Replay data: replay.md + - UI library: ui.md + - Explanation: + - Architecture: architecture.md + - Determinism: determinism.md + - Validation: validation.md + +markdown_extensions: + - admonition + - attr_list + - md_in_html + - tables + - toc: + permalink: true + - pymdownx.details + - pymdownx.highlight + - pymdownx.inlinehilite + - pymdownx.superfences + +validation: + links: + anchors: warn diff --git a/pyproject.toml b/pyproject.toml index 7304b63..701e391 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "nnnotes" description = "Offline data toolkit for BanG Dream! Our Notes game files: catalogs, bundles, master data, stories, Live2D, audio and live charts" -readme = "README.en.md" +readme = "README.md" license = "MIT" license-files = ["LICENSE"] requires-python = ">=3.11" @@ -28,6 +28,7 @@ dynamic = ["version"] [project.optional-dependencies] test = ["pytest>=8", "ruff>=0.15", "jsonschema>=4.18", "import-linter>=2.1"] fonts = ["fonttools>=4.50", "freetype-py>=2.5", "scipy>=1.13"] +docs = ["mkdocs>=1.6,<2", "mkdocs-material>=9.5"] # the deck model of music-data: one nnnotes version carries one ournotes-deck release (.github/workflows/deck.yml moves # the pin when ournotes-deck publishes a newer one) deck = ["ournotes-sim==0.0.8"] @@ -37,6 +38,7 @@ nnnotes = "nnnotes.cli:main" [project.urls] Homepage = "https://github.com/MetaSekaiLab/nnnotes" +Documentation = "https://metasekailab.github.io/nnnotes/" Source = "https://github.com/MetaSekaiLab/nnnotes" Issues = "https://github.com/MetaSekaiLab/nnnotes/issues" @@ -47,7 +49,7 @@ path = "src/nnnotes/__init__.py" packages = ["src/nnnotes"] [tool.hatch.build.targets.sdist] -include = ["/src", "/tests", "/docs", "/scripts", "/CHANGELOG.md", "/README.md", "/README.en.md", "/LICENSE"] +include = ["/src", "/tests", "/docs", "/scripts", "/CHANGELOG.md", "/README.md", "/README.zh-CN.md", "/mkdocs.yml", "/LICENSE"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/src/nnnotes/cli/reference.py b/src/nnnotes/cli/reference.py index 8410465..5c63563 100644 --- a/src/nnnotes/cli/reference.py +++ b/src/nnnotes/cli/reference.py @@ -29,7 +29,7 @@ def _choices(sub: argparse._SubParsersAction) -> dict[str, str]: def argument(a: argparse.Action) -> dict: - """An argument: its flags (or name), value name, help and the facts the help does not say.""" + """An argument: its flags (or name), value name, help and its choices, default and repetition.""" d = {"flags": list(a.option_strings) or [a.metavar or a.dest], "help": _help(a)} takes = a.nargs != 0 if a.option_strings and takes: @@ -92,10 +92,11 @@ def _usage(d: dict) -> str: def _facts(d: dict) -> str: + """The choices, default and repetition of an argument, those its help does not already state.""" facts = [] if "choices" in d: facts.append("one of " + ", ".join(f"`{c}`" for c in d["choices"])) - if "default" in d: + if "default" in d and "default" not in d["help"]: facts.append(f"default `{d['default']}`") if d.get("repeatable") and "repeatable" not in d["help"]: facts.append("repeatable") diff --git a/tests/test_repo.py b/tests/test_repo.py index dd81a9b..1d3065b 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -1,4 +1,4 @@ -"""Repository metadata: the package page's readme and the commit message check.""" +"""Repository metadata: the readmes, the documentation site and the commit message check.""" import re import tomllib @@ -6,22 +6,55 @@ import pytest +from nnnotes import schemas + ROOT = Path(__file__).resolve().parents[1] REPO_BLOB = "https://github.com/MetaSekaiLab/nnnotes/blob/main/" +DOCS_SITE = "https://metasekailab.github.io/nnnotes/" -def test_package_readme_is_english_with_absolute_links(): - meta = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))["project"] - assert meta["readme"] == "README.en.md" - text = (ROOT / meta["readme"]).read_text(encoding="utf-8") - links = re.findall(r"\]\(([^)\s]+)\)", text) - assert links - relative = [u for u in links if not re.match(r"https?://", u)] - assert relative == [] # a package index has no repository to resolve them in - for u in links: +def links(text: str) -> list[str]: + return re.findall(r"\]\(([^)\s]+)\)", text) + + +@pytest.mark.parametrize("name", ["README.md", "README.zh-CN.md"]) +def test_readmes_have_absolute_links(name): + text = (ROOT / name).read_text(encoding="utf-8") + found = links(text) + assert found + # a package index has no repository to resolve relative links in + assert [u for u in found if not re.match(r"https?://", u)] == [] + for u in found: if u.startswith(REPO_BLOB): assert (ROOT / u[len(REPO_BLOB) :]).is_file(), u - assert (ROOT / "README.md").is_file() # the repository's own page stays the Chinese one + if u.startswith(DOCS_SITE): + page = u[len(DOCS_SITE) :].split("#")[0].strip("/") + assert (ROOT / "docs" / f"{page or 'index'}.md").is_file(), u + + +def test_the_package_page_is_the_english_readme(): + meta = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))["project"] + assert meta["readme"] == "README.md" + assert meta["urls"]["Documentation"] == DOCS_SITE + + +def test_the_translation_follows_the_readme(): + def shape(name): + text = (ROOT / name).read_text(encoding="utf-8") + return [len(h) for h in re.findall(r"^(#+) ", text, re.M)], [u for u in links(text) if DOCS_SITE in u] + + assert shape("README.zh-CN.md") == shape("README.md") + + +def test_every_docs_page_is_in_the_site_navigation(): + nav = re.findall(r"^\s+- [^:]+: ([\w-]+\.md)$", (ROOT / "mkdocs.yml").read_text(encoding="utf-8"), re.M) + assert sorted(nav) == sorted(p.name for p in (ROOT / "docs").glob("*.md")) + assert len(set(nav)) == len(nav) + + +def test_the_schema_page_lists_every_schema(): + text = (ROOT / "docs" / "schemas.md").read_text(encoding="utf-8") + assert re.findall(r"^\| `([\w-]+)` \|", text, re.M) == schemas.names() def workflow_triggers(text: str) -> list[str]: @@ -53,7 +86,13 @@ def test_the_deck_pin_pull_request_gets_its_checks(): assert f'gh workflow run {name} --repo "$GITHUB_REPOSITORY" --ref deck/update' in deck -@pytest.mark.parametrize("name", ["README.md", "README.en.md"]) +@pytest.mark.parametrize("name", ["README.md", "README.zh-CN.md"]) def test_determinism_is_stated_with_its_condition(name): text = (ROOT / name).read_text(encoding="utf-8") assert "Pillow" in text and "1e999" in text + + +def test_the_changelog_starts_with_the_generated_header(): + header = tomllib.loads((ROOT / "cliff.toml").read_text(encoding="utf-8"))["changelog"]["header"] + assert "Semantic Versioning" in header + assert (ROOT / "CHANGELOG.md").read_text(encoding="utf-8").startswith(header.rstrip("\n") + "\n\n## ")