From 5f57d6c2111bdfdd868d67874004329020e8af49 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Tue, 13 Oct 2026 21:15:12 +0800 Subject: [PATCH 1/3] build: ship nnnotes as a pure Python package with the deck model as the extra deck nnnotes is now one pure Python wheel (py3-none-any) built with hatchling, its version read from src/nnnotes/__init__.py. The deck model of music-data is the package ournotes-sim, which ournotes-deck publishes with each release; the extra `deck` pins one release (ournotes-sim==0.0.7), so one nnnotes version still carries one deck model. `pip install 'nnnotes[deck]'` installs it; without it music-data names the extra and --no-deck. - musicdata.Deck imports ournotes_sim; provenance.deck keeps the same fields (name, version, source, commit, sourceSha256, format). - The Deck model workflow reads the pin from pyproject.toml and moves it once the latest ournotes-deck release has published its WASM packages and its ournotes-sim wheels. - CI installs `.[test,deck]`; the release builds the sdist and the wheel, checks them with twine and tests the wheel with the extra in a fresh environment. - The tests check that the installed model is the release the extra pins. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/smoke_wheel.py | 17 +- .github/workflows/ci.yml | 24 +-- .github/workflows/deck.yml | 39 +++-- .github/workflows/release.yml | 68 ++------ .gitignore | 3 - README.en.md | 5 +- README.md | 4 +- docs/commands.md | 4 +- docs/music-data.md | 18 +-- pyproject.toml | 24 +-- rust/Cargo.lock | 285 --------------------------------- rust/Cargo.toml | 24 --- rust/build.rs | 33 ---- rust/rustfmt.toml | 2 - rust/src/lib.rs | 107 ------------- src/nnnotes/musicdata.py | 10 +- tests/test_musicdata.py | 25 +-- 17 files changed, 86 insertions(+), 606 deletions(-) delete mode 100644 rust/Cargo.lock delete mode 100644 rust/Cargo.toml delete mode 100644 rust/build.rs delete mode 100644 rust/rustfmt.toml delete mode 100644 rust/src/lib.rs diff --git a/.github/scripts/smoke_wheel.py b/.github/scripts/smoke_wheel.py index 2befd5a..9033a07 100644 --- a/.github/scripts/smoke_wheel.py +++ b/.github/scripts/smoke_wheel.py @@ -1,4 +1,4 @@ -"""Install one wheel in a fresh environment and exercise its CLI and extension.""" +"""Install the wheel with the deck model in a fresh environment and exercise its CLI and the deck model.""" from pathlib import Path import os import subprocess @@ -13,20 +13,21 @@ import sys import nnnotes -from nnnotes import _deck, deckdata +import ournotes_sim +from nnnotes import deckdata, musicdata assert Path(nnnotes.__file__).resolve().is_relative_to(Path(sys.prefix).resolve()) -assert Path(_deck.__file__).resolve().is_relative_to(Path(sys.prefix).resolve()) -info = _deck.info() +deck = musicdata.Deck(workers=1, aptitude=False) +info = deck.info() data = { - "format": info["dataFormat"], + "format": ournotes_sim.info()["dataFormat"], "master": {name: {"columns": [], "rows": []} for name, _ in deckdata.TABLES}, "charts": [], } -stats = json.loads(_deck.chart_stats(json.dumps(data), workers=1, aptitude=False)) +stats = json.loads(ournotes_sim.chart_stats(json.dumps(data), workers=1, aptitude=False)) assert stats["format"] == info["format"] assert stats["charts"] == [] -print(f"nnnotes {nnnotes.__version__}: {info['name']} {info['version']}") +print(f"nnnotes {nnnotes.__version__}: {info['name']} {info['version']} ({info['commit']})") """ @@ -38,7 +39,7 @@ def main(directory: str) -> None: venv.EnvBuilder(with_pip=True).create(environment) scripts = environment / ("Scripts" if os.name == "nt" else "bin") python = scripts / ("python.exe" if os.name == "nt" else "python") - subprocess.run([python, "-I", "-m", "pip", "install", str(wheel)], cwd=root, check=True) + subprocess.run([python, "-I", "-m", "pip", "install", f"{wheel}[deck]"], cwd=root, check=True) subprocess.run([python, "-I", "-c", SMOKE], cwd=root, check=True) subprocess.run([scripts / ("nnnotes.exe" if os.name == "nt" else "nnnotes"), "--help"], cwd=root, check=True) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 280ff7a..34b02b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,13 +29,10 @@ jobs: python-version: ${{ matrix.python-version }} cache: pip cache-dependency-path: pyproject.toml - - uses: Swatinem/rust-cache@v2 - with: - workspaces: rust - - name: Install (builds the extension module nnnotes._deck) + - name: Install (with the deck model) run: | python -m pip install --upgrade pip - python -m pip install -e ".[test]" + python -m pip install -e ".[test,deck]" - name: Lint run: python -m pyflakes src tests - name: Command line @@ -56,9 +53,6 @@ jobs: cache-dependency-path: | pyproject.toml .github/goldens-constraints.txt - - uses: Swatinem/rust-cache@v2 - with: - workspaces: rust - name: Install (pinned) run: | python -m pip install --upgrade pip @@ -67,17 +61,3 @@ jobs: env: GOLDENS_STRICT: "1" run: python -m pytest tests/test_goldens.py tests/test_objexport.py - - rust: - # the extension module's own code (rust/); ournotes-deck is tested in its repository - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - uses: Swatinem/rust-cache@v2 - with: - workspaces: rust - - name: Format and lints - working-directory: rust - run: | - cargo fmt --check - cargo clippy --locked --all-targets -- -D warnings diff --git a/.github/workflows/deck.yml b/.github/workflows/deck.yml index c0a7a97..284e10e 100644 --- a/.github/workflows/deck.yml +++ b/.github/workflows/deck.yml @@ -1,11 +1,11 @@ name: Deck model -# nnnotes builds the deck model, the crate ournotes-sim of the ournotes-deck repository, at the commit rust/Cargo.toml -# pins, so that one nnnotes version always carries one deck model. The pin follows ournotes-deck's releases: when the -# latest release has published its replay and recommendation WASM packages (the engines of music-data --replay-engine -# and --recommend-engine) and its commit is past the pinned one, this opens a pull request that moves the pin and -# dispatches CI and the commit message check on its branch (a pull request opened with the workflow token starts no -# workflow by itself); their checks land on the pull request. +# The extra `deck` pins the deck model, the package ournotes-sim that ournotes-deck publishes with each release, so +# that one nnnotes version always carries one deck model. The pin follows ournotes-deck's releases: when the latest +# release has published its replay and recommendation WASM packages (the engines of music-data --replay-engine and +# --recommend-engine) and its ournotes-sim wheels, and it is past the pinned release, this opens a pull request that +# moves the pin and dispatches CI and the commit message check on its branch (a pull request opened with the workflow +# token starts no workflow by itself); their checks land on the pull request. on: schedule: - cron: "17 3 * * *" @@ -21,35 +21,32 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - - name: Compare the pinned commit with ournotes-deck's latest release + - name: Compare the pinned release with ournotes-deck's latest release id: deck env: GH_TOKEN: ${{ github.token }} run: | - pinned=$(sed -nE 's/^ournotes-sim = .*rev = "([0-9a-f]{40})".*/\1/p' rust/Cargo.toml) - [ -n "$pinned" ] || { echo "rust/Cargo.toml pins no ournotes-sim commit" >&2; exit 1; } + pinned=$(sed -nE 's/^deck = \["ournotes-sim==([0-9]+\.[0-9]+\.[0-9]+)"\]$/\1/p' pyproject.toml) + [ -n "$pinned" ] || { echo "pyproject.toml pins no ournotes-sim release" >&2; exit 1; } release=$(gh api repos/empty-sekai/ournotes-deck/releases/latest) tag=$(jq -r .tag_name <<< "$release") - head=$(gh api "repos/empty-sekai/ournotes-deck/commits/$tag" --jq .sha) + version=${tag#v} published=true for module in replay recommend; do jq -e --arg name "ournotes-$module-wasm-$tag.tar.gz" 'any(.assets[]; .name == $name)' <<< "$release" \ > /dev/null || published=false done - # ahead: the release contains the pinned commit and more; behind or diverged: leave the pin to a person - # (with no common history the comparison itself fails) - status=$(gh api "repos/empty-sekai/ournotes-deck/compare/$pinned...$head" --jq .status) - echo "pinned $pinned, $tag $head: $status, WASM packages published: $published" + curl -fsS -o /dev/null "https://pypi.org/pypi/ournotes-sim/$version/json" || published=false + # ahead: the release contains the pinned release and more; behind or diverged: leave the pin to a person + status=$(gh api "repos/empty-sekai/ournotes-deck/compare/v$pinned...$tag" --jq .status) + echo "pinned $pinned, latest $version: $status, WASM packages and wheels published: $published" echo "pinned=$pinned" >> "$GITHUB_OUTPUT" - echo "head=$head" >> "$GITHUB_OUTPUT" + echo "version=$version" >> "$GITHUB_OUTPUT" echo "tag=$tag" >> "$GITHUB_OUTPUT" echo "move=$([ "$status" = ahead ] && [ "$published" = true ] && echo true || echo false)" >> "$GITHUB_OUTPUT" - name: Move the pin if: steps.deck.outputs.move == 'true' - working-directory: rust - run: | - sed -i -E 's/(rev = ")${{ steps.deck.outputs.pinned }}(")/\1${{ steps.deck.outputs.head }}\2/' Cargo.toml - cargo update -p ournotes-sim + run: sed -i -E 's/^(deck = \["ournotes-sim==)${{ steps.deck.outputs.pinned }}("\])$/\1${{ steps.deck.outputs.version }}\2/' pyproject.toml - uses: peter-evans/create-pull-request@v8 id: pr if: steps.deck.outputs.move == 'true' @@ -58,9 +55,9 @@ jobs: commit-message: "build(deck): ournotes-deck ${{ steps.deck.outputs.tag }}" title: "build(deck): ournotes-deck ${{ steps.deck.outputs.tag }}" body: | - Moves the deck model to ournotes-deck ${{ steps.deck.outputs.tag }} (${{ steps.deck.outputs.head }}), + Moves the deck model to ournotes-deck ${{ steps.deck.outputs.tag }}, https://github.com/empty-sekai/ournotes-deck/releases/tag/${{ steps.deck.outputs.tag }} - Changes: https://github.com/empty-sekai/ournotes-deck/compare/${{ steps.deck.outputs.pinned }}...${{ steps.deck.outputs.head }} + Changes: https://github.com/empty-sekai/ournotes-deck/compare/v${{ steps.deck.outputs.pinned }}...${{ steps.deck.outputs.tag }} Music data files made after this change name the new commit in `provenance.deck.commit`; their deck statistics may differ. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7ba8ba0..1e9cc3b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,13 +13,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 - - name: Check the tag against the package and extension versions + - name: Check the tag against the package version if: startsWith(github.ref, 'refs/tags/v') run: | version=$(python3 -c 'import re, pathlib; print(re.search(r"__version__ = \"([^\"]+)\"", pathlib.Path("src/nnnotes/__init__.py").read_text()).group(1))') - cargo=$(python3 -c 'import re, pathlib; print(re.search(r"^version = \"([^\"]+)\"", pathlib.Path("rust/Cargo.toml").read_text(), re.M).group(1))') - if [ "v$version" != "$GITHUB_REF_NAME" ] || [ "$cargo" != "$version" ]; then - echo "tag $GITHUB_REF_NAME, package $version and rust/Cargo.toml $cargo differ" >&2 + if [ "v$version" != "$GITHUB_REF_NAME" ]; then + echo "tag $GITHUB_REF_NAME and package $version differ" >&2 exit 1 fi if ! grep -qF "## [$version]" CHANGELOG.md; then @@ -31,68 +30,23 @@ jobs: needs: check uses: ./.github/workflows/ci.yml - # one abi3 wheel (CPython 3.11 and later) per platform with the extension module nnnotes._deck, and the sdist - wheels: - needs: check - runs-on: ${{ matrix.os }} - strategy: - fail-fast: true - matrix: - include: - - {os: ubuntu-latest, target: x86_64, manylinux: "2_28"} - - {os: ubuntu-24.04-arm, target: aarch64, manylinux: "2_28"} - - {os: windows-latest, target: x64} - - {os: macos-latest, target: aarch64} - - {os: macos-15-intel, target: x86_64} - steps: - - uses: actions/checkout@v7 - - uses: actions/setup-python@v7 - with: - python-version: "3.13" - - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.target }} - manylinux: ${{ matrix.manylinux || 'auto' }} - args: --release --locked --out dist - - name: Test the installed wheel - run: python .github/scripts/smoke_wheel.py dist - - uses: actions/upload-artifact@v7 - with: - name: wheel-${{ matrix.os }}-${{ matrix.target }} - path: dist/ - if-no-files-found: error - - sdist: - needs: check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - uses: PyO3/maturin-action@v1 - with: - command: sdist - args: --out dist - - uses: actions/upload-artifact@v7 - with: - name: sdist - path: dist/ - if-no-files-found: error - + # the sdist and one pure Python wheel; the wheel is tested with the deck model the extra pins build: - needs: [quality, wheels, sdist] + needs: [check, quality] runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@v8 - with: - path: dist/ - merge-multiple: true + - uses: actions/checkout@v7 - uses: actions/setup-python@v7 with: python-version: "3.13" - - name: Check the distributions + - name: Build and check the distributions run: | - python -m pip install --upgrade pip twine + python -m pip install --upgrade pip build twine + python -m build --outdir dist python -m twine check --strict dist/* ls dist + - name: Test the installed wheel + run: python .github/scripts/smoke_wheel.py dist - uses: actions/upload-artifact@v7 with: name: dist diff --git a/.gitignore b/.gitignore index 0c92583..a179109 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,3 @@ nnnotes.toml .env out/ cache/ -rust/target/ -src/nnnotes/_deck*.pyd -src/nnnotes/_deck*.so diff --git a/README.en.md b/README.en.md index b6e3155..59ed006 100644 --- a/README.en.md +++ b/README.en.md @@ -28,7 +28,8 @@ nnnotes config check # each setting's origin and whether it is valid - Scripts and AI agents get no questions: `nnnotes config init --user --no-input --set SECTION.KEY=VALUE ...` writes the file at once, `nnnotes config set SECTION.KEY VALUE` changes one value (VALUE `-` reads it from standard input, so a key need not appear on the command line), and `nnnotes config check --json` reports the states for machines. -- `--fonts game` also needs `pip install 'nnnotes[fonts]'`; the APK, the master data and the external tools are listed +- `--fonts game` also needs `pip install 'nnnotes[fonts]'`, and the deck statistics of `music-data` need + `pip install 'nnnotes[deck]'`; the APK, the master data and the external tools are listed under Requirements below. ## Features @@ -54,7 +55,7 @@ nnnotes config check # each setting's origin and whether it is valid | `ui` (experimental) | APK set and its embedded catalog | an offline serialized prefab library with dependency roots, AnimatorControllers, stable references and PNG/font resources for the optional ournotes-player UI preview ([format and limits](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/ui.md)) | | `live` | music ID + difficulty | a full chart directory: chart and runtime notes, 3D scene, note and effect assets, BGM and sounds, sound routing | | `web` | `--pair music:difficulty` (repeatable) or `--all`; `--live2d model` (repeatable) or `--all-live2d`; `--story episode` (repeatable) or `--all-stories`; `--region region` (repeatable) or `--all-regions` | an ournotes-player static site: shared player + per-chart / per-model / per-episode manifests + content-addressed assets; the Live2D models of the stories are built first as models, listed in `models.json`, and the story manifests reference them; one site can serve several regions, with listing texts in five languages; a story's interface texts are grouped by language, with TextMeshPro font assets generated from open fonts (the game's fonts with `--fonts game`); compressible assets (JSON, shaders, moc3, ...) are stored gzip encoded by default (`--compress br` / `none`) | -| `music-data` | master data files (`--master-files` directory or `--apk-master`), or decoded master data with its manifest (`--decoded-master`) | one JSON file with every song and chart: titles and credits in five languages, bands, vocal characters, category, tags, release time, score ranks, BGM length; the Gekisou catalog (member cards, snaps and their Gekisou skills and support skills); per difficulty the level, note counts, BPM, chart times, skill events and fever ranges; and the chart statistics the deck model ournotes-deck (built into nnnotes) measures on its whole-live simulation (the no-skill score, the weight of every score-up skill kind at every position), plus single-shape Gekisou aptitude (increments and standard errors, not an optimal deck; omit with `--no-gekisou-aptitude`). `--full` adds the deck model's input: every chart's runtime notes and the master data tables about cards, skills, bonuses, scores and events ([format](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/music-data.md)) | +| `music-data` | master data files (`--master-files` directory or `--apk-master`), or decoded master data with its manifest (`--decoded-master`) | one JSON file with every song and chart: titles and credits in five languages, bands, vocal characters, category, tags, release time, score ranks, BGM length; the Gekisou catalog (member cards, snaps and their Gekisou skills and support skills); per difficulty the level, note counts, BPM, chart times, skill events and fever ranges; and the chart statistics the deck model ournotes-deck (installed with `pip install 'nnnotes[deck]'`) measures on its whole-live simulation (the no-skill score, the weight of every score-up skill kind at every position), plus single-shape Gekisou aptitude (increments and standard errors, not an optimal deck; omit with `--no-gekisou-aptitude`). `--full` adds the deck model's input: every chart's runtime notes and the master data tables about cards, skills, bonuses, scores and events ([format](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/music-data.md)) | Export conventions: diff --git a/README.md b/README.md index 11a675b..c27afea 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ nnnotes config check # 每项设置的来源和格式是否有效， - 设置值（密钥、区服的 CDN 与 API 地址、本地数据与工具的路径）都来自使用者自己的游戏客户端，见[配置](#配置)。 - 在脚本或 AI 代理中不会有交互提问：`nnnotes config init --user --no-input --set 段.键=值 ...` 一次写好，之后用 `nnnotes config set 段.键 值` 修改单项（值写 `-` 时从标准输入读取，密钥不必出现在命令行里），`nnnotes config check --json` 给出机器可读的状态。 -- `--fonts game` 另需 `pip install 'nnnotes[fonts]'`；APK、masterdata 与外部工具见[需要准备](#需要准备)。 +- `--fonts game` 另需 `pip install 'nnnotes[fonts]'`，`music-data` 的组卡统计另需 `pip install 'nnnotes[deck]'`；APK、masterdata 与外部工具见[需要准备](#需要准备)。 ## 功能 @@ -41,7 +41,7 @@ nnnotes config check # 每项设置的来源和格式是否有效， | `ui`（实验性） | APK 集合及内嵌 catalog | 离线序列化预制体库，包含依赖根对象、AnimatorController、稳定引用与 PNG／字体资源，供 ournotes-player 可选 UI 预览使用（[格式与边界](https://github.com/MetaSekaiLab/nnnotes/blob/main/docs/ui.md)） | | `live` | 曲目 ID + 难度 | 完整谱面目录：谱面与运行时音符、3D 场景、音符与特效资源、BGM 与音效、声音路由 | | `web` | `--pair 曲目:难度`（可重复）或 `--all`；`--live2d 模型`（可重复）或 `--all-live2d`；`--story 剧情 ID`（可重复）或 `--all-stories`；`--region 区服`（可重复）或 `--all-regions` | ournotes-player 静态站点：共享播放器 + 每谱 / 每模型 / 每集剧情清单 + 内容寻址资源；剧情用到的 Live2D 模型先按模型构建并列入 `models.json`，剧情清单引用它们；一个站点可服务多个区服，列表文本含五种语言；剧情的界面文字按语言分组，字形由开源字体生成 TextMeshPro 字体资源（`--fonts game` 时用游戏字体）；可压缩的资源（JSON、着色器、moc3 等）默认以 gzip 存储，`--compress br` / `none` 可改 | -| `music-data` | masterdata 文件（`--master-files` 目录或 `--apk-master`），或带清单的解码 masterdata（`--decoded-master`） | 全部歌曲与谱面的单个 JSON：五语标题与作词作曲编曲、乐队、演唱角色、分类、标签、上线时间、评级线、BGM 时长；激走目录（成员卡、小卡及其激走技能与激走支援技能）；每个难度的等级、音符数、BPM、谱面时间、技能事件与 fever 区间；以及组卡模型 ournotes-deck（内置于 nnnotes）在整场模拟上实测的谱面统计（无技能得分、每种加分技能在每个演出位的权重），以及单个激走技能形状的谱面适性（增量与标准误，不选最佳编成；`--no-gekisou-aptitude` 可跳过）。`--full` 另附组卡模型的输入：每张谱面的运行时音符与卡牌、技能、加成、分数、活动相关的 masterdata 表（[格式](docs/music-data.md)） | +| `music-data` | masterdata 文件（`--master-files` 目录或 `--apk-master`），或带清单的解码 masterdata（`--decoded-master`） | 全部歌曲与谱面的单个 JSON：五语标题与作词作曲编曲、乐队、演唱角色、分类、标签、上线时间、评级线、BGM 时长；激走目录（成员卡、小卡及其激走技能与激走支援技能）；每个难度的等级、音符数、BPM、谱面时间、技能事件与 fever 区间；以及组卡模型 ournotes-deck（`pip install 'nnnotes[deck]'` 安装）在整场模拟上实测的谱面统计（无技能得分、每种加分技能在每个演出位的权重），以及单个激走技能形状的谱面适性（增量与标准误，不选最佳编成；`--no-gekisou-aptitude` 可跳过）。`--full` 另附组卡模型的输入：每张谱面的运行时音符与卡牌、技能、加成、分数、活动相关的 masterdata 表（[格式](docs/music-data.md)） | 导出约定： diff --git a/docs/commands.md b/docs/commands.md index b3db6e8..8ebfd0a 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -693,8 +693,8 @@ characters, bands and Gekisou skills and support skills: missions, highest level difficulty the chart facts (level and display level, full combo count, note counts, BPM, note times, the live's music length, skill event times, fever ranges) and the chart's deck statistics: the no-skill score and the weight of every score-up skill kind at every performance position, with Gekisou on (a Gekisou live at rank 1, with what every other -rank and the Perfect play need) and off (a solo live), measured by the deck model ournotes-deck (built into nnnotes as -`nnnotes._deck`) on its whole-live simulation and checked against the chart facts and the master data. `--full` also writes the deck model's input: every +rank and the Perfect play need) and off (a solo live), measured by the deck model ournotes-deck (the package ournotes-sim, +installed with `pip install 'nnnotes[deck]'`) on its whole-live simulation and checked against the chart facts and the master data. `--full` also writes the deck model's input: every `MasterLiveMusicScore` row's chart as the client builds it at runtime (notes, skill events, fever ranges) and the master data tables about cards, skills, bonuses, scores and events. Single-skill Gekisou aptitude is included by default as independent nominal expectations; `--no-gekisou-aptitude` omits it. diff --git a/docs/music-data.md b/docs/music-data.md index dd6ad75..da305dc 100644 --- a/docs/music-data.md +++ b/docs/music-data.md @@ -5,8 +5,8 @@ listing shows (titles and credits in every language, bands, vocal characters, ca ranks, jacket, BGM length), the Gekisou catalog (member cards, snaps and their Gekisou skills), per difficulty the chart facts (level, note counts, BPM, chart times, skill events, fevers), and per chart its **deck statistics**: what the chart contributes to the live score whatever the deck, in a solo live (Gekisou off) and in a Gekisou live at every -rank, measured by the deck model [ournotes-deck](https://github.com/empty-sekai/ournotes-deck), which nnnotes carries -as its extension module `nnnotes._deck`. The format is `nnnotes.music-data/2`; its JSON Schema is +rank, measured by the deck model [ournotes-deck](https://github.com/empty-sekai/ournotes-deck), the package +`ournotes-sim` that the extra `deck` installs (`pip install 'nnnotes[deck]'`). The format is `nnnotes.music-data/2`; its JSON Schema is [schema/music-data.schema.json](schema/music-data.schema.json). ``` @@ -377,14 +377,12 @@ The deck model reads the same content under the format name `nnnotes.deck-data/1 ## Building -The deck model is the Rust crate ournotes-sim of the ournotes-deck repository, pinned to the commit of an ournotes-deck -release in `rust/Cargo.toml` -(and `rust/Cargo.lock`) and built into the extension module `nnnotes._deck` with [maturin](https://www.maturin.rs/) -(PyO3, the stable ABI of Python 3.11 and later: one wheel per platform). The release workflow builds the wheels; -`pip install .` or `pip install -e .` in a checkout builds the module with the Rust toolchain. The same nnnotes -version always carries the same deck model: the commit moves only through a pull request (`.github/workflows/deck.yml` -opens one when ournotes-deck publishes a newer release with its WASM packages), and `provenance.deck.commit` names it in -every file. `provenance.deck.sourceSha256` is the SHA-256 of the deck model's sources (`ournotes_sim::SOURCE_SHA256`): +The deck model is the Python package [ournotes-sim](https://pypi.org/project/ournotes-sim/) that each ournotes-deck +release publishes: the Rust crate ournotes-sim of that release as an extension module, one wheel per platform. The +extra `deck` pins one release (`pip install 'nnnotes[deck]'`, or `pip install -e ".[deck]"` in a checkout), so the +same nnnotes version always carries the same deck model: the pin moves only through a pull request +(`.github/workflows/deck.yml` opens one when ournotes-deck publishes a newer release with its WASM packages and its +wheels), and `provenance.deck.version` and `provenance.deck.commit` name the release in every file. `provenance.deck.sourceSha256` is the SHA-256 of the deck model's sources (`ournotes_sim::SOURCE_SHA256`): releases with the same value measure the same statistics. The replay and recommendation engines of `--replay-engine` and `--recommend-engine` are that release's WASM packages. diff --git a/pyproject.toml b/pyproject.toml index 3889b27..c2620db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [build-system] -requires = ["maturin>=1.9,<2"] -build-backend = "maturin" +requires = ["hatchling>=1.26"] +build-backend = "hatchling.build" [project] name = "nnnotes" @@ -28,6 +28,9 @@ dynamic = ["version"] [project.optional-dependencies] test = ["pytest>=8", "pyflakes>=3.2", "jsonschema>=4.18"] fonts = ["fonttools>=4.50", "freetype-py>=2.5", "scipy>=1.13"] +# the deck model of music-data: one nnnotes version carries one ournotes-deck release (.github/workflows/deck.yml moves +# the pin when ournotes-deck publishes a newer one) +deck = ["ournotes-sim==0.0.7"] [project.scripts] nnnotes = "nnnotes.cli:main" @@ -37,15 +40,14 @@ Homepage = "https://github.com/MetaSekaiLab/nnnotes" Source = "https://github.com/MetaSekaiLab/nnnotes" Issues = "https://github.com/MetaSekaiLab/nnnotes/issues" -[tool.maturin] -# the pure Python package src/nnnotes with the extension module nnnotes._deck (rust/: ournotes-sim of ournotes-deck, -# pinned by commit); the version is rust/Cargo.toml's, kept equal to nnnotes.__version__ (tests/test_musicdata.py) -python-source = "src" -module-name = "nnnotes._deck" -manifest-path = "rust/Cargo.toml" -profile = "release" -locked = true -include = [{ path = "CHANGELOG.md", format = "sdist" }] +[tool.hatch.version] +path = "src/nnnotes/__init__.py" + +[tool.hatch.build.targets.wheel] +packages = ["src/nnnotes"] + +[tool.hatch.build.targets.sdist] +include = ["/src", "/tests", "/docs", "/scripts", "/CHANGELOG.md", "/README.md", "/README.en.md", "/LICENSE"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/rust/Cargo.lock b/rust/Cargo.lock deleted file mode 100644 index 0373076..0000000 --- a/rust/Cargo.lock +++ /dev/null @@ -1,285 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "block-buffer" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" -dependencies = [ - "hybrid-array", -] - -[[package]] -name = "cfg-if" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" - -[[package]] -name = "cpufeatures" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" -dependencies = [ - "libc", -] - -[[package]] -name = "crypto-common" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" -dependencies = [ - "hybrid-array", -] - -[[package]] -name = "digest" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" -dependencies = [ - "block-buffer", - "crypto-common", -] - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "hybrid-array" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" -dependencies = [ - "typenum", -] - -[[package]] -name = "itoa" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" - -[[package]] -name = "libc" -version = "0.2.189" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" - -[[package]] -name = "memchr" -version = "2.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" - -[[package]] -name = "nnnotes-deck" -version = "0.1.4" -dependencies = [ - "ournotes-sim", - "pyo3", - "serde_json", -] - -[[package]] -name = "once_cell" -version = "1.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - -[[package]] -name = "ournotes-sim" -version = "0.0.6" -source = "git+https://github.com/empty-sekai/ournotes-deck?rev=46cdedf3015f0bf894f3a6fd6994886d55bfea6b#46cdedf3015f0bf894f3a6fd6994886d55bfea6b" -dependencies = [ - "serde", - "serde_json", - "sha2", -] - -[[package]] -name = "portable-atomic" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" - -[[package]] -name = "proc-macro2" -version = "1.0.107" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "pyo3" -version = "0.29.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4688ddedf473e32662b9b067670129a8afb8c18e351482c70d62ba4a88171e8b" -dependencies = [ - "libc", - "once_cell", - "portable-atomic", - "pyo3-build-config", - "pyo3-ffi", - "pyo3-macros", -] - -[[package]] -name = "pyo3-build-config" -version = "0.29.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f41027e41b4bd03f6e60f9f417fe24a6341a6bb744edd62b6f709f2a52ea30e9" -dependencies = [ - "target-lexicon", -] - -[[package]] -name = "pyo3-ffi" -version = "0.29.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e591a95526fead067432c3b3a33fc74770b87b1e04e73671090d9c2055a2b327" -dependencies = [ - "libc", - "pyo3-build-config", -] - -[[package]] -name = "pyo3-macros" -version = "0.29.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73225868fc1cd84eef2c3c230ddb91273bf1de46aeb8a4248da76d32a0924a1c" -dependencies = [ - "proc-macro2", - "pyo3-macros-backend", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "pyo3-macros-backend" -version = "0.29.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "571575aa3749fa6216757dd47d2a3e7ef360f329a40f0666a9fbd14889024952" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "quote" -version = "1.0.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "serde" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde_core" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.6", -] - -[[package]] -name = "serde_json" -version = "1.0.151" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" -dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", -] - -[[package]] -name = "sha2" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "syn" -version = "2.0.119" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "3.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "target-lexicon" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" - -[[package]] -name = "typenum" -version = "1.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" - -[[package]] -name = "unicode-ident" -version = "1.0.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" - -[[package]] -name = "zmij" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/rust/Cargo.toml b/rust/Cargo.toml deleted file mode 100644 index 053d12e..0000000 --- a/rust/Cargo.toml +++ /dev/null @@ -1,24 +0,0 @@ -[package] -name = "nnnotes-deck" -# the nnnotes version (src/nnnotes/__init__.py); the wheel takes its version from here -version = "0.1.4" -edition = "2024" -rust-version = "1.85" -description = "nnnotes._deck: the ournotes-sim chart statistics as a Python extension module" -license = "MIT" -publish = false - -[lib] -name = "_deck" -crate-type = ["cdylib"] - -[dependencies] -pyo3 = { version = "0.29", features = ["abi3-py311"] } -# the deck model, the crate ournotes-sim of the ournotes-deck repository, pinned to the commit of a release; -# .github/workflows/deck.yml opens a pull request when ournotes-deck publishes a newer release -ournotes-sim = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "46cdedf3015f0bf894f3a6fd6994886d55bfea6b" } -serde_json = { version = "1", features = ["raw_value", "arbitrary_precision"] } - -[profile.release] -lto = "thin" -codegen-units = 1 diff --git a/rust/build.rs b/rust/build.rs deleted file mode 100644 index a7ef6e8..0000000 --- a/rust/build.rs +++ /dev/null @@ -1,33 +0,0 @@ -//! Records the ournotes-sim package the module is built with (version, repository and git commit, from Cargo.lock) -//! as the environment variables DECK_VERSION, DECK_SOURCE and DECK_COMMIT of the crate. - -use std::path::Path; - -fn main() { - let dir = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR"); - let lock = Path::new(&dir).join("Cargo.lock"); - println!("cargo:rerun-if-changed={}", lock.display()); - let text = std::fs::read_to_string(&lock).expect("rust/Cargo.lock (the deck commit is read from it)"); - let (mut version, mut source) = (None, None); - for block in text.split("[[package]]") { - let field = |k: &str| { - block.lines().find_map(|l| { - let v = l.trim().strip_prefix(k)?.trim_start().strip_prefix('=')?.trim(); - Some(v.trim_matches('"').to_string()) - }) - }; - if field("name").as_deref() == Some("ournotes-sim") { - version = field("version"); - source = field("source"); - } - } - let version = version.expect("Cargo.lock has no ournotes-sim package"); - let source = source.expect("ournotes-sim has no source in Cargo.lock"); - // git+https://github.com/empty-sekai/ournotes-deck?rev=# - let (url, commit) = source.split_once('#').expect("ournotes-sim is not a git dependency"); - let url = url.strip_prefix("git+").unwrap_or(url); - let url = url.split_once('?').map_or(url, |(u, _)| u); - println!("cargo:rustc-env=DECK_VERSION={version}"); - println!("cargo:rustc-env=DECK_SOURCE={url}"); - println!("cargo:rustc-env=DECK_COMMIT={commit}"); -} diff --git a/rust/rustfmt.toml b/rust/rustfmt.toml deleted file mode 100644 index c775577..0000000 --- a/rust/rustfmt.toml +++ /dev/null @@ -1,2 +0,0 @@ -max_width = 120 -use_small_heuristics = "Max" diff --git a/rust/src/lib.rs b/rust/src/lib.rs deleted file mode 100644 index 793b441..0000000 --- a/rust/src/lib.rs +++ /dev/null @@ -1,107 +0,0 @@ -//! `nnnotes._deck`: the chart statistics of the deck model ournotes-sim (`ournotes-deck.chart-stats/3`) on a deck -//! data document held in memory, the charts measured in parallel. - -use std::sync::Mutex; -use std::sync::atomic::{AtomicUsize, Ordering}; - -use ournotes_sim::chartstats::{self, ChartStats, Options, REPLAY_SEEDS}; -use ournotes_sim::data::DeckData; -use ournotes_sim::error::Error; -use pyo3::exceptions::PyValueError; -use pyo3::prelude::*; -use pyo3::types::PyDict; - -fn value_error(e: impl std::fmt::Display) -> PyErr { - PyValueError::new_err(e.to_string()) -} - -/// The statistics of every chart, in chart order, on `workers` threads. -fn measure(data: &DeckData, options: &Options, workers: usize) -> Result, Error> { - let kinds = chartstats::kinds(&data.master); - let n = data.charts.len(); - let next = AtomicUsize::new(0); - let out: Mutex>>> = Mutex::new((0..n).map(|_| None).collect()); - std::thread::scope(|s| { - for _ in 0..workers.clamp(1, n.max(1)) { - s.spawn(|| { - loop { - let i = next.fetch_add(1, Ordering::Relaxed); - if i >= n { - break; - } - let c = &data.charts[i]; - // as chartstats::document names a chart's domain errors - let r = chartstats::chart_stats_with(&data.master, c, &kinds, options).map_err(|e| match e { - Error::Domain(m) => Error::Domain(format!("chart {}: {m}", c.score_id)), - e => e, - }); - let failed = r.is_err(); - out.lock().expect("results")[i] = Some(r); - if failed { - next.store(n, Ordering::Relaxed); - } - } - }); - } - }); - // charts are taken in order and a taken chart always completes, so every chart before the first failure has - // its result: the error returned is the first in chart order, as the command's - let mut stats = Vec::with_capacity(n); - for r in out.into_inner().expect("results") { - stats.push(r.expect("a chart before the first failure has run")?); - } - Ok(stats) -} - -/// chart_stats(data, seeds=None, workers=None, aptitude=True) -> str -/// -/// The `ournotes-deck.chart-stats/3` document (JSON text) of a deck data document (`nnnotes.deck-data/1` JSON text), -/// as `ournotes-deck chart-stats` writes it. `seeds`: the replay seed count of charts with a luck range (default -/// 8); `workers`: threads measuring charts (default: the available parallelism). Raises ValueError for data the -/// deck model cannot read or a chart whose check deck fails. `aptitude`: measure single Gekisou skill shapes -/// (default true). Gekisou statistics use independent nominal lottery and skill probabilities. -#[pyfunction] -#[pyo3(signature = (data, seeds=None, workers=None, aptitude=true))] -fn chart_stats( - py: Python<'_>, - data: &str, - seeds: Option, - workers: Option, - aptitude: bool, -) -> PyResult { - let options = Options { replay_seeds: seeds.unwrap_or(REPLAY_SEEDS), aptitude }; - let workers = workers.unwrap_or_else(|| std::thread::available_parallelism().map_or(1, |n| n.get())); - py.detach(|| { - let mut data = DeckData::from_json(data).map_err(value_error)?; - let charts = std::mem::take(&mut data.charts); - // the document of no chart: the format, source, model and kinds, exactly as the command writes them - let mut doc = chartstats::document_with(&data, &options).map_err(value_error)?; - data.charts = charts; - let stats = measure(&data, &options, workers).map_err(value_error)?; - doc["charts"] = serde_json::to_value(stats).map_err(value_error)?; - serde_json::to_string(&doc).map_err(value_error) - }) -} - -/// info() -> dict: the deck model of this module (name: the ournotes-deck repository; version, source and commit: -/// its crate ournotes-sim as built; sourceSha256: the SHA-256 of that crate's sources, equal for builds that run the -/// same model) and the formats it reads and writes. -#[pyfunction] -fn info(py: Python<'_>) -> PyResult> { - let d = PyDict::new(py); - d.set_item("name", "ournotes-deck")?; - d.set_item("version", env!("DECK_VERSION"))?; - d.set_item("source", env!("DECK_SOURCE"))?; - d.set_item("commit", env!("DECK_COMMIT"))?; - d.set_item("sourceSha256", ournotes_sim::SOURCE_SHA256)?; - d.set_item("dataFormat", ournotes_sim::data::FORMAT)?; - d.set_item("format", chartstats::FORMAT)?; - Ok(d) -} - -#[pymodule] -fn _deck(m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_function(wrap_pyfunction!(chart_stats, m)?)?; - m.add_function(wrap_pyfunction!(info, m)?)?; - Ok(()) -} diff --git a/src/nnnotes/musicdata.py b/src/nnnotes/musicdata.py index 58a7e81..ffd4623 100644 --- a/src/nnnotes/musicdata.py +++ b/src/nnnotes/musicdata.py @@ -4,7 +4,7 @@ skills), per difficulty the chart facts (level, note counts, BPM, chart times, skill events, fever ranges) and the chart's deck statistics, what the chart contributes to the live score whatever the deck, with Gekisou on (a Gekisou live, every rank) and off (a solo live), measured by the deck model ournotes-deck (the extension module -nnnotes._deck). +the package ournotes-sim). Master data is read from the files as served (deckdata.master_files / apk_master: SHA-256 checked against the manifest, decoded with master.decode). Charts are the TextAssets `Live/MusicScore/` converted by @@ -303,7 +303,7 @@ def skill(r: dict, max_level: dict) -> dict: class Deck: - """The deck model (nnnotes._deck): chart statistics of a deck input document, on `workers` threads (None: the + """The deck model (the package ournotes-sim): chart statistics of a deck input document, on `workers` threads (None: the available parallelism), with independent nominal lottery and skill probabilities. `seeds` gives the number of replay seeds for a chart with a luck range; `aptitude` includes every chart's Gekisou aptitude and cross terms. With a `cache` directory, a chart's statistics are kept there under the SHA-256 of @@ -314,10 +314,10 @@ def __init__(self, seeds: int = DECK_SEEDS, workers: int | None = None, module=N cache: Path | None = None): if module is None: try: - from . import _deck as module + import ournotes_sim as module except ImportError: - raise MusicDataError("the deck model (nnnotes._deck) is not built into this installation: install " - "nnnotes from a wheel or build it (maturin), or pass --no-deck") from None + raise MusicDataError("the deck model (the package ournotes-sim) is not installed: install " + "'nnnotes[deck]', or pass --no-deck") from None self.module, self.seeds, self.workers = module, seeds, workers self.aptitude = aptitude self.cache = None if cache is None else Path(cache) diff --git a/tests/test_musicdata.py b/tests/test_musicdata.py index 4fb3333..1145214 100644 --- a/tests/test_musicdata.py +++ b/tests/test_musicdata.py @@ -7,6 +7,7 @@ import json import re import sys +import tomllib from pathlib import Path import pytest @@ -262,7 +263,7 @@ def aptitude_of(s, positions): class FakeDeck: - """The interface of nnnotes._deck: statistics made from the deck input as the model reports them, with the Gekisou + """The interface of ournotes_sim: statistics made from the deck input as the model reports them, with the Gekisou aptitude (SHAPES; `aptitude`) unless it is left out.""" COMMIT = "7e5d84b5998d28c21541ce3f2e0a3dfb1439f4f6" SOURCE = "5" * 64 @@ -657,30 +658,30 @@ def test_deck_errors(tmp_path, monkeypatch): with pytest.raises(musicdata.MusicDataError, match="deck model: chart 30: the check deck scores"): export(tmp_path, deck=FakeDeck(fail="chart 30: the check deck scores 1, the chart statistics predict 2")) assert not (tmp_path / "music.json").exists() - monkeypatch.setitem(sys.modules, "nnnotes._deck", None) # an installation without the module - with pytest.raises(musicdata.MusicDataError, match="not built into this installation.*--no-deck"): + monkeypatch.setitem(sys.modules, "ournotes_sim", None) # an installation without the package + with pytest.raises(musicdata.MusicDataError, match=r"not installed.*nnnotes\[deck\].*--no-deck"): musicdata.Deck() +def deck_pin() -> str: + """The ournotes-sim version the extra `deck` pins.""" + extras = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))["project"]["optional-dependencies"] + pin, = extras["deck"] + return re.fullmatch(r"ournotes-sim==(\d+\.\d+\.\d+)", pin).group(1) + + def test_deck_module(): - deck = pytest.importorskip("nnnotes._deck") + deck = pytest.importorskip("ournotes_sim") info = deck.info() assert info["name"] == "ournotes-deck" and info["format"] == "ournotes-deck.chart-stats/3" assert info["dataFormat"] == deckdata.DECK_FORMAT and re.fullmatch(r"[0-9a-f]{40}", info["commit"]) assert re.fullmatch(r"[0-9a-f]{64}", info["sourceSha256"]) - lock = (ROOT / "rust" / "Cargo.lock").read_text(encoding="utf-8") - assert f"#{info['commit']}\"" in lock # the commit Cargo.lock pins + assert info["version"] == deck_pin() # the release the extra pins with pytest.raises(ValueError, match="not a deck data file"): deck.chart_stats('{"format": "x"}') assert musicdata.Deck().info()["commit"] == info["commit"] -def test_the_extension_version_is_the_package_version(): - from nnnotes import __version__ - cargo = (ROOT / "rust" / "Cargo.toml").read_text(encoding="utf-8") - assert re.search(r'^version = "([^"]+)"', cargo, re.M).group(1) == __version__ - - # ---------------------------------------------------------------- the deck input (--full) def test_full(tmp_path): asked = [] From baeb58d74ea959034bcbb5e051fb73e7a52ab37f Mon Sep 17 00:00:00 2001 From: nichinichisou0609 <128159543+nichinichisou0609@users.noreply.github.com> Date: Tue, 13 Oct 2026 21:21:58 +0800 Subject: [PATCH 2/3] feat(music-data): keep native simulation programs and export Snap labels The deck statistics of music-data use the deck model's native program cache, and the replay bundle carries the Snap labels of the same master data snapshot. - `--stats-cache DIR` keeps the simulation programs and runs of the deck model (ournotes_sim.chart_stats_cached) in DIR across builds; entries completed before an interrupted build stay valid. The export summary's `deckStats` reports the cache counters: requests, hits, computed, writes, invalid and bytes. - `--seeds` must be a positive integer and `--workers` a nonnegative one (0 selects the available parallelism). - Every chart's expectations are checked against the model's exact bounds; `validate_aptitude_statistics` checks the Gekisou aptitude of a statistics document. - The replay bundle writes `snap-labels.json` (`nnnotes.replay-labels/1`): the names, description templates, per-level effects and artwork ids of the Snap and card tables, each bound to the SHA-256 the master data provenance records; the manifest names it in `snapLabels`. Labels from another region, master version or table hash are rejected. Co-Authored-By: Claude Opus 5.5 --- docs/music-data.md | 281 ++++++++++----- docs/replay.md | 30 +- docs/schema/music-data.schema.json | 159 ++++---- src/nnnotes/cli.py | 6 +- src/nnnotes/musicdata.py | 560 +++++++++++++++-------------- src/nnnotes/replaydata.py | 45 ++- tests/test_musicdata.py | 378 +++++++++++-------- tests/test_replaydata.py | 63 +++- tests/test_statscache.py | 160 +++++++++ 9 files changed, 1094 insertions(+), 588 deletions(-) create mode 100644 tests/test_statscache.py diff --git a/docs/music-data.md b/docs/music-data.md index da305dc..52fd78c 100644 --- a/docs/music-data.md +++ b/docs/music-data.md @@ -26,15 +26,26 @@ nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--ful - `--full`: also write the deck model's input, every chart's runtime notes and the master data tables about cards, skills, bonuses, scores and events ([the deck input](#the-deck-input---full)), for tools that run a deck model of their own. -- `--no-deck`: leave every chart's `deck` null. `--workers N` sets the measurement threads - (default: every processor). `--seeds N` sets the replay seed count for a chart with a luck range (default 8); - nominal expectation measurements are independent of this count. -- `--no-gekisou-aptitude`: omit single-skill aptitude measurements while retaining baseline statistics. -- `--stats-cache DIR`: keep every chart's deck statistics in `DIR`, one file per chart named by the SHA-256 of what - they are a function of: the deck model's sources (`provenance.deck.sourceSha256`), `--seeds` and the aptitude - options, the master data tables of the deck input and the chart's runtime notes. A chart found there is not measured - again, so an export after a deck model release that keeps the model's sources, or after a master data update that - keeps those tables, measures only the charts that changed. `DIR` then holds this file's charts only. +- `--no-deck`: do not run the deck model; every chart's `deck` is null. A full statistics build evaluates many + complete lives per chart. `--workers N` sets the native worker count (default: every available processor). +- `--seeds N`: list this many replay seeds for a chart with a luck range (default 8). Expectations and aptitude + use the model's independent nominal probabilities, not a sample of these seeds. Increasing this option changes + the replay examples without changing the expected scores or their numerical enclosures. +- `--no-gekisou-aptitude`: keep the ordinary deck statistics and skip single-skill aptitude measurements. +- `--stats-cache DIR`: use the native model's persistent program and run cache. Reuse follows the complete + compiled simulation inputs, so unchanged programs and runs can survive a change to the skill catalog. New + shapes need their own measurements; changing a shape's numeric index does not by itself identify a new program. + Every invocation still passes the complete current master data and charts to the native model, which constructs + the current header and assembles the requested statistics. Python does not reuse a chart-sized JSON result or + replace the model's dependency analysis with a table allowlist. + Cache records are isolated by cache schema and model source SHA-256. Each completed result is saved through a + unique temporary file, a file sync and an atomic rename. Invalid or unreadable records are measured again; + interrupted work retains its completed records. Concurrent processes may compute the same missing record, but + publish complete records atomically. Old inputs remain available for later reuse: there is no automatic garbage + collection, and unused model-source directories can be removed to reclaim space. A record larger than 64 MiB is + not persisted. The command's `deckStats` summary reports native requests, hits, completed computations, writes, + invalid entries and written bytes; these count cache operations, not charts. Cache counters are not part of the + published music data. This cache does not import the previous Python chart cache. - `--no-bgm`: do not read the cue sheets (every `bgm.length` is null). - `--jackets DIR`: also write every song's jacket, the Texture2D `Image/Jacket/`, as `DIR/.webp` (WebP quality 88, scaled down with Lanczos to at most 320 pixels on the longer side, without alpha when opaque); a @@ -99,7 +110,7 @@ A **text** is an object with one string per language of `languages` (`{"ja": ... | `master.tables..sha256` | SHA-256 of each table's file as served, before decoding: the song tables (`MasterLiveMusic`, `MasterLiveMusicScore`, `MasterText`, `MasterBand`, `MasterCharacter`, `MasterTag`, `MasterLiveMusicCategory`, `MasterSound`, `MasterSoundCueSheet`, `MasterLiveScoreRank`), the tables of the Gekisou catalog (`MasterMemberCard`, `MasterSupportCard`, `MasterSupportCardRank`, `MasterGekisouSkill`, `MasterGekisouSkillEffect`, `MasterGekisouSupportSkill`, `MasterGekisouSupportSkillEffect`) and, when the deck model runs or with `--full`, the tables of [the deck input](#the-deck-input---full) | | `exporter.name`, `exporter.version` | `nnnotes` and its version | | `exporter.chartFormat` | the format of the chart converter the notes come from, `nnnotes.live-score/1` | -| `deck` | the deck model: `{name, version, source, commit, format}`, `ournotes-deck`, the package version of its crate ournotes-sim, the repository, the git commit nnnotes is built with and the statistics format (`ournotes-deck.chart-stats/3`); null with `--no-deck` | +| `deck` | the deck model: `{name, version, source, commit, sourceSha256, format}`, `ournotes-deck`, the package version of its crate ournotes-sim, the repository, the git commit nnnotes is built with and the statistics format (`ournotes-deck.chart-stats/3`); null with `--no-deck` | ### bands, characters, tags, categories @@ -172,100 +183,159 @@ plays, `musicLengthMs` is the length the score code uses; a listing chooses the ## Deck statistics -The deck model reports `ournotes-deck.chart-stats/3`. Gekisou measurements use independent nominal -lottery and skill probabilities. An interval is `[center, half-width]`, enclosing the expectation under -that law with outward probability arithmetic and score rounding bounds. Deterministic judgement counters -are exact integers. The law describes the model; agreement with a particular game's seed distribution -is a separate question. +The native statistics format is `ournotes-deck.chart-stats/3`. It evaluates complete lives on the game's default +frame schedule in two scenarios: -Serialized point centers use millipoints, consumed result counts use five fractional digits, and weights -use twelve. Half-widths round outward at the same precision and include the center's rounding. +- **Gekisou on** (`expectation`): each note is judged at its exact time, Just inside a Just-count range and Perfect + elsewhere, with rank 1 in each range. Random lotteries and probabilistic skills follow independent nominal + probabilities. The numbers enclose the expectation under that stated model. +- **Gekisou off** (`offSeeds`): each note is Perfect, with seed 0 and no Just, luck, Gekisou combo or rank bonus. + This remains a deterministic measurement. A chart with more than three fevers can still play here. -The best play judges each note at its exact time, Just in Just-count ranges and Perfect elsewhere. -The Perfect play supplies the other endpoint, with every Just judged Perfect. Both include rank 1 bonuses. -Free Live has its own deterministic Perfect run, without Gekisou, and remains available on charts with -more than three fever ranges. +### Expected values and their intervals + +An **Estimate** is `[center, radius]`, denoting the interval `[center - radius, center + radius]`. Both values and +both endpoints must be finite, and the radius must be nonnegative. The radius bounds numerical uncertainty; it is +not a standard error, a confidence interval or variation between replay seeds. Point values are serialized at +millipoint precision, lottery counts at five decimal places and weights at twelve, with radii expanded outwards +so rounding keeps the enclosure. A zero radius is a point interval. + +The no-skill expected score is measured at `deck.model.power` (300000). For each ordinary **score-up kind** and +performance position, `expectation.weights[kind][position]` encloses the expected gain from one effect at factor +1, divided by that power. Effects run through the simulator's own conditions, frames and appliers. A deck whose +ordinary live skills are represented by these kinds has the score approximation + +``` +P * (score / model.power + sum_k factor_k * weights[kind_k][k]) +``` + +where `P` is its power. Arithmetic on estimates must carry their radii. The approximation also has a separate +integer-floor error, checked by a complete nominal evaluation at `model.checkPower`; a numerical radius does not +replace that error bound. The factor follows the game's value conversion: effect type 2000 uses +`floor(value / 10000f * 1e5) / 1e5`, type 2005 uses `floor(value / -10000f * 1e5) / 1e5`, and types 2002 and 2004 +round the binary32 quotient `value / 10000f` half to even. Other effect types, including cumulative scoring, life +and judgement conversion, have no ordinary score-up weights. + +`replaySeeds` is separate from these expectations. A playable chart without a luck range lists `[0]`; otherwise +it lists the first `--seeds` values of the published sequence. Candidate `k` is the low 32 bits, interpreted as a +signed integer, of output `k + 1` of SplitMix64 started at `0x6765_6B69_736F_7531` (`gekisou1`). A candidate is kept +when its effective stream-seed pair (`|b|`, `|b ^ 0x9E3779B9|`, with -2^31 treated as 2^31 - 1) differs from all +previous pairs. A shorter list is a prefix of a longer list. These values select concrete replay examples; the +exporter never averages them to obtain `expectation` or `gekisouAptitude`. + +### Ranks and Just judgements + +A range at rank `r` awards `trunc(rangeScore * p(r) / 100)`, where +`p(r) = ranges[i].rankBonusPercents[r - 1]`. `rankBonus` is the expected result of that integer operation at rank 1. +In general, **the expectation of a truncated score is not the truncation of the expected score**. Readers must +keep the separately measured `rankBonus` and `rankBonusPerfect`; neither is reconstructed by truncating an +expectation's center. + +When rank changes do not alter range scoring, the other-rank approximation replaces the rank-1 bonus by +`E[rangeScore] * p(r) / 100` and adjusts an ordinary weight by +`sum_i (p_i(r_i) - p_i(1)) / 100 * rangeWeights[kind][position][i]`. The native rank check includes the resulting +integer-truncation slack in its bound. `rangeWeights` is null when overlapping ranges break that independence; +a kind's row is null when its conditions read a confirmed rank (condition 7012). These nulls mark a limit of the +rank formula, not a zero contribution. Such scenarios require full simulation. + +`scorePerfect` and `rangeScorePerfect` use the same nominal model with every Just judgement replaced by Perfect. +`rankBonusPerfect` is measured on that play too. A chart with `justNotes == 0` has matching best and Perfect +expectations. Interpolating between these endpoints is a consumer's approximation, not an additional measured +Just-rate curve. ### deck -`model` describes the play, probability law, measurement `power`, `checkPower`, `unitValue` and score formulas. -`kinds` groups ordinary score-up rows of type 2000, 2002, 2004 or 2005 by duration, targets, conditions, -release and execution limits. A weight is the expected increment per unit of deck power and effect factor. -Type 2000 uses `value / 10000`; 2002 adds its intrinsic half factor; 2004 and 2005 use the effect's -half-to-even rounded activation factor. A card's kind is identified by its effect row's parameters. +| Field | Content | +|---|---| +| `model` | the native model's description: engine, play, score formula, measurement and check powers, effect unit value, nominal expectation law, rank approximation, Perfect play, Free Live and single-shape aptitude | +| `kinds[]` | ordinary score-up kinds from `MasterLiveSkillEffect` types 2000, 2002, 2004 or 2005 without a cumulative condition. Each has a continuous zero-based `id`, effect type, activation time, duration, targets, condition/release/reset groups, effect limits, source `rows` and distinct sorted `values` | +| `gekisouAptitude` | the independently validated single-shape catalog below, or null when aptitude is disabled | + +A kind's duration is `ceil(activationTimeSecond * 1000f)` milliseconds. Match a card's ordinary live-skill effect at +its actual level to the kind's structural fields. Kind ids and shape ids are indices in the current document; +they are not stable identifiers across master-data releases. ### charts[].deck | Field | Content | |---|---| -| `convertedNoteCount`, `skip` | the score divisor and skipped-live score per unit of power | -| `events`, `positions` | `[position, timeMs]` skill events and the number of measured performance positions | -| `ranges` | index, mission, start/end times and `rankBonusPercents` for ranks 1..5 | -| `justNotes` | exact Just count of the best play | -| `expectation` | `score`, `scorePerfect`, `ranges`, `weights`, `rangeWeights`, `check`, `rankCheck`; null when Gekisou is unplayable | -| `replaySeeds` | seeds for per-play replay: `[0]` without a luck range, otherwise the first `--seeds` published seeds; empty when Gekisou is unplayable | -| `offSeeds` | one deterministic Free Live run: `seed: 0`, integer `score`, numeric `weights` and `{deck, exact, predicted, bound}` check | -| `unplayable` | null or the reason Gekisou cannot play the chart | - -`expectation.score` and `scorePerfect` are point intervals at `model.power`. Each expected range has -`rangeScore`, `rangeScorePerfect`, `rankBonus`, `rankBonusPerfect` and `luckPoints` intervals, integer -`maxCombo` and `justCount`, and four `lotResults` intervals: expected consumed Miss, Hit, Super Hit and -Critical counts. `weights[kind][position]` and `rangeWeights[kind][position][range]` hold intervals in -the weight unit. A range-weight kind is null when its conditions read confirmed rank; the whole field -is null when overlapping ranges prevent a linear rank transformation. - -A check has `{deck, ranks, expected, predicted, bound}`. Its deck contains `[kind, value]` or null per -position, its score intervals use `checkPower`, and the greatest separation of their endpoints must be -within `bound`. The baseline check uses `ranks: null`; `rankCheck` supplies explicit fixed rank arrivals -and is null when no linear rank check is available. The exporter checks chart facts, dimensions, finite -intervals, counters, replay seeds and every check before writing the file. - -### Ranks and accuracy - -For disjoint ranges whose effects do not read confirmed rank, an arbitrary-rank expectation uses +| `convertedNoteCount` | the converted note count used by the score formula | +| `skip` | points per unit of power for a skipped live: every note Great, combo 0, no skills | +| `events` | `[[position, timeMs], ...]` in chart order | +| `positions` | largest event position plus one; the size of each position dimension | +| `ranges[]` | `index`, `mission` (1 combo, 2 luck, 3 Just count), `startMs`, `endMs`, `rankBonusPercents` for ranks 1–5, and the rank-1 `rankBonusPercent` | +| `justNotes` | notes judged Just on the best Gekisou play | +| `expectation` | the expected Gekisou statistics below, or null when unplayable | +| `replaySeeds` | concrete replay seeds, empty when unplayable | +| `offSeeds` | exactly one deterministic Free Live result: seed 0, integer `score`, scalar `weights[kind][position]`, and `check` with `deck`, integer `exact`, scalar `predicted` and `bound`; a weight row is null when the kind reads unavailable Gekisou state | +| `unplayable` | null, or why Gekisou cannot play this chart; a fourth fever is unsupported by the game. Free Live statistics remain present | +| `gekisouAptitude` | per-range factors and single-shape increments below, or null | + +The expectation object contains: -``` -score_r = score - sum(rankBonus_i) + sum(E[trunc(rangeScore_i * p_i(r_i) / 100)]) -weight_r = weight + sum((p_i(r_i) - p_i(1)) / 100 * rangeWeight_i) -``` - -The expected truncation can be enclosed by scaling the range-score expectation and adding a rounding -allowance. Truncating the center alone does not compute an expected rank bonus. At rank 1 the measured -bonus interval is available directly. `rankBonusPerfect` supplies the Perfect endpoint's bonus. -Just-rate interpolation and Great-share scaling are accuracy approximations; the summary does not -describe arbitrary misses or combo breaks. Per-note replay runs an explicit play with its own seed. +| Field | Content | +|---|---| +| `score`, `scorePerfect` | Estimate of no-skill points, including rank-1 bonuses, on the best and Perfect plays | +| `ranges[]` | `rangeScore`, `rankBonus`, `rankBonusPerfect`, `rangeScorePerfect` and `luckPoints` as Estimates; deterministic integer `maxCombo` and `justCount`; four `lotResults` Estimates in Miss, Hit, Super Hit, Critical order | +| `weights` | `[kind][position]` Estimates of points per unit of deck power and effect factor | +| `rangeWeights` | `[kind][position][range]` Estimates for rank adjustment, or null; individual kind rows may be null | +| `check` | `deck`: `[kind, value]` or null at each position; `ranks: null`; `expected` and `predicted` Estimates in points at `model.checkPower`; scalar nonnegative `bound` | +| `rankCheck` | the same check deck evaluated at ranks 1–5 per range; the same Estimate fields and scalar bound, or null when no rank check applies | + +The exporter checks native chart identities and times against the chart facts, and checks each range's rank +percentages against `MasterLiveGekisouRankingScoreBonus`. It validates array dimensions, finite estimates, +nonnegative radii, deterministic counters, replay seeds, unavailable rank domains, best/Perfect agreement without +Just notes, and all check bounds. Checks compare the most distant endpoints; the serialized check allows only +the outward millipoint rounding expansion of its two intervals in addition to the native bound. Lottery counts +and luck points outside a luck range must enclose zero. ### Gekisou skill aptitude -The file's `deck.gekisouAptitude` has `plainKind`, `host`, `law` and `shapes`. Each shape groups equivalent -score-relevant parameters and retains its source, mission, band condition, normalized effect rows and -skill/level references. Member skills use their highest level; support skills use the highest Snap rank's -level. Support target differences share a shape while each skill retains `memberTargetIds` and `bandIds`. -A support skill is measured with an empty member skill of the same mission. +`deck.gekisouAptitude` describes the shapes measured, and each chart's `deck.gekisouAptitude` describes the expected +increment with **one** such shape equipped. Ordinary `expectation` has no card Gekisou skills. Single-shape gains +cannot be added to estimate several interacting skills; complete formations still require simulation. -Each chart has `{factors, variants}` or null when there are no applicable ranges/shapes or aptitude is -disabled. Factors give judged, Just, Perfect and tail note counts, starting combo and the expected number -of consumed lotteries (`lotteries`, an interval). Each applicable shape has one variant, or matching and -nonmatching band variants. A variant gives: +The header has `plainKind`, `host`, `law` and `shapes`. `plainKind` is the ordinary, unconditional, whole-team +five-second score-up kind used for cross terms, or null. `host` describes the paired member used for a support +skill. `law` is exactly `independent nominal lottery and skill probabilities`. + +Each shape has a continuous zero-based `id`, `source` (`member` or `support`), `mission` (1 combo, 2 luck, 3 Just, +4 all), `bandCondition`, normalized `effects` and `skills`. Each skill reference is +`{id, level, memberTargetIds, bandIds}` and joins to the Gekisou catalog for its text. Members use their skill's +highest level; snaps use their support skills' levels at the highest rank. Effects preserve master order and +expose type, trigger, duration, value, limits, targets, four condition groups and cumulative condition, as specified +by the Schema. Equivalent parameters share a shape. Support condition 5000's target is normalized away; +`memberTargetIds` and `bandIds` retain the targets and bands for matching, or are null without that condition. + +A chart's aptitude is `{factors, variants}`, or null when disabled, unplayable, without ranges, or without shapes. +`factors` has one object per range: deterministic `judgedNotes`, `justNotes`, `perfectNotes`, `tailNotes` and +`comboAtStart`, plus an Estimate of baseline `lotteries`. That interval must agree with the sum of the baseline +range's four expected lottery-result counts. The factors of overlapping ranges may refer to the same notes. + +Variants appear in shape-id order for the chart's missions and mission 4. A band-conditioned shape has a +`bandMatch: true` variant followed by `false`; other shapes use null. | Field | Content | |---|---| -| `score`, `scorePerfect`, `tail`, `tailPerfect`, `converted` | expected gains with the skill minus without it; `[center, half-width]` | -| `ranges` | `rangeScore`, `rankBonus`, `rangeScorePerfect`, `rankBonusPerfect`, `maxCombo`, `justCount`, `luckPoints` gain intervals | -| `weights` | the plain kind's weight changes, `[position][interval]`; null without that kind | -| `rangeWeights` | its range-weight changes, `[position][range][interval]`; null outside the linear domain | -| `check` | the full expected score and linear prediction for a random plain-skill deck at explicit ranks | - -The score and tail centers satisfy `tail = score - sum(rangeScore + rankBonus)`, and the Perfect endpoint -uses the corresponding Perfect fields. Interval radii propagate by outward arithmetic. Conversion, -combo and Just counters have zero radius after proving their independence from lotteries. - -Each variant measures one skill alone. Several gains cannot be added to estimate combined skills: -combo caps, lottery gauges, Rush and Just-triggered support effects interact. Plain-skill cross weights -are measured on the best play. Below 100% Just, a page can interpolate the no-ordinary-skill endpoints; -it cannot derive complete cross weights at that accuracy from this summary. A full play uses replay. -The exporter verifies shape coverage, normalized effect rows, targets, missions, variant ordering, -intervals, tail identities and expectation checks against the supplied master data. +| `shape`, `bandMatch` | the current shape index and measured band condition | +| `score`, `scorePerfect` | Estimate of the total point increment on the best and Perfect plays | +| `tail`, `tailPerfect` | Estimate of the increment outside the range scores and their bonuses | +| `converted` | deterministic integer converted-note increment as `[value, 0]` | +| `ranges[]` | Estimates of `rangeScore`, `rankBonus`, `rankBonusPerfect`, `rangeScorePerfect` and `luckPoints`; deterministic integer `maxCombo` and `justCount` increments as `[value, 0]` | +| `weights` | `[position]` Estimates of cross terms with the plain kind, or null without it | +| `rangeWeights` | `[position][range]` Estimates of cross terms, or null without a plain kind or when the chart or shape cannot use linear rank adjustment | +| `check` | plain-kind `deck`, explicit `ranks`, expected/predicted point Estimates and scalar bound; a nonlinear-rank shape is checked at rank 1 in every range | + +Point increments are measured at `deck.model.power`. Their intervals must be consistent with +`score = tail + sum(rangeScore + rankBonus)` and +`scorePerfect = tailPerfect + sum(rangeScorePerfect + rankBonusPerfect)`. These are interval identities, not +identities between rounded centers. A shape that reads confirmed rank in any condition group or cumulative +condition has no linear rank cross terms, even when the chart's ordinary range weights are available. + +The exporter independently checks the shape catalog against the complete master data, including skill and level +coverage, normalized effects, member targets and bands. It then checks variant order and coverage, factor counts, +interval identities, dimensions, deterministic integer increments and expectation bounds. Aptitude has no +sample-count, standard-error target or convergence-cap fields in this format. ## The deck input (`--full`) @@ -383,14 +453,37 @@ extra `deck` pins one release (`pip install 'nnnotes[deck]'`, or `pip install -e same nnnotes version always carries the same deck model: the pin moves only through a pull request (`.github/workflows/deck.yml` opens one when ournotes-deck publishes a newer release with its WASM packages and its wheels), and `provenance.deck.version` and `provenance.deck.commit` name the release in every file. `provenance.deck.sourceSha256` is the SHA-256 of the deck model's sources (`ournotes_sim::SOURCE_SHA256`): -releases with the same value measure the same statistics. The replay and recommendation engines of `--replay-engine` and `--recommend-engine` are that release's WASM -packages. +releases with the same value measure the same statistics. Replay bundles take the engines of the same release: +the replay and recommendation engines of `--replay-engine` and `--recommend-engine` are its WASM packages, whose +`build-info.json` names the release commit that `provenance.deck.commit` names; engines of another release are +rejected. ## Versions -`format` names the major version. `nnnotes.music-data/2` carries deck statistics -`ournotes-deck.chart-stats/3`. Additive fields and tables retain their existing meanings within a major -version; readers ignore unknown keys and reject unsupported major versions. Deck model provenance names -the exact statistics format and source digest. The optional runtime input keeps `nnnotes.deck-data/1`. - -`--replay-dir OUT/replay --replay-engine PKG [--recommend-engine PKG]` writes normalized runtime inputs and the pinned model's WASM release packages (the replay engine and, optionally, the recommendation engine) as described in [replay.md](replay.md). The music data stays compact and carries the SHA-bound `replay.manifestUrl` pointer. No original chart/master blobs or native binary are included in this artifact bundle. +`format` names the major version. Within `nnnotes.music-data/2`, fields and tables are only added, never renamed, +removed or given another meaning, and readers ignore keys they do not know. Readers must reject an unknown major +version. This version carries `ournotes-deck.chart-stats/3` in `provenance.deck.format` and requires that exact +statistics format from the extension module. + +`nnnotes.music-data/2` replaces the per-seed Gekisou statistics and sampled aptitude of `/1` with nominal +expectations and outward numerical intervals. `seeds` becomes `expectation` plus separate `replaySeeds`; the +aptitude `seedRule` becomes `law`, sampling flags disappear, checks use interval `expected`/`predicted` values, and +Perfect-play range bonuses are explicit. These are changes of meaning as well as shape: an older reader must not +interpret a radius as a standard error. Free Live `offSeeds`, chart facts and the optional `nnnotes.deck-data/1` +input retain their existing semantics. The old aptitude sampling-cap CLI options are no longer accepted. + +The persistent statistics cache also starts a new format. The native cache uses model-source namespaces and +fine-grained program/run identities; no old Python chart-cache entry is migrated or trusted. Removing a cache +changes the amount of work, not the music data bytes. + +Deploy readers supporting `/2` and nominal expectations before publishing the new format. The reference page +is pinned in `.github/workflows/music-data-region.yml`; any configured override must support the same format. +The native release packages, Python producer and deployed consumers must move together as described in +[the publication workflow](../.github/MUSIC_DATA.md#before-the-first-run). + +`--replay-dir OUT/replay --replay-engine PKG [--recommend-engine PKG]` writes normalized runtime inputs, the pinned +model's WASM release packages (the replay engine and, optionally, the recommendation engine) and same-snapshot Snap +label/reference-card metadata as described in [replay.md](replay.md). The music data stays compact and carries +the SHA-bound `replay.manifestUrl` pointer; its manifest optionally references `snap-labels.json`. The selected +decoded label tables retain names/templates, original per-level effects and artwork IDs. No encrypted master/chart +blobs or native binary are included in this artifact bundle. Label metadata does not change the score model. diff --git a/docs/replay.md b/docs/replay.md index d785863..fc56546 100644 --- a/docs/replay.md +++ b/docs/replay.md @@ -1,3 +1,5 @@ + + # Shared Rust/WASM replay inputs `nnnotes music-data --replay-dir OUT/replay --replay-engine PKG [--recommend-engine PKG] -o OUT/music-data.json` writes the normalized input of the pinned deck model. Each engine is an ournotes-deck WASM release package of the pinned model, given as the downloaded archive or its extracted directory: `ournotes-replay-wasm-vVERSION.tar.gz` for `--replay-engine` and, optionally, `ournotes-recommend-wasm-vVERSION.tar.gz` for `--recommend-engine`. Its `build-info.json` must describe that module's WASM package (`kind:"wasm"`, `module`), name the pinned model's `commit`, and list in `files` the SHA-256 of the web binding's JS and WASM files, which must match the package files. The web binding (`web/ournotes__wasm.js` and `web/ournotes__wasm_bg.wasm`) and `build-info.json` are copied byte for byte. Actual ACB cue length is required. Original encrypted master files, chart blobs and native binaries are not copied. @@ -16,14 +18,14 @@ The music-data pointer is: "deckData": {"format":"nnnotes.deck-data/1","url":"deck-data.json","sha256":"","bytes":0}, "charts": [{"scoreId":10000303,"url":"charts/10000303.json","sha256":"","bytes":0,"musicLengthMs":120557,"noteCount":0,"assetSha256":""}], "engine": { - "model": {"name":"ournotes-deck","version":"","source":"","commit":"","format":"ournotes-deck.chart-stats/3"}, + "model": {"name":"ournotes-deck","version":"","source":"","commit":"","sourceSha256":"","format":"ournotes-deck.chart-stats/3"}, "requestFormat":"ournotes.replay/1","class":"ReplaySession","methods":["describeChart","template","run"], "js":{"url":"engine/ournotes_replay_wasm.js","sha256":"","bytes":0}, "wasm":{"url":"engine/ournotes_replay_wasm_bg.wasm","sha256":"","bytes":0}, "build":{"url":"engine/build-info.json","sha256":"","bytes":0} }, "recommendEngine": { - "model": {"name":"ournotes-deck","version":"","source":"","commit":"","format":"ournotes-deck.chart-stats/3"}, + "model": {"name":"ournotes-deck","version":"","source":"","commit":"","sourceSha256":"","format":"ournotes-deck.chart-stats/3"}, "js":{"url":"recommend/ournotes_recommend_wasm.js","sha256":"","bytes":0}, "wasm":{"url":"recommend/ournotes_recommend_wasm_bg.wasm","sha256":"","bytes":0}, "build":{"url":"recommend/build-info.json","sha256":"","bytes":0} @@ -38,3 +40,27 @@ Numbers and SHA placeholders above illustrate the schema; actual manifests conta Each chart resource is `{format:"nnnotes.replay-chart/1",scoreId,musicLengthMs,chart}`. `chart` retains the existing DeckData record: `asset:{key,sha256}`, equal-length `notes:{id,op,judgementType,timeMs}` arrays in native enumeration order, `skillEvents:{timeMs}` and `fevers:{startMs,endMs}`. `deck-data.json` uses the existing `nnnotes.deck-data/1` schema and adds `provenance.replay.musicLengthsMs:{"":}`. Score-table length is a separate native value derived by the shared Rust chart implementation; it must not be replaced by audio length. The page loads and hashes the module/data once, calls `new ReplaySession(deckDataJson)`, then parses JSON returned by `describeChart(scoreId)`, `template(scoreId,power,fps)` and `run(requestJson)`. Template construction stays in Rust. It preserves simultaneous note order, supplies Perfect for judgement notes and native Pass for other operations, and does not guess Just. The request has `format:"ournotes.replay/1"`, explicit `scoreId`, `power`, audio and score-table lengths, `seed`, five performers, a `skillOrder` permutation, mode, and frames `{timeMs,deltaSeconds,judgements:[{noteId,judgement,judgementTimeMs}]}`. Complete input is required; missing notes are not silently changed to Miss. Advanced skill/rank scenarios use the same request JSON and engine. +## Snap labels and reference cards + +Replay exports also write `snap-labels.json`, referenced by the optional `manifest.snapLabels` resource +(`format`, relative `url`, SHA-256 and byte length). Its format is `nnnotes.replay-labels/1`. Older manifests +without the resource remain valid; a consumer must not fill the gap with unbound tables from a newer global +master snapshot. + +The resource contains `region`, `masterVersion` and `tables. = {sha256, rows}` for 13 original decoded +tables: ordinary/Gekisou support definitions and per-level effects, `MasterText`, condition sets/conditions, +cumulative conditions, skill targets, characters, bands, member cards and support cards. Each table hash is the +served-file hash in the same `MasterManifest.json`/runtime master provenance. The producer rejects missing or +inconsistent hashes. This retains names, description templates, every effect level, condition target names and +card artwork IDs. No highest-level growth state or scoring coefficients are inferred. `MasterSkillIcon` is +omitted because the runtime provenance does not currently bind that table. + +Consumers first verify the resource's SHA/bytes and its region/version/table hashes against the loaded replay +data. They can reuse their existing text/skill/card normalization with the chosen level; named skill targets +resolve through `MasterSkillTarget` to a character or band. Descriptions remain presentation metadata. Actual +Snap scoring uses the loaded Rust evaluator and explicit paired member predicates, skill levels, mode, +judgement plan, execution order and seed. The label file cannot enable an unsupported effect or substitute a +linear coefficient for whole-live evaluation. Reference-card maximum-power fields are not the evaluated deck's power. + +The publisher validates this optional resource as part of the replay inventory and reads it back before +publishing the replay manifest/music-data pointer. Its content-addressed tree retains the prior resource files. diff --git a/docs/schema/music-data.schema.json b/docs/schema/music-data.schema.json index 9ffdec6..148c345 100644 --- a/docs/schema/music-data.schema.json +++ b/docs/schema/music-data.schema.json @@ -152,6 +152,7 @@ "version", "source", "commit", + "sourceSha256", "format" ], "properties": { @@ -175,8 +176,8 @@ "description": "the SHA-256 of the deck model's sources; releases with the same value measure the same statistics" }, "format": { - "type": "string", - "description": "the statistics format, ournotes-deck.chart-stats/3" + "const": "ournotes-deck.chart-stats/3", + "description": "The native nominal-expectation statistics format." } } } @@ -424,7 +425,8 @@ ], "required": [ "model", - "kinds" + "kinds", + "gekisouAptitude" ], "description": "the deck model's measurement and score-up kinds; null with --no-deck", "properties": { @@ -693,7 +695,9 @@ "justNotes", "unplayable", "expectation", - "replaySeeds" + "replaySeeds", + "offSeeds", + "gekisouAptitude" ], "properties": { "convertedNoteCount": { @@ -732,7 +736,8 @@ "mission", "startMs", "endMs", - "rankBonusPercent" + "rankBonusPercent", + "rankBonusPercents" ], "properties": { "index": { @@ -764,7 +769,8 @@ } }, "justNotes": { - "type": "integer" + "type": "integer", + "minimum": 0 }, "offSeeds": { "type": "array", @@ -779,7 +785,7 @@ ], "properties": { "seed": { - "type": "integer" + "const": 0 }, "score": { "type": "integer", @@ -828,14 +834,16 @@ } } } - } + }, + "minItems": 1, + "maxItems": 1 }, "unplayable": { "type": [ "string", "null" ], - "description": "why Gekisou cannot play the chart; expectation is null and replaySeeds is empty" + "description": "Why the game cannot play the chart with Gekisou; expectation is then null and replaySeeds empty, while offSeeds remains available." }, "gekisouAptitude": { "$ref": "#/$defs/chartGekisouAptitude" @@ -843,19 +851,22 @@ "expectation": { "anyOf": [ { - "$ref": "#/$defs/chartExpectation" + "$ref": "#/$defs/expectedStats" }, { "type": "null" } - ] + ], + "description": "Expected Gekisou scores under independent nominal lottery and skill probabilities; null when unplayable." }, "replaySeeds": { "type": "array", "items": { - "type": "integer" + "type": "integer", + "minimum": -2147483648, + "maximum": 2147483647 }, - "description": "Seeds for deterministic replays." + "description": "Whole-live replay examples. Changing this prefix does not change expectations." } } }, @@ -1507,9 +1518,7 @@ } }, "law": { - "type": "string", - "minLength": 1, - "description": "The independent nominal lottery and skill probability law." + "const": "independent nominal lottery and skill probabilities" } } }, @@ -1570,11 +1579,11 @@ "required": [ "rangeScore", "rankBonus", + "rankBonusPerfect", "rangeScorePerfect", "maxCombo", "justCount", - "luckPoints", - "rankBonusPerfect" + "luckPoints" ], "properties": { "rangeScore": { @@ -1698,8 +1707,6 @@ }, "estimate": { "type": "array", - "minItems": 2, - "maxItems": 2, "prefixItems": [ { "type": "number" @@ -1710,7 +1717,9 @@ } ], "items": false, - "description": "The center and outward half-width of a nominal expectation interval." + "minItems": 2, + "maxItems": 2, + "description": "[center, outward interval half-width]; the radius is a numerical enclosure, not a standard error." }, "expectationCheck": { "type": "object", @@ -1728,9 +1737,16 @@ "anyOf": [ { "type": "array", - "items": { - "type": "integer" - }, + "prefixItems": [ + { + "type": "integer", + "minimum": 0 + }, + { + "type": "integer" + } + ], + "items": false, "minItems": 2, "maxItems": 2 }, @@ -1761,10 +1777,55 @@ "type": "number", "minimum": 0 } - }, - "description": "A full nominal expectation and its linear prediction; the worst endpoint separation is within bound." + } }, - "chartExpectation": { + "expectedRange": { + "type": "object", + "required": [ + "rangeScore", + "rankBonus", + "rankBonusPerfect", + "rangeScorePerfect", + "maxCombo", + "justCount", + "luckPoints", + "lotResults" + ], + "properties": { + "rangeScore": { + "$ref": "#/$defs/estimate" + }, + "rankBonus": { + "$ref": "#/$defs/estimate" + }, + "rankBonusPerfect": { + "$ref": "#/$defs/estimate" + }, + "rangeScorePerfect": { + "$ref": "#/$defs/estimate" + }, + "maxCombo": { + "type": "integer", + "minimum": 0 + }, + "justCount": { + "type": "integer", + "minimum": 0 + }, + "luckPoints": { + "$ref": "#/$defs/estimate" + }, + "lotResults": { + "type": "array", + "items": { + "$ref": "#/$defs/estimate" + }, + "minItems": 4, + "maxItems": 4 + } + } + }, + "expectedStats": { "type": "object", "required": [ "score", @@ -1785,49 +1846,7 @@ "ranges": { "type": "array", "items": { - "type": "object", - "required": [ - "rangeScore", - "rankBonus", - "rankBonusPerfect", - "rangeScorePerfect", - "luckPoints", - "maxCombo", - "justCount", - "lotResults" - ], - "properties": { - "rangeScore": { - "$ref": "#/$defs/estimate" - }, - "rankBonus": { - "$ref": "#/$defs/estimate" - }, - "rankBonusPerfect": { - "$ref": "#/$defs/estimate" - }, - "rangeScorePerfect": { - "$ref": "#/$defs/estimate" - }, - "luckPoints": { - "$ref": "#/$defs/estimate" - }, - "maxCombo": { - "type": "integer" - }, - "justCount": { - "type": "integer" - }, - "lotResults": { - "type": "array", - "items": { - "$ref": "#/$defs/estimate" - }, - "minItems": 4, - "maxItems": 4, - "description": "Expected consumed result counts: Miss, Hit, Super Hit, Critical." - } - } + "$ref": "#/$defs/expectedRange" } }, "weights": { diff --git a/src/nnnotes/cli.py b/src/nnnotes/cli.py index abe9dd3..6d3e5b9 100644 --- a/src/nnnotes/cli.py +++ b/src/nnnotes/cli.py @@ -986,14 +986,14 @@ def target(m, what): c.add_argument("--no-deck", action="store_true", help="do not run the deck model (every chart's deck is null)") c.add_argument("--seeds", type=int, default=8, metavar="N", - help="replay seeds of a chart with a luck range (default 8); statistics use nominal expectations") + help="replay seeds for a chart with a luck range (default 8); expectations do not use sampling") c.add_argument("--workers", type=int, metavar="N", help="threads measuring charts (default: every processor)") c.add_argument("--no-gekisou-aptitude", action="store_true", help="leave out the charts' Gekisou aptitude (every gekisouAptitude is null)") c.add_argument("--stats-cache", type=Path, metavar="DIR", - help="keep the charts' deck statistics in DIR: a chart whose model sources, options, master tables " - "and chart are unchanged is not measured again; DIR then holds this export's charts only") + help="reuse native simulation programs and runs in DIR across builds; " + "completed cache entries survive interrupted builds") c.add_argument("--no-bgm", action="store_true", help="do not read the BGM cue sheets (every song's bgm.length is null)") c.add_argument("--replay-dir", metavar="DIR", diff --git a/src/nnnotes/musicdata.py b/src/nnnotes/musicdata.py index ffd4623..e988cb0 100644 --- a/src/nnnotes/musicdata.py +++ b/src/nnnotes/musicdata.py @@ -26,7 +26,9 @@ import io import json import math +import sys from collections import Counter +from decimal import Decimal, localcontext from pathlib import Path from typing import Callable @@ -299,96 +301,80 @@ def skill(r: dict, max_level: dict) -> dict: # ---------------------------------------------------------------- the deck model -DECK_SEEDS = 8 # replay seed count of a chart with a luck range +DECK_SEEDS = 8 # replay seeds of a chart with a luck range +DECK_STATS_FORMAT = "ournotes-deck.chart-stats/3" class Deck: - """The deck model (the package ournotes-sim): chart statistics of a deck input document, on `workers` threads (None: the - available parallelism), with independent nominal lottery and skill probabilities. `seeds` gives the number of - replay seeds for a chart with a luck range; `aptitude` includes every chart's Gekisou aptitude and cross terms. - With a `cache` directory, a chart's statistics are kept there under the SHA-256 of - what they are a function of: the model's sources (its `sourceSha256`), the options above, the master tables and - the chart; a chart found there is not measured again.""" + """The deck model (the package ournotes-sim): native chart expectations. `seeds` selects replay examples; it does + not sample expectations. + With `cache`, the native model persists reusable simulation programs and runs. Python passes the complete + current input on every call and publishes the native cache counters separately from the music data. + """ def __init__(self, seeds: int = DECK_SEEDS, workers: int | None = None, module=None, aptitude: bool = True, cache: Path | None = None): + native_max = sys.maxsize * 2 + 1 + if not _int(seeds) or not 1 <= seeds <= native_max: + raise MusicDataError("--seeds must be a positive integer supported by the native model") + if workers is not None and (not _int(workers) or not 0 <= workers <= native_max): + raise MusicDataError("--workers must be a nonnegative integer supported by the native model (0: automatic)") if module is None: try: import ournotes_sim as module except ImportError: raise MusicDataError("the deck model (the package ournotes-sim) is not installed: install " "'nnnotes[deck]', or pass --no-deck") from None - self.module, self.seeds, self.workers = module, seeds, workers + self.module, self.seeds, self.workers = module, seeds, None if workers == 0 else workers self.aptitude = aptitude self.cache = None if cache is None else Path(cache) - self.counts: dict | None = None # {cached, measured} charts of the last stats() + self.counts: dict | None = None # native counters, or None when no cache was requested def info(self) -> dict: - """{name, version, source, commit, sourceSha256, format} of the deck model.""" - i = self.module.info() - return {k: i[k] for k in ("name", "version", "source", "commit", "sourceSha256", "format")} + """The identity and statistics format of the installed model.""" + info = self.module.info() + return {k: info[k] for k in ("name", "version", "source", "commit", "sourceSha256", "format")} - def _measure(self, deck_input: dict) -> dict: + def stats(self, deck_input: dict) -> dict: + """The current chart-statistics document, preserving every number exactly as the model writes it.""" + info = self.info() + if info["format"] != DECK_STATS_FORMAT: + raise MusicDataError(f"deck model: format {info['format']!r} is not supported; install the ournotes-sim " + f"release the extra deck pins ({DECK_STATS_FORMAT})") + data = deckdata.encode(deck_input).decode("utf-8") try: - text = self.module.chart_stats(deckdata.encode(deck_input).decode("utf-8"), self.seeds, self.workers, - aptitude=self.aptitude) + if self.cache is None: + text = self.module.chart_stats(data, self.seeds, self.workers, aptitude=self.aptitude) + counts = None + else: + text, counts = self.module.chart_stats_cached(data, self.seeds, self.workers, aptitude=self.aptitude, + cache_dir=str(self.cache)) + if not isinstance(counts, dict): + raise MusicDataError("deck model: native cache counters are not an object") + except MusicDataError: + raise except ValueError as e: raise MusicDataError(f"deck model: {e}") from None - doc = json.loads(text, parse_float=deckdata._Num) - if doc.get("format") != self.info()["format"]: - raise MusicDataError(f"deck model: wrote {doc.get('format')!r}, expected {self.info()['format']!r}") - return doc - - def cache_keys(self, deck_input: dict) -> list[str]: - """The cache key of each chart of a deck input document, in chart order.""" - info = self.info() - base = {"sourceSha256": info["sourceSha256"], "format": info["format"], "seeds": self.seeds, - "aptitude": self.aptitude, - "master": hashlib.sha256(deckdata.encode(deck_input["master"])).hexdigest()} - return [hashlib.sha256(deckdata.encode({**base, "chart": hashlib.sha256(deckdata.encode(c)).hexdigest()})) - .hexdigest() for c in deck_input["charts"]] - - def _cached_stats(self, deck_input: dict) -> dict: - """stats() through the cache: measure the charts it lacks, add them, then keep only this document's charts.""" - from .cache import write_atomic - keys = self.cache_keys(deck_input) - found = {} - for k in keys: - path = self.cache / f"{k}.json" - if path.is_file(): - found[k] = json.loads(path.read_bytes(), parse_float=deckdata._Num) - missing = [c for c, k in zip(deck_input["charts"], keys) if k not in found] - doc = self._measure({**deck_input, "charts": missing}) - if len(doc.get("charts") or []) != len(missing): + try: + doc = json.loads(text, parse_float=deckdata._Num) + except (TypeError, ValueError): + raise MusicDataError("deck model: the statistics are not a JSON document") from None + if not isinstance(doc, dict) or doc.get("format") != DECK_STATS_FORMAT: + wrote = doc.get("format") if isinstance(doc, dict) else type(doc).__name__ + raise MusicDataError(f"deck model: wrote {wrote!r}, expected {DECK_STATS_FORMAT!r}") + charts = doc.get("charts") + asked = [chart["scoreId"] for chart in deck_input["charts"]] + if (not isinstance(charts, list) or any(not isinstance(chart, dict) or not _int(chart.get("scoreId")) + for chart in charts) + or [chart["scoreId"] for chart in charts] != asked): raise MusicDataError("deck model: the charts measured differ from the charts asked for") - self.cache.mkdir(parents=True, exist_ok=True) - measured = iter(doc["charts"]) - for k in keys: - if k not in found: - found[k] = next(measured) - write_atomic(self.cache / f"{k}.json", deckdata.encode(found[k])) - for path in self.cache.glob("*.json"): - if path.stem not in found: - path.unlink() - doc["charts"] = [found[k] for k in keys] - self.counts = {"cached": len(keys) - len(missing), "measured": len(missing)} - return doc - - def stats(self, deck_input: dict) -> dict: - """The chart statistics document of a deck input document (deckdata.build), its numbers as written. With a - cache, the cache then holds exactly this document's charts.""" - if self.cache is None: - doc = self._measure(deck_input) - self.counts = {"cached": 0, "measured": len(deck_input["charts"])} - else: - doc = self._cached_stats(deck_input) + self.counts = counts return doc # the keys of a chart's statistics carried by `deck` (the others are checked against the chart facts) DECK_CHART_KEYS = ("convertedNoteCount", "skip", "events", "positions", "ranges", "justNotes", "expectation", - "replaySeeds", "offSeeds", - "unplayable", "gekisouAptitude") + "replaySeeds", "offSeeds", "unplayable", "gekisouAptitude") RANKS = 5 # the ranks of a Gekisou range (a Gekisou live has up to five players) GEKISOU_RANGES = 3 # the Gekisou ranges of a live (its first three fevers) @@ -416,12 +402,6 @@ def rank_bonus_percents(rows: list[dict], missions) -> list[list]: return out -def _trunc_percent(score: int, percent: int) -> int: - """trunc(score * percent / 100), the rank bonus of a range score.""" - p = score * percent - return p // 100 if p >= 0 else -(-p // 100) - - def _int(v) -> bool: return isinstance(v, int) and not isinstance(v, bool) @@ -429,7 +409,7 @@ def _int(v) -> bool: def _numbers(v, n: int) -> bool: """Whether v is a list of n numbers (as the deck model writes them).""" return isinstance(v, list) and len(v) == n and all( - isinstance(x, (int, deckdata._Num)) and not isinstance(x, bool) for x in v) + _number(x) for x in v) def _check_bound(c, what: str, where: str) -> None: @@ -477,95 +457,133 @@ def eff(x: int) -> int: def chart_seeds(stats: dict, seeds: int) -> list[int]: - """The chart's replay seeds: none when unplayable with Gekisou, 0 without a luck range, otherwise the first - `seeds` published seeds. Nominal expectation measurements are independent of this list.""" + """Replay examples: none when unplayable, seed 0 without luck, otherwise the published prefix.""" if stats.get("unplayable") is not None: return [] return published_seeds(seeds) if any(r["mission"] == MISSION_LUCK for r in stats["ranges"]) else [0] -def _check_interval_bound(c, what: str, where: str) -> None: - """The largest separation of the expected and predicted intervals is within the stated bound.""" - if not isinstance(c, dict) or any(k not in c for k in CHECK_KEYS): - raise MusicDataError(f"{where}: {what}: no expectation check") - if not (_estimate(c["expected"]) and _estimate(c["predicted"]) and _number(c["bound"]) - and float(c["bound"]) >= 0): - raise MusicDataError(f"{where}: {what}: invalid expected, predicted or bound") - error = abs(float(c["expected"][0]) - float(c["predicted"][0])) + \ - float(c["expected"][1]) + float(c["predicted"][1]) - if error > float(c["bound"]) and not _same(error, float(c["bound"])): - raise MusicDataError(f"{where}: {what}: expected {c['expected']}, predicted {c['predicted']} " - f"beyond the bound {c['bound']}") - - -def _check_expectation_shapes(stats: dict, kinds: int, where: str) -> None: - """Expected intervals and their array dimensions, deterministic counters and validation decks.""" - positions, n = stats["positions"], len(stats["ranges"]) - expected = stats.get("expectation") - if stats.get("unplayable") is not None: - if expected is not None: - raise MusicDataError(f"{where}: an expectation on an unplayable Gekisou chart") - else: - if not isinstance(expected, dict): - raise MusicDataError(f"{where}: no nominal expectation") - if not all(_estimate(expected.get(k)) for k in ("score", "scorePerfect")): - raise MusicDataError(f"{where}: expectation scores are not [center, half-width]") - w = expected.get("weights") - if not (isinstance(w, list) and len(w) == kinds and all(isinstance(x, list) and len(x) == positions - and all(_estimate(y) for y in x) for x in w)): - raise MusicDataError(f"{where}: expectation weights are not [kind][position] intervals") - rs = expected.get("ranges") - if not isinstance(rs, list) or len(rs) != n: - raise MusicDataError(f"{where}: expectation range results differ from the {n} ranges") - for i, r in enumerate(rs): - if not (isinstance(r, dict) and all(_estimate(r.get(k)) for k in - ("rangeScore", "rangeScorePerfect", "rankBonus", "rankBonusPerfect", "luckPoints")) - and _int(r.get("maxCombo")) and _int(r.get("justCount")) - and isinstance(r.get("lotResults"), list) and len(r["lotResults"]) == 4 - and all(_estimate(x) for x in r["lotResults"])): - raise MusicDataError(f"{where}: expectation range {i}: invalid scores, counters or lottery results") - rw = expected.get("rangeWeights") - if rw is not None and not (isinstance(rw, list) and len(rw) == kinds and all( - k is None or (isinstance(k, list) and len(k) == positions and all(isinstance(x, list) and len(x) == n - and all(_estimate(y) for y in x) - for x in k)) for k in rw)): - raise MusicDataError(f"{where}: expectation rangeWeights are not [kind][position][range] intervals") - for name, c in (("the check deck", expected.get("check")), ("rank check", expected.get("rankCheck"))): - if c is None and name == "rank check": - continue - _check_interval_bound(c, name, where) - if not (isinstance(c["deck"], list) and len(c["deck"]) == positions and all( - x is None or isinstance(x, list) and len(x) == 2 and _int(x[0]) and 0 <= x[0] < kinds - and _int(x[1]) for x in c["deck"])): - raise MusicDataError(f"{where}: {name}: invalid check deck") - ranks = c["ranks"] - if name == "rank check": - if not (isinstance(ranks, list) and len(ranks) == n - and all(_int(r) and 1 <= r <= RANKS for r in ranks)): - raise MusicDataError(f"{where}: rank check ranks {ranks!r}") - elif ranks is not None: - raise MusicDataError(f"{where}: the baseline check deck has fixed rank arrivals") +def _check_deck_slots(deck, positions: int, allowed: set[int], where: str) -> None: + if not (isinstance(deck, list) and len(deck) == positions and all( + slot is None or (isinstance(slot, list) and len(slot) == 2 and _int(slot[0]) and slot[0] in allowed + and _int(slot[1])) for slot in deck)): + raise MusicDataError(f"{where}: check deck is not [kind, value] or null per position") + + +def _check_expectation_check(check, positions: int, allowed: set[int], ranges: int, where: str, *, + ranked: bool = False, linear: bool = True) -> None: + if not (isinstance(check, dict) and all(key in check for key in CHECK_KEYS) + and _estimate(check["expected"]) and _estimate(check["predicted"]) + and _number(check["bound"]) and float(check["bound"]) >= 0): + raise MusicDataError(f"{where}: invalid expectation check (expected, predicted and bound)") + _check_deck_slots(check["deck"], positions, allowed, where) + ranks = check["ranks"] + if ranked: + if not (isinstance(ranks, list) and len(ranks) == ranges + and all(_int(rank) and 1 <= rank <= RANKS for rank in ranks) + and (linear or all(rank == 1 for rank in ranks))): + raise MusicDataError(f"{where}: check ranks {ranks!r}") + elif ranks is not None: + raise MusicDataError(f"{where}: an unranked expectation check has ranks {ranks!r}") + with localcontext() as context: + context.prec = 800 # exact arithmetic for the finite binary64 decimal tokens + expected, predicted = check["expected"], check["predicted"] + error = abs(Decimal(expected[0]) - Decimal(predicted[0])) + Decimal(expected[1]) + Decimal(predicted[1]) + # Each of the two serialized score intervals rounds its center to millipoints and expands its radius + # outwards. This can add at most four millipoints to the pre-serialization check's endpoint distance. + if error > Decimal(check["bound"]) + Decimal("0.00400001"): + raise MusicDataError(f"{where}: expectation check exceeds its bound {check['bound']}: {error}") + + +def _check_expectation(stats: dict, kinds: int, where: str) -> None: + """Validate /3 expectations and the independent deterministic Free Live measurements.""" + positions, infos = stats.get("positions"), stats.get("ranges") + if not _int(positions) or positions < 0: + raise MusicDataError(f"{where}: invalid number of performance positions") + if not (isinstance(infos, list) and all(isinstance(row, dict) and _int(row.get("mission")) for row in infos)): + raise MusicDataError(f"{where}: invalid Gekisou range descriptions") + if not (all(_int(stats.get(key)) and stats[key] >= 0 for key in ("judgedNotes", "justNotes")) + and stats["justNotes"] <= stats["judgedNotes"]): + raise MusicDataError(f"{where}: invalid chart judgement counts") + n = len(infos) off = stats.get("offSeeds") - if not isinstance(off, list) or len(off) != 1: + if not (isinstance(off, list) and len(off) == 1 and isinstance(off[0], dict) + and _int(off[0].get("seed")) and off[0]["seed"] == 0 and _int(off[0].get("score"))): raise MusicDataError(f"{where}: the deck model gives no Gekisou off statistics (offSeeds)") - for seed in off: - w = seed.get("weights") - if not (isinstance(w, list) and len(w) == kinds and all(x is None or _numbers(x, positions) for x in w)): - raise MusicDataError(f"{where}: Gekisou off: weights are not [kind][position]") - _check_bound(seed["check"], "Gekisou off: the check deck", where) + weights = off[0].get("weights") + if not (isinstance(weights, list) and len(weights) == kinds + and all(row is None or _numbers(row, positions) for row in weights)): + raise MusicDataError(f"{where}: Gekisou off: weights are not [kind][position]") + _check_bound(off[0].get("check"), "Gekisou off: the check deck", where) + _check_deck_slots(off[0]["check"].get("deck"), positions, + {i for i, row in enumerate(weights) if row is not None}, f"{where}: Gekisou off") + if "expectation" not in stats: + raise MusicDataError(f"{where}: no Gekisou expectation field") + expected = stats["expectation"] + if stats.get("unplayable") is not None: + if expected is not None: + raise MusicDataError(f"{where}: an unplayable Gekisou chart has an expectation") + return + if not isinstance(expected, dict): + raise MusicDataError(f"{where}: no Gekisou expectation") + if any(key not in expected for key in ("rangeWeights", "rankCheck")): + raise MusicDataError(f"{where}: no expectation rangeWeights or rankCheck field") + if not all(_estimate(expected.get(key)) for key in ("score", "scorePerfect")): + raise MusicDataError(f"{where}: expected scores are not [center, outward radius]") + ranges = expected.get("ranges") + if not (isinstance(ranges, list) and len(ranges) == n and all(isinstance(row, dict) for row in ranges)): + raise MusicDataError(f"{where}: expectation ranges do not match the chart's ranges") + for i, (row, info) in enumerate(zip(ranges, stats["ranges"])): + at = f"{where}: expectation range {i}" + if not all(_estimate(row.get(key)) for key in EXPECTED_RANGE_ESTIMATES): + raise MusicDataError(f"{at}: a range score or luck points is not [center, outward radius]") + if not all(_int(row.get(key)) and row[key] >= 0 for key in ("maxCombo", "justCount")): + raise MusicDataError(f"{at}: invalid judgement counters") + lots = row.get("lotResults") + if not (isinstance(lots, list) and len(lots) == 4 and all(_estimate(value) for value in lots)): + raise MusicDataError(f"{at}: lottery results are not four [center, outward radius] estimates") + if info["mission"] != MISSION_LUCK and any(not _consistent(value, [[0, 0]]) for value in [row["luckPoints"], *lots]): + raise MusicDataError(f"{at}: lottery results or luck points outside a luck range") + if stats["justNotes"] == 0 and any(not _consistent(row[a], [row[b]]) for a, b in ( + ("rangeScore", "rangeScorePerfect"), ("rankBonus", "rankBonusPerfect"))): + raise MusicDataError(f"{at}: a chart without Just notes differs on its Perfect play") + if stats["justNotes"] == 0 and not _consistent(expected["score"], [expected["scorePerfect"]]): + raise MusicDataError(f"{where}: a chart without Just notes differs on its Perfect play") + weights, rw = expected.get("weights"), expected.get("rangeWeights") + if not (isinstance(weights, list) and len(weights) == kinds + and all(isinstance(row, list) and len(row) == positions and all(_estimate(v) for v in row) + for row in weights)): + raise MusicDataError(f"{where}: expectation weights are not [kind][position] of estimates") + if rw is not None and not (isinstance(rw, list) and len(rw) == kinds and all( + kind is None or (isinstance(kind, list) and len(kind) == positions and all( + isinstance(row, list) and len(row) == n and all(_estimate(value) for value in row) for row in kind)) + for kind in rw)): + raise MusicDataError(f"{where}: expectation rangeWeights are not [kind][position][range] of estimates") + _check_expectation_check(expected.get("check"), positions, set(range(kinds)), n, f"{where}: expectation") + check, rank_check = expected["check"], expected.get("rankCheck") + can_rank = rw is not None and n > 0 and all(slot is None or rw[slot[0]] is not None for slot in check["deck"]) + if can_rank != (rank_check is not None): + raise MusicDataError(f"{where}: rankCheck does not match the range-weight domain") + if rank_check is not None: + _check_expectation_check(rank_check, positions, set(range(kinds)), n, + f"{where}: rank expectation", ranked=True) + if rank_check["deck"] != check["deck"]: + raise MusicDataError(f"{where}: rankCheck does not use the expectation's check deck") + # ---------------------------------------------------------------- the Gekisou aptitude PLAIN_EFFECT_TYPE, PLAIN_MS = 2000, 5000 # the plain kind: score up on the whole deck for 5 s, nothing else CONDITION_MEMBER_TARGET = 5000 # the skill condition on the member a snap's support skill is paired with APTITUDE_KEYS = ("plainKind", "host", "law", "shapes") +EXPECTATION_LAW = "independent nominal lottery and skill probabilities" SHAPE_KEYS = ("id", "source", "mission", "bandCondition", "effects", "skills") FACTOR_COUNTS = ("judgedNotes", "justNotes", "perfectNotes", "tailNotes", "comboAtStart") -VARIANT_MEANS = ("score", "scorePerfect", "tail", "tailPerfect", "converted") -RANGE_MEANS = ("rangeScore", "rankBonus", "rankBonusPerfect", "rangeScorePerfect", "maxCombo", "justCount", "luckPoints") +VARIANT_ESTIMATES = ("score", "scorePerfect", "tail", "tailPerfect", "converted") +EXPECTED_RANGE_ESTIMATES = ("rangeScore", "rankBonus", "rankBonusPerfect", "rangeScorePerfect", "luckPoints") +RANGE_ESTIMATES = (*EXPECTED_RANGE_ESTIMATES, "maxCombo", "justCount") CHECK_KEYS = ("ranks", "deck", "expected", "predicted", "bound") CONDITION_GROUPS = (("trigger", "_skillTriggerConditionGroup"), ("condition", "_skillConditionGroup"), ("release", "_skillReleaseConditionGroup"), ("reset", "_effectExecuteLimitResetConditionGroup")) -MEAN_TOLERANCE = 1e-9 # relative: a mean against its recomputation (binary64 sums) def plain_kind(kinds: list[dict]) -> int | None: @@ -591,13 +609,20 @@ def _number(v) -> bool: return False -def _estimate(v) -> bool: - """Whether v is `[center, half-width]`: two finite numbers, the radius not negative.""" - return isinstance(v, list) and len(v) == 2 and all(_number(x) for x in v) and float(v[1]) >= 0 +def _estimate(value) -> bool: + """A finite [center, outward interval half-width], including its two finite endpoints.""" + return (isinstance(value, list) and len(value) == 2 and all(_number(v) for v in value) + and float(value[1]) >= 0 and math.isfinite(float(value[0]) - float(value[1])) + and math.isfinite(float(value[0]) + float(value[1]))) -def _same(a: float, b: float, scale: float = 1.0) -> bool: - return abs(a - b) <= MEAN_TOLERANCE * max(1.0, abs(scale), abs(a), abs(b)) +def _consistent(value, terms) -> bool: + """Whether an estimate and a sum of estimates have a common value; radii are bounds, never sample errors.""" + with localcontext() as context: + context.prec = 800 + center = sum((Decimal(term[0]) for term in terms), Decimal(0)) + radius = sum((Decimal(term[1]) for term in terms), Decimal(0)) + return abs(Decimal(value[0]) - center) <= Decimal(value[1]) + radius def _f32(v) -> float: @@ -711,7 +736,7 @@ def _effects_equal(got, want) -> bool: def check_header(self, info, kinds: list[dict], model: dict) -> dict | None: """`deck.gekisouAptitude` after checking it (None when the deck model leaves the aptitude out): the plain kind - of `kinds`, a description of the host (and `model.gekisouAptitude`), the seed rule, and the shape table: + of `kinds`, a description of the host (and `model.gekisouAptitude`), the expectation law and shape table: ids 0..n-1, each shape's skills in the catalog (a member card's skill at its highest level, a snap's support skill at its highest rank level) with the shape's mission, their effect rows as the master data has them, their member targets and bands, and every skill and level of the catalog in exactly one shape.""" @@ -728,8 +753,8 @@ def check_header(self, info, kinds: list[dict], model: dict) -> dict | None: raise MusicDataError(f"{where}: the support skills' host is not described (host)") if not (isinstance(model.get("gekisouAptitude"), str) and model["gekisouAptitude"]): raise MusicDataError("deck model: the model does not describe the aptitude (model.gekisouAptitude)") - if not isinstance(info["law"], str) or not info["law"]: - raise MusicDataError(f"{where}: the nominal probability law is not described (law)") + if info["law"] != EXPECTATION_LAW: + raise MusicDataError(f"{where}: unsupported expectation law {info['law']!r}") shapes = info["shapes"] if not isinstance(shapes, list) or [s.get("id") if isinstance(s, dict) else s for s in shapes] != list( range(len(shapes))): @@ -780,118 +805,121 @@ def check_header(self, info, kinds: list[dict], model: dict) -> dict | None: self.header, self.shapes = info, shapes return {k: info[k] for k in APTITUDE_KEYS} - # ------------------------------------------------ a chart's aptitude def chart(self, where: str, stats: dict, plain: int | None) -> dict | None: - """A chart's `gekisouAptitude` after checking it: null exactly when the aptitude is left out, the master data - has no shape, the chart is unplayable with Gekisou or has no Gekisou range; else a factor per range and a - variant per shape of the chart's missions (both band match results of a shape with a band condition), each - with its intervals shaped and consistent (the best and Perfect tails) and its check within - its bound.""" - apt = stats.get("gekisouAptitude") - if self.header is None or not self.shapes or stats.get("unplayable") is not None or not stats["ranges"]: - if apt is not None: - why = ("the aptitude left out" if self.header is None else "no shape" if not self.shapes - else "a chart unplayable with Gekisou" if stats.get("unplayable") is not None - else "a chart without Gekisou ranges") - raise MusicDataError(f"{where}: the deck model gives a Gekisou aptitude with {why}") - return None - where = f"{where}: Gekisou aptitude" - if not (isinstance(apt, dict) and isinstance(apt.get("factors"), list) and isinstance(apt.get("variants"), - list)): - raise MusicDataError(f"{where}: none (gekisouAptitude {{factors, variants}})") - ranges, base = stats["ranges"], stats["expectation"] - self._factors(where, apt["factors"], stats) - missions = {r["mission"] for r in ranges} - want = [(s["id"], b) for s in self.shapes if s["mission"] == MISSION_ALL or s["mission"] in missions - for b in ((True, False) if s["bandCondition"] else (None,))] - if any(not isinstance(v, dict) or not _int(v.get("shape")) - or v.get("bandMatch") is not None and not isinstance(v.get("bandMatch"), bool) - or "bandMatch" not in v for v in apt["variants"]): - raise MusicDataError(f"{where}: invalid shape or bandMatch") - got = [(v.get("shape"), v.get("bandMatch")) if isinstance(v, dict) else v for v in apt["variants"]] - if got != want: - raise MusicDataError(f"{where}: variants (shape, bandMatch) {got!r}, the shapes of the chart's missions " - f"{sorted(missions)} give {want!r}") - for v in apt["variants"]: - self._variant(f"{where}: shape {v['shape']} band match {v['bandMatch']!r}", v, stats, base, plain) - return {"factors": apt["factors"], "variants": apt["variants"]} - - def _factors(self, where: str, factors: list, stats: dict) -> None: - ranges, base = stats["ranges"], stats["expectation"] - if len(factors) != len(ranges): - raise MusicDataError(f"{where}: {len(factors)} factors for {len(ranges)} ranges") - just = 0 - for j, (f, r) in enumerate(zip(factors, ranges)): - at = f"{where}: range {j} factors" - if not (isinstance(f, dict) and all(_int(f.get(k)) and f[k] >= 0 for k in FACTOR_COUNTS) - and _estimate(f.get("lotteries"))): - raise MusicDataError(f"{at}: not {{{', '.join(FACTOR_COUNTS)}, lotteries}}: {f!r}") - if r["mission"] != MISSION_JUST and (f["justNotes"], f["perfectNotes"]) != (0, 0): - raise MusicDataError(f"{at}: {f['justNotes']} Just and {f['perfectNotes']} Perfect notes outside a " - f"Just count range") - if f["justNotes"] + f["perfectNotes"] > f["judgedNotes"] or f["comboAtStart"] > stats["judgedNotes"]: - raise MusicDataError(f"{at}: note counts {f!r} beyond the range's or the chart's") - just += f["justNotes"] - estimates = base["ranges"][j]["lotResults"] - center = sum(float(x[0]) for x in estimates) if r["mission"] == MISSION_LUCK else 0.0 - radius = sum(float(x[1]) for x in estimates) if r["mission"] == MISSION_LUCK else 0.0 - if abs(float(f["lotteries"][0]) - center) > float(f["lotteries"][1]) + radius + MEAN_TOLERANCE: - raise MusicDataError(f"{at}: lotteries {f['lotteries']}, expected counts give [{center}, {radius}]") - if just > stats["justNotes"]: - raise MusicDataError(f"{where}: {just} Just notes in the ranges, {stats['justNotes']} on the play") - - def _variant(self, at: str, v: dict, stats: dict, base: dict, plain: int | None) -> None: - n, positions = len(stats["ranges"]), stats["positions"] - pairs = [(k, v.get(k)) for k in VARIANT_MEANS] - rs = v.get("ranges") - if not (isinstance(rs, list) and len(rs) == n and all(isinstance(r, dict) for r in rs)): - raise MusicDataError(f"{at}: invalid range results for {n} ranges") - pairs += [(f"ranges[{j}].{k}", r.get(k)) for j, r in enumerate(rs) for k in RANGE_MEANS] - w, rw = v.get("weights"), v.get("rangeWeights") - linear = base.get("rangeWeights") is not None - if plain is None: - if w is not None or rw is not None: - raise MusicDataError(f"{at}: weights without a plain kind") - else: - if not isinstance(w, list) or len(w) != positions: - raise MusicDataError(f"{at}: weights are not [position] intervals") - pairs += [(f"weights[{k}]", x) for k, x in enumerate(w)] - if (rw is not None) != linear: - raise MusicDataError(f"{at}: rangeWeights differ from the baseline's range weight domain") - if rw is not None: - if not (isinstance(rw, list) and len(rw) == positions - and all(isinstance(x, list) and len(x) == n for x in rw)): - raise MusicDataError(f"{at}: rangeWeights are not [position][range] intervals") - pairs += [(f"rangeWeights[{k}][{j}]", y) for k, x in enumerate(rw) for j, y in enumerate(x)] - for name, x in pairs: - if not _estimate(x): - raise MusicDataError(f"{at}: {name} {x!r} is not [center, half-width]") - if name == "converted" or name.endswith(("maxCombo", "justCount")): - if float(x[1]) != 0 or not float(x[0]).is_integer(): - raise MusicDataError(f"{at}: {name} must be an exact integer counter") - for score, tail, range_score, rank_bonus in ( - ("score", "tail", "rangeScore", "rankBonus"), - ("scorePerfect", "tailPerfect", "rangeScorePerfect", "rankBonusPerfect")): - inside = sum(float(r[k][0]) for r in rs for k in (range_score, rank_bonus)) - # Each point center is serialized to millipoints; every field encloses its own rounding. - tolerance = (2 * n + 2) * 0.0005 + MEAN_TOLERANCE * max(1, abs(float(v[score][0]))) - if abs(float(v[tail][0]) - (float(v[score][0]) - inside)) > tolerance: - raise MusicDataError(f"{at}: {tail} is not {score} minus the ranges' scores and rank bonuses") - c = v.get("check") - _check_interval_bound(c, "the check deck", at) - if not (isinstance(c["ranks"], list) and len(c["ranks"]) == n - and all(_int(r) and 1 <= r <= RANKS for r in c["ranks"]) - and (linear or all(r == 1 for r in c["ranks"]))): - raise MusicDataError(f"{at}: check ranks {c['ranks']!r}") - if not (isinstance(c["deck"], list) and len(c["deck"]) == positions and all( - x is None or (plain is not None and isinstance(x, list) and len(x) == 2 and x[0] == plain - and _int(x[1])) for x in c["deck"])): - raise MusicDataError(f"{at}: check deck {c['deck']!r} is not [plain kind ({plain}), value] or null per position") + return validate_aptitude_statistics(self.header, stats, where) + + +def validate_aptitude_statistics(header: dict | None, stats: dict, where: str) -> dict | None: + """A chart's `gekisouAptitude` after checking it: null exactly when the aptitude is left out, the master data + has no shape, the chart is unplayable with Gekisou or has no Gekisou range; else a factor per range and a + variant per shape of the chart's missions (both band match results of a shape with a band condition), each + with finite outward estimates, consistent tail identities and its expectation check within + its bound.""" + shapes = header["shapes"] if header is not None else [] + plain = header["plainKind"] if header is not None else None + if "gekisouAptitude" not in stats: + raise MusicDataError(f"{where}: no gekisouAptitude field") + apt = stats["gekisouAptitude"] + if header is None or not shapes or stats.get("unplayable") is not None or not stats["ranges"]: + if apt is not None: + why = ("the aptitude left out" if header is None else "no shape" if not shapes + else "a chart unplayable with Gekisou" if stats.get("unplayable") is not None + else "a chart without Gekisou ranges") + raise MusicDataError(f"{where}: the deck model gives a Gekisou aptitude with {why}") + return None + where = f"{where}: Gekisou aptitude" + if not (isinstance(apt, dict) and isinstance(apt.get("factors"), list) and isinstance(apt.get("variants"), + list)): + raise MusicDataError(f"{where}: none (gekisouAptitude {{factors, variants}})") + ranges, base = stats["ranges"], stats["expectation"] + _validate_aptitude_factors(where, apt["factors"], stats) + missions = {r["mission"] for r in ranges} + want = [(s["id"], b) for s in shapes if s["mission"] == MISSION_ALL or s["mission"] in missions + for b in ((True, False) if s["bandCondition"] else (None,))] + if any(not isinstance(v, dict) or not _int(v.get("shape")) + or v.get("bandMatch") is not None and not isinstance(v.get("bandMatch"), bool) + or "bandMatch" not in v for v in apt["variants"]): + raise MusicDataError(f"{where}: invalid shape or bandMatch") + got = [(v.get("shape"), v.get("bandMatch")) if isinstance(v, dict) else v for v in apt["variants"]] + if got != want: + raise MusicDataError(f"{where}: variants (shape, bandMatch) {got!r}, the shapes of the chart's missions " + f"{sorted(missions)} give {want!r}") + for v in apt["variants"]: + _validate_aptitude_variant(f"{where}: shape {v['shape']} band match {v['bandMatch']!r}", v, stats, base, plain, shapes) + return {"factors": apt["factors"], "variants": apt["variants"]} + + +def _validate_aptitude_factors(where: str, factors: list, stats: dict) -> None: + ranges, base = stats["ranges"], stats["expectation"] + if len(factors) != len(ranges): + raise MusicDataError(f"{where}: {len(factors)} factors for {len(ranges)} ranges") + for j, (f, r) in enumerate(zip(factors, ranges)): + at = f"{where}: range {j} factors" + if not (isinstance(f, dict) and all(_int(f.get(k)) and f[k] >= 0 for k in FACTOR_COUNTS) + and _estimate(f.get("lotteries"))): + raise MusicDataError(f"{at}: not {{{', '.join(FACTOR_COUNTS)}, lotteries}}: {f!r}") + if r["mission"] != MISSION_JUST and (f["justNotes"], f["perfectNotes"]) != (0, 0): + raise MusicDataError(f"{at}: {f['justNotes']} Just and {f['perfectNotes']} Perfect notes outside a " + f"Just count range") + if f["justNotes"] + f["perfectNotes"] > f["judgedNotes"] or f["comboAtStart"] > stats["judgedNotes"]: + raise MusicDataError(f"{at}: note counts {f!r} beyond the range's or the chart's") + lotteries = base["ranges"][j]["lotResults"] + if not _consistent(f["lotteries"], lotteries): + raise MusicDataError(f"{at}: lotteries {f['lotteries']} do not enclose the sum of expected results") + if f["justNotes"] > stats["justNotes"]: + raise MusicDataError(f"{at}: more Just notes than the chart's play") + + +def _validate_aptitude_variant(at: str, value: dict, stats: dict, base: dict, plain: int | None, shapes: list) -> None: + n, positions = len(stats["ranges"]), stats["positions"] + effects = shapes[value["shape"]]["effects"] + reads_rank = any( + any(condition["type"] == 7012 for key, _ in CONDITION_GROUPS for group in effect[key] + for condition in group) or (effect["cumulative"] or {}).get("type") == 7012 for effect in effects) + linear = base.get("rangeWeights") is not None and not reads_rank + pairs = [(key, value.get(key)) for key in VARIANT_ESTIMATES] + ranges = value.get("ranges") + if not (isinstance(ranges, list) and len(ranges) == n and all(isinstance(row, dict) for row in ranges)): + raise MusicDataError(f"{at}: range results do not match the chart's {n} ranges") + pairs += [(f"ranges[{i}].{key}", row.get(key)) for i, row in enumerate(ranges) for key in RANGE_ESTIMATES] + if "weights" not in value or "rangeWeights" not in value: + raise MusicDataError(f"{at}: no weights or rangeWeights field") + weights, rw = value["weights"], value["rangeWeights"] + if plain is None: + if weights is not None or rw is not None: + raise MusicDataError(f"{at}: weights without a plain kind") + else: + if not (isinstance(weights, list) and len(weights) == positions): + raise MusicDataError(f"{at}: weights are not [position] of estimates") + pairs += [(f"weights[{i}]", weight) for i, weight in enumerate(weights)] + if (rw is not None) != linear: + raise MusicDataError(f"{at}: rangeWeights do not match the chart and shape's rank-linear domain") + if rw is not None: + if not (isinstance(rw, list) and len(rw) == positions + and all(isinstance(row, list) and len(row) == n for row in rw)): + raise MusicDataError(f"{at}: rangeWeights are not [position][range] of estimates") + pairs += [(f"rangeWeights[{i}][{j}]", estimate) for i, row in enumerate(rw) + for j, estimate in enumerate(row)] + for name, estimate in pairs: + if not _estimate(estimate): + raise MusicDataError(f"{at}: {name} {estimate!r} is not [center, outward radius]") + for name, estimate in [("converted", value["converted"])] + [ + (f"ranges[{i}].{key}", row[key]) for i, row in enumerate(ranges) for key in ("maxCombo", "justCount")]: + if float(estimate[1]) != 0 or not float(estimate[0]).is_integer(): + raise MusicDataError(f"{at}: {name} is not a deterministic integer counter") + for score, tail, range_score, bonus in (("score", "tail", "rangeScore", "rankBonus"), + ("scorePerfect", "tailPerfect", "rangeScorePerfect", "rankBonusPerfect")): + terms = [value[tail]] + [row[key] for row in ranges for key in (range_score, bonus)] + if not _consistent(value[score], terms): + raise MusicDataError(f"{at}: {tail} interval is inconsistent with {score} and the range increments") + _check_expectation_check(value.get("check"), positions, set() if plain is None else {plain}, n, + f"{at}: expectation", ranked=True, linear=linear) + def chart_deck(song: dict, chart: dict, stats: dict, kinds: int, percents: list[list], seeds: int, aptitude: Aptitude, plain: int | None) -> dict: """A chart's `deck` from its statistics, after checking them against the song, the chart facts, the song's rank - bonus percentages (`percents`: rank_bonus_percents), the replay seed set (chart_seeds) + bonus percentages (`percents`: rank_bonus_percents), and the replay seed set (`seeds` with a luck range) and its Gekisou aptitude (Aptitude.chart, `plain` the plain kind), `kinds` the number of score-up kinds.""" where = f"chart {chart['scoreId']} ({song['id']} {chart['difficulty']})" checks = ( @@ -916,15 +944,10 @@ def chart_deck(song: dict, chart: dict, stats: dict, kinds: int, percents: list[ raise MusicDataError(f"{where}: range {i}: the deck model's rank bonus percentages {got!r} " f"({r['rankBonusPercent']!r}) differ from MasterLiveGekisouRankingScoreBonus {want!r}") want = chart_seeds(stats, seeds) - if stats.get("replaySeeds") != want: - raise MusicDataError(f"{where}: replay seeds {stats.get('replaySeeds')!r} are not the chart's seed set {want}") - _check_expectation_shapes(stats, kinds, where) - expected = stats.get("expectation") - if expected is not None and stats["justNotes"] == 0: - pairs = [(expected["score"], expected["scorePerfect"])] + [ - (r["rangeScore"], r["rangeScorePerfect"]) for r in expected["ranges"]] - if any(abs(float(a[0]) - float(b[0])) > float(a[1]) + float(b[1]) + MEAN_TOLERANCE for a, b in pairs): - raise MusicDataError(f"{where}: a chart without Just notes scores otherwise on the Perfect play") + replay = stats.get("replaySeeds") + if not isinstance(replay, list) or not all(_int(seed) for seed in replay) or replay != want: + raise MusicDataError(f"{where}: replay seeds {replay!r} are not the chart's seed set {want}") + _check_expectation(stats, kinds, where) return dict({k: stats.get(k) for k in DECK_CHART_KEYS}, gekisouAptitude=aptitude.chart(where, stats, plain)) @@ -1111,7 +1134,8 @@ def export(out, src: deckdata.MasterSource, key, fetch: Callable[[str], bytes], replay_files = None if replay_dir is not None: from . import replaydata - replay_files, replay_manifest = replaydata.bundle(doc, replay_engine, recommend_engine) + replay_files, replay_manifest = replaydata.bundle(doc, replay_engine, recommend_engine, + label_source=replaydata.labels(tables, shas, doc["provenance"])) manifest_sha = hashlib.sha256(replay_files["manifest.json"]).hexdigest() replay_target = Path(replay_dir) / manifest_sha try: diff --git a/src/nnnotes/replaydata.py b/src/nnnotes/replaydata.py index d345b6b..a231944 100644 --- a/src/nnnotes/replaydata.py +++ b/src/nnnotes/replaydata.py @@ -3,6 +3,7 @@ import hashlib import json +import re import tarfile from pathlib import Path @@ -12,12 +13,36 @@ FORMAT = "nnnotes.replay-manifest/1" CHART_FORMAT = "nnnotes.replay-chart/1" BUILD_INFO = "build-info.json" +LABEL_FORMAT = "nnnotes.replay-labels/1" +LABEL_TABLES = ("MasterSupportSkill", "MasterSupportSkillEffect", "MasterGekisouSupportSkill", + "MasterGekisouSupportSkillEffect", "MasterText", "MasterSkillConditionSet", "MasterSkillCondition", + "MasterSkillCumulativeCondition", "MasterSkillTarget", "MasterCharacter", "MasterMemberCard", "MasterSupportCard", "MasterBand") def _json(value) -> bytes: return deckdata.encode(deckdata._value(value, "replay input")) +def labels(tables: dict, hashes: dict, provenance: dict) -> dict: + """Original names/templates, level effects and artwork IDs from the same served master tables. + + Consumers use their existing localization/description formatter. No formula or inferred growth is exported. + MasterSkillIcon is deliberately absent until the producer also records its source hash. + """ + master = provenance.get("master") or {} + recorded = master.get("tables") or {} + result = {} + for name in LABEL_TABLES: + digest = hashes.get(name) + if name not in tables or not isinstance(tables[name], list): + raise deckdata.DeckDataError(f"replay labels: missing source table {name}") + if not isinstance(digest, str) or not re.fullmatch(r"[0-9a-f]{64}", digest) or recorded.get(name, {}).get("sha256") != digest: + raise deckdata.DeckDataError(f"replay labels: {name} hash differs from master provenance") + result[name] = {"sha256": digest, "rows": tables[name]} + return {"format": LABEL_FORMAT, "region": provenance["region"], "masterVersion": master.get("version"), + "tables": result} + + def _read_package(source: Path, paths: tuple[str, ...], name: str) -> dict[str, bytes]: """Those of the files `paths` that a release package holds: a directory, or a .tar.gz archive holding the package directory.""" @@ -69,8 +94,8 @@ def _package(files: dict[str, bytes], source: Path, model: dict | None, module: return paths -def bundle(music: dict, engine_dir: Path | None = None, - recommend_dir: Path | None = None) -> tuple[dict[str, bytes], dict]: +def bundle(music: dict, engine_dir: Path | None = None, recommend_dir: Path | None = None, *, + label_source: dict | None = None) -> tuple[dict[str, bytes], dict]: """Canonical normalized --full inputs and SHA manifest. Actual ACB length is mandatory. Runtime arrays retain native enumeration order. ReplaySession.template/run in Rust produce frames and scores. @@ -101,6 +126,21 @@ def bundle(music: dict, engine_dir: Path | None = None, def resource(path: str) -> dict: return {"url": path, "sha256": hashlib.sha256(files[path]).hexdigest(), "bytes": len(files[path])} + label_ref = None + if label_source is not None: + if (label_source.get("format") != LABEL_FORMAT or label_source.get("region") != provenance.get("region") + or label_source.get("masterVersion") != (provenance.get("master") or {}).get("version")): + raise deckdata.DeckDataError("replay labels: snapshot identity differs from runtime inputs") + source_tables = label_source.get("tables") or {} + recorded = (provenance.get("master") or {}).get("tables") or {} + for name in LABEL_TABLES: + if (not isinstance(source_tables.get(name), dict) or not isinstance(source_tables[name].get("rows"), list) + or not re.fullmatch(r"[0-9a-f]{64}", str(source_tables[name].get("sha256"))) + or source_tables[name].get("sha256") != recorded.get(name, {}).get("sha256")): + raise deckdata.DeckDataError(f"replay labels: {name} differs from runtime master provenance") + files["snap-labels.json"] = _json(label_source) + label_ref = {"format": LABEL_FORMAT, **resource("snap-labels.json")} + charts = [] for record in records: sid = record["scoreId"] @@ -122,6 +162,7 @@ def resource(path: str) -> dict: recommend = {"model": model, "js": resource(js), "wasm": resource(wasm), "build": resource(build)} manifest = {"format": FORMAT, "deckData": {"format": deckdata.DECK_FORMAT, **resource("deck-data.json")}, "charts": charts, "engine": engine, + **({"snapLabels": label_ref} if label_ref else {}), **({"recommendEngine": recommend} if recommend is not None else {}), "unlistedScoreIds": [c["scoreId"] for c in source_records if str(c["scoreId"]) not in lengths], "clock": "Explicit frames from ReplaySession.template; no Python/JS scoring or scheduling"} diff --git a/tests/test_musicdata.py b/tests/test_musicdata.py index 1145214..5bf23cd 100644 --- a/tests/test_musicdata.py +++ b/tests/test_musicdata.py @@ -8,6 +8,7 @@ import re import sys import tomllib +from copy import deepcopy from pathlib import Path import pytest @@ -206,65 +207,47 @@ def shape_skill(i, level, targets=None, bands=None): {"id": 4, "source": "support", "mission": 1, "bandCondition": False, "effects": [effect_row(5)], "skills": [shape_skill(32, 5)]}, ] -NOMINAL_LAW = "independent nominal lottery and skill probabilities" - - -def expected_stats(positions, percents, missions): - ranges = [] - for j, p in enumerate(percents): - points = 101 * (j + 1) - luck = missions[j] == musicdata.MISSION_LUCK - ranges.append({"rangeScore": [points, 0], "rangeScorePerfect": [points, 0], - "rankBonus": [trunc_percent(points, p[0]), 0], - "rankBonusPerfect": [trunc_percent(points, p[0]), 0], - "maxCombo": 1, "justCount": 0, "luckPoints": [11, 0.35] if luck else [0, 0], - "lotResults": [[2, 0.01], [1, 0.01], [0.5, 0.01], [0.1, 0.01]] if luck else [[0, 0]] * 4}) - check = {"deck": [[0, 5000]] + [None] * (positions - 1), "ranks": None, - "expected": [2000, 0.25], "predicted": [2000.25, 0.1], "bound": 7} - return {"score": [1234, 0.5], "scorePerfect": [1234, 0.5], "ranges": ranges, - "weights": [[["W", 0]] * positions], "rangeWeights": [[[["W", 0]] * len(ranges)] * positions], - "check": check, "rankCheck": dict(check, ranks=[2] * len(ranges)) if ranges else None} - - -def aptitude_of(s, positions): - """Synthetic nominal intervals: known range gains and an exact seven-point tail.""" - ranges, base = s["ranges"], s["expectation"] - missions = {r["mission"] for r in ranges} - factors = [{"judgedNotes": 5, "justNotes": 0, "perfectNotes": 5 if r["mission"] == 3 else 0, "tailNotes": 1, - "comboAtStart": 0, - "lotteries": [sum(x[i] for x in base["ranges"][j]["lotResults"]) for i in range(2)]} - for j, r in enumerate(ranges)] +def expectation_check(positions, ranks=None): + return {"deck": ([[0, 5000]] + [None] * (positions - 1)) if positions else [], "ranks": ranks, + "expected": [2000, 0.001], "predicted": [2000.25, 0.001], "bound": 7.0} + + +def aptitude_of(stats, positions): + """Independently authored interval fixtures, not native numerical results or sampling estimates.""" + ranges, base = stats["ranges"], stats["expectation"] + missions = {row["mission"] for row in ranges} + factors = [{"judgedNotes": 5, "justNotes": 0, "perfectNotes": 5 if row["mission"] == 3 else 0, + "tailNotes": 1, "comboAtStart": 0, + "lotteries": [sum(value[0] for value in base["ranges"][j]["lotResults"]), + sum(value[1] for value in base["ranges"][j]["lotResults"])]} + for j, row in enumerate(ranges)] variants = [] for shape in SHAPES: if shape["mission"] != 4 and shape["mission"] not in missions: continue for band in ((True, False) if shape["bandCondition"] else (None,)): - offset, radius = (0.25, 0.5) if shape["id"] in (2, 3) or 2 in missions else (0, 0) - rs, inside, inside_p = [], 0, 0 - for j, (r, r0) in enumerate(zip(ranges, base["ranges"])): - d = 10 * (j + 1) - rb = trunc_percent(r0["rangeScore"][0] + d, r["rankBonusPercent"]) - r0["rankBonus"][0] - rbp = trunc_percent(r0["rangeScorePerfect"][0] + d, r["rankBonusPercent"]) - r0["rankBonusPerfect"][0] - rs.append({"rangeScore": [d + offset, radius], "rankBonus": [rb + offset, radius], - "rangeScorePerfect": [d + offset, radius], "rankBonusPerfect": [rbp + offset, radius], - "maxCombo": [0, 0], "justCount": [1, 0], "luckPoints": [offset, radius]}) - inside += d + rb + 2 * offset - inside_p += d + rbp + 2 * offset - variants.append({ - "shape": shape["id"], "bandMatch": band, - "score": [inside + 7, radius], "scorePerfect": [inside_p + 7, radius], "tail": [7, radius], - "tailPerfect": [7, radius], "converted": [0, 0], "ranges": rs, - "weights": [[0.01, radius / 100]] * positions, - "rangeWeights": [[[0.001, 0]] * len(ranges)] * positions if base["rangeWeights"] is not None else None, - "check": {"ranks": [2 if base["rangeWeights"] is not None else 1] * len(ranges), - "deck": [[0, 5000]] + [None] * (positions - 1), "expected": [2000, 0.25], - "predicted": [2000.25, 0.1], "bound": 7}}) + rs, inside, inside_perfect = [], 0, 0 + for j, (row, baseline) in enumerate(zip(ranges, base["ranges"])): + gain = 10 * (j + 1) + bonus = trunc_percent(baseline["rangeScore"][0] + gain, row["rankBonusPercent"]) - baseline["rankBonus"][0] + perfect_bonus = trunc_percent(baseline["rangeScorePerfect"][0] + gain, row["rankBonusPercent"]) - baseline["rankBonusPerfect"][0] + rs.append({"rangeScore": [gain, 0], "rankBonus": [bonus, 0], "rankBonusPerfect": [perfect_bonus, 0], + "rangeScorePerfect": [gain, 0], "maxCombo": [0, 0], "justCount": [1, 0], + "luckPoints": [0, 0]}) + inside += gain + bonus + inside_perfect += gain + perfect_bonus + linear = base["rangeWeights"] is not None + variants.append({"shape": shape["id"], "bandMatch": band, + "score": [inside + 7, 0], "scorePerfect": [inside_perfect + 7, 0], + "tail": [7, 0], "tailPerfect": [7, 0], "converted": [0, 0], "ranges": rs, + "weights": [[0.01, 1e-12]] * positions, + "rangeWeights": [[[0.001, 1e-12]] * len(ranges)] * positions if linear else None, + "check": expectation_check(positions, [2 if linear else 1] * len(ranges))}) return {"factors": factors, "variants": variants} class FakeDeck: - """The interface of ournotes_sim: statistics made from the deck input as the model reports them, with the Gekisou - aptitude (SHAPES; `aptitude`) unless it is left out.""" + """The /3 extension interface with synthetic expectations and independently constructed outward intervals.""" COMMIT = "7e5d84b5998d28c21541ce3f2e0a3dfb1439f4f6" SOURCE = "5" * 64 FORMAT = "ournotes-deck.chart-stats/3" @@ -280,56 +263,72 @@ def info(self): def chart_stats(self, data, seeds, workers, aptitude=True): if self.fail: raise ValueError(self.fail) + if seeds < 1: + raise ValueError("an empty seed set") doc = json.loads(data) self.inputs.append((doc, seeds, workers)) self.options.append(aptitude) assert doc["format"] == deckdata.DECK_FORMAT and list(doc["master"]) == [t for t, _ in deckdata.TABLES] - m = doc["master"] - with_aptitude = aptitude + master = doc["master"] def table(name): - return [dict(zip(m[name]["columns"], r)) for r in m[name]["rows"]] - levels = {r["_id"]: r["_musicScoreLevel"] for r in table("MasterLiveMusicScore")} - bonus = {(r["_missionPattern"], r["_count"], r["_rank"]): r["_scoreBonusPercent"] - for r in table("MasterLiveGekisouRankingScoreBonus")} + return [dict(zip(master[name]["columns"], row)) for row in master[name]["rows"]] + levels = {row["_id"]: row["_musicScoreLevel"] for row in table("MasterLiveMusicScore")} + bonuses = {(row["_missionPattern"], row["_count"], row["_rank"]): row["_scoreBonusPercent"] + for row in table("MasterLiveGekisouRankingScoreBonus")} songs = {} - for r in table("MasterLiveMusic"): - for d in musicdata.DIFFICULTIES: - if r[f"_{d}ID"]: - songs[r[f"_{d}ID"]] = (r["_id"], d, [r["_gekisouMission1"], r["_gekisouMission2"], - r["_gekisouMission3"]]) + for row in table("MasterLiveMusic"): + for difficulty in musicdata.DIFFICULTIES: + if row[f"_{difficulty}ID"]: + songs[row[f"_{difficulty}ID"]] = (row["_id"], difficulty, + [row["_gekisouMission1"], row["_gekisouMission2"], row["_gekisouMission3"]]) charts = [] - for c in doc["charts"]: - music, d, missions = songs[c["scoreId"]] - last = max(c["notes"]["timeMs"]) - fevers = list(zip(c["fevers"]["startMs"], c["fevers"]["endMs"]))[:3] - positions = min(len(c["skillEvents"]["timeMs"]), 5) + for chart in doc["charts"]: + music, difficulty, missions = songs[chart["scoreId"]] + last = max(chart["notes"]["timeMs"]) + fevers = list(zip(chart["fevers"]["startMs"], chart["fevers"]["endMs"]))[:3] + positions = min(len(chart["skillEvents"]["timeMs"]), 5) pattern = musicdata.mission_pattern(missions) - percents = [[bonus.get((pattern, i + 1, k), 0) for k in range(1, 6)] for i in range(len(fevers))] - check = {"deck": [[0, 5000]], "exact": 2000, "predicted": 2000.25, "bound": 7.0} + percents = [[bonuses.get((pattern, i + 1, rank), 0) for rank in range(1, 6)] for i in range(len(fevers))] luck = musicdata.MISSION_LUCK in missions[:len(fevers)] - - s = {"scoreId": c["scoreId"], "musicId": music, "difficulty": d, "level": levels[c["scoreId"]], - "judgedNotes": sum(score.is_judgement_note(op) for op in c["notes"]["op"]), - "convertedNoteCount": len(c["notes"]["id"]), "lastNoteMs": last, "musicLengthMs": last + 1000, - "skip": 1.5, "events": [[i % 5, t] for i, t in enumerate(c["skillEvents"]["timeMs"])], - "positions": positions, "missions": missions, - "ranges": [{"index": i, "mission": missions[i], "startMs": a, "endMs": b, - "rankBonusPercent": percents[i][0], "rankBonusPercents": percents[i]} - for i, (a, b) in enumerate(fevers)], - "justNotes": 0, - "expectation": expected_stats(positions, percents, missions), - "replaySeeds": musicdata.published_seeds(seeds) if luck else [0], - "offSeeds": [{"seed": 0, "score": 1000, "weights": [["W"] * positions], "check": dict(check)}]} - s["gekisouAptitude"] = aptitude_of(s, positions) if with_aptitude and fevers else None + expected_ranges = [{"rangeScore": [101 * (j + 1), 0], + "rankBonus": [trunc_percent(101 * (j + 1), percent[0]), 0], + "rankBonusPerfect": [trunc_percent(101 * (j + 1), percent[0]), 0], + "rangeScorePerfect": [101 * (j + 1), 0], "maxCombo": 1, "justCount": 0, + "luckPoints": [10, 0] if missions[j] == 2 else [0, 0], + "lotResults": [[1, 0], [1, 0], [0, 0], [0, 0]] if missions[j] == 2 else [[0, 0]] * 4} + for j, percent in enumerate(percents)] + expected = {"score": [1234, 0], "scorePerfect": [1234, 0], "ranges": expected_ranges, + "weights": [[["W", 0]] * positions], + "rangeWeights": [[[["W", 0]] * len(fevers)] * positions], + "check": expectation_check(positions), + "rankCheck": expectation_check(positions, [2] * len(fevers)) if fevers else None} + off_check = {"deck": ([[0, 5000]] + [None] * (positions - 1)) if positions else [], + "exact": 2000, "predicted": 2000.25, "bound": 7.0} + stats = {"scoreId": chart["scoreId"], "musicId": music, "difficulty": difficulty, + "level": levels[chart["scoreId"]], + "judgedNotes": sum(score.is_judgement_note(op) for op in chart["notes"]["op"]), + "convertedNoteCount": len(chart["notes"]["id"]), "lastNoteMs": last, "musicLengthMs": last + 1000, + "skip": 1.5, "events": [[i % 5, time] for i, time in enumerate(chart["skillEvents"]["timeMs"])], + "positions": positions, "missions": missions, + "ranges": [{"index": i, "mission": missions[i], "startMs": a, "endMs": b, + "rankBonusPercent": percents[i][0], "rankBonusPercents": percents[i]} + for i, (a, b) in enumerate(fevers)], + "justNotes": 0, "expectation": expected, + "replaySeeds": musicdata.published_seeds(seeds) if luck else [0], + "offSeeds": [{"seed": 0, "score": 1000, "weights": [["W"] * positions], "check": off_check}]} + stats["gekisouAptitude"] = aptitude_of(stats, positions) if aptitude and fevers else None if self.change: - self.change(s) - charts.append(s) - out = {"format": self.FORMAT, "source": {}, - "model": {"power": 300000, "gekisouAptitude": "each Gekisou (support) skill shape alone"}, + self.change(stats) + charts.append(stats) + out = {"format": self.FORMAT, "source": doc["provenance"], + "model": {"power": 300000, "checkPower": 1000003, "unitValue": 10000, + "expectation": "independent nominal probabilities with outward intervals", + "gekisouAptitude": "each Gekisou (support) skill shape alone"}, "gekisouAptitude": {"plainKind": 0, "host": "a synthetic member without a Gekisou skill", - "law": NOMINAL_LAW, "shapes": SHAPES} if aptitude else None, - "kinds": [{"id": 0, "effectType": 2000, "activationTimeSecond": 7.5}], "charts": charts} + "law": musicdata.EXPECTATION_LAW, "shapes": SHAPES} if aptitude else None, + "kinds": [{"id": 0, "effectType": 2000, "activationTimeSecond": 5.0, "durationMs": 5000}], + "charts": charts} out = json.loads(json.dumps(out)) if self.header: self.header(out) @@ -342,44 +341,6 @@ def export(tmp_path, rows=TABLE_ROWS, charts=CHARTS, bgm=bgm, out="music.json", deck=musicdata.Deck(module=deck, workers=3) if deck is not None else None, **kw) -def test_the_stats_cache_measures_only_charts_it_lacks(tmp_path): - cache = tmp_path / "stats" - - def run(name, deck, seeds=musicdata.DECK_SEEDS, stats_cache=cache): - (tmp_path / name).mkdir() - d = master_dir(tmp_path / name) - return musicdata.export(tmp_path / name / "music.json", deckdata.master_files(d), KEY, CHARTS.__getitem__, - bgm, **PROV, deck=musicdata.Deck(module=deck, seeds=seeds, cache=stats_cache)) - plain = run("plain", FakeDeck(), stats_cache=None) - fake = FakeDeck() - first = run("first", fake) - n = plain["charts"] - assert len(fake.inputs[0][0]["charts"]) == n and first["deckStats"] == {"cached": 0, "measured": n} - assert plain["deckStats"] == {"cached": 0, "measured": n} and len(list(cache.glob("*.json"))) == n - again = run("again", fake) - assert fake.inputs[1][0]["charts"] == [] and again["deckStats"] == {"cached": n, "measured": 0} - assert plain["sha256"] == first["sha256"] == again["sha256"] - # the key names the model's sources and the options: changing either measures every chart again, and the - # cache then holds the new statistics only - other = FakeDeck() - other.SOURCE = "6" * 64 - assert run("other", other)["deckStats"] == {"cached": 0, "measured": n} and len(list(cache.glob("*.json"))) == n - assert run("seeds", FakeDeck(), seeds=4)["deckStats"] == {"cached": 0, "measured": n} - - -def test_the_stats_cache_key_reads_the_master_tables_and_the_chart(): - deck = musicdata.Deck(module=FakeDeck()) - doc = {"master": {"MasterLiveSetting": {"columns": ["_id"], "rows": [[1]]}}, - "charts": [{"scoreId": 1, "notes": [1]}, {"scoreId": 2, "notes": [2]}]} - keys = deck.cache_keys(doc) - assert len(set(keys)) == 2 and all(re.fullmatch(r"[0-9a-f]{64}", k) for k in keys) - assert deck.cache_keys({**doc, "charts": doc["charts"][1:]}) == keys[1:] - changed = {**doc, "charts": [doc["charts"][0], {"scoreId": 2, "notes": [3]}]} - assert deck.cache_keys(changed)[0] == keys[0] and deck.cache_keys(changed)[1] != keys[1] - other = {**doc, "master": {"MasterLiveSetting": {"columns": ["_id"], "rows": [[2]]}}} - assert not set(deck.cache_keys(other)) & set(keys) - - def schema_validator(): jsonschema = pytest.importorskip("jsonschema") doc = json.loads((ROOT / "docs" / "schema" / "music-data.schema.json").read_text(encoding="utf-8")) @@ -506,22 +467,22 @@ def test_deck(tmp_path): (lambda s: s.update(difficulty="hard"), "difficulty 'hard' differs"), (lambda s: s["ranges"] and s["ranges"][0]["rankBonusPercents"].__setitem__(2, 99), "range 0: .*percentages"), (lambda s: s["ranges"] and s["ranges"][0].update(rankBonusPercent=1), "range 0: .*percentages"), - (lambda s: s["ranges"] and s["expectation"]["ranges"][0].update(rankBonus=[1, -1]), "invalid scores"), - (lambda s: s["expectation"]["ranges"].pop(), "range results differ"), + (lambda s: s["ranges"] and s["expectation"]["ranges"][0].update(rankBonus=-1), "range score or luck points"), + (lambda s: s["expectation"]["ranges"].pop(), "expectation ranges do not match"), (lambda s: s.pop("offSeeds"), "no Gekisou off statistics"), (lambda s: s["offSeeds"].append(s["offSeeds"][0]), "no Gekisou off statistics"), (lambda s: s["offSeeds"][0]["weights"].append(None), "Gekisou off: weights"), (lambda s: s["offSeeds"][0]["check"].update(exact=0), "Gekisou off: the check deck scores 0"), - (lambda s: s["expectation"].update(scorePerfect=[1, 0]), "otherwise on the Perfect play"), + (lambda s: s["expectation"].update(scorePerfect=[1, 0]), "without Just notes differs"), (lambda s: s["expectation"]["weights"][0].pop(), "weights are not"), (lambda s: s["expectation"]["weights"].append(None), "weights are not"), (lambda s: s["expectation"].update(rangeWeights=[[]]), "rangeWeights are not"), - (lambda s: s["expectation"]["check"].update(bound=0.1), "the check deck: expected"), - (lambda s: s["ranges"] and s["expectation"]["rankCheck"].update(expected=[0, 0]), "rank check: expected"), - (lambda s: s["ranges"] and s["expectation"]["rankCheck"].update(ranks=[6, 1]), "rank check ranks"), - (lambda s: s["expectation"]["ranges"][0].pop("luckPoints"), "chart 10 .*range 0: invalid scores"), - (lambda s: s.update(replaySeeds=[3]), r"chart 10 .*replay seeds \[3\] are not the chart's seed set \[0\]"), - (lambda s: s["replaySeeds"].append(5), r"chart 10 .*replay seeds \[0, 5\] are not the chart's seed"), + (lambda s: s["expectation"]["check"].update(bound=0.1), "expectation check exceeds"), + (lambda s: s["ranges"] and s["expectation"]["rankCheck"].update(expected=[0, 0]), "expectation check exceeds"), + (lambda s: s["ranges"] and s["expectation"]["rankCheck"].update(ranks=[6, 1]), "check ranks"), + (lambda s: s["expectation"]["ranges"][0].pop("luckPoints"), "expectation range 0: a range score or luck points"), + (lambda s: s.update(replaySeeds=[3]), r"chart 10 .*seeds \[3\] are not the chart's seed set \[0\]"), + (lambda s: s["replaySeeds"].append(5), r"chart 10 .*seeds \[0, 5\] are not the chart's seed"), ]) def test_deck_checks(tmp_path, change, match): with pytest.raises(musicdata.MusicDataError, match=match): @@ -529,12 +490,84 @@ def test_deck_checks(tmp_path, change, match): assert not (tmp_path / "music.json").exists() +@pytest.fixture +def expectation_fixture(): + """One complete /3 chart from the controlled interface, decoded without narrowing native numbers.""" + tables = {name: all_rows(name) for name, _ in deckdata.TABLES} + charts = deckdata.charts([tables["MasterLiveMusicScore"][1]], CHARTS.__getitem__) + source = deckdata.build(tables, charts, {}) + document = FakeDeck().chart_stats(deckdata.encode(source).decode("utf-8"), 8, 1) + return json.loads(document, parse_float=deckdata._Num)["charts"][0] + + +@pytest.mark.parametrize("estimate", [ + 100, [100], [100, 0, 0], [True, 0], [100, False], [100, -1], + [deckdata._Num("NaN"), 0], [100, deckdata._Num("Infinity")], + [deckdata._Num("1e308"), deckdata._Num("1e308")], +]) +def test_expectation_rejects_invalid_enclosures(expectation_fixture, estimate): + expectation_fixture["expectation"]["score"] = estimate + with pytest.raises(musicdata.MusicDataError, match="outward radius"): + musicdata._check_expectation(expectation_fixture, 1, "synthetic") + + +@pytest.mark.parametrize("change, message", [ + (lambda c: c.update(justNotes=True), "judgement counts"), + (lambda c: c.update(justNotes=c["judgedNotes"] + 1), "judgement counts"), + (lambda c: c.pop("expectation"), "expectation field"), + (lambda c: c["expectation"].pop("rangeWeights"), "rangeWeights or rankCheck field"), + (lambda c: c["expectation"].pop("rankCheck"), "rangeWeights or rankCheck field"), + (lambda c: c["expectation"]["ranges"][0].pop("rankBonusPerfect"), "range score"), + (lambda c: c["expectation"]["ranges"][0].update(maxCombo=True), "judgement counters"), + (lambda c: c["expectation"]["ranges"][0].update(lotResults=[[1, 0]] * 4), "outside a luck range"), + (lambda c: c["expectation"]["check"].update(ranks=[]), "unranked expectation check"), + (lambda c: c["expectation"]["rankCheck"].update(deck=[None, None]), "check deck"), +]) +def test_expectation_requires_the_new_contract(expectation_fixture, change, message): + change(expectation_fixture) + with pytest.raises(musicdata.MusicDataError, match=message): + musicdata._check_expectation(expectation_fixture, 1, "synthetic") + + +def test_expected_bonus_is_not_truncation_of_the_expected_score(expectation_fixture): + # An equally likely score of 99 or 100 has E[score] = 99.5 and E[trunc(score * .35)] = 34.5. + # Truncating the mean would incorrectly produce 34. Native /3 explicitly measures both bonuses. + row = expectation_fixture["expectation"]["ranges"][0] + row.update(rangeScore=[deckdata._Num("99.5"), 0], rangeScorePerfect=[deckdata._Num("99.5"), 0], + rankBonus=[deckdata._Num("34.5"), 0], rankBonusPerfect=[deckdata._Num("34.5"), 0]) + musicdata._check_expectation(expectation_fixture, 1, "synthetic") + assert trunc_percent(float(row["rangeScore"][0]), 35) != float(row["rankBonus"][0]) + + +def test_expectation_check_bounds_both_interval_endpoints(expectation_fixture): + check = expectation_fixture["expectation"]["check"] + check.update(expected=[1000, 1], predicted=[1000, 1], bound=1) + # Identical centers alone cannot certify a bound smaller than the farthest endpoint separation. + with pytest.raises(musicdata.MusicDataError, match="expectation check exceeds"): + musicdata._check_expectation(expectation_fixture, 1, "synthetic") + check["bound"] = 2 + musicdata._check_expectation(expectation_fixture, 1, "synthetic") + + +def test_unplayable_keeps_only_free_live_statistics(expectation_fixture): + original = deepcopy(expectation_fixture["offSeeds"]) + expectation_fixture.update(unplayable="synthetic unsupported fourth fever", expectation=None, + replaySeeds=[], gekisouAptitude=None) + musicdata._check_expectation(expectation_fixture, 1, "synthetic") + assert musicdata.validate_aptitude_statistics(None, expectation_fixture, "synthetic") is None + assert expectation_fixture["offSeeds"] == original + expectation_fixture.pop("expectation") + with pytest.raises(musicdata.MusicDataError, match="expectation field"): + musicdata._check_expectation(expectation_fixture, 1, "synthetic") + + def test_deck_without_range_weights(tmp_path): """A chart without range weights, a kind without them and a kind without Gekisou off weights are carried.""" def change(s): s["expectation"]["rangeWeights"] = None if s["scoreId"] == 10 else [None] s["expectation"]["rankCheck"] = None s["offSeeds"][0]["weights"] = [None] + s["offSeeds"][0]["check"]["deck"] = [None] * s["positions"] if s["scoreId"] == 10: for v in s["gekisouAptitude"]["variants"]: v["rangeWeights"] = None @@ -845,13 +878,17 @@ def test_command_decoded_master(tmp_path, capsys, monkeypatch): @pytest.fixture def aptitude_fixture(): """A header and a two-range chart, independently authored above, through JSON as the extension returns it.""" - tables = {t: TABLE_ROWS.get(t, rows_of(t)) for t in musicdata.tables_of(True, False)} + tables = deepcopy({t: TABLE_ROWS.get(t, rows_of(t)) for t in musicdata.tables_of(True, False)}) a = musicdata.Aptitude(tables, musicdata.gekisou_catalog(tables, musicdata.Texts(tables['MasterText']))) - header = json.loads(json.dumps({'plainKind': 0, 'host': 'synthetic host', 'law': NOMINAL_LAW, 'shapes': SHAPES}), + header = json.loads(json.dumps({'plainKind': 0, 'host': 'synthetic host', 'law': musicdata.EXPECTATION_LAW, 'shapes': SHAPES}), parse_float=deckdata._Num) chart = {'positions': 2, 'judgedNotes': 20, 'justNotes': 0, 'ranges': [ {'mission': 1, 'rankBonusPercent': 35}, {'mission': 3, 'rankBonusPercent': 40}], - 'expectation': expected_stats(2, [[35], [40]], [1, 3])} + 'expectation': {'rangeWeights': [], 'ranges': [ + {'rangeScore': [101, 0], 'rangeScorePerfect': [101, 0], 'rankBonus': [35, 0], + 'rankBonusPerfect': [35, 0], 'lotResults': [[0, 0]] * 4}, + {'rangeScore': [202, 0], 'rangeScorePerfect': [202, 0], 'rankBonus': [80, 0], + 'rankBonusPerfect': [80, 0], 'lotResults': [[0, 0]] * 4}]}} chart['gekisouAptitude'] = aptitude_of(chart, 2) chart = json.loads(json.dumps(chart), parse_float=deckdata._Num) return a, header, chart @@ -887,8 +924,9 @@ def test_aptitude_fixture(aptitude_fixture): lambda h: h['shapes'].pop(), lambda h: h.update(plainKind=1), lambda h: h.update(host=''), - lambda h: h.update(law=''), + lambda h: h.update(law='sampled means'), lambda h: h.update(law=None), + lambda h: h.pop('law'), ]) def test_aptitude_header_rejects(aptitude_fixture, change): a, h, _ = aptitude_fixture @@ -911,10 +949,13 @@ def test_aptitude_header_rejects(aptitude_fixture, change): lambda c: c['gekisouAptitude']['variants'][0].update(score=[deckdata._Num('NaN'), 0]), lambda c: c['gekisouAptitude']['variants'][0].update(score=[10**400, 0]), lambda c: c['gekisouAptitude']['variants'][0].update(score=[1, -1]), - lambda c: c['gekisouAptitude']['variants'][0].update(converted=[0.25, 0]), + lambda c: c['gekisouAptitude']['variants'][0].update(tail=[7, deckdata._Num('Infinity')]), lambda c: c['gekisouAptitude']['variants'][0].update(tail=[8, 0]), lambda c: c['gekisouAptitude']['variants'][0]['ranges'].pop(), - lambda c: c['gekisouAptitude']['variants'][0]['ranges'][0].update(justCount=[1, 0.1]), + lambda c: c['gekisouAptitude']['variants'][0].update(converted=[deckdata._Num('0.5'), 0]), + lambda c: c['gekisouAptitude']['variants'][0]['ranges'][0].pop('rankBonusPerfect'), + lambda c: c['gekisouAptitude']['variants'][2].update(tailPerfect=[9, 0]), + lambda c: c['gekisouAptitude']['variants'][2]['weights'].__setitem__(0, [0, -1]), lambda c: c['gekisouAptitude']['variants'][0]['check'].update(bound=-1), lambda c: c['gekisouAptitude']['variants'][0]['check'].update(predicted=deckdata._Num('NaN')), lambda c: c['gekisouAptitude']['variants'][0]['check'].update(expected=[0, 0]), @@ -933,13 +974,62 @@ def test_aptitude_rounding(aptitude_fixture): a, h, c = aptitude_fixture aptitude_header(a, h) v = c['gekisouAptitude']['variants'][2] - v['tail'][0] = deckdata._Num('7.000000001') - a.chart('synthetic', c, 0) # binary64 sum tolerance - v['tail'][0] = deckdata._Num('7.004') + v['tail'] = [deckdata._Num('7.002'), deckdata._Num('0.002')] + a.chart('synthetic', c, 0) # outward radius, not a standard error or an implicit rounding tolerance + v['tail'][1] = deckdata._Num('0.001') with pytest.raises(musicdata.MusicDataError, match='tail'): a.chart('synthetic', c, 0) +def test_aptitude_perfect_tail_uses_the_measured_perfect_bonus(aptitude_fixture): + a, h, c = aptitude_fixture + header = aptitude_header(a, h) + v = c['gekisouAptitude']['variants'][0] + c['justNotes'] = 1 + v['ranges'][0]['rankBonusPerfect'][0] = deckdata._Num('4.25') + v['scorePerfect'][0] = deckdata._Num('49.25') + assert v['ranges'][0]['rankBonus'][0] == 3 + # The Perfect and best channels have different independently measured bonuses. + # Their separate interval identities must both hold, without a seed or standard-error field. + assert musicdata.validate_aptitude_statistics(header, c, 'synthetic') == a.chart('synthetic', c, 0) + v['ranges'][0]['rankBonusPerfect'][0] = v['ranges'][0]['rankBonus'][0] + with pytest.raises(musicdata.MusicDataError, match='tailPerfect'): + musicdata.validate_aptitude_statistics(header, c, 'synthetic') + + +@pytest.mark.parametrize('group', ['trigger', 'condition', 'release', 'reset', 'cumulative']) +def test_confirmed_rank_shape_has_no_linear_rank_cross_terms(aptitude_fixture, group): + a, h, c = aptitude_fixture + source = a.rows[('member', 1, 5)][0] + effect = h['shapes'][0]['effects'][0] + if group == 'cumulative': + source['_skillCumulativeConditionID'] = 77 + a.cumulative[77] = {'_skillCumulativeConditionType': 7012, '_conditionValues': [1], + '_conditionTargetIDs': [], '_maxCumulativeCount': 1} + effect[group] = {'type': 7012, 'values': [1], 'targetIds': [], 'maxCumulativeCount': 1} + else: + column = dict(musicdata.CONDITION_GROUPS)[group] + source[column] = 77 + a.sets[77] = [{'_id': 77, '_conditionIds': [77]}] + a.conditions[77] = {'_conditionType': 7012, '_conditionValues': [1], + '_conditionTargetIDs': [], '_isPositive': True} + effect[group] = [[{'type': 7012, 'values': [1], 'targetIds': [], 'positive': True}]] + header = aptitude_header(a, h) # Still checked against independent master rows. + v = c['gekisouAptitude']['variants'][0] + old_weights = v['rangeWeights'] + v['rangeWeights'] = None + v['check']['ranks'] = [1, 1] + assert c['expectation']['rangeWeights'] is not None + assert musicdata.validate_aptitude_statistics(header, c, 'synthetic') == c['gekisouAptitude'] + v['check']['ranks'][0] = 2 + with pytest.raises(musicdata.MusicDataError, match='check ranks'): + a.chart('synthetic', c, 0) + v['check']['ranks'][0] = 1 + v['rangeWeights'] = old_weights + with pytest.raises(musicdata.MusicDataError, match='rank-linear domain'): + musicdata.validate_aptitude_statistics(header, c, 'synthetic') + + def test_aptitude_no_plain(aptitude_fixture): a, h, c = aptitude_fixture h['plainKind'] = None diff --git a/tests/test_replaydata.py b/tests/test_replaydata.py index e4a1b37..2465e66 100644 --- a/tests/test_replaydata.py +++ b/tests/test_replaydata.py @@ -29,6 +29,15 @@ def test_export_replay_keeps_main_file_small_and_preserves_runtime_input(tmp_pat assert chart["chart"] == record assert chart["musicLengthMs"] == data["provenance"]["replay"]["musicLengthsMs"][str(record["scoreId"])] assert manifest["engine"] is None + labels = json.loads((manifest_path.parent / manifest["snapLabels"]["url"]).read_bytes()) + assert labels["format"] == replaydata.LABEL_FORMAT + assert labels["region"] == data["provenance"]["region"] + assert labels["masterVersion"] == data["provenance"]["master"]["version"] + assert set(labels["tables"]) == set(replaydata.LABEL_TABLES) + assert "MasterSkillIcon" not in labels["tables"] + for name, table in labels["tables"].items(): + assert table["sha256"] == data["provenance"]["master"]["tables"][name]["sha256"] + assert hashlib.sha256((manifest_path.parent / manifest["snapLabels"]["url"]).read_bytes()).hexdigest() == manifest["snapLabels"]["sha256"] def test_replay_requires_actual_audio_length_and_contains_no_guess(tmp_path): @@ -37,6 +46,50 @@ def test_replay_requires_actual_audio_length_and_contains_no_guess(tmp_path): assert not (tmp_path / "music.json").exists() +@pytest.mark.parametrize("change", ["region", "version", "table_hash", "missing_table"]) +def test_replay_label_binding_rejects_other_snapshots(tmp_path, change): + import copy + from test_musicdata import TABLE_ROWS + from nnnotes import deckdata + hashes = {name: hashlib.sha256(name.encode()).hexdigest() for name in replaydata.LABEL_TABLES} + provenance = {"region": "jp", "master": {"version": "v", "tables": {name: {"sha256": digest} for name, digest in hashes.items()}}} + rows = {name: TABLE_ROWS.get(name, []) for name in replaydata.LABEL_TABLES} + label_source = replaydata.labels(rows, hashes, provenance) + export(tmp_path, replay_dir=tmp_path / "replay") + doc = json.loads((tmp_path / "music.json").read_bytes()) + runtime = json.loads((tmp_path / doc["replay"]["manifestUrl"]).parent.joinpath("deck-data.json").read_bytes()) + music = {**doc, "master": runtime["master"], "charts": runtime["charts"]} + music["provenance"] = {**copy.deepcopy(doc["provenance"]), **provenance} + if change == "region": + label_source["region"] = "tw" + elif change == "version": + label_source["masterVersion"] = "other" + elif change == "table_hash": + label_source["tables"]["MasterText"]["sha256"] = "f" * 64 + else: + del label_source["tables"]["MasterText"] + with pytest.raises(deckdata.DeckDataError, match="replay labels"): + replaydata.bundle(music, label_source=label_source) + + +def test_raw_label_rows_preserve_names_levels_and_bind_artwork(tmp_path): + from test_musicdata import TABLE_ROWS + import copy + rows = {name: copy.deepcopy(TABLE_ROWS.get(name, [])) for name in replaydata.LABEL_TABLES} + rows["MasterSupportSkill"] = [{"_id": 1, "_nameTextID": "first", "_descriptionTextFormatID": "duration"}] + rows["MasterSupportSkillEffect"] = [{"_id": 1, "_supportSkillID": 1, "_level": 1, "_effectValue": 250}, + {"_id": 2, "_supportSkillID": 1, "_level": 2, "_effectValue": 500}] + hashes = {name: hashlib.sha256(name.encode()).hexdigest() for name in replaydata.LABEL_TABLES} + provenance = {"region": "tw", "master": {"version": "v", "tables": {name: {"sha256": digest} for name, digest in hashes.items()}}} + labels = replaydata.labels(rows, hashes, provenance) + assert labels["tables"]["MasterSupportSkill"]["rows"] == rows["MasterSupportSkill"] + assert [row["_level"] for row in labels["tables"]["MasterSupportSkillEffect"]["rows"]] == [1, 2] + assert labels["tables"]["MasterSupportCard"]["rows"] == rows["MasterSupportCard"] + hashes["MasterText"] = "f" * 64 + with pytest.raises(deckdata.DeckDataError, match="hash differs"): + replaydata.labels(rows, hashes, provenance) + + WASM_MAGIC = b"\x00asm\x01\x00\x00\x00" @@ -159,11 +212,11 @@ def test_recommend_engine_rejects_another_build(tmp_path, change, message): assert not (tmp_path / "music.json").exists() and not (tmp_path / "replay").exists() -def test_final_export_checks_the_complete_expectation_interval(tmp_path): - def unmet(chart): +def test_final_export_rejects_an_invalid_expectation_before_publishing(tmp_path): + def invalid(chart): apt = chart.get("gekisouAptitude") if apt and apt["variants"]: - apt["variants"][0]["check"]["expected"] = [2000, 100] - with pytest.raises(musicdata.MusicDataError, match="beyond the bound"): - export(tmp_path, deck=FakeDeck(change=unmet)) + apt["variants"][0]["check"]["expected"] = [0, 0] + with pytest.raises(musicdata.MusicDataError, match="expectation check exceeds"): + export(tmp_path, deck=FakeDeck(change=invalid)) assert not (tmp_path / "music.json").exists() diff --git a/tests/test_statscache.py b/tests/test_statscache.py new file mode 100644 index 0000000..598bce5 --- /dev/null +++ b/tests/test_statscache.py @@ -0,0 +1,160 @@ +"""Python/native cache boundary tests, using controlled /3 documents rather than a cache implementation. + +The Rust tests exercise program/run reuse, persistence and concurrency. These tests establish that Python passes +complete inputs and options to that implementation, preserves its reported counters and publishes identical data. +""" +from copy import deepcopy +import gzip +import json +from pathlib import Path + +import pytest + +from nnnotes import deckdata, musicdata +from test_musicdata import CHARTS, KEY, PROV, FakeDeck, all_rows, bgm, master_dir, run + + +@pytest.fixture +def deck_input(): + tables = {name: all_rows(name) for name, _ in deckdata.TABLES} + scores = [row for row in tables["MasterLiveMusicScore"] if row["_id"] != 40] + return deckdata.build(tables, deckdata.charts(scores, CHARTS.__getitem__), {"region": "xx", "tables": {}}) + + +class CacheBridge(FakeDeck): + """An instrumented native interface, not a simulation of native cache hits or performance.""" + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.cache_calls = [] + self.counters = {"requests": 29, "hits": 19, "computed": 10, "writes": 10, "invalid": 2, "bytes": 5000} + + def chart_stats_cached(self, data, seeds, workers, aptitude=True, cache_dir=None): + self.cache_calls.append((json.loads(data), seeds, workers, aptitude, cache_dir)) + return self.chart_stats(data, seeds, workers, aptitude=aptitude), dict(self.counters) + + +def test_cached_calls_pass_the_whole_current_input_and_report_native_counters(tmp_path, deck_input): + module = CacheBridge() + directory = tmp_path / "native cache" + directory.mkdir() + retained = directory / "old-entry.json" + retained.write_bytes(b"native-owned entry") + before = deckdata.encode(deck_input) + deck = musicdata.Deck(module=module, seeds=4, workers=3, aptitude=False, cache=directory) + result = deck.stats(deck_input) + assert module.cache_calls == [(json.loads(before), 4, 3, False, str(directory))] + assert deck.counts == module.counters + assert result["format"] == "ournotes-deck.chart-stats/3" + assert deckdata.encode(deck_input) == before and retained.read_bytes() == b"native-owned entry" + assert list(directory.iterdir()) == [retained] # Python neither replaces nor garbage-collects native records. + current = deepcopy(deck_input) + current["provenance"]["revision"] = "next" + current["master"]["MasterEvent"] = {"columns": ["_id"], "rows": [[99]]} + result = deck.stats(current) + assert module.cache_calls[-1][0] == json.loads(deckdata.encode(current)) + assert result["source"] == current["provenance"] + + +def test_uncached_calls_do_not_invent_native_cache_counters(deck_input): + module = CacheBridge() + deck = musicdata.Deck(module=module, seeds=2, workers=1) + deck.stats(deck_input) + assert module.cache_calls == [] and len(module.inputs) == 1 + assert module.inputs[0][1:] == (2, 1) and module.options == [True] + assert deck.counts is None + + +@pytest.mark.parametrize("options, flag", [ + ({"seeds": -1}, "--seeds"), ({"seeds": 0}, "--seeds"), ({"seeds": True}, "--seeds"), + ({"seeds": 1 << 100}, "--seeds"), ({"workers": -1}, "--workers"), + ({"workers": False}, "--workers"), ({"workers": 1 << 100}, "--workers"), +]) +def test_invalid_options_are_user_errors_before_native_or_export_work(tmp_path, capsys, options, flag): + with pytest.raises(musicdata.MusicDataError, match=flag): + musicdata.Deck(**options) + value = next(iter(options.values())) + if not isinstance(value, bool): + output = tmp_path / "music.json" + code, out, error = run(["music-data", "--decoded-master", "-o", str(output), flag, str(value)], capsys) + assert code == 1 and not out and flag in error and "Traceback" not in error + assert not output.exists() + + +def test_zero_workers_selects_native_automatic_parallelism(deck_input): + module = CacheBridge() + deck = musicdata.Deck(module=module, seeds=1, workers=0) + deck.stats(deck_input) + assert module.inputs[0][1:] == (1, None) + + +def test_sampling_format_is_rejected_before_calling_the_model(deck_input): + module = FakeDeck() + module.FORMAT = "ournotes-deck.chart-stats/2" + with pytest.raises(musicdata.MusicDataError, match="format .*chart-stats/2.* is not supported"): + musicdata.Deck(module=module).stats(deck_input) + assert module.inputs == [] + + +@pytest.mark.parametrize("change", [ + lambda doc: doc.update(format="ournotes-deck.chart-stats/2"), + lambda doc: doc["charts"].reverse(), + lambda doc: doc["charts"].append(doc["charts"][0]), + lambda doc: doc["charts"][0].update(scoreId=True), +]) +def test_model_output_format_and_chart_identity_are_checked(deck_input, change): + with pytest.raises(musicdata.MusicDataError): + musicdata.Deck(module=FakeDeck(header=change)).stats(deck_input) + + +@pytest.mark.parametrize("response", ["{", "[]", "null", "{}"]) +def test_malformed_native_documents_raise_a_domain_error(deck_input, response): + class BadDocument(FakeDeck): + def chart_stats(self, *args, **kwargs): + return response + with pytest.raises(musicdata.MusicDataError, match="deck model:"): + musicdata.Deck(module=BadDocument()).stats(deck_input) + + +def test_cache_interface_failure_does_not_replace_an_existing_export(tmp_path): + source = master_dir(tmp_path) + out = tmp_path / "music.json" + out.write_bytes(b"previous published artifact\n") + with pytest.raises(musicdata.MusicDataError, match="interrupted native run"): + musicdata.export(out, deckdata.master_files(source), KEY, CHARTS.__getitem__, bgm, **PROV, + deck=musicdata.Deck(module=CacheBridge(fail="interrupted native run"), cache=tmp_path / "stats")) + assert out.read_bytes() == b"previous published artifact\n" + + +@pytest.mark.parametrize("compressed", [False, True]) +def test_native_cache_transport_does_not_change_exported_bytes(tmp_path, compressed): + source = master_dir(tmp_path) + sources = {path.name: path.read_bytes() for path in source.iterdir()} + outputs, summaries = [], [] + for name, cache in (("uncached", None), ("cached", tmp_path / "stats")): + out = tmp_path / (name + (".json.gz" if compressed else ".json")) + summaries.append(musicdata.export(out, deckdata.master_files(source), KEY, CHARTS.__getitem__, bgm, **PROV, + full=True, deck=musicdata.Deck(module=CacheBridge(), cache=cache, workers=2))) + outputs.append(out.read_bytes()) + assert outputs[0] == outputs[1] and summaries[0]["sha256"] == summaries[1]["sha256"] + assert summaries[0]["deckStats"] is None + assert summaries[1]["deckStats"] == CacheBridge().counters + doc = json.loads(gzip.decompress(outputs[0]) if compressed else outputs[0]) + assert doc["format"] == "nnnotes.music-data/2" + assert doc["provenance"]["deck"]["format"] == "ournotes-deck.chart-stats/3" + assert "deckStats" not in doc + assert {path.name: path.read_bytes() for path in source.iterdir()} == sources + + +@pytest.mark.parametrize("option", ["--aptitude-max-seeds", "--aptitude-cross-seeds", "--allow-unconverged-aptitude"]) +def test_removed_sampling_options_are_not_silently_accepted(capsys, option): + argv = ["music-data", "--decoded-master", "-o", "out.json", option] + argv += [] if option == "--allow-unconverged-aptitude" else ["32"] + code, _, error = run(argv, capsys) + assert code == 2 and "unrecognized arguments" in error + + +def test_the_native_cached_entry_point_rejects_bad_input(tmp_path): + native = pytest.importorskip("ournotes_sim") + with pytest.raises(ValueError, match="not a deck data file"): + native.chart_stats_cached('{"format":"invalid"}', cache_dir=str(tmp_path / "stats")) + assert not list(Path(tmp_path).glob("stats/**/*.json")) From 735fd2d67c2637004a2835e7220f386fd0aa6bf4 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Tue, 13 Oct 2026 21:24:24 +0800 Subject: [PATCH 3/3] feat(schemas): ship the JSON Schemas with the package as nnnotes.schemas The JSON Schemas of every document nnnotes writes move from docs/schema/ into the package, src/nnnotes/schemas/, so an installed nnnotes carries the schemas of its own formats. nnnotes.schemas.names() lists them and nnnotes.schemas.load(name) returns one, for example load("music-data"). The documentation links to the new location. Co-Authored-By: Claude Opus 5.5 --- docs/contracts.md | 42 +++++++++---------- docs/music-data.md | 2 +- src/nnnotes/contract.py | 2 +- src/nnnotes/schemas/__init__.py | 19 +++++++++ .../nnnotes/schemas}/addresses.schema.json | 0 .../nnnotes/schemas}/artifact.schema.json | 0 .../nnnotes/schemas}/artifacts.schema.json | 0 .../nnnotes/schemas}/catalog-diff.schema.json | 0 .../schemas}/catalog-index.schema.json | 0 .../nnnotes/schemas}/catalogs.schema.json | 0 .../nnnotes/schemas}/census.schema.json | 0 .../nnnotes/schemas}/common.schema.json | 0 .../nnnotes/schemas}/cost.schema.json | 0 .../nnnotes/schemas}/coverage.schema.json | 0 .../nnnotes/schemas}/failures.schema.json | 0 .../nnnotes/schemas}/layout.schema.json | 0 .../nnnotes/schemas}/music-data.schema.json | 0 .../nnnotes/schemas}/plan.schema.json | 0 .../nnnotes/schemas}/result.schema.json | 0 .../nnnotes/schemas}/run.schema.json | 0 .../nnnotes/schemas}/scripts.schema.json | 0 .../nnnotes/schemas}/task.schema.json | 0 .../nnnotes/schemas}/view.schema.json | 0 tests/test_contract.py | 4 +- tests/test_musicdata.py | 4 +- 25 files changed, 46 insertions(+), 27 deletions(-) create mode 100644 src/nnnotes/schemas/__init__.py rename {docs/schema => src/nnnotes/schemas}/addresses.schema.json (100%) rename {docs/schema => src/nnnotes/schemas}/artifact.schema.json (100%) rename {docs/schema => src/nnnotes/schemas}/artifacts.schema.json (100%) rename {docs/schema => src/nnnotes/schemas}/catalog-diff.schema.json (100%) rename {docs/schema => src/nnnotes/schemas}/catalog-index.schema.json (100%) rename {docs/schema => src/nnnotes/schemas}/catalogs.schema.json (100%) rename {docs/schema => src/nnnotes/schemas}/census.schema.json (100%) rename {docs/schema => src/nnnotes/schemas}/common.schema.json (100%) rename {docs/schema => src/nnnotes/schemas}/cost.schema.json (100%) rename {docs/schema => src/nnnotes/schemas}/coverage.schema.json (100%) rename {docs/schema => src/nnnotes/schemas}/failures.schema.json (100%) rename {docs/schema => src/nnnotes/schemas}/layout.schema.json (100%) rename {docs/schema => src/nnnotes/schemas}/music-data.schema.json (100%) rename {docs/schema => src/nnnotes/schemas}/plan.schema.json (100%) rename {docs/schema => src/nnnotes/schemas}/result.schema.json (100%) rename {docs/schema => src/nnnotes/schemas}/run.schema.json (100%) rename {docs/schema => src/nnnotes/schemas}/scripts.schema.json (100%) rename {docs/schema => src/nnnotes/schemas}/task.schema.json (100%) rename {docs/schema => src/nnnotes/schemas}/view.schema.json (100%) diff --git a/docs/contracts.md b/docs/contracts.md index 929ca1a..5c3f55a 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -2,8 +2,8 @@ The asset stages exchange documents in fixed formats. They are the public interface of the extraction: a task description can be run on another machine, a result can be read without nnnotes, and every output layout can be -derived from the records. JSON Schemas of all documents are in [schema/](schema/) (draft 2020-12, each with -`$id` `urn:nnnotes:schema:`; shared definitions in [schema/common.schema.json](schema/common.schema.json)). +derived from the records. JSON Schemas of all documents are in [src/nnnotes/schemas](../src/nnnotes/schemas/), the package `nnnotes.schemas` (draft 2020-12, each with +`$id` `urn:nnnotes:schema:`; shared definitions in [common.schema.json](../src/nnnotes/schemas/common.schema.json)). ## Versioning @@ -47,7 +47,7 @@ bundles of a catalog have the same stable name, both keep their full file names. ## Task: nnnotes.task/1 One stage applied to one subject, with everything needed to run it -([schema/task.schema.json](schema/task.schema.json)): +([task.schema.json](../src/nnnotes/schemas/task.schema.json)): ```json {"schema": "nnnotes.task/1", "id": "unity.export:membercard_assets_membercard_100101", @@ -90,7 +90,7 @@ Locator kinds, tried in order to find an input's bytes (each checked against the ## Artifact record: nnnotes.artifact/1 -One output file ([schema/artifact.schema.json](schema/artifact.schema.json)): +One output file ([artifact.schema.json](../src/nnnotes/schemas/artifact.schema.json)): ```json {"id": "CAB-…:-4153…#image", @@ -112,7 +112,7 @@ Roles: an object's primary artifact is the one whose role comes first in `image` ## Result: nnnotes.result/1 The result of a task, written after the bytes of all its artifacts are stored: its presence is the commit -([schema/result.schema.json](schema/result.schema.json)). +([result.schema.json](../src/nnnotes/schemas/result.schema.json)). ```json {"schema": "nnnotes.result/1", "key": "…", "keyParts": {…}, "status": "ok", @@ -139,7 +139,7 @@ stage, an input that cannot be read, a worker process that dies) leaves no resul ## Run manifest: nnnotes.run/1 -What one run covered ([schema/run.schema.json](schema/run.schema.json)): the context (region, language, the catalog +What one run covered ([run.schema.json](../src/nnnotes/schemas/run.schema.json)): the context (region, language, the catalog version, and the master data its views read: the version named by `MasterManifest.json` and the sha256 of each table), the selection, the normalized parameters per stage, the stage versions, every task with its key (`null` when it could not be described) and status (`hit`: the result was in the store, `ran`, `failed`), the @@ -166,7 +166,7 @@ Task failure codes (failed items carry the reason codes of their stages): ## Plan: nnnotes.plan/1 -What a run would do, without running anything ([schema/plan.schema.json](schema/plan.schema.json)): every task as +What a run would do, without running anything ([plan.schema.json](../src/nnnotes/schemas/plan.schema.json)): every task as a node with its key, status (`hit`, `run`, `unknown`), reasons and cost estimate, the edges between tasks, and a summary (counts, estimated CPU seconds and peak memory for the given workers and budget, tasks above the budget, and per layout the files it would write, remove and keep). Reasons compare the key parts with those of the same @@ -191,7 +191,7 @@ task id in the previous run manifest: ## Layout manifest: nnnotes.layout/1 The files an output layout owns in its directory, as `manifest.json` there -([schema/layout.schema.json](schema/layout.schema.json)): name, parameters, and `entries` `{path, sha256, size, id}` +([layout.schema.json](../src/nnnotes/schemas/layout.schema.json)): name, parameters, and `entries` `{path, sha256, size, id}` sorted by path, one per artifact (artifacts of equal content may share a path). A layout changes neither artifact bytes nor keys; the file names of `original` and `cas` both follow from the records, and `cas` also from the `original` manifest alone. Documents derived for `original` from records and paths (the views with the files of @@ -199,14 +199,14 @@ their objects) are the exception: `original` has the derived documents, `cas` th ## Reports -- Coverage, `nnnotes.coverage/1` ([schema/coverage.schema.json](schema/coverage.schema.json)): per class the census +- Coverage, `nnnotes.coverage/1` ([coverage.schema.json](../src/nnnotes/schemas/coverage.schema.json)): per class the census total, the objects without an item (`missing`) and the count of every item status; per reason code its count and the first five object ids. An object with items in several results (a sprite whose image `sprite.crop` makes beside its `unity.export` item) counts once, with the most severe of its statuses (`failed`, `unsupported`, `generic`, `exported`, `contained`, in that order) and that item's reason. -- Failures, `nnnotes.failures/1` ([schema/failures.schema.json](schema/failures.schema.json)): `{task, stage, input, +- Failures, `nnnotes.failures/1` ([failures.schema.json](../src/nnnotes/schemas/failures.schema.json)): `{task, stage, input, object, code, message}` for failed tasks (`object` null) and failed items, sorted by task and object. -- Cost, `nnnotes.cost/1` ([schema/cost.schema.json](schema/cost.schema.json)): the measured CPU and wall seconds +- Cost, `nnnotes.cost/1` ([cost.schema.json](../src/nnnotes/schemas/cost.schema.json)): the measured CPU and wall seconds and peak resident memory of a task's last run, and the resident memory of its process when it started (a worker keeps memory from its earlier tasks: the task's own need is the difference). @@ -214,20 +214,20 @@ Documents written by particular stages (artifacts of no object: their ids are `< | Document | Stage | Schema | |---|---|---| -| `nnnotes.catalog-index/1` | `catalog.index` | [schema/catalog-index.schema.json](schema/catalog-index.schema.json) | -| `nnnotes.census/1` | `unity.census` | [schema/census.schema.json](schema/census.schema.json) | -| `nnnotes.scripts/1` | `link.scripts` | [schema/scripts.schema.json](schema/scripts.schema.json) | -| `nnnotes.addresses/1` | `link.addresses` | [schema/addresses.schema.json](schema/addresses.schema.json) | -| `nnnotes.artifacts/1` | `link.artifacts` | [schema/artifacts.schema.json](schema/artifacts.schema.json) | -| `nnnotes.view/1` | `view.` | [schema/view.schema.json](schema/view.schema.json) | - -The catalog versions of a store, `nnnotes.catalogs/1` ([schema/catalogs.schema.json](schema/catalogs.schema.json), +| `nnnotes.catalog-index/1` | `catalog.index` | [catalog-index.schema.json](../src/nnnotes/schemas/catalog-index.schema.json) | +| `nnnotes.census/1` | `unity.census` | [census.schema.json](../src/nnnotes/schemas/census.schema.json) | +| `nnnotes.scripts/1` | `link.scripts` | [scripts.schema.json](../src/nnnotes/schemas/scripts.schema.json) | +| `nnnotes.addresses/1` | `link.addresses` | [addresses.schema.json](../src/nnnotes/schemas/addresses.schema.json) | +| `nnnotes.artifacts/1` | `link.artifacts` | [artifacts.schema.json](../src/nnnotes/schemas/artifacts.schema.json) | +| `nnnotes.view/1` | `view.` | [view.schema.json](../src/nnnotes/schemas/view.schema.json) | + +The catalog versions of a store, `nnnotes.catalogs/1` ([catalogs.schema.json](../src/nnnotes/schemas/catalogs.schema.json), `/catalogs/index.json`), and the difference between two of them, `nnnotes.catalog-diff/1` -([schema/catalog-diff.schema.json](schema/catalog-diff.schema.json), `nnnotes catalogs diff --json`), are written +([catalog-diff.schema.json](../src/nnnotes/schemas/catalog-diff.schema.json), `nnnotes catalogs diff --json`), are written by the commands. The music data file of `nnnotes music-data`, `nnnotes.music-data/2` -([schema/music-data.schema.json](schema/music-data.schema.json)), names its format in a `format` field; it is +([music-data.schema.json](../src/nnnotes/schemas/music-data.schema.json)), names its format in a `format` field; it is described in [music-data.md](music-data.md). ## Store diff --git a/docs/music-data.md b/docs/music-data.md index 52fd78c..64c7e00 100644 --- a/docs/music-data.md +++ b/docs/music-data.md @@ -7,7 +7,7 @@ chart facts (level, note counts, BPM, chart times, skill events, fevers), and pe the chart contributes to the live score whatever the deck, in a solo live (Gekisou off) and in a Gekisou live at every rank, measured by the deck model [ournotes-deck](https://github.com/empty-sekai/ournotes-deck), the package `ournotes-sim` that the extra `deck` installs (`pip install 'nnnotes[deck]'`). The format is `nnnotes.music-data/2`; its JSON Schema is -[schema/music-data.schema.json](schema/music-data.schema.json). +[music-data.schema.json](../src/nnnotes/schemas/music-data.schema.json). ``` nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--full] [--no-deck] [--seeds N] diff --git a/src/nnnotes/contract.py b/src/nnnotes/contract.py index 6ace46e..c932422 100644 --- a/src/nnnotes/contract.py +++ b/src/nnnotes/contract.py @@ -1,6 +1,6 @@ """The contract of the asset stages: canonical JSON, identities, keys and the documents stages exchange. -Everything here is a public format (docs/contracts.md, JSON Schemas in docs/schema/): +Everything here is a public format (docs/contracts.md, JSON Schemas in nnnotes.schemas): canon/1 UTF-8, LF, one trailing newline, non-ASCII as is. Contract documents: keys sorted, indent 1. Object documents (typetree JSON): the object's own field order, indent 1. Non-finite floats as diff --git a/src/nnnotes/schemas/__init__.py b/src/nnnotes/schemas/__init__.py new file mode 100644 index 0000000..8f7c9ef --- /dev/null +++ b/src/nnnotes/schemas/__init__.py @@ -0,0 +1,19 @@ +"""The JSON Schemas (draft 2020-12) of the documents nnnotes writes, shipped with the package: `.schema.json`, +each with the `$id` `urn:nnnotes:schema:` that the others reference.""" +import json +from importlib import resources + +SUFFIX = ".schema.json" + + +def names() -> list[str]: + """The schema names (`music-data`, `plan`, ...), sorted.""" + return sorted(p.name[:-len(SUFFIX)] for p in resources.files(__package__).iterdir() if p.name.endswith(SUFFIX)) + + +def load(name: str) -> dict: + """The schema `name` (`music-data` for music-data.schema.json).""" + path = resources.files(__package__) / f"{name}{SUFFIX}" + if not path.is_file(): + raise KeyError(f"no schema {name!r} (schemas: {', '.join(names())})") + return json.loads(path.read_text(encoding="utf-8")) diff --git a/docs/schema/addresses.schema.json b/src/nnnotes/schemas/addresses.schema.json similarity index 100% rename from docs/schema/addresses.schema.json rename to src/nnnotes/schemas/addresses.schema.json diff --git a/docs/schema/artifact.schema.json b/src/nnnotes/schemas/artifact.schema.json similarity index 100% rename from docs/schema/artifact.schema.json rename to src/nnnotes/schemas/artifact.schema.json diff --git a/docs/schema/artifacts.schema.json b/src/nnnotes/schemas/artifacts.schema.json similarity index 100% rename from docs/schema/artifacts.schema.json rename to src/nnnotes/schemas/artifacts.schema.json diff --git a/docs/schema/catalog-diff.schema.json b/src/nnnotes/schemas/catalog-diff.schema.json similarity index 100% rename from docs/schema/catalog-diff.schema.json rename to src/nnnotes/schemas/catalog-diff.schema.json diff --git a/docs/schema/catalog-index.schema.json b/src/nnnotes/schemas/catalog-index.schema.json similarity index 100% rename from docs/schema/catalog-index.schema.json rename to src/nnnotes/schemas/catalog-index.schema.json diff --git a/docs/schema/catalogs.schema.json b/src/nnnotes/schemas/catalogs.schema.json similarity index 100% rename from docs/schema/catalogs.schema.json rename to src/nnnotes/schemas/catalogs.schema.json diff --git a/docs/schema/census.schema.json b/src/nnnotes/schemas/census.schema.json similarity index 100% rename from docs/schema/census.schema.json rename to src/nnnotes/schemas/census.schema.json diff --git a/docs/schema/common.schema.json b/src/nnnotes/schemas/common.schema.json similarity index 100% rename from docs/schema/common.schema.json rename to src/nnnotes/schemas/common.schema.json diff --git a/docs/schema/cost.schema.json b/src/nnnotes/schemas/cost.schema.json similarity index 100% rename from docs/schema/cost.schema.json rename to src/nnnotes/schemas/cost.schema.json diff --git a/docs/schema/coverage.schema.json b/src/nnnotes/schemas/coverage.schema.json similarity index 100% rename from docs/schema/coverage.schema.json rename to src/nnnotes/schemas/coverage.schema.json diff --git a/docs/schema/failures.schema.json b/src/nnnotes/schemas/failures.schema.json similarity index 100% rename from docs/schema/failures.schema.json rename to src/nnnotes/schemas/failures.schema.json diff --git a/docs/schema/layout.schema.json b/src/nnnotes/schemas/layout.schema.json similarity index 100% rename from docs/schema/layout.schema.json rename to src/nnnotes/schemas/layout.schema.json diff --git a/docs/schema/music-data.schema.json b/src/nnnotes/schemas/music-data.schema.json similarity index 100% rename from docs/schema/music-data.schema.json rename to src/nnnotes/schemas/music-data.schema.json diff --git a/docs/schema/plan.schema.json b/src/nnnotes/schemas/plan.schema.json similarity index 100% rename from docs/schema/plan.schema.json rename to src/nnnotes/schemas/plan.schema.json diff --git a/docs/schema/result.schema.json b/src/nnnotes/schemas/result.schema.json similarity index 100% rename from docs/schema/result.schema.json rename to src/nnnotes/schemas/result.schema.json diff --git a/docs/schema/run.schema.json b/src/nnnotes/schemas/run.schema.json similarity index 100% rename from docs/schema/run.schema.json rename to src/nnnotes/schemas/run.schema.json diff --git a/docs/schema/scripts.schema.json b/src/nnnotes/schemas/scripts.schema.json similarity index 100% rename from docs/schema/scripts.schema.json rename to src/nnnotes/schemas/scripts.schema.json diff --git a/docs/schema/task.schema.json b/src/nnnotes/schemas/task.schema.json similarity index 100% rename from docs/schema/task.schema.json rename to src/nnnotes/schemas/task.schema.json diff --git a/docs/schema/view.schema.json b/src/nnnotes/schemas/view.schema.json similarity index 100% rename from docs/schema/view.schema.json rename to src/nnnotes/schemas/view.schema.json diff --git a/tests/test_contract.py b/tests/test_contract.py index 48425f1..ce22e1a 100644 --- a/tests/test_contract.py +++ b/tests/test_contract.py @@ -14,7 +14,7 @@ from test_stages import PIPELINE, toy_files, toy_stages ROOT = Path(__file__).resolve().parents[1] -SCHEMAS = ROOT / "docs" / "schema" +SCHEMAS = ROOT / "src" / "nnnotes" / "schemas" SHA = "0" * 64 @@ -237,4 +237,4 @@ def test_the_contract_is_documented(): missing = [w for w in words if not re.search(rf"(?