From 9be0b6fb5b0dacbb9fdffb35f68bf968f7ec3b9b Mon Sep 17 00:00:00 2001 From: nichinichisou0609 <128159543+nichinichisou0609@users.noreply.github.com> Date: Sat, 10 Oct 2026 04:11:17 +0000 Subject: [PATCH 1/3] build(deck): ournotes-deck v0.0.4 --- rust/Cargo.lock | 4 ++-- rust/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 3146643..5192313 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -95,8 +95,8 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "ournotes-sim" -version = "0.0.3" -source = "git+https://github.com/empty-sekai/ournotes-deck?rev=31d74487c4fd9e5dc3c1d4f34be8dee8f1a75b0e#31d74487c4fd9e5dc3c1d4f34be8dee8f1a75b0e" +version = "0.0.4" +source = "git+https://github.com/empty-sekai/ournotes-deck?rev=efbf286ad60f98e50c906605e76a1ad67c18ea4b#efbf286ad60f98e50c906605e76a1ad67c18ea4b" dependencies = [ "serde", "serde_json", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 54ef161..b3ee408 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -16,7 +16,7 @@ crate-type = ["cdylib"] pyo3 = { version = "0.29", features = ["abi3-py311"] } # the deck model, the crate ournotes-sim of the ournotes-deck repository, pinned to the commit of a release; # .github/workflows/deck.yml opens a pull request when ournotes-deck publishes a newer release -ournotes-sim = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "31d74487c4fd9e5dc3c1d4f34be8dee8f1a75b0e" } +ournotes-sim = { git = "https://github.com/empty-sekai/ournotes-deck", rev = "efbf286ad60f98e50c906605e76a1ad67c18ea4b" } serde_json = { version = "1", features = ["raw_value", "arbitrary_precision"] } [profile.release] From 66f654cb99cadacde9a89c0b17a370432f7b713d Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Sat, 10 Oct 2026 12:16:33 +0800 Subject: [PATCH 2/3] feat(music-data): measure chart statistics with nominal lottery and skill probabilities Chart statistics follow the ournotes-deck chart-stats/3 document: Gekisou statistics use independent nominal lottery and skill probabilities, and charts with a luck range take a replay seed count. The music-data document, its schema, the command help and the replay data tests follow that format. Co-Authored-By: Claude Sonnet 5.5 --- docs/commands.md | 9 +- docs/contracts.md | 2 +- docs/music-data.md | 248 +++++--------- docs/replay.md | 4 +- docs/schema/music-data.schema.json | 533 ++++++++++++----------------- rust/src/lib.rs | 27 +- src/nnnotes/cli.py | 12 +- src/nnnotes/musicdata.py | 299 +++++++--------- tests/test_musicdata.py | 163 +++++---- tests/test_replaydata.py | 6 +- 10 files changed, 539 insertions(+), 764 deletions(-) diff --git a/docs/commands.md b/docs/commands.md index 1280243..fd22e1b 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -678,7 +678,7 @@ in the printed summary and in `SITE.failures.json`, models that fail in the summ ``` nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--full] [--no-deck] [--seeds N] - [--workers N] [--no-gekisou-aptitude] [--aptitude-max-seeds N] [--aptitude-cross-seeds N] + [--workers N] [--no-gekisou-aptitude] [--stats-cache DIR] [--no-bgm] [--jackets DIR] -o FILE ``` @@ -694,10 +694,9 @@ rank and the Perfect play need) and off (a solo live), measured by the deck mode `nnnotes._deck`) on its whole-live simulation and checked against the chart facts and the master data. `--full` also writes the deck model's input: every `MasterLiveMusicScore` row's chart as the client builds it at runtime (notes, skill events, fever ranges) and the master data tables about cards, skills, bonuses, scores and events. Single-skill Gekisou aptitude is included by -default (not an optimal deck); `--no-gekisou-aptitude` omits it, `--aptitude-max-seeds N` (1024) and -`--aptitude-cross-seeds N` (64) cap its sampling and cross terms. `--no-deck` skips the deck model (every chart's -`deck` is null); `--seeds N` (default 8) and `--workers N` (default: every processor) set its seeds on charts with a -luck range and its threads. `--stats-cache DIR` keeps every chart's statistics in `DIR` under the SHA-256 of what +default as independent nominal expectations; `--no-gekisou-aptitude` omits it. +`--no-deck` leaves every chart's `deck` null. `--seeds N` (default 8) gives replay seeds for luck charts; +`--workers N` sets measurement threads (default: every processor). `--stats-cache DIR` keeps every chart's statistics in `DIR` under the SHA-256 of what they are a function of (the deck model's sources, these options, the master data tables and the chart) and measures only the charts it lacks; `DIR` then holds this file's charts only. The master data is decoded from the files as served: `--master-files DIR` reads `DIR/MasterManifest.json` and the `.bin` files it lists (`master download`; the file's region is `[catalog] region`), diff --git a/docs/contracts.md b/docs/contracts.md index 2cf4e7f..929ca1a 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -226,7 +226,7 @@ The catalog versions of a store, `nnnotes.catalogs/1` ([schema/catalogs.schema.j ([schema/catalog-diff.schema.json](schema/catalog-diff.schema.json), `nnnotes catalogs diff --json`), are written by the commands. -The music data file of `nnnotes music-data`, `nnnotes.music-data/1` +The music data file of `nnnotes music-data`, `nnnotes.music-data/2` ([schema/music-data.schema.json](schema/music-data.schema.json)), names its format in a `format` field; it is described in [music-data.md](music-data.md). diff --git a/docs/music-data.md b/docs/music-data.md index 8b8290c..dd6ad75 100644 --- a/docs/music-data.md +++ b/docs/music-data.md @@ -6,12 +6,12 @@ ranks, jacket, BGM length), the Gekisou catalog (member cards, snaps and their G chart facts (level, note counts, BPM, chart times, skill events, fevers), and per chart its **deck statistics**: what the chart contributes to the live score whatever the deck, in a solo live (Gekisou off) and in a Gekisou live at every rank, measured by the deck model [ournotes-deck](https://github.com/empty-sekai/ournotes-deck), which nnnotes carries -as its extension module `nnnotes._deck`. The format is `nnnotes.music-data/1`; its JSON Schema is +as its extension module `nnnotes._deck`. The format is `nnnotes.music-data/2`; its JSON Schema is [schema/music-data.schema.json](schema/music-data.schema.json). ``` nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--full] [--no-deck] [--seeds N] - [--workers N] [--no-gekisou-aptitude] [--aptitude-max-seeds N] [--aptitude-cross-seeds N] + [--workers N] [--no-gekisou-aptitude] [--stats-cache DIR] [--no-bgm] [--jackets DIR] -o FILE ``` @@ -26,13 +26,10 @@ nnnotes music-data (--master-files DIR | --apk-master | --decoded-master) [--ful - `--full`: also write the deck model's input, every chart's runtime notes and the master data tables about cards, skills, bonuses, scores and events ([the deck input](#the-deck-input---full)), for tools that run a deck model of their own. -- `--no-deck`: do not run the deck model; every chart's `deck` is null. The deck model measures every chart on a - whole-live simulation, dozens of lives per chart: a full run takes processor time in proportion to the number of - charts. `--workers N` sets the threads it uses (default: every processor), `--seeds N` the seeds measured on a chart - with a luck range (default 8). -- `--no-gekisou-aptitude`: keep the existing deck statistics but skip single-skill aptitude measurements. - `--aptitude-max-seeds N` caps their samples (default 65536), and `--aptitude-cross-seeds N` caps cross-term samples - (default 64). Lower caps reduce work but may leave the standard-error target unmet. +- `--no-deck`: leave every chart's `deck` null. `--workers N` sets the measurement threads + (default: every processor). `--seeds N` sets the replay seed count for a chart with a luck range (default 8); + nominal expectation measurements are independent of this count. +- `--no-gekisou-aptitude`: omit single-skill aptitude measurements while retaining baseline statistics. - `--stats-cache DIR`: keep every chart's deck statistics in `DIR`, one file per chart named by the SHA-256 of what they are a function of: the deck model's sources (`provenance.deck.sourceSha256`), `--seeds` and the aptitude options, the master data tables of the deck input and the chart's runtime notes. A chart found there is not measured @@ -67,7 +64,7 @@ the wheels on PyPI carry it. See [Building](#building). ```json { - "format": "nnnotes.music-data/1", + "format": "nnnotes.music-data/2", "provenance": {"region": ..., "client": {...}, "catalog": {...}, "master": {...}, "exporter": {...}, "deck": {...}}, "languages": ["ja", "en", "zh-Hant", "zh-Hans", "ko"], "bands": [...], "characters": [...], "tags": [...], "categories": [...], @@ -102,7 +99,7 @@ A **text** is an object with one string per language of `languages` (`{"ja": ... | `master.tables..sha256` | SHA-256 of each table's file as served, before decoding: the song tables (`MasterLiveMusic`, `MasterLiveMusicScore`, `MasterText`, `MasterBand`, `MasterCharacter`, `MasterTag`, `MasterLiveMusicCategory`, `MasterSound`, `MasterSoundCueSheet`, `MasterLiveScoreRank`), the tables of the Gekisou catalog (`MasterMemberCard`, `MasterSupportCard`, `MasterSupportCardRank`, `MasterGekisouSkill`, `MasterGekisouSkillEffect`, `MasterGekisouSupportSkill`, `MasterGekisouSupportSkillEffect`) and, when the deck model runs or with `--full`, the tables of [the deck input](#the-deck-input---full) | | `exporter.name`, `exporter.version` | `nnnotes` and its version | | `exporter.chartFormat` | the format of the chart converter the notes come from, `nnnotes.live-score/1` | -| `deck` | the deck model: `{name, version, source, commit, format}`, `ournotes-deck`, the package version of its crate ournotes-sim, the repository, the git commit nnnotes is built with and the statistics format (`ournotes-deck.chart-stats/2`); null with `--no-deck` | +| `deck` | the deck model: `{name, version, source, commit, format}`, `ournotes-deck`, the package version of its crate ournotes-sim, the repository, the git commit nnnotes is built with and the statistics format (`ournotes-deck.chart-stats/3`); null with `--no-deck` | ### bands, characters, tags, categories @@ -175,157 +172,100 @@ plays, `musicLengthMs` is the length the score code uses; a listing chooses the ## Deck statistics -The deck model plays every chart of a song on its whole-live simulation in the theoretical best play, the frame -times of the game's default schedule, in two scenarios: +The deck model reports `ournotes-deck.chart-stats/3`. Gekisou measurements use independent nominal +lottery and skill probabilities. An interval is `[center, half-width]`, enclosing the expectation under +that law with outward probability arithmetic and score rounding bounds. Deterministic judgement counters +are exact integers. The law describes the model; agreement with a particular game's seed distribution +is a separate question. -- **Gekisou on** (`seeds`), as a Gekisou live (Battle Live, up to five players) plays: every judged note at its exact - time, Just inside the Just-count ranges and Perfect elsewhere, rank 1 in every Gekisou range. Other ranks follow - from the same numbers (below). -- **Gekisou off** (`offSeeds`), as a solo live (Free Live, Challenge Live) plays: every judged note Perfect at its - exact time, seed 0, no Just, luck, Gekisou combo or rank bonus. A chart with more than three fevers plays here too. +Serialized point centers use millipoints, consumed result counts use five fractional digits, and weights +use twelve. Half-widths round outward at the same precision and include the center's rounding. -In each it measures, at deck power `model.power` (300000): - -- `score`: the exact score without skills (with Gekisou on, the rank 1 bonuses of the Gekisou ranges included); -- for every **score-up kind** (`deck.kinds`) and every performance position `k`, `weights[kind][k]`: the exact score - a deck gains when its position-`k` member has one effect of that kind at factor 1, divided by the deck power. The - effect runs through the simulation's own updaters, conditions, frames and appliers, so the weight carries every rule - of the game (execute and finish frames, the 40 ms score frames, combo and Gekisou combo factors, Just scores, luck - rushes, the rank bonuses of the ranges it overlaps). - -A deck whose live skills are all score-up kinds then scores, up to the floors, - -``` -P * (score / power + sum over positions k of factor_k * weights[kind_k][k]) -``` - -with `P` the deck power and `factor_k` the effect's factor as the game's applier converts its value: effect type 2000 -`floor(value / 10000f * 1e5) / 1e5`, 2005 `floor(value / -10000f * 1e5) / 1e5`, 2002 and 2004 the same quotient -rounded half to even (`value / 10000f` in binary32). Each seed also plays a **check deck**, random kinds at real -master values at another power (`model.checkPower`), and the command fails when its exact score leaves the bound of -the formula. Effects of other types (cumulative score 2001 / 2003, life, judgement conversion, Gekisou and snap -skills) are not linear in the chart alone and have no weights: a deck's score with them comes from the simulation. - -A luck range draws lottery results from the play's random seed, so the Gekisou on measurements are given **per -seed**: one seed (0) when no range is a luck range, else the first `--seeds` seeds of the deck model's published seed -set. That set is a fixed sequence: candidate `k` is the low 32 bits (signed) of output `k + 1` of SplitMix64 started at -`0x6765_6B69_736F_7531` (`gekisou1`), kept when its pair of stream seeds (`|b|` and `|b ^ 0x9E3779B9|`, -2^31 as -2^31 - 1) differs from those of every seed kept before it, so a smaller set is a prefix of a larger one. The seed -set is not the game's seed law (which is unknown); a mean over it is not the game's expectation. - -### Ranks - -In a Gekisou live, range `i` takes a rank `r_i` from 1 to 5 among the room's players, and its rank bonus is -`trunc(rangeScore_i * p_i(r_i) / 100)` with `p_i(r) = ranges[i].rankBonusPercents[r - 1]`. The bonus is a fixed score -at the range's end: it changes no factor and no note score, and a later range's score holds it at both ends, so the -range scores do not depend on the ranks. At ranks `r` a seed's numbers are therefore - -``` -score_r = score - sum_i rankBonus_i + sum_i trunc(rangeScore_i * p_i(r_i) / 100) (exact) -weights_r[kind][k] = weights[kind][k] + sum_i (p_i(r_i) - p_i(1)) / 100 * rangeWeights[kind][k][i] -``` - -(`rankBonus_i`, `rangeScore_i`: `seeds[].ranges[i]`), and a deck scores `P * (score_r / power + sum_k factor_k * -weights_r[kind_k][k])` as above; `weights_r` is within `2 * ranges / power` per unit of factor of the exact weight at -those ranks. At rank 1 everywhere these are `score` and `weights`. `rangeWeights` is null when a range's bonus can fall -inside another range's score frames (overlapping ranges), and a kind's entry is null when its conditions read the -confirmed rank (condition 7012): the ranks do not follow linearly there. `rankCheck` plays the seed's check deck at -random ranks through the simulation's explicit rank confirmations and bounds it against these formulas. - -### Just rate - -`scorePerfect` and `ranges[i].rangeScorePerfect` are the no-skill score (rank 1 bonuses included) and the range -scores of the same play with every Just judged Perfect (the Just judgement is enabled only inside the Just-count -ranges, so nothing else changes). Between a Just rate of 1 (`score`, `rangeScore`) and 0 (`scorePerfect`, -`rangeScorePerfect`) a page can interpolate; the rank bonus of the Perfect play is `trunc(rangeScorePerfect_i * -p_i(r_i) / 100)`. A chart without Just notes (`justNotes` 0) has `scorePerfect == score`. +The best play judges each note at its exact time, Just in Just-count ranges and Perfect elsewhere. +The Perfect play supplies the other endpoint, with every Just judged Perfect. Both include rank 1 bonuses. +Free Live has its own deterministic Perfect run, without Gekisou, and remains available on charts with +more than three fever ranges. ### deck -| Field | Content | -|---|---| -| `model` | the deck model's description of the measurement: `engine`, `play`, `score` (the formula), `power`, `checkPower`, `unitValue` (the effect value of factor 1, 10000), `seeds`, `ranks` (the rank formulas), `perfect` (the Perfect play), `off` (the Gekisou off scenario) | -| `kinds[]` | the score-up kinds of the master data: `MasterLiveSkillEffect` rows of type 2000, 2002, 2004 or 2005 without a cumulative condition, grouped by what shapes their score. `id` (the index in `weights`), `effectType`, `activationTimeSecond`, `durationMs` (`ceil(activationTimeSecond * 1000f)`), `skillTargetIds`, `skillConditionGroup`, `skillReleaseConditionGroup`, `effectLimitCount`, `effectExecuteLimitCount`, `effectExecuteLimitResetConditionGroup`; `rows` (master rows of the kind) and `values` (their distinct `_effectValue`s, ascending) | - -The kind of a card's live skill is found by matching its `MasterLiveSkillEffect` row (at the skill level) on these -fields; `values` lists what the master data uses. +`model` describes the play, probability law, measurement `power`, `checkPower`, `unitValue` and score formulas. +`kinds` groups ordinary score-up rows of type 2000, 2002, 2004 or 2005 by duration, targets, conditions, +release and execution limits. A weight is the expected increment per unit of deck power and effect factor. +Type 2000 uses `value / 10000`; 2002 adds its intrinsic half factor; 2004 and 2005 use the effect's +half-to-even rounded activation factor. A card's kind is identified by its effect row's parameters. ### charts[].deck | Field | Content | |---|---| -| `convertedNoteCount` | the note count the score formula divides by (converted notes) | -| `skip` | score per unit of deck power of a skipped live (every note Great, combo 0, no skills) | -| `events` | `[[position, timeMs], ...]`: the skill events in chart order with the performance position each fires | -| `positions` | the performance positions the events fire (the largest position + 1): the length of every `weights[kind]` | -| `ranges[]` | the Gekisou ranges: `index`, `mission` (1 combo, 2 luck, 3 Just count), `startMs`, `endMs`, `rankBonusPercents` (the rank bonus percentages of ranks 1..5 of the song's mission pattern, `MasterLiveGekisouRankingScoreBonus`), `rankBonusPercent` (the rank 1 percentage, `rankBonusPercents[0]`) | -| `justNotes` | notes judged Just on the Gekisou on play | -| `seeds[]` | Gekisou on, per seed: `seed`; `score` (points at `model.power`, rank 1 bonuses included); `ranges[]` (`rangeScore`: the points gained inside the range, `rankBonus`: its rank 1 bonus in points, `maxCombo`, `justCount`, `luckPoints`: the luck points gained (`TotalBonusPoint`; a combo, Just count or luck range ranks the room by `maxCombo`, `justCount` or `luckPoints`), `lotResults`: lottery results Miss, Hit, Super Hit, Critical, `rangeScorePerfect`: `rangeScore` on the Perfect play); `weights[kind][position]` (points per unit of deck power and of factor); `check` (`deck`: `[kind, value]` or null per position, `exact`, `predicted`, `bound`: points at `model.checkPower`); `scorePerfect` (`score` on the Perfect play); `rangeWeights[kind][position][range]` (range points per unit of deck power and of factor, or null; a kind null); `rankCheck` (`ranks`: 1..5 per range, `exact`, `predicted`, `bound`; null without ranges or range weights) | -| `offSeeds[]` | Gekisou off, one seed: `seed` (0), `score`, `weights[kind][position]` (a kind null when its conditions read the Gekisou state, which a solo live does not have) and `check`, as in `seeds[]` | -| `unplayable` | null, or why the game cannot play the chart with Gekisou (more than three fevers: the game fails when the fourth starts); `seeds` is then empty, `offSeeds` is not | - -The deck model's chart facts are checked against the file's: the song, difficulty, level, judged note count, last -note time, music length, Gekisou missions, skill event times and fever ranges must agree, and are not repeated in -`deck`. Its numbers are checked against the master data and themselves: every range's `rankBonusPercents` are the -`MasterLiveGekisouRankingScoreBonus` rows of the song's mission pattern (0 without a row), every `rankBonus` is -`trunc(rangeScore * rankBonusPercent / 100)`, a chart without Just notes has the same scores on the Perfect play, -every array has its shape (`[kind][position]`, `[kind][position][range]`, one range result per range with its luck -points, one Gekisou off seed), the seeds are the chart's seed set (none on a chart unplayable with Gekisou, seed 0 -without a luck range, else the first `--seeds` seeds of the published seed set, which nnnotes computes itself), and -every check and rank check is within its bound. +| `convertedNoteCount`, `skip` | the score divisor and skipped-live score per unit of power | +| `events`, `positions` | `[position, timeMs]` skill events and the number of measured performance positions | +| `ranges` | index, mission, start/end times and `rankBonusPercents` for ranks 1..5 | +| `justNotes` | exact Just count of the best play | +| `expectation` | `score`, `scorePerfect`, `ranges`, `weights`, `rangeWeights`, `check`, `rankCheck`; null when Gekisou is unplayable | +| `replaySeeds` | seeds for per-play replay: `[0]` without a luck range, otherwise the first `--seeds` published seeds; empty when Gekisou is unplayable | +| `offSeeds` | one deterministic Free Live run: `seed: 0`, integer `score`, numeric `weights` and `{deck, exact, predicted, bound}` check | +| `unplayable` | null or the reason Gekisou cannot play the chart | + +`expectation.score` and `scorePerfect` are point intervals at `model.power`. Each expected range has +`rangeScore`, `rangeScorePerfect`, `rankBonus`, `rankBonusPerfect` and `luckPoints` intervals, integer +`maxCombo` and `justCount`, and four `lotResults` intervals: expected consumed Miss, Hit, Super Hit and +Critical counts. `weights[kind][position]` and `rangeWeights[kind][position][range]` hold intervals in +the weight unit. A range-weight kind is null when its conditions read confirmed rank; the whole field +is null when overlapping ranges prevent a linear rank transformation. + +A check has `{deck, ranks, expected, predicted, bound}`. Its deck contains `[kind, value]` or null per +position, its score intervals use `checkPower`, and the greatest separation of their endpoints must be +within `bound`. The baseline check uses `ranks: null`; `rankCheck` supplies explicit fixed rank arrivals +and is null when no linear rank check is available. The exporter checks chart facts, dimensions, finite +intervals, counters, replay seeds and every check before writing the file. + +### Ranks and accuracy + +For disjoint ranges whose effects do not read confirmed rank, an arbitrary-rank expectation uses + +``` +score_r = score - sum(rankBonus_i) + sum(E[trunc(rangeScore_i * p_i(r_i) / 100)]) +weight_r = weight + sum((p_i(r_i) - p_i(1)) / 100 * rangeWeight_i) +``` + +The expected truncation can be enclosed by scaling the range-score expectation and adding a rounding +allowance. Truncating the center alone does not compute an expected rank bonus. At rank 1 the measured +bonus interval is available directly. `rankBonusPerfect` supplies the Perfect endpoint's bonus. +Just-rate interpolation and Great-share scaling are accuracy approximations; the summary does not +describe arbitrary misses or combo breaks. Per-note replay runs an explicit play with its own seed. ### Gekisou skill aptitude -`deck.gekisouAptitude` describes the shapes measured, and each chart's `deck.gekisouAptitude` describes how its -score changes with **one** such shape equipped. It does not select a deck, and increments measured separately must -not be added to estimate several skills together. The existing `deck.seeds` still measures Gekisou **without card -Gekisou skills**; a solo/free live uses `offSeeds`, without Gekisou. These are model results, not a guarantee that -they reproduce the game. - -The file header has `plainKind` (the ordinary, unconditional, whole-team five-second score-up kind used for cross -terms, or null), `host` (how a support skill's paired member is measured), `seedRule` and `shapes`. A shape has a -continuous zero-based `id`, `source` (`member` or `support`), `mission` (1 combo, 2 luck, 3 Just, 4 all), -`bandCondition`, normalized `effects`, and `skills`. Each skill is `{id, level, memberTargetIds, bandIds}`; join its -id to `gekisouCatalog.skills` or `supportSkills` for its name. Members use their skill's highest level; snaps use -the support skill level at their highest rank. Effect rows preserve the master order and expose their effect, -trigger, duration, value, limits, targets, four condition groups and cumulative condition (the exact fields are in -the JSON Schema). Equivalent parameters share a shape. Support condition 5000's target is normalized away; -`memberTargetIds` and `bandIds` retain each skill's targets and bands for matching, or are null without that condition. - -Each chart has `{factors, variants}`, or null when aptitude is disabled, the chart is unplayable with Gekisou, there -are no ranges, or the master has no shapes. `factors`, in range order, has `judgedNotes`, `justNotes`, `perfectNotes`, -`tailNotes`, `comboAtStart`, and `lotteries`: the mean and standard error of the number of lotteries without a skill, -on `deck.seeds` (zero outside luck ranges). - -Variants are in shape-id order, only for the chart's missions or mission 4. A band-conditioned shape appears twice, -`bandMatch: true` then `false`; otherwise `bandMatch` is null. Each variant has: +The file's `deck.gekisouAptitude` has `plainKind`, `host`, `law` and `shapes`. Each shape groups equivalent +score-relevant parameters and retains its source, mission, band condition, normalized effect rows and +skill/level references. Member skills use their highest level; support skills use the highest Snap rank's +level. Support target differences share a shape while each skill retains `memberTargetIds` and `bandIds`. +A support skill is measured with an empty member skill of the same mission. + +Each chart has `{factors, variants}` or null when there are no applicable ranges/shapes or aptitude is +disabled. Factors give judged, Just, Perfect and tail note counts, starting combo and the expected number +of consumed lotteries (`lotteries`, an interval). Each applicable shape has one variant, or matching and +nonmatching band variants. A variant gives: | Field | Content | |---|---| -| `shape`, `bandMatch` | shape and the measured band condition | -| `deterministic`, `seeds`, `seTargetMet`, `crossSeeds` | whether random dependencies were excluded and four initial plays agreed, number of measured seeds, whether the standard-error target was met, number used for cross terms | -| `score`, `scorePerfect`, `tail`, `tailPerfect`, `converted` | score increments, the increments after the ranges, and the conversion increment; each `[mean, standard error]` | -| `ranges` | one object per range: `rangeScore`, `rankBonus`, `rangeScorePerfect`, `maxCombo`, `justCount`, `luckPoints`, each `[mean, standard error]` | -| `weights` | cross terms `[position][mean, standard error]`, null without a plain kind | -| `rangeWeights` | cross terms `[position][range][mean, standard error]`, null without a plain kind or a linear rank model | -| `check` | first seed's check: `seed`, `ranks`, `deck` (plain kind/value or null per slot), `exact`, `predicted`, `bound` | - -Point increments are measured at `deck.model.power`, using the same seed for the equipped and unequipped plays. -`score.mean = tail.mean + sum(rangeScore.mean + rankBonus.mean)`; **standard errors cannot be added this way**. -Point means and errors are rounded to 0.001. With R ranges this equality allows `(2+2R)*0.0005 + 1e-8` points of -independent rounding error. Weights, lottery statistics and the check's prediction and bound keep binary64 precision. -A deterministic result uses one seed and zero standard errors. Charts with luck ranges, effects 11000–11005, -or probability condition 4011 in any condition group always use random sampling, even if the first observations -agree; four matching observations alone do not establish determinism. Others expand through `seedRule.batches` (normally -32, 64, 128, 256, 512, 1024), stopping when the unrounded score error is at most the larger of `relative` (0.01) times -the absolute mean increment and `baseline` (0.001) times the no-skill mean on the same seeds. A result at the cap may -have `seTargetMet: false`. Cross terms use the first `min(seeds, seedRule.crossSeeds)` seeds (normally capped at 64). -The sampled increment is an estimate, not an exact expectation; consult its error and the check bound. - -The exporter checks the shape table against independently read master data, its skill/level coverage and bands, -the variants' references, mission and band coverage, finite `[mean, se]` pairs with nonnegative errors, deterministic -zero errors, array lengths, the tail identity and each check's bound. New exports always include the aptitude keys; -the Schema still accepts older `/1` files without them. +| `score`, `scorePerfect`, `tail`, `tailPerfect`, `converted` | expected gains with the skill minus without it; `[center, half-width]` | +| `ranges` | `rangeScore`, `rankBonus`, `rangeScorePerfect`, `rankBonusPerfect`, `maxCombo`, `justCount`, `luckPoints` gain intervals | +| `weights` | the plain kind's weight changes, `[position][interval]`; null without that kind | +| `rangeWeights` | its range-weight changes, `[position][range][interval]`; null outside the linear domain | +| `check` | the full expected score and linear prediction for a random plain-skill deck at explicit ranks | + +The score and tail centers satisfy `tail = score - sum(rangeScore + rankBonus)`, and the Perfect endpoint +uses the corresponding Perfect fields. Interval radii propagate by outward arithmetic. Conversion, +combo and Just counters have zero radius after proving their independence from lotteries. + +Each variant measures one skill alone. Several gains cannot be added to estimate combined skills: +combo caps, lottery gauges, Rush and Just-triggered support effects interact. Plain-skill cross weights +are measured on the best play. Below 100% Just, a page can interpolate the no-ordinary-skill endpoints; +it cannot derive complete cross weights at that accuracy from this summary. A full play uses replay. +The exporter verifies shape coverage, normalized effect rows, targets, missions, variant ordering, +intervals, tail identities and expectation checks against the supplied master data. ## The deck input (`--full`) @@ -450,17 +390,9 @@ packages. ## Versions -`format` names the major version. Within `nnnotes.music-data/1`, fields and tables are only added, never renamed, -removed or given another meaning, and readers ignore keys they do not know. A reader rejects a file with a major -version it does not know. A change that breaks readers is a new major version, `nnnotes.music-data/2`. The deck -statistics follow the deck model's format (`provenance.deck.format`): a new major version of it is a new major -version of this file. - -`nnnotes.music-data/1` replaces the `nnnotes.songs/1` file of `nnnotes songs` (its fields are the songs, charts and -their facts here) and the `nnnotes.deck-data/1` file of `nnnotes deck-data` (its content is the deck input of -`--full`). - - -Final aptitude exports require the standard-error target for both `score` and `scorePerfect`, each against its own paired no-skill baseline. Sampling extends the same published seed prefix through geometric batches, stopping as soon as both targets agree with the unchanged max(1% of increment, 0.1% of baseline) rule. The 65,536-seed cap is a failure guard, not a requirement to run every seed. An unmet cap aborts a normal export before writing artifacts. `--allow-unconverged-aptitude` is an explicit diagnostic option; it retains real SE values and unmet flags. +`format` names the major version. `nnnotes.music-data/2` carries deck statistics +`ournotes-deck.chart-stats/3`. Additive fields and tables retain their existing meanings within a major +version; readers ignore unknown keys and reject unsupported major versions. Deck model provenance names +the exact statistics format and source digest. The optional runtime input keeps `nnnotes.deck-data/1`. `--replay-dir OUT/replay --replay-engine PKG [--recommend-engine PKG]` writes normalized runtime inputs and the pinned model's WASM release packages (the replay engine and, optionally, the recommendation engine) as described in [replay.md](replay.md). The music data stays compact and carries the SHA-bound `replay.manifestUrl` pointer. No original chart/master blobs or native binary are included in this artifact bundle. diff --git a/docs/replay.md b/docs/replay.md index 7a4c747..d785863 100644 --- a/docs/replay.md +++ b/docs/replay.md @@ -16,14 +16,14 @@ The music-data pointer is: "deckData": {"format":"nnnotes.deck-data/1","url":"deck-data.json","sha256":"","bytes":0}, "charts": [{"scoreId":10000303,"url":"charts/10000303.json","sha256":"","bytes":0,"musicLengthMs":120557,"noteCount":0,"assetSha256":""}], "engine": { - "model": {"name":"ournotes-deck","version":"","source":"","commit":"","format":"ournotes-deck.chart-stats/2"}, + "model": {"name":"ournotes-deck","version":"","source":"","commit":"","format":"ournotes-deck.chart-stats/3"}, "requestFormat":"ournotes.replay/1","class":"ReplaySession","methods":["describeChart","template","run"], "js":{"url":"engine/ournotes_replay_wasm.js","sha256":"","bytes":0}, "wasm":{"url":"engine/ournotes_replay_wasm_bg.wasm","sha256":"","bytes":0}, "build":{"url":"engine/build-info.json","sha256":"","bytes":0} }, "recommendEngine": { - "model": {"name":"ournotes-deck","version":"","source":"","commit":"","format":"ournotes-deck.chart-stats/2"}, + "model": {"name":"ournotes-deck","version":"","source":"","commit":"","format":"ournotes-deck.chart-stats/3"}, "js":{"url":"recommend/ournotes_recommend_wasm.js","sha256":"","bytes":0}, "wasm":{"url":"recommend/ournotes_recommend_wasm_bg.wasm","sha256":"","bytes":0}, "build":{"url":"recommend/build-info.json","sha256":"","bytes":0} diff --git a/docs/schema/music-data.schema.json b/docs/schema/music-data.schema.json index 44e3cdb..9ffdec6 100644 --- a/docs/schema/music-data.schema.json +++ b/docs/schema/music-data.schema.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "urn:nnnotes:schema:music-data", - "title": "nnnotes.music-data/1", + "title": "nnnotes.music-data/2", "description": "Every live song and chart of one master data version: song metadata in every language, chart facts and the deck model's chart statistics, with --full also the deck model's input (every chart's runtime notes and a subset of the master data tables as columns and rows). Written by `nnnotes music-data`; described in docs/music-data.md. Within the arrays of a deck input chart's `notes` (and of its `fevers`) the i-th elements belong to the same note (range): the arrays have equal lengths, which a schema cannot state. Readers ignore keys they do not know.", "type": "object", "required": [ @@ -17,7 +17,7 @@ ], "properties": { "format": { - "const": "nnnotes.music-data/1" + "const": "nnnotes.music-data/2" }, "provenance": { "type": "object", @@ -176,7 +176,7 @@ }, "format": { "type": "string", - "description": "the statistics format, ournotes-deck.chart-stats/2" + "description": "the statistics format, ournotes-deck.chart-stats/3" } } } @@ -691,8 +691,9 @@ "positions", "ranges", "justNotes", - "seeds", - "unplayable" + "unplayable", + "expectation", + "replaySeeds" ], "properties": { "convertedNoteCount": { @@ -765,165 +766,6 @@ "justNotes": { "type": "integer" }, - "seeds": { - "type": "array", - "items": { - "type": "object", - "required": [ - "seed", - "score", - "ranges", - "weights", - "check" - ], - "properties": { - "seed": { - "type": "integer" - }, - "score": { - "type": "integer", - "description": "the exact no-skill score at model.power" - }, - "ranges": { - "type": "array", - "items": { - "type": "object", - "required": [ - "rangeScore", - "rankBonus", - "maxCombo", - "justCount", - "lotResults" - ], - "properties": { - "rangeScore": { - "type": "integer" - }, - "rankBonus": { - "type": "integer" - }, - "maxCombo": { - "type": "integer" - }, - "justCount": { - "type": "integer" - }, - "luckPoints": { - "type": "integer", - "description": "the luck points gained in the range (TotalBonusPoint), a luck range's rank figure" - }, - "lotResults": { - "type": "array", - "items": { - "type": "integer" - }, - "minItems": 4, - "maxItems": 4 - }, - "rangeScorePerfect": { - "type": "integer", - "description": "the range score on the Perfect play (every Just judged Perfect)" - } - } - } - }, - "weights": { - "type": "array", - "items": { - "type": "array", - "items": { - "type": "number" - } - }, - "description": "weights[kind][position]: score per unit of deck power and of factor" - }, - "check": { - "type": "object", - "required": [ - "deck", - "exact", - "predicted", - "bound" - ], - "properties": { - "deck": { - "type": "array", - "items": { - "type": [ - "array", - "null" - ] - } - }, - "exact": { - "type": "integer" - }, - "predicted": { - "type": "number" - }, - "bound": { - "type": "number" - } - } - }, - "scorePerfect": { - "type": "integer", - "description": "the exact no-skill score at model.power on the Perfect play (every Just judged Perfect), its rank 1 bonuses included" - }, - "rangeWeights": { - "type": [ - "array", - "null" - ], - "items": { - "type": [ - "array", - "null" - ], - "items": { - "type": "array", - "items": { - "type": "number" - } - } - }, - "description": "rangeWeights[kind][position][range]: range score per unit of deck power and of factor; null when the ranks do not follow linearly on the chart, a kind null when its conditions read the confirmed rank" - }, - "rankCheck": { - "type": [ - "object", - "null" - ], - "required": [ - "ranks", - "exact", - "predicted", - "bound" - ], - "description": "the check deck played at random ranks; null without ranges or range weights", - "properties": { - "ranks": { - "type": "array", - "items": { - "type": "integer", - "minimum": 1, - "maximum": 5 - } - }, - "exact": { - "type": "integer" - }, - "predicted": { - "type": "number" - }, - "bound": { - "type": "number" - } - } - } - } - } - }, "offSeeds": { "type": "array", "description": "the measurements with Gekisou off (a solo live), one seed", @@ -993,10 +835,27 @@ "string", "null" ], - "description": "why the game cannot play the chart with Gekisou (seeds is then empty; offSeeds is not)" + "description": "why Gekisou cannot play the chart; expectation is null and replaySeeds is empty" }, "gekisouAptitude": { "$ref": "#/$defs/chartGekisouAptitude" + }, + "expectation": { + "anyOf": [ + { + "$ref": "#/$defs/chartExpectation" + }, + { + "type": "null" + } + ] + }, + "replaySeeds": { + "type": "array", + "items": { + "type": "integer" + }, + "description": "Seeds for deterministic replays." } } }, @@ -1372,21 +1231,6 @@ } } }, - "aptitudeMeanSE": { - "type": "array", - "prefixItems": [ - { - "type": "number" - }, - { - "type": "number", - "minimum": 0 - } - ], - "items": false, - "minItems": 2, - "maxItems": 2 - }, "aptitudeCondition": { "type": "object", "required": [ @@ -1637,7 +1481,7 @@ "required": [ "plainKind", "host", - "seedRule", + "law", "shapes" ], "properties": { @@ -1656,47 +1500,16 @@ "type": "string", "minLength": 1 }, - "seedRule": { - "type": "object", - "required": [ - "deterministicTest", - "batches", - "relative", - "baseline", - "crossSeeds" - ], - "properties": { - "deterministicTest": { - "type": "integer", - "minimum": 1 - }, - "batches": { - "type": "array", - "items": { - "type": "integer", - "minimum": 1 - }, - "minItems": 1 - }, - "relative": { - "type": "number", - "minimum": 0 - }, - "baseline": { - "type": "number", - "minimum": 0 - }, - "crossSeeds": { - "type": "integer", - "minimum": 1 - } - } - }, "shapes": { "type": "array", "items": { "$ref": "#/$defs/aptitudeShape" } + }, + "law": { + "type": "string", + "minLength": 1, + "description": "The independent nominal lottery and skill probability law." } } }, @@ -1710,10 +1523,6 @@ "required": [ "shape", "bandMatch", - "deterministic", - "seeds", - "seTargetMet", - "crossSeeds", "score", "scorePerfect", "tail", @@ -1739,34 +1548,20 @@ } ] }, - "deterministic": { - "type": "boolean" - }, - "seeds": { - "type": "integer", - "minimum": 1 - }, - "seTargetMet": { - "type": "boolean" - }, - "crossSeeds": { - "type": "integer", - "minimum": 1 - }, "score": { - "$ref": "#/$defs/aptitudeMeanSE" + "$ref": "#/$defs/estimate" }, "scorePerfect": { - "$ref": "#/$defs/aptitudeMeanSE" + "$ref": "#/$defs/estimate" }, "tail": { - "$ref": "#/$defs/aptitudeMeanSE" + "$ref": "#/$defs/estimate" }, "tailPerfect": { - "$ref": "#/$defs/aptitudeMeanSE" + "$ref": "#/$defs/estimate" }, "converted": { - "$ref": "#/$defs/aptitudeMeanSE" + "$ref": "#/$defs/estimate" }, "ranges": { "type": "array", @@ -1778,26 +1573,30 @@ "rangeScorePerfect", "maxCombo", "justCount", - "luckPoints" + "luckPoints", + "rankBonusPerfect" ], "properties": { "rangeScore": { - "$ref": "#/$defs/aptitudeMeanSE" + "$ref": "#/$defs/estimate" }, "rankBonus": { - "$ref": "#/$defs/aptitudeMeanSE" + "$ref": "#/$defs/estimate" }, "rangeScorePerfect": { - "$ref": "#/$defs/aptitudeMeanSE" + "$ref": "#/$defs/estimate" }, "maxCombo": { - "$ref": "#/$defs/aptitudeMeanSE" + "$ref": "#/$defs/estimate" }, "justCount": { - "$ref": "#/$defs/aptitudeMeanSE" + "$ref": "#/$defs/estimate" }, "luckPoints": { - "$ref": "#/$defs/aptitudeMeanSE" + "$ref": "#/$defs/estimate" + }, + "rankBonusPerfect": { + "$ref": "#/$defs/estimate" } } } @@ -1807,7 +1606,7 @@ { "type": "array", "items": { - "$ref": "#/$defs/aptitudeMeanSE" + "$ref": "#/$defs/estimate" } }, { @@ -1822,7 +1621,7 @@ "items": { "type": "array", "items": { - "$ref": "#/$defs/aptitudeMeanSE" + "$ref": "#/$defs/estimate" } } }, @@ -1832,63 +1631,7 @@ ] }, "check": { - "type": "object", - "required": [ - "seed", - "ranks", - "deck", - "exact", - "predicted", - "bound" - ], - "properties": { - "seed": { - "type": "integer" - }, - "ranks": { - "type": "array", - "items": { - "type": "integer", - "minimum": 1, - "maximum": 5 - } - }, - "deck": { - "type": "array", - "items": { - "anyOf": [ - { - "type": "array", - "prefixItems": [ - { - "type": "integer", - "minimum": 0 - }, - { - "type": "integer" - } - ], - "items": false, - "minItems": 2, - "maxItems": 2 - }, - { - "type": "null" - } - ] - } - }, - "exact": { - "type": "integer" - }, - "predicted": { - "type": "number" - }, - "bound": { - "type": "number", - "minimum": 0 - } - } + "$ref": "#/$defs/expectationCheck" } } }, @@ -1935,7 +1678,7 @@ "minimum": 0 }, "lotteries": { - "$ref": "#/$defs/aptitudeMeanSE" + "$ref": "#/$defs/estimate" } } } @@ -1952,6 +1695,182 @@ "type": "null" } ] + }, + "estimate": { + "type": "array", + "minItems": 2, + "maxItems": 2, + "prefixItems": [ + { + "type": "number" + }, + { + "type": "number", + "minimum": 0 + } + ], + "items": false, + "description": "The center and outward half-width of a nominal expectation interval." + }, + "expectationCheck": { + "type": "object", + "required": [ + "deck", + "ranks", + "expected", + "predicted", + "bound" + ], + "properties": { + "deck": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer" + }, + "minItems": 2, + "maxItems": 2 + }, + { + "type": "null" + } + ] + } + }, + "ranks": { + "type": [ + "array", + "null" + ], + "items": { + "type": "integer", + "minimum": 1, + "maximum": 5 + } + }, + "expected": { + "$ref": "#/$defs/estimate" + }, + "predicted": { + "$ref": "#/$defs/estimate" + }, + "bound": { + "type": "number", + "minimum": 0 + } + }, + "description": "A full nominal expectation and its linear prediction; the worst endpoint separation is within bound." + }, + "chartExpectation": { + "type": "object", + "required": [ + "score", + "scorePerfect", + "ranges", + "weights", + "rangeWeights", + "check", + "rankCheck" + ], + "properties": { + "score": { + "$ref": "#/$defs/estimate" + }, + "scorePerfect": { + "$ref": "#/$defs/estimate" + }, + "ranges": { + "type": "array", + "items": { + "type": "object", + "required": [ + "rangeScore", + "rankBonus", + "rankBonusPerfect", + "rangeScorePerfect", + "luckPoints", + "maxCombo", + "justCount", + "lotResults" + ], + "properties": { + "rangeScore": { + "$ref": "#/$defs/estimate" + }, + "rankBonus": { + "$ref": "#/$defs/estimate" + }, + "rankBonusPerfect": { + "$ref": "#/$defs/estimate" + }, + "rangeScorePerfect": { + "$ref": "#/$defs/estimate" + }, + "luckPoints": { + "$ref": "#/$defs/estimate" + }, + "maxCombo": { + "type": "integer" + }, + "justCount": { + "type": "integer" + }, + "lotResults": { + "type": "array", + "items": { + "$ref": "#/$defs/estimate" + }, + "minItems": 4, + "maxItems": 4, + "description": "Expected consumed result counts: Miss, Hit, Super Hit, Critical." + } + } + } + }, + "weights": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/estimate" + } + } + }, + "rangeWeights": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "array", + "null" + ], + "items": { + "type": "array", + "items": { + "$ref": "#/$defs/estimate" + } + } + } + }, + "check": { + "$ref": "#/$defs/expectationCheck" + }, + "rankCheck": { + "anyOf": [ + { + "$ref": "#/$defs/expectationCheck" + }, + { + "type": "null" + } + ] + } + } } } } diff --git a/rust/src/lib.rs b/rust/src/lib.rs index aec368c..6405055 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -1,10 +1,10 @@ -//! `nnnotes._deck`: the chart statistics of the deck model ournotes-sim (`ournotes-deck.chart-stats/2`) on a deck +//! `nnnotes._deck`: the chart statistics of the deck model ournotes-sim (`ournotes-deck.chart-stats/3`) on a deck //! data document held in memory, the charts measured in parallel. use std::sync::Mutex; use std::sync::atomic::{AtomicUsize, Ordering}; -use ournotes_sim::chartstats::{self, AptitudeOptions, ChartStats, GEKISOU_SEEDS, Options}; +use ournotes_sim::chartstats::{self, ChartStats, REPLAY_SEEDS, Options}; use ournotes_sim::data::DeckData; use ournotes_sim::error::Error; use pyo3::exceptions::PyValueError; @@ -53,34 +53,23 @@ fn measure(data: &DeckData, options: &Options, workers: usize) -> Result str +/// chart_stats(data, seeds=None, workers=None, aptitude=True) -> str /// -/// The `ournotes-deck.chart-stats/2` document (JSON text) of a deck data document (`nnnotes.deck-data/1` JSON text), -/// as `ournotes-deck chart-stats` writes it. `seeds`: the size of the seed set of charts with a luck range (default +/// The `ournotes-deck.chart-stats/3` document (JSON text) of a deck data document (`nnnotes.deck-data/1` JSON text), +/// as `ournotes-deck chart-stats` writes it. `seeds`: the replay seed count of charts with a luck range (default /// 8); `workers`: threads measuring charts (default: the available parallelism). Raises ValueError for data the /// deck model cannot read or a chart whose check deck fails. `aptitude`: measure single Gekisou skill shapes -/// (default true); `aptitude_max_seeds` and `aptitude_cross_seeds`: sample caps (defaults 65536 and 64). +/// (default true). Gekisou statistics use independent nominal lottery and skill probabilities. #[pyfunction] -#[pyo3(signature = (data, seeds=None, workers=None, aptitude=true, aptitude_max_seeds=None, aptitude_cross_seeds=None))] +#[pyo3(signature = (data, seeds=None, workers=None, aptitude=true))] fn chart_stats( py: Python<'_>, data: &str, seeds: Option, workers: Option, aptitude: bool, - aptitude_max_seeds: Option, - aptitude_cross_seeds: Option, ) -> PyResult { - let seeds = seeds.unwrap_or(GEKISOU_SEEDS); - let defaults = AptitudeOptions::default(); - let options = Options { - seeds, - aptitude: aptitude.then_some(AptitudeOptions { - max_seeds: aptitude_max_seeds.unwrap_or(defaults.max_seeds), - cross_seeds: aptitude_cross_seeds.unwrap_or(defaults.cross_seeds), - }), - }; + let options = Options { replay_seeds: seeds.unwrap_or(REPLAY_SEEDS), aptitude }; let workers = workers.unwrap_or_else(|| std::thread::available_parallelism().map_or(1, |n| n.get())); py.detach(|| { let mut data = DeckData::from_json(data).map_err(value_error)?; diff --git a/src/nnnotes/cli.py b/src/nnnotes/cli.py index cf4f083..b812280 100644 --- a/src/nnnotes/cli.py +++ b/src/nnnotes/cli.py @@ -678,8 +678,7 @@ def cmd_music_data(args, cfg): try: deck = None if args.no_deck else musicdata.Deck( seeds=args.seeds, workers=args.workers, aptitude=not args.no_gekisou_aptitude, - aptitude_max_seeds=args.aptitude_max_seeds, aptitude_cross_seeds=args.aptitude_cross_seeds, - require_convergence=not args.allow_unconverged_aptitude, cache=args.stats_cache) + cache=args.stats_cache) if args.apk_master: src, region = deckdata.apk_master(apk), deckdata.EMBEDDED elif args.decoded_master: # decoded elsewhere: no master key @@ -987,18 +986,11 @@ def target(m, what): c.add_argument("--no-deck", action="store_true", help="do not run the deck model (every chart's deck is null)") c.add_argument("--seeds", type=int, default=8, metavar="N", - help="seeds measured on a chart with a luck range (default 8)") + help="replay seeds of a chart with a luck range (default 8); statistics use nominal expectations") c.add_argument("--workers", type=int, metavar="N", help="threads measuring charts (default: every processor)") c.add_argument("--no-gekisou-aptitude", action="store_true", help="leave out the charts' Gekisou aptitude (every gekisouAptitude is null)") - c.add_argument("--aptitude-max-seeds", type=int, metavar="N", - help="seeds of a Gekisou aptitude variant at most (default: the deck model's, 65536); " - "sampling stops earlier when both score targets converge") - c.add_argument("--allow-unconverged-aptitude", action="store_true", - help="diagnostic export only: retain unmet SE flags at the sample cap; final exports reject them") - c.add_argument("--aptitude-cross-seeds", type=int, metavar="N", - help="seeds of a Gekisou aptitude variant's cross terms (default: the deck model's, 64)") c.add_argument("--stats-cache", type=Path, metavar="DIR", help="keep the charts' deck statistics in DIR: a chart whose model sources, options, master tables " "and chart are unchanged is not measured again; DIR then holds this export's charts only") diff --git a/src/nnnotes/musicdata.py b/src/nnnotes/musicdata.py index a32d900..58a7e81 100644 --- a/src/nnnotes/musicdata.py +++ b/src/nnnotes/musicdata.py @@ -1,5 +1,5 @@ """Music data: one JSON file with every live song and chart of one master data version (format -`nnnotes.music-data/1`, docs/music-data.md): titles and credits in every language, bands, vocal characters, category, +`nnnotes.music-data/2`, docs/music-data.md): titles and credits in every language, bands, vocal characters, category, tags, release time, score ranks, the live BGM's length, the Gekisou catalog (member cards, snaps and their Gekisou skills), per difficulty the chart facts (level, note counts, BPM, chart times, skill events, fever ranges) and the chart's deck statistics, what the chart contributes to the live score whatever the deck, with Gekisou on (a Gekisou @@ -33,7 +33,7 @@ from . import deckdata from .languages import LANGUAGES -FORMAT = "nnnotes.music-data/1" +FORMAT = "nnnotes.music-data/2" DIFFICULTIES = ("easy", "normal", "hard", "expert") MUSIC_LENGTH_TAIL_MS = 1000 # the live's music length: the last note time + 1000 ms (LiveScore skip path) @@ -299,20 +299,19 @@ def skill(r: dict, max_level: dict) -> dict: # ---------------------------------------------------------------- the deck model -DECK_SEEDS = 8 # seed set size of a chart with a luck range (ournotes-deck's default) +DECK_SEEDS = 8 # replay seed count of a chart with a luck range class Deck: """The deck model (nnnotes._deck): chart statistics of a deck input document, on `workers` threads (None: the - available parallelism), `seeds` seeds for a chart with a luck range; with `aptitude` every chart's Gekisou aptitude - too, a variant on at most `aptitude_max_seeds` seeds and its cross terms on at most `aptitude_cross_seeds` (None: - the deck model's defaults). With a `cache` directory, a chart's statistics are kept there under the SHA-256 of + available parallelism), with independent nominal lottery and skill probabilities. `seeds` gives the number of + replay seeds for a chart with a luck range; `aptitude` includes every chart's Gekisou aptitude and cross terms. + With a `cache` directory, a chart's statistics are kept there under the SHA-256 of what they are a function of: the model's sources (its `sourceSha256`), the options above, the master tables and the chart; a chart found there is not measured again.""" def __init__(self, seeds: int = DECK_SEEDS, workers: int | None = None, module=None, aptitude: bool = True, - aptitude_max_seeds: int | None = None, aptitude_cross_seeds: int | None = None, - require_convergence: bool = True, cache: Path | None = None): + cache: Path | None = None): if module is None: try: from . import _deck as module @@ -320,9 +319,7 @@ def __init__(self, seeds: int = DECK_SEEDS, workers: int | None = None, module=N raise MusicDataError("the deck model (nnnotes._deck) is not built into this installation: install " "nnnotes from a wheel or build it (maturin), or pass --no-deck") from None self.module, self.seeds, self.workers = module, seeds, workers - self.aptitude, self.aptitude_max_seeds, self.aptitude_cross_seeds = \ - aptitude, aptitude_max_seeds, aptitude_cross_seeds - self.require_convergence = require_convergence + self.aptitude = aptitude self.cache = None if cache is None else Path(cache) self.counts: dict | None = None # {cached, measured} charts of the last stats() @@ -334,8 +331,7 @@ def info(self) -> dict: def _measure(self, deck_input: dict) -> dict: try: text = self.module.chart_stats(deckdata.encode(deck_input).decode("utf-8"), self.seeds, self.workers, - aptitude=self.aptitude, aptitude_max_seeds=self.aptitude_max_seeds, - aptitude_cross_seeds=self.aptitude_cross_seeds) + aptitude=self.aptitude) except ValueError as e: raise MusicDataError(f"deck model: {e}") from None doc = json.loads(text, parse_float=deckdata._Num) @@ -347,8 +343,7 @@ def cache_keys(self, deck_input: dict) -> list[str]: """The cache key of each chart of a deck input document, in chart order.""" info = self.info() base = {"sourceSha256": info["sourceSha256"], "format": info["format"], "seeds": self.seeds, - "aptitude": self.aptitude, "aptitudeMaxSeeds": self.aptitude_max_seeds, - "aptitudeCrossSeeds": self.aptitude_cross_seeds, + "aptitude": self.aptitude, "master": hashlib.sha256(deckdata.encode(deck_input["master"])).hexdigest()} return [hashlib.sha256(deckdata.encode({**base, "chart": hashlib.sha256(deckdata.encode(c)).hexdigest()})) .hexdigest() for c in deck_input["charts"]] @@ -387,21 +382,12 @@ def stats(self, deck_input: dict) -> dict: self.counts = {"cached": 0, "measured": len(deck_input["charts"])} else: doc = self._cached_stats(deck_input) - if self.require_convergence and self.aptitude: - unmet = [(c.get("scoreId"), v.get("shape"), v.get("bandMatch"), v.get("seeds")) - for c in doc.get("charts", []) - for v in (c.get("gekisouAptitude") or {}).get("variants", []) - if v.get("seTargetMet") is not True] - if unmet: - chart, shape, band, seeds = unmet[0] - raise MusicDataError(f"deck model: {len(unmet)} aptitude variants did not meet BOTH score SE targets; " - f"chart {chart}, shape {shape}, bandMatch {band}, seeds {seeds}; " - "increase --aptitude-max-seeds; no final artifact written") return doc # the keys of a chart's statistics carried by `deck` (the others are checked against the chart facts) -DECK_CHART_KEYS = ("convertedNoteCount", "skip", "events", "positions", "ranges", "justNotes", "seeds", "offSeeds", +DECK_CHART_KEYS = ("convertedNoteCount", "skip", "events", "positions", "ranges", "justNotes", "expectation", + "replaySeeds", "offSeeds", "unplayable", "gekisouAptitude") RANKS = 5 # the ranks of a Gekisou range (a Gekisou live has up to five players) GEKISOU_RANGES = 3 # the Gekisou ranges of a live (its first three fevers) @@ -458,12 +444,6 @@ def _check_bound(c, what: str, where: str) -> None: f"{c['bound']}") -def _check_rank_check(rc: dict, n: int, s: str, where: str) -> None: - if len(rc["ranks"]) != n or not all(1 <= r <= RANKS for r in rc["ranks"]): - raise MusicDataError(f"{where}: {s}: rank check ranks {rc['ranks']!r}") - _check_bound(rc, f"{s}: the check deck at ranks {rc['ranks']!r}", where) - - # ournotes-sim's published seed set (live::seeds): candidate k is the low 32 bits (a signed integer) of output k + 1 # of SplitMix64 started at SEED_ORIGIN (the ASCII bytes of `gekisou1`); a candidate is kept when its pair of effective # stream seeds (|b| and |b ^ 0x9E3779B9|, -2^31 taken as 2^31 - 1) differs from that of every seed kept before it @@ -497,37 +477,74 @@ def eff(x: int) -> int: def chart_seeds(stats: dict, seeds: int) -> list[int]: - """The seeds of a chart's Gekisou on statistics: none when it is unplayable with Gekisou, seed 0 when no range is - a luck range (the play draws nothing), else the first `seeds` published seeds.""" + """The chart's replay seeds: none when unplayable with Gekisou, 0 without a luck range, otherwise the first + `seeds` published seeds. Nominal expectation measurements are independent of this list.""" if stats.get("unplayable") is not None: return [] return published_seeds(seeds) if any(r["mission"] == MISSION_LUCK for r in stats["ranges"]) else [0] -def _check_seed_shapes(stats: dict, kinds: int, where: str) -> None: - """The array shapes and the checks of a chart's statistics: `weights[kind][position]` (a kind null only with - Gekisou off), `rangeWeights[kind][position][range]` (null, or a kind null), one Gekisou off seed, the seeds' - ranges one per range with their luck points, and every check (and rank check) within its bound.""" +def _check_interval_bound(c, what: str, where: str) -> None: + """The largest separation of the expected and predicted intervals is within the stated bound.""" + if not isinstance(c, dict) or any(k not in c for k in CHECK_KEYS): + raise MusicDataError(f"{where}: {what}: no expectation check") + if not (_estimate(c["expected"]) and _estimate(c["predicted"]) and _number(c["bound"]) + and float(c["bound"]) >= 0): + raise MusicDataError(f"{where}: {what}: invalid expected, predicted or bound") + error = abs(float(c["expected"][0]) - float(c["predicted"][0])) + \ + float(c["expected"][1]) + float(c["predicted"][1]) + if error > float(c["bound"]) and not _same(error, float(c["bound"])): + raise MusicDataError(f"{where}: {what}: expected {c['expected']}, predicted {c['predicted']} " + f"beyond the bound {c['bound']}") + + +def _check_expectation_shapes(stats: dict, kinds: int, where: str) -> None: + """Expected intervals and their array dimensions, deterministic counters and validation decks.""" positions, n = stats["positions"], len(stats["ranges"]) - for seed in stats["seeds"]: - s = f"seed {seed['seed']}" - w = seed.get("weights") - if not (isinstance(w, list) and len(w) == kinds and all(_numbers(x, positions) for x in w)): - raise MusicDataError(f"{where}: {s}: weights are not [kind][position]") - if len(seed["ranges"]) != n: - raise MusicDataError(f"{where}: {s}: {len(seed['ranges'])} range results for {n} ranges") - for i, r in enumerate(seed["ranges"]): - if not _int(r.get("luckPoints")): - raise MusicDataError(f"{where}: {s} range {i}: no luck points (luckPoints)") - rw = seed.get("rangeWeights") + expected = stats.get("expectation") + if stats.get("unplayable") is not None: + if expected is not None: + raise MusicDataError(f"{where}: an expectation on an unplayable Gekisou chart") + else: + if not isinstance(expected, dict): + raise MusicDataError(f"{where}: no nominal expectation") + if not all(_estimate(expected.get(k)) for k in ("score", "scorePerfect")): + raise MusicDataError(f"{where}: expectation scores are not [center, half-width]") + w = expected.get("weights") + if not (isinstance(w, list) and len(w) == kinds and all(isinstance(x, list) and len(x) == positions + and all(_estimate(y) for y in x) for x in w)): + raise MusicDataError(f"{where}: expectation weights are not [kind][position] intervals") + rs = expected.get("ranges") + if not isinstance(rs, list) or len(rs) != n: + raise MusicDataError(f"{where}: expectation range results differ from the {n} ranges") + for i, r in enumerate(rs): + if not (isinstance(r, dict) and all(_estimate(r.get(k)) for k in + ("rangeScore", "rangeScorePerfect", "rankBonus", "rankBonusPerfect", "luckPoints")) + and _int(r.get("maxCombo")) and _int(r.get("justCount")) + and isinstance(r.get("lotResults"), list) and len(r["lotResults"]) == 4 + and all(_estimate(x) for x in r["lotResults"])): + raise MusicDataError(f"{where}: expectation range {i}: invalid scores, counters or lottery results") + rw = expected.get("rangeWeights") if rw is not None and not (isinstance(rw, list) and len(rw) == kinds and all( - k is None or (isinstance(k, list) and len(k) == positions and all(_numbers(x, n) for x in k)) - for k in rw)): - raise MusicDataError(f"{where}: {s}: rangeWeights are not [kind][position][range]") - _check_bound(seed["check"], f"{s}: the check deck", where) - rc = seed.get("rankCheck") - if rc is not None: - _check_rank_check(rc, n, s, where) + k is None or (isinstance(k, list) and len(k) == positions and all(isinstance(x, list) and len(x) == n + and all(_estimate(y) for y in x) + for x in k)) for k in rw)): + raise MusicDataError(f"{where}: expectation rangeWeights are not [kind][position][range] intervals") + for name, c in (("the check deck", expected.get("check")), ("rank check", expected.get("rankCheck"))): + if c is None and name == "rank check": + continue + _check_interval_bound(c, name, where) + if not (isinstance(c["deck"], list) and len(c["deck"]) == positions and all( + x is None or isinstance(x, list) and len(x) == 2 and _int(x[0]) and 0 <= x[0] < kinds + and _int(x[1]) for x in c["deck"])): + raise MusicDataError(f"{where}: {name}: invalid check deck") + ranks = c["ranks"] + if name == "rank check": + if not (isinstance(ranks, list) and len(ranks) == n + and all(_int(r) and 1 <= r <= RANKS for r in ranks)): + raise MusicDataError(f"{where}: rank check ranks {ranks!r}") + elif ranks is not None: + raise MusicDataError(f"{where}: the baseline check deck has fixed rank arrivals") off = stats.get("offSeeds") if not isinstance(off, list) or len(off) != 1: raise MusicDataError(f"{where}: the deck model gives no Gekisou off statistics (offSeeds)") @@ -537,17 +554,15 @@ def _check_seed_shapes(stats: dict, kinds: int, where: str) -> None: raise MusicDataError(f"{where}: Gekisou off: weights are not [kind][position]") _check_bound(seed["check"], "Gekisou off: the check deck", where) - # ---------------------------------------------------------------- the Gekisou aptitude PLAIN_EFFECT_TYPE, PLAIN_MS = 2000, 5000 # the plain kind: score up on the whole deck for 5 s, nothing else CONDITION_MEMBER_TARGET = 5000 # the skill condition on the member a snap's support skill is paired with -APTITUDE_KEYS = ("plainKind", "host", "seedRule", "shapes") -SEED_RULE_KEYS = ("deterministicTest", "batches", "relative", "baseline", "crossSeeds") +APTITUDE_KEYS = ("plainKind", "host", "law", "shapes") SHAPE_KEYS = ("id", "source", "mission", "bandCondition", "effects", "skills") FACTOR_COUNTS = ("judgedNotes", "justNotes", "perfectNotes", "tailNotes", "comboAtStart") VARIANT_MEANS = ("score", "scorePerfect", "tail", "tailPerfect", "converted") -RANGE_MEANS = ("rangeScore", "rankBonus", "rangeScorePerfect", "maxCombo", "justCount", "luckPoints") -CHECK_KEYS = ("seed", "ranks", "deck", "exact", "predicted", "bound") +RANGE_MEANS = ("rangeScore", "rankBonus", "rankBonusPerfect", "rangeScorePerfect", "maxCombo", "justCount", "luckPoints") +CHECK_KEYS = ("ranks", "deck", "expected", "predicted", "bound") CONDITION_GROUPS = (("trigger", "_skillTriggerConditionGroup"), ("condition", "_skillConditionGroup"), ("release", "_skillReleaseConditionGroup"), ("reset", "_effectExecuteLimitResetConditionGroup")) MEAN_TOLERANCE = 1e-9 # relative: a mean against its recomputation (binary64 sums) @@ -576,8 +591,8 @@ def _number(v) -> bool: return False -def _mean_se(v) -> bool: - """Whether v is `[mean, standard error]`: two finite numbers, the error not negative.""" +def _estimate(v) -> bool: + """Whether v is `[center, half-width]`: two finite numbers, the radius not negative.""" return isinstance(v, list) and len(v) == 2 and all(_number(x) for x in v) and float(v[1]) >= 0 @@ -585,13 +600,6 @@ def _same(a: float, b: float, scale: float = 1.0) -> bool: return abs(a - b) <= MEAN_TOLERANCE * max(1.0, abs(scale), abs(a), abs(b)) -def mean_se(xs: list[int]) -> tuple[float, float]: - """The mean of xs and its standard error (the sample standard deviation / sqrt(n); 0 for one value).""" - n = len(xs) - m = sum(xs) / n - return m, (math.sqrt(sum((x - m) ** 2 for x in xs) / (n - 1) / n) if n > 1 else 0.0) - - def _f32(v) -> float: import numpy as np return float(np.float32(float(v))) @@ -720,15 +728,8 @@ def check_header(self, info, kinds: list[dict], model: dict) -> dict | None: raise MusicDataError(f"{where}: the support skills' host is not described (host)") if not (isinstance(model.get("gekisouAptitude"), str) and model["gekisouAptitude"]): raise MusicDataError("deck model: the model does not describe the aptitude (model.gekisouAptitude)") - rule = info["seedRule"] - if not (isinstance(rule, dict) and all(k in rule for k in SEED_RULE_KEYS) - and _int(rule["deterministicTest"]) and rule["deterministicTest"] >= 1 - and isinstance(rule["batches"], list) and rule["batches"] and all(_int(b) for b in rule["batches"]) - and rule["batches"][0] >= 2 and all(a < b for a, b in zip(rule["batches"], rule["batches"][1:])) - and _number(rule["relative"]) and float(rule["relative"]) >= 0 - and _number(rule["baseline"]) and float(rule["baseline"]) >= 0 - and _int(rule["crossSeeds"]) and rule["crossSeeds"] >= 1): - raise MusicDataError(f"{where}: seed rule {rule!r}") + if not isinstance(info["law"], str) or not info["law"]: + raise MusicDataError(f"{where}: the nominal probability law is not described (law)") shapes = info["shapes"] if not isinstance(shapes, list) or [s.get("id") if isinstance(s, dict) else s for s in shapes] != list( range(len(shapes))): @@ -784,7 +785,7 @@ def chart(self, where: str, stats: dict, plain: int | None) -> dict | None: """A chart's `gekisouAptitude` after checking it: null exactly when the aptitude is left out, the master data has no shape, the chart is unplayable with Gekisou or has no Gekisou range; else a factor per range and a variant per shape of the chart's missions (both band match results of a shape with a band condition), each - with its means shaped and consistent (the tail, a deterministic variant's rank bonuses) and its check within + with its intervals shaped and consistent (the best and Perfect tails) and its check within its bound.""" apt = stats.get("gekisouAptitude") if self.header is None or not self.shapes or stats.get("unplayable") is not None or not stats["ranges"]: @@ -798,7 +799,7 @@ def chart(self, where: str, stats: dict, plain: int | None) -> dict | None: if not (isinstance(apt, dict) and isinstance(apt.get("factors"), list) and isinstance(apt.get("variants"), list)): raise MusicDataError(f"{where}: none (gekisouAptitude {{factors, variants}})") - ranges, base = stats["ranges"], stats["seeds"] + ranges, base = stats["ranges"], stats["expectation"] self._factors(where, apt["factors"], stats) missions = {r["mission"] for r in ranges} want = [(s["id"], b) for s in self.shapes if s["mission"] == MISSION_ALL or s["mission"] in missions @@ -816,14 +817,14 @@ def chart(self, where: str, stats: dict, plain: int | None) -> dict | None: return {"factors": apt["factors"], "variants": apt["variants"]} def _factors(self, where: str, factors: list, stats: dict) -> None: - ranges, base = stats["ranges"], stats["seeds"] + ranges, base = stats["ranges"], stats["expectation"] if len(factors) != len(ranges): raise MusicDataError(f"{where}: {len(factors)} factors for {len(ranges)} ranges") just = 0 for j, (f, r) in enumerate(zip(factors, ranges)): at = f"{where}: range {j} factors" if not (isinstance(f, dict) and all(_int(f.get(k)) and f[k] >= 0 for k in FACTOR_COUNTS) - and _mean_se(f.get("lotteries"))): + and _estimate(f.get("lotteries"))): raise MusicDataError(f"{at}: not {{{', '.join(FACTOR_COUNTS)}, lotteries}}: {f!r}") if r["mission"] != MISSION_JUST and (f["justNotes"], f["perfectNotes"]) != (0, 0): raise MusicDataError(f"{at}: {f['justNotes']} Just and {f['perfectNotes']} Perfect notes outside a " @@ -831,109 +832,66 @@ def _factors(self, where: str, factors: list, stats: dict) -> None: if f["justNotes"] + f["perfectNotes"] > f["judgedNotes"] or f["comboAtStart"] > stats["judgedNotes"]: raise MusicDataError(f"{at}: note counts {f!r} beyond the range's or the chart's") just += f["justNotes"] - m, se = mean_se([sum(s["ranges"][j]["lotResults"]) for s in base]) if base else (0.0, 0.0) - if r["mission"] != MISSION_LUCK: - m, se = 0.0, 0.0 - if not (_same(float(f["lotteries"][0]), m) and _same(float(f["lotteries"][1]), se, m)): - raise MusicDataError(f"{at}: lotteries {f['lotteries']}, the chart's seeds give [{m!r}, {se!r}]") + estimates = base["ranges"][j]["lotResults"] + center = sum(float(x[0]) for x in estimates) if r["mission"] == MISSION_LUCK else 0.0 + radius = sum(float(x[1]) for x in estimates) if r["mission"] == MISSION_LUCK else 0.0 + if abs(float(f["lotteries"][0]) - center) > float(f["lotteries"][1]) + radius + MEAN_TOLERANCE: + raise MusicDataError(f"{at}: lotteries {f['lotteries']}, expected counts give [{center}, {radius}]") if just > stats["justNotes"]: raise MusicDataError(f"{where}: {just} Just notes in the ranges, {stats['justNotes']} on the play") - def _variant(self, at: str, v: dict, stats: dict, base: list, plain: int | None) -> None: - rule, n, positions = self.header["seedRule"], len(stats["ranges"]), stats["positions"] - if not (isinstance(v.get("deterministic"), bool) and isinstance(v.get("seTargetMet"), bool) - and _int(v.get("seeds")) and _int(v.get("crossSeeds"))): - raise MusicDataError(f"{at}: deterministic, seeds, seTargetMet, crossSeeds " - f"{[v.get(k) for k in ('deterministic', 'seeds', 'seTargetMet', 'crossSeeds')]!r}") - det = v["deterministic"] - effects = self.shapes[v["shape"]]["effects"] - random = any(r["mission"] == MISSION_LUCK for r in stats["ranges"]) or any( - 11000 <= e["effectType"] <= 11005 or any( - c["type"] == 4011 for key, _ in CONDITION_GROUPS for group in e[key] for c in group) - for e in effects) - if det and random: - raise MusicDataError(f"{at}: a deterministic variant with a random dependency") - if det and (v["seeds"], v["seTargetMet"]) != (1, True): - raise MusicDataError(f"{at}: deterministic on {v['seeds']} seeds (seTargetMet {v['seTargetMet']})") - if not det and (v["seeds"] not in rule["batches"] - or not v["seTargetMet"] and v["seeds"] != rule["batches"][-1]): - raise MusicDataError(f"{at}: {v['seeds']} seeds (seTargetMet {v['seTargetMet']}), not a batch of the seed " - f"rule {rule['batches']}") - if v["crossSeeds"] != min(v["seeds"], rule["crossSeeds"]): - raise MusicDataError(f"{at}: {v['crossSeeds']} cross seeds, not min({v['seeds']}, {rule['crossSeeds']})") + def _variant(self, at: str, v: dict, stats: dict, base: dict, plain: int | None) -> None: + n, positions = len(stats["ranges"]), stats["positions"] pairs = [(k, v.get(k)) for k in VARIANT_MEANS] rs = v.get("ranges") if not (isinstance(rs, list) and len(rs) == n and all(isinstance(r, dict) for r in rs)): - raise MusicDataError(f"{at}: {len(rs) if isinstance(rs, list) else rs!r} range results for {n} ranges") + raise MusicDataError(f"{at}: invalid range results for {n} ranges") pairs += [(f"ranges[{j}].{k}", r.get(k)) for j, r in enumerate(rs) for k in RANGE_MEANS] w, rw = v.get("weights"), v.get("rangeWeights") - linear = bool(base) and base[0].get("rangeWeights") is not None + linear = base.get("rangeWeights") is not None if plain is None: if w is not None or rw is not None: raise MusicDataError(f"{at}: weights without a plain kind") else: - if not (isinstance(w, list) and len(w) == positions): - raise MusicDataError(f"{at}: weights are not [position] of [mean, se]") + if not isinstance(w, list) or len(w) != positions: + raise MusicDataError(f"{at}: weights are not [position] intervals") pairs += [(f"weights[{k}]", x) for k, x in enumerate(w)] if (rw is not None) != linear: - raise MusicDataError(f"{at}: rangeWeights {'given' if rw is not None else 'null'}, the chart's seeds " - f"have {'them' if linear else 'none'}") + raise MusicDataError(f"{at}: rangeWeights differ from the baseline's range weight domain") if rw is not None: - if not (isinstance(rw, list) and len(rw) == positions and all(isinstance(x, list) and len(x) == n - for x in rw)): - raise MusicDataError(f"{at}: rangeWeights are not [position][range] of [mean, se]") + if not (isinstance(rw, list) and len(rw) == positions + and all(isinstance(x, list) and len(x) == n for x in rw)): + raise MusicDataError(f"{at}: rangeWeights are not [position][range] intervals") pairs += [(f"rangeWeights[{k}][{j}]", y) for k, x in enumerate(rw) for j, y in enumerate(x)] for name, x in pairs: - if not _mean_se(x): - raise MusicDataError(f"{at}: {name} {x!r} is not [mean, standard error]") - if det and float(x[1]) != 0: - raise MusicDataError(f"{at}: {name} {x!r}: a deterministic variant with a standard error") - m = {k: float(v[k][0]) for k in VARIANT_MEANS} - terms = [float(r[k][0]) for r in rs for k in ("rangeScore", "rankBonus")] - # Every serialized point mean is independently rounded to 0.001 (score, tail and two terms per range). - if abs(m["tail"] - (m["score"] - sum(terms))) > (len(terms) + 2) * 0.0005 + 1e-8: - raise MusicDataError(f"{at}: tail {v['tail'][0]} is not score {v['score'][0]} - {sum(terms)!r} (the " - f"ranges' scores and rank bonuses)") - if det: - ints = [float(x[0]) for name, x in pairs if not name.startswith(("weights", "rangeWeights"))] - if not all(x.is_integer() for x in ints): - raise MusicDataError(f"{at}: a deterministic variant with a fraction of a point") - perfect = 0 # the Perfect play's range scores and rank bonuses - for j, (r, info, r0) in enumerate(zip(rs, stats["ranges"], base[0]["ranges"])): - p = info["rankBonusPercent"] - want = _trunc_percent(r0["rangeScore"] + int(float(r["rangeScore"][0])), p) - r0["rankBonus"] - if int(float(r["rankBonus"][0])) != want: - raise MusicDataError(f"{at}: range {j}: rank bonus {r['rankBonus'][0]} is not trunc((" - f"{r0['rangeScore']} + {r['rangeScore'][0]}) * {p} / 100) - " - f"{r0['rankBonus']} = {want}") - d, r0p = int(float(r["rangeScorePerfect"][0])), r0["rangeScorePerfect"] - perfect += d + _trunc_percent(r0p + d, p) - _trunc_percent(r0p, p) - if m["tailPerfect"] != m["scorePerfect"] - perfect: - raise MusicDataError(f"{at}: tail on the Perfect play {v['tailPerfect'][0]} is not scorePerfect " - f"{v['scorePerfect'][0]} - {perfect} (the ranges' Perfect play scores and rank " - f"bonuses)") + if not _estimate(x): + raise MusicDataError(f"{at}: {name} {x!r} is not [center, half-width]") + if name == "converted" or name.endswith(("maxCombo", "justCount")): + if float(x[1]) != 0 or not float(x[0]).is_integer(): + raise MusicDataError(f"{at}: {name} must be an exact integer counter") + for score, tail, range_score, rank_bonus in ( + ("score", "tail", "rangeScore", "rankBonus"), + ("scorePerfect", "tailPerfect", "rangeScorePerfect", "rankBonusPerfect")): + inside = sum(float(r[k][0]) for r in rs for k in (range_score, rank_bonus)) + # Each point center is serialized to millipoints; every field encloses its own rounding. + tolerance = (2 * n + 2) * 0.0005 + MEAN_TOLERANCE * max(1, abs(float(v[score][0]))) + if abs(float(v[tail][0]) - (float(v[score][0]) - inside)) > tolerance: + raise MusicDataError(f"{at}: {tail} is not {score} minus the ranges' scores and rank bonuses") c = v.get("check") - if not (isinstance(c, dict) and all(k in c for k in CHECK_KEYS)): - raise MusicDataError(f"{at}: no check {{{', '.join(CHECK_KEYS)}}}") - first = base[0]["seed"] if det else published_seeds(1)[0] - if not _int(c["seed"]) or c["seed"] != first: - raise MusicDataError(f"{at}: the check plays seed {c['seed']!r}, not the variant's first {first}") - if not (isinstance(c["ranks"], list) and len(c["ranks"]) == n and all(_int(r) and 1 <= r <= RANKS - for r in c["ranks"]) + _check_interval_bound(c, "the check deck", at) + if not (isinstance(c["ranks"], list) and len(c["ranks"]) == n + and all(_int(r) and 1 <= r <= RANKS for r in c["ranks"]) and (linear or all(r == 1 for r in c["ranks"]))): raise MusicDataError(f"{at}: check ranks {c['ranks']!r}") if not (isinstance(c["deck"], list) and len(c["deck"]) == positions and all( x is None or (plain is not None and isinstance(x, list) and len(x) == 2 and x[0] == plain and _int(x[1])) for x in c["deck"])): - raise MusicDataError(f"{at}: check deck {c['deck']!r} is not [plain kind ({plain}), value] or null per " - f"position") - _check_bound(c, "the check deck", at) - + raise MusicDataError(f"{at}: check deck {c['deck']!r} is not [plain kind ({plain}), value] or null per position") def chart_deck(song: dict, chart: dict, stats: dict, kinds: int, percents: list[list], seeds: int, aptitude: Aptitude, plain: int | None) -> dict: """A chart's `deck` from its statistics, after checking them against the song, the chart facts, the song's rank - bonus percentages (`percents`: rank_bonus_percents), the seed set (chart_seeds: `seeds` seeds with a luck range) + bonus percentages (`percents`: rank_bonus_percents), the replay seed set (chart_seeds) and its Gekisou aptitude (Aptitude.chart, `plain` the plain kind), `kinds` the number of score-up kinds.""" where = f"chart {chart['scoreId']} ({song['id']} {chart['difficulty']})" checks = ( @@ -958,20 +916,15 @@ def chart_deck(song: dict, chart: dict, stats: dict, kinds: int, percents: list[ raise MusicDataError(f"{where}: range {i}: the deck model's rank bonus percentages {got!r} " f"({r['rankBonusPercent']!r}) differ from MasterLiveGekisouRankingScoreBonus {want!r}") want = chart_seeds(stats, seeds) - if [s["seed"] for s in stats["seeds"]] != want: - raise MusicDataError(f"{where}: seeds {[s['seed'] for s in stats['seeds']]} are not the chart's seed set " - f"{want}") - _check_seed_shapes(stats, kinds, where) - for seed in stats["seeds"]: - for i, (r, info) in enumerate(zip(seed["ranges"], ranges)): - bonus = _trunc_percent(r["rangeScore"], info["rankBonusPercent"]) - if r["rankBonus"] != bonus: - raise MusicDataError(f"{where}: seed {seed['seed']} range {i}: rank bonus {r['rankBonus']} is not " - f"trunc({r['rangeScore']} * {info['rankBonusPercent']} / 100) = {bonus}") - if stats["justNotes"] == 0 and (seed.get("scorePerfect") != seed["score"] or any( - r.get("rangeScorePerfect") != r["rangeScore"] for r in seed["ranges"])): - raise MusicDataError(f"{where}: seed {seed['seed']}: a chart without Just notes scores otherwise on the " - f"Perfect play ({seed.get('scorePerfect')} for {seed['score']})") + if stats.get("replaySeeds") != want: + raise MusicDataError(f"{where}: replay seeds {stats.get('replaySeeds')!r} are not the chart's seed set {want}") + _check_expectation_shapes(stats, kinds, where) + expected = stats.get("expectation") + if expected is not None and stats["justNotes"] == 0: + pairs = [(expected["score"], expected["scorePerfect"])] + [ + (r["rangeScore"], r["rangeScorePerfect"]) for r in expected["ranges"]] + if any(abs(float(a[0]) - float(b[0])) > float(a[1]) + float(b[1]) + MEAN_TOLERANCE for a, b in pairs): + raise MusicDataError(f"{where}: a chart without Just notes scores otherwise on the Perfect play") return dict({k: stats.get(k) for k in DECK_CHART_KEYS}, gekisouAptitude=aptitude.chart(where, stats, plain)) diff --git a/tests/test_musicdata.py b/tests/test_musicdata.py index db077db..4fb3333 100644 --- a/tests/test_musicdata.py +++ b/tests/test_musicdata.py @@ -205,54 +205,59 @@ def shape_skill(i, level, targets=None, bands=None): {"id": 4, "source": "support", "mission": 1, "bandCondition": False, "effects": [effect_row(5)], "skills": [shape_skill(32, 5)]}, ] -SEED_RULE = {"deterministicTest": 4, "batches": [32, 64, 128, 256, 512, 1024], "relative": 0.01, "baseline": 0.001, - "crossSeeds": 64} -NONDETERMINISTIC = {2, 3} # the shapes the stand-in measures on 32 seeds - - -def mean_se(xs): - m = sum(xs) / len(xs) - return [m, (sum((x - m) ** 2 for x in xs) / (len(xs) - 1) / len(xs)) ** 0.5 if len(xs) > 1 else 0.0] +NOMINAL_LAW = "independent nominal lottery and skill probabilities" + + +def expected_stats(positions, percents, missions): + ranges = [] + for j, p in enumerate(percents): + points = 101 * (j + 1) + luck = missions[j] == musicdata.MISSION_LUCK + ranges.append({"rangeScore": [points, 0], "rangeScorePerfect": [points, 0], + "rankBonus": [trunc_percent(points, p[0]), 0], + "rankBonusPerfect": [trunc_percent(points, p[0]), 0], + "maxCombo": 1, "justCount": 0, "luckPoints": [11, 0.35] if luck else [0, 0], + "lotResults": [[2, 0.01], [1, 0.01], [0.5, 0.01], [0.1, 0.01]] if luck else [[0, 0]] * 4}) + check = {"deck": [[0, 5000]] + [None] * (positions - 1), "ranks": None, + "expected": [2000, 0.25], "predicted": [2000.25, 0.1], "bound": 7} + return {"score": [1234, 0.5], "scorePerfect": [1234, 0.5], "ranges": ranges, + "weights": [[["W", 0]] * positions], "rangeWeights": [[[["W", 0]] * len(ranges)] * positions], + "check": check, "rankCheck": dict(check, ranks=[2] * len(ranges)) if ranges else None} def aptitude_of(s, positions): - """The stand-in's Gekisou aptitude of a chart's statistics `s`: per range 5 judged notes (Perfect in a Just count - range), per variant +10 * (j + 1) in range j and 7 after the ranges; a nondeterministic shape's means a quarter - point off with a standard error of 0.5.""" - ranges, base = s["ranges"], s["seeds"][0] + """Synthetic nominal intervals: known range gains and an exact seven-point tail.""" + ranges, base = s["ranges"], s["expectation"] missions = {r["mission"] for r in ranges} factors = [{"judgedNotes": 5, "justNotes": 0, "perfectNotes": 5 if r["mission"] == 3 else 0, "tailNotes": 1, "comboAtStart": 0, - "lotteries": mean_se([sum(x["ranges"][j]["lotResults"]) for x in s["seeds"]]) if r["mission"] == 2 - else [0, 0]} for j, r in enumerate(ranges)] + "lotteries": [sum(x[i] for x in base["ranges"][j]["lotResults"]) for i in range(2)]} + for j, r in enumerate(ranges)] variants = [] for shape in SHAPES: if shape["mission"] != 4 and shape["mission"] not in missions: continue for band in ((True, False) if shape["bandCondition"] else (None,)): - det = shape["id"] not in NONDETERMINISTIC and 2 not in missions - off, se = (0, 0) if det else (0.25, 0.5) + offset, radius = (0.25, 0.5) if shape["id"] in (2, 3) or 2 in missions else (0, 0) rs, inside, inside_p = [], 0, 0 for j, (r, r0) in enumerate(zip(ranges, base["ranges"])): d = 10 * (j + 1) - rb = trunc_percent(r0["rangeScore"] + d, r["rankBonusPercent"]) - r0["rankBonus"] - rbp = trunc_percent(r0["rangeScorePerfect"] + d, r["rankBonusPercent"]) - \ - trunc_percent(r0["rangeScorePerfect"], r["rankBonusPercent"]) - rs.append({"rangeScore": [d + off, se], "rankBonus": [rb + off, se], "rangeScorePerfect": [d + off, se], - "maxCombo": [0, 0], "justCount": [1 + off, se], "luckPoints": [off, se]}) - inside += d + rb + 2 * off - inside_p += d + rbp - seeds = 1 if det else 32 + rb = trunc_percent(r0["rangeScore"][0] + d, r["rankBonusPercent"]) - r0["rankBonus"][0] + rbp = trunc_percent(r0["rangeScorePerfect"][0] + d, r["rankBonusPercent"]) - r0["rankBonusPerfect"][0] + rs.append({"rangeScore": [d + offset, radius], "rankBonus": [rb + offset, radius], + "rangeScorePerfect": [d + offset, radius], "rankBonusPerfect": [rbp + offset, radius], + "maxCombo": [0, 0], "justCount": [1, 0], "luckPoints": [offset, radius]}) + inside += d + rb + 2 * offset + inside_p += d + rbp + 2 * offset variants.append({ - "shape": shape["id"], "bandMatch": band, "deterministic": det, "seeds": seeds, "seTargetMet": True, - "crossSeeds": min(seeds, SEED_RULE["crossSeeds"]), - "score": [inside + 7, se], "scorePerfect": [inside_p + 7 + off, se], "tail": [7, se], - "tailPerfect": [7, se], "converted": [0, 0], "ranges": rs, - "weights": [[0.01, se / 100]] * positions, + "shape": shape["id"], "bandMatch": band, + "score": [inside + 7, radius], "scorePerfect": [inside_p + 7, radius], "tail": [7, radius], + "tailPerfect": [7, radius], "converted": [0, 0], "ranges": rs, + "weights": [[0.01, radius / 100]] * positions, "rangeWeights": [[[0.001, 0]] * len(ranges)] * positions if base["rangeWeights"] is not None else None, - "check": {"seed": base["seed"] if det else musicdata.published_seeds(1)[0], "ranks": [2 if base["rangeWeights"] is not None else 1] * len(ranges), - "deck": [[0, 5000]] + [None] * (positions - 1), "exact": 2000, "predicted": 2000.25, - "bound": 7.0}}) + "check": {"ranks": [2 if base["rangeWeights"] is not None else 1] * len(ranges), + "deck": [[0, 5000]] + [None] * (positions - 1), "expected": [2000, 0.25], + "predicted": [2000.25, 0.1], "bound": 7}}) return {"factors": factors, "variants": variants} @@ -261,7 +266,7 @@ class FakeDeck: aptitude (SHAPES; `aptitude`) unless it is left out.""" COMMIT = "7e5d84b5998d28c21541ce3f2e0a3dfb1439f4f6" SOURCE = "5" * 64 - FORMAT = "ournotes-deck.chart-stats/2" + FORMAT = "ournotes-deck.chart-stats/3" def __init__(self, change=None, fail=None, header=None): self.change, self.fail, self.header, self.inputs, self.options = change, fail, header, [], [] @@ -271,12 +276,12 @@ def info(self): "commit": self.COMMIT, "sourceSha256": self.SOURCE, "dataFormat": deckdata.DECK_FORMAT, "format": self.FORMAT} - def chart_stats(self, data, seeds, workers, aptitude=True, aptitude_max_seeds=None, aptitude_cross_seeds=None): + def chart_stats(self, data, seeds, workers, aptitude=True): if self.fail: raise ValueError(self.fail) doc = json.loads(data) self.inputs.append((doc, seeds, workers)) - self.options.append((aptitude, aptitude_max_seeds, aptitude_cross_seeds)) + self.options.append(aptitude) assert doc["format"] == deckdata.DECK_FORMAT and list(doc["master"]) == [t for t, _ in deckdata.TABLES] m = doc["master"] with_aptitude = aptitude @@ -303,16 +308,6 @@ def table(name): check = {"deck": [[0, 5000]], "exact": 2000, "predicted": 2000.25, "bound": 7.0} luck = musicdata.MISSION_LUCK in missions[:len(fevers)] - def seed_stats(i, seed): - return {"seed": seed, "score": 1234, - "ranges": [{"rangeScore": 101 * (j + 1), "rankBonus": trunc_percent(101 * (j + 1), p[0]), - "maxCombo": 1, "justCount": 0, "luckPoints": 0, - "lotResults": [i % 3, 1, 0, 0] if luck else [0, 0, 0, 0], - "rangeScorePerfect": 101 * (j + 1)} for j, p in enumerate(percents)], - "weights": [["W"] * positions], "check": dict(check), "scorePerfect": 1234, - "rangeWeights": [[["W"] * len(fevers)] * positions], - "rankCheck": {"ranks": [2] * len(fevers), "exact": 2001, "predicted": 2000.5, - "bound": 7.0} if fevers else None} s = {"scoreId": c["scoreId"], "musicId": music, "difficulty": d, "level": levels[c["scoreId"]], "judgedNotes": sum(score.is_judgement_note(op) for op in c["notes"]["op"]), "convertedNoteCount": len(c["notes"]["id"]), "lastNoteMs": last, "musicLengthMs": last + 1000, @@ -322,7 +317,8 @@ def seed_stats(i, seed): "rankBonusPercent": percents[i][0], "rankBonusPercents": percents[i]} for i, (a, b) in enumerate(fevers)], "justNotes": 0, - "seeds": [seed_stats(i, x) for i, x in enumerate(musicdata.published_seeds(seeds) if luck else [0])], + "expectation": expected_stats(positions, percents, missions), + "replaySeeds": musicdata.published_seeds(seeds) if luck else [0], "offSeeds": [{"seed": 0, "score": 1000, "weights": [["W"] * positions], "check": dict(check)}]} s["gekisouAptitude"] = aptitude_of(s, positions) if with_aptitude and fevers else None if self.change: @@ -331,7 +327,7 @@ def seed_stats(i, seed): out = {"format": self.FORMAT, "source": {}, "model": {"power": 300000, "gekisouAptitude": "each Gekisou (support) skill shape alone"}, "gekisouAptitude": {"plainKind": 0, "host": "a synthetic member without a Gekisou skill", - "seedRule": SEED_RULE, "shapes": SHAPES} if aptitude else None, + "law": NOMINAL_LAW, "shapes": SHAPES} if aptitude else None, "kinds": [{"id": 0, "effectType": 2000, "activationTimeSecond": 7.5}], "charts": charts} out = json.loads(json.dumps(out)) if self.header: @@ -400,7 +396,7 @@ def test_document(tmp_path): doc = json.loads(raw) assert list(doc) == ["format", "provenance", "languages", "bands", "characters", "tags", "categories", "gekisouCatalog", "deck", "songs"] - assert doc["format"] == "nnnotes.music-data/1" and doc["languages"] == ["ja", "en", "zh-Hant", "zh-Hans", "ko"] + assert doc["format"] == "nnnotes.music-data/2" and doc["languages"] == ["ja", "en", "zh-Hant", "zh-Hans", "ko"] p = doc["provenance"] assert list(p) == ["region", "client", "catalog", "master", "exporter", "deck"] and p["deck"] is None # without the deck model: the song and the Gekisou catalog tables only @@ -485,9 +481,10 @@ def test_deck(tmp_path): assert c["deck"]["unplayable"] is None and c["deck"]["skip"] == 1.5 # the ranks: the song's mission pattern (3) rows of every range assert [r["rankBonusPercents"] for r in c["deck"]["ranges"]] == [[35, 34, 33, 32, 31], [40, 38, 36, 34, 32]] - seed = c["deck"]["seeds"][0] - assert seed["rankCheck"]["ranks"] == [2, 2] and seed["scorePerfect"] == seed["score"] - assert [r["rankBonus"] for r in seed["ranges"]] == [35, 80] and [r["luckPoints"] for r in seed["ranges"]] == [0, 0] + expected = c["deck"]["expectation"] + assert expected["rankCheck"]["ranks"] == [2, 2] and expected["scorePerfect"] == expected["score"] + assert [r["rankBonus"] for r in expected["ranges"]] == [[35, 0], [80, 0]] + assert [r["luckPoints"] for r in expected["ranges"]] == [[0, 0], [0, 0]] assert c["deck"]["offSeeds"][0]["score"] == 1000 # the deck model's numbers as it writes them: a binary64 value is not narrowed to binary32 assert b'"weights":[[' + WEIGHT.encode() + b',' + WEIGHT.encode() + b']]' in raw and b'"predicted":2000.25' in raw @@ -508,22 +505,22 @@ def test_deck(tmp_path): (lambda s: s.update(difficulty="hard"), "difficulty 'hard' differs"), (lambda s: s["ranges"] and s["ranges"][0]["rankBonusPercents"].__setitem__(2, 99), "range 0: .*percentages"), (lambda s: s["ranges"] and s["ranges"][0].update(rankBonusPercent=1), "range 0: .*percentages"), - (lambda s: s["ranges"] and s["seeds"][0]["ranges"][0].update(rankBonus=-1), "rank bonus -1 is not trunc"), - (lambda s: s["seeds"][0]["ranges"].pop(), "range results for"), + (lambda s: s["ranges"] and s["expectation"]["ranges"][0].update(rankBonus=[1, -1]), "invalid scores"), + (lambda s: s["expectation"]["ranges"].pop(), "range results differ"), (lambda s: s.pop("offSeeds"), "no Gekisou off statistics"), (lambda s: s["offSeeds"].append(s["offSeeds"][0]), "no Gekisou off statistics"), (lambda s: s["offSeeds"][0]["weights"].append(None), "Gekisou off: weights"), (lambda s: s["offSeeds"][0]["check"].update(exact=0), "Gekisou off: the check deck scores 0"), - (lambda s: s["seeds"][0].update(scorePerfect=1), "otherwise on the Perfect play"), - (lambda s: s["seeds"][0]["weights"][0].pop(), "weights are not"), - (lambda s: s["seeds"][0]["weights"].append(None), "weights are not"), - (lambda s: s["seeds"][0].update(rangeWeights=[[]]), "rangeWeights are not"), - (lambda s: s["seeds"][0]["check"].update(bound=0.1), "the check deck scores 2000"), - (lambda s: s["ranges"] and s["seeds"][0]["rankCheck"].update(exact=0), "at ranks"), - (lambda s: s["ranges"] and s["seeds"][0]["rankCheck"].update(ranks=[6, 1]), "rank check ranks"), - (lambda s: s["seeds"][0]["ranges"][0].pop("luckPoints"), "chart 10 .*seed 0 range 0: no luck points"), - (lambda s: s["seeds"][0].update(seed=3), r"chart 10 .*seeds \[3\] are not the chart's seed set \[0\]"), - (lambda s: s["seeds"].append(dict(s["seeds"][0], seed=5)), r"chart 10 .*seeds \[0, 5\] are not the chart's seed"), + (lambda s: s["expectation"].update(scorePerfect=[1, 0]), "otherwise on the Perfect play"), + (lambda s: s["expectation"]["weights"][0].pop(), "weights are not"), + (lambda s: s["expectation"]["weights"].append(None), "weights are not"), + (lambda s: s["expectation"].update(rangeWeights=[[]]), "rangeWeights are not"), + (lambda s: s["expectation"]["check"].update(bound=0.1), "the check deck: expected"), + (lambda s: s["ranges"] and s["expectation"]["rankCheck"].update(expected=[0, 0]), "rank check: expected"), + (lambda s: s["ranges"] and s["expectation"]["rankCheck"].update(ranks=[6, 1]), "rank check ranks"), + (lambda s: s["expectation"]["ranges"][0].pop("luckPoints"), "chart 10 .*range 0: invalid scores"), + (lambda s: s.update(replaySeeds=[3]), r"chart 10 .*replay seeds \[3\] are not the chart's seed set \[0\]"), + (lambda s: s["replaySeeds"].append(5), r"chart 10 .*replay seeds \[0, 5\] are not the chart's seed"), ]) def test_deck_checks(tmp_path, change, match): with pytest.raises(musicdata.MusicDataError, match=match): @@ -534,8 +531,8 @@ def test_deck_checks(tmp_path, change, match): def test_deck_without_range_weights(tmp_path): """A chart without range weights, a kind without them and a kind without Gekisou off weights are carried.""" def change(s): - s["seeds"][0]["rangeWeights"] = None if s["scoreId"] == 10 else [None] - s["seeds"][0]["rankCheck"] = None + s["expectation"]["rangeWeights"] = None if s["scoreId"] == 10 else [None] + s["expectation"]["rankCheck"] = None s["offSeeds"][0]["weights"] = [None] if s["scoreId"] == 10: for v in s["gekisouAptitude"]["variants"]: @@ -544,7 +541,7 @@ def change(s): export(tmp_path, deck=FakeDeck(change)) doc = json.loads((tmp_path / "music.json").read_bytes()) decks = {c["scoreId"]: c["deck"] for s in doc["songs"] for c in s["charts"]} - assert decks[10]["seeds"][0]["rangeWeights"] is None and decks[20]["seeds"][0]["rangeWeights"] == [None] + assert decks[10]["expectation"]["rangeWeights"] is None and decks[20]["expectation"]["rangeWeights"] == [None] assert decks[30]["offSeeds"][0]["weights"] == [None] schema_validator().validate(doc) @@ -636,11 +633,11 @@ def test_luck_chart(tmp_path): export(tmp_path, rows=rows, deck=FakeDeck()) doc = json.loads((tmp_path / "music.json").read_bytes()) deck = doc["songs"][0]["charts"][0]["deck"] - assert [s["seed"] for s in deck["seeds"]] == musicdata.published_seeds(8) + assert deck["replaySeeds"] == musicdata.published_seeds(8) schema_validator().validate(doc) (tmp_path / "m").rename(tmp_path / "m1") with pytest.raises(musicdata.MusicDataError, match=r"chart 10 .*seeds .* are not the chart's seed set"): - export(tmp_path, rows=rows, deck=FakeDeck(on(10, lambda s: s["seeds"].pop()))) + export(tmp_path, rows=rows, deck=FakeDeck(on(10, lambda s: s["replaySeeds"].pop()))) def test_rank_bonus_percents(): @@ -668,7 +665,7 @@ def test_deck_errors(tmp_path, monkeypatch): def test_deck_module(): deck = pytest.importorskip("nnnotes._deck") info = deck.info() - assert info["name"] == "ournotes-deck" and info["format"] == "ournotes-deck.chart-stats/2" + assert info["name"] == "ournotes-deck" and info["format"] == "ournotes-deck.chart-stats/3" assert info["dataFormat"] == deckdata.DECK_FORMAT and re.fullmatch(r"[0-9a-f]{40}", info["commit"]) assert re.fullmatch(r"[0-9a-f]{64}", info["sourceSha256"]) lock = (ROOT / "rust" / "Cargo.lock").read_text(encoding="utf-8") @@ -849,13 +846,11 @@ def aptitude_fixture(): """A header and a two-range chart, independently authored above, through JSON as the extension returns it.""" tables = {t: TABLE_ROWS.get(t, rows_of(t)) for t in musicdata.tables_of(True, False)} a = musicdata.Aptitude(tables, musicdata.gekisou_catalog(tables, musicdata.Texts(tables['MasterText']))) - header = json.loads(json.dumps({'plainKind': 0, 'host': 'synthetic host', 'seedRule': SEED_RULE, 'shapes': SHAPES}), + header = json.loads(json.dumps({'plainKind': 0, 'host': 'synthetic host', 'law': NOMINAL_LAW, 'shapes': SHAPES}), parse_float=deckdata._Num) chart = {'positions': 2, 'judgedNotes': 20, 'justNotes': 0, 'ranges': [ {'mission': 1, 'rankBonusPercent': 35}, {'mission': 3, 'rankBonusPercent': 40}], - 'seeds': [{'seed': 0, 'rangeWeights': [], 'ranges': [ - {'rangeScore': 101, 'rangeScorePerfect': 101, 'rankBonus': 35, 'lotResults': [0, 0, 0, 0]}, - {'rangeScore': 202, 'rangeScorePerfect': 202, 'rankBonus': 80, 'lotResults': [0, 0, 0, 0]}]}]} + 'expectation': expected_stats(2, [[35], [40]], [1, 3])} chart['gekisouAptitude'] = aptitude_of(chart, 2) chart = json.loads(json.dumps(chart), parse_float=deckdata._Num) return a, header, chart @@ -891,9 +886,8 @@ def test_aptitude_fixture(aptitude_fixture): lambda h: h['shapes'].pop(), lambda h: h.update(plainKind=1), lambda h: h.update(host=''), - lambda h: h['seedRule'].update(batches=[32, 16]), - lambda h: h['seedRule'].update(crossSeeds=0), - lambda h: h['seedRule'].update(relative=deckdata._Num('NaN')), + lambda h: h.update(law=''), + lambda h: h.update(law=None), ]) def test_aptitude_header_rejects(aptitude_fixture, change): a, h, _ = aptitude_fixture @@ -916,16 +910,13 @@ def test_aptitude_header_rejects(aptitude_fixture, change): lambda c: c['gekisouAptitude']['variants'][0].update(score=[deckdata._Num('NaN'), 0]), lambda c: c['gekisouAptitude']['variants'][0].update(score=[10**400, 0]), lambda c: c['gekisouAptitude']['variants'][0].update(score=[1, -1]), - lambda c: c['gekisouAptitude']['variants'][0].update(tail=[7, 1]), + lambda c: c['gekisouAptitude']['variants'][0].update(converted=[0.25, 0]), lambda c: c['gekisouAptitude']['variants'][0].update(tail=[8, 0]), lambda c: c['gekisouAptitude']['variants'][0]['ranges'].pop(), - lambda c: c['gekisouAptitude']['variants'][0].update(seeds=2), - lambda c: c['gekisouAptitude']['variants'][0].update(crossSeeds=2), - lambda c: c['gekisouAptitude']['variants'][2].update(seeds=31), - lambda c: c['gekisouAptitude']['variants'][2].update(seTargetMet=False), + lambda c: c['gekisouAptitude']['variants'][0]['ranges'][0].update(justCount=[1, 0.1]), lambda c: c['gekisouAptitude']['variants'][0]['check'].update(bound=-1), lambda c: c['gekisouAptitude']['variants'][0]['check'].update(predicted=deckdata._Num('NaN')), - lambda c: c['gekisouAptitude']['variants'][0]['check'].update(exact=0), + lambda c: c['gekisouAptitude']['variants'][0]['check'].update(expected=[0, 0]), lambda c: c['gekisouAptitude']['variants'][0]['check'].update(ranks=[0, 2]), lambda c: c['gekisouAptitude']['variants'][0]['check'].update(deck=[[9, 100], None]), ]) @@ -941,8 +932,8 @@ def test_aptitude_rounding(aptitude_fixture): a, h, c = aptitude_fixture aptitude_header(a, h) v = c['gekisouAptitude']['variants'][2] - v['tail'][0] = deckdata._Num('7.002') - a.chart('synthetic', c, 0) # independently rounded terms, two ranges: 0.00300001 absolute tolerance + v['tail'][0] = deckdata._Num('7.000000001') + a.chart('synthetic', c, 0) # binary64 sum tolerance v['tail'][0] = deckdata._Num('7.004') with pytest.raises(musicdata.MusicDataError, match='tail'): a.chart('synthetic', c, 0) @@ -962,9 +953,9 @@ def test_aptitude_disabled(tmp_path): fake = FakeDeck() d = master_dir(tmp_path) musicdata.export(tmp_path / 'music.json', deckdata.master_files(d), KEY, CHARTS.__getitem__, bgm, **PROV, - deck=musicdata.Deck(module=fake, aptitude=False, aptitude_max_seeds=32, aptitude_cross_seeds=8)) + deck=musicdata.Deck(module=fake, aptitude=False)) doc = json.loads((tmp_path / 'music.json').read_bytes()) - assert fake.options == [(False, 32, 8)] + assert fake.options == [False] assert doc['deck']['gekisouAptitude'] is None assert all(c['deck']['gekisouAptitude'] is None for s in doc['songs'] for c in s['charts']) schema_validator().validate(doc) diff --git a/tests/test_replaydata.py b/tests/test_replaydata.py index dd1b02d..e4a1b37 100644 --- a/tests/test_replaydata.py +++ b/tests/test_replaydata.py @@ -159,11 +159,11 @@ def test_recommend_engine_rejects_another_build(tmp_path, change, message): assert not (tmp_path / "music.json").exists() and not (tmp_path / "replay").exists() -def test_final_export_rejects_unmet_sampling_instead_of_publishing_flag(tmp_path): +def test_final_export_checks_the_complete_expectation_interval(tmp_path): def unmet(chart): apt = chart.get("gekisouAptitude") if apt and apt["variants"]: - apt["variants"][0]["seTargetMet"] = False - with pytest.raises(musicdata.MusicDataError, match="did not meet BOTH score SE targets"): + apt["variants"][0]["check"]["expected"] = [2000, 100] + with pytest.raises(musicdata.MusicDataError, match="beyond the bound"): export(tmp_path, deck=FakeDeck(change=unmet)) assert not (tmp_path / "music.json").exists() From 5ca35c6a9e1a0f78866167d5d619e178bd973ad2 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Sat, 10 Oct 2026 12:19:30 +0800 Subject: [PATCH 3/3] style(rust): order chartstats imports Co-Authored-By: Claude Sonnet 5.5 --- rust/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 6405055..793b441 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -4,7 +4,7 @@ use std::sync::Mutex; use std::sync::atomic::{AtomicUsize, Ordering}; -use ournotes_sim::chartstats::{self, ChartStats, REPLAY_SEEDS, Options}; +use ournotes_sim::chartstats::{self, ChartStats, Options, REPLAY_SEEDS}; use ournotes_sim::data::DeckData; use ournotes_sim::error::Error; use pyo3::exceptions::PyValueError;