From 988ba4c5aaf3f5d63369fb8af724e950f18c83da Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Sun, 4 Oct 2026 10:00:02 +0800 Subject: [PATCH] feat(music-data): publish the recommendation engine in the replay manifest `nnnotes music-data --recommend-engine DIR` copies a wasm-bindgen `--target web` package of the deck model's `wasm/recommend` (`ournotes_recommend.js`, `ournotes_recommend_bg.wasm`, `build.json`) into the replay bundle beside the replay engine. Its `build.json` (`ournotes.recommend-engine/1`) must name the pinned model commit and the SHA-256 of both files, else the export stops before writing anything. The replay manifest lists it as `recommendEngine`: the model identity and the `js`, `wasm` and `build` resources with their URL relative to the manifest, SHA-256 and size. The model is the one of `engine` and of `deck-data.json`, so a page loads the recommendation module and the deck data it reads from one model commit and one master snapshot. Without the option the key is absent. Co-Authored-By: Claude Opus 5.5 --- docs/music-data.md | 2 +- docs/replay.md | 10 ++++-- src/nnnotes/cli.py | 5 ++- src/nnnotes/musicdata.py | 6 ++-- src/nnnotes/replaydata.py | 63 ++++++++++++++++++++------------ tests/test_replaydata.py | 75 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 132 insertions(+), 29 deletions(-) diff --git a/docs/music-data.md b/docs/music-data.md index b619d2c..ef4caf9 100644 --- a/docs/music-data.md +++ b/docs/music-data.md @@ -454,4 +454,4 @@ their facts here) and the `nnnotes.deck-data/1` file of `nnnotes deck-data` (its Final aptitude exports require the standard-error target for both `score` and `scorePerfect`, each against its own paired no-skill baseline. Sampling extends the same published seed prefix through geometric batches, stopping as soon as both targets agree with the unchanged max(1% of increment, 0.1% of baseline) rule. The 65,536-seed cap is a failure guard, not a requirement to run every seed. An unmet cap aborts a normal export before writing artifacts. `--allow-unconverged-aptitude` is an explicit diagnostic option; it retains real SE values and unmet flags. -`--replay-dir OUT/replay --replay-engine WASM_PKG` writes normalized runtime inputs and pinned WASM assets as described in [replay.md](replay.md). The music data stays compact and carries the SHA-bound `replay.manifestUrl` pointer. No original chart/master blobs or native binary are included in this artifact bundle. +`--replay-dir OUT/replay --replay-engine WASM_PKG [--recommend-engine WASM_PKG]` writes normalized runtime inputs and pinned WASM assets (the replay engine and, optionally, the recommendation engine) as described in [replay.md](replay.md). The music data stays compact and carries the SHA-bound `replay.manifestUrl` pointer. No original chart/master blobs or native binary are included in this artifact bundle. diff --git a/docs/replay.md b/docs/replay.md index 8ab44ea..867afc7 100644 --- a/docs/replay.md +++ b/docs/replay.md @@ -1,6 +1,6 @@ # Shared Rust/WASM replay inputs -`nnnotes music-data --replay-dir OUT/replay --replay-engine WASM_PKG -o OUT/music-data.json` writes the normalized input of the pinned deck model. The engine package must contain `ournotes_replay.js`, `ournotes_replay_bg.wasm` and `build.json`; its commit and both file hashes must match the measured model. Actual ACB cue length is required. Original encrypted master files, chart blobs and native binaries are not copied. +`nnnotes music-data --replay-dir OUT/replay --replay-engine WASM_PKG [--recommend-engine WASM_PKG] -o OUT/music-data.json` writes the normalized input of the pinned deck model. The replay engine package must contain `ournotes_replay.js`, `ournotes_replay_bg.wasm` and `build.json` (`format:"ournotes.replay-engine/1"`); the optional recommendation engine package, the wasm-bindgen `--target web` build of the deck model's `wasm/recommend`, must contain `ournotes_recommend.js`, `ournotes_recommend_bg.wasm` and `build.json` (`format:"ournotes.recommend-engine/1"`). Each `build.json` records `commit`, `jsSha256` and `wasmSha256`; the commit and both file hashes must match the measured model and the package files. Actual ACB cue length is required. Original encrypted master files, chart blobs and native binaries are not copied. The music-data pointer is: @@ -22,12 +22,18 @@ The music-data pointer is: "wasm":{"url":"engine/ournotes_replay_bg.wasm","sha256":"","bytes":0}, "build":{"url":"engine/build.json","sha256":"","bytes":0} }, + "recommendEngine": { + "model": {"name":"ournotes-deck","version":"","source":"","commit":"","format":"ournotes-deck.chart-stats/2"}, + "js":{"url":"recommend/ournotes_recommend.js","sha256":"","bytes":0}, + "wasm":{"url":"recommend/ournotes_recommend_bg.wasm","sha256":"","bytes":0}, + "build":{"url":"recommend/build.json","sha256":"","bytes":0} + }, "unlistedScoreIds":[], "clock":"Explicit frames from ReplaySession.template; no Python/JS scoring or scheduling" } ``` -Numbers and SHA placeholders above illustrate the schema; actual manifests contain measured sizes, complete chart entries and computed hashes. A data-only export can leave `engine` null. A publishable interactive bundle supplies the pinned engine. Normalized runtime rows belonging to no listed live song are omitted and recorded in `unlistedScoreIds`. +Numbers and SHA placeholders above illustrate the schema; actual manifests contain measured sizes, complete chart entries and computed hashes. A data-only export can leave `engine` null. A publishable interactive bundle supplies the pinned engine. `recommendEngine` is present only with `--recommend-engine`; its `model` is the same pinned model as `engine.model` and `deck-data.json`'s `provenance.deck`, so the recommendation module and the deck data it reads come from one model commit and one master snapshot. The module's exported API is the one of `wasm/recommend` at `model.commit`. Normalized runtime rows belonging to no listed live song are omitted and recorded in `unlistedScoreIds`. Each chart resource is `{format:"nnnotes.replay-chart/1",scoreId,musicLengthMs,chart}`. `chart` retains the existing DeckData record: `asset:{key,sha256}`, equal-length `notes:{id,op,judgementType,timeMs}` arrays in native enumeration order, `skillEvents:{timeMs}` and `fevers:{startMs,endMs}`. `deck-data.json` uses the existing `nnnotes.deck-data/1` schema and adds `provenance.replay.musicLengthsMs:{"":}`. Score-table length is a separate native value derived by the shared Rust chart implementation; it must not be replaced by audio length. diff --git a/src/nnnotes/cli.py b/src/nnnotes/cli.py index 5bc0f2a..3bc659e 100644 --- a/src/nnnotes/cli.py +++ b/src/nnnotes/cli.py @@ -693,7 +693,8 @@ def cmd_music_data(args, cfg): catalog=deckdata.catalog_info(cat, cli_assets.store_root(args, cfg)), deck=deck, full=args.full, jacket=musicdata.catalog_jacket(cat) if args.jackets else None, - jackets_dir=args.jackets, replay_dir=args.replay_dir, replay_engine=args.replay_engine) + jackets_dir=args.jackets, replay_dir=args.replay_dir, replay_engine=args.replay_engine, + recommend_engine=args.recommend_engine) except (deckdata.DeckDataError, musicdata.MusicDataError) as e: sys.exit(f"nnnotes: {e}") _print_json(r) @@ -1003,6 +1004,8 @@ def target(m, what): help="write canonical runtime DeckData, per-chart inputs and replay manifest under the output directory") c.add_argument("--replay-engine", metavar="DIR", help="copy pinned wasm-bindgen JS/WASM + build.json into --replay-dir") + c.add_argument("--recommend-engine", metavar="DIR", + help="copy the pinned wasm-bindgen recommendation JS/WASM + build.json into --replay-dir") c.add_argument("--jackets", metavar="DIR", help="also write every song's jacket as DIR/.webp (at most 320 px on the longer side)") _out(c, "output file (.json, or .json.gz for gzip)") diff --git a/src/nnnotes/musicdata.py b/src/nnnotes/musicdata.py index f4cf3f6..b2ef42e 100644 --- a/src/nnnotes/musicdata.py +++ b/src/nnnotes/musicdata.py @@ -1090,7 +1090,7 @@ def tables_of(deck: bool, full: bool) -> tuple[str, ...]: def export(out, src: deckdata.MasterSource, key, fetch: Callable[[str], bytes], bgm: Callable[[str, str], dict] | None, *, region: str, client: dict, catalog: dict, deck: Deck | None = None, full: bool = False, jacket: Callable[[str], bytes] | None = None, - jackets_dir=None, replay_dir=None, replay_engine=None) -> dict: + jackets_dir=None, replay_dir=None, replay_engine=None, recommend_engine=None) -> dict: """Read the master data, every chart and every BGM cue sheet, measure the charts with `deck`, then write the file `out` (gzip when it ends in `.gz`) through a temporary file and a rename; with `jacket` and `jackets_dir`, first every song's jacket as `/.webp`. Returns the summary.""" @@ -1099,6 +1099,8 @@ def export(out, src: deckdata.MasterSource, key, fetch: Callable[[str], bytes], try: if replay_engine is not None and replay_dir is None: raise deckdata.DeckDataError("--replay-engine needs --replay-dir") + if recommend_engine is not None and replay_dir is None: + raise deckdata.DeckDataError("--recommend-engine needs --replay-dir") if replay_dir is not None and bgm is None: raise deckdata.DeckDataError("replay export requires actual BGM lengths; --no-bgm is incompatible") tables, shas = deckdata.read_master(src, key, tables_of(deck is not None, full or replay_dir is not None)) @@ -1107,7 +1109,7 @@ def export(out, src: deckdata.MasterSource, key, fetch: Callable[[str], bytes], replay_files = None if replay_dir is not None: from . import replaydata - replay_files, replay_manifest = replaydata.bundle(doc, replay_engine) + replay_files, replay_manifest = replaydata.bundle(doc, replay_engine, recommend_engine) manifest_sha = hashlib.sha256(replay_files["manifest.json"]).hexdigest() replay_target = Path(replay_dir) / manifest_sha try: diff --git a/src/nnnotes/replaydata.py b/src/nnnotes/replaydata.py index 267819e..9615d1d 100644 --- a/src/nnnotes/replaydata.py +++ b/src/nnnotes/replaydata.py @@ -12,13 +12,43 @@ CHART_FORMAT = "nnnotes.replay-chart/1" JS = "ournotes_replay.js" WASM = "ournotes_replay_bg.wasm" +RECOMMEND_JS = "ournotes_recommend.js" +RECOMMEND_WASM = "ournotes_recommend_bg.wasm" def _json(value) -> bytes: return deckdata.encode(deckdata._value(value, "replay input")) -def bundle(music: dict, engine_dir: Path | None = None) -> tuple[dict[str, bytes], dict]: +def _package(files: dict[str, bytes], source: Path, model: dict | None, name: str, build_format: str, prefix: str, + js: str, wasm: str) -> tuple[str, str, str]: + """Copy a wasm-bindgen package of the pinned deck model (`js`, `wasm`, build.json) into `files` under `prefix`; + build.json must name the model's commit and both file hashes. Returns the JS, WASM and build.json paths.""" + source = Path(source) + try: + built = json.loads((source / "build.json").read_text(encoding="utf8")) + except (OSError, ValueError): + raise deckdata.DeckDataError(f"{name}: missing or malformed build.json") from None + if built.get("format") != build_format or not model or built.get("commit") != model.get("commit"): + raise deckdata.DeckDataError(f"{name}: build.json commit differs from the pinned deck model") + js_path, wasm_path, build_path = f"{prefix}/{js}", f"{prefix}/{wasm}", f"{prefix}/build.json" + for file, path in ((js, js_path), (wasm, wasm_path)): + if not (source / file).is_file(): + raise deckdata.DeckDataError(f"{name}: missing {file} in {source}") + files[path] = (source / file).read_bytes() + if not files[wasm_path].startswith(b"\x00asm\x01\x00\x00\x00"): + raise deckdata.DeckDataError(f"{name}: not a WASM v1 module") + if not model.get("commit"): + raise deckdata.DeckDataError(f"{name}: the pinned deck model identity is required") + if built.get("jsSha256") != hashlib.sha256(files[js_path]).hexdigest() or \ + built.get("wasmSha256") != hashlib.sha256(files[wasm_path]).hexdigest(): + raise deckdata.DeckDataError(f"{name}: JS/WASM SHA differs from build.json") + files[build_path] = _json(built) + return js_path, wasm_path, build_path + + +def bundle(music: dict, engine_dir: Path | None = None, + recommend_dir: Path | None = None) -> tuple[dict[str, bytes], dict]: """Canonical normalized --full inputs and SHA manifest. Actual ACB length is mandatory. Runtime arrays retain native enumeration order. ReplaySession.template/run in Rust produce frames and scores. @@ -60,32 +90,19 @@ def resource(path: str) -> dict: model = music["provenance"].get("deck") engine = None if engine_dir is not None: - engine_dir = Path(engine_dir) - try: - built = json.loads((engine_dir / "build.json").read_text(encoding="utf8")) - except (OSError, ValueError): - raise deckdata.DeckDataError("replay engine: missing or malformed build.json") from None - if built.get("format") != "ournotes.replay-engine/1" or not model or built.get("commit") != model.get("commit"): - raise deckdata.DeckDataError("replay engine: build.json commit differs from the pinned deck model") - for name in (JS, WASM): - source = engine_dir / name - if not source.is_file(): - raise deckdata.DeckDataError(f"replay engine: missing {name} in {engine_dir}") - files[f"engine/{name}"] = source.read_bytes() - if not files[f"engine/{WASM}"].startswith(b"\x00asm\x01\x00\x00\x00"): - raise deckdata.DeckDataError("replay engine: not a WASM v1 module") - if not model or not model.get("commit"): - raise deckdata.DeckDataError("replay engine: the pinned deck model identity is required") - if built.get("jsSha256") != hashlib.sha256(files[f"engine/{JS}"]).hexdigest() or \ - built.get("wasmSha256") != hashlib.sha256(files[f"engine/{WASM}"]).hexdigest(): - raise deckdata.DeckDataError("replay engine: JS/WASM SHA differs from build.json") - files["engine/build.json"] = _json(built) + js, wasm, build = _package(files, engine_dir, model, "replay engine", "ournotes.replay-engine/1", "engine", + JS, WASM) engine = {"model": model, "requestFormat": "ournotes.replay/1", "class": "ReplaySession", "methods": ["describeChart", "template", "run"], - "js": resource(f"engine/{JS}"), "wasm": resource(f"engine/{WASM}"), - "build": resource("engine/build.json")} + "js": resource(js), "wasm": resource(wasm), "build": resource(build)} + recommend = None + if recommend_dir is not None: + js, wasm, build = _package(files, recommend_dir, model, "recommend engine", "ournotes.recommend-engine/1", + "recommend", RECOMMEND_JS, RECOMMEND_WASM) + recommend = {"model": model, "js": resource(js), "wasm": resource(wasm), "build": resource(build)} manifest = {"format": FORMAT, "deckData": {"format": deckdata.DECK_FORMAT, **resource("deck-data.json")}, "charts": charts, "engine": engine, + **({"recommendEngine": recommend} if recommend is not None else {}), "unlistedScoreIds": [c["scoreId"] for c in source_records if str(c["scoreId"]) not in lengths], "clock": "Explicit frames from ReplaySession.template; no Python/JS scoring or scheduling"} files["manifest.json"] = _json(manifest) diff --git a/tests/test_replaydata.py b/tests/test_replaydata.py index 915ad54..f1f2276 100644 --- a/tests/test_replaydata.py +++ b/tests/test_replaydata.py @@ -75,6 +75,81 @@ def test_engine_manifest_rejects_stale_pin_and_changed_wasm(tmp_path): export(changed, deck=FakeDeck(), replay_dir=changed / "replay", replay_engine=engine) +def package(directory, js_name, wasm_name, build_format, commit, js=b"export class Session {}"): + """A synthetic wasm-bindgen package with its build.json.""" + directory.mkdir(parents=True, exist_ok=True) + wasm = b"\x00asm\x01\x00\x00\x00" + (directory / js_name).write_bytes(js) + (directory / wasm_name).write_bytes(wasm) + built = {"format": build_format, "commit": commit, "workingTreeDirty": False, + "jsSha256": hashlib.sha256(js).hexdigest(), "wasmSha256": hashlib.sha256(wasm).hexdigest()} + (directory / "build.json").write_text(json.dumps(built)) + return built + + +def replay_package(tmp_path): + return package(tmp_path / "replay-pkg", replaydata.JS, replaydata.WASM, "ournotes.replay-engine/1", FakeDeck.COMMIT) + + +def recommend_package(tmp_path, commit=FakeDeck.COMMIT, build_format="ournotes.recommend-engine/1"): + return package(tmp_path / "recommend-pkg", replaydata.RECOMMEND_JS, replaydata.RECOMMEND_WASM, build_format, commit, + b"export class RecommendationSession {}") + + +def test_recommend_engine_shares_the_model_of_the_deck_data(tmp_path): + replay_package(tmp_path) + built = recommend_package(tmp_path) + plain = tmp_path / "plain" + plain.mkdir() + result = export(plain, deck=FakeDeck(), replay_dir=plain / "replay", replay_engine=tmp_path / "replay-pkg") + assert "recommendEngine" not in json.loads((plain / result["replay"]["manifestUrl"]).read_bytes()) + out = tmp_path / "out" + out.mkdir() + result = export(out, deck=FakeDeck(), replay_dir=out / "replay", replay_engine=tmp_path / "replay-pkg", + recommend_engine=tmp_path / "recommend-pkg") + manifest_path = out / result["replay"]["manifestUrl"] + manifest = json.loads(manifest_path.read_bytes()) + entry = manifest["recommendEngine"] + data = json.loads((manifest_path.parent / manifest["deckData"]["url"]).read_bytes()) + assert list(entry) == ["model", "js", "wasm", "build"] + assert entry["model"] == manifest["engine"]["model"] == data["provenance"]["deck"] + assert entry["model"]["commit"] == FakeDeck.COMMIT + assert [entry[k]["url"] for k in ("js", "wasm", "build")] == [ + "recommend/ournotes_recommend.js", "recommend/ournotes_recommend_bg.wasm", "recommend/build.json"] + assert manifest["engine"]["build"]["url"] == "engine/build.json" + for key, name in (("js", replaydata.RECOMMEND_JS), ("wasm", replaydata.RECOMMEND_WASM)): + payload = (manifest_path.parent / entry[key]["url"]).read_bytes() + assert payload == (tmp_path / "recommend-pkg" / name).read_bytes() + assert (entry[key]["sha256"], entry[key]["bytes"]) == (hashlib.sha256(payload).hexdigest(), len(payload)) + assert (entry["js"]["sha256"], entry["wasm"]["sha256"]) == (built["jsSha256"], built["wasmSha256"]) + build = (manifest_path.parent / entry["build"]["url"]).read_bytes() + assert json.loads(build) == built + assert (entry["build"]["sha256"], entry["build"]["bytes"]) == (hashlib.sha256(build).hexdigest(), len(build)) + + +@pytest.mark.parametrize("change,message", [ + ("commit", "recommend engine: build.json commit differs"), + ("format", "recommend engine: build.json commit differs"), + ("wasm", "recommend engine: JS/WASM SHA differs"), + ("missing", "recommend engine: missing ournotes_recommend.js"), + ("no-replay-dir", "--recommend-engine needs --replay-dir"), +]) +def test_recommend_engine_rejects_another_build(tmp_path, change, message): + replay_package(tmp_path) + recommend_package(tmp_path, commit="0" * 40 if change == "commit" else FakeDeck.COMMIT, + build_format="ournotes.replay-engine/1" if change == "format" else "ournotes.recommend-engine/1") + pkg = tmp_path / "recommend-pkg" + if change == "wasm": + (pkg / replaydata.RECOMMEND_WASM).write_bytes(b"\x00asm\x01\x00\x00\x00changed") + if change == "missing": + (pkg / replaydata.RECOMMEND_JS).unlink() + replay = {} if change == "no-replay-dir" else {"replay_dir": tmp_path / "replay", + "replay_engine": tmp_path / "replay-pkg"} + with pytest.raises(musicdata.MusicDataError, match=message): + export(tmp_path, deck=FakeDeck(), recommend_engine=pkg, **replay) + assert not (tmp_path / "music.json").exists() and not (tmp_path / "replay").exists() + + def test_final_export_rejects_unmet_sampling_instead_of_publishing_flag(tmp_path): def unmet(chart): apt = chart.get("gekisouAptitude")