From b83c5f95db2c8162ea1d40fb306b987332b62ede Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Sat, 17 Oct 2026 14:57:16 +0800 Subject: [PATCH 1/2] fix(story): include CRI categories in sound products Read each cue's category names from the ACB when assembling story.sound. Record nonempty lists in cues.json and version the stage so cached sound products are recomputed. Audio files and decoded stream records retain their existing contracts. --- docs/products.md | 4 ++- docs/stages.md | 2 +- src/nnnotes/formats/acb.py | 22 ++++++++++++ src/nnnotes/pipeline/story.py | 11 ++++-- tests/formats/test_acb_categories.py | 52 ++++++++++++++++++++++++++++ tests/pipeline/test_story.py | 40 +++++++++++++++++++-- 6 files changed, 124 insertions(+), 7 deletions(-) create mode 100644 tests/formats/test_acb_categories.py diff --git a/docs/products.md b/docs/products.md index 9af78fe..eb469cf 100644 --- a/docs/products.md +++ b/docs/products.md @@ -372,7 +372,9 @@ hold. `story.sound` reads its cue sheet's `cri.audio` files; `cues.json` maps each cue name to the first stream of that name (in stream order): `file` (the sound, next to `cues.json`), `sampleRate`, `channels`, `samples` (frames at `sampleRate`, without encoder padding) and, for a looping stream, `loopStart` and `loopEnd` -(sample frames). The sounds are in the web format of the parameter `audio`, as for `live.music`: `aac` (the default) +(sample frames). A cue whose ACB sequence names CRI categories also has `categories`, the ordered category names +(for example `["Voice", "VoiceConfig"]`), read from sequence command 65 and the ACB's ACF category references. +The sounds are in the web format of the parameter `audio`, as for `live.music`: `aac` (the default) records `encoderDelay`, the priming samples of the MP4 edit list; `flac` keeps the decoded FLAC. The `product` layout makes the `story.sound` product of every cue sheet key the selection holds. diff --git a/docs/stages.md b/docs/stages.md index 0cc2805..68b59d1 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -36,7 +36,7 @@ codec and channel of a USM) and `catalog.index` (version 2: gzip input and Japan | `live.noteeffect` | a note effect set (`effect001`, `effect001Light`) | `objects`: the object table of the effect settings | `noteeffect.json` and the object table | | `live.scene` | `default` | `objects`: the object table of the scene, the bands' start timelines and backgrounds, the lane skins, the UI assets and the built-in graphics settings ([products](products.md#live-scene)); `master`: the master record of its settings, lane skins and bands | `scene.json` and the object table | | `live` | a live: `/[/]` | `objects`: the object table of the chart and the jacket ([products](products.md#lives)); `master`: the live's master record and the manifest paths of its parts | `live.json`, `chart.json`, `notes.json`, `notes.mirror.json` and the object table | -| `story.sound` | a cue sheet name | `objects`; `acb:` and `sheets`: the cue sheet and its `cri.audio` files ([products](products.md#stories)); parameter `audio` (`aac`) | `cues.json` and the cues' sounds in the web format | +| `story.sound` | a cue sheet name | `objects`; `acb:` and `sheets`: the cue sheet and its `cri.audio` files ([products](products.md#stories)); parameter `audio` (`aac`) | `cues.json` (each cue's nonempty CRI `categories` included) and the cues' sounds in the web format | | `story.crilips` | `default` | `objects` (empty); `apk:lib/arm64-v8a/libcri_lips_unity.so` | `crilips.json` and `crilips.bin` | | `story.scene` | `default` | `objects`: the object table of the ADV scene, its prefabs and settings and the built-in graphics settings ([products](products.md#stories)) | `scene.json` and the object table | | `story.asset` | a catalog key a story loads | `objects`: the object table of what the key reaches | `asset.json` and the object table | diff --git a/src/nnnotes/formats/acb.py b/src/nnnotes/formats/acb.py index c6d661a..21c337d 100644 --- a/src/nnnotes/formats/acb.py +++ b/src/nnnotes/formats/acb.py @@ -97,6 +97,28 @@ def commands(b: bytes) -> list[tuple[int, bytes]]: return out +def cue_categories(acb: bytes) -> dict[str, list[str]]: + """{cue name: the CRI category names its sequence puts it in} (command 65 of the cue's sequence, the names of the + ACF category references the ACB lists). A cue that is not a sequence, or whose sequence has no category command, + has an empty list.""" + _, T = tables(acb) + names = {r["Id"]: r["Name"] for r in T.get("AcfReferenceTable", []) if r["Type"] == 3} + cue_names = {r["CueIndex"]: r["CueName"] for r in T.get("CueNameTable", [])} + seqs = T.get("SequenceTable", []) + cmds = T.get("SeqCommandTable", []) + out: dict[str, list[str]] = {} + for i, cue in enumerate(T.get("CueTable", [])): + found: list[str] = [] + if cue["ReferenceType"] == REF_SEQUENCE and cue["ReferenceIndex"] < len(seqs): + ci = seqs[cue["ReferenceIndex"]]["CommandIndex"] + if ci != 65535 and ci < len(cmds): + for code, v in commands(cmds[ci]["Command"]): + if code == 65: + found += [names[struct.unpack_from(">I", v, j)[0]] for j in range(0, len(v) - 3, 4)] + out[cue_names[i]] = found + return out + + def u16s(b: bytes) -> list[int]: return [struct.unpack_from(">H", b, i)[0] for i in range(0, len(b) - 1, 2)] diff --git a/src/nnnotes/pipeline/story.py b/src/nnnotes/pipeline/story.py index 72ae6a1..0257911 100644 --- a/src/nnnotes/pipeline/story.py +++ b/src/nnnotes/pipeline/story.py @@ -5,7 +5,8 @@ story.sound subject a cue sheet name (the `_cueSheetName` of a story's cue sheet table): "#cues.json" (kind story.sound: cue name -> the cue's first stream: `file`, `sampleRate`, `channels`, `samples`, - `loopStart`, `loopEnd` and, in AAC, `encoderDelay`) and "#" for each of those streams + `loopStart`, `loopEnd`, the nonempty CRI `categories` from the ACB and, in AAC, `encoderDelay`) and + "#" for each of those streams (kind audio.story) in the web format of the parameter `audio` (encoders.WEB_AUDIO; default aac: AAC-LC in MP4, each file recording its encoder delay; flac: the decoded FLAC as it is). Its root is the cue sheet's catalog key `Cri/Sound/` and its manifest cuesheets/.json. The atom @@ -57,7 +58,7 @@ from ..content import liveaudio, storyasset, storychat, storyepisode, storyscene, storyvideo from ..content.livescene import GRAPHICS -from ..formats import crilips +from ..formats import acb, crilips from ..formats.encoders import DEFAULT_AUDIO_FORMAT, WEB_AUDIO, mp4_priming, web_encode from ..unity.atoms import Unsupported from . import contract @@ -127,7 +128,7 @@ class SoundStage(ProductStage): """story.sound: a cue sheet -> cues.json and its streams in the web format (module documentation).""" name = SOUND - version = 1 + version = 2 KIND = "StorySound" PARAMS = {"audio": DEFAULT_AUDIO_FORMAT} @@ -161,6 +162,10 @@ def assemble(self, task, objects) -> Assembly: sheet = objects.sheet(liveaudio.sheet_key(_sheet(task.subject))) ext = WEB_AUDIO[fmt][0] doc, sources = cues(sheet.streams(), ext) + categories = acb.cue_categories(sheet.acb()) + for cue, entry in doc.items(): + if categories.get(cue): + entry["categories"] = categories[cue] files, delays = {}, {} for name, src in sources.items(): if WEB_AUDIO[fmt][1] is None: diff --git a/tests/formats/test_acb_categories.py b/tests/formats/test_acb_categories.py new file mode 100644 index 0000000..ac75775 --- /dev/null +++ b/tests/formats/test_acb_categories.py @@ -0,0 +1,52 @@ +"""The CRI categories of the cues of a synthetic ACB (no game data).""" + +import struct + +from nnnotes.formats import acb +from tests.pipeline.test_voices import tlv, utf + + +def acb_with_categories() -> bytes: + ref = lambda i, t, n: {"Id": i, "Type": t, "Name": n} # noqa: E731 + tables = { + "CueNameTable": utf( + "CueName", + [("CueName", 0xA), ("CueIndex", 2)], + [{"CueName": n, "CueIndex": i} for i, n in enumerate(["voice", "plain", "wave"])], + ), + "CueTable": utf( + "Cue", + [("CueId", 4), ("ReferenceType", 0), ("ReferenceIndex", 2), ("Length", 4)], + [ + {"CueId": 0, "ReferenceType": 3, "ReferenceIndex": 0, "Length": 10}, + {"CueId": 1, "ReferenceType": 3, "ReferenceIndex": 1, "Length": 10}, + {"CueId": 2, "ReferenceType": 1, "ReferenceIndex": 0, "Length": 10}, + ], + ), + "SequenceTable": utf( + "Sequence", + [("Type", 0), ("TrackIndex", 0xB), ("CommandIndex", 2)], + [ + {"Type": 0, "TrackIndex": b"", "CommandIndex": 0}, + {"Type": 0, "TrackIndex": b"", "CommandIndex": 65535}, + ], + ), + "SeqCommandTable": utf( + "SeqCommand", [("Command", 0xB)], [{"Command": tlv(65, struct.pack(">II", 7, 9)) + tlv(146, bytes(4))}] + ), + "AcfReferenceTable": utf( + "AcfReference", + [("Id", 2), ("Type", 0), ("Name", 0xA)], + [ref(7, 3, "Voice"), ref(9, 3, "VoiceConfig"), ref(1, 2, "Bus")], + ), + } + cols = [(k, 0xB) for k in tables] + [("Name", 0xA)] + return utf("Header", cols, [{**tables, "Name": "sheet"}]) + + +def test_a_cue_names_the_categories_its_sequence_puts_it_in(): + assert acb.cue_categories(acb_with_categories()) == { + "voice": ["Voice", "VoiceConfig"], # command 65 of its sequence + "plain": [], # a sequence without commands + "wave": [], # not a sequence + } diff --git a/tests/pipeline/test_story.py b/tests/pipeline/test_story.py index e825de4..7767bdc 100644 --- a/tests/pipeline/test_story.py +++ b/tests/pipeline/test_story.py @@ -20,12 +20,14 @@ StoryStage, VideoStage, ) +from tests.formats.test_acb_categories import acb_with_categories from tests.formats.test_crilips import _elf_with_rodata, _synthetic_version from tests.formats.test_usm import KEY from tests.helpers import story as sh from tests.pipeline.test_assemble import addresses from tests.pipeline.test_cristages import export_result, fake_tools, planning, usm_file from tests.pipeline.test_live import master_record, run, stored, table +from tests.pipeline.test_voices import utf from tests.site.test_story_site import mp4 # the streams of a voice sheet as cri.audio lists them: cue v2 has two layers, cue loop loops @@ -38,10 +40,11 @@ ] # fmt: skip -def sheet(depot, name, streams=STREAMS) -> tuple[list, dict]: +def sheet(depot, name, streams=STREAMS, *, acb_bytes=None) -> tuple[list, dict]: """The inputs "acb:" and "sheets" of the cue sheet `name` decoded as `streams` (FLAC bytes per stream).""" key = f"Cri/Sound/{name}" - acb = stored(depot, f"@UTF {name}".encode(), f"acb:{key}") + body = utf("Header", [("Name", 0xA)], [{"Name": name}]) if acb_bytes is None else acb_bytes + acb = stored(depot, body, f"acb:{key}") files = {} for s in streams: data = f"fLaC {name} {s['stream']}".encode() @@ -113,6 +116,39 @@ def test_cues_take_the_first_stream_of_each_name(): assert files == {"v1.m4a": "v1.flac", "v2.m4a": "v2.flac", "loop.m4a": "loop.flac"} +@pytest.mark.parametrize("fmt", ["flac", "aac"]) +def test_story_sound_records_the_nonempty_acb_categories_with_the_first_stream(tmp_path, monkeypatch, fmt): + depot = Depot(tmp_path / "d") + streams = [ + {**STREAMS[0], "name": "voice"}, + {**STREAMS[1], "name": "voice"}, + {**STREAMS[2], "name": "plain"}, + {**STREAMS[3], "name": "wave"}, + ] + inputs, desc = sheet(depot, "story_sheet", streams, acb_bytes=acb_with_categories()) + atoms = {} + if fmt == "aac": + monkeypatch.setattr(cristages, "tool_id", lambda which, exe: f"{which}-fake/1") + monkeypatch.setattr(story, "_ffmpeg", lambda: "ffmpeg") + monkeypatch.setattr(story, "web_encode", lambda ffmpeg, src, dst, format: mp4(1024)) + atoms = {"audio.encode": "ffmpeg-fake/1"} + stage = SoundStage() + task, _, arts = run(depot, stage, "story_sheet", [table(depot), *inputs], {"audio": fmt}, atoms) + doc = read_json(depot, arts["cues.json"]) + ext = ".m4a" if fmt == "aac" else ".flac" + expected = story.cues(streams, ext)[0] + expected["voice"]["categories"] = ["Voice", "VoiceConfig"] + if fmt == "aac": + for entry in expected.values(): + entry["encoderDelay"] = 1024 + assert doc == expected + files = desc["Cri/Sound/story_sheet"]["files"] + assert contract.loads(depot.read(files["streams.json"]["sha256"])) == streams + assert doc["voice"]["samples"] == streams[0]["samples"] + assert stage.version == task.version == 2 + assert contract.Task(task.stage, 1, task.subject, task.params, task.atoms, task.inputs).key != task.key + + # ---------------------------------------------------------------- the products def test_the_sound_product_keeps_the_decoded_flac(tmp_path): depot = Depot(tmp_path / "d") From 403aff0dc02e6b09762aea2da668ffa1693e4902 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Sat, 17 Oct 2026 15:01:00 +0800 Subject: [PATCH 2/2] test(story): isolate AAC category checks from encoder discovery Use the test encoder for the shared audio atom lookup as well as the sound assembler so synthetic category tests run without an installed ffmpeg executable. --- tests/pipeline/test_story.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pipeline/test_story.py b/tests/pipeline/test_story.py index 7767bdc..dc59de6 100644 --- a/tests/pipeline/test_story.py +++ b/tests/pipeline/test_story.py @@ -130,6 +130,7 @@ def test_story_sound_records_the_nonempty_acb_categories_with_the_first_stream(t if fmt == "aac": monkeypatch.setattr(cristages, "tool_id", lambda which, exe: f"{which}-fake/1") monkeypatch.setattr(story, "_ffmpeg", lambda: "ffmpeg") + monkeypatch.setattr(live, "_ffmpeg", lambda: "ffmpeg") monkeypatch.setattr(story, "web_encode", lambda ffmpeg, src, dst, format: mp4(1024)) atoms = {"audio.encode": "ffmpeg-fake/1"} stage = SoundStage()