diff --git a/docs/products.md b/docs/products.md index 9af78fe..eb469cf 100644 --- a/docs/products.md +++ b/docs/products.md @@ -372,7 +372,9 @@ hold. `story.sound` reads its cue sheet's `cri.audio` files; `cues.json` maps each cue name to the first stream of that name (in stream order): `file` (the sound, next to `cues.json`), `sampleRate`, `channels`, `samples` (frames at `sampleRate`, without encoder padding) and, for a looping stream, `loopStart` and `loopEnd` -(sample frames). The sounds are in the web format of the parameter `audio`, as for `live.music`: `aac` (the default) +(sample frames). A cue whose ACB sequence names CRI categories also has `categories`, the ordered category names +(for example `["Voice", "VoiceConfig"]`), read from sequence command 65 and the ACB's ACF category references. +The sounds are in the web format of the parameter `audio`, as for `live.music`: `aac` (the default) records `encoderDelay`, the priming samples of the MP4 edit list; `flac` keeps the decoded FLAC. The `product` layout makes the `story.sound` product of every cue sheet key the selection holds. diff --git a/docs/stages.md b/docs/stages.md index 0cc2805..68b59d1 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -36,7 +36,7 @@ codec and channel of a USM) and `catalog.index` (version 2: gzip input and Japan | `live.noteeffect` | a note effect set (`effect001`, `effect001Light`) | `objects`: the object table of the effect settings | `noteeffect.json` and the object table | | `live.scene` | `default` | `objects`: the object table of the scene, the bands' start timelines and backgrounds, the lane skins, the UI assets and the built-in graphics settings ([products](products.md#live-scene)); `master`: the master record of its settings, lane skins and bands | `scene.json` and the object table | | `live` | a live: `/[/]` | `objects`: the object table of the chart and the jacket ([products](products.md#lives)); `master`: the live's master record and the manifest paths of its parts | `live.json`, `chart.json`, `notes.json`, `notes.mirror.json` and the object table | -| `story.sound` | a cue sheet name | `objects`; `acb:` and `sheets`: the cue sheet and its `cri.audio` files ([products](products.md#stories)); parameter `audio` (`aac`) | `cues.json` and the cues' sounds in the web format | +| `story.sound` | a cue sheet name | `objects`; `acb:` and `sheets`: the cue sheet and its `cri.audio` files ([products](products.md#stories)); parameter `audio` (`aac`) | `cues.json` (each cue's nonempty CRI `categories` included) and the cues' sounds in the web format | | `story.crilips` | `default` | `objects` (empty); `apk:lib/arm64-v8a/libcri_lips_unity.so` | `crilips.json` and `crilips.bin` | | `story.scene` | `default` | `objects`: the object table of the ADV scene, its prefabs and settings and the built-in graphics settings ([products](products.md#stories)) | `scene.json` and the object table | | `story.asset` | a catalog key a story loads | `objects`: the object table of what the key reaches | `asset.json` and the object table | diff --git a/src/nnnotes/formats/acb.py b/src/nnnotes/formats/acb.py index c6d661a..21c337d 100644 --- a/src/nnnotes/formats/acb.py +++ b/src/nnnotes/formats/acb.py @@ -97,6 +97,28 @@ def commands(b: bytes) -> list[tuple[int, bytes]]: return out +def cue_categories(acb: bytes) -> dict[str, list[str]]: + """{cue name: the CRI category names its sequence puts it in} (command 65 of the cue's sequence, the names of the + ACF category references the ACB lists). A cue that is not a sequence, or whose sequence has no category command, + has an empty list.""" + _, T = tables(acb) + names = {r["Id"]: r["Name"] for r in T.get("AcfReferenceTable", []) if r["Type"] == 3} + cue_names = {r["CueIndex"]: r["CueName"] for r in T.get("CueNameTable", [])} + seqs = T.get("SequenceTable", []) + cmds = T.get("SeqCommandTable", []) + out: dict[str, list[str]] = {} + for i, cue in enumerate(T.get("CueTable", [])): + found: list[str] = [] + if cue["ReferenceType"] == REF_SEQUENCE and cue["ReferenceIndex"] < len(seqs): + ci = seqs[cue["ReferenceIndex"]]["CommandIndex"] + if ci != 65535 and ci < len(cmds): + for code, v in commands(cmds[ci]["Command"]): + if code == 65: + found += [names[struct.unpack_from(">I", v, j)[0]] for j in range(0, len(v) - 3, 4)] + out[cue_names[i]] = found + return out + + def u16s(b: bytes) -> list[int]: return [struct.unpack_from(">H", b, i)[0] for i in range(0, len(b) - 1, 2)] diff --git a/src/nnnotes/pipeline/story.py b/src/nnnotes/pipeline/story.py index 72ae6a1..0257911 100644 --- a/src/nnnotes/pipeline/story.py +++ b/src/nnnotes/pipeline/story.py @@ -5,7 +5,8 @@ story.sound subject a cue sheet name (the `_cueSheetName` of a story's cue sheet table): "#cues.json" (kind story.sound: cue name -> the cue's first stream: `file`, `sampleRate`, `channels`, `samples`, - `loopStart`, `loopEnd` and, in AAC, `encoderDelay`) and "#" for each of those streams + `loopStart`, `loopEnd`, the nonempty CRI `categories` from the ACB and, in AAC, `encoderDelay`) and + "#" for each of those streams (kind audio.story) in the web format of the parameter `audio` (encoders.WEB_AUDIO; default aac: AAC-LC in MP4, each file recording its encoder delay; flac: the decoded FLAC as it is). Its root is the cue sheet's catalog key `Cri/Sound/` and its manifest cuesheets/.json. The atom @@ -57,7 +58,7 @@ from ..content import liveaudio, storyasset, storychat, storyepisode, storyscene, storyvideo from ..content.livescene import GRAPHICS -from ..formats import crilips +from ..formats import acb, crilips from ..formats.encoders import DEFAULT_AUDIO_FORMAT, WEB_AUDIO, mp4_priming, web_encode from ..unity.atoms import Unsupported from . import contract @@ -127,7 +128,7 @@ class SoundStage(ProductStage): """story.sound: a cue sheet -> cues.json and its streams in the web format (module documentation).""" name = SOUND - version = 1 + version = 2 KIND = "StorySound" PARAMS = {"audio": DEFAULT_AUDIO_FORMAT} @@ -161,6 +162,10 @@ def assemble(self, task, objects) -> Assembly: sheet = objects.sheet(liveaudio.sheet_key(_sheet(task.subject))) ext = WEB_AUDIO[fmt][0] doc, sources = cues(sheet.streams(), ext) + categories = acb.cue_categories(sheet.acb()) + for cue, entry in doc.items(): + if categories.get(cue): + entry["categories"] = categories[cue] files, delays = {}, {} for name, src in sources.items(): if WEB_AUDIO[fmt][1] is None: diff --git a/tests/formats/test_acb_categories.py b/tests/formats/test_acb_categories.py new file mode 100644 index 0000000..ac75775 --- /dev/null +++ b/tests/formats/test_acb_categories.py @@ -0,0 +1,52 @@ +"""The CRI categories of the cues of a synthetic ACB (no game data).""" + +import struct + +from nnnotes.formats import acb +from tests.pipeline.test_voices import tlv, utf + + +def acb_with_categories() -> bytes: + ref = lambda i, t, n: {"Id": i, "Type": t, "Name": n} # noqa: E731 + tables = { + "CueNameTable": utf( + "CueName", + [("CueName", 0xA), ("CueIndex", 2)], + [{"CueName": n, "CueIndex": i} for i, n in enumerate(["voice", "plain", "wave"])], + ), + "CueTable": utf( + "Cue", + [("CueId", 4), ("ReferenceType", 0), ("ReferenceIndex", 2), ("Length", 4)], + [ + {"CueId": 0, "ReferenceType": 3, "ReferenceIndex": 0, "Length": 10}, + {"CueId": 1, "ReferenceType": 3, "ReferenceIndex": 1, "Length": 10}, + {"CueId": 2, "ReferenceType": 1, "ReferenceIndex": 0, "Length": 10}, + ], + ), + "SequenceTable": utf( + "Sequence", + [("Type", 0), ("TrackIndex", 0xB), ("CommandIndex", 2)], + [ + {"Type": 0, "TrackIndex": b"", "CommandIndex": 0}, + {"Type": 0, "TrackIndex": b"", "CommandIndex": 65535}, + ], + ), + "SeqCommandTable": utf( + "SeqCommand", [("Command", 0xB)], [{"Command": tlv(65, struct.pack(">II", 7, 9)) + tlv(146, bytes(4))}] + ), + "AcfReferenceTable": utf( + "AcfReference", + [("Id", 2), ("Type", 0), ("Name", 0xA)], + [ref(7, 3, "Voice"), ref(9, 3, "VoiceConfig"), ref(1, 2, "Bus")], + ), + } + cols = [(k, 0xB) for k in tables] + [("Name", 0xA)] + return utf("Header", cols, [{**tables, "Name": "sheet"}]) + + +def test_a_cue_names_the_categories_its_sequence_puts_it_in(): + assert acb.cue_categories(acb_with_categories()) == { + "voice": ["Voice", "VoiceConfig"], # command 65 of its sequence + "plain": [], # a sequence without commands + "wave": [], # not a sequence + } diff --git a/tests/pipeline/test_story.py b/tests/pipeline/test_story.py index e825de4..dc59de6 100644 --- a/tests/pipeline/test_story.py +++ b/tests/pipeline/test_story.py @@ -20,12 +20,14 @@ StoryStage, VideoStage, ) +from tests.formats.test_acb_categories import acb_with_categories from tests.formats.test_crilips import _elf_with_rodata, _synthetic_version from tests.formats.test_usm import KEY from tests.helpers import story as sh from tests.pipeline.test_assemble import addresses from tests.pipeline.test_cristages import export_result, fake_tools, planning, usm_file from tests.pipeline.test_live import master_record, run, stored, table +from tests.pipeline.test_voices import utf from tests.site.test_story_site import mp4 # the streams of a voice sheet as cri.audio lists them: cue v2 has two layers, cue loop loops @@ -38,10 +40,11 @@ ] # fmt: skip -def sheet(depot, name, streams=STREAMS) -> tuple[list, dict]: +def sheet(depot, name, streams=STREAMS, *, acb_bytes=None) -> tuple[list, dict]: """The inputs "acb:" and "sheets" of the cue sheet `name` decoded as `streams` (FLAC bytes per stream).""" key = f"Cri/Sound/{name}" - acb = stored(depot, f"@UTF {name}".encode(), f"acb:{key}") + body = utf("Header", [("Name", 0xA)], [{"Name": name}]) if acb_bytes is None else acb_bytes + acb = stored(depot, body, f"acb:{key}") files = {} for s in streams: data = f"fLaC {name} {s['stream']}".encode() @@ -113,6 +116,40 @@ def test_cues_take_the_first_stream_of_each_name(): assert files == {"v1.m4a": "v1.flac", "v2.m4a": "v2.flac", "loop.m4a": "loop.flac"} +@pytest.mark.parametrize("fmt", ["flac", "aac"]) +def test_story_sound_records_the_nonempty_acb_categories_with_the_first_stream(tmp_path, monkeypatch, fmt): + depot = Depot(tmp_path / "d") + streams = [ + {**STREAMS[0], "name": "voice"}, + {**STREAMS[1], "name": "voice"}, + {**STREAMS[2], "name": "plain"}, + {**STREAMS[3], "name": "wave"}, + ] + inputs, desc = sheet(depot, "story_sheet", streams, acb_bytes=acb_with_categories()) + atoms = {} + if fmt == "aac": + monkeypatch.setattr(cristages, "tool_id", lambda which, exe: f"{which}-fake/1") + monkeypatch.setattr(story, "_ffmpeg", lambda: "ffmpeg") + monkeypatch.setattr(live, "_ffmpeg", lambda: "ffmpeg") + monkeypatch.setattr(story, "web_encode", lambda ffmpeg, src, dst, format: mp4(1024)) + atoms = {"audio.encode": "ffmpeg-fake/1"} + stage = SoundStage() + task, _, arts = run(depot, stage, "story_sheet", [table(depot), *inputs], {"audio": fmt}, atoms) + doc = read_json(depot, arts["cues.json"]) + ext = ".m4a" if fmt == "aac" else ".flac" + expected = story.cues(streams, ext)[0] + expected["voice"]["categories"] = ["Voice", "VoiceConfig"] + if fmt == "aac": + for entry in expected.values(): + entry["encoderDelay"] = 1024 + assert doc == expected + files = desc["Cri/Sound/story_sheet"]["files"] + assert contract.loads(depot.read(files["streams.json"]["sha256"])) == streams + assert doc["voice"]["samples"] == streams[0]["samples"] + assert stage.version == task.version == 2 + assert contract.Task(task.stage, 1, task.subject, task.params, task.atoms, task.inputs).key != task.key + + # ---------------------------------------------------------------- the products def test_the_sound_product_keeps_the_decoded_flac(tmp_path): depot = Depot(tmp_path / "d")