fix(story): include CRI categories in sound products

docs/products.md

@@ -372,7 +372,9 @@ hold.
 `story.sound` reads its cue sheet's `cri.audio` files; `cues.json` maps each cue name to the
 first stream of that name (in stream order): `file` (the sound, next to `cues.json`), `sampleRate`, `channels`,
 `samples` (frames at `sampleRate`, without encoder padding) and, for a looping stream, `loopStart` and `loopEnd`
-(sample frames). The sounds are in the web format of the parameter `audio`, as for `live.music`: `aac` (the default)
+(sample frames). A cue whose ACB sequence names CRI categories also has `categories`, the ordered category names
+(for example `["Voice", "VoiceConfig"]`), read from sequence command 65 and the ACB's ACF category references.
+The sounds are in the web format of the parameter `audio`, as for `live.music`: `aac` (the default)
 records `encoderDelay`, the priming samples of the MP4 edit list; `flac` keeps the decoded FLAC. The `product`
 layout makes the `story.sound` product of every cue sheet key the selection holds.
 

docs/stages.md

@@ -36,7 +36,7 @@ codec and channel of a USM) and `catalog.index` (version 2: gzip input and Japan
 | `live.noteeffect` | a note effect set (`effect001`, `effect001Light`) | `objects`: the object table of the effect settings | `noteeffect.json` and the object table |
 | `live.scene` | `default` | `objects`: the object table of the scene, the bands' start timelines and backgrounds, the lane skins, the UI assets and the built-in graphics settings ([products](products.md#live-scene)); `master`: the master record of its settings, lane skins and bands | `scene.json` and the object table |
 | `live` | a live: `<musicId>/<difficulty>[/<band>]` | `objects`: the object table of the chart and the jacket ([products](products.md#lives)); `master`: the live's master record and the manifest paths of its parts | `live.json`, `chart.json`, `notes.json`, `notes.mirror.json` and the object table |
-| `story.sound` | a cue sheet name | `objects`; `acb:<key>` and `sheets`: the cue sheet and its `cri.audio` files ([products](products.md#stories)); parameter `audio` (`aac`) | `cues.json` and the cues' sounds in the web format |
+| `story.sound` | a cue sheet name | `objects`; `acb:<key>` and `sheets`: the cue sheet and its `cri.audio` files ([products](products.md#stories)); parameter `audio` (`aac`) | `cues.json` (each cue's nonempty CRI `categories` included) and the cues' sounds in the web format |
 | `story.crilips` | `default` | `objects` (empty); `apk:lib/arm64-v8a/libcri_lips_unity.so` | `crilips.json` and `crilips.bin` |
 | `story.scene` | `default` | `objects`: the object table of the ADV scene, its prefabs and settings and the built-in graphics settings ([products](products.md#stories)) | `scene.json` and the object table |
 | `story.asset` | a catalog key a story loads | `objects`: the object table of what the key reaches | `asset.json` and the object table |

src/nnnotes/formats/acb.py

@@ -97,6 +97,28 @@ def commands(b: bytes) -> list[tuple[int, bytes]]:
     return out
 
 
+def cue_categories(acb: bytes) -> dict[str, list[str]]:
+    """{cue name: the CRI category names its sequence puts it in} (command 65 of the cue's sequence, the names of the
+    ACF category references the ACB lists). A cue that is not a sequence, or whose sequence has no category command,
+    has an empty list."""
+    _, T = tables(acb)
+    names = {r["Id"]: r["Name"] for r in T.get("AcfReferenceTable", []) if r["Type"] == 3}
+    cue_names = {r["CueIndex"]: r["CueName"] for r in T.get("CueNameTable", [])}
+    seqs = T.get("SequenceTable", [])
+    cmds = T.get("SeqCommandTable", [])
+    out: dict[str, list[str]] = {}
+    for i, cue in enumerate(T.get("CueTable", [])):
+        found: list[str] = []
+        if cue["ReferenceType"] == REF_SEQUENCE and cue["ReferenceIndex"] < len(seqs):
+            ci = seqs[cue["ReferenceIndex"]]["CommandIndex"]
+            if ci != 65535 and ci < len(cmds):
+                for code, v in commands(cmds[ci]["Command"]):
+                    if code == 65:
+                        found += [names[struct.unpack_from(">I", v, j)[0]] for j in range(0, len(v) - 3, 4)]
+        out[cue_names[i]] = found
+    return out
+
+
 def u16s(b: bytes) -> list[int]:
     return [struct.unpack_from(">H", b, i)[0] for i in range(0, len(b) - 1, 2)]
 

src/nnnotes/pipeline/story.py

@@ -5,7 +5,8 @@
 
 story.sound    subject a cue sheet name (the `_cueSheetName` of a story's cue sheet table): "<task id>#cues.json"
                (kind story.sound: cue name -> the cue's first stream: `file`, `sampleRate`, `channels`, `samples`,
-               `loopStart`, `loopEnd` and, in AAC, `encoderDelay`) and "<task id>#<file>" for each of those streams
+               `loopStart`, `loopEnd`, the nonempty CRI `categories` from the ACB and, in AAC, `encoderDelay`) and
+               "<task id>#<file>" for each of those streams
                (kind audio.story) in the web format of the parameter `audio` (encoders.WEB_AUDIO; default aac:
                AAC-LC in MP4, each file recording its encoder delay; flac: the decoded FLAC as it is). Its root is the
                cue sheet's catalog key `Cri/Sound/<sheet>` and its manifest cuesheets/<sheet>.json. The atom
@@ -57,7 +58,7 @@
 
 from ..content import liveaudio, storyasset, storychat, storyepisode, storyscene, storyvideo
 from ..content.livescene import GRAPHICS
-from ..formats import crilips
+from ..formats import acb, crilips
 from ..formats.encoders import DEFAULT_AUDIO_FORMAT, WEB_AUDIO, mp4_priming, web_encode
 from ..unity.atoms import Unsupported
 from . import contract
@@ -127,7 +128,7 @@ class SoundStage(ProductStage):
     """story.sound: a cue sheet -> cues.json and its streams in the web format (module documentation)."""
 
     name = SOUND
-    version = 1
+    version = 2
     KIND = "StorySound"
     PARAMS = {"audio": DEFAULT_AUDIO_FORMAT}
 
@@ -161,6 +162,10 @@ def assemble(self, task, objects) -> Assembly:
         sheet = objects.sheet(liveaudio.sheet_key(_sheet(task.subject)))
         ext = WEB_AUDIO[fmt][0]
         doc, sources = cues(sheet.streams(), ext)
+        categories = acb.cue_categories(sheet.acb())
+        for cue, entry in doc.items():
+            if categories.get(cue):
+                entry["categories"] = categories[cue]
         files, delays = {}, {}
         for name, src in sources.items():
             if WEB_AUDIO[fmt][1] is None:

tests/formats/test_acb_categories.py

@@ -0,0 +1,52 @@
+"""The CRI categories of the cues of a synthetic ACB (no game data)."""
+
+import struct
+
+from nnnotes.formats import acb
+from tests.pipeline.test_voices import tlv, utf
+
+
+def acb_with_categories() -> bytes:
+    ref = lambda i, t, n: {"Id": i, "Type": t, "Name": n}  # noqa: E731
+    tables = {
+        "CueNameTable": utf(
+            "CueName",
+            [("CueName", 0xA), ("CueIndex", 2)],
+            [{"CueName": n, "CueIndex": i} for i, n in enumerate(["voice", "plain", "wave"])],
+        ),
+        "CueTable": utf(
+            "Cue",
+            [("CueId", 4), ("ReferenceType", 0), ("ReferenceIndex", 2), ("Length", 4)],
+            [
+                {"CueId": 0, "ReferenceType": 3, "ReferenceIndex": 0, "Length": 10},
+                {"CueId": 1, "ReferenceType": 3, "ReferenceIndex": 1, "Length": 10},
+                {"CueId": 2, "ReferenceType": 1, "ReferenceIndex": 0, "Length": 10},
+            ],
+        ),
+        "SequenceTable": utf(
+            "Sequence",
+            [("Type", 0), ("TrackIndex", 0xB), ("CommandIndex", 2)],
+            [
+                {"Type": 0, "TrackIndex": b"", "CommandIndex": 0},
+                {"Type": 0, "TrackIndex": b"", "CommandIndex": 65535},
+            ],
+        ),
+        "SeqCommandTable": utf(
+            "SeqCommand", [("Command", 0xB)], [{"Command": tlv(65, struct.pack(">II", 7, 9)) + tlv(146, bytes(4))}]
+        ),
+        "AcfReferenceTable": utf(
+            "AcfReference",
+            [("Id", 2), ("Type", 0), ("Name", 0xA)],
+            [ref(7, 3, "Voice"), ref(9, 3, "VoiceConfig"), ref(1, 2, "Bus")],
+        ),
+    }
+    cols = [(k, 0xB) for k in tables] + [("Name", 0xA)]
+    return utf("Header", cols, [{**tables, "Name": "sheet"}])
+
+
+def test_a_cue_names_the_categories_its_sequence_puts_it_in():
+    assert acb.cue_categories(acb_with_categories()) == {
+        "voice": ["Voice", "VoiceConfig"],  # command 65 of its sequence
+        "plain": [],  # a sequence without commands
+        "wave": [],  # not a sequence
+    }

tests/pipeline/test_story.py

@@ -20,12 +20,14 @@
     StoryStage,
     VideoStage,
 )
+from tests.formats.test_acb_categories import acb_with_categories
 from tests.formats.test_crilips import _elf_with_rodata, _synthetic_version
 from tests.formats.test_usm import KEY
 from tests.helpers import story as sh
 from tests.pipeline.test_assemble import addresses
 from tests.pipeline.test_cristages import export_result, fake_tools, planning, usm_file
 from tests.pipeline.test_live import master_record, run, stored, table
+from tests.pipeline.test_voices import utf
 from tests.site.test_story_site import mp4
 
 # the streams of a voice sheet as cri.audio lists them: cue v2 has two layers, cue loop loops
@@ -38,10 +40,11 @@
 ]  # fmt: skip
 
 
-def sheet(depot, name, streams=STREAMS) -> tuple[list, dict]:
+def sheet(depot, name, streams=STREAMS, *, acb_bytes=None) -> tuple[list, dict]:
     """The inputs "acb:<key>" and "sheets" of the cue sheet `name` decoded as `streams` (FLAC bytes per stream)."""
     key = f"Cri/Sound/{name}"
-    acb = stored(depot, f"@UTF {name}".encode(), f"acb:{key}")
+    body = utf("Header", [("Name", 0xA)], [{"Name": name}]) if acb_bytes is None else acb_bytes
+    acb = stored(depot, body, f"acb:{key}")
     files = {}
     for s in streams:
         data = f"fLaC {name} {s['stream']}".encode()
@@ -113,6 +116,40 @@ def test_cues_take_the_first_stream_of_each_name():
     assert files == {"v1.m4a": "v1.flac", "v2.m4a": "v2.flac", "loop.m4a": "loop.flac"}
 
 
+@pytest.mark.parametrize("fmt", ["flac", "aac"])
+def test_story_sound_records_the_nonempty_acb_categories_with_the_first_stream(tmp_path, monkeypatch, fmt):
+    depot = Depot(tmp_path / "d")
+    streams = [
+        {**STREAMS[0], "name": "voice"},
+        {**STREAMS[1], "name": "voice"},
+        {**STREAMS[2], "name": "plain"},
+        {**STREAMS[3], "name": "wave"},
+    ]
+    inputs, desc = sheet(depot, "story_sheet", streams, acb_bytes=acb_with_categories())
+    atoms = {}
+    if fmt == "aac":
+        monkeypatch.setattr(cristages, "tool_id", lambda which, exe: f"{which}-fake/1")
+        monkeypatch.setattr(story, "_ffmpeg", lambda: "ffmpeg")
+        monkeypatch.setattr(live, "_ffmpeg", lambda: "ffmpeg")
+        monkeypatch.setattr(story, "web_encode", lambda ffmpeg, src, dst, format: mp4(1024))
+        atoms = {"audio.encode": "ffmpeg-fake/1"}
+    stage = SoundStage()
+    task, _, arts = run(depot, stage, "story_sheet", [table(depot), *inputs], {"audio": fmt}, atoms)
+    doc = read_json(depot, arts["cues.json"])
+    ext = ".m4a" if fmt == "aac" else ".flac"
+    expected = story.cues(streams, ext)[0]
+    expected["voice"]["categories"] = ["Voice", "VoiceConfig"]
+    if fmt == "aac":
+        for entry in expected.values():
+            entry["encoderDelay"] = 1024
+    assert doc == expected
+    files = desc["Cri/Sound/story_sheet"]["files"]
+    assert contract.loads(depot.read(files["streams.json"]["sha256"])) == streams
+    assert doc["voice"]["samples"] == streams[0]["samples"]
+    assert stage.version == task.version == 2
+    assert contract.Task(task.stage, 1, task.subject, task.params, task.atoms, task.inputs).key != task.key
+
+
 # ---------------------------------------------------------------- the products
 def test_the_sound_product_keeps_the_decoded_flac(tmp_path):
     depot = Depot(tmp_path / "d")