From 187f7efc6f430922e61fec2eeb25e4d6df13edf5 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Wed, 14 Oct 2026 16:59:10 +0800 Subject: [PATCH] refactor(cli): run export live2d, audio and shader through the store The single-subject exports of a Live2D model, a cue sheet and the shaders of a key now run their stages through the store over the files the subject's catalog keys load, then copy the subject's artifacts into the output directory. A second export of the same subject is a store hit. - export live2d places the files of the live2d.model task. - export audio runs cri.audio with a new `format` parameter (flac, ogg, wav); its atoms follow the format. Ogg is encoded with ffmpeg's bit-exact flags, so the same samples give the same file. - export shader writes the Shader artifacts of the unity.export tasks of the key's closure; the first shader of each name in load order wins. - shader.safe_name is public; the docs describe the store requirement. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 8 +- docs/commands.md | 15 +- docs/configuration.md | 6 +- docs/stages.md | 2 +- src/nnnotes/cli/assets.py | 90 +++++++++++ src/nnnotes/cli/export.py | 102 +++++++++--- src/nnnotes/formats/cri.py | 16 +- src/nnnotes/formats/shader.py | 6 +- src/nnnotes/pipeline/cristages.py | 51 ++++-- tests/cli/test_cli.py | 4 +- tests/cli/test_subjects.py | 247 ++++++++++++++++++++++++++++++ tests/pipeline/test_cristages.py | 15 +- tests/unity/test_speed_exact.py | 10 +- 13 files changed, 507 insertions(+), 65 deletions(-) create mode 100644 tests/cli/test_subjects.py diff --git a/docs/architecture.md b/docs/architecture.md index e259fe3..513f079 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -36,9 +36,11 @@ decrypted bundles in the cache directory. ## Two ways to export -- **One subject.** `export episode`, `export story`, `export live`, `export live2d`, `export spot`, `export room`, - `export shader`, `export audio`, `export ui`, `site build` and `music-data` read what one subject needs through the - Session and write their output directory directly ([Export stories and lives](export.md)). +- **One subject.** `export live2d`, `export shader` and `export audio` run the stages of their subject through the + store, over the files the subject's catalog keys load, and copy the subject's files from the store into the output + directory; a second export of the same subject is a hit of the store. `export episode`, `export story`, + `export live`, `export spot`, `export room`, `export ui`, `site build` and `music-data` read what one subject needs + through the Session and write their output directory directly ([Export stories and lives](export.md)). - **Every asset.** `export assets` splits the work into stages ([stages](stages.md)). A task is one stage applied to one subject (a bundle, a content id, a view); its key covers exactly what its output depends on: the stage and converter versions, the input contents and the options ([contracts](contracts.md)). Results and their files are diff --git a/docs/commands.md b/docs/commands.md index 7dc367c..b73ef86 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -285,7 +285,9 @@ OUT/motions/.motion3.json motions (and motions/_fades.json) OUT/.model3.json file references, EyeBlink / LipSync groups ``` -Needs `[paths] apk` (component classes are resolved through the APK). Prints a JSON summary. +Needs `[paths] apk` (component classes are resolved through the APK). The model is the `live2d.model` task of the +store ([stages](stages.md)): the bundles of the model's closure are fetched into the cache, the task runs once and +its files are copied from the store. Prints a JSON summary. ## export spot @@ -348,6 +350,10 @@ OUT///sp

__. every compiled sub-program (G OUT/shaders.json index with the keywords of every variant ``` +The shaders are the `Shader` artifacts of the `unity.export` tasks of those bundles in the store (the export limited +to the class `Shader`). A name met again later in the bundle order (the closure in the order the catalog loads it) +keeps its first shader; `shaders.json` names the bundle file each shader comes from. + ## export audio ``` @@ -356,9 +362,10 @@ nnnotes export audio CUE_SHEET -o OUT [--format flac|ogg|wav] [--flac-level N] Decodes the CRI cue sheet of the key `Cri/Sound/`: one file per stream (a name repeated within the sheet gets `_`), `cues.json` (first stream per name: file, sample rate, channels, samples, loop -points) and `streams.json` (every stream in order). `flac` (default) keeps the decoded PCM bit-exact; `ogg` is lossy. -`--flac-level` (0 to 12, default 8) is ffmpeg's FLAC compression level; every level decodes to the same samples. -The HCA keycode is read from `[paths] apk`. +points) and `streams.json` (every stream in order). `flac` (default) keeps the decoded PCM bit-exact; `ogg` is lossy +(encoded with ffmpeg's bit-exact flags: the same samples give the same file). `--flac-level` (0 to 12, default 8) is +ffmpeg's FLAC compression level; every level decodes to the same samples. The HCA keycode is read from the boot data +of `[paths] apk`. The sheet is the `cri.audio` task of its ACB in the store, with the format as a parameter. ## voices diff --git a/docs/configuration.md b/docs/configuration.md index 2cfcb5a..78ca0cd 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -166,11 +166,11 @@ story text); it is not a setting. | `servers` | `[bootstrap] api`; the client version: `[client] version` or `[paths] apk` | | `export episode` | catalog, master data | | `export story` | catalog, `[catalog] language`, master data, `[paths] apk`, `ffmpeg` (audio and videos), `vgmstream` (not with `--no-audio`) | -| `export live2d` | catalog, `[paths] apk` | +| `export live2d` | the store (`[paths] store` or `[paths] cache`), catalog, `[paths] apk` | | `export spot` | catalog, master data, `[paths] apk` | | `export room` | catalog | -| `export shader` | catalog; `--apk-bundle` also `[paths] apk` | -| `export audio` | catalog, `[paths] apk`, `vgmstream`, `ffmpeg` (not for `--format wav`) | +| `export shader` | the store, catalog; `--apk-bundle` also `[paths] apk` | +| `export audio` | the store, catalog, `[paths] apk`, `vgmstream`, `ffmpeg` (not for `--format wav`) | | `apk hca-key` | `[paths] apk` | | `apk settings` | `[paths] apk` | | `export live` | catalog, `[catalog] language`, master data, `[paths] apk`, `vgmstream`, `ffmpeg` | diff --git a/docs/stages.md b/docs/stages.md index 16d9e9f..e288a6d 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -111,7 +111,7 @@ bundle. | Stage | Parameters | Atoms | Artifacts (`#`) | |---|---|---|---| -| `cri.audio` | `flac.level` (8): ffmpeg's FLAC compression level 0-12 (every level decodes to the same samples) | `hca.decode` (vgmstream), `flac.encode` (ffmpeg) | one FLAC per vgmstream stream (a name repeated within the sheet gets `_`; facts: stream, name, sample rate, channels, samples, loop points), `cues.json` (the first stream of each name), `streams.json` (every stream in order): the bytes `nnnotes export audio` writes for the sheet (FLAC) | +| `cri.audio` | `format`: `flac` (default; with `flac.level` (8): ffmpeg's FLAC compression level 0-12, every level decodes to the same samples), `ogg` (Vorbis, ffmpeg's bit-exact flags) or `wav` | `hca.decode` (vgmstream); `flac.encode` or `ogg.encode` (ffmpeg) | one file per vgmstream stream in the format (a name repeated within the sheet gets `_`; facts: stream, name, sample rate, channels, samples, loop points), `cues.json` (the first stream of each name), `streams.json` (every stream in order): the files `nnnotes export audio` writes for the sheet | | `cri.movie` | `format`: `mkv` (default; with `flac.level`) or `webm` | `usm.demux` (nnnotes: `numpy-/2`), `movie.mux` (ffmpeg), `hca.decode` (vgmstream) | per stream as stored and unmasked (a stream stored without its mask is recognised by its structure), named by its kind and, past channel 0, `_`: `video.` and `alpha.` (`ivf`: VP9 in IVF; `m1v`: MPEG-1 elementary stream, CRI's Sofdec.Prime; `h264`: H.264 Annex B; facts: codec, size, display size, frame rate, frames, an alpha stream's alpha type), `audio.` (`adx`; `hca`, still enciphered as stored; facts: channels, sample rate, samples or blocks, cipher), `subtitle.json` (the records as stored: language, time unit, start, duration, the text decoded as UTF-8 or `textHex`) with `subtitle.srt` and `subtitle.vtt`; `movie.mkv`: every video and alpha stream copied (the alpha as a second video track titled `alpha`), every audio stream as FLAC (HCA decoded by vgmstream with the boot data's key), every subtitle channel as a SubRip track (facts: the tracks); with `format: webm` `movie.webm`, the story's form: the first video stream, VP9 copied, or re-encoded as VP9 (libvpx, CRF 20) when it is not VP9 or has an alpha stream, which becomes its alpha channel cropped to the display size; the first audio stream as Opus; no subtitles | `cri.movie` in players: a player shows the first video track of `movie.mkv` and no player composites the `alpha` diff --git a/src/nnnotes/cli/assets.py b/src/nnnotes/cli/assets.py index 3536ff9..48a486f 100644 --- a/src/nnnotes/cli/assets.py +++ b/src/nnnotes/cli/assets.py @@ -1097,6 +1097,96 @@ def parse_layouts(spec: str) -> list[str]: return names +# ---------------------------------------------------------------- single subjects +def _literal(name: str) -> str: + """A stable name as a `bundle:` glob that matches that name only.""" + return "".join(f"[{c}]" if c in "*?[" else c for c in name) + + +def key_selection(index: dict, keys=(), files=()) -> list[str]: + """The selection of the bundles and raw files the catalog keys `keys` load (every location of each key and its + dependency closure) and of the files `files` (stable names), by stable name.""" + from ..pipeline import catalogdb + + want, reach = set(keys), catalogdb.closures(index) + names = set(files) + for loc in index["locations"]: + if loc["primaryKey"] in want: + names |= {stable for _, stable in reach[loc["id"]]} + return parse_selection([f"bundle:{_literal(stable)}" for stable in names]) + + +def subject_pipeline(stages: dict[str, Stage], want) -> list[str]: + """The stages `want` and the stages they read (Stage.after, transitively), in the order of the pipeline.""" + need, stack = set(), list(want) + while stack: + name = stack.pop() + if name not in need: + need.add(name) + stack += [a for a in stages[name].after if a in stages] + return [n for n in stages if n in need or n == "catalog.index"] + + +class SubjectRun: + """The stages of one command's subjects, run in this process: `run(want, params, keys=, files=)` runs the stages + `want` and those they read over the files the catalog keys `keys` load and the files `files` (key_selection); + `result(task id)` is a task's result. The store, catalog and settings are those of `export assets`.""" + + def __init__(self, args, cfg): + self.ws = Workspace(args, cfg) + self.ws.run_index() + + @property + def store(self): + return self.ws.store + + def index(self) -> dict: + return self.ws.index() + + def run(self, want, params: dict | None = None, *, keys=(), files=()): + from ..pipeline.orchestrate import Orchestrator + + ws = self.ws + ws.selection = key_selection(ws.index(), keys, files) + pipeline = subject_pipeline(ws.installed.stages, want) + facts = ws.facts(pipeline, fetch=True) + self.orch = Orchestrator(ws.store, ws.scheduled, workers=0, log=ws.log) + self.last = self.orch.run(pipeline, params=params, facts=facts, context=ws.context(), selection=ws.selection) + return self.last + + def result(self, tid: str) -> dict: + """The result of task `tid` of the last run; when it failed or did not run (a file it reads could not be + read), the command exits with status 1 naming why.""" + t = self.last.tasks.get(tid) + if t is None or t["status"] == "failed": + why = [f"{f['code']}: {f['message']}" for f in self.last.failures_ if f["task"] == tid] + why += [f"{p['name']}: {p['message']}" for p in self.ws.problems()] + sys.exit(f"nnnotes: {tid}: no result" + (f" ({'; '.join(why)})" if why else "")) + return self.store.committed(t["key"], tid) + + def names(self, stage: str) -> dict: + """{subject: catalog names} of a stage that names its tasks' files (Workspace.names), after the last run.""" + self.ws.done({tid: t["key"] for tid, t in self.last.tasks.items() if t["status"] != "failed"}) + return self.ws.names(stage) + + +def place(store, doc: dict, out: Path, roles=None) -> list[str]: + """Write the artifacts of a result into the directory `out`, each as the file its role names (the part of its + id after the task id), relative to `out`; `roles`: only those. Returns the files written, sorted.""" + from ..pipeline.store import write_file + + written = [] + for a in doc["artifacts"]: + _, role = contract.parse_artifact_id(a["id"]) + if roles is not None and role not in roles: + continue + dst = Path(out, *role.split("/")) + dst.parent.mkdir(parents=True, exist_ok=True) + write_file(dst, store.read(a["content"]["sha256"])) + written.append(role) + return sorted(written) + + # ---------------------------------------------------------------- export class Totals: """What export reports of the results besides the run's counts, from its one reading of each (Run.scan): the diff --git a/src/nnnotes/cli/export.py b/src/nnnotes/cli/export.py index a9bbfb6..c0a80a6 100644 --- a/src/nnnotes/cli/export.py +++ b/src/nnnotes/cli/export.py @@ -9,6 +9,7 @@ from pathlib import Path from ..core.jsonio import write_json +from ..pipeline.store import write_file from ..source.session import Session from . import assets from .common import ( @@ -122,16 +123,24 @@ def cmd_live(args, cfg): def cmd_live2d(args, cfg): + from ..pipeline import contract + from ..pipeline.modelstages import LIVE2D from ..site import webmodel - from ..unity import live2d cfg.require_path("paths", "apk") # the Cubism component classes are read from the APK - cat = Session(cfg).catalog() + run = assets.SubjectRun(args, cfg) + prefix = webmodel.LIVE2D_PREFIX + keys = {loc["primaryKey"] for loc in run.index()["locations"] if loc["primaryKey"].startswith(prefix)} try: # a model id or a model key - (key,) = webmodel.catalog_models(cat, [args.key]).values() + (key,) = webmodel.select([args.key], webmodel.model_keys(sorted(keys))).values() except ValueError as e: usage(args, str(e)) - report(args, live2d.extract_model(cat, key, Path(args.out))) + run.run([LIVE2D], keys=[key]) + doc = run.result(contract.task_id(LIVE2D, key)) + out = Path(args.out) + assets.place(run.store, doc, out) + (summary,) = [a["semantics"]["facts"] for a in doc["artifacts"] if a["semantics"]["kind"] == "live2d.model3"] + report(args, summary) def cmd_spot(args, cfg): @@ -176,35 +185,82 @@ def cmd_room(args, cfg): def cmd_shader(args, cfg): from ..formats import shader - - cat = Session(cfg).catalog() + from ..pipeline import contract + from ..pipeline.modelstages import Closures + from ..pipeline.objexport import EXPORT + + if not args.key: + cfg.require_path("paths", "apk") + run = assets.SubjectRun(args, cfg) + index = run.index() if args.key: - known_keys(args, cat, [args.key]) - bundles = cat.fetch_key(args.key) + if not any(loc["primaryKey"] == args.key for loc in index["locations"]): + usage(args, f"not a key of the catalog: {args.key}") + order = Closures(index).closure_bundles(args.key) # the bundles of the key's closure, as the catalog loads them else: - if cat.apk is None: - cfg.require_path("paths", "apk") - bundles = [] + order = [] for s in args.apk_bundle: - try: - bundles.append(cat.apk_bundle(s)) - except KeyError as e: # none or several bundles match - usage(args, f"--apk-bundle {e.args[0]}") - r = shader.dump(bundles, Path(args.out)) + names = [b["stable"] for b in index["bundles"] if not b["remote"] and s in b["name"]] + if len(names) != 1: + usage(args, f"--apk-bundle {s!r}: {len(names)} APK bundles match") + order += names + run.run([EXPORT], {EXPORT: {"classes": ["Shader"]}}, keys=[args.key] if args.key else (), files=order) + out = Path(args.out) + names = {b["stable"]: b["name"] for b in index["bundles"]} + records: list = [] # shaders.json: the first shader of each name, in bundle and object order + for stable in order: + doc = run.result(contract.task_id(EXPORT, stable)) + arts = {a["id"]: a for a in doc["artifacts"]} + shaders = [it for it in doc["items"] if it.get("class") == "Shader"] + for it in sorted(shaders, key=lambda it: contract.parse_object_id(it["object"])): + if it["status"] != "exported": + sys.exit(f"nnnotes: shader {it['object']} of {names[stable]}: {it.get('why', {}).get('message')}") + obj = it["object"] + index_doc = contract.loads(run.store.read(arts[contract.artifact_id(obj, "index")]["content"]["sha256"])) + name = index_doc["name"] + if any(r["name"] == name for r in records): + continue + safe = shader.safe_name(name) + parsed = run.store.read(arts[contract.artifact_id(obj, "json")]["content"]["sha256"]) + out.mkdir(parents=True, exist_ok=True) + write_file(out / f"{safe}.json", parsed) + variants = [] + for v in index_doc["variants"]: + code = run.store.read(arts[contract.artifact_id(obj, v["role"])]["content"]["sha256"]) + write_file(out.joinpath(*v["file"].split("/")), code) + variants.append({k: x for k, x in v.items() if k != "role"}) + records.append({"name": name, "source": names[stable], "parsed": f"{safe}.json", "variants": variants}) + r = shader.write_index(records, out) report(args, r, lambda r: [f"{r['count']} shaders, {r['variants']} variants", *(f" {n}" for n in r["names"])]) def cmd_audio(args, cfg): - from ..formats import cri + from ..pipeline import contract + from ..pipeline.cristages import AUDIO flac = flac_options(args) - cfg.require_path("paths", "apk") # the HCA keycode is read from the APK - cat = Session(cfg).catalog() - if not cat.has(f"Cri/Sound/{args.cue_sheet}"): - usage(args, f"cue sheet {args.cue_sheet}: no key Cri/Sound/{args.cue_sheet} in the catalog") + cfg.require_path("paths", "apk") # the HCA keycode is read from the APK's boot data + run = assets.SubjectRun(args, cfg) + key = f"Cri/Sound/{args.cue_sheet}" + if not any(loc["primaryKey"] == key for loc in run.index()["locations"]): + usage(args, f"cue sheet {args.cue_sheet}: no key {key} in the catalog") + params = {"format": args.format} + if "flac_level" in flac: + params["flac"] = {"level": flac["flac_level"]} + if args.format == "flac": + del params["format"] + run.run([AUDIO], {AUDIO: params}, keys=[key]) + subjects = [s for s, names in run.names(AUDIO).items() if key in names] + if len(subjects) != 1: + sys.exit(f"nnnotes: cue sheet {args.cue_sheet}: {len(subjects)} ACBs have the key {key}") + doc = run.result(contract.task_id(AUDIO, subjects[0])) + (item,) = doc["items"] + if item["status"] != "exported": + sys.exit(f"nnnotes: cue sheet {args.cue_sheet}: {item['why']['message']}") out = Path(args.out) - cues = cri.decode(cat, args.cue_sheet, out, fmt=args.format, **flac) - report(args, {"out": str(out), "cues": len(cues)}, lambda r: [f"{r['out']} ({r['cues']} cues)"]) + assets.place(run.store, doc, out) + (cues,) = [a["semantics"]["facts"]["cues"] for a in doc["artifacts"] if a["semantics"]["kind"] == "cri.cues"] + report(args, {"out": str(out), "cues": cues}, lambda r: [f"{r['out']} ({r['cues']} cues)"]) def cmd_ui(args, cfg): diff --git a/src/nnnotes/formats/cri.py b/src/nnnotes/formats/cri.py index c617df5..3fef535 100644 --- a/src/nnnotes/formats/cri.py +++ b/src/nnnotes/formats/cri.py @@ -219,7 +219,8 @@ def decode( """Decode every stream of a cue sheet. Returns {streamName: file of its first stream}. flac (default) keeps the decoded PCM bit-exact (`flac_level`: ffmpeg's compression level; any level decodes to - the same samples); ogg re-encodes (lossy). `workers`: streams decoded at once (default default_workers()). + the same samples); ogg re-encodes (lossy; bit-exact flags, so the same samples give the same file). `workers`: + streams decoded at once (default default_workers()). `also` = (extension, ffmpeg codec options): every stream is encoded a second time from the same decoded samples into `` next to its file (encode_args: no metadata, bit-exact flags), e.g. the web format of a BGM, so that it need not be transcoded from the first file later; the manifests name the first file only. @@ -360,14 +361,15 @@ def one(item) -> Path: if fmt == "wav": dst = out_dir / wav.name shutil.move(str(wav), dst) - else: - dst = out_dir / f"{stem}.{fmt}" - codec = { - "flac": ["-c:a", "flac", "-compression_level", str(int(flac_level))], - "ogg": ["-c:a", "libvorbis", "-q:a", "5"], - }[fmt] + elif fmt == "flac": + dst = out_dir / f"{stem}.flac" + codec = ["-c:a", "flac", "-compression_level", str(int(flac_level))] subprocess.run([encoder, "-y", "-loglevel", "error", "-i", str(wav), *codec, str(dst)], check=True) wav.unlink() + else: # ogg: bit-exact, so that the same samples give the same file + dst = out_dir / f"{stem}.ogg" + subprocess.run([encoder, *encode_args(wav, dst, ["-c:a", "libvorbis", "-q:a", "5"])], check=True) + wav.unlink() return dst with ThreadPoolExecutor(n) as ex: diff --git a/src/nnnotes/formats/shader.py b/src/nnnotes/formats/shader.py index aac32f8..b91bb10 100644 --- a/src/nnnotes/formats/shader.py +++ b/src/nnnotes/formats/shader.py @@ -61,7 +61,7 @@ STAGES = ("progVertex", "progFragment", "progGeometry", "progHull", "progDomain") -def _safe(name: str) -> str: +def safe_name(name: str) -> str: return re.sub(r"[^0-9A-Za-z._-]", "_", name) @@ -157,7 +157,7 @@ def _parsed_summary(tt: dict) -> dict: def _render(o, tt: dict, name: str) -> tuple[str, list[dict], list[bytes]]: """(parsed summary JSON text, variant records, variant program bytes) of one Shader object.""" - safe = _safe(name) + safe = safe_name(name) text = dumps(_parsed_summary(tt), indent=1, ensure_ascii=False, default=str) recs, codes = [], [] for plat, si, pi, stage, n, t, kws, code in variants(tt, platform_blobs(o.read())): @@ -217,7 +217,7 @@ def dump_objects(shaders, out_dir: Path, source: str, index: list | None = None) if name in done: continue done.add(name) - safe = _safe(name) + safe = safe_name(name) (out_dir / f"{safe}.json").write_text(text, encoding="utf-8", newline="\n") # as write_json writes it rec = {"name": name, "source": source, "parsed": f"{safe}.json", "variants": []} for v, code in zip(recs, codes): diff --git a/src/nnnotes/pipeline/cristages.py b/src/nnnotes/pipeline/cristages.py index 87f314d..989d285 100644 --- a/src/nnnotes/pipeline/cristages.py +++ b/src/nnnotes/pipeline/cristages.py @@ -10,8 +10,10 @@ itself never enters a key, a record or a log. Atoms: `hca.decode` (vgmstream) and `flac.encode` (ffmpeg), each named by the version the tool reports and the sha256 of its executable. Artifacts "cri.audio:#": one FLAC per vgmstream stream, `cues.json` and `streams.json`, the bytes - cri.decode writes for the sheet. Parameter `flac.level` (ffmpeg's FLAC compression level; every level - decodes to the same samples). + cri.decode writes for the sheet. Parameters `format`: flac (the default), ogg (Vorbis, encoded with + ffmpeg's bit-exact flags) or wav (the decoded samples as vgmstream writes them); `flac.level` (ffmpeg's + FLAC compression level; every level decodes to the same samples). The parameters of the default format + are `{"flac": {"level"}}`, of the others `{"format"}`. cri.movie subject: the sha256 of a USM: a raw file of the catalog (fact "raw", magic `CRID`) or held in a bundle (the `cri.usm` artifacts of unity.export: a movie asset's CriSerializedBytesAssetImpl bytes), so a movie stored in several places is converted once. Inputs "usm", "boot" (the USM masks and the HCA cipher use @@ -308,22 +310,42 @@ class AudioStage(_CriStage): version = 1 after = (EXPORT,) CLASS = "ACB" - PARAMS = {"flac": {"level": FLAC_LEVEL}} + PARAMS = {"format": "flac", "flac": {"level": FLAC_LEVEL}} + FORMATS = ("flac", "ogg", "wav") @property def ATOMS(self) -> dict: - return { - "hca.decode": tool_id("vgmstream", _tool("vgmstream")), - "flac.encode": tool_id("ffmpeg", _tool("ffmpeg")), - } + return self.atoms("flac") + + @staticmethod + def atoms(fmt: str) -> dict: + """The atoms of a task that writes `fmt`: the decoder, and the encoder of a format other than wav.""" + out = {"hca.decode": tool_id("vgmstream", _tool("vgmstream"))} + if fmt != "wav": + out[f"{fmt}.encode"] = tool_id("ffmpeg", _tool("ffmpeg")) + return out + + def _atoms(self, fmt: str) -> dict: + return self.ATOMS if fmt == "flac" else self.atoms(fmt) def normalize(self, params: dict | None) -> dict: p = dict(params or {}) unknown = sorted(set(p) - set(self.PARAMS)) if unknown: raise ValueError(f"stage {self.name}: unknown parameters {', '.join(unknown)}") + fmt = p.get("format", "flac") + if fmt not in self.FORMATS: + raise ValueError(f"stage {self.name}: format {fmt!r}: expected {', '.join(self.FORMATS)}") + if fmt != "flac": + if "flac" in p: + raise ValueError(f"stage {self.name}: flac parameters with format {fmt}") + return {"format": fmt} return {"flac": _flac_params(p.get("flac"))} + @staticmethod + def format(params: dict) -> str: + return params.get("format", "flac") + def index(self, env) -> dict: """{content id: {acb, awb, depends, names}} of the raw ACBs and the ACB artifacts of the unity.export results.""" @@ -371,7 +393,7 @@ def inputs(self, subject: str, env) -> list[Input]: return [e["acb"], *([e["awb"]] if e["awb"] is not None else []), _boot(env)] def uses(self, subject: str, env) -> dict: - return self.ATOMS + return self._atoms(self.format(env.params)) def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: """From the ACB's size (a whole catalog: about 2 CPU seconds per MB, the tools' processes included).""" @@ -381,8 +403,9 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: def run(self, task, store) -> Output: from ..formats import cri - vgm, ff = _tool("vgmstream"), _tool("ffmpeg") - self._check_atoms(task, {"hca.decode": tool_id("vgmstream", vgm), "flac.encode": tool_id("ffmpeg", ff)}) + fmt = self.format(task.params) + self._check_atoms(task, self._atoms(fmt)) + vgm, ff = _tool("vgmstream"), _tool("ffmpeg") if fmt != "wav" else None if any(i.role == "awb" for i in task.inputs): return self._item( task, @@ -392,14 +415,14 @@ def run(self, task, store) -> Output: ) acb = store.input_bytes(task.input("acb")) key = hca_key(store, task.input("boot")) - level = int(task.params["flac"]["level"]) + level = int(task.params["flac"]["level"]) if fmt == "flac" else FLAC_LEVEL with tempfile.TemporaryDirectory(prefix="nnnotes-cri-") as tmp: out = Path(tmp) d = cri.decode_acb_bytes( acb, None, key, - "flac", + fmt, out, name=ACB_NAME, flac_level=level, @@ -413,9 +436,9 @@ def run(self, task, store) -> Output: store, s["file"], (out / s["file"]).read_bytes(), - "flac", + fmt, "audio.stream", - "flac", + fmt, {k: v for k, v in s.items() if k != "file"}, ) for s in d.streams diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 36cb2cf..c0de70e 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -436,7 +436,6 @@ def test_flac_level_reaches_the_decode(data, tmp_path, capsys, monkeypatch): from types import SimpleNamespace from nnnotes.content import live, story - from nnnotes.formats import cri from nnnotes.source.session import Session from nnnotes.unity import player @@ -444,11 +443,10 @@ def test_flac_level_reaches_the_decode(data, tmp_path, capsys, monkeypatch): monkeypatch.setattr(Session, "catalog", lambda self: SimpleNamespace(has=lambda key: True)) monkeypatch.setattr(Session, "apk", lambda self, region=None: None) monkeypatch.setattr(player, "PlayerData", lambda apk: None) - monkeypatch.setattr(cri, "decode", lambda *a, **kw: seen.append(kw.get("flac_level")) or []) monkeypatch.setattr(story, "build", lambda *a, **kw: seen.append(kw["audio_options"].get("flac_level")) or {}) monkeypatch.setattr(live, "resolve_band", lambda *a, **kw: None) monkeypatch.setattr(live, "build", lambda *a, **kw: seen.append(kw.get("flac_level")) or {}) - for argv in (["export", "audio", "s"], ["export", "story", "1"], ["export", "live", "7"]): + for argv in (["export", "story", "1"], ["export", "live", "7"]): for flag, level in (([], None), (["--flac-level", "5"], 5)): seen.clear() code, _, err = run(data + argv + flag + ["-o", str(tmp_path / "o")], capsys) diff --git a/tests/cli/test_subjects.py b/tests/cli/test_subjects.py new file mode 100644 index 0000000..89a4f54 --- /dev/null +++ b/tests/cli/test_subjects.py @@ -0,0 +1,247 @@ +"""The single-subject export commands run their stage through the store, as `export assets` does, and place the +artifacts of the subject's task in the output directory.""" + +import json +import zipfile + +from nnnotes import cli +from nnnotes.cli import assets +from nnnotes.core import jsonio +from nnnotes.formats import shader +from nnnotes.pipeline import contract +from nnnotes.pipeline.stages import Output +from nnnotes.source.catalog import Catalog +from tests.cli.test_assets import FakeExport, bundle_bytes, cdoc, obj +from tests.helpers import synth + +M1 = "Character/Live2D/g1/m1/model/m1" +PREFAB = "Assets/AddressableResources/Character/Live2D/g1/m1/model/m1.prefab" +MODEL_BUNDLE = f"model_m1_{'1' * 32}.bundle" +TEX_BUNDLE = f"tex_{'2' * 32}.bundle" +REMOTE = [ + (M1, PREFAB, [1, 2, 3]), + (MODEL_BUNDLE, synth.remote(MODEL_BUNDLE), []), + (TEX_BUNDLE, synth.remote(TEX_BUNDLE), []), + ("monoscripts.bundle", synth.local("monoscripts.bundle"), []), +] +APK = [("monoscripts.bundle", synth.local("monoscripts.bundle"), [])] + + +def run(argv, capsys): + code = 0 + try: + cli.main(argv) + except SystemExit as e: + code = e.code if isinstance(e.code, int) else 1 + out, err = capsys.readouterr() + return code, out, err + + +def settings(tmp_path) -> list[str]: + """Global options of a catalog whose remote bundles are in the cache and an APK with a local bundle.""" + catalog = tmp_path / "catalog_main_ja.bin" + catalog.write_bytes(synth.CatalogWriter().build(REMOTE)) + bundles = tmp_path / "cache" / "bundles" + bundles.mkdir(parents=True) + for name in (MODEL_BUNDLE, TEX_BUNDLE): + (bundles / name).write_bytes(b"UnityFS " + name.encode()) + apk = tmp_path / "base.apk" + with zipfile.ZipFile(apk, "w") as z: + z.writestr("assets/aa/catalog.bin", synth.CatalogWriter().build(APK)) + z.writestr("assets/aa/Android/monoscripts.bundle", b"UnityFS scripts") + return [ + "--catalog", + str(catalog), + "--cache", + str(tmp_path / "cache"), + "--region", + "zz", + "--language", + "ja", + "--apk", + str(apk), + ] + + +def fake_extract_model(calls): + def extract_model(cat, key, out_dir): + data = b"|".join(p.read_bytes() for p in cat.fetch_key(key)) + calls.append(key) + (out_dir / "textures").mkdir() + (out_dir / "m1.moc3").write_bytes(b"MOC3" + data) + (out_dir / "textures" / "texture_00.png").write_bytes(b"\x89PNG page") + (out_dir / "m1.model3.json").write_text('{"Version": 3}\n', encoding="utf-8") + return {"name": "m1", "model3": "m1.model3.json", "textures": ["textures/texture_00.png"]} + + return extract_model + + +def test_export_live2d_places_the_files_of_the_model_task(tmp_path, capsys, monkeypatch): + from nnnotes.unity import live2d + + calls = [] + monkeypatch.setattr(live2d, "extract_model", fake_extract_model(calls)) + base = settings(tmp_path) + out = tmp_path / "out" + code, stdout, err = run([*base, "--json", "export", "live2d", "m1", "-o", str(out)], capsys) + assert code == 0, err + assert sorted(p.relative_to(out).as_posix() for p in out.rglob("*") if p.is_file()) == [ + "m1.moc3", + "m1.model3.json", + "textures/texture_00.png", + ] + # the bundles of the model's closure, in the order the catalog resolves them (as the extractor reads them) + cat = Catalog((tmp_path / "catalog_main_ja.bin").read_bytes(), tmp_path / "cache", apk=tmp_path / "base.apk") + data = {MODEL_BUNDLE: b"UnityFS " + MODEL_BUNDLE.encode(), TEX_BUNDLE: b"UnityFS " + TEX_BUNDLE.encode()} + order = [data.get(b.name, b"UnityFS scripts") for b in cat.resolve(M1)] + assert len(order) == 3 and (out / "m1.moc3").read_bytes() == b"MOC3" + b"|".join(order) + assert json.loads(stdout) == {"model3": "m1.model3.json", "name": "m1", "textures": ["textures/texture_00.png"]} + assert calls == [M1] + # the second export of the same model is a hit of the store: the extractor does not run again + code, _, err = run([*base, "export", "live2d", M1, "-o", str(tmp_path / "again")], capsys) + assert code == 0, err + assert calls == [M1] and (tmp_path / "again" / "m1.moc3").read_bytes() == (out / "m1.moc3").read_bytes() + + +ACB = b"@UTF" + b"cue sheet bytes" +RAW = f"cri_assets_cri/sound/s_{'3' * 32}" + + +def audio_settings(tmp_path) -> list[str]: + """Global options of a catalog whose key Cri/Sound/s loads a raw ACB in the cache, and an APK with boot data.""" + catalog = tmp_path / "catalog_main_ja.bin" + entries = [("Cri/Sound/s", "Assets/Cri/Sound/s.acb.bytes", [1]), (RAW, synth.remote(RAW), [])] + catalog.write_bytes(synth.CatalogWriter().build(entries)) + raw = tmp_path / "cache" / "raw" / "asset" / "Android" / RAW + raw.parent.mkdir(parents=True) + raw.write_bytes(ACB) + apk = tmp_path / "base.apk" + with zipfile.ZipFile(apk, "w") as z: + z.writestr("assets/aa/catalog.bin", synth.CatalogWriter().build(APK)) + z.writestr("assets/bin/Data/data.unity3d", b"UnityFS boot data") + return [ + "--catalog", + str(catalog), + "--cache", + str(tmp_path / "cache"), + "--region", + "zz", + "--language", + "ja", + "--apk", + str(apk), + ] + + +def test_export_audio_decodes_the_cue_sheet_task_in_its_format(tmp_path, capsys): + from tests.pipeline.test_cristages import fake_tools + + base = audio_settings(tmp_path) + with fake_tools() as fake: + code, stdout, err = run([*base, "--json", "export", "audio", "s", "-o", str(tmp_path / "flac")], capsys) + assert code == 0, err + assert json.loads(stdout) == {"out": str(tmp_path / "flac"), "cues": 3} + files = sorted(p.name for p in (tmp_path / "flac").iterdir()) + assert files == ["a.flac", "a_3.flac", "b.flac", "c.flac", "c_5.flac", "cues.json", "streams.json"] + level = ["-compression_level", "8"] + assert all(b"|".join(x.encode() for x in level) in (tmp_path / "flac" / f).read_bytes() for f in files[:5]) + calls = len(fake.calls) + # the same cue sheet again: a hit of the store (no tool runs); another format or level: another task + code, _, err = run([*base, "export", "audio", "s", "-o", str(tmp_path / "again")], capsys) + assert code == 0 and len(fake.calls) == calls, err + code, _, err = run([*base, "export", "audio", "s", "--flac-level", "0", "-o", str(tmp_path / "l0")], capsys) + assert code == 0, err + assert b"-compression_level|0" in (tmp_path / "l0" / "a.flac").read_bytes() + code, _, err = run([*base, "export", "audio", "s", "--format", "wav", "-o", str(tmp_path / "wav")], capsys) + assert code == 0, err + assert sorted(p.suffix for p in (tmp_path / "wav").iterdir()) == [".json"] * 2 + [".wav"] * 5 + + +# ---------------------------------------------------------------- shaders: the Shader artifacts of unity.export +class FakeShaders(FakeExport): + """unity.export of the synthetic bundles: each Shader object as the artifacts the real converter writes (the + parsed form `json`, one program, the variant `index`).""" + + def run(self, task, store): + doc = contract.loads(store.input_bytes(task.input("census"))) + assert task.params["classes"] == ["Shader"] + arts, items = [], [] + for f in doc["files"]: + for o in f["objects"]: + if o["class"] != "Shader": + continue + oid = contract.object_id(f["name"], o["pathId"]) + safe, role = shader.safe_name(o["name"]), "program:gles3/s0p0_vertex_0" + rec = { + "file": f"{safe}/gles3/s0p0_vertex_0.glsl", + "platform": "gles3", + "subShader": 0, + "pass": 0, + "stage": "vertex", + "type": "kShaderGpuProgramGLES3", + "keywords": [f["name"]], + } + parts = [ + ("json", json.dumps({"name": o["name"], "file": f["name"]}).encode(), "json"), + (role, f"// {o['name']} of {f['name']}".encode(), "glsl"), + ("index", contract.encode({"name": o["name"], "variants": [{**rec, "role": role}]}), "json"), + ] + ids = [] + for r, data, ext in parts: + ids.append(contract.artifact_id(oid, r)) + content = store.add(data, ext) + arts.append(contract.artifact(ids[-1], content, contract.provenance(task), {"kind": "shader"})) + items.append(contract.item(oid, "exported", artifacts=ids, cls="Shader")) + return Output(arts, items) + + +REAL = {s.name: s for s in assets.STAGE_SOURCES} +SHADER_SOURCES = ( + REAL["catalog.index"], + assets.StageSource("unity.census", "tests.cli.test_assets:FakeCensus"), + REAL["link.scripts"], + assets.StageSource("unity.export", "tests.cli.test_subjects:FakeShaders", output=True, root="_bundles/{subject}"), +) +FX = {n: f"fx_{n}_{n * 32}.bundle" for n in ("a", "b")} + + +def test_export_shader_writes_the_first_shader_of_each_name_from_the_export_results(tmp_path, capsys, monkeypatch): + monkeypatch.setattr(assets, "STAGE_SOURCES", SHADER_SOURCES) + docs = { + "a": cdoc("CAB-a", [obj(1, "AssetBundle", "ab"), obj(7, "Shader", "Game/Glow"), obj(2, "Shader", "Game/Rim")]), + "b": cdoc("CAB-b", [obj(1, "AssetBundle", "ab"), obj(3, "Shader", "Game/Glow"), obj(9, "Mesh", "m")]), + } + cache = tmp_path / "cache" / "bundles" + cache.mkdir(parents=True) + for n, doc in docs.items(): + (cache / FX[n]).write_bytes(bundle_bytes(doc)) + entries = [("Fx/x", "Assets/Game/Fx/x.mat", [1, 2]), (FX["a"], synth.remote(FX["a"]), [])] + entries.append((FX["b"], synth.remote(FX["b"]), [])) + catalog = tmp_path / "catalog_main_ja.bin" + catalog.write_bytes(synth.CatalogWriter().build(entries)) + base = ["--catalog", str(catalog), "--cache", str(tmp_path / "cache"), "--region", "zz", "--language", "ja"] + out = tmp_path / "out" + code, stdout, err = run([*base, "--json", "export", "shader", "--key", "Fx/x", "-o", str(out)], capsys) + assert code == 0, err + order = [b.name for b in Catalog(catalog.read_bytes(), tmp_path / "cache").resolve("Fx/x")] + assert order == [FX["b"], FX["a"]] # the closure as the catalog loads it: b's Glow comes first + records = json.loads((out / "shaders.json").read_text(encoding="utf-8")) + assert [(r["name"], r["source"], r["parsed"]) for r in records] == [ + ("Game/Glow", FX["b"], "Game_Glow.json"), + ("Game/Rim", FX["a"], "Game_Rim.json"), + ] + assert records[0]["variants"] == [ + { + "file": "Game_Glow/gles3/s0p0_vertex_0.glsl", + "platform": "gles3", + "subShader": 0, + "pass": 0, + "stage": "vertex", + "type": "kShaderGpuProgramGLES3", + "keywords": ["CAB-b"], + } + ] + assert json.loads((out / "Game_Glow.json").read_bytes()) == {"name": "Game/Glow", "file": "CAB-b"} + assert (out / "Game_Rim" / "gles3" / "s0p0_vertex_0.glsl").read_bytes() == b"// Game/Rim of CAB-a" + assert json.loads(stdout) == {"count": 2, "names": ["Game/Glow", "Game/Rim"], "variants": 2} + assert (out / "shaders.json").read_text(encoding="utf-8") == jsonio.dumps(records, indent=1, ensure_ascii=False) diff --git a/tests/pipeline/test_cristages.py b/tests/pipeline/test_cristages.py index 89d9002..5e336cb 100644 --- a/tests/pipeline/test_cristages.py +++ b/tests/pipeline/test_cristages.py @@ -307,9 +307,22 @@ def test_audio_two_runs_and_levels(tmp_path, tools): assert ta.key == tb.key and contract.encode(da) == contract.encode(db) t0, d0 = run(AudioStage(), ea, contract.sha256(ACB), {"flac": {"level": 0}}) assert t0.key != ta.key and load(a, d0, f"{t0.id}#a.flac") != load(a, da, f"{ta.id}#a.flac") - for bad in ({"flac": {"level": 13}}, {"flac": {"speed": 1}}, {"format": "ogg"}): + for bad in ({"flac": {"level": 13}}, {"flac": {"speed": 1}}, {"format": "mp3"}, {"format": "ogg", "flac": {}}): with pytest.raises(ValueError): AudioStage().normalize(bad) + assert AudioStage().normalize({"format": "flac"}) == AudioStage().normalize(None) == {"flac": {"level": 8}} + + +@pytest.mark.parametrize("fmt, atoms", [("ogg", {"hca.decode", "ogg.encode"}), ("wav", {"hca.decode"})]) +def test_audio_formats_write_their_files_with_their_atoms(tmp_path, tools, fmt, atoms): + store = Store(tmp_path / "s") + env = planning(store, tmp_path, {"s/a": ACB}) + t, doc = run(AudioStage(), env, contract.sha256(ACB), {"format": fmt}) + assert t.params == {"format": fmt} and set(t.atoms) == atoms + streams = [a for a in doc["artifacts"] if a["semantics"]["kind"] == "audio.stream"] + assert streams and all(a["content"]["ext"] == fmt and a["semantics"]["format"] == fmt for a in streams) + flac, _ = run(AudioStage(), env, contract.sha256(ACB)) + assert flac.key != t.key def test_an_acb_with_an_external_awb_is_unsupported(tmp_path, tools): diff --git a/tests/unity/test_speed_exact.py b/tests/unity/test_speed_exact.py index 24fa7c5..72cbd93 100644 --- a/tests/unity/test_speed_exact.py +++ b/tests/unity/test_speed_exact.py @@ -424,7 +424,7 @@ def ref_dump_objects(shaders, out_dir: Path, source: str, index: list) -> list: if name in done: continue done.add(name) - safe = shader._safe(name) + safe = shader.safe_name(name) jsonio.write_json(out_dir / f"{safe}.json", shader._parsed_summary(tt), default=str) rec = {"name": name, "source": source, "parsed": f"{safe}.json", "variants": []} for plat, si, pi, stage, n, t, kws, code in shader.variants(tt, shader.platform_blobs(o.read())): @@ -526,8 +526,12 @@ def meta(i): wav = work / f"{stem}.wav" cri.subprocess.run([vgm, "-i", "-s", str(i), "-o", str(wav), str(acb)], check=True, capture_output=True) dst = out_dir / f"{stem}.{fmt}" - codec = {"flac": ["-c:a", "flac", "-compression_level", str(level)], "ogg": ["-c:a", "libvorbis", "-q:a", "5"]} - cri.subprocess.run([ffmpeg, "-y", "-loglevel", "error", "-i", str(wav), *codec[fmt], str(dst)], check=True) + if fmt == "flac": + codec = ["-c:a", "flac", "-compression_level", str(level)] + cri.subprocess.run([ffmpeg, "-y", "-loglevel", "error", "-i", str(wav), *codec, str(dst)], check=True) + else: # ogg, bit-exact + vorbis = ["-c:a", "libvorbis", "-q:a", "5"] + cri.subprocess.run([ffmpeg, *encoders.encode_args(wav, dst, vorbis)], check=True) wav.unlink() entry = { "file": dst.name,