From 823e0e93a2690c97fd9d954dd24c829aabb3bb28 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Wed, 14 Oct 2026 17:51:56 +0800 Subject: [PATCH] refactor(cli): run export episode through the store The adv.episode stage exports one episode (a MasterAdv id a command asks for) into the store: adv.extract runs through a task catalog that holds the closures of the episode's script and shards and of the ADV player settings, and answers which resource keys exist from the catalog's keys under the prefixes of the episode's resources. A key outside those prefixes leaves the task unchanged; the master tables the extractor reads are inputs. TaskCatalog serves several keys: their closures by stable name, the names resolve lists, the internal ids extractors read and further known keys. The shared stage helpers (master table inputs, closure context, the master directory of a task) are public in modelstages. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 8 +- docs/commands.md | 2 + docs/configuration.md | 2 +- docs/stages.md | 1 + src/nnnotes/cli/assets.py | 3 +- src/nnnotes/cli/export.py | 17 ++- src/nnnotes/pipeline/advstages.py | 123 ++++++++++++++++++++ src/nnnotes/pipeline/modelstages.py | 171 +++++++++++++++++----------- tests/cli/test_cli.py | 4 +- tests/cli/test_subjects.py | 102 +++++++++++++++++ tests/pipeline/test_modelstages.py | 27 ++++- 11 files changed, 378 insertions(+), 82 deletions(-) create mode 100644 src/nnnotes/pipeline/advstages.py diff --git a/docs/architecture.md b/docs/architecture.md index cb5862f..3c10ad5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -36,10 +36,10 @@ decrypted bundles in the cache directory. ## Two ways to export -- **One subject.** `export live2d`, `export spot`, `export room`, `export shader` and `export audio` run the stages - of their subject through the store, over the files the subject's catalog keys load, and copy the subject's files - from the store into the output directory; a second export of the same subject is a hit of the store. `export - episode`, `export story`, `export live`, `export ui`, `site build` and `music-data` read what one subject needs +- **One subject.** `export episode`, `export live2d`, `export spot`, `export room`, `export shader` and `export + audio` run the stages of their subject through the store, over the files the subject's catalog keys load, and copy + the subject's files from the store into the output directory; a second export of the same subject is a hit of the + store. `export story`, `export live`, `export ui`, `site build` and `music-data` read what one subject needs through the Session and write their output directory directly ([Export stories and lives](export.md)). - **Every asset.** `export assets` splits the work into stages ([stages](stages.md)). A task is one stage applied to one subject (a bundle, a content id, a view); its key covers exactly what its output depends on: the stage and diff --git a/docs/commands.md b/docs/commands.md index cb5f5c7..68499b8 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -153,6 +153,8 @@ takes the player settings' default transition, which is read from `[paths] apk`) `chatwindow` and `chaticon` from the `MasterAdvChat` row of the chat rows' chat id; `video` from the episode's video row of a Movie / Clip row's video id. +The file is the result of the store's `adv.episode` task of the episode ([stages](stages.md)). + ## export story ``` diff --git a/docs/configuration.md b/docs/configuration.md index 0974ae4..0b0d6d4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -164,7 +164,7 @@ story text); it is not a setting. | `master download` | `[catalog] region` and that region's `cdn`; `--latest` also that region's `api` and the client version | | `master version` | `[catalog] region` and that region's `api`; the client version: `[client] version` or `[paths] apk` | | `servers` | `[bootstrap] api`; the client version: `[client] version` or `[paths] apk` | -| `export episode` | catalog, master data | +| `export episode` | the store, catalog, master data | | `export story` | catalog, `[catalog] language`, master data, `[paths] apk`, `ffmpeg` (audio and videos), `vgmstream` (not with `--no-audio`) | | `export live2d` | the store (`[paths] store` or `[paths] cache`), catalog, `[paths] apk` | | `export spot` | the store, catalog, master data, `[paths] apk` | diff --git a/docs/stages.md b/docs/stages.md index 2b14266..d0febb3 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -28,6 +28,7 @@ gzip input and Japanese remote asset placeholders). | `live2d.model` | a Live2D model key (`Character/Live2D///model/`) whose bundle is selected | `bundle:`: the bundles of the key's dependency closure in the order the extractor loads them (the order is part of the key); context `internalId` (the key's location); atom `live2d.extract_model` (the extractor of `nnnotes export live2d`, run unchanged) | `live2d.model:#`: every file `nnnotes export live2d` writes for the model, `` relative to the model's directory; placed under `/` | | `room.scene` | a background key a command asks for (`export room`, `export spot`) | `bundle:`: the bundles of the key's closure in load order, as `live2d.model`; atom `room.extract_room` (the glTF writer of `nnnotes export room`) | `room.scene:#room.glb`: the scene (facts: `meshCount`, `inactiveMeshes`); `#room.json`: its report, naming the scene `room.glb` (`export room` names its own file) | | `spot.scene` | a Spot id (`MasterHomeSpot`) a command asks for (`export spot`), with master data | `master:`: `MasterHomeSpot`, `MasterStoryHomeSpotTapTalkEpisode`, `MasterText`; `bundle:`: the bundles of the Spot's situation key in load order; context `resources` (the bundle names of the closures of the situation and background keys, which `spot.json` lists); atom `spot.extract` | `spot.scene:#spot.json` (facts: `tapTargets`, `skeletons`) and `#spine/`: the files `nnnotes export spot` writes for the Spot besides its room and shaders | +| `adv.episode` | a MasterAdv id a command asks for (`export episode`), with master data | `master:
`: `MasterAdv`, `MasterAdvChat`, `MasterText`; `bundle:`: the bundles of the closures of the episode's keys the catalog has (the episode asset, its `-Text`, `-Sound`, `-SoundCueSheet` and `-Video` shards, the ADV player settings); `keys`: the catalog's keys under the prefixes of the episode's resources (whether the catalog has a resource is part of the episode); context `closures` (each key's bundles in load order); atom `adv.extract` | `adv.episode:#episode.json`: the file `nnnotes export episode` writes (facts: `commands`, `resources`) | | `spine.skeleton` | a SkeletonDataAsset of a selected bundle, by its stable address `:` (in a bundle of several serialized files the file's tag follows the bundle name, as in `link.addresses`) | `bundle:`: the bundles holding the serialized files the asset's file reaches through its externals; context `object` (the asset's object id); atom `spot.write_skeleton` (the Spine writer of `nnnotes export spot`) | `spine.skeleton:#`: the skeleton (`.json` or `.skel`), its atlases (`.atlas`) and their page images, the bytes `nnnotes export spot` writes into `spine/`; one item for the asset | | `link.addresses` | global | catalog index, censuses | key -> objects, serialized file -> bundle, bundle -> keys | | `link.artifacts` | global | the above, the results | object id -> artifact ids | diff --git a/src/nnnotes/cli/assets.py b/src/nnnotes/cli/assets.py index bf17d46..300ab0e 100644 --- a/src/nnnotes/cli/assets.py +++ b/src/nnnotes/cli/assets.py @@ -76,7 +76,7 @@ REPORTS = "_reports" SELECT_KINDS = ("group", "key", "bundle") RAW_STAGES = ("cri.audio", "cri.movie") -MASTER_STAGES = ("spot.scene",) # stages that read master data (besides the views) +MASTER_STAGES = ("spot.scene", "adv.episode") # stages that read master data (besides the views) EXPORT_STAGE = "unity.export" CATALOG_SUBJECT = "main" EXIT_FAILED, EXIT_USAGE, EXIT_ABORTED, EXIT_INCOMPATIBLE = 1, 2, 3, 4 @@ -111,6 +111,7 @@ def matches(self, stage: str) -> bool: StageSource("spine.skeleton", "nnnotes.pipeline.modelstages:SpineStage", output=True), StageSource("room.scene", "nnnotes.pipeline.modelstages:RoomStage", output=True), StageSource("spot.scene", "nnnotes.pipeline.modelstages:SpotStage", output=True), + StageSource("adv.episode", "nnnotes.pipeline.advstages:EpisodeStage", output=True), StageSource("link.artifacts", "nnnotes.pipeline.link:ArtifactsStage"), StageSource("view.", "nnnotes.pipeline.views:stages", output=True), StageSource("view.voices", "nnnotes.pipeline.voices:VoiceStage", output=True), diff --git a/src/nnnotes/cli/export.py b/src/nnnotes/cli/export.py index f566c27..0156ec9 100644 --- a/src/nnnotes/cli/export.py +++ b/src/nnnotes/cli/export.py @@ -32,16 +32,23 @@ def cmd_episode(args, cfg): - from ..content import adv + from ..pipeline import contract + from ..pipeline.advstages import EPISODE, episode_keys + from ..source import master md = Session(cfg).master_dir() known_row(args, md, "MasterAdv", args.adv_id, "episode") - cat = Session(cfg).catalog() - doc = adv.to_json(adv.extract(cat, md, args.adv_id)) + (row,) = [r for r in master.table(md, "MasterAdv") if r["_id"] == args.adv_id] + run = assets.SubjectRun(args, cfg) + subject = str(args.adv_id) + run.run([EPISODE], keys=episode_keys(row["_advEpisodeAsset"]), requests={EPISODE: [subject]}) + doc = run.result(contract.task_id(EPISODE, subject)) out = Path(args.out) out.parent.mkdir(parents=True, exist_ok=True) - write_json(out, doc) - r = {"out": str(out), "commands": doc["commandCount"], "resources": len(doc["resources"])} + (art,) = doc["artifacts"] + write_file(out, run.store.read(art["content"]["sha256"])) + facts = art["semantics"]["facts"] + r = {"out": str(out), "commands": facts["commands"], "resources": facts["resources"]} report(args, r, lambda r: [f"{r['out']} ({r['commands']} commands, {r['resources']} resources)"]) diff --git a/src/nnnotes/pipeline/advstages.py b/src/nnnotes/pipeline/advstages.py new file mode 100644 index 0000000..7c7717e --- /dev/null +++ b/src/nnnotes/pipeline/advstages.py @@ -0,0 +1,123 @@ +"""The ADV stages: an episode's script and shards as `nnnotes export episode` writes them. + +adv.episode, one task per MasterAdv id a command asks for (the fact "requests"), with master data (the fact "master"): + inputs "master:
": MasterAdv, MasterAdvChat and MasterText (the tables adv.extract reads); + "bundle:": the bundles of the closures of the episode's keys the catalog has (the + episode asset Adv/Episode//, its -Text, -Sound, -SoundCueSheet and -Video shards, and + the ADV player settings, which name the default transition); "keys": the keys of the catalog that + start with a prefix of the episode's resources (key_prefixes: whether the catalog has a resource is + part of episode.json) + context {"closures": {key: [stable names of its bundles in load order]}} + artifacts "adv.episode:#episode.json": the episode as `nnnotes export episode` writes it (facts: commands, + resources) + +The task runs adv.extract through TaskCatalog, its one atom ("adv.extract"); this module imports the extractor (and +UnityPy) only when it plans or runs a task. +""" + +from __future__ import annotations + +import json +import tempfile +from pathlib import Path + +from . import contract +from .atoms import impl_id +from .contract import Cost, Input +from .modelstages import LIBRARIES, ClosureStage, TaskCatalog, check_atoms, file_artifact, requested, write_master +from .stages import Output + +EPISODE = "adv.episode" +EPISODE_TABLES = ("MasterAdv", "MasterAdvChat", "MasterText") +SHARDS = ("", "-Text", "-Sound", "-SoundCueSheet", "-Video") + + +def key_prefixes() -> list[str]: + """The prefixes of the addresses adv.closure asks the catalog about (adv.RESOURCE_PREFIX and the resources + derived from other fields), sorted.""" + from ..content import adv + + talk = adv.TALK_WINDOW_PREFIX + return sorted( + { + *(prefix for _kind, prefix in adv.RESOURCE_PREFIX.values()), + adv.TRANSITION_PREFIX, + talk, + adv.EMBEDDED_PREFIX + talk, + adv.CHAT_WINDOW_PREFIX, + adv.CHAT_ICON_PREFIX, + adv.VIDEO_ADDRESS.format(""), + } + ) + + +def episode_keys(asset: str) -> list[str]: + """The keys adv.extract reads for the episode asset `asset` (its script and shards, the player settings).""" + from ..content import adv + + base = f"Adv/Episode/{asset}/{asset}" + return [base + s for s in SHARDS] + [adv.PLAYER_SETTINGS_KEY] + + +def _rows(path: Path) -> list[dict]: + return json.loads(path.read_bytes())["_allData"] + + +def _stored(store, role: str, data: bytes) -> Input: + return Input(role, store.put(data), len(data), None, ({"kind": "store"},)) + + +class EpisodeStage(ClosureStage): + """adv.episode: a MasterAdv id -> episode.json of `nnnotes export episode` (module documentation). Subjects: + the ids a command asks for (the fact "requests"), when the fact "master" names the master data.""" + + name = EPISODE + version = 1 + ATOMS = {"adv.extract": impl_id(LIBRARIES, 1)} + + def subjects(self, env) -> list[str]: + if env.facts.get("master") is None: + return [] + return requested(env, self.name) + + def row(self, subject: str, env) -> dict: + p = Path(env.fact("master")) / f"{EPISODE_TABLES[0]}.json" + for r in _rows(p): + if r.get("_id") == int(subject): + return r + raise ValueError(f"{self.name}: no MasterAdv row {subject}") + + def keys(self, subject: str, env) -> list[str]: + """The keys of the episode the catalog has.""" + c = self.closures(env) + return [k for k in episode_keys(self.row(subject, env)["_advEpisodeAsset"]) if k in c.by_key] + + def inputs(self, subject: str, env) -> list[Input]: + what = f"{self.name} {subject}" + bundles, _ = self.closure_context(self.keys(subject, env), env, what) + prefixes = tuple(key_prefixes()) + known = [k for k in self.closures(env).keys() if k.startswith(prefixes)] + keys = _stored(env.store, "keys", contract.encode({"prefixes": list(prefixes), "keys": known})) + return self.master_inputs(EPISODE_TABLES, env, what) + bundles + [keys] + + def context(self, subject: str, env) -> dict: + return self.closure_context(self.keys(subject, env), env, f"{self.name} {subject}")[1] + + def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: + size = sum(i.size for i in inputs) + return Cost(0.3 + size * 1e-7, (320 << 20) + 8 * size) + + def run(self, task, store) -> Output: + check_atoms(task, self.ATOMS) + from ..content import adv + from ..core.jsonio import write_json + + known = contract.loads(store.input_bytes(task.input("keys")))["keys"] + cat = TaskCatalog.of_closures(task, store, known=known) + with tempfile.TemporaryDirectory(prefix="nnnotes-adv-") as d: + mdir = write_master(task, store, Path(d, "master")) + doc = adv.to_json(adv.extract(cat, mdir, int(task.subject))) # pyright: ignore[reportArgumentType] + path = write_json(Path(d, "episode.json"), doc) + facts = {"commands": doc["commandCount"], "resources": len(doc["resources"])} + arts = [file_artifact(task, store, "episode.json", path.read_bytes(), "adv.episode", "json", facts)] + return Output(arts, []) diff --git a/src/nnnotes/pipeline/modelstages.py b/src/nnnotes/pipeline/modelstages.py index 5360707..fd2074c 100644 --- a/src/nnnotes/pipeline/modelstages.py +++ b/src/nnnotes/pipeline/modelstages.py @@ -46,7 +46,9 @@ from __future__ import annotations import json +import os import re +import shutil import tempfile from pathlib import Path @@ -95,7 +97,7 @@ def _ext(name: str, default: str = "bin") -> str: return ext if _EXT.fullmatch(ext) else default -def _check_atoms(task, atoms: dict) -> None: +def check_atoms(task, atoms: dict) -> None: if task.atoms != atoms: raise IncompatibleTask(f"task {task.id}: this installation runs {atoms}, the task names {task.atoms}") @@ -120,12 +122,26 @@ def _ordered(task, prefix: str = "bundle:") -> list[Input]: return sorted(ins, key=lambda i: int(i.role[len(prefix) :])) +def write_master(task, store, mdir: Path) -> Path: + """The inputs "master:
" of a task as the decoded master data directory `mdir` (
.json, each a + link to the checked file, else a copy), for the extractors that read master data from a directory.""" + mdir.mkdir(parents=True) + for i in task.inputs: + if i.role.startswith(MASTER_ROLE): + src, dst = store.open_input(i), mdir / f"{i.role[len(MASTER_ROLE) :]}.json" + try: + os.link(src, dst) + except OSError: + shutil.copyfile(src, dst) + return mdir + + def requested(env, stage: str) -> list[str]: """The subjects a command asks of `stage` (the fact "requests": {stage: [subject]}; none without it), sorted.""" return sorted(set(env.facts.get("requests", {}).get(stage, ()))) -def _artifact(task, store, name: str, data: bytes, kind: str, ext: str, facts: dict | None = None) -> dict: +def file_artifact(task, store, name: str, data: bytes, kind: str, ext: str, facts: dict | None = None) -> dict: return contract.artifact( contract.artifact_id(task.id, name), store.add(data, ext), @@ -192,69 +208,86 @@ def closure_bundles(self, key: str) -> list[str]: # the value of TaskCatalog.apk: not None, the APK's bundles of the closure are inputs of the task APK_FROM_INPUTS = "" +BUNDLE_ROLE = "bundle:" class TaskCatalog: """What the extractors ask of a Catalog (live2d.extract_model through export.Exporter, room.extract_room, - spot.extract: fetch_key, resolve, entry, apk), answered from a task: its key, the internal id of the key's - location (None: the extractor does not ask) and the bundles of the key's closure in load order, each bundle read - through the store (Store.open_input: its locators, checked against its content id). `names` ({key: [bundle - names in load order]}) gives the names `resolve` lists for the key and for keys whose bundles the extractor only - names. Another key is not in it (KeyError). `apk` is not None: the extractors need the APK's bundles, and those - of the closure are inputs of the task like the others.""" + spot.extract, adv.extract: fetch_key, resolve, has, keys, entry, apk), answered from a task: + + closures {key: the bundles of its closure in load order}, each bundle read through the store + (Store.open_input: its locators, checked against its content id) + internal_ids {key: the internal id of its location}, for the keys whose entry the extractor reads + names {key: [bundle names in load order]}: the names `resolve` lists, also for keys whose bundles the + extractor only names (else a bundle's input file name, its content id when it has none) + known further keys the catalog has (`has` and `keys` answer from them and the keys of `closures`) + + A key the task does not hold raises KeyError (`has` answers False). `apk` is not None: the extractors need the + APK's bundles, and those of the closures are inputs of the task like the others.""" def __init__( self, - key: str, - internal_id: str | None, - bundles: list[Input], + closures: dict[str, list[Input]], store, + *, + internal_ids: dict[str, str] | None = None, names: dict[str, list[str]] | None = None, + known=(), ): - self.key, self.internal_id, self.bundles, self.store = key, internal_id, list(bundles), store + self.closures = {k: list(v) for k, v in closures.items()} + self.store = store + self.internal_ids = dict(internal_ids or {}) self.names = dict(names or {}) - if key in self.names and len(self.names[key]) != len(self.bundles): - raise ValueError(f"{key}: {len(self.names[key])} bundle names for {len(self.bundles)} bundles") + for k, v in self.names.items(): + if k in self.closures and len(v) != len(self.closures[k]): + raise ValueError(f"{k}: {len(v)} bundle names for {len(self.closures[k])} bundles") + self.known = set(known) | set(self.closures) self.apk = APK_FROM_INPUTS @classmethod def from_task(cls, task, store) -> TaskCatalog: """The catalog of a live2d.model task: its subject, context internalId and inputs "bundle:".""" - return cls(task.subject, task.context["internalId"], _ordered(task), store) + return cls({task.subject: _ordered(task)}, store, internal_ids={task.subject: task.context["internalId"]}) - def _check(self, key: str) -> None: - if key != self.key: - raise KeyError(key) + @classmethod + def of_closures(cls, task, store, **kw) -> TaskCatalog: + """The catalog of a task whose context "closures" lists each key's bundles by stable name in load order + ({key: [stable names]}) and whose inputs are "bundle:" (closure_context).""" + by_stable = {i.role[len(BUNDLE_ROLE) :]: i for i in task.inputs if i.role.startswith(BUNDLE_ROLE)} + closures = {k: [by_stable[s] for s in stables] for k, stables in task.context["closures"].items()} + return cls(closures, store, **kw) def has(self, key: str) -> bool: - return key == self.key + return key in self.known def keys(self, prefix: str = "") -> list[str]: - return [self.key] if self.key.startswith(prefix) else [] + return sorted(k for k in self.known if k.startswith(prefix)) def entry(self, key: str) -> dict: - self._check(key) - if self.internal_id is None: - raise KeyError(f"{key}: the task has no internal id") - return {"primary_key": key, "internal_id": self.internal_id} + if key not in self.internal_ids: + raise KeyError(f"{key}: the task has no internal id of it") + return {"primary_key": key, "internal_id": self.internal_ids[key]} def entries_for(self, key: str) -> list[dict]: - return [self.entry(key)] if key == self.key else [] + return [self.entry(key)] if key in self.internal_ids else [] def resolve(self, key: str) -> list[Bundle]: """The closure as Bundles in load order: offset = the position, name = its name in `names`, else the input's file name (its content id when it has none); internal_id is empty and remote True (a task has a file's content, not its place in a catalog; every bundle is read from the inputs).""" - if key != self.key and key in self.names: - return [Bundle(i, "", name, True) for i, name in enumerate(self.names[key])] - self._check(key) - names = self.names.get(key) or [inp.name or inp.sha256 for inp in self.bundles] + if key in self.names: + names = self.names[key] + elif key in self.closures: + names = [inp.name or inp.sha256 for inp in self.closures[key]] + else: + raise KeyError(key) return [Bundle(i, "", name, True) for i, name in enumerate(names)] def fetch_key(self, key: str) -> list[Path]: - """The files of the closure's bundles, in load order.""" - self._check(key) - return [self.store.open_input(inp) for inp in self.bundles] + """The files of the bundles of the key's closure, in load order.""" + if key not in self.closures: + raise KeyError(key) + return [self.store.open_input(inp) for inp in self.closures[key]] # ---------------------------------------------------------------- live2d.model @@ -271,7 +304,7 @@ def _tree(root: Path) -> list[tuple[str, Path]]: return sorted((p.relative_to(root).as_posix(), p) for p in root.rglob("*") if p.is_file()) -class _ClosureStage(Stage): +class ClosureStage(Stage): """A stage whose task reads the bundles of a key's closure in load order (inputs "bundle:"), from the facts "index" (the catalog index) and "bundles" ({stable name: Input "bundle"}).""" @@ -307,8 +340,33 @@ def closure_names(self, key: str, env) -> list[str]: names = {b["stable"]: b["name"] for b in c.index["bundles"]} return [names[stable] for stable in c.closure_bundles(key)] + def closure_context(self, keys, env, what: str) -> tuple[list[Input], dict]: + """The inputs "bundle:" of the bundles of the closures of `keys` (each bundle once) and the + context entry "closures": {key: [stable names in load order]} (TaskCatalog.of_closures).""" + bundles, c = env.fact("bundles"), self.closures(env) + closures = {k: c.closure_bundles(k) for k in sorted(set(keys))} + inputs = {} + for stables in closures.values(): + for stable in stables: + if stable not in inputs: + inp = _fact_input(bundles, stable, what) + inputs[stable] = Input(BUNDLE_ROLE + stable, inp.sha256, inp.size, inp.name, inp.locators) + return [inputs[s] for s in sorted(inputs)], {"closures": closures} + + def master_inputs(self, tables, env, what: str) -> list[Input]: + """The inputs "master:
" of the master tables `tables` (the fact "master": the decoded tables).""" + mdir = Path(env.fact("master")) + out = [] + for t in tables: + p = mdir / f"{t}.json" + if not p.is_file(): + raise ValueError(f"{what}: master table {t} is not in the master data") + sha, size = env.store.identify(p) + out.append(Input(MASTER_ROLE + t, sha, size, p.name, ({"kind": "file", "path": str(p)},))) + return out + -class Live2DStage(_ClosureStage): +class Live2DStage(ClosureStage): """live2d.model: a Live2D model key -> the files of `nnnotes export live2d` (module documentation). Subjects and inputs come from the facts "index" (the catalog index), "selected" and "bundles" ({stable name: Input "bundle"}).""" @@ -334,7 +392,7 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: return Cost(0.9 + size * 2.2e-7, (420 << 20) + 10 * size) def run(self, task, store) -> Output: - _check_atoms(task, self.ATOMS) + check_atoms(task, self.ATOMS) from ..unity import live2d cat = TaskCatalog.from_task(task, store) @@ -345,7 +403,7 @@ def run(self, task, store) -> Output: for rel, p in _tree(Path(d)): kind = live2d_kind(rel) facts = {k: summary[k] for k in sorted(summary)} if kind == "live2d.model3" else {} - arts.append(_artifact(task, store, rel, p.read_bytes(), kind, _ext(rel), facts)) + arts.append(file_artifact(task, store, rel, p.read_bytes(), kind, _ext(rel), facts)) return Output(arts, []) @@ -470,7 +528,7 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: return Cost(0.27 + size * 1.9e-7, (440 << 20) + 20 * size) def run(self, task, store) -> Output: - _check_atoms(task, self.ATOMS) + check_atoms(task, self.ATOMS) from ..content import spot from ..unity import reader as unity @@ -513,7 +571,7 @@ def run(self, task, store) -> Output: # ---------------------------------------------------------------- room.scene -class RoomStage(_ClosureStage): +class RoomStage(ClosureStage): """room.scene: a background key -> the glTF scene of `nnnotes export room` and its report (module documentation). Subjects: the keys of the catalog a command asks for (the fact "requests").""" @@ -534,17 +592,17 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: return Cost(1.0 + size * 3e-7, (480 << 20) + 12 * size) def run(self, task, store) -> Output: - _check_atoms(task, self.ATOMS) + check_atoms(task, self.ATOMS) from ..content import room - cat = TaskCatalog(task.subject, None, _ordered(task), store) + cat = TaskCatalog({task.subject: _ordered(task)}, store) with tempfile.TemporaryDirectory(prefix="nnnotes-room-") as d: glb = Path(d) / "room.glb" doc = room.extract_room(cat, task.subject, glb) # pyright: ignore[reportArgumentType] facts = {"meshCount": doc["meshCount"], "inactiveMeshes": doc["inactiveMeshes"]} arts = [ - _artifact(task, store, "room.glb", glb.read_bytes(), "room.glb", "glb", facts), - _artifact(task, store, "room.json", glb.with_suffix(".json").read_bytes(), "room.report", "json"), + file_artifact(task, store, "room.glb", glb.read_bytes(), "room.glb", "glb", facts), + file_artifact(task, store, "room.json", glb.with_suffix(".json").read_bytes(), "room.report", "json"), ] return Output(arts, []) @@ -557,7 +615,7 @@ def _spot_row(rows: list[dict], spot_id: int) -> dict: raise ValueError(f"{SPOT}: no MasterHomeSpot row {spot_id}") -class SpotStage(_ClosureStage): +class SpotStage(ClosureStage): """spot.scene: a Spot id -> spot.json and the Spine files of `nnnotes export spot` (module documentation). Subjects: the ids a command asks for (the fact "requests"), when the fact "master" names the master data.""" @@ -570,23 +628,12 @@ def subjects(self, env) -> list[str]: return [] return requested(env, self.name) - def master_inputs(self, subject: str, env) -> list[Input]: - mdir = Path(env.fact("master")) - out = [] - for t in SPOT_TABLES: - p = mdir / f"{t}.json" - if not p.is_file(): - raise ValueError(f"{self.name} {subject}: master table {t} is not in the master data") - sha, size = env.store.identify(p) - out.append(Input(MASTER_ROLE + t, sha, size, p.name, ({"kind": "file", "path": str(p)},))) - return out - def row(self, subject: str, env) -> dict: p = Path(env.fact("master")) / f"{SPOT_TABLES[0]}.json" return _spot_row(json.loads(p.read_bytes())["_allData"], int(subject)) def inputs(self, subject: str, env) -> list[Input]: - masters = self.master_inputs(subject, env) + masters = self.master_inputs(SPOT_TABLES, env, f"{self.name} {subject}") key = self.row(subject, env)["_situationAssetPath"] return masters + self.closure_inputs(key, env, f"{self.name} {subject}") @@ -601,27 +648,23 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: return Cost(1.0 + size * 2e-7, (480 << 20) + 12 * size) def run(self, task, store) -> Output: - _check_atoms(task, self.ATOMS) + check_atoms(task, self.ATOMS) from ..content import spot spot_id = int(task.subject) with tempfile.TemporaryDirectory(prefix="nnnotes-spot-") as d: - mdir, out = Path(d, "master"), Path(d, "out") - mdir.mkdir() - for i in task.inputs: - if i.role.startswith(MASTER_ROLE): - (mdir / f"{i.role[len(MASTER_ROLE) :]}.json").write_bytes(store.input_bytes(i)) + mdir, out = write_master(task, store, Path(d, "master")), Path(d, "out") rows = json.loads((mdir / f"{SPOT_TABLES[0]}.json").read_bytes())["_allData"] key = _spot_row(rows, spot_id)["_situationAssetPath"] names = task.context["resources"] - cat = TaskCatalog(key, None, _ordered(task), store, names=names) + cat = TaskCatalog({key: _ordered(task)}, store, names=names) doc = spot.extract(cat, mdir, spot_id, out) # pyright: ignore[reportArgumentType] facts = {"tapTargets": len(doc["characters"]), "skeletons": len(doc["skeletons"])} arts = [] for rel, p in _tree(out): if rel == "spot.json": - arts.append(_artifact(task, store, rel, p.read_bytes(), "spot.json", "json", facts)) + arts.append(file_artifact(task, store, rel, p.read_bytes(), "spot.json", "json", facts)) else: ext = _ext(rel) - arts.append(_artifact(task, store, rel, p.read_bytes(), "spine.file", ext)) + arts.append(file_artifact(task, store, rel, p.read_bytes(), "spine.file", ext)) return Output(arts, []) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index c0de70e..ca2bb3f 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -316,7 +316,9 @@ def test_crikey_write_creates_the_directory(tmp_path, capsys, monkeypatch): # ---------------------------------------------------------------- unknown keys and ids: usage errors MASTER_TABLES = { - "MasterAdv": [{"_id": 1}], + "MasterAdv": [{"_id": 1, "_advEpisodeAsset": "e1"}], + "MasterAdvChat": [], + "MasterText": [], "MasterHomeSpot": [{"_id": 1}], "MasterLiveMusic": [{"_id": 7, "_expertID": 70, "_easyID": 0, "_vocalCharacterIDs": [1]}], "MasterLiveMusicScore": [{"_id": 70}], diff --git a/tests/cli/test_subjects.py b/tests/cli/test_subjects.py index cd7afe4..862d9ea 100644 --- a/tests/cli/test_subjects.py +++ b/tests/cli/test_subjects.py @@ -410,3 +410,105 @@ def test_export_spot_names_a_key_of_the_spot_the_catalog_lacks(tmp_path, capsys, code, out, err = run([*scene_settings(tmp_path), "export", "spot", "2", "-o", str(tmp_path / "o")], capsys) assert code == 2 and out == "", err assert "nnnotes export spot: error: not a key of the catalog: Spot/none" in err + + +# ---------------------------------------------------------------- adv.episode +EP = "Adv/Episode/e1/e1" +EP_BUNDLES = {n: f"adv_{n}_{n * 32}.bundle" for n in ("e", "t", "c")} +EP_MASTER = { + "MasterAdv": [{"_id": 1, "_advEpisodeAsset": "e1", "_titleTextId": ""}], + "MasterAdvChat": [], + "MasterText": [{"_id": "t", "_japanese": "title"}], +} + + +def episode_settings(tmp_path, extra=()) -> list[str]: + """A catalog with the episode e1 (its script, a -Text shard and a model) and the ADV player settings in the + APK; `extra`: more (key, bundle letter) entries.""" + cache = tmp_path / "cache" / "bundles" + cache.mkdir(parents=True, exist_ok=True) + for name in EP_BUNDLES.values(): + (cache / name).write_bytes(b"UnityFS " + name.encode()) + entries = [ + (EP, "Assets/Adv/e1.asset", [3]), + (EP + "-Text", "Assets/Adv/e1-Text.json", [4]), + ("Character/Live2D/g/m/model/m", "Assets/m.prefab", [5]), + ] + entries += [(EP_BUNDLES[n], synth.remote(EP_BUNDLES[n]), []) for n in ("e", "t", "c")] + entries += [(key, f"Assets/{key}.prefab", [5]) for key in extra] + catalog = tmp_path / "catalog_main_ja.bin" + catalog.write_bytes(synth.CatalogWriter().build(entries)) + md = tmp_path / "master" + md.mkdir(exist_ok=True) + for name, rows in EP_MASTER.items(): + (md / f"{name}.json").write_text(json.dumps({"_allData": rows}), encoding="utf-8") + apk = tmp_path / "base.apk" + settings = [ + ("EmbCommon/Adv/Settings/AdvPlayerSettings", "Assets/Emb/s.asset", [1]), + ("emb.bundle", synth.local("emb.bundle"), []), + ] + with zipfile.ZipFile(apk, "w") as z: + z.writestr("assets/aa/catalog.bin", synth.CatalogWriter().build(settings)) + z.writestr("assets/aa/Android/emb.bundle", b"UnityFS player settings") + return [ + "--catalog", + str(catalog), + "--cache", + str(tmp_path / "cache"), + "--region", + "zz", + "--language", + "ja", + "--master", + str(md), + "--apk", + str(apk), + ] + + +def test_export_episode_writes_the_episode_task_and_reruns_on_its_keys(tmp_path, capsys, monkeypatch): + from nnnotes.content import adv + + calls = [] + + def extract(cat, md, adv_id): + (row,) = [r for r in json.loads((md / "MasterAdv.json").read_bytes())["_allData"] if r["_id"] == adv_id] + assert row["_advEpisodeAsset"] == "e1" and (md / "MasterText.json").is_file() + calls.append(adv_id) + return { + "script": [p.read_bytes() for p in cat.fetch_key(EP)][0].decode(), + "text": cat.has(EP + "-Text") and [p.read_bytes() for p in cat.fetch_key(EP + "-Text")][0].decode(), + "sound": cat.has(EP + "-Sound"), + "settings": [p.read_bytes() for p in cat.fetch_key(adv.PLAYER_SETTINGS_KEY)][-1].decode(), + "model": cat.has("Character/Live2D/g/m/model/m"), + "stage": cat.has("Adv/Stage/s1"), + "commandCount": 2, + "resources": [{"kind": "live2d"}], + } + + monkeypatch.setattr(adv, "extract", extract) + monkeypatch.setattr(adv, "to_json", lambda x: x) + base = episode_settings(tmp_path) + out = tmp_path / "e.json" + code, stdout, err = run([*base, "--json", "export", "episode", "1", "-o", str(out)], capsys) + assert code == 0, err + doc = { + "script": "UnityFS " + EP_BUNDLES["e"], + "text": "UnityFS " + EP_BUNDLES["t"], + "sound": False, + "settings": "UnityFS player settings", + "model": True, + "stage": False, + "commandCount": 2, + "resources": [{"kind": "live2d"}], + } + assert out.read_text(encoding="utf-8") == jsonio.dumps(doc, indent=1, ensure_ascii=False) + assert json.loads(stdout) == {"out": str(out), "commands": 2, "resources": 1} and calls == [1] + # a key outside the prefixes of the episode's resources: a hit; a resource key: the episode again + base = episode_settings(tmp_path, extra=["Other/k"]) + code, _, err = run([*base, "export", "episode", "1", "-o", str(out)], capsys) + assert code == 0 and calls == [1], err + base = episode_settings(tmp_path, extra=["Other/k", "Adv/Stage/s1"]) + code, _, err = run([*base, "export", "episode", "1", "-o", str(out)], capsys) + assert code == 0 and calls == [1, 1], err + assert json.loads(out.read_bytes())["stage"] is True diff --git a/tests/pipeline/test_modelstages.py b/tests/pipeline/test_modelstages.py index 0df3759..f136c52 100644 --- a/tests/pipeline/test_modelstages.py +++ b/tests/pipeline/test_modelstages.py @@ -171,12 +171,27 @@ def test_task_catalog_answers_from_the_task(tmp_path): f(M2) # inputs are ordered by position, not by role text many = [Input(f"bundle:{i}", contract.sha256(b"x%d" % i), 2) for i in (10, 2, 1)] - assert [ - i.role - for i in TaskCatalog( - "k", "Assets/k", modelstages._ordered(Task(Live2DStage.name, 1, "k", inputs=tuple(many))), store - ).bundles - ] == ["bundle:1", "bundle:2", "bundle:10"] + ordered = modelstages._ordered(Task(Live2DStage.name, 1, "k", inputs=tuple(many))) + assert [i.role for i in TaskCatalog({"k": ordered}, store).closures["k"]] == ["bundle:1", "bundle:2", "bundle:10"] + + +def test_task_catalog_of_several_keys_answers_from_their_closures(tmp_path): + store = Store(tmp_path / "s") + data = {s: s.encode() for s in ("a", "b", "c")} + ins = {s: Input(f"bundle:{s}", store.put(d), len(d), f"{s}.bundle", ({"kind": "store"},)) for s, d in data.items()} + task = Task( + "x.y", 1, "1", inputs=tuple(ins.values()), context={"closures": {"K1": ["b", "a"], "K2": ["c", "b", "c"]}} + ) + cat = TaskCatalog.of_closures(task, store, names={"Bg": ["n1", "n2"]}, known=["Other/x"]) + assert [p.read_bytes() for p in cat.fetch_key("K1")] == [b"b", b"a"] + assert [p.read_bytes() for p in cat.fetch_key("K2")] == [b"c", b"b", b"c"] + assert [b.name for b in cat.resolve("K2")] == ["c.bundle", "b.bundle", "c.bundle"] + assert [b.name for b in cat.resolve("Bg")] == ["n1", "n2"] + assert cat.has("K1") and cat.has("Other/x") and not cat.has("Bg") and not cat.has("Nope") + assert cat.keys("K") == ["K1", "K2"] and cat.keys("") == ["K1", "K2", "Other/x"] + for f in (cat.fetch_key, cat.entry): + with pytest.raises(KeyError): + f("Other/x") def fake_extract(calls):