diff --git a/docs/cli.md b/docs/cli.md index b6fc062..e6788a2 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -195,7 +195,7 @@ an ADV episode -> JSON ## nnnotes export story -an ADV episode -> the story product and the products it plays with (scene, assets, Live2D models, cue sheets, chat, CRI Lips) +an ADV episode -> the story product and the products it plays with (scene, assets, Live2D models, cue sheets, videos, chat, CRI Lips) | Argument | Description | | --- | --- | diff --git a/docs/commands.md b/docs/commands.md index 85560d9..1caac51 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -188,6 +188,7 @@ OUT/storyassets/.json each stage, frame, effect, post effect, still, ti chat icon and stamp the episode loads (`story.asset`; : its key, "/" as "~") OUT/models/.json each Live2D model of the episode (`live2d.model`) OUT/cuesheets/.json each cue sheet of the episode, decoded (`story.sound`) +OUT/videos/.json each video of the episode, as WebM (`story.video`; : its asset, "/" as "~") OUT/storychat/default.json the shared chat rows, when the episode shows chat (`story.chat`) OUT/crilips/default.json the CRI Lips analysis data, when a voice can reach it (`story.crilips`) OUT/assets/. the files the manifests name @@ -197,7 +198,9 @@ OUT/assets/. the files the manifests name `story.json` names the episode's chat rows, its effect instances and, in `parts`, the manifest of each product above it plays with. The command runs twice: it asks for the story, reads its `story.json`, then asks for the story and every product `parts` names and places them. It stops with a usage error before the second run when a key of such a -product is not in the catalog. The episode's talk windows and videos are listed in `episode.json` `resources`. +product is not in the catalog. The episode's talk windows are listed in `episode.json` `resources`. The videos are +the `movie.webm` of `cri.movie` with `format` `webm`: VP9 as stored, re-encoded when the video is in another codec or +has an alpha stream. A product several stories play with (the scene, a stage, a model, a cue sheet) is one product: stories exported into one directory share its manifest and files, and the depot keeps every result for the next export. diff --git a/docs/export.md b/docs/export.md index 5a1aefa..3dddf4d 100644 --- a/docs/export.md +++ b/docs/export.md @@ -17,8 +17,8 @@ nnnotes export story 10462 -o out/stories/10462 # everything the epis `export story` writes the story product of an episode and the products it plays with, as a product layout below `-o`: the story index in `stories/.json`, the stage scene in `storyscenes/default.json`, each stage, frame, effect and -other asset in `storyassets/`, each Live2D model in `models/`, each cue sheet in `cuesheets/`, and, when the episode -uses them, the shared chat rows in `storychat/default.json` and the CRI Lips data in `crilips/default.json`. The files +other asset in `storyassets/`, each Live2D model in `models/`, each cue sheet in `cuesheets/`, each video in +`videos/` and, when the episode uses them, the shared chat rows in `storychat/default.json` and the CRI Lips data in `crilips/default.json`. The files themselves are under `assets/`. Products that several stories share (the scene, assets, models and cue sheets) are placed once, so exporting several diff --git a/docs/products.md b/docs/products.md index 6b12f90..9af78fe 100644 --- a/docs/products.md +++ b/docs/products.md @@ -23,7 +23,10 @@ A product stage may also read cue sheets: the catalog keys `cue_sheets` gives (e sheet is the ACB its key loads, held in a bundle (the `acb` file of an object of the key) or a raw file of the catalog that `cri.audio` decoded. The task's inputs then hold each ACB (`acb:`) and, as `sheets`, the files `cri.audio` made of it (`{key: {acb, status, reason, files: {name: {sha256, size, ext}}}}`); it depends on those -`cri.audio` tasks, and a product file may be one of those files as stored, without copying its bytes. A stage that +`cri.audio` tasks, and a product file may be one of those files as stored, without copying its bytes. Movies work the +same way: the catalog keys `movies` gives, each the USM its key loads (the `usm` file of an object of the key, or a raw +file of the catalog that `cri.movie` decoded), as `usm:` and, in `movies`, the files `cri.movie` made of it with +their facts (`{key: {usm, status, reason, files: {name: {sha256, size, ext, facts}}}}`). A stage that reads files of the APK names them in `APK`: inputs `apk:`, from the fact `apk`. All are read from the planning facts alone, so a command can ask for what its products need before anything runs: @@ -360,10 +363,12 @@ A story episode plays with products that several episodes share, made by `nnnote | `story.crilips` | `default` | `crilips/default.json` | none (it reads the APK) | `crilips.json` and `crilips.bin` | | `story.scene` | `default` | `storyscenes/default.json` | the ADV scene `EmbScene/Adv`, the prefabs `EmbCommon/Prefab/CameraManager`, `EmbCommon/Adv/Prefab/AdvCharacterField`, `AdvBackgroundField` and `AdvGlobalVolume`, the settings `EmbCommon/Adv/Settings/AdvPlayerSettings` and `AdvMasterIdSettings`, and the built-in `GraphicsSettings`, `QualitySettings` and `PlayerSettings` | `scene.json` and the object table | | `story.asset` | a catalog key a story loads: a stage (`Adv/Stage/`), frame, effect, post effect, still, timeline, chat window, chat icon or stamp, transition | `storyassets/.json`, `` the key with `/` as `~` | the key | `asset.json` and the object table | +| `story.video` | a video key `Cri/Video/` (the video of a Movie / Clip command) | `videos/.json`, `` the asset with `/` as `~` | the key | `video.json` and `movie.webm` | | `story.chat` | `default` | `storychat/default.json` | the shared chat shards `Adv/Chat/AdvChat-Text`, `Adv/Chat/AdvChat-Sound` and `Adv/Chat/AdvChat-SoundCueSheet` | `chat.json` | | `story` | a `MasterAdv` id | `stories/.json` | the episode key `Adv/Episode//` (``: the row's `_advEpisodeAsset`), its shard keys (the key with `-Text`, `-Sound`, `-SoundCueSheet`, `-Video`) and `EmbCommon/Adv/Settings/AdvPlayerSettings` | `episode.json` and `story.json` | -`story.sound`, `story.crilips`, `story.chat` and `story` keep no objects: their files are all they hold. +`story.sound`, `story.crilips`, `story.video`, `story.chat` and `story` keep no objects: their files are all they +hold. `story.sound` reads its cue sheet's `cri.audio` files; `cues.json` maps each cue name to the first stream of that name (in stream order): `file` (the sound, next to `cues.json`), `sampleRate`, `channels`, `samples` (frames at `sampleRate`, without encoder padding) and, for a looping stream, `loopStart` and `loopEnd` @@ -391,8 +396,18 @@ the product `unsupported` (`unsupported.crilips.version`). Only a request makes The renderers' shaders and post-processing textures are reached through the graphics settings' references (the render pipeline asset, its renderer data and their features). `story.asset` holds the objects of one catalog key and what they reach; an asset several stories load is one product. `asset.json` has `format` 1, `key`, `objects` -(the key's object ids), `bindings` and `shaders` as above. The CRI Lips data and the story assets are made only when -a request names them; the `product` layout makes `story.scene` when the selection holds every key of it. +(the key's object ids), `bindings` and `shaders` as above. The CRI Lips data, the story assets and the videos are +made only when a request names them; the `product` layout makes `story.scene` when the selection holds every key of +it. + +`story.video` is one movie of a story. Its key loads a CriWare.Assets MonoBehaviour describing the movie and the movie +itself, a USM; `movie.webm` is the `movie.webm` that `cri.movie` makes of the USM with its parameter `format` `webm` +([stages](stages.md#cri-stages)): the first video stream, VP9 as stored or re-encoded as VP9 when it is in another +codec or has an alpha stream (the alpha becoming the WebM's alpha channel), and the first audio stream as Opus. +`video.json` has `format` 1, `file` (`movie.webm`), `codec` (the stored video codec: `vp9`, `h264` or +`sofdec.prime`), `width` and `height` (the coded size), `displayWidth` and `displayHeight`, `frameRate` +(`[numerator, denominator]`), `frames` and `tracks`, the tracks of `movie.webm` (`kind`, `channel`, `codec` and, for a +converted stream, its `source` codec). `story.chat` holds the rows of the chat shards every chat of a story reads. `chat.json` has `format` 1, `text` (the chat message texts), `sounds` (their sound rows) and `cueSheets` (the cue sheet rows those name), each the `_allData` @@ -412,11 +427,11 @@ language, the voices, sounds, cue sheets and videos the commands name, and the r | `episode` | `"episode.json"`, the file next to it | | `chats` | `{chat id: MasterAdvChat row}`: each chat id the commands name (`TargetChatID`) | | `effects` | `{TargetName: "Adv/Effect/"}`: the effect instances, the first asset an Effect command loads under a name | -| `parts` | the manifests of the products the story plays with: `scene` (`storyscenes/default.json`), `assets` (`{address: manifest}` of each `story.asset`), `models` (`{address: manifest}` of each `live2d.model`), `sounds` (`{cue sheet: manifest}` of each `story.sound`), `chat` (`storychat/default.json` when a command names a chat, else null) and `crilips` (`crilips/default.json` when a voice command has a target and voice cues and sets neither `IgnoreData` nor `IgnoreLipSync`, else null) | +| `parts` | the manifests of the products the story plays with: `scene` (`storyscenes/default.json`), `assets` (`{address: manifest}` of each `story.asset`), `models` (`{address: manifest}` of each `live2d.model`), `sounds` (`{cue sheet: manifest}` of each `story.sound`), `videos` (`{address: manifest}` of each `story.video`), `chat` (`storychat/default.json` when a command names a chat, else null) and `crilips` (`crilips/default.json` when a voice command has a target and voice cues and sets neither `IgnoreData` nor `IgnoreLipSync`, else null) | A model's MotionSync (model.json `motionSync`) decides whether a lip-synced voice runs the CRI Lips analysis, so -`crilips` names the data whenever a voice can reach it. The episode's talk windows and videos are listed in -`episode.json` `resources`. `parts` depend on the episode's commands, which only the story's result holds: a command +`crilips` names the data whenever a voice can reach it. The episode's talk windows are listed in `episode.json` +`resources`. `parts` depend on the episode's commands, which only the story's result holds: a command asks for the `story` product first, reads its `story.json`, then asks for the story with every product `parts` names (`nnnotes.pipeline.story.part_products`); `export story` works this way. The `product` layout makes the story of every episode whose key the selection holds. diff --git a/docs/quickstart.md b/docs/quickstart.md index 7658031..2da1d10 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -68,8 +68,8 @@ nnnotes export story 10462 -o out/story_10462 `export episode` writes the episode's command list with its lines in five languages and the index of its voices, sounds and videos. `export story` writes the story product of the episode and the products it plays with below `out/story_10462`: the story index, the stage scene, the assets, the Live2D models, the cue sheets decoded to audio -(AAC by default, `--format` chooses another web format), and the shared chat rows and CRI Lips data when the episode -uses them. The files are under `assets/`. Add `--json` to any command to get its report as JSON. +(AAC by default, `--format` chooses another web format), the videos as WebM, and the shared chat rows and CRI Lips +data when the episode uses them. The files are under `assets/`. Add `--json` to any command to get its report as JSON. ## Next diff --git a/docs/stages.md b/docs/stages.md index b129b3d..0cc2805 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -40,6 +40,7 @@ codec and channel of a USM) and `catalog.index` (version 2: gzip input and Japan | `story.crilips` | `default` | `objects` (empty); `apk:lib/arm64-v8a/libcri_lips_unity.so` | `crilips.json` and `crilips.bin` | | `story.scene` | `default` | `objects`: the object table of the ADV scene, its prefabs and settings and the built-in graphics settings ([products](products.md#stories)) | `scene.json` and the object table | | `story.asset` | a catalog key a story loads | `objects`: the object table of what the key reaches | `asset.json` and the object table | +| `story.video` | a video key `Cri/Video/` | `objects`: the object table of the key; `usm:` and `movies`: the key's USM and its `cri.movie` files with their facts ([products](products.md#stories)); the `cri.movie` task takes `format` `webm` | `video.json` and `movie.webm` | | `story.chat` | `default` | `objects`: the object table of the shared chat shards | `chat.json` | | `story` | a `MasterAdv` id | `objects`: the object table of the episode's command list, its shards and the ADV player settings ([products](products.md#stories)); `master`: the episode's master record | `episode.json` and `story.json` | | `link.addresses` | global | catalog index, censuses (the built-in files' too) | key -> objects (a scene key: the objects of the scene's file outside a hierarchy), Resources.Load path -> objects, built-in selector (class, MonoBehaviour script class) -> objects, serialized file -> bundle or built-in file, bundle -> keys | diff --git a/src/nnnotes/cli/assets.py b/src/nnnotes/cli/assets.py index 22f8b4a..952930a 100644 --- a/src/nnnotes/cli/assets.py +++ b/src/nnnotes/cli/assets.py @@ -131,6 +131,7 @@ def matches(self, stage: str) -> bool: StageSource("story.crilips", "nnnotes.pipeline.story:CriLipsStage", output=True, product=True), StageSource("story.scene", "nnnotes.pipeline.story:SceneStage", output=True, product=True), StageSource("story.asset", "nnnotes.pipeline.story:AssetStage", output=True, product=True), + StageSource("story.video", "nnnotes.pipeline.story:VideoStage", output=True, product=True), StageSource("story.chat", "nnnotes.pipeline.story:ChatStage", output=True, product=True), StageSource("story", "nnnotes.pipeline.story:StoryStage", output=True, product=True), StageSource("view.", "nnnotes.pipeline.views:stages", output=True), @@ -658,6 +659,9 @@ def views(self) -> list[str]: def request(self, products: dict, selection: list[str]) -> None: """Ask for the products `products` {product stage: subjects} alone, with the selection `selection` (what their census needs) instead of the command's: only what the products reach is exported.""" + unknown = sorted(n for n in products if n not in self.installed.stages) + if unknown: + raise ValueError(f"no product stage {', '.join(unknown)} is installed") self.products, self.selection = {k: sorted(v) for k, v in products.items()}, list(selection) def _wanted(self, name: str) -> bool: @@ -686,6 +690,10 @@ def params(self, pipeline: list[str]) -> dict: if fmt is not None: for n in [n for n in pipeline if "audio" in stages[n].PARAMS]: out.setdefault(n, {})["audio"] = fmt + fmt = getattr(self.args, "movie_format", None) + if fmt is not None: + for n in [n for n in pipeline if "format" in stages[n].PARAMS]: + out.setdefault(n, {})["format"] = fmt level = getattr(self.args, "flac_level", None) if level is not None: if not 0 <= level <= 12: diff --git a/src/nnnotes/cli/export.py b/src/nnnotes/cli/export.py index bd0a140..3bd64ab 100644 --- a/src/nnnotes/cli/export.py +++ b/src/nnnotes/cli/export.py @@ -288,7 +288,7 @@ def register(sub) -> None: "story", cmd_story, "an ADV episode -> the story product and the products it plays with (scene, assets, Live2D models, " - "cue sheets, chat, CRI Lips)", + "cue sheets, videos, chat, CRI Lips)", ) c.add_argument("adv_id", type=int, help="MasterAdv id of the episode") c.add_argument( @@ -302,6 +302,7 @@ def register(sub) -> None: flac_level_arg(c, "of the decoded sounds") out_arg(c, "output directory") product_args(c) + c.set_defaults(movie_format="webm") # the story videos: cri.movie's movie.webm c = command( es, diff --git a/src/nnnotes/content/storyepisode.py b/src/nnnotes/content/storyepisode.py index efdf28a..60237c6 100644 --- a/src/nnnotes/content/storyepisode.py +++ b/src/nnnotes/content/storyepisode.py @@ -160,13 +160,14 @@ def chat_ids(episode: dict) -> list[int]: def parts(episode: dict) -> dict: """What the story plays with, from episode.json: the addresses of the ASSET_KINDS resources, the addresses of the - live2d models, the cue sheets of the episode, whether it shows chat and whether a voice reaches the CRI Lips - analysis. Talk windows and videos are products of their own and are not listed.""" + live2d models, the cue sheets of the episode, the addresses of its videos, whether it shows chat and whether a + voice reaches the CRI Lips analysis. Talk windows are not listed.""" resources = episode["resources"] return { "assets": sorted(r["address"] for r in resources if r["kind"] in ASSET_KINDS), "models": sorted(r["address"] for r in resources if r["kind"] == "live2d"), "sounds": sorted({cs["_cueSheetName"] for cs in episode["cuesheets"].values() if cs.get("_cueSheetName")}), + "videos": sorted(r["address"] for r in resources if r["kind"] == "video"), "chat": bool(chat_ids(episode)), "crilips": lip_synced(episode), } @@ -176,7 +177,7 @@ def story(episode: dict, record: dict, paths: dict) -> dict: """The story.json document: the MasterAdvChat row of each chat id of the episode (KeyError when there is none), the effect instances (TargetName -> "Adv/Effect/" + TargetAssetName) and `parts`. `paths` are the manifest paths of the products the story plays with, given by the caller: {"scene", "assets": {address: path}, "models": {address: path}, - "sounds": {cue sheet: path}, "chat", "crilips"}.""" + "sounds": {cue sheet: path}, "videos": {address: path}, "chat", "crilips"}.""" chats = {} for chat_id in chat_ids(episode): hits = [r for r in record["MasterAdvChat"] if r["_id"] == chat_id] diff --git a/src/nnnotes/content/storyvideo.py b/src/nnnotes/content/storyvideo.py new file mode 100644 index 0000000..305b6fd --- /dev/null +++ b/src/nnnotes/content/storyvideo.py @@ -0,0 +1,57 @@ +"""The document of one story video: a Movie / Clip command's `Cri/Video/` key. + +The key loads a CriWare.Assets MonoBehaviour that describes the movie (`movieInfo`: size, display size, frame rate, +frame count, codec) and the movie itself, a USM. cri.movie converts the USM to `movie.webm` (VP9 as stored, or +re-encoded as VP9 when the video is in another codec or has an alpha stream, the alpha becoming the WebM's alpha +channel; the first audio stream as Opus). video.json carries the movie's description and the tracks of `movie.webm`, +so the product keeps no objects. +""" + +from __future__ import annotations + +import re + +from .prefabs import Objects, has_json + +__all__ = ["CODECS", "KEY_PREFIX", "VIDEO_FORMAT", "manifest_name", "video"] + +VIDEO_FORMAT = 1 # video.json "format" +KEY_PREFIX = "Cri/Video/" +ASSET = re.compile(r"[A-Za-z0-9_.-]+(/[A-Za-z0-9_.-]+)*") # the asset part of a video key +CODECS = {1: "sofdec.prime", 5: "h264", 9: "vp9"} # movieInfo.codecType (CriMana.CodecType) + + +def manifest_name(key: str) -> str: + """The manifest name of the video key `key` (`Cri/Video/`): `` with "/" as "~" (ValueError for + another key).""" + asset = key[len(KEY_PREFIX) :] if key.startswith(KEY_PREFIX) else "" + if not ASSET.fullmatch(asset): + raise ValueError(f"{key!r}: not a video key ({KEY_PREFIX})") + return asset.replace("/", "~") + + +def _info(objects: Objects, key: str) -> dict: + """The `movieInfo` of the CriWare.Assets MonoBehaviour among the objects of `key`.""" + hits = [o for o in objects.key(key) if has_json(objects, o) and "movieInfo" in objects.json(o)] + if len(hits) != 1: + raise ValueError(f"{key}: {len(hits)} MonoBehaviours with a movieInfo, expected one") + return objects.json(hits[0])["movieInfo"] + + +def video(objects: Objects, key: str, tracks: list[dict]) -> dict: + """video.json of the video key `key`: `format`, `file` (movie.webm), the movie's `codec` (as stored), `width` and + `height` (the coded size), `displayWidth` and `displayHeight`, `frameRate` [numerator, denominator], `frames`, and + `tracks`, those of movie.webm (cri.movie: kind, channel, codec and, for a converted stream, its `source` codec).""" + mi = _info(objects, key) + return { + "format": VIDEO_FORMAT, + "file": "movie.webm", + "codec": CODECS.get(mi["codecType"], f"codecType {mi['codecType']}"), + "width": mi["width"], + "height": mi["height"], + "displayWidth": mi["dispWidth"], + "displayHeight": mi["dispHeight"], + "frameRate": [mi["framerateN"], mi["framerateD"]], + "frames": mi["totalFrames"], + "tracks": tracks, + } diff --git a/src/nnnotes/pipeline/assemble.py b/src/nnnotes/pipeline/assemble.py index 89f3fac..586b943 100644 --- a/src/nnnotes/pipeline/assemble.py +++ b/src/nnnotes/pipeline/assemble.py @@ -13,17 +13,19 @@ A task's inputs are the object table of what it reaches ("objects", nnnotes.objects/1: every object with its class, name, item status and files, the contained ones with the object holding them, and the root objects of each of its keys), the APK files the stage names in `APK` ("apk:", from the fact "apk"), the cue sheets of the catalog -keys `cue_sheets(subject)` gives and what `more_inputs` adds (a master record as "master": master_input; parameters' -data); it depends on the unity.export and sprite.crop tasks it read and the cri.audio tasks of its cue sheets. A cue -sheet is the ACB the key loads: held in a bundle (the `acb` file of an object of the key) or a raw file of the catalog -(a dependency of the key that cri.audio decoded); its inputs are the ACB ("acb:") and, in "sheets", the files -cri.audio made of it ({key: {acb, status, reason?, files}}). - -`assemble` makes the product's own files from the table, the bytes of its objects and its cue sheets (Sheet); a file -may be content the depot already holds (Stored: a decoded waveform), recorded without reading it again. The result -holds them as "#", the table of the objects they use as "#objects", and one item for the -subject. A product layout (layout.product_entries) places the files and the objects' files and writes the product's -manifest. +keys `cue_sheets(subject)` gives, the movies of the catalog keys `movies(subject)` gives and what `more_inputs` adds (a +master record as "master": master_input; parameters' data); it depends on the unity.export and sprite.crop tasks it +read and the cri.audio and cri.movie tasks of its cue sheets and movies. A cue sheet is the ACB the key loads, a movie +the USM: held in a bundle (the `acb` / `usm` file of an object of the key) or a raw file of the catalog (a dependency +of the key that cri.audio / cri.movie decoded); the inputs are that file ("acb:" / "usm:") and, in "sheets" +/ "movies", the files the stage made of it ({key: {acb or usm, status, reason?, files}}; a movie's files also carry +their facts). + +`assemble` makes the product's own files from the table, the bytes of its objects, its cue sheets (Sheet) and movies +(Movie); a file may be content the depot already holds (Stored: a decoded waveform, a movie), recorded without reading +it again. The result holds them as "#", the table of the objects they use as "#objects", and one +item for the subject. A product layout (layout.product_entries) places the files and the objects' files and writes the +product's manifest. """ from __future__ import annotations @@ -38,13 +40,30 @@ from .orchestrate import SEVERITY from .stages import Output, Stage -EXPORT, CROP, AUDIO = "unity.export", "sprite.crop", "cri.audio" +EXPORT, CROP, AUDIO, MOVIE = "unity.export", "sprite.crop", "cri.audio", "cri.movie" ADDRESSES = units.ADDRESSES TABLE_KIND = "product.objects" -SHEETS_INPUT = "sheets" +SHEETS_INPUT, MOVIES_INPUT = "sheets", "movies" MASTER_INPUT = "master" +@dataclass(frozen=True) +class Decodes: + """How a product reads the results of a CRI stage: the file role of the content in the object table and of its + input (":"), the stage that decodes it, the input that lists the stage's files, what the content is + called and whether the files carry their facts.""" + + role: str + stage: str + input: str + what: str + facts: bool = False + + +SHEETS = Decodes("acb", AUDIO, SHEETS_INPUT, "cue sheet") +MOVIES = Decodes("usm", MOVIE, MOVIES_INPUT, "movie", facts=True) + + def master_input(env, record: dict) -> Input: """The input "master" of a product task: the master record `record` (the rows it reads), stored in the depot.""" body = contract.encode(record) @@ -200,46 +219,77 @@ class Stored: size: int -class Sheet: - """A cue sheet of a product task: its ACB and the files cri.audio made of it (one per stream, `cues.json`, - `streams.json`).""" +class _Decoded: + """The content of a catalog key a CRI stage decoded for a product task, and the files the stage made of it.""" - def __init__(self, key: str, entry: dict, acb: Input, depot): - self.key, self.entry, self._acb, self.depot = key, entry, acb, depot + WHAT = "" + REASON = "" + + def __init__(self, key: str, entry: dict, source: Input, depot): + self.key, self.entry, self._source, self.depot = key, entry, source, depot def _check(self) -> None: if self.entry["status"] != "exported": - reason = self.entry.get("reason") or "unsupported.cri.sheet" - raise Unsupported(reason, f"cue sheet {self.key} was not decoded ({self.entry['status']})") + reason = self.entry.get("reason") or self.REASON + raise Unsupported(reason, f"{self.WHAT} {self.key} was not decoded ({self.entry['status']})") - def acb(self) -> bytes: - return self.depot.input_bytes(self._acb) - - def file(self, name: str) -> Stored: + def _file(self, name: str) -> dict: self._check() f = self.entry["files"].get(name) if f is None: - raise KeyError(f"cue sheet {self.key} has no file {name!r}") + raise KeyError(f"{self.WHAT} {self.key} has no file {name!r}") + return f + + def file(self, name: str) -> Stored: + f = self._file(name) return Stored(f["sha256"], f["size"]) def bytes(self, name: str) -> bytes: return self.depot.read(self.file(name).sha256) + +class Sheet(_Decoded): + """A cue sheet of a product task: its ACB and the files cri.audio made of it (one per stream, `cues.json`, + `streams.json`).""" + + WHAT, REASON = "cue sheet", "unsupported.cri.sheet" + + def acb(self) -> bytes: + return self.depot.input_bytes(self._source) + def streams(self) -> list[dict]: """streams.json: every stream in decoder order ({stream, name, file, sampleRate, channels, samples, ...}).""" return contract.loads(self.bytes("streams.json")) +class Movie(_Decoded): + """A movie of a product task: its USM and the files cri.movie made of it (its streams, `movie.`), each + with its facts.""" + + WHAT, REASON = "movie", "unsupported.cri.movie" + + def _file(self, name: str) -> dict: + """A movie whose subtitles cannot be read is `unsupported` and keeps its other files: a file it made is + there.""" + f = self.entry["files"].get(name) + return f if f is not None else super()._file(name) + + def facts(self, name: str) -> dict: + """The facts cri.movie recorded for its file `name`.""" + return self._file(name).get("facts") or {} + + class Objects: """The object table of a product task, the bytes of its objects' files (from the depot) and its cue sheets.""" - def __init__(self, doc: dict, depot, sheets: dict | None = None): + def __init__(self, doc: dict, depot, sheets: dict | None = None, movies: dict | None = None): self.doc, self.depot = doc, depot self.roots: list[str] = doc["roots"] self._keys: dict[str, list[str]] = doc["keys"] self._objects: dict = doc["objects"] self._json: dict = {} self._sheets: dict[str, Sheet] = sheets or {} + self._movies: dict[str, Movie] = movies or {} def sheet(self, key: str) -> Sheet: """The cue sheet of the catalog key `key` (one of the stage's cue_sheets).""" @@ -248,6 +298,13 @@ def sheet(self, key: str) -> Sheet: except KeyError: raise KeyError(f"cue sheet {key} is not an input of the product") from None + def movie(self, key: str) -> Movie: + """The movie of the catalog key `key` (one of the stage's movies).""" + try: + return self._movies[key] + except KeyError: + raise KeyError(f"movie {key} is not an input of the product") from None + def key(self, key: str) -> list[str]: """The root objects of the catalog key `key` (one of the stage's keys), sorted.""" try: @@ -321,7 +378,7 @@ class ProductStage(Stage): fact "products". A stage that reads master tables names them in `MASTER` (its inputs come from the fact "master"), one that reads files of the APK names them in `APK`.""" - after = (ADDRESSES, EXPORT, CROP, AUDIO) + after = (ADDRESSES, EXPORT, CROP, AUDIO, MOVIE) KIND = "product" MASTER: tuple[str, ...] = () APK: tuple[str, ...] = () @@ -368,6 +425,10 @@ def cue_sheets(self, subject: str, env) -> list[str]: """The catalog keys of the cue sheets `subject` reads (from the facts alone; each also one of its `keys`).""" return [] + def movies(self, subject: str, env) -> list[str]: + """The catalog keys of the movies `subject` reads (from the facts alone; each also one of its `keys`).""" + return [] + def more_inputs(self, subject: str, env) -> list[Input]: """Inputs besides the object table.""" return [] @@ -399,30 +460,37 @@ def reach(self, subject: str, env) -> Reach: def sheets(self, subject: str, env) -> dict[str, tuple[Input, str]]: """{catalog key: (its ACB as input "acb:", the cri.audio task id)} of the cue sheets of `subject`.""" - keys = self.cue_sheets(subject, env) + return self._decodes(SHEETS, self.cue_sheets(subject, env), subject, env) + + def movie_files(self, subject: str, env) -> dict[str, tuple[Input, str]]: + """{catalog key: (its USM as input "usm:", the cri.movie task id)} of the movies of `subject`.""" + return self._decodes(MOVIES, self.movies(subject, env), subject, env) + + def _decodes(self, d: Decodes, keys: list[str], subject: str, env) -> dict[str, tuple[Input, str]]: + """{catalog key: (the content it loads as input ":", the task id of stage d.stage)} of `keys`.""" if not keys: return {} objects = self.reach(subject, env).table["objects"] out = {} for key in keys: - held: dict[str, Input] = {} # content id -> the ACB + held: dict[str, Input] = {} # content id -> the file for o in units.root_objects(env, [key]): - f = (objects.get(o, {}).get("files") or {}).get("acb") + f = (objects.get(o, {}).get("files") or {}).get(d.role) if f is not None: - held[f["sha256"]] = Input(f"acb:{key}", f["sha256"], f["size"], None, ({"kind": "depot"},)) - if not held: # a raw file of the catalog that cri.audio decodes + held[f["sha256"]] = Input(f"{d.role}:{key}", f["sha256"], f["size"], None, ({"kind": "depot"},)) + if not held: # a raw file of the catalog that the stage decodes raw = env.facts.get("raw") or {} for stable in _raw_files(env.fact("index")).get(key, ()): if stable not in raw: - raise ValueError(f"cue sheet {key}: the request does not hold its file {stable}") + raise ValueError(f"{d.what} {key}: the request does not hold its file {stable}") i = raw[stable] - if env.known(contract.task_id(AUDIO, i.sha256)): - held[i.sha256] = Input(f"acb:{key}", i.sha256, i.size, i.name, i.locators) + if env.known(contract.task_id(d.stage, i.sha256)): + held[i.sha256] = Input(f"{d.role}:{key}", i.sha256, i.size, i.name, i.locators) if len(held) != 1: - what = "no ACB" if not held else f"{len(held)} ACBs" - raise ValueError(f"cue sheet {key}: {what} in what the request holds") - (acb,) = held.values() - out[key] = (acb, contract.task_id(AUDIO, acb.sha256)) + what = f"no {d.role.upper()}" if not held else f"{len(held)} {d.role.upper()}s" + raise ValueError(f"{d.what} {key}: {what} in what the request holds") + (src,) = held.values() + out[key] = (src, contract.task_id(d.stage, src.sha256)) return out def inputs(self, subject: str, env) -> list[Input]: @@ -435,30 +503,16 @@ def inputs(self, subject: str, env) -> list[Input]: if m not in have: raise ValueError(f"{self.name} reads {m} of the APK: set [paths] apk to an APK that holds it") out.append(have[m]) - sheets = self.sheets(subject, env) - if sheets: - doc = {} - for key, (acb, tid) in sheets.items(): - res = env.result(tid) - (item,) = res["items"] - e = {"acb": acb.sha256, "status": item["status"]} - if "reason" in item: - e["reason"] = item["reason"]["code"] - e["files"] = { - contract.parse_artifact_id(a["id"])[1]: {k: a["content"][k] for k in ("sha256", "size", "ext")} - for a in res["artifacts"] - } - doc[key] = e - out.append(acb) - data = contract.encode(doc) - out.append(Input(SHEETS_INPUT, env.depot.put(data), len(data), None, ({"kind": "depot"},))) + for d, held in ((SHEETS, self.sheets(subject, env)), (MOVIES, self.movie_files(subject, env))): + if held: + out += _decodes_inputs(d, held, env) return out + self.more_inputs(subject, env) def depends(self, subject: str, env) -> list[str]: named = self.keys(subject, env) or self.resources(subject, env) or self.builtins(subject, env) own = [contract.task_id(ADDRESSES, s) for s in sorted(env.fact("catalogs"))] if named else [] - audio = sorted({tid for _, tid in self.sheets(subject, env).values()}) - return own + self.reach(subject, env).read + audio + cri = {tid for _, tid in (*self.sheets(subject, env).values(), *self.movie_files(subject, env).values())} + return own + self.reach(subject, env).read + sorted(cri) def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: """From the bytes of the objects' files: a product reads each of them about once.""" @@ -469,12 +523,13 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: def run(self, task, depot) -> Output: data = depot.input_bytes(task.input("objects")) doc = contract.loads(data) - sheets = {} - if any(i.role == SHEETS_INPUT for i in task.inputs): - for key, e in contract.loads(depot.input_bytes(task.input(SHEETS_INPUT))).items(): - sheets[key] = Sheet(key, e, task.input(f"acb:{key}"), depot) + decoded: dict = {SHEETS_INPUT: {}, MOVIES_INPUT: {}} + for d, cls in ((SHEETS, Sheet), (MOVIES, Movie)): + if any(i.role == d.input for i in task.inputs): + for key, e in contract.loads(depot.input_bytes(task.input(d.input))).items(): + decoded[d.input][key] = cls(key, e, task.input(f"{d.role}:{key}"), depot) try: - made = self.assemble(task, Objects(doc, depot, sheets)) + made = self.assemble(task, Objects(doc, depot, decoded[SHEETS_INPUT], decoded[MOVIES_INPUT])) except Unsupported as e: why = contract.reason(e.code, e.message) return Output([], [contract.item(task.id, "unsupported", why=why, cls=self.KIND)]) @@ -500,6 +555,29 @@ def run(self, task, depot) -> Output: return Output(arts, [item]) +def _decodes_inputs(d: Decodes, held: dict[str, tuple[Input, str]], env) -> list[Input]: + """The inputs of the contents `held` (ProductStage._decodes): each content as ":" and the input d.input + that lists, per key, the status and the files of its d.stage result.""" + doc, out = {}, [] + for key, (src, tid) in held.items(): + res = env.result(tid) + (item,) = res["items"] + e = {d.role: src.sha256, "status": item["status"]} + if "reason" in item: + e["reason"] = item["reason"]["code"] + files = {} + for a in res["artifacts"]: + f = {k: a["content"][k] for k in ("sha256", "size", "ext")} + if d.facts and a["semantics"].get("facts"): + f["facts"] = a["semantics"]["facts"] + files[contract.parse_artifact_id(a["id"])[1]] = f + e["files"] = files + doc[key] = e + out.append(src) + data = contract.encode(doc) + return [*out, Input(d.input, env.depot.put(data), len(data), None, ({"kind": "depot"},))] + + _RAW: dict = {} # id of the last catalog index -> (the index, {catalog key: the raw files its locations depend on}) diff --git a/src/nnnotes/pipeline/story.py b/src/nnnotes/pipeline/story.py index fda88a6..72ae6a1 100644 --- a/src/nnnotes/pipeline/story.py +++ b/src/nnnotes/pipeline/story.py @@ -1,7 +1,7 @@ """The story products: an ADV episode (story) and the products it plays with. -All are product stages (pipeline.assemble); story.sound, story.crilips, story.chat and story keep no objects (their -files are all they hold): +All are product stages (pipeline.assemble); story.sound, story.crilips, story.video, story.chat and story keep no +objects (their files are all they hold): story.sound subject a cue sheet name (the `_cueSheetName` of a story's cue sheet table): "#cues.json" (kind story.sound: cue name -> the cue's first stream: `file`, `sampleRate`, `channels`, `samples`, @@ -27,6 +27,12 @@ the object table of what the key reaches. Its manifest is storyassets/.json, the key with "/" as "~" (storyasset.manifest_name). An asset several stories load is one product. +story.video subject a video key `Cri/Video/` (a Movie / Clip command's video): "#video.json" (kind + story.video: content.storyvideo.video, the movie's description and the tracks of movie.webm) and + "#movie.webm" (kind video.story), the movie.webm of the key's USM that cri.movie made with its + parameter `format` webm. Its root is the key (the CriWare.Assets MonoBehaviour describing the movie) and + its manifest videos/.json, the asset with "/" as "~" (storyvideo.manifest_name). + story.chat subject `default`: "#chat.json" (kind story.chat: content.storychat.chat, the rows of the shared chat shards). Its manifest is storychat/default.json. @@ -40,8 +46,8 @@ (part_products). `select` (the `product` layout) makes the story.sound product of every cue sheet key the selection holds, story.scene -and story.chat when it holds every key of theirs, and the story of every episode whose key it holds; the CRI Lips data -and the story assets are made only when a request names them. +and story.chat when it holds every key of theirs, and the story of every episode whose key it holds; the CRI Lips data, +the story assets and the videos are made only when a request names them. """ from __future__ import annotations @@ -49,7 +55,7 @@ import tempfile from pathlib import Path -from ..content import liveaudio, storyasset, storychat, storyepisode, storyscene +from ..content import liveaudio, storyasset, storychat, storyepisode, storyscene, storyvideo from ..content.livescene import GRAPHICS from ..formats import crilips from ..formats.encoders import DEFAULT_AUDIO_FORMAT, WEB_AUDIO, mp4_priming, web_encode @@ -70,18 +76,20 @@ "SCENE", "SOUND", "STORY", + "VIDEO", "AssetStage", "ChatStage", "CriLipsStage", "SceneStage", "SoundStage", "StoryStage", + "VideoStage", "cues", "part_products", ] SOUND, CRILIPS, SCENE, ASSET = "story.sound", "story.crilips", "story.scene", "story.asset" -CHAT, STORY = "story.chat", "story" +CHAT, STORY, VIDEO = "story.chat", "story", "story.video" DEFAULT = "default" SHEET_PREFIX = "Cri/Sound/" LIBRARY = crilips.LIB_ENTRY_SUFFIX @@ -245,6 +253,43 @@ def assemble(self, task, objects) -> Assembly: return Assembly({"asset.json": (contract.encode(doc), "json", ASSET)}, keep, roles) +class VideoStage(ProductStage): + """story.video: a story's movie as WebM and its description (module documentation).""" + + name = VIDEO + version = 1 + KIND = "StoryVideo" + WEBM = "movie.webm" + + def keys(self, subject: str, env) -> list[str]: + storyvideo.manifest_name(subject) + return [subject] + + def movies(self, subject: str, env) -> list[str]: + return self.keys(subject, env) + + def manifest(self, subject: str) -> str: + return f"videos/{storyvideo.manifest_name(subject)}.json" + + def inputs(self, subject: str, env) -> list: + for key, (_, tid) in self.movie_files(subject, env).items(): + res = env.result(tid) + if not any(a["id"] == contract.artifact_id(tid, self.WEBM) for a in res["artifacts"]): + if res["items"][0]["status"] == "exported": + raise ValueError(f"{self.name} {key}: {tid} made no {self.WEBM} (cri.movie takes format webm)") + return super().inputs(subject, env) + + def assemble(self, task, objects) -> Assembly: + movie = objects.movie(task.subject) + webm = movie.file(self.WEBM) + doc = storyvideo.video(objects, task.subject, movie.facts(self.WEBM).get("tracks", [])) + files = { + "video.json": (contract.encode(doc), "json", VIDEO), + self.WEBM: (webm, "webm", "video.story"), + } + return Assembly(files, set()) + + class ChatStage(ProductStage): """story.chat: the rows of the shared chat shards (module documentation).""" @@ -280,6 +325,7 @@ def part_paths(parts: dict) -> dict: "assets": {a: AssetStage().manifest(a) for a in parts["assets"]}, "models": {a: ModelStage().manifest(a) for a in parts["models"]}, "sounds": {s: SoundStage().manifest(s) for s in parts["sounds"]}, + "videos": {v: VideoStage().manifest(v) for v in parts["videos"]}, "chat": ChatStage().manifest(DEFAULT) if parts["chat"] else None, "crilips": CriLipsStage().manifest(DEFAULT) if parts["crilips"] else None, } @@ -293,6 +339,7 @@ def part_products(story: dict) -> dict[str, list[str]]: ASSET: sorted(parts["assets"]), MODEL: sorted(parts["models"]), SOUND: sorted(parts["sounds"]), + VIDEO: sorted(parts["videos"]), CHAT: [DEFAULT] if parts["chat"] else [], CRILIPS: [DEFAULT] if parts["crilips"] else [], } @@ -305,6 +352,7 @@ def part_products(story: dict) -> dict[str, list[str]]: ASSET: AssetStage, MODEL: ModelStage, SOUND: SoundStage, + VIDEO: VideoStage, CHAT: ChatStage, CRILIPS: CriLipsStage, } diff --git a/tests/cli/test_assets.py b/tests/cli/test_assets.py index 757f2eb..f1c465e 100644 --- a/tests/cli/test_assets.py +++ b/tests/cli/test_assets.py @@ -1617,3 +1617,20 @@ def hooked(args, cfg, ws, expand=None): # the command's export takes the hook; manifest = contract.loads((tmp_path / "out" / "live2d.cubism" / f"{quote(MODEL_KEY, safe='')}.json").read_bytes()) assert manifest["product"] == "live2d.cubism" and manifest["subject"] == MODEL_KEY assert not (tmp_path / "out" / "fake.product").exists() # the first run's product is not placed + + +def test_every_product_a_story_plays_with_is_installed(): + from nnnotes.pipeline import story + + names = {s.name for s in assets.STAGE_SOURCES} + assert {story.STORY, *story.PART_STAGES} <= names + + +def test_a_request_for_a_product_no_stage_makes_is_an_error(): + from types import SimpleNamespace + + ws = SimpleNamespace(installed=SimpleNamespace(stages={"story": object()})) + with pytest.raises(ValueError, match="no product stage story.nothing is installed"): + assets.Workspace.request(ws, {"story": ["1"], "story.nothing": ["x"]}, []) + assets.Workspace.request(ws, {"story": ["2", "1"]}, ["key:k"]) + assert ws.products == {"story": ["1", "2"]} and ws.selection == ["key:k"] diff --git a/tests/content/test_story_kinds.py b/tests/content/test_story_kinds.py index 660e1cd..fb04919 100644 --- a/tests/content/test_story_kinds.py +++ b/tests/content/test_story_kinds.py @@ -94,6 +94,19 @@ def test_story_takes_the_web_audio_format_of_its_cue_sheets(): assert cli.parse_args(["export", "story", "10001", "-o", "out", "--format", "flac"]).audio_format == "flac" +def test_story_takes_the_webm_format_of_its_movies(): + from types import SimpleNamespace + + from nnnotes.cli import assets + from nnnotes.pipeline import cristages + + args = cli.parse_args(["export", "story", "10001", "-o", "out"]) + assert args.movie_format == "webm" + installed = SimpleNamespace(stages={"cri.movie": cristages.MovieStage(), "cri.audio": cristages.AudioStage()}) + ws = SimpleNamespace(args=args, installed=installed) + assert assets.Workspace.params(ws, ["cri.movie", "cri.audio"]) == {"cri.movie": {"format": "webm"}} + + class _Parsed: def __init__(self, name): self.m_ParsedForm = type("P", (), {"m_Name": name})() diff --git a/tests/content/test_storyepisode.py b/tests/content/test_storyepisode.py index 8d967c2..10111f6 100644 --- a/tests/content/test_storyepisode.py +++ b/tests/content/test_storyepisode.py @@ -237,6 +237,7 @@ def test_parts_lists_the_products_the_story_plays_with(): "assets": ["Adv/Stage/s", "Adv/Transition/t"], "models": ["Character/Live2D/a", "Character/Live2D/b"], "sounds": ["sheet_a", "sheet_b"], + "videos": ["Cri/Video/v"], "chat": True, "crilips": True, } @@ -245,6 +246,7 @@ def test_parts_lists_the_products_the_story_plays_with(): "assets": ["Adv/Stage/s", "Adv/Transition/t"], "models": ["Character/Live2D/a", "Character/Live2D/b"], "sounds": [], + "videos": ["Cri/Video/v"], "chat": False, "crilips": False, } diff --git a/tests/content/test_storyvideo.py b/tests/content/test_storyvideo.py new file mode 100644 index 0000000..8921311 --- /dev/null +++ b/tests/content/test_storyvideo.py @@ -0,0 +1,109 @@ +"""storyvideo: the manifest names and the video.json document of a synthetic CriWare.Assets movie description.""" + +import pytest + +from nnnotes.content import storyvideo + + +class FakeObjects: + """An object table: {oid: (info, {role: value})} with catalog keys {key: [oid]}.""" + + def __init__(self, objs: dict, keys: dict): + self._objs = objs + self._keys = keys + self.roots: list[str] = [] + + def __contains__(self, oid): + return oid in self._objs + + def ids(self, cls=None): + return sorted(o for o, (info, _) in self._objs.items() if cls is None or info["class"] == cls) + + def key(self, key): + return list(self._keys.get(key, [])) + + def info(self, oid): + return self._objs[oid][0] + + def bytes(self, oid, role): + return self._objs[oid][1][role] + + def json(self, oid, role="json"): + return self._objs[oid][1][role] + + +def movie_info(**over) -> dict: + info = { + "codecType": 9, + "width": 1280, + "height": 720, + "dispWidth": 1280, + "dispHeight": 720, + "framerateN": 30000, + "framerateD": 1001, + "totalFrames": 300, + } + info.update(over) + return info + + +def table(key: str, *behaviours: dict) -> FakeObjects: + """A table whose key `key` holds one MonoBehaviour per document in `behaviours`.""" + objs, ids = {}, [] + for n, doc in enumerate(behaviours): + oid = f"globalgamemanagers.assets:{n + 1}" + objs[oid] = ({"class": "MonoBehaviour", "files": {"json": {}}}, {"json": doc}) + ids.append(oid) + return FakeObjects(objs, {key: ids}) + + +TRACKS = [{"kind": "video", "channel": 0, "codec": "vp9"}, {"kind": "audio", "channel": 0, "codec": "opus"}] + + +def test_the_manifest_name_is_the_asset_with_slashes_as_tildes(): + assert storyvideo.manifest_name("Cri/Video/adv/movie_a/movie_a") == "adv~movie_a~movie_a" + assert storyvideo.manifest_name("Cri/Video/intro.v-2") == "intro.v-2" + + +@pytest.mark.parametrize("key", ["Cri/Sound/adv/m", "Cri/Video/", "Cri/Video/a b", "Cri/Video//a", "Cri/Video/a/"]) +def test_a_key_that_is_not_a_video_key_has_no_manifest_name(key): + with pytest.raises(ValueError, match="not a video key"): + storyvideo.manifest_name(key) + + +def test_video_describes_the_movie_and_carries_the_tracks_of_the_webm(): + key = "Cri/Video/adv/m/m" + objects = table(key, {"movieInfo": movie_info()}) + doc = storyvideo.video(objects, key, TRACKS) + assert doc == { + "format": 1, + "file": "movie.webm", + "codec": "vp9", + "width": 1280, + "height": 720, + "displayWidth": 1280, + "displayHeight": 720, + "frameRate": [30000, 1001], + "frames": 300, + "tracks": TRACKS, + } + + +def test_video_names_the_codecs_it_knows_and_keeps_the_others_by_number(): + key = "Cri/Video/adv/m/m" + for codec, name in ((1, "sofdec.prime"), (5, "h264"), (9, "vp9")): + doc = storyvideo.video(table(key, {"movieInfo": movie_info(codecType=codec)}), key, []) + assert doc["codec"] == name + doc = storyvideo.video(table(key, {"movieInfo": movie_info(codecType=7)}), key, []) + assert doc["codec"] == "codecType 7" + + +def test_video_reads_the_one_movie_description_of_the_key(): + key = "Cri/Video/adv/m/m" + other = {"Other": True} + assert storyvideo.video(table(key, other, {"movieInfo": movie_info()}), key, [])["frames"] == 300 + with pytest.raises(ValueError, match="0 MonoBehaviours with a movieInfo"): + storyvideo.video(table(key, other), key, []) + two = {"movieInfo": movie_info()} + with pytest.raises(ValueError, match="2 MonoBehaviours with a movieInfo"): + storyvideo.video(table(key, two, two), key, []) diff --git a/tests/pipeline/test_story.py b/tests/pipeline/test_story.py index b845b06..e825de4 100644 --- a/tests/pipeline/test_story.py +++ b/tests/pipeline/test_story.py @@ -6,13 +6,25 @@ from nnnotes.formats import crilips from nnnotes.pipeline import assemble, contract, cristages, live, story from nnnotes.pipeline.assemble import Objects -from nnnotes.pipeline.contract import IncompatibleTask +from nnnotes.pipeline.contract import IncompatibleTask, Input +from nnnotes.pipeline.cristages import MovieStage from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.live2d import MODEL -from nnnotes.pipeline.stages import Env -from nnnotes.pipeline.story import AssetStage, ChatStage, CriLipsStage, SceneStage, SoundStage, StoryStage +from nnnotes.pipeline.stages import Env, describe, execute, registry +from nnnotes.pipeline.story import ( + AssetStage, + ChatStage, + CriLipsStage, + SceneStage, + SoundStage, + StoryStage, + VideoStage, +) from tests.formats.test_crilips import _elf_with_rodata, _synthetic_version +from tests.formats.test_usm import KEY from tests.helpers import story as sh +from tests.pipeline.test_assemble import addresses +from tests.pipeline.test_cristages import export_result, fake_tools, planning, usm_file from tests.pipeline.test_live import master_record, run, stored, table from tests.site.test_story_site import mp4 @@ -268,6 +280,7 @@ def test_the_story_product_holds_the_episode_and_the_story_document(tmp_path): }, "models": {}, "sounds": {}, + "videos": {"Cri/Video/adv/movie_a/movie_a": "videos/adv~movie_a~movie_a.json"}, "chat": "storychat/default.json", "crilips": None, }, @@ -320,6 +333,7 @@ def test_part_paths_give_the_manifest_of_each_part(): "assets": ["Adv/Stage/st_a/st_a", "Adv/Still/st"], "models": ["Character/Live2D/g/m/model/m"], "sounds": ["sheet_a"], + "videos": ["Cri/Video/adv/m/m"], "chat": True, "crilips": False, } @@ -331,15 +345,17 @@ def test_part_paths_give_the_manifest_of_each_part(): }, "models": {"Character/Live2D/g/m/model/m": "models/m.json"}, "sounds": {"sheet_a": "cuesheets/sheet_a.json"}, + "videos": {"Cri/Video/adv/m/m": "videos/adv~m~m.json"}, "chat": "storychat/default.json", "crilips": None, } - quiet = {**parts, "assets": [], "models": [], "sounds": [], "chat": False, "crilips": True} + quiet = {**parts, "assets": [], "models": [], "sounds": [], "videos": [], "chat": False, "crilips": True} assert story.part_paths(quiet) == { "scene": "storyscenes/default.json", "assets": {}, "models": {}, "sounds": {}, + "videos": {}, "chat": None, "crilips": "crilips/default.json", } @@ -350,6 +366,7 @@ def test_part_products_list_the_subjects_of_each_product_the_story_needs(): "assets": ["Adv/Stage/z", "Adv/Stage/a"], "models": ["Character/Live2D/g/m/model/m"], "sounds": ["sheet_a"], + "videos": ["Cri/Video/adv/m/m"], "chat": True, "crilips": True, } @@ -359,14 +376,212 @@ def test_part_products_list_the_subjects_of_each_product_the_story_needs(): story.ASSET: ["Adv/Stage/a", "Adv/Stage/z"], MODEL: ["Character/Live2D/g/m/model/m"], story.SOUND: ["sheet_a"], + story.VIDEO: ["Cri/Video/adv/m/m"], story.CHAT: ["default"], story.CRILIPS: ["default"], } def test_part_products_omit_the_products_a_story_does_not_need(): - parts = {"assets": [], "models": [], "sounds": [], "chat": False, "crilips": False} + parts = {"assets": [], "models": [], "sounds": [], "videos": [], "chat": False, "crilips": False} assert story.part_products({"parts": story.part_paths(parts)}) == {story.SCENE: ["default"]} parts["sounds"] = ["sheet_a"] doc = {"parts": story.part_paths(parts)} assert story.part_products(doc) == {story.SCENE: ["default"], story.SOUND: ["sheet_a"]} + + +# ---------------------------------------------------------------- the video product +VIDEO_KEY = "Cri/Video/adv/m/m" +WEBM = b"\x1aE\xdf\xa3 synthetic webm" +TRACKS = [ + {"kind": "video", "channel": 0, "codec": "vp9"}, + {"kind": "audio", "channel": 1, "codec": "opus", "source": "adx"}, +] + + +def video_table(depot, key=VIDEO_KEY, info=None) -> Input: + """The object table of the video key `key`: one MonoBehaviour whose json carries `movieInfo`.""" + info = info or { + "codecType": 9, + "width": 640, + "height": 360, + "dispWidth": 640, + "dispHeight": 360, + "framerateN": 30, + "framerateD": 1, + "totalFrames": 90, + } + data = contract.encode({"movieInfo": info}) + oid = "globalgamemanagers.assets:1" + objects = { + oid: {"status": "exported", "class": "MonoBehaviour", "files": {"json": {"sha256": depot.put(data), + "size": len(data), "ext": "json"}}}, + } # fmt: skip + doc = {"schema": contract.OBJECTS, "roots": [oid], "keys": {key: [oid]}, "objects": objects, "missing": []} + return stored(depot, contract.encode(doc), "objects") + + +def movie_inputs(depot, key=VIDEO_KEY, entry=None) -> list[Input]: + """The inputs "usm:" and "movies" of a movie decoded as movie.webm (with its tracks).""" + usm = b"synthetic usm of " + key.encode() + if entry is None: + entry = { + "usm": contract.sha256(usm), + "status": "exported", + "files": {"movie.webm": {"sha256": depot.put(WEBM), "size": len(WEBM), "ext": "webm", + "facts": {"tracks": TRACKS}}}, + } # fmt: skip + doc = {key: entry} + return [ + stored(depot, usm, f"usm:{key}"), + stored(depot, contract.encode(doc), assemble.MOVIES_INPUT), + ] + + +def test_a_video_key_names_its_manifest(): + stage = VideoStage() + assert stage.keys(VIDEO_KEY, None) == [VIDEO_KEY] + assert stage.movies(VIDEO_KEY, None) == [VIDEO_KEY] + assert stage.manifest("Cri/Video/adv/movie_a/movie_a") == "videos/adv~movie_a~movie_a.json" + for bad in ("Cri/Sound/adv/m", "Cri/Video/", "Cri/Video/a b/c", "adv/m"): + with pytest.raises(ValueError, match="not a video key"): + stage.keys(bad, None) + with pytest.raises(ValueError, match="not a video key"): + stage.manifest(bad) + + +def test_the_video_product_holds_the_movie_as_webm_and_its_description(tmp_path): + depot = Depot(tmp_path / "d") + inputs = movie_inputs(depot) + _, res, arts = run(depot, VideoStage(), VIDEO_KEY, [video_table(depot), *inputs]) + assert {p: a["semantics"]["kind"] for p, a in arts.items()} == { + "movie.webm": "video.story", + "objects": "product.objects", + "video.json": "story.video", + } + (item,) = res["items"] + assert item["status"] == "exported" + webm = arts["movie.webm"]["content"] + assert webm["sha256"] == contract.sha256(WEBM) and depot.read(webm["sha256"]) == WEBM + assert read_json(depot, arts["video.json"]) == { + "format": 1, + "file": "movie.webm", + "codec": "vp9", + "width": 640, + "height": 360, + "displayWidth": 640, + "displayHeight": 360, + "frameRate": [30, 1], + "frames": 90, + "tracks": TRACKS, + } + assert read_json(depot, arts["objects"])["objects"] == {} + + +def test_a_movie_that_is_unsupported_keeps_the_webm_it_made(tmp_path): + depot = Depot(tmp_path / "d") + usm_sha = contract.sha256(b"synthetic usm of " + VIDEO_KEY.encode()) + entry = { + "usm": usm_sha, + "status": "unsupported", + "reason": "unsupported.usm.subtitle", + "files": {"movie.webm": {"sha256": depot.put(WEBM), "size": len(WEBM), "ext": "webm", + "facts": {"tracks": TRACKS}}}, + } # fmt: skip + _, res, arts = run(depot, VideoStage(), VIDEO_KEY, [video_table(depot), *movie_inputs(depot, entry=entry)]) + (item,) = res["items"] + assert item["status"] == "exported" + assert arts["movie.webm"]["content"]["sha256"] == contract.sha256(WEBM) + assert read_json(depot, arts["video.json"])["tracks"] == TRACKS + + +def test_an_unsupported_movie_without_its_webm_makes_the_video_product_unsupported(tmp_path): + depot = Depot(tmp_path / "d") + usm_sha = contract.sha256(b"synthetic usm of " + VIDEO_KEY.encode()) + entry = {"usm": usm_sha, "status": "unsupported", "reason": "unsupported.usm.codec", "files": {}} + _, res, arts = run(depot, VideoStage(), VIDEO_KEY, [video_table(depot), *movie_inputs(depot, entry=entry)]) + (item,) = res["items"] + assert item["status"] == "unsupported" and item["reason"]["code"] == "unsupported.usm.codec" + assert arts == {} + + +class Movies(VideoStage): + """A product reading the movies of its keys (the planning of a product that holds several movies).""" + + def __init__(self, keys): + super().__init__() + self.movie_keys = list(keys) + + def keys(self, subject, env) -> list[str]: + return self.movie_keys + + def movies(self, subject, env) -> list[str]: + return self.movie_keys + + +def test_the_movies_of_a_product_are_the_usms_the_request_holds(tmp_path): + """A USM held in a bundle is the `usm` file of an object of its key; a raw one is the raw file of the key that + cri.movie decoded. The product's inputs are the movies' USMs and cri.movie's files (movie.webm), and it depends on + both cri.movie tasks.""" + depot = Depot(tmp_path / "d") + held_key, raw_key = "Cri/Video/adv/m/held", "Cri/Video/adv/m/raw" + held, _ = usm_file(audio=False) + raw, _ = usm_file() + with fake_tools(key=KEY): + env = planning(depot, tmp_path, {"cri_assets_cri/video/raw": raw}, keys={raw_key: ["cri_assets_cri/video/raw"]}) + export_result(env, "movies_a", {}, {"Held": held}) + env.facts["catalogs"] = {"main": []} + addresses(depot, env, {held_key: ["CAB-movies_a:100"], raw_key: []}) + stage = Movies([held_key, raw_key]) + env.facts["products"] = {stage.name: ["x"]} + env.facts["roots"] = assemble.request({stage.name: stage}, {stage.name: ["x"]}, env) + movie = MovieStage() + for s in movie.subjects(env): + t = describe(movie, s, {"format": "webm"}, env) + execute(t, depot, registry(movie), force=True) + env.done(t.id, t.key) + sha, raw_sha = contract.sha256(held), contract.sha256(raw) + files = stage.movie_files("x", env) + assert {k: (i.role, i.sha256, tid) for k, (i, tid) in files.items()} == { + held_key: (f"usm:{held_key}", sha, f"cri.movie:{sha}"), + raw_key: (f"usm:{raw_key}", raw_sha, f"cri.movie:{raw_sha}"), + } + assert {f"cri.movie:{sha}", f"cri.movie:{raw_sha}"} <= set(stage.depends("x", env)) + t = describe(stage, "x", None, env) + assert sorted(i.role for i in t.inputs) == sorted([f"usm:{held_key}", f"usm:{raw_key}", "objects", "movies"]) + + +def test_a_movie_the_request_does_not_hold_is_an_error(tmp_path): + depot = Depot(tmp_path / "d") + with fake_tools(key=KEY): + env = planning(depot, tmp_path, {}, keys={"Cri/Video/adv/m/raw": ["cri_assets_cri/video/raw"]}) + env.facts["catalogs"], env.facts["bundles"] = {"main": []}, [] + env.facts["selected"] = [] + addresses(depot, env, {"Cri/Video/adv/m/held": []}) + env.facts["roots"] = {"keys": [], "objects": []} + with pytest.raises(ValueError, match="movie Cri/Video/adv/m/held: no USM"): + Movies(["Cri/Video/adv/m/held"]).movie_files("x", env) + with pytest.raises(ValueError, match="does not hold its file cri_assets_cri/video/raw"): + Movies(["Cri/Video/adv/m/raw"]).movie_files("x", env) + + +def test_a_video_product_needs_the_webm_of_each_movie(tmp_path): + """cri.movie made only movie.mkv (its default format): the video product names the format it needs.""" + depot = Depot(tmp_path / "d") + raw, _ = usm_file() + key = "Cri/Video/adv/m/raw" + with fake_tools(key=KEY): + env = planning(depot, tmp_path, {"cri_assets_cri/video/raw": raw}, keys={key: ["cri_assets_cri/video/raw"]}) + env.facts["catalogs"], env.facts["bundles"] = {"main": []}, [] + env.facts["selected"] = [] + addresses(depot, env, {key: []}) + stage = Movies([key]) + env.facts["products"] = {stage.name: ["x"]} + env.facts["roots"] = assemble.request({stage.name: stage}, {stage.name: ["x"]}, env) + movie = MovieStage() + for s in movie.subjects(env): + t = describe(movie, s, None, env) + execute(t, depot, registry(movie), force=True) + env.done(t.id, t.key) + with pytest.raises(ValueError, match="format webm"): + stage.inputs("x", env)