From eda7767ec73543cf18720e54acce25566b1116a5 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Thu, 15 Oct 2026 04:01:45 +0800 Subject: [PATCH] feat(export)!: run unity.export once per unit instead of once per bundle unity.export (version 2) runs one task per unit: a prefab hierarchy (a root GameObject with the objects its prefab holds) or an object outside any hierarchy, named by the stable address of its root object, `:`. The unit table (nnnotes.pipeline.units) comes from the censuses: a run exports the units of the selected bundles and every unit their objects reference, and `--only-class` selects units by the class of their root object instead of entering the task keys. - The tasks of one bundle form one batch: a worker loads the bundle once and shares its object index, hierarchies, atlases and last decoded textures across them. A unit's result does not depend on the other units of the batch or their order. - A task key covers the bundle, the unit's root, the converters of its classes and the script entries and shipped type trees its objects use, so a converter or script change runs again only the units holding it. - MonoScript and AssetBundle objects get the typetree JSON like any other class. - The original layout roots objects without a container path at `_bundles/` from the unit table; cri.audio, cri.movie, sprite.crop and view.voices find bundle-held content through it. BREAKING CHANGE: unity.export task ids are `unity.export::`. The bundle-level artifacts `scripts.json` and `bundle.json`, the link.artifacts stage and the nnnotes.artifacts/1 schema are removed. Co-Authored-By: Claude Opus 5.5 --- docs/assets.md | 8 +- docs/cli.md | 4 +- docs/contracts.md | 17 +- docs/schemas.md | 1 - docs/stages.md | 55 ++- src/nnnotes/cli/assets.py | 81 +-- src/nnnotes/pipeline/cristages.py | 32 +- src/nnnotes/pipeline/link.py | 130 +---- src/nnnotes/pipeline/objexport.py | 574 +++++++++++----------- src/nnnotes/pipeline/units.py | 198 ++++++++ src/nnnotes/pipeline/voices.py | 28 +- src/nnnotes/schemas/artifacts.schema.json | 57 --- src/nnnotes/unity/convert/__init__.py | 16 +- src/nnnotes/unity/convert/generic.py | 20 +- src/nnnotes/unity/convert/objects.py | 59 ++- src/nnnotes/unity/convert/textures.py | 31 +- tests/cli/test_assets.py | 132 +++-- tests/golden/unity.export.json | 68 +-- tests/helpers/units.py | 76 +++ tests/pipeline/test_cristages.py | 47 +- tests/pipeline/test_link.py | 75 --- tests/pipeline/test_objexport.py | 520 ++++++++++---------- tests/pipeline/test_voices_story.py | 38 +- 23 files changed, 1145 insertions(+), 1122 deletions(-) create mode 100644 src/nnnotes/pipeline/units.py delete mode 100644 src/nnnotes/schemas/artifacts.schema.json create mode 100644 tests/helpers/units.py diff --git a/docs/assets.md b/docs/assets.md index c94bb36..96e04b3 100644 --- a/docs/assets.md +++ b/docs/assets.md @@ -45,7 +45,7 @@ nnnotes depot verify [--quick] [--workers N] | `--stored-catalog` | a catalog version of the depot (`catalog versions`) instead of the current catalog | | `--png-level N` | zlib level 0-9 of the PNG files (default 6); part of the task keys | | `--flac-level N` | FLAC compression level 0-12 of the CRI audio and the movies' audio (default 8; every level decodes to the same samples; 12 writes files 0.2-1.3 % smaller in about five times the encoding time); part of the task keys | -| `--only-class C,...` | export only objects of these Unity classes; part of the task keys | +| `--only-class C,...` | export only the units whose root object is of these Unity classes (a prefab by `GameObject`, [stages.md](stages.md#units)) | | `--link` | how the layout files are made from the depot (below): `auto` (default; `[export] link`), `clone`, `hard` or `copy` | | `--strict` | exit 1 when a view has gaps: a required role whose address names no catalog key or no fitting sub-object (empty values and roles that do not apply are not gaps); in `voices`, a voice whose cue sheet has no catalog key or whose decoded sheet lacks its cue, and a story episode without a catalog key | | `--dry-run` | print the plan and stop | @@ -53,10 +53,12 @@ nnnotes depot verify [--quick] [--workers N] A run resolves the context (region, language, catalog version) and indexes the catalog, identifies the selected bundles (the sha256 of each decrypted bundle, remembered by path, size and modification time; bundles not -downloaded yet are fetched into the depot's `downloads/`), takes the census of each bundle, links the scripts and addresses, exports the bundles, runs the +downloaded yet are fetched into the depot's `downloads/`), takes the census of each bundle, links the scripts and +addresses, exports the units (one task per hierarchy or object outside any, [stages.md](stages.md#units)), runs the stages that need those results (atlas sprites, views), writes the layouts and the reports. The selected raw files (CRI audio and movies) not downloaded yet are fetched while the bundle stages run. The census also covers -the bundles the selected ones depend on, so script classes and atlas textures in other bundles resolve. +the bundles the selected ones depend on, so script classes and atlas textures in other bundles resolve, and the +export covers the units of those bundles that the selected ones reference. Tasks run in spawned worker processes that stay alive between tasks. The tasks of a stage that read the same loaded data (a stage's batch, such as one bundle) form one job: one worker runs them one after another and loads the diff --git a/docs/cli.md b/docs/cli.md index 469c685..ec38edb 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -174,7 +174,7 @@ every object of the selected bundles -> common formats (the depot and its layout | `--stored-catalog` `LABEL\|SHA` | a catalog version of the depot (`catalog versions`; default: the current catalog) | | `--png-level` `N` | PNG compression level 0-9 (changes the output) | | `--flac-level` `N` | FLAC compression level 0-12 of the CRI audio (default 8; every level decodes to the same samples; changes the files) | -| `--only-class` `C,...` | export only objects of these classes | +| `--only-class` `C,...` | export only the units whose root object is of these classes | | `--workers` `N` | worker processes (default: the CPUs this process may use; 0: this process) | | `--memory` `GiB` | memory budget of the running tasks and workers (default: 80 % of the physical memory) | | `--fetch-workers` `N` | parallel downloads (default 8) | @@ -317,7 +317,7 @@ what `export assets` would run and why, without running it (--json: nnnotes.plan | `--stored-catalog` `LABEL\|SHA` | a catalog version of the depot (`catalog versions`; default: the current catalog) | | `--png-level` `N` | PNG compression level 0-9 (changes the output) | | `--flac-level` `N` | FLAC compression level 0-12 of the CRI audio (default 8; every level decodes to the same samples; changes the files) | -| `--only-class` `C,...` | export only objects of these classes | +| `--only-class` `C,...` | export only the units whose root object is of these classes | | `--workers` `N` | worker processes (default: the CPUs this process may use; 0: this process) | | `--memory` `GiB` | memory budget of the running tasks and workers (default: 80 % of the physical memory) | | `--fetch-workers` `N` | parallel downloads (default 8) | diff --git a/docs/contracts.md b/docs/contracts.md index 5c5569b..369222e 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -36,14 +36,15 @@ The canonical JSON (`nnnotes.core.canon`) of every document nnnotes writes: | object | `:` | `CAB-2f1e…:-4153…`, `unity default resources:10` | | artifact of an object | `#` | `CAB-2f1e…:-4153…#image` | | artifact of no object | `:#` | `cri.audio:#bgm_x.flac` | -| task | `:` | `unity.export:membercard_assets_membercard_100101` | +| task | `:` | `unity.census:membercard_assets_membercard_100101`, `unity.export:membercard_assets_membercard_100101:-4153…` | | stable bundle name | the bundle file name without `_<32 hex>` and `.bundle` | `membercard_assets_membercard_100101` | | stable address | `/` (`[]` for a sub-object), else `:` | `membercard_assets_membercard_100101/assets/…/member_full.png` | Stage names contain no `:` or `#`; an artifact's owner contains no `#` (its role may). A bundle's serialized file names change whenever its content changes, while its path ids stay: an object id names one content version of an object, and the stable address finds the same object in another catalog version. Bundle stages use the stable -bundle name as their subject, so a task keeps its id across versions while its key follows the content. When two +bundle name as their subject and object stages the `:` address of their object, so a task +keeps its id across versions while its key follows the content. When two bundles of a catalog have the same stable name, both keep their full file names. ## Task: nnnotes.task/1 @@ -52,13 +53,14 @@ One stage applied to one subject, with everything needed to run it ([task.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/task.schema.json)): ```json -{"schema": "nnnotes.task/1", "id": "unity.export:membercard_assets_membercard_100101", - "stage": {"name": "unity.export", "version": 1}, - "params": {"png": {"encoder": "pillow", "level": 6}, "blobMin": 4096}, +{"schema": "nnnotes.task/1", "id": "unity.export:membercard_assets_membercard_100101:-4153…", + "stage": {"name": "unity.export", "version": 2}, + "params": {"blobMin": 4096, "png": {"level": 6}}, "atoms": {"png.encode": "pillow-12.3.0/1", "texture.decode": "unitypy-1.25.3/1"}, "inputs": [{"role": "bundle", "sha256": "…", "size": 123456, "name": "membercard_…_3fa…c2.bundle", "locators": [{"kind": "depot"}, {"kind": "download", "path": "bundles/membercard_…_3fa…c2.bundle"}]}], - "context": {"scripts": {"CAB-…:12345": "Assembly-CSharp|Fwk.Sound|SplitAcbData"}}, + "context": {"unit": "CAB-2f1e…:-4153…", "bundle": "membercard_assets_membercard_100101", + "scripts": {"CAB-…:12345": "Assembly-CSharp|Fwk.Sound|SplitAcbData"}}, "key": "…", "cost": {"cpuSeconds": 1.2, "peakBytes": 450000000}} ``` @@ -97,7 +99,7 @@ One output file ([artifact.schema.json](https://github.com/MetaSekaiLab/nnnotes/ ```json {"id": "CAB-…:-4153…#image", "content": {"sha256": "…", "size": 1234, "mediaType": "image/png", "ext": "png"}, - "provenance": {"stage": {"name": "unity.export", "version": 1, "params": ""}, + "provenance": {"stage": {"name": "unity.export", "version": 2, "params": ""}, "atoms": {"texture.decode": "…", "png.encode": "…"}, "inputs": [{"role": "bundle", "sha256": "…"}], "object": {"file": "CAB-…", "pathId": -4153, "classId": 28, "class": "Texture2D", @@ -221,7 +223,6 @@ Documents written by particular stages (artifacts of no object: their ids are `< | `nnnotes.census/2` | `unity.census` | [census.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/census.schema.json) | | `nnnotes.scripts/1` | `link.scripts` | [scripts.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/scripts.schema.json) | | `nnnotes.addresses/1` | `link.addresses` | [addresses.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/addresses.schema.json) | -| `nnnotes.artifacts/1` | `link.artifacts` | [artifacts.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/artifacts.schema.json) | | `nnnotes.view/1` | `view.` | [view.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/view.schema.json) | The catalog versions of a depot, `nnnotes.catalogs/1` ([catalogs.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/catalogs.schema.json), diff --git a/docs/schemas.md b/docs/schemas.md index f0930e7..0fa3405 100644 --- a/docs/schemas.md +++ b/docs/schemas.md @@ -16,7 +16,6 @@ schemas.load("plan") # the schema as a dict |---|---|---| | `addresses` | `nnnotes.addresses/1` | the address table of a catalog version and the censuses of its bundles | | `artifact` | `nnnotes.artifact/1` | the content, provenance and semantics of one output file | -| `artifacts` | `nnnotes.artifacts/1` | the artifact table: the artifacts of every object, with content id, size and extension | | `catalog-diff` | `nnnotes.catalog-diff/1` | the difference between two catalog versions (`catalog diff`) | | `catalog-index` | `nnnotes.catalog-index/1` | the index of a catalog pair: remote and APK catalogs, every location in catalog order | | `catalogs` | `nnnotes.catalogs/1` | the catalog versions of a depot (`catalog versions`) | diff --git a/docs/stages.md b/docs/stages.md index a350faf..7d46629 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -2,34 +2,32 @@ The asset export is built from three layers: -- **Stages** apply one unit of work to one subject (a bundle, a content id, a model key, a view). Each has a name and +- **Stages** apply one unit of work to one subject (a bundle, an object, a content id, a model key, a view). Each has a name and a version; a task is a stage applied to one subject, and its key covers exactly what its output depends on ([contracts.md](contracts.md)). - **Converters** turn the objects of one class inside `unity.export` into artifacts. Each carries its own version; a - bundle's task includes only the converters of the classes the bundle holds. + task includes only the converters of the classes its objects have. - **Atoms** are the pure functions stages and converters are made of: no file or network IO, no threads, the same output for the same input. Each atom has exactly one reference implementation. ## Stages Every stage is at version 1, except `unity.census` (version 3: it lists what every object references and the -hierarchy holding it) -and `cri.movie` (version 2: every stream kind, codec and channel of a USM), and `catalog.index` (version 2: -gzip input and Japanese remote asset placeholders). +hierarchy holding it), `unity.export` (version 2: one task per unit), `cri.movie` (version 2: every stream kind, +codec and channel of a USM) and `catalog.index` (version 2: gzip input and Japanese remote asset placeholders). | Stage | Subject | Inputs | Outputs | |---|---|---|---| | `catalog.index` | a catalog (remote + APK) | the catalog files | every location (with the bundle hash, crc and size), bundles and raw remote files | | `unity.census` | a bundle, or a built-in file of the APK | the bundle | serialized files, externals, objects (file, path id, class, size, type hash, name, script, the scripts an AnimationClip's bindings name, the objects it references, the hierarchy holding it), container, scene hashes, dependencies, script fields | | `link.scripts` | global | the censuses holding MonoScripts | serialized file and path id -> assembly, namespace, class | -| `unity.export` | a bundle, or a built-in file of the APK | the bundle, the script entries its MonoBehaviours use (and, for files without type trees, the type trees nnnotes ships for their type hashes) | the objects' artifacts and one item per object | +| `unity.export` | a unit of a bundle or of a built-in file of the APK (below), by the stable address of its root object | the bundle; context `unit` (the root's object id), `bundle`, the script entries its objects use and, for files without type trees, the type trees nnnotes ships for their type hashes | its objects' artifacts and one item per object | | `sprite.crop` | a SpriteAtlas | the atlas' JSON, the images of its textures, the metas of the sprites of other bundles it packs | the sprites' images | | `cri.audio` | ACB content | `acb` (a raw file of the catalog, or the `acb` artifact of a cue sheet held in a bundle), `awb` (an AWB the catalog pairs with it), `boot` (the game's boot data: the key's source) | FLAC per stream, `cues.json`, `streams.json` | | `cri.movie` | USM content | `usm` (a raw file of the catalog, or the `usm` artifact of a movie held in a bundle), `boot` | every stream as stored (video, alpha, audio, subtitles), `movie.mkv` (or `movie.webm`) | | `live2d.model` | a Live2D model key (`Character/Live2D///model/`) whose bundle is selected | `bundle:`: the bundles of the key's dependency closure in the order the extractor loads them (the order is part of the key); context `internalId` (the key's location); atom `live2d.extract_model` (the extractor of `nnnotes export live2d`, run unchanged) | `live2d.model:#`: every file `nnnotes export live2d` writes for the model, `` relative to the model's directory; placed under `/` | | `spine.skeleton` | a SkeletonDataAsset of a selected bundle, by its stable address `:` (in a bundle of several serialized files the file's tag follows the bundle name, as in `link.addresses`) | `bundle:`: the bundles holding the serialized files the asset's file reaches through its externals; context `object` (the asset's object id); atom `spot.write_skeleton` (the Spine writer of `nnnotes export spot`) | `spine.skeleton:#`: the skeleton (`.json` or `.skel`), its atlases (`.atlas`) and their page images, the bytes `nnnotes export spot` writes into `spine/`; one item for the asset | | `link.addresses` | global | catalog index, censuses (the built-in files' too) | key -> objects, serialized file -> bundle or built-in file, bundle -> keys | -| `link.artifacts` | global | the above, the results | object id -> artifact ids | | `view.` | a catalog (`main`) | `master:` per master table it reads, `rules` (the view's rule and the resolvers it uses), `addresses` (the part of the address table it read: the keys looked up and the prefixes listed); no parameters, atoms or context | `view.:#view` (facts: rows, entries, gaps, unreferenced) | | `view.voices` | a catalog (`main`) | `master:
` per master table its rules read, `rules`, `addresses`, `sheets` (each cue sheet the `cri.audio` results are known by that a voice names or the rules' prefixes list: its task and ACB content id), `acb:` (the ACB of each decoded one), `episodes` (per story episode and part, the `unity.export` artifact holding the object its key names), `adv:` (those contents); context `audio` (the keys of those `cri.audio` results); runs after `unity.export` and `cri.audio` | `view.voices:#view` (facts: rows, entries, gaps, unreferenced streams) | @@ -40,8 +38,8 @@ gzip input and Japanese remote asset placeholders). ## Converters (`unity.export`) The converters are the library `nnnotes.unity.convert`, one module per group of classes; `unity.export` -(`nnnotes.pipeline.objexport`) runs them over one bundle read alone and gives every object of it exactly one item -status: `exported`, +(`nnnotes.pipeline.objexport`) runs them over one unit of a bundle read alone and gives every object of the unit +exactly one item status: `exported`, `contained` (part of another artifact), `generic` (typetree JSON because the specialized form could not be made; the reason says why), `unsupported` (a class the stage does not convert; its typetree JSON is written as well) or `failed` (not even the typetree JSON could be made). A converter's documented limit gives the generic typetree JSON @@ -50,7 +48,7 @@ with its reason code; any other error of a converter gives it with `generic.erro | Class | Converter | Artifacts | Atoms | Notes | |---|---|---|---|---| | Texture2D | `tex.png/2` | `image`: PNG of the first mip level (HDR ASTC: `.astc`), `json`: the typetree without `image data` (settings, colour space, mip count, stream data) | `texture.decode`, `png.encode`, `astc.container` | further mip levels as facts; a texture without pixels: `empty.texture` | -| Sprite | `sprite.png/2` | `image`: PNG on the sprite's rect (packing undone, tight mask applied), `json`: the typetree, geometry, render data and image (see below) | `texture.decode`, `sprite.crop`, `png.encode`, `mesh.arrays` | the texture is decoded once per task; a sprite whose SpriteAtlas is in another bundle gets its `json` here (with its geometry and object facts) and its `image` from `sprite.crop` | +| Sprite | `sprite.png/2` | `image`: PNG on the sprite's rect (packing undone, tight mask applied), `json`: the typetree, geometry, render data and image (see below) | `texture.decode`, `sprite.crop`, `png.encode`, `mesh.arrays` | the tasks of a bundle share the textures it decoded last; a sprite whose SpriteAtlas is in another bundle gets its `json` here (with its geometry and object facts) and its `image` from `sprite.crop` | | Mesh | `mesh.glb/2` | `mesh`: binary glTF in the mesh's local space, `json`: the typetree without the vertex, index and compressed mesh data | `mesh.arrays`, `gltf.write` | see `gltf.write` below; limits of the skin and blend shapes as issues of the artifact | | GameObject (a root) | `prefab.json/2` | `json`: the hierarchy's nodes (path, path ids, transform) with their components' fields | | the objects below it are `contained`; a root of a scene's file names its scene | | Material | `material.json/1` | `json`: shader, keywords, textures, floats, colours | | | @@ -60,9 +58,7 @@ with its reason code; any other error of a converter gives it with `generic.erro | MonoBehaviour (not in a hierarchy) | `mono.json/2`, `cri.usm/1` | `json`: `$script` (assembly, namespace, class) and the typetree; a cue sheet asset also `acb`, a movie asset holding its USM also `usm` (see below) | `sniff` | the class from the task's script table | | TextAsset | `data/2` | `data`: the stored bytes, `json`: the typetree without `m_Script` | `sniff` | extension from the container path, else from the content | | Font | `font/2` | `font`: the font file, `json`: the typetree without `m_FontData` | `sniff` | | -| MonoScript | `scripts.json/1` | `#scripts.json`: every MonoScript of the bundle | | | -| AssetBundle | `bundle.json/1` | `#bundle.json`: serialized files, externals, the AssetBundle objects | | | -| any other class | `generic.json/2` | `json`: the typetree | `sniff` | an object that can be read only in part (see below): that part, `unsupported.typetree` | +| any other class (a MonoScript, an AssetBundle, ...) | `generic.json/2` | `json`: the typetree | `sniff` | an object that can be read only in part (see below): that part, `unsupported.typetree` | Every object a converter takes has a `json` artifact or is contained in another object's. Each item lists in `refs` the objects its documents reference (by object id, other bundles' objects included, the objects of its own @@ -105,10 +101,25 @@ again. A MonoBehaviour without one is `unsupported.typetree` with its header (`m issue. PlayerSettings, whose type tree reads the stored layout right up to `m_ActiveColorSpace` (by its stored type hash), gives the fields up to it. -A task's key includes the atoms `reader` and `sniff`, the converter `generic.json`, and the converters (with their -atoms) of the classes the bundle's census lists, so a converter's version bump runs again exactly the bundles holding -its classes. `classes` (a list of class names) limits a task to the objects of those classes and the objects their -hierarchies contain. +### Units + +A unit is what one `unity.export` task converts: a hierarchy (a root GameObject with the GameObjects, Transforms and +components of its prefab: the objects whose census record names it as their `unit`) or an object outside any +hierarchy. Its subject is the stable address of its root object, `:` (in a bundle of +several serialized files the file's tag follows the bundle name, as in `link.addresses`): path ids stay when a +bundle's content changes, so a unit keeps its task across catalog versions. + +A run exports the units holding an object of the selected bundles and every unit reachable from those through what +their objects reference (the census `refs`, across the censuses of the run); with `--only-class` only the units whose +root object is of those classes. The selection is not part of any key: a unit's task is the same whichever request +reaches it. + +A task's key includes the atoms `reader` and `sniff`, the converter `generic.json`, the converters (with their +atoms) of the classes of the unit's objects, its bundle, its root and the script entries and shipped type trees its +objects use, so a converter's version bump runs again exactly the units holding its classes. The tasks of one bundle +are one batch: a worker loads the bundle once for them and keeps what depends on the bundle alone (the index of its +objects, its hierarchies, its atlases and the textures it decoded last) until the batch ends. A unit's result +depends on neither the other units of the batch nor their order. Object JSON is canon/1 in the object's own field order: @@ -139,7 +150,7 @@ hierarchy that plays the clip, which the clip alone does not give. A cue sheet held in a bundle is a MonoBehaviour in one of two layouts; either gives the artifact `#acb` (kind `cri.acb`, facts: `cueSheet`, `layout`), which `cri.audio` decodes: -- a SplitAcbData (fields `_cueSheetName`, `_chunks`): its chunk TextAssets (of the same bundle, exported as well) joined +- a SplitAcbData (fields `_cueSheetName`, `_chunks`): its chunk TextAssets (of the same bundle, units it references) joined in order, every byte XORed with the one mask that gives the ACB signature `@UTF` (layout `split`; its refs name the chunks); - an asset named like the sheet whose `implementation` managed reference is a CriSerializedBytesAssetImpl: its @@ -148,11 +159,11 @@ A cue sheet held in a bundle is a MonoBehaviour in one of two layouts; either gi A movie asset (CriWare.Assets `CriManaUsmAsset`) whose `implementation` is a CriSerializedBytesAssetImpl holds its USM the same way: its `data.data` bytes, whatever their size, are the artifact `#usm` (kind `cri.usm`, facts: -`movie`, the asset's name), which `cri.movie` converts. Converter `cri.usm/1` does this; it is in the key of a bundle -only when the bundle's script context names that class, so no other bundle's key changes with it. +`movie`, the asset's name), which `cri.movie` converts. Converter `cri.usm/1` does this; it is in the key of a unit +only when the scripts of its objects name that class, so no other unit's key changes with it. -`sprite.crop` makes one task per SpriteAtlas that sprites of other bundles use (subject `:`). Its inputs are the atlas' JSON, the `json` of each such sprite and the images of the +`sprite.crop` makes one task per SpriteAtlas that sprites of other bundles use (subject: the `unity.export` +subject of the atlas' unit). Its inputs are the atlas' JSON, the `json` of each such sprite and the images of the atlas textures they use; it writes `#image` with the same bytes `unity.export` writes for a sprite whose atlas is in its own bundle. diff --git a/src/nnnotes/cli/assets.py b/src/nnnotes/cli/assets.py index 6580554..81d2ad4 100644 --- a/src/nnnotes/cli/assets.py +++ b/src/nnnotes/cli/assets.py @@ -104,13 +104,12 @@ def matches(self, stage: str) -> bool: StageSource("unity.census", "nnnotes.pipeline.census:CensusStage"), StageSource("link.scripts", "nnnotes.pipeline.link:ScriptsStage"), StageSource("link.addresses", "nnnotes.pipeline.link:AddressesStage"), - StageSource("unity.export", "nnnotes.pipeline.objexport:ExportStage", output=True, root="_bundles/{subject}"), + StageSource("unity.export", "nnnotes.pipeline.objexport:ExportStage", output=True), StageSource("sprite.crop", "nnnotes.pipeline.objexport:SpriteCropStage", output=True), StageSource("cri.audio", "nnnotes.pipeline.cristages:AudioStage", output=True), StageSource("cri.movie", "nnnotes.pipeline.cristages:MovieStage", output=True), StageSource("live2d.model", "nnnotes.pipeline.modelstages:Live2DStage", output=True), StageSource("spine.skeleton", "nnnotes.pipeline.modelstages:SpineStage", output=True), - StageSource("link.artifacts", "nnnotes.pipeline.link:ArtifactsStage"), StageSource("view.", "nnnotes.pipeline.views:stages", output=True), StageSource("view.voices", "nnnotes.pipeline.voices:VoiceStage", output=True), ) @@ -640,18 +639,6 @@ def params(self, pipeline: list[str]) -> dict: usage(self.args, "--flac-level: no stage of the pipeline encodes FLAC") for n in takers: out.setdefault(n, {})["flac"] = {**stages[n].PARAMS["flac"], "level": level} - only = getattr(self.args, "only_class", None) - if only: - classes = sorted({c.strip() for c in only.split(",") if c.strip()}) - takers = [n for n in pipeline if "classes" in stages[n].PARAMS] - if not classes or not takers: - usage( - self.args, - "--only-class: " - + ("no class given" if not classes else "no stage of the pipeline selects classes"), - ) - for n in takers: - out.setdefault(n, {})["classes"] = classes for n, p in out.items(): try: stages[n].normalize(p) @@ -659,6 +646,16 @@ def params(self, pipeline: list[str]) -> dict: usage(self.args, str(e)) return out + def classes(self) -> list[str] | None: + """The classes of --only-class (the root objects of the units to export), None for every class.""" + only = getattr(self.args, "only_class", None) + if only is None: + return None + classes = sorted({c.strip() for c in only.split(",") if c.strip()}) + if not classes: + usage(self.args, "--only-class: no class given") + return classes + def workers(self) -> int: w = getattr(self.args, "workers", None) return max(0, w) if w is not None else usable_cpus() @@ -695,6 +692,7 @@ def facts(self, pipeline: list[str], fetch: bool, background: bool = False) -> d "master": self.master() if any(n.startswith("view.") for n in pipeline) else None, "views": sorted(n[5:] for n in pipeline if n.startswith("view.")), "bundles": inputs, + "classes": self.classes(), "raw": Inputs({}), } if any(n in pipeline for n in RAW_STAGES): @@ -924,12 +922,12 @@ def layout_entries(self, name: str, placed: LayoutInput, *, store_derived: bool if name != "original": return layout.entries(name, sources) derives = lambda s: hasattr(stages[contract.parse_task_id(s.task)[0]], "derived") # noqa: E731 - roots = {} # every object of a bundle under its export task's root - for s in sources: - if s.task.startswith(EXPORT_STAGE + ":") and s.root is not None: - for f in s.result["files"]: - roots.setdefault(f, s.root) - entries, report = layout.entries(name, [s for s in sources if not derives(s)], case=case, object_roots=roots) + table = self.units() + files = table.files if table is not None else {} + roots = {f: f"_bundles/{b}" for f, b in files.items()} # every object of a bundle under one root + entries, report = layout.entries( + name, [s for s in sources if not derives(s)], case=case, bundles=files, object_roots=roots + ) report["names"] = named extra = self._derived(sources, entries, store_derived, placed.objects) taken = {e["path"].casefold() for e in entries} @@ -943,6 +941,22 @@ def done(self, tasks: dict) -> None: """The tasks whose results the layouts place, {task id: key} (after a run: every task with a result).""" self._done, self._names = dict(tasks), {} + def units(self): + """The unit table of the request (pipeline.units) for the tasks given to done(), a census without a result + counted as failed; None when the run has no censuses.""" + from ..pipeline import units + + env = Env(self.depot, self._facts) + for tid, key in sorted(self._done.items()): + env.done(tid, key) + census = [contract.task_id("unity.census", b) for b in self._facts.get("bundles") or ()] + if not any(t in self._done for t in census): + return None + for t in census: + if t not in self._done: + env.waiting(t, failed=True) + return units.table(env) + def names(self, stage_name: str) -> dict: """{subject: catalog names} of a stage that names its tasks' files (stage.names(env): the CRI stages root a content at each catalog name that refers to it, bundle-held content through the unity.export results; @@ -1014,11 +1028,7 @@ def add(self, tid: str, doc: dict) -> None: if obj: slim["provenance"] = {"object": obj} arts.append(slim) - files = {} - if tid.startswith(EXPORT_STAGE + ":"): - for it in doc["items"]: - files.setdefault(contract.parse_object_id(it["object"])[0], None) - self.results.append((tid, {"artifacts": arts, "files": list(files)})) + self.results.append((tid, {"artifacts": arts})) class _Asked: @@ -1118,23 +1128,19 @@ def parse_layouts(spec: str) -> list[str]: # ---------------------------------------------------------------- export class Totals: """What export reports of the results besides the run's counts, from its one reading of each (Run.scan): the - census totals of `subjects` ({class: objects}, the census artifacts' facts) and the results of the view tasks - (small; their master data tables and gaps).""" + census totals (`classes`: {class: objects} of the requested units, pipeline.units) and the results of the view + tasks (small; their master data tables and gaps).""" - def __init__(self, subjects): - self.want = {contract.task_id("unity.census", s) for s in subjects} - self.classes: Counter = Counter() + def __init__(self, classes: dict | None): + self.classes = None if classes is None else dict(sorted(classes.items())) self.views: list[tuple[str, dict]] = [] def add(self, tid: str, doc: dict) -> None: - if tid in self.want: - for a in doc["artifacts"]: - self.classes.update(a["semantics"].get("facts", {}).get("classes", {})) if tid.startswith("view."): self.views.append((tid, doc)) - def census(self) -> dict[str, int]: - return dict(sorted(self.classes.items())) + def census(self) -> dict[str, int] | None: + return self.classes def view_gaps(results) -> int: @@ -1237,7 +1243,8 @@ def cmd_export(args, cfg): ws.done(done) output = {tid for tid in done if ws._output(contract.parse_task_id(tid)[0])} placed = ws.layout_input({tid: done[tid] for tid in output}) - totals = Totals(sel.selected) + table = ws.units() + totals = Totals(table.classes if table is not None else None) def visit(tid, doc): totals.add(tid, doc) @@ -1531,7 +1538,7 @@ def run_args(c) -> None: help="FLAC compression level 0-12 of the CRI audio (default 8; every level decodes to the same " "samples; changes the files)", ) - c.add_argument("--only-class", metavar="C,...", help="export only objects of these classes") + c.add_argument("--only-class", metavar="C,...", help="export only the units whose root object is of these classes") c.add_argument( "--workers", type=nonnegative, diff --git a/src/nnnotes/pipeline/cristages.py b/src/nnnotes/pipeline/cristages.py index fd7f1b9..077e5ce 100644 --- a/src/nnnotes/pipeline/cristages.py +++ b/src/nnnotes/pipeline/cristages.py @@ -44,7 +44,7 @@ from ..formats.encoders import FLAC_LEVEL from ..unity.atoms import impl_id -from . import builtins, contract +from . import builtins, contract, units from .contract import Cost, IncompatibleTask, Input from .stages import Output, Pending, Stage @@ -306,10 +306,7 @@ def normalize(self, params: dict | None) -> dict: def index(self, env) -> dict: """{content id: {acb, awb, depends, names}} of the raw ACBs and the ACB artifacts of the unity.export results.""" - pending = env.pending(EXPORT) - if pending: - raise Pending(pending[0]) - tids = env.tasks(EXPORT) + tids = _holders(env) raw = _raw(env) cat = self.catalog(env) sig = (tuple((s, i.sha256) for s, (i, _m) in raw.items()), tuple(env.key(t) for t in tids), id(cat)) @@ -426,6 +423,24 @@ def run(self, task, depot) -> Output: return self._item(task, arts) +def _holders(env) -> list[str]: + """The unity.export tasks with a result whose unit is a MonoBehaviour outside any hierarchy (the units that may + hold an ACB or a USM); Pending while one has not run.""" + pending = env.pending(EXPORT) + if pending: + raise Pending(pending[0]) + tids = env.tasks(EXPORT) + if not tids: + return [] + table = units.table(env) + out = [] + for tid in tids: + u = table.units.get(contract.parse_task_id(tid, EXPORT)[1]) + if u is not None and u.cls == "MonoBehaviour": + out.append(tid) + return out + + def _export_artifacts(depot, key: str, kind: str) -> list[dict]: """The artifact records of semantics kind `kind` (cri.acb, cri.usm) of a unity.export result (a result without the kind is not parsed).""" @@ -480,10 +495,7 @@ def normalize(self, params: dict | None) -> dict: def index(self, env) -> dict: """{content id: {usm, depends, names}} of the raw USMs and the USM artifacts of the unity.export results.""" - pending = env.pending(EXPORT) - if pending: - raise Pending(pending[0]) - tids = env.tasks(EXPORT) + tids = _holders(env) raw = _raw(env) cat = self.catalog(env) sig = (tuple((s, i.sha256) for s, (i, _m) in raw.items()), tuple(env.key(t) for t in tids), id(cat)) @@ -499,7 +511,7 @@ def entry(inp: Input) -> dict: e = entry(Input("usm", inp.sha256, inp.size, inp.name, inp.locators)) e["names"] |= cat.keys.get(stable) or {stable} for tid in tids: - bundle = contract.parse_task_id(tid, EXPORT)[1] + bundle = units.table(env).unit(contract.parse_task_id(tid, EXPORT)[1]).bundle for a in _export_artifacts(env.depot, env.key(tid), USM_KIND): c = a["content"] e = entry(Input("usm", c["sha256"], c["size"], None, ({"kind": "depot"},))) diff --git a/src/nnnotes/pipeline/link.py b/src/nnnotes/pipeline/link.py index 88363db..bf41aa9 100644 --- a/src/nnnotes/pipeline/link.py +++ b/src/nnnotes/pipeline/link.py @@ -27,20 +27,10 @@ "|" ("apk/data.unity3d|level0:3"). Objects that no container path names are not listed; `address_of` gives any object's address. -link.artifacts (nnnotes.artifacts/1), one global task: the results of the stages that export objects (unity.export, -sprite.crop) -> where each object went: - objects {object id: [artifact ids]} of the objects with artifacts (exported, generic, unsupported with some) - contained {object id: the artifact it is part of} - artifacts {artifact id: {sha256, size, ext}} -Its context names the results it reads ({"results": {task id: key}}); results are functions of their keys, so the -table is a function of its key. With a layout manifest (its entries' ids) it gives every object's files. The task -reads the results one at a time and writes the table's text entry by entry (encode_artifacts), so its memory is the -table's entries, not the results nor the whole text. """ from __future__ import annotations -import json import re from ..unity import scripttypes @@ -51,8 +41,6 @@ SCRIPTS = "nnnotes.scripts/1" ADDRESSES = "nnnotes.addresses/1" -ARTIFACTS = "nnnotes.artifacts/1" -EXPORT_STAGES = ("unity.export", "sprite.crop") _CAB = re.compile(r"^CAB-[0-9a-f]+") @@ -92,7 +80,9 @@ def file_tags(files) -> dict[str, str]: return tags -def _path_address(stable: str, tags: dict, file: str, pid: int) -> str: +def path_address(stable: str, tags: dict, file: str, pid: int) -> str: + """The path-id stable address of object `pid` of serialized file `file` of bundle `stable` (`tags`: file_tags of + the bundle's files).""" tag = tags.get(file, "|" + file) if tags else "" return contract.stable_address(stable + tag, path_id=pid) @@ -124,7 +114,7 @@ def cls(e): if name and len(same) == 1: out.append((contract.stable_address(stable, path, name=name), e)) else: - out.append((_path_address(stable, tags, e["file"], e["pathId"]), e)) + out.append((path_address(stable, tags, e["file"], e["pathId"]), e)) return out @@ -233,7 +223,7 @@ def address_of(doc: dict, oid: str, container_objects: dict | None = None) -> st stable = doc["files"].get(file) if stable is None: return None - return _path_address(stable, file_tags(doc["bundles"][stable]["files"]), file, pid) + return path_address(stable, file_tags(doc["bundles"][stable]["files"]), file, pid) def reverse(doc: dict) -> dict[str, str]: @@ -343,113 +333,3 @@ def run(self, task, depot) -> Output: {"kind": "link.addresses", "format": "json", "facts": facts}, ) return Output([rec], []) - - -def artifact_table(results) -> tuple[dict, dict, dict]: - """The parts of the artifact table of results [(task id, result)], each result read once and not kept: - ({object id: set of artifact ids}, {object id: the artifact it is part of}, {artifact id: {sha256, size, ext}}), - unsorted. The ids an object is contained in repeat (every part of a prefab names it): one string each.""" - objects, contained, arts, same = {}, {}, {}, {} - for _, doc in results: - for a in doc["artifacts"]: - c = a["content"] - arts[a["id"]] = {"sha256": c["sha256"], "size": c["size"], "ext": c["ext"]} - for it in doc["items"]: - if it.get("artifacts"): - objects.setdefault(it["object"], set()).update(it["artifacts"]) - elif "in" in it: - contained[it["object"]] = same.setdefault(it["in"], it["in"]) - return objects, contained, arts - - -def artifacts(results) -> dict: - """The artifact table (nnnotes.artifacts/1) of results [(task id, result)].""" - objects, contained, arts = artifact_table(results) - return { - "schema": ARTIFACTS, - "objects": {o: sorted(v) for o, v in sorted(objects.items())}, - "contained": dict(sorted(contained.items())), - "artifacts": dict(sorted(arts.items())), - } - - -def encode_artifacts(objects: dict, contained: dict, arts: dict, chunk: int = 1 << 20): - """contract.encode(artifacts(...)) of the parts artifact_table gives, as byte strings of about `chunk` bytes: - the canonical text (keys sorted, indent 1, non-ASCII as is, one trailing newline) written entry by entry, so the - whole document is never built.""" - - def text(v) -> str: - return json.dumps(v, indent=1, ensure_ascii=False, sort_keys=True) - - buf, size = ["{"], 0 - tables = ( - ("artifacts", arts, arts.get), - ("contained", contained, contained.get), - ("objects", objects, lambda k: sorted(objects[k])), - ) - for i, (name, table, value) in enumerate(tables): - buf.append(("\n " if i == 0 else ",\n ") + text(name) + ": ") - if not table: - buf.append("{}") - continue - sep = "{\n " - for k in sorted(table): - s = sep + text(k) + ": " + text(value(k)).replace("\n", "\n ") - buf.append(s) - size += len(s) - sep = ",\n " - if size >= chunk: - yield "".join(buf).encode("utf-8") - buf, size = [], 0 - buf.append("\n }") - buf.append(",\n " + text("schema") + ": " + text(ARTIFACTS) + "\n}\n") - yield "".join(buf).encode("utf-8") - - -class ArtifactsStage(Stage): - """link.artifacts: the results of the export stages (EXPORT_STAGES, those of the pipeline) -> the artifact table. - One subject ("all"); no inputs, context {"results": {task id: key}}. Waits for export tasks that have not run; - skips failed ones. Artifact "link.artifacts:all#artifacts".""" - - name = "link.artifacts" - version = 1 - after = ("unity.export",) - - def subjects(self, env) -> list[str]: - return ["all"] - - def _tasks(self, env) -> list[str]: - for stage in EXPORT_STAGES: - pending = env.pending(stage) - if pending: - raise Pending(pending[0]) - return [t for stage in EXPORT_STAGES for t in env.tasks(stage)] - - def depends(self, subject: str, env) -> list[str]: - return self._tasks(env) - - def context(self, subject: str, env) -> dict: - return {"results": {t: env.key(t) for t in self._tasks(env)}} - - def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: - n = len(self._tasks(env)) - return Cost(0.05 + 0.002 * n, (60 << 20) + 200_000 * n) - - def run(self, task, depot) -> Output: - def results(): - for tid, key in sorted(task.context.get("results", {}).items()): - doc = depot.result(key) - if doc is None: - raise FileNotFoundError(f"result of {tid} ({key}) is not in the depot") - yield tid, doc - - objects, contained, arts = artifact_table(results()) - content = depot.add_chunks(encode_artifacts(objects, contained, arts), "json") - facts = {"objects": len(objects), "contained": len(contained), "artifacts": len(arts)} - rec = contract.artifact( - contract.artifact_id(task.id, "artifacts"), - content, - contract.provenance(task), - {"kind": "link.artifacts", "format": "json", "facts": facts}, - ) - return Output([rec], []) diff --git a/src/nnnotes/pipeline/objexport.py b/src/nnnotes/pipeline/objexport.py index 0c40d9f..1644ff1 100644 --- a/src/nnnotes/pipeline/objexport.py +++ b/src/nnnotes/pipeline/objexport.py @@ -1,29 +1,33 @@ -"""Stages `unity.export` (one bundle -> an artifact per object) and `sprite.crop` (the images of the sprites whose atlas -is in another bundle). - -unity.export reads one bundle alone (atom `reader`) and runs the converters of nnnotes.unity.convert over it, giving -every object exactly one item (contract.item): `exported` (its artifacts), `contained` (part of another artifact: -the objects of a hierarchy in its root's prefab, the MonoScripts in `scripts.json`, the AssetBundle objects in -`bundle.json`), `generic` (the object's typetree JSON, with the reason its converter gave for not making the -specialized form), `unsupported` (a class this stage does not convert; its typetree JSON is written all the same) or -`failed` (not even the typetree JSON could be made). A converter's documented limit (atoms.Unsupported) gives the -typetree JSON with its reason code; any other error of a converter gives it with `generic.error` and the error's -message. The reason codes are listed in docs/stages.md. A bundle's task key includes only the converters of the -classes its census lists. - -A texture is converted before the sprites cut from it (they read its decoded image) and a hierarchy before the -MonoBehaviours outside it; the objects no converter took get their typetree JSON last. - -sprite.crop: a sprite whose SpriteAtlas is in another bundle gets its `meta` from unity.export (with the sprite's -mesh and object facts); one sprite.crop task per atlas then reads the atlas' JSON, the images of its textures and the -metas, and makes "#image" exactly as unity.export makes the image of a sprite whose atlas is in its -own bundle. +"""Stages `unity.export` (one unit of a bundle -> the artifacts of its objects) and `sprite.crop` (the images of the +sprites whose atlas is in another bundle). + +unity.export: one task per unit (pipeline.units: a hierarchy, or an object outside any hierarchy) of a request. A +task reads its bundle alone (atom `reader`) and runs the converters of nnnotes.unity.convert over its unit, giving +every object of the unit exactly one item (contract.item): `exported` (its artifacts), `contained` (an object of a +hierarchy, in its root's prefab), `generic` (the object's typetree JSON, with the reason its converter gave for not +making the specialized form), `unsupported` (a class this stage does not convert; its typetree JSON is written all +the same) or `failed` (not even the typetree JSON could be made). A converter's documented limit +(atoms.Unsupported) gives the typetree JSON with its reason code; any other error of a converter gives it with +`generic.error` and the error's message. The reason codes are listed in docs/stages.md. A task's key holds the +bundle, the unit's root object, the converters of the classes of its objects and the script entries and shipped +type trees they use, so a unit converts again only when one of those changes. + +The tasks of one bundle are one batch (Stage.batch): a worker loads the bundle once for them and keeps, until the +batch ends, what depends on the bundle alone: the index of its objects, its hierarchies, its atlases' render data +and the last textures it decoded. A unit's result depends on neither the other units of its batch nor their order. +A sprite cut from a texture of its bundle reads the texture's decoded image; a hierarchy that cannot be walked +gives each of its objects the typetree JSON with the reason. + +sprite.crop: a sprite whose SpriteAtlas is in another bundle gets its `json` from unity.export (with the sprite's +mesh and object facts) and no image; one sprite.crop task per atlas then reads the atlas' JSON, the images of its +textures and those sprites' documents, and makes "#image" exactly as unity.export makes the image +of a sprite whose atlas is in its own bundle. """ from __future__ import annotations import io -from collections import defaultdict +from collections import OrderedDict, defaultdict from ..unity import scripttypes from ..unity.atoms import ATOMS as ATOM_REGISTRY @@ -48,22 +52,21 @@ ) from ..unity.convert.objects import ( TYPETREE, + BundleIndex, BundleObjects, ReadError, - encode_object, label, message, ref_object_id, ) -from . import builtins, contract, link -from . import census as census_mod +from . import builtins, contract, link, units from .contract import Cost, IncompatibleTask, Input from .stages import Output, Pending, Stage EXPORT, CROP, CENSUS, SCRIPTS = "unity.export", "sprite.crop", "unity.census", "link.scripts" SCRIPTS_TASK = contract.task_id(SCRIPTS, "all") ATOM_NAMES = frozenset(ATOM_REGISTRY) -# class -> (converter, function), in the order they run after the textures and sprites +# class -> (converter, function) of the classes converted by one function alone _SIMPLE = ( ("Mesh", "mesh.glb", files.mesh), ("TextAsset", "data", files.text), @@ -74,26 +77,80 @@ ("AnimatorController", "controller.json", animation.controller), ("AnimatorOverrideController", "controller.json", animation.controller), ) - - -# ---------------------------------------------------------------- one bundle -class BundleRun: - """unity.export on one bundle: the converters over its objects (the task's `classes` only), one item per - object, the artifacts in the depot.""" - - def __init__(self, view, task, depot, fns: dict): - self.task, self.depot = task, depot +DECODED_PIXELS = 1 << 25 # the decoded textures a batch keeps: the last ones, up to this many pixels + + +# ---------------------------------------------------------------- one bundle in a worker +class _Bundle: + """What the tasks of one bundle share in a worker: the bundle's view and BundleIndex, its hierarchies (valid + roots by GameObject key, and the objects of the hierarchies that cannot be walked with the reason), its atlases' + render data and the last decoded textures.""" + + def __init__(self, sha: str, view): + self.sha, self.view = sha, view + self.index = BundleIndex(view) + self.atlases: dict = {} + self._h = None + self.roots: dict[tuple, prefab.Root] = {} + self.broken: dict[tuple, str] = {} + self._decoded: OrderedDict = OrderedDict() + self._pixels = 0 + + def hierarchies(self, ctx: BundleObjects): + if self._h is None: + h = prefab.hierarchies(ctx) + for root in h.roots: + go = (root.file, root.game_object) + info = self.index.info.get(go) + if root.error is None and info is not None and info.class_name == "GameObject": + self.roots[go] = root + for root in h.roots: + if (root.file, root.game_object) in self.roots: + continue + why = root.error or f"transform {root.transform}: no GameObject" + for k in root.members: + if k in self.index.info and k not in h.member: + self.broken.setdefault(k, f"hierarchy at transform {root.transform}: {why}") + self._h = h + return self._h + + def decoded(self, ctx: BundleObjects, info) -> dict: + """A texture's decoded image or the error (textures.decode_into), kept while it is among the last + DECODED_PIXELS pixels decoded.""" + k = (info.file, info.path_id) + holder = self._decoded.get(k) + if holder is not None: + self._decoded.move_to_end(k) + return holder + holder = {} + textures.decode_into(ctx, info, holder) + img = holder.get("image") + self._decoded[k] = holder + self._pixels += img.width * img.height if img is not None else 0 + while self._pixels > DECODED_PIXELS and len(self._decoded) > 1: + _, old = self._decoded.popitem(last=False) + img = old.get("image") + self._pixels -= img.width * img.height if img is not None else 0 + return holder + + +# ---------------------------------------------------------------- one unit +class UnitRun: + """unity.export on one unit: its converters over its objects, one item per object, the artifacts in the + depot.""" + + def __init__(self, bundle: _Bundle, task, depot, fns: dict): + self.task, self.depot, self.bundle = task, depot, bundle self.ctx = BundleObjects( - view, + bundle.view, fns, scripts=task.context.get("scripts", {}), blob_min=int(task.params["blobMin"]), png_level=int(task.params["png"]["level"]), emit=self._emit, typetrees=task.context.get("typetrees", {}), + index=bundle.index, ) - classes = task.params.get("classes") - self.selected = None if classes is None else frozenset(classes) self.usm_assets = USM_CONVERTER in task.atoms self.items: dict[str, dict] = {} self.artifacts: list[dict] = [] @@ -175,8 +232,6 @@ def _need(self, conv: str) -> None: def _convert(self, info, conv: str, fn, *args, **kw) -> bool: """Run a converter on one object: its artifacts (exported), else the typetree JSON with the reason.""" - if info.id in self.items: - return False self._need(conv) self._begin(conv) try: @@ -213,124 +268,61 @@ def _generic(self, info) -> None: else: self._commit(info, "generic", contract.reason(TYPETREE, label(why))) - def _selected(self, cls: str) -> bool: - return self.selected is None or cls in self.selected - - def _of(self, cls: str) -> list: - """The objects of `cls` this task converts (none when the task's `classes` leave the class out).""" - return self.ctx.of(cls) if self._selected(cls) else [] - - # ---------------------------------------------------------------- the bundle + # ---------------------------------------------------------------- the unit def run(self) -> tuple[list, list]: - self._textures_sprites() - for cls, conv, fn in _SIMPLE: - for i in self._of(cls): - self._convert(i, conv, fn) - self._prefabs() - for i in self._of("MonoBehaviour"): - self._convert(i, "mono.json", mono.mono, usm_assets=self.usm_assets) - for conv, (cls, kind) in generic.LOOSE.items(): - self._loose(cls, conv, kind) - for i in self.ctx.infos: - if i.id in self.items or not self._selected(i.class_name): - continue - if i.class_name in UNSUPPORTED_CLASSES: - self._fallback( - i, f"unsupported.class.{i.class_name}", f"{i.class_name} is not converted", status="unsupported" - ) - else: - self._generic(i) + key = contract.parse_object_id(self.task.context["unit"]) + info = self.ctx.info.get(key) + if info is None: + raise IncompatibleTask(f"task {self.task.id}: object {self.task.context['unit']} is not in the bundle") + cls = info.class_name + self.bundle.hierarchies(self.ctx) + simple = {c: (conv, fn) for c, conv, fn in _SIMPLE} + if key in self.bundle.broken: + self._fallback(info, "generic.error", self.bundle.broken[key]) + elif key in self.bundle.roots: + self._prefab(info, self.bundle.roots[key]) + elif cls == "Sprite": + self._sprite(info) + elif cls == "Texture2D": + self._convert(info, "tex.png", textures.texture, self.bundle.decoded(self.ctx, info)) + elif cls in simple: + self._convert(info, *simple[cls]) + elif cls == "MonoBehaviour": + self._convert(info, "mono.json", mono.mono, usm_assets=self.usm_assets) + elif cls in UNSUPPORTED_CLASSES: + self._fallback(info, f"unsupported.class.{cls}", f"{cls} is not converted", status="unsupported") + else: + self._generic(info) return self.artifacts, list(self.items.values()) - def _textures_sprites(self) -> None: - waiting, atlases = defaultdict(list), {} - for s in self._of("Sprite"): - self._need("sprite.png") - self._begin("sprite.png") - try: - plan = textures.sprite_plan(self.ctx, (s.file, s.path_id), atlases) - except Unsupported as e: - self._fallback(s, e.code, e.message) - continue - except Exception as e: - self._fallback(s, "generic.error", message(e)) - continue - if plan["external"]: - self._convert(s, "sprite.png", textures.sprite_external, plan, CROP) - else: - waiting[plan["texture"]].append((s, plan)) - export_textures = self._selected("Texture2D") - for t in self.ctx.of("Texture2D"): - sprites = waiting.pop((t.file, t.path_id), ()) - if not sprites and not export_textures: - continue - holder: dict = {} - if export_textures: - self._convert(t, "tex.png", textures.texture, holder) - else: - textures.decode_into(self.ctx, t, holder) - for s, plan in sprites: - self._convert(s, "sprite.png", textures.sprite, plan, holder) - holder.clear() - for tkey, rest in waiting.items(): - for s, _plan in rest: - self._fallback(s, "no_data.sprite", f"texture {contract.object_id(*tkey)} is not decoded") - - def _prefabs(self) -> None: - if not self._selected("GameObject"): + def _sprite(self, info) -> None: + self._need("sprite.png") + self._begin("sprite.png") + try: + plan = textures.sprite_plan(self.ctx, (info.file, info.path_id), self.bundle.atlases) + except Unsupported as e: + self._fallback(info, e.code, e.message) return - h = prefab.hierarchies(self.ctx) - info_of = self.ctx.info - for root in h.roots: - info = info_of.get((root.file, root.game_object)) - if root.error is not None or info is None or info.class_name != "GameObject": - why = root.error or f"transform {root.transform}: no GameObject" - for k in root.members: - if k in info_of and info_of[k].id not in self.items: - self._fallback(info_of[k], "generic.error", f"hierarchy at transform {root.transform}: {why}") - continue - if self._convert(info, "prefab.json", prefab.prefab, h, root): - within = contract.artifact_id(info.id, "json") - for k in root.members: - i = info_of[k] - if i.id != info.id: - self.items[i.id] = contract.item(i.id, "contained", within=within, cls=i.class_name) - else: - why = self.items[info.id].get("reason") or {"code": "generic.error", "message": ""} - for k in root.members: - i = info_of[k] - if i.id not in self.items: - self._fallback(i, why["code"], f"hierarchy of {info.id}: {why['message']}") - - def _loose(self, cls: str, conv: str, kind: str) -> None: - """Objects of `cls` in one task-level artifact "#" (contained in it).""" - infos = [i for i in self._of(cls) if i.id not in self.items] - if not infos: + except Exception as e: + self._fallback(info, "generic.error", message(e)) return - self._need(conv) - self._begin(conv) - entries, done, failed = {}, [], [] - for i in infos: - mark = len(self.staged) - key = (i.file, i.path_id) - self.ctx.seen.clear() - try: - entries[i.id] = self.ctx.tree(self.ctx.readers[key], self.ctx.tt(key)) - done.append((i, self._refs(i))) - except Exception as e: - del self.staged[mark:] - failed.append((i, e)) - if done: - doc = encode_object(generic.loose_doc(self.ctx, conv, entries)) - aid = self._emit(None, conv, doc, "json", kind, "json", {"objects": len(done)}, owner=self.task.id)[0] - self._publish() - for i, refs in done: - self.items[i.id] = contract.item(i.id, "contained", within=aid, cls=i.class_name, refs=refs) - for i, e in failed: - if isinstance(e, ReadError): - self._failed(i, e) - else: - self._fallback(i, "generic.error", message(e)) + if plan["external"]: + self._convert(info, "sprite.png", textures.sprite_external, plan, CROP) + else: + holder = self.bundle.decoded(self.ctx, self.ctx.info[plan["texture"]]) + self._convert(info, "sprite.png", textures.sprite, plan, holder) + + def _prefab(self, info, root: prefab.Root) -> None: + h = self.bundle.hierarchies(self.ctx) + members = [self.ctx.info[k] for k in root.members if k != (info.file, info.path_id)] + if self._convert(info, "prefab.json", prefab.prefab, h, root): + within = contract.artifact_id(info.id, "json") + for i in members: + self.items[i.id] = contract.item(i.id, "contained", within=within, cls=i.class_name) + return + why = self.items[info.id].get("reason") or {"code": "generic.error", "message": ""} + for i in members: + self._fallback(i, why["code"], f"hierarchy of {info.id}: {why['message']}") # ---------------------------------------------------------------- stages @@ -361,7 +353,7 @@ def _check_atoms(task, reader: Impl | None = None) -> None: class _Docs: - """A few JSON documents of the depot by content id (the planning reads each census several times).""" + """A few JSON documents of the depot by content id (the planning reads some several times).""" def __init__(self, size: int = 64): self.size, self.docs = size, {} @@ -377,29 +369,27 @@ def get(self, depot, sha: str) -> dict: class ExportStage(Stage): - """unity.export: one bundle -> an artifact per object, one item per object (see the module documentation). - - Subjects: the selected bundles (the fact "selected", stable bundle names) and, when a selected bundle holds - sprites, the bundles of the fact "bundles" (the selection's closure) that hold a SpriteAtlas: a sprite reaches its - atlas texture through the atlas' bundle, whose export makes the texture's image (sprite.crop reads it). Inputs: - "bundles" {subject: Input (role "bundle")}, as unity.census. It reads the census of its bundle - (unity.census:) for the classes (the converters and atoms of its key) and the MonoScripts its objects - reference, and the script table (link.scripts:all) for the entries of those: its context - {"scripts": {object id: "assembly|namespace|class"}}, and for the MonoBehaviours of files that store no type trees - (the APK's built-in files) the type trees nnnotes ships for their type hashes: "typetrees" {type hash: nodes}. - Parameters: `blobMin` (bytes; byte arrays of this size or - more are blobs), `png.level` (zlib level of the PNG files). `reader`: another implementation of the reader atom - (its id enters the keys).""" + """unity.export: one unit -> the artifacts of its objects, one item per object (see the module documentation). + + Subjects: the units of the request (pipeline.units.table: the facts "selected", "bundles" and "classes", and + the censuses of the bundles). Inputs: the unit's bundle ("bundles" {stable name: Input (role "bundle")}, as + unity.census). It reads the script table (link.scripts:all) for the entries of the MonoScripts its objects + name. Context: "unit" (the root object's id), "bundle" (the stable bundle name), "scripts" {object id: + "assembly|namespace|class"} and, for the MonoBehaviours of files that store no type trees (the APK's built-in + files), "typetrees" {type hash: nodes}: the type trees nnnotes ships for their type hashes. Parameters: + `blobMin` (bytes; byte arrays of this size or more are blobs), `png.level` (zlib level of the PNG files). + `reader`: another implementation of the reader atom (its id enters the keys). Batch: the bundle's content id.""" name = EXPORT - version = 1 + version = 2 after = (CENSUS, SCRIPTS) - PARAMS = {"blobMin": 4096, "classes": None, "png": {"level": 6}} - LAYOUT_ROOT = "_bundles/{subject}" + PARAMS = {"blobMin": 4096, "png": {"level": 6}} def __init__(self, reader: Impl | None = None): self.reader = reader self._docs = _Docs() + self._atoms_memo: dict[tuple, dict] = {} + self._loaded: _Bundle | None = None def __getstate__(self): return {"reader": self.reader} @@ -426,113 +416,104 @@ def normalize(self, params: dict | None) -> dict: blob = p.get("blobMin", 4096) if not isinstance(blob, int) or isinstance(blob, bool) or blob < 1: raise ValueError(f"stage {self.name}: blobMin {blob!r}: expected a positive integer") - classes = p.get("classes") - if classes is not None: - if isinstance(classes, str) or not all(isinstance(c, str) and c for c in classes): - raise ValueError(f"stage {self.name}: classes {classes!r}: expected a list of class names") - classes = sorted(set(classes)) - return {"blobMin": blob, "classes": classes, "png": _png_params(p.get("png"))} + return {"blobMin": blob, "png": _png_params(p.get("png"))} def subjects(self, env) -> list[str]: - selected = set(env.fact("selected")) - out = set(selected) - if any("Sprite" in _census_classes(env, s) for s in sorted(selected)): - out.update(s for s in env.fact("bundles") if s not in out and "SpriteAtlas" in _census_classes(env, s)) - return sorted(out) + return units.table(env).subjects() + + def unit(self, subject: str, env) -> units.Unit: + return units.table(env).unit(subject) def inputs(self, subject: str, env) -> list[Input]: - inp = env.fact("bundles")[subject] + u = self.unit(subject, env) + inp = env.fact("bundles")[u.bundle] if inp.role != "bundle": raise ValueError(f"{self.name} {subject}: input role {inp.role!r}, expected 'bundle'") return [inp] - def census(self, subject: str, env) -> dict: - inp = env.input_of(contract.task_id(CENSUS, subject), "census") - return self._docs.get(env.depot, inp.sha256) - def uses(self, subject: str, env) -> dict: - """The atoms and converters of the classes the bundle's census lists; with the task's `classes` (env.params - while the task is described), of those of them only; converter cri.usm when a MonoBehaviour of the bundle is - a movie asset (its script context names the class).""" - classes = list(self.census(subject, env)["classes"]) - only = (getattr(env, "params", None) or {}).get("classes") - if only is not None: - keep = set(only) - classes = [c for c in classes if c in keep] - atoms = export_atoms(classes, self.reader_id()) - if "MonoBehaviour" in classes and USM_ASSET_SCRIPT in self.context(subject, env).get("scripts", {}).values(): - atoms = dict(sorted({**atoms, USM_CONVERTER: converter_id(USM_CONVERTER)}.items())) + """The atoms and converters of the classes of the unit's objects; converter cri.usm when one of them is a + movie asset (its script context names the class).""" + u = self.unit(subject, env) + usm = ( + "MonoBehaviour" in u.classes and USM_ASSET_SCRIPT in self.context(subject, env).get("scripts", {}).values() + ) + k = (u.classes, usm) + atoms = self._atoms_memo.get(k) + if atoms is None: + atoms = export_atoms(u.classes, self.reader_id()) + if usm: + atoms = dict(sorted({**atoms, USM_CONVERTER: converter_id(USM_CONVERTER)}.items())) + self._atoms_memo[k] = atoms return atoms def context(self, subject: str, env) -> dict: - doc = self.census(subject, env) - out = {} - refs = census_mod.script_refs(doc) - if refs: + u = self.unit(subject, env) + out: dict = {"unit": u.root, "bundle": u.bundle} + if u.scripts: table = self._docs.get(env.depot, env.input_of(SCRIPTS_TASK, "scripts").sha256)["scripts"] - out["scripts"] = link.script_context(table, refs) - types = _script_types(doc) - if types: - out["typetrees"] = types + scripts = link.script_context(table, u.scripts) + if scripts: + out["scripts"] = scripts + trees = {} + for version, h in u.trees: + nodes = scripttypes.script_types(version).get(h) + if nodes is not None: + trees[h] = nodes + if trees: + out["typetrees"] = dict(sorted(trees.items())) return out def depends(self, subject: str, env) -> list[str]: - return [contract.task_id(CENSUS, subject), SCRIPTS_TASK] + return [contract.task_id(CENSUS, self.unit(subject, env).bundle), SCRIPTS_TASK] def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: - """From the bundle's size, its census' object count and the size of its resource files (least squares over - a whole catalog: CPU seconds; peak resident bytes of a worker process, which grow with the objects).""" + """CPU: the unit's share of loading its bundle (from the bundle's size, among the requested units of the + bundle) and its own (its objects and their bytes); peak: the worker's for the whole bundle (from its size + and object count), which the tasks of its batch hold together.""" + u = self.unit(subject, env) + t = units.table(env) size = sum(i.size for i in inputs) - try: - doc = self.census(subject, env) - n = int(doc["objects"]) - res = sum(int(r.get("size") or 0) for r in doc.get("resources") or ()) - except Exception: - n = res = 0 - return Cost(0.12 + size * 3.3e-7 + n * 8.8e-5 + res * 6.6e-8, (80 << 20) + 80 * size + 17500 * n) + n = t.objects.get(u.bundle, 0) + load = 0.12 + size * 3.3e-7 + cpu = load / max(1, t.per_bundle[u.bundle]) + len(u.members) * 8.8e-5 + u.size * 6.6e-8 + return Cost(cpu, (80 << 20) + 80 * size + 17500 * n) + + def batch(self, task) -> str: + return task.input("bundle").sha256 + + def end_batch(self, batch: str) -> None: + if self._loaded is not None and self._loaded.sha == batch: + self._loaded = None def run(self, task, depot) -> Output: reader = self.reader if self.reader is not None else resolve("reader") _check_atoms(task, reader) - view = reader.fn(depot.input_bytes(task.input("bundle")), builtins.serialized_name(task.subject)) + inp = task.input("bundle") + if self._loaded is None or self._loaded.sha != inp.sha256: + self._loaded = None + name = builtins.serialized_name(task.context["bundle"]) + self._loaded = _Bundle(inp.sha256, reader.fn(depot.input_bytes(inp), name)) fns = {n: resolve(n).fn for n in task.atoms if n in ATOM_NAMES and n != "reader"} - arts, items = BundleRun(view, task, depot, fns).run() + arts, items = UnitRun(self._loaded, task, depot, fns).run() return Output(arts, items) -def _script_types(doc: dict) -> dict: - """{type hash: nodes} of the embedded type trees (nnnotes.unity.scripttypes, of each file's Unity version) of the - type hashes of the MonoBehaviours in the census' files that store no type trees ({} when it has none).""" - out = {} - for f in doc["files"]: - if f["typetree"]: - continue - types = scripttypes.script_types(f["unityVersion"] or "") - for o in f["objects"]: - h = o["typeHash"] if o["class"] == "MonoBehaviour" else None - if h in types: - out[h] = types[h] - return dict(sorted(out.items())) - - -def _census_classes(env, subject: str) -> dict: - """{class: objects} of a bundle's census ({} when its census has no result).""" - try: - return env.artifact(contract.task_id(CENSUS, subject), "census")["semantics"]["facts"]["classes"] - except (Pending, KeyError): - return {} - - def _store_input(role: str, record: dict) -> Input: c = record["content"] return Input(role, c["sha256"], c["size"], None, ({"kind": "depot"},)) +def _artifact(env, tid: str, aid: str) -> dict | None: + return next((a for a in env.result(tid)["artifacts"] if a["id"] == aid), None) + + class SpriteCropStage(Stage): - """sprite.crop: the images of the sprites whose SpriteAtlas is in another bundle. One task per atlas, subject - ":"; inputs: "atlas" (the atlas' JSON), "sprite:" (each sprite's meta), "texture:" (the image of each atlas texture the sprites use), all artifacts - of unity.export results. Waits for unity.export tasks that have not run. Artifacts "#image".""" + """sprite.crop: the images of the sprites whose SpriteAtlas is in another bundle. One task per atlas, subject the + unity.export subject of the atlas' unit; inputs: "atlas" (the atlas' JSON), "sprite:" (each sprite's + JSON), "texture:" (the image of each atlas texture the sprites use), all artifacts of unity.export + results. The sprites are those of the unity.export tasks of Sprite units whose JSON names an atlas of another + bundle; waits for unity.export tasks that have not run. Artifacts "#image".""" name = CROP version = 1 @@ -561,51 +542,40 @@ def normalize(self, params: dict | None) -> dict: raise ValueError(f"stage {self.name}: unknown parameters {', '.join(unknown)}") return {"png": _png_params(p.get("png"))} - @staticmethod - def _may_hold(env, tid: str) -> bool: - subject = contract.parse_task_id(tid, EXPORT)[1] - try: - classes = env.artifact(contract.task_id(CENSUS, subject), "census")["semantics"]["facts"]["classes"] - except Exception: - return True - return "Sprite" in classes or "SpriteAtlas" in classes - def index(self, env) -> dict: - """{subjects: {subject: (atlas object id, [(sprite object id, export task id, meta record)])}, atlases, - textures, missing: atlas object ids that sprites name but no result holds}.""" + """{subjects: {atlas subject: (atlas object id, [(sprite object id, export task id, its JSON record)])}, + missing: atlas object ids that sprites name but no unity.export result holds}.""" pending = env.pending(EXPORT) if pending: raise Pending(pending[0]) - tids = env.tasks(EXPORT) - sig = tuple(env.key(t) for t in tids) + table = units.table(env) + tids = [] + for t in env.tasks(EXPORT): + u = table.units.get(contract.parse_task_id(t, EXPORT)[1]) + if u is not None and u.cls == "Sprite": + tids.append(t) + sig = (id(table), tuple(env.key(t) for t in tids)) if self._memo is not None and self._memo[0] == sig: return self._memo[1] - atlases, textures, requests = {}, {}, defaultdict(list) + requests = defaultdict(list) for tid in tids: - if not self._may_hold(env, tid): - continue for a in env.result(tid)["artifacts"]: - owner, role = contract.parse_artifact_id(a["id"]) - sem, obj = a["semantics"], a["provenance"].get("object") or {} - facts = sem.get("facts") or {} - if sem["kind"] == "sprite.meta" and facts.get("crop") == CROP: - requests[facts["atlas"]].append((owner, tid, a)) - elif role == "json" and obj.get("class") == "SpriteAtlas": - atlases[owner] = (tid, a) - elif role == "image" and sem["kind"] == "texture.image": - textures[owner] = (tid, a) - subjects = {} - for atlas, reqs in requests.items(): - if atlas in atlases: - tid = atlases[atlas][0] - subject = f"{contract.parse_task_id(tid, EXPORT)[1]}:{contract.parse_object_id(atlas)[1]}" - subjects[subject] = (atlas, sorted(reqs, key=lambda r: r[0])) - idx = { - "subjects": subjects, - "atlases": atlases, - "textures": textures, - "missing": sorted(set(requests) - set(atlases)), - } + facts = a["semantics"].get("facts") or {} + if a["semantics"]["kind"] == "sprite.meta" and facts.get("crop") == CROP: + requests[facts["atlas"]].append((contract.parse_artifact_id(a["id"])[0], tid, a)) + subjects, missing = {}, [] + for atlas, reqs in sorted(requests.items()): + subject = table.subject_of(atlas) + if subject is None or not env.known(contract.task_id(EXPORT, subject)): + missing.append(atlas) + continue + try: + env.key(contract.task_id(EXPORT, subject)) + except Pending: + missing.append(atlas) + continue + subjects[subject] = (atlas, sorted(reqs, key=lambda r: r[0])) + idx = {"subjects": subjects, "missing": missing} self._memo = (sig, idx) return idx @@ -616,31 +586,53 @@ def _subject(self, subject: str, env): idx = self.index(env) if subject not in idx["subjects"]: raise ValueError(f"{self.name}: no sprites wait for atlas {subject}") - return idx, *idx["subjects"][subject] - - def inputs(self, subject: str, env) -> list[Input]: - idx, atlas, reqs = self._subject(subject, env) - rec = idx["atlases"][atlas][1] - ins = [_store_input("atlas", rec)] + return idx["subjects"][subject] + + def _textures(self, subject: str, env) -> tuple[dict, list]: + """(the atlas' JSON record, [(texture object id, the export task holding it)]) of the textures the + sprites use.""" + atlas, reqs = self._subject(subject, env) + rec = _artifact(env, contract.task_id(EXPORT, subject), contract.artifact_id(atlas, "json")) + if rec is None: + raise ValueError(f"{self.name} {subject}: the atlas {atlas} has no JSON") entries = textures.atlas_entries(self._docs.get(env.depot, rec["content"]["sha256"])) needed = set() - for owner, _tid, meta in reqs: - ins.append(_store_input("sprite:" + owner, meta)) + for _owner, _tid, meta in reqs: doc = self._docs.get(env.depot, meta["content"]["sha256"]) entry = textures.atlas_entry(entries, doc["sprite"].get("m_RenderDataKey")) if entry is not None: needed.update( t for t in (ref_object_id(entry.get("texture")), ref_object_id(entry.get("alphaTexture"))) if t ) - ins += [_store_input("texture:" + t, idx["textures"][t][1]) for t in sorted(needed) if t in idx["textures"]] + table = units.table(env) + out = [] + for t in sorted(needed): + s = table.subject_of(t) + if s is not None and env.known(contract.task_id(EXPORT, s)): + out.append((t, contract.task_id(EXPORT, s))) + return rec, out + + def inputs(self, subject: str, env) -> list[Input]: + _atlas, reqs = self._subject(subject, env) + rec, texs = self._textures(subject, env) + ins = [_store_input("atlas", rec)] + ins += [_store_input("sprite:" + owner, meta) for owner, _tid, meta in reqs] + for t, tid in texs: + try: + img = _artifact(env, tid, contract.artifact_id(t, "image")) + except Pending: + continue + if img is not None and img["semantics"]["kind"] == "texture.image": + ins.append(_store_input("texture:" + t, img)) return ins def uses(self, subject: str, env) -> dict: return dict(self.ATOMS) def depends(self, subject: str, env) -> list[str]: - idx, atlas, reqs = self._subject(subject, env) - return sorted({idx["atlases"][atlas][0], *(tid for _o, tid, _m in reqs)}) + _atlas, reqs = self._subject(subject, env) + _rec, texs = self._textures(subject, env) + return sorted({contract.task_id(EXPORT, subject), *(tid for _o, tid, _m in reqs), *(tid for _t, tid in texs)}) def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: tex = sum(i.size for i in inputs if i.role.startswith("texture:")) diff --git a/src/nnnotes/pipeline/units.py b/src/nnnotes/pipeline/units.py new file mode 100644 index 0000000..bfce293 --- /dev/null +++ b/src/nnnotes/pipeline/units.py @@ -0,0 +1,198 @@ +"""The units of unity.export, from the censuses: what one export task converts, and which of them a request needs. + +A unit is a hierarchy (a root GameObject with the GameObjects, Transforms and components its prefab holds: the +objects whose census record names it as their `unit`) or an object outside any hierarchy. Its subject is the +path-id stable address of its root object (link.path_address: ":", with the file after the bundle in +a bundle of several serialized files), which stays when the bundle's content changes; its object ids do not. + +The request (`table`): every unit holding an object of the selected bundles (fact "selected"), and every unit +reachable from those through what their objects reference (census `refs`) across the censuses of the bundles of +the fact "bundles"; with the fact "classes" (a list of class names; absent or None: every class), only the units +whose root object is of those classes. A bundle whose census failed has no units; a reference into a file no +census holds is left out. +""" + +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass + +from . import contract, link +from .census import CensusStage +from .stages import Pending + +CENSUS = CensusStage.name + + +@dataclass(frozen=True) +class Unit: + """One unit: its subject, stable bundle name, root object id and class, member object ids (the root first), + ((class, objects)) of its members by class, the MonoScripts they name (m_Script and an AnimationClip's binding + scripts), the (Unity version, type hash) of its MonoBehaviours in files without type trees, and the byte size of + its members.""" + + subject: str + bundle: str + root: str + cls: str + members: tuple + counts: tuple + scripts: tuple + trees: tuple + size: int + + @property + def classes(self) -> tuple: + return tuple(c for c, _ in self.counts) + + +class Units: + """The units of a request: `units` {subject: Unit} of the requested units; `of` {object id: subject} of the + objects of every unit of the censuses read; `files` {serialized file: stable bundle name}; `objects` {stable + bundle name: its objects}; `per_bundle` the number of requested units per bundle; `classes` the objects of the + requested units per class.""" + + def __init__(self, units: dict, of: dict, files: dict, objects: dict): + self.units, self.of, self.files, self.objects = units, of, files, objects + self.per_bundle = Counter(u.bundle for u in units.values()) + self.classes: Counter = Counter() + for u in units.values(): + for c, n in u.counts: + self.classes[c] += n + + def subjects(self) -> list[str]: + return sorted(self.units) + + def unit(self, subject: str) -> Unit: + try: + return self.units[subject] + except KeyError: + raise ValueError(f"unity.export: no requested unit {subject}") from None + + def subject_of(self, oid: str) -> str | None: + """The subject of the unit holding object `oid` (None when no census read holds it).""" + return self.of.get(oid) + + +def _ref(r: dict | None) -> str | None: + """The object id of a census reference record (None for null or an unknown file index).""" + if not r or "file" not in r: + return None + return contract.object_id(r["file"], r["pathId"]) + + +def _census_units(stable: str, doc: dict, need_refs: bool): + """(units, {object id: subject}, {file: stable}, {subject: referenced object ids}) of one census.""" + tags = link.file_tags([f["name"] for f in doc["files"]]) + groups: dict[str, list] = {} # root object id -> [(file record, object record)], the root first + for f in doc["files"]: + for o in f["objects"]: + oid = contract.object_id(f["name"], o["pathId"]) + root = o.get("unit", oid) + entry = (f, o) + if root == oid: + groups.setdefault(root, []).insert(0, entry) + else: + groups.setdefault(root, []).append(entry) + units, of, refs = {}, {}, {} + for root, members in groups.items(): + rf, ro = members[0] + if contract.object_id(rf["name"], ro["pathId"]) != root: + continue # members of a root the census does not list + subject = link.path_address(stable, tags, rf["name"], ro["pathId"]) + ids, classes, scripts, trees, size, out = [], Counter(), set(), set(), 0, set() + for f, o in members: + oid = contract.object_id(f["name"], o["pathId"]) + ids.append(oid) + of[oid] = subject + classes[o["class"]] += 1 + size += int(o.get("byteSize") or 0) + for s in [o.get("script"), *(o.get("bindingScripts") or ())]: + r = _ref(s) + if r is not None: + scripts.add(r) + if o["class"] == "MonoBehaviour" and not f["typetree"] and o.get("typeHash"): + trees.add((f["unityVersion"] or "", o["typeHash"])) + if need_refs: + out.update(o.get("refs") or ()) + units[subject] = Unit( + subject, + stable, + root, + ro["class"], + tuple(ids), + tuple(sorted(classes.items())), + tuple(sorted(scripts)), + tuple(sorted(trees)), + size, + ) + if need_refs: + refs[subject] = out.difference(ids) + return units, of, {f["name"]: stable for f in doc["files"]}, refs + + +def build(censuses, bundles, selected, classes=None) -> Units: + """The Units of a request over `censuses` [(stable bundle name, census document)] of the bundles `bundles` + (module documentation).""" + selected = set(selected) + every = selected >= set(bundles) # no reference can add a unit: none is read + units: dict[str, Unit] = {} + of: dict[str, str] = {} + files: dict[str, str] = {} + refs: dict[str, set] = {} + objects: dict[str, int] = {} + for stable, doc in censuses: + objects[stable] = int(doc["objects"]) + u, o, f, r = _census_units(stable, doc, need_refs=not every) + units.update(u) + of.update(o) + files.update(f) + refs.update(r) + if every: + want = set(units) + else: + want = {s for s, u in units.items() if u.bundle in selected} + todo = list(want) + while todo: + for oid in refs.get(todo.pop(), ()): + s = of.get(oid) + if s is not None and s not in want: + want.add(s) + todo.append(s) + if classes is not None: + keep = set(classes) + want = {s for s in want if units[s].cls in keep} + return Units({s: units[s] for s in sorted(want)}, of, files, objects) + + +_MEMO: list = [] # [(signature, Units)] of the last table + + +def table(env) -> Units: + """The Units of the request of a planning environment: the censuses of the fact "bundles" (Pending while one + has not run; a failed one is skipped), the facts "selected" and "classes". Kept for the planning state it was + made in.""" + bundles = sorted(env.fact("bundles")) + keys = [] + for b in bundles: + tid = contract.task_id(CENSUS, b) + try: + keys.append((b, env.key(tid))) + except Pending as p: + if not p.failed: + raise + classes = env.facts.get("classes") + sig = (id(env.depot), tuple(keys), tuple(env.fact("selected")), None if classes is None else tuple(classes)) + if _MEMO and _MEMO[0][0] == sig: + return _MEMO[0][1] + + def censuses(): + for b, key in keys: + tid = contract.task_id(CENSUS, b) + aid = contract.artifact_id(tid, "census") + rec = next(a for a in env.depot.committed(key, tid)["artifacts"] if a["id"] == aid) + yield b, contract.loads(env.depot.read(rec["content"]["sha256"])) + + t = build(censuses(), bundles, env.fact("selected"), classes) + _MEMO[:] = [(sig, t)] + return t diff --git a/src/nnnotes/pipeline/voices.py b/src/nnnotes/pipeline/voices.py index 52eefa9..147b060 100644 --- a/src/nnnotes/pipeline/voices.py +++ b/src/nnnotes/pipeline/voices.py @@ -62,7 +62,7 @@ from ..content import advcommand, storyvoices from ..formats import acb from ..source import languages -from . import contract, views +from . import contract, units, views from .contract import Cost, Input from .stages import Output, Pending, Stage @@ -820,16 +820,15 @@ def _sheet_name(name: str) -> str | None: return name if "/" not in name else None -_address_cache: dict = {} # content id of an address table -> (AddressIndex, files), the latest one +_address_cache: dict = {} # content id of an address table -> its AddressIndex, the latest one -def _address_parts(env, tid: str) -> tuple: - """(AddressIndex, {serialized file: stable bundle name}) of the result of the link.addresses task `tid`.""" +def _address_index(env, tid: str): + """The AddressIndex of the result of the link.addresses task `tid`.""" sha = env.artifact(tid, "addresses")["content"]["sha256"] hit = _address_cache.get(sha) if hit is None: - doc = contract.loads(env.depot.read(sha)) - hit = (views.AddressIndex.from_addresses(doc), dict(doc.get("files") or {})) + hit = views.AddressIndex.from_addresses(contract.loads(env.depot.read(sha))) _address_cache.clear() _address_cache[sha] = hit return hit @@ -930,7 +929,7 @@ def _plan(self, subject: str, env) -> tuple: ) if self._memo is not None and self._memo[0] == sig: return self._memo[1] - index, files = _address_parts(env, link) + index = _address_index(env, link) tables = { i.role[len(views.MASTER_ROLE) :]: views._cached_rows(i.sha256, Path(i.locators[0]["path"]).read_bytes) for i in masters @@ -945,15 +944,18 @@ def _plan(self, subject: str, env) -> tuple: } sheets = {s: v for s, v in found.items() if s in wanted} used = {v["acb"] for v in sheets.values()} - episodes, contents, read = self._episodes(env, rec, tables, files, set(exports)) + episodes, contents, read = self._episodes(env, rec, tables, set(exports)) plan = (masters, rec.subset(), sheets, [acbs[a] for a in sorted(acbs) if a in used], episodes, contents, read) self._memo = (sig, plan) return plan - def _episodes(self, env, index, tables, files: dict, exports: set) -> tuple[dict, list[Input], list[str]]: + def _episodes(self, env, index, tables, exports: set) -> tuple[dict, list[Input], list[str]]: """The story parts the unity.export results hold: the object a part's key names (the first of its class, - STORY_OBJECTS) in the bundle of its serialized file (the address table's `files`), and the artifact with - its content. ({asset: {part: {object, artifact, sha256}}}, the content inputs, the tasks read.)""" + STORY_OBJECTS) in the task of its unit (pipeline.units), and the artifact with its content. ({asset: + {part: {object, artifact, sha256}}}, the content inputs, the tasks read.)""" + if not exports: + return {}, [], [] + table = units.table(env) arts: dict[str, dict] = {} out: dict = {} contents: dict[str, Input] = {} @@ -965,8 +967,8 @@ def _episodes(self, env, index, tables, files: dict, exports: set) -> tuple[dict if not pick: continue oid = pick[0].id - stable = files.get(contract.parse_object_id(oid)[0]) - tid = contract.task_id(EXPORT_STAGE, stable) if stable is not None else None + subject = table.subject_of(oid) + tid = contract.task_id(EXPORT_STAGE, subject) if subject is not None else None if tid is None or tid not in exports: continue if tid not in arts: diff --git a/src/nnnotes/schemas/artifacts.schema.json b/src/nnnotes/schemas/artifacts.schema.json deleted file mode 100644 index c8ce2d3..0000000 --- a/src/nnnotes/schemas/artifacts.schema.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "urn:nnnotes:schema:artifacts", - "title": "nnnotes.artifacts/1", - "description": "The artifact table (stage link.artifacts): the artifact ids of every object with artifacts, the artifact each contained object is part of, and per artifact its content id, size and extension. With a layout manifest it gives every object's files.", - "type": "object", - "required": [ - "schema", - "objects", - "contained", - "artifacts" - ], - "properties": { - "schema": { - "const": "nnnotes.artifacts/1" - }, - "objects": { - "type": "object", - "additionalProperties": { - "type": "array", - "items": { - "$ref": "urn:nnnotes:schema:common#/$defs/artifactId" - }, - "minItems": 1 - } - }, - "contained": { - "type": "object", - "additionalProperties": { - "$ref": "urn:nnnotes:schema:common#/$defs/artifactId" - } - }, - "artifacts": { - "type": "object", - "additionalProperties": { - "type": "object", - "required": [ - "sha256", - "size", - "ext" - ], - "properties": { - "sha256": { - "$ref": "urn:nnnotes:schema:common#/$defs/sha256" - }, - "size": { - "type": "integer", - "minimum": 0 - }, - "ext": { - "type": "string" - } - } - } - } - } -} diff --git a/src/nnnotes/unity/convert/__init__.py b/src/nnnotes/unity/convert/__init__.py index ce85c0c..e0cb8f7 100644 --- a/src/nnnotes/unity/convert/__init__.py +++ b/src/nnnotes/unity/convert/__init__.py @@ -2,8 +2,8 @@ A converter reads one bundle alone through its `BundleObjects` (objects.py) and writes its artifacts through the `emit` function the caller gives it; the caller (the unity.export stage) keeps the items, the depot and the -provenance. Each converter carries a version (CONVERTERS); a task's key includes the converters of the classes its -bundle holds, with the atoms they use (CONVERTER_ATOMS). +provenance. Each converter carries a version (CONVERTERS); a task's key includes the converters of the classes of the +objects it converts, with the atoms they use (CONVERTER_ATOMS). Texture2D tex.png `image`: PNG of the first mip level (HDR ASTC: the .astc file) Sprite sprite.png `image`: PNG on the sprite's rect; `json`: typetree, geometry, render data @@ -21,9 +21,7 @@ also `usm`: its USM (kind cri.usm) TextAsset data `data`: the stored bytes Font font `font`: the font file - MonoScript scripts.json "#scripts.json": every MonoScript of the bundle - AssetBundle bundle.json "#bundle.json": serialized files, externals, AssetBundle objects - any other class generic.json `json`: the object's typetree + any other class generic.json `json`: the object's typetree (a MonoScript, an AssetBundle, ...) Every object a converter takes has a `json` artifact (or is contained in one): the converter's document, or the typetree JSON (a Texture2D, Mesh, TextAsset or Font without the bytes its other artifact holds). @@ -77,8 +75,6 @@ "mono.json": 2, "data": 2, "font": 2, - "scripts.json": 1, - "bundle.json": 1, "shader.json": 1, "generic.json": 2, "cri.usm": 1, @@ -96,8 +92,6 @@ "MonoBehaviour": ("mono.json",), "TextAsset": ("data",), "Font": ("font",), - "MonoScript": ("scripts.json",), - "AssetBundle": ("bundle.json",), "Shader": ("shader.json",), } ALWAYS = ("generic.json",) @@ -134,7 +128,7 @@ def converter_id(name: str) -> str: def converters_for(classes) -> list[str]: - """The converters a bundle holding objects of `classes` may use.""" + """The converters objects of `classes` may use.""" out: set[str] = set(ALWAYS) for c in classes: out.update(CLASS_CONVERTERS.get(c, ())) @@ -142,7 +136,7 @@ def converters_for(classes) -> list[str]: def export_atoms(classes, reader_id: str | None = None) -> dict: - """The atoms and converters a bundle holding `classes` uses (the `atoms` of its unity.export task).""" + """The atoms and converters objects of `classes` use (the `atoms` of a unity.export task).""" convs = converters_for(classes) names = set(BASE_ATOMS) for c in convs: diff --git a/src/nnnotes/unity/convert/generic.py b/src/nnnotes/unity/convert/generic.py index c69e4b2..dfa9715 100644 --- a/src/nnnotes/unity/convert/generic.py +++ b/src/nnnotes/unity/convert/generic.py @@ -1,14 +1,10 @@ -"""Objects without a converter of their own (generic.json: the typetree JSON), and the bundle's AssetBundle objects -(bundle.json) and MonoScripts (scripts.json), which go into one document each.""" +"""Objects without a converter of their own (generic.json: the typetree JSON).""" from __future__ import annotations from .objects import BundleObjects, encode_object -__all__ = ["LOOSE", "generic", "loose_doc"] - -# converter of the objects that go into one document -> (class, kind of the document) -LOOSE = {"bundle.json": ("AssetBundle", "unity.bundle"), "scripts.json": ("MonoScript", "unity.scripts")} +__all__ = ["generic"] def generic(ctx: BundleObjects, info) -> str | None: @@ -18,15 +14,3 @@ def generic(ctx: BundleObjects, info) -> str | None: doc = ctx.tree(ctx.readers[key], ctx.tt(key, whole=False)) ctx.emit(key, "json", encode_object(doc), "json", "unity.object", "json", {}) return ctx.partial.get(key) - - -def loose_doc(ctx: BundleObjects, conv: str, entries: dict) -> dict: - """The document of converter `conv` (LOOSE) for `entries` ({object id: typetree JSON}): bundle.json also lists - the bundle's serialized files with their Unity versions and externals.""" - if conv == "bundle.json": - files = [ - {"name": f, "unityVersion": ctx.versions.get(f), "externals": list(ctx.view.externals(f))} - for f in ctx.files - ] - return {"files": files, "assetBundles": entries} - return {"scripts": entries} diff --git a/src/nnnotes/unity/convert/objects.py b/src/nnnotes/unity/convert/objects.py index d6383cf..ec167a7 100644 --- a/src/nnnotes/unity/convert/objects.py +++ b/src/nnnotes/unity/convert/objects.py @@ -34,6 +34,7 @@ "NO_SCRIPT", "PREFIXES", "TYPETREE", + "BundleIndex", "BundleObjects", "Capture", "ReadError", @@ -188,11 +189,37 @@ class Capture: facts: dict = field(default_factory=dict) +class BundleIndex: + """What the converters read of a bundle that depends on its bytes alone (`view`, a reader atom's BundleView): + its objects and their readers, files, externals, Unity versions and container paths, with memos of names and + type tree nodes. The BundleObjects of the tasks of one bundle share it.""" + + def __init__(self, view): + self.view = view + self.infos = view.objects() + self.info = {(i.file, i.path_id): i for i in self.infos} + self.readers = {k: view.reader(i) for k, i in self.info.items()} + self.files = view.files() + self.ext_files = {f: [_file_name(p) for p in view.externals(f)] for f in self.files} + self.versions = {f: view.unity_version(f) for f in self.files} + self.stripped = frozenset(f for f in self.files if not view.stores_typetrees(f)) + self.containers: dict[tuple, str] = {} + for path, file, pid in view.container(): + k = (file, int(pid)) + if k not in self.containers or path < self.containers[k]: + self.containers[k] = path + self.names: dict[tuple, str | None] = {} + self.special_memo: dict[int, tuple] = {} + self.children_memo: dict[int, tuple] = {} + self.heads: dict[int, tuple] = {} + + class BundleObjects: - """The objects of one bundle (`view`, a reader atom's BundleView). `fns`: the atom functions by name; `scripts`: - the script table entries the bundle's objects use ({MonoScript object id: "assembly|namespace|class"}); - `typetrees`: the type trees of script classes for the files that store none ({type hash: nodes}); `blob_min`: - byte arrays of this size or more are blobs; `png_level`: the zlib level of PNG files.""" + """The objects of one bundle (`view`, a reader atom's BundleView; `index`: its BundleIndex when one is shared). + `fns`: the atom functions by name; `scripts`: the script table entries the bundle's objects use ({MonoScript + object id: "assembly|namespace|class"}); `typetrees`: the type trees of script classes for the files that store + none ({type hash: nodes}); `blob_min`: byte arrays of this size or more are blobs; `png_level`: the zlib level of + PNG files.""" def __init__( self, @@ -204,30 +231,22 @@ def __init__( png_level: int, emit: Emit, typetrees: dict | None = None, + index: BundleIndex | None = None, ): + idx = index if index is not None else BundleIndex(view) + if idx.view is not view: + raise ValueError("the index is of another view") self.view, self.fns, self.emit = view, fns, emit self.scripts, self.blob_min, self.png_level = dict(scripts), int(blob_min), int(png_level) self.typetrees = dict(typetrees or {}) - self.infos = view.objects() - self.info = {(i.file, i.path_id): i for i in self.infos} - self.readers = {k: view.reader(i) for k, i in self.info.items()} - self.files = view.files() - self.ext_files = {f: [_file_name(p) for p in view.externals(f)] for f in self.files} - self.versions = {f: view.unity_version(f) for f in self.files} - self.stripped = frozenset(f for f in self.files if not view.stores_typetrees(f)) + self.infos, self.info, self.readers = idx.infos, idx.info, idx.readers + self.files, self.ext_files, self.versions, self.stripped = idx.files, idx.ext_files, idx.versions, idx.stripped + self.containers, self.names = idx.containers, idx.names + self._special_memo, self._children_memo, self._heads = idx.special_memo, idx.children_memo, idx.heads self.partial: dict[tuple, str] = {} # objects read in part -> why self._nodes: dict[tuple, object] = {} # objects read with a type tree of `typetrees` -> its node self._script_nodes: dict[str, object] = {} # type hash -> node - self.containers: dict[tuple, str] = {} - for path, file, pid in view.container(): - k = (file, int(pid)) - if k not in self.containers or path < self.containers[k]: - self.containers[k] = path - self.names: dict[tuple, str | None] = {} self._mono_scripts: dict[tuple, str | None] = {} - self._special_memo: dict[int, tuple] = {} - self._children_memo: dict[int, tuple] = {} - self._heads: dict[int, tuple] = {} self._captures: dict[str, Capture] = {} self._local: Callable[[tuple], dict | None] | None = None self.seen: set[str] = set() diff --git a/src/nnnotes/unity/convert/textures.py b/src/nnnotes/unity/convert/textures.py index ed05194..79763ba 100644 --- a/src/nnnotes/unity/convert/textures.py +++ b/src/nnnotes/unity/convert/textures.py @@ -1,7 +1,8 @@ """Texture2D (tex.png) and Sprite (sprite.png). -A texture's image is decoded once per bundle: `texture` (or `decode_into`, when the task leaves textures out) puts it -in a holder that the sprites cut from it read. Where a sprite's image comes from is its plan (`sprite_plan`). +A texture's image is decoded by `decode_into` into a holder, which `texture` and the sprites cut from the texture read, +so the caller decides how long a decoded image is kept. Where a sprite's image comes from is its plan +(`sprite_plan`). A sprite's `json` holds its typetree JSON (`sprite`), its decoded geometry (`mesh`: positions in the sprite's units, triangles per submesh, texture coordinates when stored; a texture coordinate of a position p is @@ -48,18 +49,14 @@ def _canon(v) -> str: return json.dumps(v, sort_keys=True, ensure_ascii=True) -def texture(ctx: BundleObjects, info, holder: dict) -> None: +def texture(ctx: BundleObjects, info, decoded: dict) -> None: """`image`: PNG of the first mip level (HDR ASTC: the .astc file); `json`: the typetree JSON without the pixels - (`image data`). The decoded image (or the error) goes into `holder` for the texture's sprites.""" + (`image data`). `decoded`: the texture's decoded image or the error, as decode_into gives them.""" from ..atoms.texture import ASTC_BLOCKS, format_name, is_hdr_astc key = (info.file, info.path_id) - try: - tt = ctx.tt(key) - inp = ctx.view.texture_input(info) - except Exception as e: - holder["error"] = e - raise + tt = ctx.tt(key) + inp = ctx.view.texture_input(info) fmt, w, h = int(inp.format), int(inp.width), int(inp.height) facts = { "width": w, @@ -69,19 +66,13 @@ def texture(ctx: BundleObjects, info, holder: dict) -> None: "exportedMips": [0], } if w and h and is_hdr_astc(fmt): - holder["error"] = Unsupported( - "unsupported.texture.format", f"{format_name(fmt)} (HDR ASTC; the texture is exported as .astc)" - ) data = ctx.fns["astc.container"](inp.data, w, h, ASTC_BLOCKS[fmt]) ctx.emit(key, "image", data, "astc", "texture.image", "astc", facts) _texture_json(ctx, key, tt) return - try: - img = ctx.fns["texture.decode"](inp) - except Exception as e: - holder["error"] = e - raise - holder["image"] = img + if "error" in decoded: + raise decoded["error"] + img = decoded["image"] facts["mode"] = img.mode data = ctx.fns["png.encode"](img, ctx.png_level) ctx.emit(key, "image", data, "png", "texture.image", "png", facts) @@ -94,7 +85,7 @@ def _texture_json(ctx: BundleObjects, key: tuple, tt: dict) -> None: def decode_into(ctx: BundleObjects, info, holder: dict) -> None: - """The texture's image for its sprites only (a task whose classes leave Texture2D out).""" + """The texture's decoded image into `holder` ("image"), or the error ("error"; HDR ASTC is not decoded).""" from ..atoms.texture import format_name, is_hdr_astc try: diff --git a/tests/cli/test_assets.py b/tests/cli/test_assets.py index 85388f5..9a1eca6 100644 --- a/tests/cli/test_assets.py +++ b/tests/cli/test_assets.py @@ -16,7 +16,7 @@ from nnnotes import cli from nnnotes.cli import assets, common from nnnotes.cli import catalog as catalog_cli -from nnnotes.pipeline import contract +from nnnotes.pipeline import contract, units from nnnotes.pipeline.census import CensusStage from nnnotes.pipeline.contract import Cost from nnnotes.pipeline.depot import Depot @@ -34,12 +34,14 @@ # ---------------------------------------------------------------- synthetic data -def obj(pid, cls, name=None, script=None): +def obj(pid, cls, name=None, script=None, unit=None): o = {"pathId": pid, "classId": 0, "class": cls, "byteSize": 10, "typeHash": None} if name is not None: o["name"] = name if script is not None: o["script"] = script + if unit is not None: + o["unit"] = unit return o @@ -82,7 +84,8 @@ def cdoc(file: str, objs: list, container=(), scripts=(), bundle_name=None) -> d def censuses(broken: bool = True) -> dict: """{bundle: census document}: two cards (a texture with a sub-sprite, a script-driven MonoBehaviour), the - shared bundle holding the MonoScript, a background bundle with an object that fails (`broken`).""" + shared bundle holding the MonoScript, a background bundle with a prefab (a GameObject and its Transform) and an + object that fails (`broken`).""" script = {"file": CABS["shared"], "pathId": 42} out = {} for n in ("card_1", "card_2"): @@ -113,7 +116,8 @@ def censuses(broken: bool = True) -> dict: ) out["bg_x"] = cdoc( CABS["bg_x"], - [obj(1, "AssetBundle", "ab"), obj(3, "Mesh", "floor")] + ([obj(4, "Broken", "bad")] if broken else []), + [obj(1, "AssetBundle", "ab"), obj(2, "Transform", "", unit=f"{CABS['bg_x']}:3"), obj(3, "GameObject", "x")] + + ([obj(4, "Broken", "bad")] if broken else []), [("assets/game/bg/x.prefab", 3)], ) return out @@ -189,26 +193,34 @@ def run(self, task, depot): class FakeExport(Stage): - """unity.export: one JSON artifact per object of the census (MonoBehaviours name their script class from the - link.scripts table), the AssetBundle object contained in the bundle's first artifact, class `Broken` failed.""" + """unity.export: one task per unit (pipeline.units), one JSON artifact for its root object (a MonoBehaviour + names its script class from the link.scripts table) that contains its other members, class `Broken` failed, + class `Crash` a ConfigError.""" name = "unity.export" version = 1 after = ("unity.census", "link.scripts") - PARAMS = {"png": {"encoder": "pillow", "level": 6}, "classes": None} + PARAMS = {"png": {"encoder": "pillow", "level": 6}} ATOMS = {"png.encode": "fake/1"} def subjects(self, env): - return list(env.fact("selected")) + return units.table(env).subjects() - def _census(self, subject): - return contract.task_id("unity.census", subject) + def _census(self, subject, env): + return contract.task_id("unity.census", units.table(env).unit(subject).bundle) def depends(self, subject, env): - return [self._census(subject), "link.scripts:all"] + return [self._census(subject, env), "link.scripts:all"] def inputs(self, subject, env): - return [env.input_of(self._census(subject), "census"), env.input_of("link.scripts:all", "scripts")] + return [env.input_of(self._census(subject, env), "census"), env.input_of("link.scripts:all", "scripts")] + + def context(self, subject, env): + unit = units.table(env).unit(subject) + return {"unit": unit.root, "members": list(unit.members)} + + def batch(self, task): + return task.input("census").sha256 def estimate(self, subject, env, inputs): return Cost(0.01, 1 << 20) @@ -216,55 +228,31 @@ def estimate(self, subject, env, inputs): def run(self, task, depot): doc = contract.loads(depot.input_bytes(task.input("census"))) scripts = contract.loads(depot.input_bytes(task.input("scripts")))["scripts"] - if any(o["class"] == "Crash" for f in doc["files"] for o in f["objects"]): - raise ConfigError("setting paths.tool is not set") container = {(e["file"], e["pathId"]): e["path"] for b in doc["assetBundles"] for e in b["container"]} - only = task.params["classes"] - arts, items, first = [], [], None - for f in doc["files"]: - for o in f["objects"]: - oid = contract.object_id(f["name"], o["pathId"]) - if only is not None and o["class"] not in only: - continue - if o["class"] == "AssetBundle": - continue - if o["class"] == "Broken": - items.append( - contract.item(oid, "failed", why=contract.reason("fake.broken", "broken"), cls=o["class"]) - ) - continue - body = {"object": oid, "class": o["class"], "level": task.params["png"]["level"]} - if o.get("script"): - body["script"] = scripts.get(contract.object_id(o["script"]["file"], o["script"]["pathId"])) - facts = { - "file": f["name"], - "pathId": o["pathId"], - "classId": 0, - "class": o["class"], - "name": o.get("name", ""), - } - if (f["name"], o["pathId"]) in container: - facts["container"] = container[(f["name"], o["pathId"])] - aid = contract.artifact_id(oid, "json") - arts.append( - contract.artifact( - aid, - depot.add(contract.encode(body), "json"), - contract.provenance(task, obj=facts), - {"kind": "fake.json"}, - ) - ) - items.append(contract.item(oid, "exported", artifacts=[aid], cls=o["class"])) - first = first or aid - for f in doc["files"]: - for o in f["objects"]: - if o["class"] == "AssetBundle" and first and (only is None or "AssetBundle" in only): - items.append( - contract.item( - contract.object_id(f["name"], o["pathId"]), "contained", within=first, cls="AssetBundle" - ) - ) - return Output(arts, items) + file, pid = contract.parse_object_id(task.context["unit"]) + (f,) = [f for f in doc["files"] if f["name"] == file] + (o,) = [o for o in f["objects"] if o["pathId"] == pid] + oid = task.context["unit"] + if o["class"] == "Crash": + raise ConfigError("setting paths.tool is not set") + if o["class"] == "Broken": + return Output( + [], [contract.item(oid, "failed", why=contract.reason("fake.broken", "broken"), cls="Broken")] + ) + body = {"object": oid, "class": o["class"], "level": task.params["png"]["level"]} + if o.get("script"): + body["script"] = scripts.get(contract.object_id(o["script"]["file"], o["script"]["pathId"])) + facts = {"file": f["name"], "pathId": o["pathId"], "classId": 0, "class": o["class"], "name": o.get("name", "")} + if (f["name"], o["pathId"]) in container: + facts["container"] = container[(f["name"], o["pathId"])] + aid = contract.artifact_id(oid, "json") + art = contract.artifact( + aid, depot.add(contract.encode(body), "json"), contract.provenance(task, obj=facts), {"kind": "fake.json"} + ) + items = [contract.item(oid, "exported", artifacts=[aid], cls=o["class"])] + cls = {contract.object_id(g["name"], m["pathId"]): m["class"] for g in doc["files"] for m in g["objects"]} + items += [contract.item(m, "contained", within=aid, cls=cls[m]) for m in task.context["members"][1:]] + return Output([art], items) REAL = {s.name: s for s in assets.STAGE_SOURCES} @@ -273,8 +261,7 @@ def run(self, task, depot): assets.StageSource("unity.census", "tests.cli.test_assets:FakeCensus"), REAL["link.scripts"], REAL["link.addresses"], - assets.StageSource("unity.export", "tests.cli.test_assets:FakeExport", output=True, root="_bundles/{subject}"), - REAL["link.artifacts"], + assets.StageSource("unity.export", "tests.cli.test_assets:FakeExport", output=True), ) @@ -370,7 +357,7 @@ def test_the_built_in_files_of_the_apk_are_exported(tmp_path, capsys, fake, monk base = [*flags(d), "--apk", apk, "depot", "plan", "--depot", tmp_path / "s", "--json"] code, out, err = nn(capsys, *base, "--select", "group:bg") ids = {n["id"] for n in json.loads(out)["nodes"]} - assert "unity.census:apk/data.unity3d" in ids and "unity.export:apk/data.unity3d" not in ids + assert "unity.census:apk/data.unity3d" in ids and not any(i.startswith("unity.export:apk/") for i in ids) code, out, err = nn( capsys, *flags(d), @@ -409,7 +396,7 @@ def test_export_writes_layouts_reports_and_hits_the_second_time(tmp_path, capsys code, out, err = export(capsys, tmp_path, d, "depot", "out") assert code == 1, err # the broken object is a failed item summary = json.loads(out) - assert summary["items"] == {"exported": 8, "contained": 4, "generic": 0, "unsupported": 0, "failed": 1} + assert summary["items"] == {"exported": 12, "contained": 1, "generic": 0, "unsupported": 0, "failed": 1} files = tree(tmp_path / "out") assert "Assets/Game/Card/1/full.png.json" in files # the catalog's case of the container path assert "Assets/Game/Card/1/full[card1].png.json" in files # the sub-sprite @@ -422,15 +409,16 @@ def test_export_writes_layouts_reports_and_hits_the_second_time(tmp_path, capsys assert [(f["code"], f["object"]) for f in fails] == [("fake.broken", f"{CABS['bg_x']}:4")] cov = contract.loads((reports / "coverage.json").read_bytes())["classes"] assert cov["Texture2D"]["total"] == 2 and cov["Texture2D"]["exported"] == 2 - assert cov["AssetBundle"] == { - "total": 4, + assert cov["Transform"] == { + "total": 1, "missing": 0, "exported": 0, - "contained": 4, + "contained": 1, "generic": 0, "unsupported": 0, "failed": 0, } + assert cov["AssetBundle"]["total"] == cov["AssetBundle"]["exported"] == 4 run = contract.loads((reports / "run.json").read_bytes()) assert Depot(tmp_path / "depot").run(run["id"]) == run assert run["context"]["catalog"]["sha"] == contract.sha256(d["catalog"].read_bytes()) @@ -518,7 +506,7 @@ def test_plan_emit_tasks_and_run_stage_give_the_exported_depot(tmp_path, capsys, code, out, err = nn(capsys, *flags(d), "depot", "run", tasks, "--depot", tmp_path / "q") assert code in (0, 1), err rounds += 1 - assert rounds == 4 # index + census; scripts + addresses; export; artifacts + assert rounds == 3 # index + census; scripts + addresses; export skip = ["inputs", "runs", "costs", "catalogs", "downloads"] assert tree(tmp_path / "orch", skip) == tree(tmp_path / "q", skip) @@ -857,7 +845,8 @@ def test_derived_documents_read_the_paths_of_the_objects_they_name(tmp_path, cap card = f"{CABS['card_1']}:5" assert doc["CAB-none:9"] == {} and doc[f"{CABS['bg_x']}:4"] == {} # unknown; failed assert doc[card] == {f"{card}#json": "Assets/Game/Card/1/full.png.json"} - assert doc[f"{CABS['card_1']}:1"] == doc[card] # contained in it + prefab = f"{CABS['bg_x']}:3" + assert doc[f"{CABS['bg_x']}:2"] == doc[prefab] == {f"{prefab}#json": "Assets/Game/Bg/x.prefab.json"} # contained assert (tmp_path / "o" / "Assets" / "Game" / "Card" / "1" / "full.png.json").is_file() code, out, _ = nn(capsys, *flags(d), "depot", "plan", "-o", tmp_path / "o", "--depot", tmp_path / "s", "--check") assert code == 0, out @@ -928,7 +917,6 @@ def test_views_are_placed_with_paths_and_every_document_validates(tmp_path, caps contract.VIEW, "nnnotes.scripts/1", "nnnotes.addresses/1", - "nnnotes.artifacts/1", }, checked code, out, _ = nn( capsys, @@ -1028,7 +1016,7 @@ def test_export_records_the_calibration_and_plan_uses_it(tmp_path, capsys, fake) d = setup_data(tmp_path) export(capsys, tmp_path, d, "s", "o") cal = tmp_path / "s" / "costs" / assets.CALIBRATION - assert contract.loads(cal.read_bytes()) == {"stages": {}} # four bundles: too few + assert contract.loads(cal.read_bytes())["stages"]["unity.export"]["samples"] == 13 # one per unit run assets.write_calibration(tmp_path / "s", {"unity.export": {"cpu": 2.0, "peak": 3.0, "samples": 50}}) code, out, err = nn( capsys, *flags(d), "depot", "plan", "--depot", tmp_path / "s", "--json", "--png-level", "1", "--explain" @@ -1208,7 +1196,7 @@ def test_names_see_the_tasks_that_ran(tmp_path, capsys, monkeypatch): code, out, err = export(capsys, tmp_path, d, "s", "o") assert code == 0, err placed = tree(tmp_path / "o") - assert "Named/4/a.flac" in placed and "Named/0/a.flac" not in placed # four bundles were exported + assert "Named/12/a.flac" in placed and "Named/0/a.flac" not in placed # the twelve units were exported code, out, _ = nn(capsys, *flags(d), "depot", "plan", "-o", tmp_path / "o", "--depot", tmp_path / "s", "--check") assert code == 0, out # plan places them the same way diff --git a/tests/golden/unity.export.json b/tests/golden/unity.export.json index 434f76a..de4fcb2 100644 --- a/tests/golden/unity.export.json +++ b/tests/golden/unity.export.json @@ -1,7 +1,6 @@ { "atoms": { "astc.container": "nnnotes/1", - "bundle.json": "bundle.json/1", "clip.json": "clip.json/2", "controller.json": "controller.json/2", "cri.usm": "cri.usm/1", @@ -16,7 +15,6 @@ "png.encode": "pillow-12.3.0/1", "prefab.json": "prefab.json/2", "reader": "fakeunity/1", - "scripts.json": "scripts.json/1", "shader.json": "shader.json/1", "sniff": "nnnotes/1", "sprite.crop": "pillow-12.3.0+numpy-2.4.6/1", @@ -25,69 +23,57 @@ "texture.decode": "unitypy-1.25.3+texture2ddecoder-1.0.6+astc-encoder-py-0.1.12+pillow-12.3.0/1" }, "fixtures": { - "all": { - "key": "bfa791aaeaa8744197413dca21b95285355376a687dcb4a5b511410b4761bd28", - "result": "d9f811e5ca222529d22d1c5e2ec9cd89328d0229eac92b7e01d0cb59dbc56077" - }, - "bundle.json": { - "key": "019bd61d3b8989c2f59289354d72d3857ed37cab1d90316359e6702ebab8a515", - "result": "f1516133579ff990af232a7b6d8b6303104c59997d20905f16aa280403d3a09e" - }, "clip.json": { - "key": "a9f5fb2a87fbcb197a5bc37f1c520b7b5a4a6fd5b5ad38b1198e9f8601dc223b", - "result": "6448026702f2239970f8041906335b3a2302a4463536d2f0394daa36906d9c79" + "key": "c2211c68d3286636469c74023637e15b3c11dade0f858ab2380498aa319171eb", + "result": "1f6ad672b184cd8b4cf9eeeb9ee0b8512ba532eaf708cb36b86c538b19910b75" }, "controller.json": { - "key": "472e6426b3af184e4fb17ccc0a147bd7b6fc9e5fc49f32d0ab2c2d79238417e1", - "result": "c43f0d2de335f10063d20f2d208b54cf7da37863860441a22781bce35f0cb1cc" + "key": "eaae782443b5874108271db45e6a0da404f91239f700dd827802081ba64dd184", + "result": "e63d57f287d5d388e1d6d2269bc03ffbf03165b11cb03a19d2ad003fb00cacf8" }, "cri.usm": { - "key": "72cba01cb4fe8fa95d052a2fd17380eaacdd722dc0e5fc802bce8a301560e7a0", - "result": "9fecf00c65642c4ebb635839f79d9001cf4308b367711c2acaf82fa53326cff5" + "key": "c69fabc7e24dd038b2375b59a0421cd514a8ea5e5dc3a21c181dda809902dc3f", + "result": "d829f22ea9a736a478f2963470f2bdca4d6a3200e40a510606956fe771d63e00" }, "data": { - "key": "1b44691876cbf5e28032ac2858d9616a8445e2357c543291e2806506c8c0a0b4", - "result": "cdaf30aeb760eef66ec41e029940a800af4d72b0ca0c4545c9c911b6c94fd331" + "key": "eec1daf7f06a873c434b64b5ab2d0629b61a71c331f16ed5434706b60ee3bee3", + "result": "493c173754a5a0088bc972fc8c373415d02b345060ab83df9ce41f8b0d0317f8" }, "font": { - "key": "7d5549079bee9a0778d9d698545f21bfd0fea5466e0efa160661cfb527a83ea9", - "result": "d58abbc2a1b9990fadca993ac0502dc2d586b79e4947bfd1b6fb0a9f7bbc41b0" + "key": "81155f5c29ca33162b4b0ee97d7559fb0460ad02c46a187097bb6e4893792b7c", + "result": "d0da497a3f6207d7e92e31ac4313f289eb483b5fcf0496476a3fb50e235b830a" }, "generic.json": { - "key": "22deb0da7906ad444ca282fb2a8f993106268f748f2fab35357c049e5b020410", - "result": "a41f325a63c894bfe54aba013c1ee7aa6d674fddd49715c5e3898b7f40e6559c" + "key": "76af5e1c7fd2b389a80c4725aa2b561ffb0ae552b0d65ce2eb4ad56710ae3201", + "result": "6d6d058a293360ee2065fbb205f35834198e7285a743f2e5b1dc847daa4cdad3" }, "material.json": { - "key": "b8b07211d9dbc578b3541a0e59653b01506a211426cc70b6e0aa2e82a948167e", - "result": "09085e48baebbda4a78a076e5dc66eeb19ae1f53710ec9f9b3553bb86dfdefde" + "key": "c391d41f8afdc880c74e24f143560ff80c122c97f82241cd2b4f0cdf8f5fad1f", + "result": "0990d8b5ccdc04c6335b2779354b3d64727ebccc701490ffe6a46346719e90e2" }, "mesh.glb": { - "key": "1c9b9a873c782e16650ef40cc0a41c17d37da55e05da6d44f47c488cc12d5027", - "result": "58304a1fd371109dbc38f99cd578ad8238453e769ae8c9d8562e1e187e2df1bc" + "key": "1e5a4fee78d3fd98334ce50a6b0b1cc56f5733252f2cb00ff974d515a362b911", + "result": "11949923ee05d67811b42f530d84313e9197afd74e203da44291761b8aa87994" }, "mono.json": { - "key": "f2109d87be61d8cf5b0b527326d2cdc474785104091c55e770a18b31cd967b23", - "result": "02bc065f80513aad7e9afc21704d0f9591e0f110c9ff6fd818f357fe139686f8" + "key": "0aeed465e4e3f8a310b4a2bd3cc06fa899fcdd5fb0ae04d3a5c2a5ccd85d09ea", + "result": "9711bc6d921e7c49ef0b86b487808c12b95e106bb791655a880fef9257ec6da3" }, "prefab.json": { - "key": "583e77337b68385457bb1f7ec789c943d69729706adc18b47d59354bd04fcf94", - "result": "3751c24516ea904f342e272234e57357b4c5d03ef6e90637c5c8bffbbca78d2a" - }, - "scripts.json": { - "key": "6e6a37fb9135463601b78a8a5f699996221242585bcc31c63863158ca0a3f9dd", - "result": "9e9b03c721f1f163ed6e81c202b767a7af84be95e662d4d3b5a9b8d46bfe3392" + "key": "0c801dd911545b249a8b629b3dfa05c7ea6c466482d7f93252f126717419dc94", + "result": "96507449031bd9d046088f01b64602d7191aa87d7cf5af29ecde677c051d20fe" }, "shader.json": { - "key": "f3c2c279ab2ed59a70932b19111c4f248593ec3ca9e0677b9402c38cb8505d03", - "result": "dac4f3d4f225dd3659ac78ab0b344daaa64e1a0a6f7dfe759f3ab49ffe3b3c43" + "key": "105e05e73efa131a339c80a936f0e8dc44a2cebe3c25d332f500b51871dfe83e", + "result": "0f670ee8b5f4ceb584329ecd3a4695d70dd5bf0b9690b4df63b87963fb83913b" }, "sprite.png": { - "key": "d35fe9220f41f10d16b8506f48f31774e9303a8ce6e7b2deba3eb2b26edc56d9", - "result": "ebe1ea68a2fb3289366624724ec4a9335ff6e42156ce4003c7669106e3981482" + "key": "ab28c0f55b23cb58aaf5b94f4357231ab1f4be8ce16c6a02e69bfc1917a3509c", + "result": "82cf3a23fe661f24f5e9fc3189e2b70db39c287e39392397556e2b081beb4c2c" }, "tex.png": { - "key": "1408f5b984cbcc1c74d23e3c6ef518288f2d112b6b2e4007aecb7f811ddfc809", - "result": "fec1a912d37045add008ea5c12941ea9c6e9eeb0cc0adb6b4d5be81c89ac26eb" + "key": "f5b96a51a6de3073767d863f7fbd7f835df241386588ae3fddbfa7eae4d02c5a", + "result": "ea26cd3f966ff9ebb9c7491f72e720aa311d5552f5ef0b136d9712958dac6ac7" } }, "libraries": { @@ -99,5 +85,5 @@ }, "provider": "tests.pipeline.test_objexport:export_golden", "stage": "unity.export", - "version": 1 + "version": 2 } diff --git a/tests/helpers/units.py b/tests/helpers/units.py new file mode 100644 index 0000000..bf36993 --- /dev/null +++ b/tests/helpers/units.py @@ -0,0 +1,76 @@ +"""The unity.census and unity.export results the stages that read unity.export results plan from: a bundle's census +(what pipeline.units reads for the unit table) and the result of one unit's task. Synthetic: one serialized file per +bundle, every object its own unit.""" + +from __future__ import annotations + +from collections import Counter + +from nnnotes.pipeline import census, contract, link +from nnnotes.pipeline.contract import Input, Task + +CENSUS = "unity.census" +EXPORT = "unity.export" + + +def census_doc(file: str, objects) -> dict: + """The census of a bundle of one serialized file `file` holding `objects` [(pathId, class)].""" + objs = [ + {"pathId": pid, "classId": 0, "class": cls, "byteSize": 10, "typeHash": None} for pid, cls in sorted(objects) + ] + return { + "schema": contract.CENSUS, + "files": [ + { + "name": file, + "unityVersion": "6000.0.0f1", + "platform": 13, + "typetree": True, + "externals": [], + "objects": objs, + } + ], + "resources": [], + "assetBundles": [], + "scripts": [], + "classes": dict(sorted(Counter(o["class"] for o in objs).items())), + "objects": len(objs), + "errors": [], + } + + +def add_census(env, bundle: str, file: str, objects) -> None: + """Commits the census of `bundle` (census_doc) and adds the bundle to the facts "bundles" and "selected".""" + depot = env.depot + data = f"bundle {bundle}".encode() + inp = Input("bundle", depot.put(data), len(data), bundle, ({"kind": "depot"},)) + t = Task(CENSUS, 3, bundle, {}, {}, (inp,)) + doc = census_doc(file, objects) + rec = contract.artifact( + contract.artifact_id(t.id, "census"), + depot.add(contract.encode(doc), "json"), + contract.provenance(t), + {"kind": CENSUS, "format": "json", "facts": census.facts(doc)}, + ) + res = contract.result(t, [rec], []) + depot.commit(res) + env.done(t.id, res["key"]) + env.facts["bundles"] = {**env.facts.get("bundles", {}), bundle: inp} + env.facts["selected"] = sorted({*env.facts.get("selected", ()), bundle}) + + +def add_unit(env, bundle: str, file: str, pid: int, artifacts, cls: str = "MonoBehaviour") -> str: + """Commits the unity.export result of the unit of object `file:pid` of `bundle` (a bundle of add_census): + artifacts [(role, bytes, extension, semantics)] of that object, one exported item; -> the task id.""" + depot = env.depot + oid = contract.object_id(file, pid) + subject = link.path_address(bundle, link.file_tags([file]), file, pid) + t = Task(EXPORT, 2, subject, {}, {"unit": oid, "bundle": bundle}, ()) + recs = [ + contract.artifact(contract.artifact_id(oid, role), depot.add(data, ext), contract.provenance(t), semantics) + for role, data, ext, semantics in artifacts + ] + res = contract.result(t, recs, [contract.item(oid, "exported", artifacts=[r["id"] for r in recs], cls=cls)]) + depot.commit(res) + env.done(t.id, res["key"]) + return t.id diff --git a/tests/pipeline/test_cristages.py b/tests/pipeline/test_cristages.py index 97b24f5..ca861b6 100644 --- a/tests/pipeline/test_cristages.py +++ b/tests/pipeline/test_cristages.py @@ -21,6 +21,7 @@ from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.stages import Env, Pending, describe, execute, registry from tests.formats.test_usm import ADX, HCA, KEY, MPEG1, VP9, audio_header, build, chunk, rand, sbt, video_header +from tests.helpers import units as unit_results from tests.unity.test_speed_exact import FakeTools ROOT = Path(__file__).resolve().parents[2] @@ -123,31 +124,29 @@ def planning(depot, tmp, files: dict, keys=None, boot=True, bundles=()) -> Env: return Env(depot, facts) -def export_result(depot, env, subject: str, acbs: dict, usms=None) -> str: - """A committed unity.export result holding `cri.acb` artifacts {cue sheet: ACB bytes} and `cri.usm` artifacts - {movie asset name: USM bytes}.""" - t = Task(EXPORT, 1, subject, {}, {}, (Input("bundle", depot.put(subject.encode()), len(subject)),)) - recs = [ - contract.artifact( - f"CAB-{subject}:{i + 1}#acb", - depot.add(data, "acb"), - contract.provenance(t), +def export_result(env, bundle: str, acbs: dict, usms=None) -> dict: + """Committed unity.export results of `bundle`: a MonoBehaviour unit per `cri.acb` artifact {cue sheet: ACB bytes} + and per `cri.usm` artifact {movie asset name: USM bytes}; -> {sheet or movie name: task id}.""" + held = [ + ( + i + 1, + sheet, + data, + "acb", {"kind": "cri.acb", "format": "acb", "facts": {"cueSheet": sheet, "layout": "embedded"}}, ) for i, (sheet, data) in enumerate(sorted(acbs.items())) ] - recs += [ - contract.artifact( - f"CAB-{subject}:{100 + i}#usm", - depot.add(data, "usm"), - contract.provenance(t), - {"kind": "cri.usm", "format": "usm", "facts": {"movie": name}}, - ) + held += [ + (100 + i, name, data, "usm", {"kind": "cri.usm", "format": "usm", "facts": {"movie": name}}) for i, (name, data) in enumerate(sorted((usms or {}).items())) ] - depot.commit(contract.result(t, recs, [])) - env.done(t.id, t.key) - return t.id + file = f"CAB-{bundle}" + unit_results.add_census(env, bundle, file, [(pid, "MonoBehaviour") for pid, *_ in held]) + return { + name: unit_results.add_unit(env, bundle, file, pid, [(role, data, role, semantics)]) + for pid, name, data, role, semantics in held + } EXPORT = "unity.export" @@ -219,12 +218,13 @@ def test_audio_subjects_by_content(tmp_path, tools): {"cri_assets_cri/sound/bgm_x": ACB, "cri_assets_cri/video/m": usm_file()[0]}, {"Cri/Sound/bgm_x": ["cri_assets_cri/sound/bgm_x"], "Cri/Video/m": ["cri_assets_cri/video/m"]}, ) - tid = export_result(depot, env, "cri_bundle", {"bgm_x": ACB, "se_live": other}) + tids = export_result(env, "cri_bundle", {"bgm_x": ACB, "se_live": other}) + tid = tids["bgm_x"] stage = AudioStage() sha, other_sha = contract.sha256(ACB), contract.sha256(other) assert stage.subjects(env) == sorted([sha, other_sha]) # the raw ACB and its bundle copy: one assert stage.names(env) == {sha: ["Cri/Sound/bgm_x"], other_sha: ["se_live"]} - assert stage.depends(sha, env) == [tid] and stage.depends(other_sha, env) == [tid] + assert stage.depends(sha, env) == [tid] and stage.depends(other_sha, env) == [tids["se_live"]] t = describe(stage, sha, None, env) assert t.id == f"{AUDIO}:{sha}" and [i.role for i in t.inputs] == ["acb", "boot"] assert t.input("acb").locators[0]["kind"] == "file" and t.atoms == FAKE_ATOMS @@ -559,12 +559,13 @@ def test_movie_held_in_a_bundle(tmp_path, tools): }, bundles=("embcri_video_common",), ) - tid = export_result(depot, env, "embcri_video_common", {}, {"warmup": held, "loose": raw}) + tids = export_result(env, "embcri_video_common", {}, {"warmup": held, "loose": raw}) + tid = tids["warmup"] stage = MovieStage() sha, raw_sha = contract.sha256(held), contract.sha256(raw) assert stage.subjects(env) == sorted([sha, raw_sha]) assert stage.names(env) == {sha: ["EmbCri/Video/common/warmup"], raw_sha: ["Cri/Video/m", "loose"]} - assert stage.depends(sha, env) == [tid] and stage.depends(raw_sha, env) == [tid] + assert stage.depends(sha, env) == [tid] and stage.depends(raw_sha, env) == [tids["loose"]] with fake_tools(key=KEY): t = describe(stage, sha, None, env) assert t.input("usm").locators == ({"kind": "depot"},) and sorted(i.role for i in t.inputs) == ["boot", "usm"] diff --git a/tests/pipeline/test_link.py b/tests/pipeline/test_link.py index 9f24dc9..eee3d1a 100644 --- a/tests/pipeline/test_link.py +++ b/tests/pipeline/test_link.py @@ -296,81 +296,6 @@ def test_link_stages(tmp_path): assert execute(at, depot, stages).status == "hit" -def test_artifacts_table_and_stage(tmp_path): - depot = Depot(tmp_path / "depot") - arts, items = [], [] - for pid, status in ((1, "exported"), (2, "contained"), (3, "failed")): - oid = contract.object_id(CAB_A, pid) - if status == "exported": - c = depot.add(b"{}\n", "json") - aid = contract.artifact_id(oid, "json") - arts.append({"id": aid, "content": c}) - items.append(contract.item(oid, "exported", artifacts=[aid])) - elif status == "contained": - items.append(contract.item(oid, "contained", within=contract.artifact_id(CAB_A + ":1", "json"))) - else: - items.append(contract.item(oid, "failed", why=contract.reason("x", "y"))) - table = link.artifacts([("unity.export:card_a", {"artifacts": arts, "items": items})]) - assert table["objects"] == {f"{CAB_A}:1": [f"{CAB_A}:1#json"]} - assert table["contained"] == {f"{CAB_A}:2": f"{CAB_A}:1#json"} - assert table["artifacts"][f"{CAB_A}:1#json"]["ext"] == "json" - stage = link.ArtifactsStage() - env = Env(depot, {}) - export = contract.Task("unity.export", 1, "card_a") - depot.commit( - contract.result( - export, - [contract.artifact(a["id"], a["content"], contract.provenance(export), {"kind": "x"}) for a in arts], - items, - ) - ) - env.waiting(export.id) - with pytest.raises(Pending): - describe(stage, "all", None, env) - env.done(export.id, export.key) - env.waiting("sprite.crop:atlas", failed=True) # failed tasks are skipped - t = describe(stage, "all", None, env) - assert t.context == {"results": {export.id: export.key}} and t.inputs == () - execute(t, depot, registry(stage)) - (rec,) = depot.result(t.key)["artifacts"] - assert rec["id"] == "link.artifacts:all#artifacts" - assert contract.loads(depot.read(rec["content"]["sha256"]))["objects"] == table["objects"] - - -def test_the_artifact_table_is_written_entry_by_entry_with_the_same_bytes(): - def res(n, oid, aid, *, within=None): - c = {"sha256": f"{n:064x}", "size": n, "ext": "png", "mediaType": "image/png"} - items = ( - [contract.item(oid, "contained", within=within)] - if within - else [contract.item(oid, "exported", artifacts=[aid, aid + "~"])] - ) - return ( - f"unity.export:{n}", - { - "artifacts": [] if within else [{"id": aid, "content": c}, {"id": aid + "~", "content": c}], - "items": items, - }, - ) - - results = [ - res(3, f"{CAB_A}:3", f"{CAB_A}:3#立ち絵"), - res(1, f"{CAB_A}:1", f"{CAB_A}:1#png"), - res(2, f"{CAB_A}:2", None, within=f"{CAB_A}:1#png"), - res(4, f"{CAB_A}:é", None, within=f"{CAB_A}:1#png"), - ] - only_exported = [results[0], results[1]] - only_contained = [ - ("unity.export:x", {"artifacts": [], "items": [contract.item(f"{CAB_A}:9", "contained", within="a#b")]}) - ] - for rs in (results, only_exported, only_contained, []): - want = contract.encode(link.artifacts(rs)) - for chunk in (1, 64, 1 << 20): - parts = list(link.encode_artifacts(*link.artifact_table(iter(rs)), chunk=chunk)) - assert b"".join(parts) == want - assert len(list(link.encode_artifacts(*link.artifact_table(rs), chunk=1))) > 1 or not rs - - def test_store_add_chunks_is_add_of_the_joined_bytes(tmp_path): s = Depot(tmp_path / "s") c = s.add_chunks(iter([b"ab", b"", "é".encode()]), ".JSON") diff --git a/tests/pipeline/test_objexport.py b/tests/pipeline/test_objexport.py index f2a1879..8ec6e27 100644 --- a/tests/pipeline/test_objexport.py +++ b/tests/pipeline/test_objexport.py @@ -1,9 +1,8 @@ -"""unity.export and sprite.crop on synthetic bundles (fakeunity environments behind a stand-in reader atom): every -object gets exactly one item, the converters' artifacts, references, byte arrays and float forms, the generic -fallback with its reason codes, class selection, determinism, the cross-bundle sprite crop, and the golden records' -providers.""" +"""unity.export and sprite.crop on synthetic bundles (fakeunity environments behind a stand-in reader atom, with +their real censuses): one task per unit, every object exactly one item, the converters' artifacts, references, byte +arrays and float forms, the generic fallback with its reason codes, unit selection, batches, determinism, the +cross-bundle sprite crop, and the golden records' providers.""" -import json import math import re import subprocess @@ -15,10 +14,10 @@ from PIL import Image from nnnotes.pipeline import census as census_mod -from nnnotes.pipeline import contract, objexport +from nnnotes.pipeline import contract, objexport, units from nnnotes.pipeline.contract import IncompatibleTask, Input, Task from nnnotes.pipeline.depot import Depot -from nnnotes.pipeline.objexport import CROP, EXPORT, ExportStage, SpriteCropStage +from nnnotes.pipeline.objexport import CROP, ExportStage, SpriteCropStage from nnnotes.pipeline.stages import Env, describe, execute, registry from nnnotes.unity.atoms import Impl, astc, estimate, gltf from nnnotes.unity.atoms.reader import UnityBundle @@ -624,26 +623,70 @@ def classes_of(name): return sorted({r.type.name for f in BUNDLES[name]().assets for r in f.objects.values()}) -def task_for(depot, name, classes=None, params=None, scripts=None, typetrees=None): - data = f"fake:{name}".encode() - sha = depot.put(data) - stage = ExportStage(FAKE_READER) - p = stage.normalize(dict(params or {}, classes=classes)) - return Task( - EXPORT, - 1, - name, - p, - export_atoms(classes_of(name), FAKE_READER.id), - (Input("bundle", sha, len(data), f"{name}.bundle", ({"kind": "depot"},)),), - {"scripts": dict(SCRIPTS if scripts is None else scripts), **({"typetrees": typetrees} if typetrees else {})}, +def planning_env(depot, names=("all", "atlas"), selected=None, scripts=SCRIPTS, classes=None): + """A planning environment over fake bundles: their inputs, their censuses (census.census_env, the shipped type + trees read for files without type trees), the script table; `selected` (default: every bundle of `names`) and + `classes` as the export command gives them.""" + names = tuple(sorted(set(names) | set(selected or ()))) + bundles = {} + for n in names: + data = f"fake:{n}".encode() + bundles[n] = Input("bundle", depot.put(data), len(data), f"{n}.bundle", ({"kind": "depot"},)) + facts = {"bundles": bundles, "selected": sorted(selected if selected is not None else names), "classes": classes} + env = Env(depot, facts) + for n in names: + t = Task(census_mod.CensusStage.name, census_mod.CensusStage.version, n, {}, {}, (bundles[n],)) + doc = census_mod.census_env(BUNDLES[n](), shipped=True) + rec = contract.artifact( + contract.artifact_id(t.id, "census"), + depot.add(contract.encode(doc), "json"), + contract.provenance(t), + {"kind": "unity.census", "format": "json", "facts": census_mod.facts(doc)}, + ) + depot.commit(contract.result(t, [rec], [])) + env.done(t.id, t.key) + doc = {"schema": "nnnotes.scripts/1", "scripts": scripts} + data = contract.encode(doc) # its input: another table is another task + table = Task("link.scripts", 1, "all", {}, {}, (Input("census", depot.put(data), len(data), None, ()),)) + committed(depot, env, table, "scripts", doc, "link.scripts") + return env + + +def committed(depot, env, task, role, doc, kind): + rec = contract.artifact( + contract.artifact_id(task.id, role), + depot.add(contract.encode(doc), "json"), + contract.provenance(task), + {"kind": kind, "format": "json", "facts": {}}, ) + depot.commit(contract.result(task, [rec], [])) + env.done(task.id, task.key) -def run(depot, task): - stage = ExportStage(FAKE_READER) - execute(task, depot, registry(stage), force=True) - return depot.result(task.key) +def unit_task(depot, env, subject, params=None): + return describe(ExportStage(FAKE_READER), subject, params, env) + + +def export(depot, env, params=None, stage=None) -> dict: + """Run every unity.export task of a planning environment (in one stage object, as a worker runs a batch): the + results merged {"items", "artifacts"} with {"tasks": {task id: Task}, "status": {task id: result status}}.""" + stage = stage or ExportStage(FAKE_READER) + out: dict = {"items": [], "artifacts": [], "tasks": {}, "status": {}} + for s in stage.subjects(env): + t = describe(stage, s, params, env) + execute(t, depot, registry(stage), force=True) + doc = depot.result(t.key) + env.done(t.id, t.key) + out["items"] += doc["items"] + out["artifacts"] += doc["artifacts"] + out["tasks"][t.id] = t + out["status"][t.id] = doc["status"] + return out + + +def run_all(depot, names=("all",), params=None, classes=None, scripts=SCRIPTS) -> dict: + """export() of the units of the bundles `names` (only those of `classes` when given).""" + return export(depot, planning_env(depot, names, scripts=scripts, classes=classes), params) def by_object(doc): @@ -669,19 +712,18 @@ def oid(pid, cab=CAB): @pytest.fixture def exported(tmp_path): depot = Depot(tmp_path / "depot") - task = task_for(depot, "all") - return depot, task, run(depot, task) + return depot, run_all(depot) # ---------------------------------------------------------------- items def test_every_object_has_exactly_one_item(exported): - depot, task, doc = exported + depot, doc = exported objects = [contract.object_id(f.name, pid) for f in bundle_all().assets for pid in f.objects] items = by_object(doc) assert sorted(items) == sorted(objects) and len(doc["items"]) == len(objects) status = {o: (i["status"], (i.get("reason") or {}).get("code")) for o, i in items.items()} assert status == { - oid(1): ("contained", None), + oid(1): ("exported", None), oid(10): ("exported", None), oid(11): ("exported", None), oid(12): ("exported", None), @@ -705,7 +747,7 @@ def test_every_object_has_exactly_one_item(exported): oid(70): ("exported", None), oid(71): ("generic", "script.unresolved"), oid(72): ("generic", "script.missing"), - oid(80): ("contained", None), + oid(80): ("exported", None), oid(90): ("exported", None), oid(91): ("generic", "generic.clip.legacy"), oid(92): ("exported", None), @@ -719,18 +761,21 @@ def test_every_object_has_exactly_one_item(exported): oid(74): ("exported", None), oid(51): ("exported", None), } - assert items[oid(61)]["in"] == f"{oid(60)}#json" and items[oid(80)]["in"] == "unity.export:all#scripts.json" - assert items[oid(1)]["in"] == "unity.export:all#bundle.json" + assert items[oid(61)]["in"] == f"{oid(60)}#json" assert items[oid(94)]["artifacts"] == [f"{oid(94)}#json"] - assert doc["status"] == "ok" and all(i["class"] for i in doc["items"]) + assert set(doc["status"].values()) == {"ok"} and all(i["class"] for i in doc["items"]) for a in doc["artifacts"]: - owner, _role = contract.parse_artifact_id(a["id"]) - assert ("object" in a["provenance"]) == (not owner.startswith("unity.export:")) - assert depot.has(a["content"]["sha256"], a["content"]["size"]) + assert "object" in a["provenance"] and depot.has(a["content"]["sha256"], a["content"]["size"]) + # one task per unit: a hierarchy (its root's subject) or an object outside any + assert sorted(doc["tasks"]) == sorted( + f"unity.export:all:{contract.parse_object_id(o)[1]}" + for o in objects + if o not in {oid(p) for p in (61, 62, 63, 64, 65)} + ) def test_provenance_facts_and_atoms(exported): - depot, task, doc = exported + depot, doc = exported a = art(doc, f"{oid(10)}#image") assert a["provenance"]["object"] == { "file": CAB, @@ -749,7 +794,9 @@ def test_provenance_facts_and_atoms(exported): "astc.container", "tex.png", } + task = doc["tasks"]["unity.export:all:10"] assert a["provenance"]["inputs"] == [{"role": "bundle", "sha256": task.inputs[0].sha256}] + assert a["provenance"]["stage"]["version"] == ExportStage.version == 2 g = art(doc, f"{oid(95)}#json") assert set(g["provenance"]["atoms"]) == {"reader", "sniff", "generic.json"} assert task.atoms["tex.png"] == "tex.png/2" and task.atoms["reader"] == "fakeunity/1" @@ -757,7 +804,7 @@ def test_provenance_facts_and_atoms(exported): # ---------------------------------------------------------------- converters def test_textures(exported): - depot, _task, doc = exported + depot, doc = exported png = load(depot, doc, f"{oid(10)}#image") img = Image.open(__import__("io").BytesIO(png)) assert img.size == (W, H) and img.mode == "RGBA" @@ -784,7 +831,7 @@ def test_textures(exported): def test_sprites_same_bundle(exported): - depot, _task, doc = exported + depot, doc = exported from nnnotes.unity.atoms import png, sprite from nnnotes.unity.atoms.mesh import arrays from nnnotes.unity.atoms.reader import mesh_input_from_typetree @@ -833,7 +880,7 @@ def test_sprites_same_bundle(exported): def test_sprite_with_its_atlas_elsewhere_gets_a_meta_only(exported): - depot, _task, doc = exported + depot, doc = exported assert by_object(doc)[oid(13)]["artifacts"] == [f"{oid(13)}#json"] a = art(doc, f"{oid(13)}#json") assert a["semantics"]["facts"] == {"crop": CROP, "atlas": f"{ATLAS_CAB}:99"} @@ -843,7 +890,7 @@ def test_sprite_with_its_atlas_elsewhere_gets_a_meta_only(exported): def test_mesh(exported): - depot, _task, doc = exported + depot, doc = exported d, body = gltf.read(load(depot, doc, f"{oid(30)}#mesh")) (prim,) = d["meshes"][0]["primitives"] assert set(prim["attributes"]) == {"POSITION", "NORMAL", "TEXCOORD_0"} and prim["mode"] == 4 @@ -858,7 +905,7 @@ def test_mesh(exported): def test_text_and_fonts(exported): - depot, _task, doc = exported + depot, doc = exported a = art(doc, f"{oid(40)}#data") assert load(depot, doc, a["id"]) == b"hello\n" and a["content"]["ext"] == "txt" assert load(depot, doc, f"{oid(41)}#data") == b"\xff\xfe\x00bin" @@ -868,7 +915,7 @@ def test_text_and_fonts(exported): def test_every_converted_object_has_its_json(exported): - depot, _task, doc = exported + depot, doc = exported tex = obj(depot, doc, f"{oid(10)}#json") assert "image data" not in tex and tex["m_Width"] == W and tex["m_TextureFormat"] == 4 assert art(doc, f"{oid(10)}#json")["semantics"] == {"kind": "unity.object", "format": "json", "facts": {}} @@ -886,12 +933,11 @@ def test_every_converted_object_has_its_json(exported): assert all( i["status"] != "exported" or any(a.endswith("#json") for a in i["artifacts"]) or i["object"] == oid(13) for i in doc["items"] - if i["class"] not in ("AssetBundle", "MonoScript") ) def test_items_name_the_objects_they_reference(exported): - _depot, _task, doc = exported + _depot, doc = exported items = by_object(doc) assert items[oid(50)]["refs"] == [oid(10), f"{ATLAS_CAB}:5"] # the material's shader and texture # a prefab: the objects its components reference outside it, other bundles' included; its own objects are named @@ -907,7 +953,7 @@ def test_items_name_the_objects_they_reference(exported): def test_prefab(exported): - depot, _task, doc = exported + depot, doc = exported p = obj(depot, doc, f"{oid(60)}#json") assert p["name"] == "root" and [n["path"] for n in p["nodes"]] == ["root", "root/child"] root, child = p["nodes"] @@ -930,7 +976,7 @@ def test_prefab(exported): def test_scriptable_objects_blobs_and_strings(exported): - depot, _task, doc = exported + depot, doc = exported m = obj(depot, doc, f"{oid(70)}#json") assert m["$script"] == "A|Game|Local" and m["m_Name"] == "moc" blob = m["_bytes"]["$blob"] @@ -949,7 +995,7 @@ def test_scriptable_objects_blobs_and_strings(exported): def test_cue_sheets_held_in_the_bundle(exported): - depot, _task, doc = exported + depot, doc = exported from nnnotes.formats import cri a = art(doc, f"{oid(73)}#acb") @@ -973,15 +1019,21 @@ def test_cue_sheets_held_in_the_bundle(exported): "$blob": {"sha256": contract.sha256(EMBEDDED_ACB), "size": len(EMBEDDED_ACB), "format": "acb"} } assert by_object(doc)[oid(74)]["artifacts"] == [f"{oid(74)}#acb", f"{oid(74)}#json"] # no blob: artifact - small = run(depot, task_for(depot, "all", classes=["MonoBehaviour"], params={"blobMin": 1 << 20})) + small = run_all(depot, classes=["MonoBehaviour"], params={"blobMin": 1 << 20}) assert load(depot, small, f"{oid(74)}#acb") == EMBEDDED_ACB # whatever its size def movie_task(depot, converter=True): - """The task of the movie bundle: converter cri.usm in its atoms, as uses() gives it for that bundle.""" - t = task_for(depot, "movie", scripts=USM_SCRIPTS) - atoms = {**t.atoms, "cri.usm": "cri.usm/1"} if converter else t.atoms - return Task(t.stage, t.version, t.subject, t.params, dict(sorted(atoms.items())), t.inputs, t.context) + """The task of the movie asset's unit: converter cri.usm in its atoms, as uses() gives it for that unit.""" + env = planning_env(depot, ("movie",), scripts=USM_SCRIPTS) + t = unit_task(depot, env, "movie:2") + atoms = t.atoms if converter else {k: v for k, v in t.atoms.items() if k != "cri.usm"} + return Task(t.stage, t.version, t.subject, t.params, atoms, t.inputs, t.context) + + +def run(depot, task): + execute(task, depot, registry(ExportStage(FAKE_READER)), force=True) + return depot.result(task.key) def test_movies_held_in_the_bundle(tmp_path): @@ -1000,22 +1052,23 @@ def test_movies_held_in_the_bundle(tmp_path): "$blob": {"sha256": contract.sha256(WARMUP), "size": len(WARMUP), "format": "usm"} } assert by_object(doc)[m]["status"] == "exported" and by_object(doc)[m]["artifacts"] == [f"{m}#json", f"{m}#usm"] - assert load(depot, doc, f"{a}#acb") == EMBEDDED_ACB old = run(depot, movie_task(depot, converter=False)) assert by_object(old)[m]["status"] == "generic" and by_object(old)[m]["reason"]["code"] == "generic.error" - assert f"{m}#usm" not in {x["id"] for x in old["artifacts"]} and load(depot, old, f"{a}#acb") == EMBEDDED_ACB + assert f"{m}#usm" not in {x["id"] for x in old["artifacts"]} + sheet = run_all(depot, ("movie",), scripts=USM_SCRIPTS) + assert load(depot, sheet, f"{a}#acb") == EMBEDDED_ACB def test_the_movie_converter_is_keyed_by_the_script_context(tmp_path): depot = Depot(tmp_path / "s") env = planning_env(depot, ("all", "movie"), scripts=USM_SCRIPTS) stage = ExportStage(FAKE_READER) - assert describe(stage, "movie", None, env).atoms["cri.usm"] == "cri.usm/1" - assert "cri.usm" not in describe(stage, "all", None, env).atoms - assert "cri.usm" not in describe(stage, "movie", {"classes": ["AssetBundle"]}, env).atoms + assert describe(stage, "movie:2", None, env).atoms["cri.usm"] == "cri.usm/1" + assert "cri.usm" not in describe(stage, "movie:3", None, env).atoms # a cue sheet asset + assert "cri.usm" not in describe(stage, "all:70", None, env).atoms assert "cri.usm" in stage.ATOMS plain = planning_env(Depot(tmp_path / "p"), ("movie",)) - assert "cri.usm" not in describe(stage, "movie", None, plain).atoms + assert "cri.usm" not in describe(stage, "movie:2", None, plain).atoms def test_acb_layouts(tmp_path): @@ -1064,14 +1117,14 @@ def test_a_split_sheet_with_a_missing_chunk_gives_the_typetree(tmp_path, monkeyp dict(real(tt), chunks=[pptr(44), pptr(99, 1)]) if real(tt) and real(tt)["layout"] == "split" else real(tt) ), ) - doc = run(depot, task_for(depot, "all", classes=["MonoBehaviour"])) + doc = run_all(depot, classes=["MonoBehaviour"]) item = by_object(doc)[oid(73)] assert item["status"] == "generic" and item["reason"]["code"] == "generic.error" assert "is not a TextAsset of the bundle" in item["reason"]["message"] def test_shader(exported): - depot, _task, doc = exported + depot, doc = exported from nnnotes.formats import shader env = bundle_all() @@ -1099,7 +1152,7 @@ def test_shader(exported): def test_clip_bindings_name_scripts_of_other_bundles(exported): - depot, _task, doc = exported + depot, doc = exported clip = obj(depot, doc, f"{oid(96)}#json") (b,) = clip["bindings"] assert b == { @@ -1114,11 +1167,11 @@ def test_clip_bindings_name_scripts_of_other_bundles(exported): "script": {"$ref": {"file": "CAB-scripts", "pathId": 12}}, "scriptClass": "Assembly-CSharp|Game|Fader", } - assert census_mod.script_refs(census_doc(open_fake(b"fake:all"))) == sorted(SCRIPTS) + ["CAB-scripts:9"] + assert census_mod.script_refs(census_mod.census_env(bundle_all())) == sorted(SCRIPTS) + ["CAB-scripts:9"] def test_material_clip_controller(exported): - depot, _task, doc = exported + depot, doc = exported mat = obj(depot, doc, f"{oid(50)}#json") assert mat["shader"] == {"$ref": {"file": ATLAS_CAB, "pathId": 5}} assert mat["textures"]["_MainTex"]["texture"]["$ref"]["pathId"] == 10 @@ -1160,19 +1213,11 @@ def test_material_clip_controller(exported): assert obj(depot, doc, f"{oid(95)}#json")["m_Speed"] == INF -def test_loose_documents(exported): - depot, task, doc = exported - s = obj(depot, doc, "unity.export:all#scripts.json") - assert list(s["scripts"]) == [oid(80)] and s["scripts"][oid(80)]["m_ClassName"] == "Local" - b = obj(depot, doc, "unity.export:all#bundle.json") - assert b["files"] == [ - { - "name": CAB, - "unityVersion": "6000.3.12f1", - "externals": ["archive:/CAB-atlas/CAB-atlas", "archive:/CAB-scripts/CAB-scripts"], - } - ] - ab = b["assetBundles"][oid(1)] +def test_scripts_and_asset_bundles_have_their_typetree(exported): + depot, doc = exported + s = obj(depot, doc, f"{oid(80)}#json") + assert s["m_ClassName"] == "Local" and by_object(doc)[oid(80)]["artifacts"] == [f"{oid(80)}#json"] + ab = obj(depot, doc, f"{oid(1)}#json") assert ab["m_Container"][0] == [ "assets/x/tex.png", { @@ -1185,33 +1230,35 @@ def test_loose_documents(exported): def test_two_runs_are_identical(tmp_path): a, b = Depot(tmp_path / "a"), Depot(tmp_path / "b") - ta, tb = task_for(a, "all"), task_for(b, "all") - assert ta.key == tb.key - da, db = run(a, ta), run(b, tb) - assert contract.encode(da) == contract.encode(db) + da, db = run_all(a), run_all(b) + assert {k: x.key for k, x in da["tasks"].items()} == {k: x.key for k, x in db["tasks"].items()} + assert contract.encode(da["items"]) == contract.encode(db["items"]) + assert contract.encode(da["artifacts"]) == contract.encode(db["artifacts"]) for x in da["artifacts"]: assert a.read(x["content"]["sha256"]) == b.read(x["content"]["sha256"]) -def test_key_follows_the_census_classes_and_params(tmp_path): +def test_a_units_key_holds_its_classes_and_the_params(tmp_path): depot = Depot(tmp_path / "s") - t = task_for(depot, "atlas") + env = planning_env(depot, ("atlas",)) + t = unit_task(depot, env, "atlas:97") # a sprite assert "mesh.glb" not in t.atoms and "mesh.arrays" in t.atoms # sprites use mesh.arrays - assert {"tex.png", "sprite.png", "generic.json", "bundle.json"} <= set(t.atoms) - assert task_for(depot, "atlas", params={"png": {"level": 3}}).key != t.key + assert set(t.atoms) == set(export_atoms(["Sprite"], FAKE_READER.id)) + assert t.context == {"unit": f"{ATLAS_CAB}:97", "bundle": "atlas"} + assert unit_task(depot, env, "atlas:97", {"png": {"level": 3}}).key != t.key assert export_atoms(["Texture2D"]) == export_atoms(["Texture2D", "Texture2D"]) assert set(export_atoms(["Mesh"])) == {"reader", "sniff", "mesh.arrays", "gltf.write", "mesh.glb", "generic.json"} with pytest.raises(ValueError, match="png level"): ExportStage().normalize({"png": {"level": 11}}) with pytest.raises(ValueError, match="unknown parameters"): - ExportStage().normalize({"mesh": "obj"}) - assert ExportStage().normalize(None) == {"blobMin": 4096, "classes": None, "png": {"level": 6}} + ExportStage().normalize({"classes": ["Mesh"]}) + assert ExportStage().normalize(None) == {"blobMin": 4096, "png": {"level": 6}} def test_png_level_changes_bytes_not_pixels(tmp_path): depot = Depot(tmp_path / "s") - d6 = run(depot, task_for(depot, "all", classes=["Texture2D"])) - d0 = run(depot, task_for(depot, "all", classes=["Texture2D"], params={"png": {"level": 0}})) + d6 = run_all(depot, classes=["Texture2D"]) + d0 = run_all(depot, classes=["Texture2D"], params={"png": {"level": 0}}) p6, p0 = load(depot, d6, f"{oid(10)}#image"), load(depot, d0, f"{oid(10)}#image") assert p6 != p0 io = __import__("io") @@ -1220,27 +1267,31 @@ def test_png_level_changes_bytes_not_pixels(tmp_path): def test_blob_threshold(tmp_path): depot = Depot(tmp_path / "s") - doc = run(depot, task_for(depot, "all", classes=["MonoBehaviour"], params={"blobMin": 1 << 20})) + doc = run_all(depot, classes=["MonoBehaviour"], params={"blobMin": 1 << 20}) m = obj(depot, doc, f"{oid(70)}#json") assert m["_bytes"] == {"$hex": MOC.hex()} -def test_class_selection(tmp_path): +def test_classes_select_units_by_their_root(tmp_path): depot = Depot(tmp_path / "s") - doc = run(depot, task_for(depot, "all", classes=["Sprite"])) + doc = run_all(depot, classes=["Sprite"]) items = by_object(doc) - assert set(items) == {oid(11), oid(12), oid(13)} # textures decoded, not exported + assert set(items) == {oid(11), oid(12), oid(13)} # their textures decoded, not exported assert all(i["status"] == "exported" for i in items.values()) - doc = run(depot, task_for(depot, "all", classes=["GameObject"])) + doc = run_all(depot, classes=["GameObject"]) assert set(by_object(doc)) == {oid(p) for p in (60, 61, 62, 63, 64, 65)} + assert list(doc["tasks"]) == ["unity.export:all:60"] + keys = {k: x.key for k, x in run_all(depot)["tasks"].items()} + assert keys["unity.export:all:60"] == doc["tasks"]["unity.export:all:60"].key # a selection is no part of keys def test_scripts_come_from_the_context_only(tmp_path): depot = Depot(tmp_path / "s") - doc = run(depot, task_for(depot, "all", classes=["MonoBehaviour"], scripts={})) + doc = run_all(depot, classes=["MonoBehaviour", "GameObject"], scripts={}) items = by_object(doc) assert items[oid(70)]["reason"]["code"] == "script.unresolved" - assert items[oid(62)]["reason"]["code"] == "script.unresolved" # no hierarchy: one by one + issues = art(doc, f"{oid(60)}#json")["semantics"]["issues"] # the components of a hierarchy + assert issues[0]["code"] == "script.unresolved" and issues[0]["message"].startswith("2 MonoBehaviours") def test_a_converter_error_gives_the_typetree(tmp_path, monkeypatch): @@ -1250,7 +1301,7 @@ def broken(tt, ref): raise KeyError("m_SavedProperties") monkeypatch.setattr(material, "material_doc", broken) - doc = run(depot, task_for(depot, "all", classes=["Material"])) + doc = run_all(depot, classes=["Material"]) item = by_object(doc)[oid(50)] assert item["status"] == "generic" and item["reason"] == { "code": "generic.error", @@ -1266,7 +1317,7 @@ def broken(*args): raise RuntimeError("component failed") monkeypatch.setattr(prefab, "_component", broken) - doc = run(depot, task_for(depot, "all", classes=["GameObject"])) + doc = run_all(depot, classes=["GameObject"]) items = by_object(doc) assert {i["status"] for i in items.values()} == {"generic"} assert all(i["reason"]["code"] == "generic.error" for i in items.values()) @@ -1283,14 +1334,16 @@ def typetree(self, o): return real(self, o) monkeypatch.setattr(UnityBundle, "typetree", typetree) - doc = run(depot, task_for(depot, "all", classes=["Animator"])) + doc = run_all(depot, classes=["Animator"]) (item,) = doc["items"] - assert item["status"] == "failed" and item["reason"]["code"] == "failed.read" and doc["status"] == "partial" + assert item["status"] == "failed" and item["reason"]["code"] == "failed.read" + assert doc["status"] == {"unity.export:all:95": "partial"} def test_task_with_other_atoms_is_refused(tmp_path): depot = Depot(tmp_path / "s") - t = task_for(depot, "all") + env = planning_env(depot, ("all",)) + t = unit_task(depot, env, "all:10") bad = Task(t.stage, t.version, t.subject, t.params, {**t.atoms, "tex.png": "tex.png/0"}, t.inputs, t.context) with pytest.raises(IncompatibleTask, match="tex.png"): execute(bad, depot, registry(ExportStage(FAKE_READER))) @@ -1299,119 +1352,90 @@ def test_task_with_other_atoms_is_refused(tmp_path): t.version, t.subject, t.params, - {k: v for k, v in t.atoms.items() if k != "mesh.glb"}, + {k: v for k, v in t.atoms.items() if k != "tex.png"}, t.inputs, t.context, ) - with pytest.raises(IncompatibleTask, match="mesh.glb"): + with pytest.raises(IncompatibleTask, match="tex.png"): execute(missing, depot, registry(ExportStage(FAKE_READER))) + other = Task(t.stage, t.version, t.subject, t.params, t.atoms, t.inputs, {**t.context, "unit": f"{CAB}:999"}) + with pytest.raises(IncompatibleTask, match="not in the bundle"): + execute(other, depot, registry(ExportStage(FAKE_READER))) # ---------------------------------------------------------------- describing tasks -def census_doc(view: UnityBundle) -> dict: - """The part of a census the export stage reads (classes, objects, m_Script references).""" - files, classes = [], {} - for f in view.files(): - objs = [] - for o in view.objects(): - if o.file != f: - continue - classes[o.class_name] = classes.get(o.class_name, 0) + 1 - rec = {"pathId": o.path_id, "classId": o.class_id, "class": o.class_name, "typeHash": o.type_hash} - tt = view.reader(o).read_typetree(check_read=False) # a header read, as the census' - ext = view.externals(f) - - def ref(s): - file = f if not s["m_FileID"] else ext[s["m_FileID"] - 1].rsplit("/", 1)[-1] - return {"file": file, "pathId": s["m_PathID"]} - - s = tt.get("m_Script") - if isinstance(s, dict) and s.get("m_PathID"): - rec["script"] = ref(s) - bindings = [ - b["script"] - for b in (tt.get("m_ClipBindingConstant") or {}).get("genericBindings", ()) - if b["script"].get("m_PathID") - ] - if bindings: - rec["bindingScripts"] = [ref(b) for b in bindings] - objs.append(rec) - files.append( - {"name": f, "unityVersion": view.unity_version(f), "typetree": view.stores_typetrees(f), "objects": objs} - ) - return { - "schema": contract.CENSUS, - "files": files, - "resources": [], - "assetBundles": [], - "scripts": [], - "classes": dict(sorted(classes.items())), - "objects": sum(classes.values()), - "errors": [], - } - - -def committed(depot, env, task, role, doc, kind): - rec = contract.artifact( - contract.artifact_id(task.id, role), - depot.add(contract.encode(doc), "json"), - contract.provenance(task), - {"kind": kind, "format": "json", "facts": {}}, - ) - depot.commit(contract.result(task, [rec], [])) - env.done(task.id, task.key) - - -def planning_env(depot, names=("all", "atlas"), selected=None, scripts=SCRIPTS): - names = tuple(sorted(set(names) | set(selected or ()))) - bundles = {} - for n in names: - data = f"fake:{n}".encode() - bundles[n] = Input("bundle", depot.put(data), len(data), f"{n}.bundle", ({"kind": "depot"},)) - env = Env(depot, {"bundles": bundles, "selected": sorted(selected if selected is not None else names)}) - for n in names: - t = Task(census_mod.CensusStage.name, 1, n, {}, {}, (bundles[n],)) - doc = census_doc(open_fake(f"fake:{n}".encode())) - rec = contract.artifact( - contract.artifact_id(t.id, "census"), - depot.add(contract.encode(doc), "json"), - contract.provenance(t), - {"kind": "unity.census", "format": "json", "facts": census_mod.facts(doc)}, - ) - depot.commit(contract.result(t, [rec], [])) - env.done(t.id, t.key) - table = Task("link.scripts", 1, "all") - committed(depot, env, table, "scripts", {"schema": "nnnotes.scripts/1", "scripts": scripts}, "link.scripts") - return env - - -def test_describe_reads_the_census_and_the_script_table(tmp_path): +def test_describe_reads_the_units_and_the_script_table(tmp_path): depot = Depot(tmp_path / "s") env = planning_env(depot) stage = ExportStage(FAKE_READER) - assert stage.subjects(env) == ["all", "atlas"] - # every closure bundle holding a SpriteAtlas joins a selection that holds sprites (a sprite reaches its - # texture through its atlas' bundle); a selection without sprites stays as it is - assert stage.subjects(planning_env(Depot(tmp_path / "t"), selected=["all"])) == ["all", "atlas"] - assert stage.subjects(planning_env(Depot(tmp_path / "u"), selected=["plain"])) == ["plain"] - t = describe(stage, "all", None, env) - assert t.atoms == export_atoms(classes_of("all"), FAKE_READER.id) - assert t.context == {"scripts": SCRIPTS} and t.key == task_for(depot, "all").key - assert stage.depends("all", env) == ["unity.census:all", "link.scripts:all"] - a = describe(stage, "atlas", None, env) - assert a.context == {} and "mesh.glb" not in a.atoms - assert t.cost.peak_bytes >= 72 << 20 + subjects = stage.subjects(env) + assert "all:60" in subjects and "all:61" not in subjects and "atlas:1" in subjects + # a selection's units and the units their objects reach in other bundles (a sprite's atlas, its texture and + # the sprites it packs); not the rest of those bundles + assert [ + s for s in stage.subjects(planning_env(Depot(tmp_path / "t"), selected=["all"])) if s.startswith("atlas") + ] == [ + "atlas:97", + "atlas:98", + "atlas:99", + ] + assert stage.subjects(planning_env(Depot(tmp_path / "u"), selected=["plain"])) == ["plain:1", "plain:2"] + t = describe(stage, "all:10", None, env) + assert t.atoms == export_atoms(["Texture2D"], FAKE_READER.id) and t.subject == "all:10" + assert t.context == {"unit": oid(10), "bundle": "all"} + assert stage.depends("all:10", env) == ["unity.census:all", "link.scripts:all"] + m = describe(stage, "all:70", None, env) + assert m.context["scripts"] == {oid(80): "A|Game|Local"} + p = describe(stage, "all:60", None, env) # a hierarchy: its objects' classes and scripts + assert set(p.atoms) == set(export_atoms(["GameObject", "MonoBehaviour", "Transform"], FAKE_READER.id)) + assert p.context["scripts"] == { + "CAB-scripts:7": SCRIPTS["CAB-scripts:7"], + "CAB-scripts:8": SCRIPTS["CAB-scripts:8"], + } + clip = describe(stage, "all:96", None, env) # its bindings' scripts + assert clip.context["scripts"] == {"CAB-scripts:12": "Assembly-CSharp|Game|Fader"} + assert t.cost.peak_bytes >= 72 << 20 and stage.batch(t) == t.inputs[0].sha256 == stage.batch(p) fresh = Env(depot, env.facts) from nnnotes.pipeline.stages import Pending fresh.waiting("unity.census:all") with pytest.raises(Pending): - describe(stage, "all", None, fresh) + stage.subjects(fresh) + + +def test_the_tasks_of_a_bundle_load_it_once_and_their_order_changes_nothing(tmp_path): + opened = [] + + def counting(data, name=None): + opened.append(bytes(data)) + return open_fake(data, name) + reader = Impl("fakeunity/1", counting, lambda facts: estimate(0.01, 1 << 20)) + depot = Depot(tmp_path / "s") + env = planning_env(depot, ("all",)) + stage = ExportStage(reader) + subjects = stage.subjects(env) + tasks = [describe(stage, s, None, env) for s in subjects] + results = {} + for t in tasks: + results[t.id] = stage.run(t, depot) + assert opened == [b"fake:all"] + stage.end_batch(stage.batch(tasks[0])) + assert stage._loaded is None + backwards = ExportStage(reader) + for t in reversed(tasks): + out = backwards.run(t, depot) + assert contract.encode(out.items) == contract.encode(results[t.id].items), t.id + assert contract.encode(out.artifacts) == contract.encode(results[t.id].artifacts), t.id + assert len(opened) == 2 + + +def test_files_without_type_trees_read_their_scripts_with_the_shipped_ones(tmp_path, monkeypatch): + from nnnotes.unity import scripttypes -def test_files_without_type_trees_read_their_scripts_with_the_shipped_ones(tmp_path): + monkeypatch.setattr(scripttypes, "script_types", lambda version: PLAYER_TYPETREES) depot = Depot(tmp_path / "s") - doc = run(depot, task_for(depot, "player", scripts=PLAYER_SCRIPTS, typetrees=PLAYER_TYPETREES)) + doc = run_all(depot, ("player",), scripts=PLAYER_SCRIPTS) items = by_object(doc) def p(pid): @@ -1469,39 +1493,35 @@ def test_the_context_holds_the_shipped_type_trees_of_its_scripts(tmp_path, monke trees = {**PLAYER_TYPETREES, shipped["typeHash"]: shipped["nodes"]} monkeypatch.setattr(scripttypes, "script_types", lambda version: trees) env = planning_env(Depot(tmp_path / "s"), names=("player",), scripts=PLAYER_SCRIPTS) - t = describe(ExportStage(FAKE_READER), "player", None, env) - assert t.context["typetrees"] == PLAYER_TYPETREES # the trees of its MonoBehaviours' type hashes - assert ( - "typetrees" not in describe(ExportStage(FAKE_READER), "all", None, planning_env(Depot(tmp_path / "t"))).context - ) - - -def test_class_selection_keeps_only_its_converters_in_the_key(tmp_path): - depot = Depot(tmp_path / "s") - env = planning_env(depot) stage = ExportStage(FAKE_READER) - full = describe(stage, "all", None, env) - assert full.key == describe(stage, "all", {"classes": None}, env).key == task_for(depot, "all").key - meshes = describe(stage, "all", {"classes": ["Mesh"]}, env) - assert meshes.atoms == export_atoms(["Mesh"], FAKE_READER.id) and meshes.params["classes"] == ["Mesh"] - assert "tex.png" not in meshes.atoms and meshes.key != full.key - assert describe(stage, "all", {"classes": ["Mesh", "Nothing"]}, env).atoms == meshes.atoms - assert describe(stage, "all", {"classes": ["Nothing"]}, env).atoms == export_atoms([], FAKE_READER.id) - assert env.params == {} # set while describing only - for classes in (["Mesh"], ["Sprite"], ["GameObject"], ["Material", "AnimationClip"]): - t = describe(stage, "all", {"classes": classes}, env) - execute(t, depot, registry(stage)) - doc = depot.result(t.key) - assert doc["items"] and all(i["class"] in {*classes, "Transform", "MonoBehaviour"} for i in doc["items"]) + # the trees of its MonoBehaviours' type hashes; a hierarchy whose component has none: no trees + assert describe(stage, "player:2", None, env).context["typetrees"] == {PLAYER_HASH: SETTINGS_ROWS} + assert describe(stage, "player:4", None, env).context["typetrees"] == {OTHER_HASH: SETTINGS_ROWS[:-1]} + assert "typetrees" not in describe(stage, "player:20", None, env).context + assert "typetrees" not in describe(stage, "all:70", None, planning_env(Depot(tmp_path / "t"))).context + + +def test_the_unit_table(tmp_path): + env = planning_env(Depot(tmp_path / "s"), ("all", "atlas", "plain"), selected=["all"], classes=["Sprite"]) + t = units.table(env) + assert t.subjects() == ["all:11", "all:12", "all:13", "atlas:97"] + u = t.unit("all:13") + assert (u.bundle, u.root, u.cls, u.members) == ("all", oid(13), "Sprite", (oid(13),)) + assert t.subject_of(oid(62)) == "all:60" and t.subject_of(f"{ATLAS_CAB}:98") == "atlas:98" + assert t.subject_of("CAB-plain:2") == "plain:2" and t.subject_of("CAB-x:1") is None + assert t.files == {CAB: "all", ATLAS_CAB: "atlas", "CAB-plain": "plain"} + assert t.per_bundle == {"all": 3, "atlas": 1} and t.classes == {"Sprite": 4} + assert units.table(env) is t # kept for the same planning state + h = units.table(planning_env(Depot(tmp_path / "t"), ("all",))).unit("all:60") + assert h.members == tuple(oid(p) for p in (60, 61, 62, 63, 64, 65)) + assert h.counts == (("GameObject", 2), ("MonoBehaviour", 2), ("Transform", 2)) + with pytest.raises(ValueError, match="no requested unit"): + t.unit("all:61") def crop_setup(depot): env = planning_env(depot) - stage = ExportStage(FAKE_READER) - for n in ("all", "atlas"): - t = describe(stage, n, None, env) - execute(t, depot, registry(stage)) - env.done(t.id, t.key) + export(depot, env) return env @@ -1512,7 +1532,11 @@ def test_sprite_crop_equals_the_same_sprite_cropped_in_its_atlas_bundle(tmp_path assert stage.subjects(env) == ["atlas:99"] t = describe(stage, "atlas:99", None, env) assert [i.role for i in t.inputs] == ["atlas", f"sprite:{oid(13)}", f"texture:{ATLAS_CAB}:98"] - assert stage.depends("atlas:99", env) == ["unity.export:all", "unity.export:atlas"] + assert stage.depends("atlas:99", env) == [ + "unity.export:all:13", + "unity.export:atlas:98", + "unity.export:atlas:99", + ] assert set(t.atoms) == {"png.encode", "sprite.crop"} execute(t, depot, registry(stage)) doc = depot.result(t.key) @@ -1520,7 +1544,7 @@ def test_sprite_crop_equals_the_same_sprite_cropped_in_its_atlas_bundle(tmp_path assert item == {"object": oid(13), "status": "exported", "artifacts": [f"{oid(13)}#image"], "class": "Sprite"} a = doc["artifacts"][0] assert a["provenance"]["object"]["name"] == "remote" and a["provenance"]["object"]["file"] == CAB - local = depot.result(env.key("unity.export:atlas")) + local = depot.result(env.key("unity.export:atlas:97")) same = art(local, f"{ATLAS_CAB}:97#image") assert a["content"] == same["content"] # the same bytes as a local crop assert Image.open(__import__("io").BytesIO(depot.read(a["content"]["sha256"]))).size == (8, 8) @@ -1529,7 +1553,7 @@ def test_sprite_crop_equals_the_same_sprite_cropped_in_its_atlas_bundle(tmp_path def test_sprite_crop_waits_for_exports(tmp_path): depot = Depot(tmp_path / "s") env = planning_env(depot) - env.waiting("unity.export:all") + env.waiting("unity.export:all:13") from nnnotes.pipeline.stages import Pending with pytest.raises(Pending): @@ -1570,34 +1594,32 @@ def test_every_reason_code_and_converter_is_documented(): # ---------------------------------------------------------------- golden records -CONVERTER_CLASSES = { - "tex.png": ["Texture2D"], - "sprite.png": ["Sprite"], - "mesh.glb": ["Mesh"], - "prefab.json": ["GameObject"], - "material.json": ["Material"], - "clip.json": ["AnimationClip"], - "controller.json": ["AnimatorController", "AnimatorOverrideController"], - "mono.json": ["MonoBehaviour"], - "data": ["TextAsset"], - "font": ["Font"], - "shader.json": ["Shader"], - "scripts.json": ["MonoScript"], - "bundle.json": ["AssetBundle"], - "generic.json": ["Animator", "AudioClip", "SpriteAtlas"], +# converter -> the unit of the synthetic bundles its golden fixture converts +CONVERTER_UNITS = { + "tex.png": "all:10", + "sprite.png": "all:12", + "mesh.glb": "all:30", + "prefab.json": "all:60", + "material.json": "all:50", + "clip.json": "all:96", + "controller.json": "all:92", + "mono.json": "all:73", + "data": "all:40", + "font": "all:42", + "shader.json": "all:51", + "generic.json": "all:95", } LIBRARIES = ["Pillow", "UnityPy", "astc-encoder-py", "numpy", "texture2ddecoder"] def export_golden(): - """Provider of tests/golden/unity.export.json: one fixture per converter (the synthetic bundle with the - classes the converter takes), and the whole bundle.""" + """Provider of tests/golden/unity.export.json: one fixture per converter (a unit of the synthetic bundles that + the converter converts).""" - def fixture(classes): - return lambda depot: task_for(depot, "all", classes=classes) + def fixture(subject): + return lambda depot: unit_task(depot, planning_env(depot, ("all",)), subject) - fixtures = {name: fixture(classes) for name, classes in CONVERTER_CLASSES.items()} - fixtures["all"] = fixture(None) + fixtures = {name: fixture(subject) for name, subject in CONVERTER_UNITS.items()} fixtures["cri.usm"] = movie_task return {"stage": ExportStage(FAKE_READER), "fixtures": fixtures, "libraries": LIBRARIES} @@ -1614,8 +1636,7 @@ def fixture(depot): def test_golden_providers_cover_every_converter(): p = export_golden() - assert set(p["fixtures"]) == set(CONVERTERS) | {"all"} - assert json.dumps(CONVERTER_CLASSES) # every converter named once + assert set(p["fixtures"]) == set(CONVERTERS) def test_a_converter_change_without_a_version_bump_fails_its_golden(tmp_path, monkeypatch): @@ -1639,5 +1660,4 @@ def changed(ctx, info): actual = golden(p["stage"], p["fixtures"], Depot(tmp_path / "s"), p["libraries"]) problems = golden_problems(expected, actual, p["stage"]) assert "unity.export: fixture font: output changed without a version bump" in problems - assert "unity.export: fixture all: output changed without a version bump" in problems assert not any("fixture tex.png" in x for x in problems) diff --git a/tests/pipeline/test_voices_story.py b/tests/pipeline/test_voices_story.py index 8fd0dd3..c6da211 100644 --- a/tests/pipeline/test_voices_story.py +++ b/tests/pipeline/test_voices_story.py @@ -10,11 +10,11 @@ from nnnotes.content import advcommand, storyvoices from nnnotes.pipeline import contract, voices -from nnnotes.pipeline.contract import Task from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.stages import Pending, describe, execute, registry from nnnotes.pipeline.views import AddressIndex, Obj, ViewError from nnnotes.source import languages +from tests.helpers import units as unit_results from tests.pipeline.test_views import addresses_doc from tests.pipeline.test_voices import KEYS, SHEETS, TABLES, Paths, audio, audio_task, planning, row, text @@ -483,28 +483,19 @@ def test_story_rules_are_checked(): # ---------------------------------------------------------------- the stage def export_results(depot, env) -> dict: - """unity.export results holding the exported episode parts, one bundle per episode: {asset: {part: oid}}.""" + """unity.export results holding the exported episode parts, one bundle per episode and a unit per part: {asset: + {part: oid}}.""" oids = {} for n, (asset, parts) in enumerate(sorted(EPISODES.items())): - t = Task("unity.export", 1, f"adv_{asset}", {}, {}, ()) - arts, items = [], [] - for m, part in enumerate(sorted(parts), 1): - oid = contract.object_id(f"CAB-{n:04x}", 10 * n + m) - role = voices.STORY_OBJECTS[part][1] - data = part_bytes(asset, part) - arts.append( - contract.artifact( - contract.artifact_id(oid, role), - depot.add(data, "json" if part == "script" else "txt"), - contract.provenance(t), - {"kind": "unity.object", "format": "json"}, - ) - ) - items.append(contract.item(oid, "exported", artifacts=[arts[-1]["id"]])) - oids.setdefault(asset, {})[part] = oid - res = contract.result(t, arts, items) - depot.commit(res) - env.done(t.id, res["key"]) + file, bundle = f"CAB-{n:04x}", f"adv_{asset}" + pids = {part: 10 * n + m for m, part in enumerate(sorted(parts), 1)} + unit_results.add_census(env, bundle, file, [(pid, voices.STORY_OBJECTS[p][0]) for p, pid in pids.items()]) + for part, pid in pids.items(): + cls, role = voices.STORY_OBJECTS[part] + ext = "json" if part == "script" else "txt" + semantics = {"kind": "unity.object", "format": "json"} + unit_results.add_unit(env, bundle, file, pid, [(role, part_bytes(asset, part), ext, semantics)], cls) + oids.setdefault(asset, {})[part] = contract.object_id(file, pid) return oids @@ -556,7 +547,8 @@ def test_stage_reads_the_exported_episodes(tmp_path): shas = {p["sha256"] for parts in eps.values() for p in parts.values()} assert sorted(i.role for i in task.inputs if i.role.startswith("adv:")) == sorted(f"adv:{s}" for s in shas) deps = stage.depends("main", env) - assert sorted(d for d in deps if d.startswith("unity.export:")) == sorted(f"unity.export:adv_{a}" for a in EPISODES) + parts = [f"unity.export:adv_{a}:{p['object'].split(':')[1]}" for a, ps in eps.items() for p in ps.values()] + assert sorted(d for d in deps if d.startswith("unity.export:")) == sorted(parts) # the unit of each part ex = execute(task, depot, registry(stage)) assert (ex.status, ex.result_status) == ("ran", "ok") res = depot.result(task.key) @@ -576,7 +568,7 @@ def test_stage_reads_the_exported_episodes(tmp_path): def test_stage_waits_for_the_exports(tmp_path): depot = Depot(tmp_path / "s") env = story_env(depot, tmp_path) - env.waiting("unity.export:adv_adv_script_x_01") + env.waiting("unity.export:adv_adv_script_x_01:11") # the unit of its script with pytest.raises(Pending, match="unity.export"): describe(voices.VoiceStage(), "main", None, env)