From 71a2dc2e1faf4aed4b54ffa2f94b827e90468063 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Thu, 15 Oct 2026 02:19:29 +0800 Subject: [PATCH] feat(export)!: census and export the APK's built-in Unity files With `[paths] apk` set, the player's own Unity files under `assets/bin/Data/` of the APK are subjects of unity.census and unity.export: `apk/data.unity3d`, `apk/datapack.unity3d` and `apk/Resources/unity default resources`. The census always covers them; everything and `--select bundle:apk/*` export them. A serialized file outside a bundle is loaded under the name references into it use, so object ids such as `unity_builtin_extra:12` and `unity default resources:10` name objects these tasks export, and link.addresses maps their files to their subjects. These files store no type trees. MonoBehaviours are read with the type trees nnnotes ships for the game's classes (nnnotes.unity.scripttypes), checked against the stored type hash and the object's size; the export task's context carries the trees its objects use. A MonoBehaviour without one gives its header with `unsupported.typetree`, and PlayerSettings gives its fields up to `m_ActiveColorSpace`, the part its type tree reads right. generic.json is version 2. The CRI stages take their `boot` input from the built-in inputs. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 2 +- docs/configuration.md | 4 +- docs/stages.md | 38 ++++++- scripts/gen_typetrees.py | 14 +-- src/nnnotes/cli/assets.py | 44 +++++--- src/nnnotes/pipeline/builtins.py | 71 +++++++++++++ src/nnnotes/pipeline/census.py | 20 ++-- src/nnnotes/pipeline/cristages.py | 33 ++---- src/nnnotes/pipeline/link.py | 47 +++++---- src/nnnotes/pipeline/objexport.py | 62 +++++++++-- src/nnnotes/unity/atoms/reader.py | 18 +++- src/nnnotes/unity/convert/__init__.py | 7 +- src/nnnotes/unity/convert/generic.py | 8 +- src/nnnotes/unity/convert/objects.py | 141 +++++++++++++++++++++----- src/nnnotes/unity/convert/prefab.py | 17 ++-- src/nnnotes/unity/player.py | 56 +--------- src/nnnotes/unity/reader.py | 22 ++-- src/nnnotes/unity/scripttypes.py | 98 ++++++++++++++++++ tests/cli/test_assets.py | 85 ++++++++++++++-- tests/golden/unity.export.json | 66 ++++++------ tests/helpers/fakeunity.py | 31 +++++- tests/pipeline/test_builtins.py | 47 +++++++++ tests/pipeline/test_census.py | 11 +- tests/pipeline/test_cristages.py | 34 +++---- tests/pipeline/test_objexport.py | 128 +++++++++++++++++++++-- tests/unity/test_typetrees.py | 13 +-- 26 files changed, 844 insertions(+), 273 deletions(-) create mode 100644 src/nnnotes/pipeline/builtins.py create mode 100644 src/nnnotes/unity/scripttypes.py create mode 100644 tests/pipeline/test_builtins.py diff --git a/docs/architecture.md b/docs/architecture.md index cd89197..c55d9c0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -12,7 +12,7 @@ subpackage of `nnnotes` and imports only the layers below it. | Products | `nnnotes.site`, `nnnotes.musicdata` | the ournotes-player site (charts, models, stories, indexes); the music data file and its deck statistics | | Pipeline | `nnnotes.pipeline` | stages, task descriptions and results, the content-addressed depot, plans, layouts, the catalog versions, the views of the master data | | Content | `nnnotes.content` | the game's subjects as exports: ADV episodes and stories, lives and scores, spots, rooms, the UI library | -| Unity objects | `nnnotes.unity` | bundle reading, object ids, the atoms (the pure functions objects are converted with: texture decoding, sprite images, mesh arrays, glTF, PNG, file sniffing), the converters (one per Unity class, `nnnotes.unity.convert`), the typetree exporter, the player data, Live2D prefabs, text styles and TextMesh Pro fonts | +| Unity objects | `nnnotes.unity` | bundle reading, object ids, the atoms (the pure functions objects are converted with: texture decoding, sprite images, mesh arrays, glTF, PNG, file sniffing), the converters (one per Unity class, `nnnotes.unity.convert`), the type trees shipped for the player's script classes (`nnnotes.unity.scripttypes`), the typetree exporter, the player data, Live2D prefabs, text styles and TextMesh Pro fonts | | File formats | `nnnotes.formats` | formats read from bytes: CRI ACB/AWB and HCA, USM movies, CRI Lips data, shaders, motions | | Sources | `nnnotes.source` | the configuration, the APK, Addressables catalogs and bundles, the game API, master data, languages, `Session` | | Core | `nnnotes.core` | the one JSON writer, canonical JSON and its hashes, media types, the process caches and the downloads cache | diff --git a/docs/configuration.md b/docs/configuration.md index 532e9b9..9aedab4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -103,7 +103,7 @@ exist when they are given. | `[paths] catalog` | `NNNOTES_PATHS_CATALOG` | `--catalog` | a catalog `.bin` file to read instead of discovering/downloading a catalog; explicit files stay usable offline | | `[paths] depot` | `NNNOTES_PATHS_DEPOT` | `--depot` | depot directory (created when missing): the downloaded game files, the results of the stages, catalog versions and runs ([commands.md](commands.md#downloads), [contracts.md](contracts.md#depot)) | | `[paths] master` | `NNNOTES_PATHS_MASTER` | `--master` | decoded master data directory, one `.json` per table; the flag overrides `[servers.] master` | -| `[paths] apk` | `NNNOTES_PATHS_APK` | `--apk` | the game's `base.apk` (the story export reads the CRI Lips library from it, or from `split_config.arm64_v8a.apk` next to it) | +| `[paths] apk` | `NNNOTES_PATHS_APK` | `--apk` | the game's `base.apk` (the story export reads the CRI Lips library from it, or from `split_config.arm64_v8a.apk` next to it; the asset export its built-in Unity files) | | `[paths] ffmpeg` | `NNNOTES_PATHS_FFMPEG` | `--ffmpeg` | `ffmpeg` executable; unset: `ffmpeg` on `PATH` | | `[paths] vgmstream` | `NNNOTES_PATHS_VGMSTREAM` | `--vgmstream` | `vgmstream-cli` executable; unset: `vgmstream-cli` on `PATH` | | `[paths] node` | `NNNOTES_PATHS_NODE` | `--node` | Node.js executable; unset: `node` on `PATH` | @@ -183,7 +183,7 @@ story text); it is not a setting. | `site build --live2d` / `--all-live2d` | catalog (bundles from the CDN of the site's first region), `[paths] apk`, `[paths] player`; not `node`; master data only for the model names (optional: without it `models.json` has no names) | | `site build --player-only` / `--reingest-json` | `[paths] player` | | `music-data` | catalog, `[master] key` + `iv` (not with `--decoded-master`); `--master-files` also `[catalog] region`; `--apk-master` also `[paths] apk`; `--decoded-master` also `[catalog] region` and master data with its `MasterManifest.json`; `[paths] apk` (optional otherwise) for the client version | -| `export assets`, `depot plan` | catalog (bundles are fetched into the depot); `[paths] apk` for the bundles inside the APK (without it they are reported as `source.absent`); master data for `--views`; with `--stored-catalog` a catalog version of the depot instead of the current catalog | +| `export assets`, `depot plan` | catalog (bundles are fetched into the depot); `[paths] apk` for the bundles inside the APK (without it they are reported as `source.absent`) and its built-in Unity files ([stages.md](stages.md#the-apks-built-in-files)); master data for `--views`; with `--stored-catalog` a catalog version of the depot instead of the current catalog | | `depot run` | the depot; `--fetch` also what fetching needs (region, `cdn`, bundle key, `[paths] apk`) | | `catalog versions` / `import` / `diff`, `depot verify` | the depot; `import` reads the APK's catalog from `[paths] apk` when it is set | | `catalog fetch` | the depot, `[catalog] region` and `language`, that region's `cdn`; its `api` (optional) for the resource version label | diff --git a/docs/stages.md b/docs/stages.md index f0473c6..c6746a8 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -19,15 +19,15 @@ gzip input and Japanese remote asset placeholders). | Stage | Subject | Inputs | Outputs | |---|---|---|---| | `catalog.index` | a catalog (remote + APK) | the catalog files | every location (with the bundle hash, crc and size), bundles and raw remote files | -| `unity.census` | a bundle | the bundle | serialized files, externals, objects (file, path id, class, size, type hash, name, script, the scripts an AnimationClip's bindings name), container, scene hashes, dependencies, script fields | +| `unity.census` | a bundle, or a built-in file of the APK | the bundle | serialized files, externals, objects (file, path id, class, size, type hash, name, script, the scripts an AnimationClip's bindings name), container, scene hashes, dependencies, script fields | | `link.scripts` | global | the censuses holding MonoScripts | serialized file and path id -> assembly, namespace, class | -| `unity.export` | a bundle | the bundle, the script entries its MonoBehaviours use | the objects' artifacts and one item per object | +| `unity.export` | a bundle, or a built-in file of the APK | the bundle, the script entries its MonoBehaviours use (and, for files without type trees, the type trees nnnotes ships for their classes) | the objects' artifacts and one item per object | | `sprite.crop` | a SpriteAtlas | the atlas' JSON, the images of its textures, the metas of the sprites of other bundles it packs | the sprites' images | | `cri.audio` | ACB content | `acb` (a raw file of the catalog, or the `acb` artifact of a cue sheet held in a bundle), `awb` (an AWB the catalog pairs with it), `boot` (the game's boot data: the key's source) | FLAC per stream, `cues.json`, `streams.json` | | `cri.movie` | USM content | `usm` (a raw file of the catalog, or the `usm` artifact of a movie held in a bundle), `boot` | every stream as stored (video, alpha, audio, subtitles), `movie.mkv` (or `movie.webm`) | | `live2d.model` | a Live2D model key (`Character/Live2D///model/`) whose bundle is selected | `bundle:`: the bundles of the key's dependency closure in the order the extractor loads them (the order is part of the key); context `internalId` (the key's location); atom `live2d.extract_model` (the extractor of `nnnotes export live2d`, run unchanged) | `live2d.model:#`: every file `nnnotes export live2d` writes for the model, `` relative to the model's directory; placed under `/` | | `spine.skeleton` | a SkeletonDataAsset of a selected bundle, by its stable address `:` (in a bundle of several serialized files the file's tag follows the bundle name, as in `link.addresses`) | `bundle:`: the bundles holding the serialized files the asset's file reaches through its externals; context `object` (the asset's object id); atom `spot.write_skeleton` (the Spine writer of `nnnotes export spot`) | `spine.skeleton:#`: the skeleton (`.json` or `.skel`), its atlases (`.atlas`) and their page images, the bytes `nnnotes export spot` writes into `spine/`; one item for the asset | -| `link.addresses` | global | catalog index, censuses | key -> objects, serialized file -> bundle, bundle -> keys | +| `link.addresses` | global | catalog index, censuses (the built-in files' too) | key -> objects, serialized file -> bundle or built-in file, bundle -> keys | | `link.artifacts` | global | the above, the results | object id -> artifact ids | | `view.` | a catalog (`main`) | `master:
` per master table it reads, `rules` (the view's rule and the resolvers it uses), `addresses` (the part of the address table it read: the keys looked up and the prefixes listed); no parameters, atoms or context | `view.:#view` (facts: rows, entries, gaps, unreferenced) | | `view.voices` | a catalog (`main`) | `master:
` per master table its rules read, `rules`, `addresses`, `sheets` (each cue sheet the `cri.audio` results are known by that a voice names or the rules' prefixes list: its task and ACB content id), `acb:` (the ACB of each decoded one), `episodes` (per story episode and part, the `unity.export` artifact holding the object its key names), `adv:` (those contents); context `audio` (the keys of those `cri.audio` results); runs after `unity.export` and `cri.audio` | `view.voices:#view` (facts: rows, entries, gaps, unreferenced streams) | @@ -61,7 +61,7 @@ with its reason code; any other error of a converter gives it with `generic.erro | Font | `font/2` | `font`: the font file, `json`: the typetree without `m_FontData` | `sniff` | | | MonoScript | `scripts.json/1` | `#scripts.json`: every MonoScript of the bundle | | | | AssetBundle | `bundle.json/1` | `#bundle.json`: serialized files, externals, the AssetBundle objects | | | -| any other class | `generic.json/1` | `json`: the typetree | `sniff` | | +| any other class | `generic.json/2` | `json`: the typetree | `sniff` | an object that can be read only in part (see below): that part, `unsupported.typetree` | Every object a converter takes has a `json` artifact or is contained in another object's. Each item lists in `refs` the objects its documents reference (by object id, other bundles' objects included, the objects of its own @@ -75,6 +75,32 @@ data, `atlas` for its SpriteAtlas' entry; `atlas`; and that render data's `textu of a position `p` is `((p.x * uvTransform.x + uvTransform.y) / width, (p.y * uvTransform.z + uvTransform.w) / height)` of the render data's texture. +### The APK's built-in files + +With `[paths] apk` set, the player's own Unity files under `assets/bin/Data/` of the APK are subjects of +`unity.census` and `unity.export` like the bundles, named `apk/`: + +| Subject | Holds | +|---|---| +| `apk/data.unity3d` | `globalgamemanagers` (player, graphics and quality settings, the resource table), `globalgamemanagers.assets` (render pipeline assets, renderer data and features, post-process data, shaders), `level0`, `resources.assets`, `sharedassets0.assets`, `unity_builtin_extra` (built-in shaders) | +| `apk/datapack.unity3d` | `resources.assets` in split builds | +| `apk/Resources/unity default resources` | the serialized file of the engine's default meshes, textures, shaders and font | + +Their objects have the ids of any serialized file's (`unity_builtin_extra:12`, `unity default resources:10`), so a +bundle's reference into them names an object these tasks export, and `link.addresses` maps their files to their +subjects. Census always covers them; everything and `--select bundle:apk/*` export them. + +These files store no type trees. Objects of the engine's classes are read with the type trees UnityPy has for the +file's Unity version; MonoBehaviours with the type trees nnnotes ships for the game's classes it reads there +(`nnnotes.unity.scripttypes`, generated by `scripts/gen_typetrees.py`): the render pipeline assets, renderer data, +renderer features, post-process data and the game's settings objects. A shipped type tree is used only when the type +hash the file stores is the one it was made for and its read consumes exactly the object's bytes; the task's +context holds the shipped type trees its MonoBehaviours use, so a new shipped tree runs those tasks again. A +MonoBehaviour without one is `unsupported.typetree` with its header (`m_GameObject`, `m_Enabled`, `m_Script`, +`m_Name`) as its typetree JSON; inside a hierarchy its component keeps that header and the prefab records the +issue. PlayerSettings, whose type tree reads the stored layout right up to `m_ActiveColorSpace` (by its stored +type hash), gives the fields up to it. + A task's key includes the atoms `reader` and `sniff`, the converter `generic.json`, and the converters (with their atoms) of the classes the bundle's census lists, so a converter's version bump runs again exactly the bundles holding its classes. `classes` (a list of class names) limits a task to the objects of those classes and the objects their @@ -153,7 +179,8 @@ and multi-audio paths are tested on synthetic movies only. The ids of the external tools are `-+sha256./` (the version as `vgmstream-cli -V` / `ffmpeg -version` report it): another build of a tool runs the tasks that use it again, and a task described for another build is refused as incompatible. The tools are `[paths] vgmstream` and `[paths] ffmpeg`, -else found on PATH. The `boot` input is the APK's `assets/bin/Data/data.unity3d` (`[paths] apk`), stored once. +else found on PATH. The `boot` input is the APK's `assets/bin/Data/data.unity3d` (`[paths] apk`), the input of +the `apk/data.unity3d` tasks. ## Atoms @@ -242,6 +269,7 @@ The codes of items that are `generic`, `unsupported` or `failed`, and of limits | `no_data.sprite` | a sprite whose render data names no texture, or whose texture has no image | | `unsupported.sprite.external_texture` | a sprite whose render data texture (not an atlas) is in another bundle | | `generic.error` | a converter failed on the object (the message says how); its typetree JSON instead | +| `unsupported.typetree` | an object of a file without type trees read in part: a MonoBehaviour without a shipped type tree of its class (its header), a class whose type tree fits only up to a field (those fields) | | `failed.read` | the object's typetree could not be read | | `failed.export` | the object's typetree JSON could not be made | | `source.absent` | a catalog entry whose file is not there (an APK entry without a file) | diff --git a/scripts/gen_typetrees.py b/scripts/gen_typetrees.py index 50e626e..6ce8f75 100644 --- a/scripts/gen_typetrees.py +++ b/scripts/gen_typetrees.py @@ -3,10 +3,11 @@ nnnotes reads a few MonoBehaviours of the APK's boot data (`assets/bin/Data/data.unity3d`): the URP pipeline assets, renderer data, renderer features and post-process data, DOTween and TextMeshPro settings, and the sound -and app-config defaults. IL2CPP player builds strip script typetrees from that data, so nnnotes ships the -typetrees of exactly these classes (CLASSES below) in `src/nnnotes/unity/typetrees/.json`, one file per -Unity version. Each entry holds the typetree nodes as [level, type, name, meta flag] rows and the type hash the -serialized file records for the class; the file also records the game version it was generated from. +and app-config defaults (the `apk/data.unity3d` tasks of `unity.export`, and PlayerData). IL2CPP player builds +strip script typetrees from that data, so nnnotes ships the typetrees of exactly these classes (CLASSES below) in +`src/nnnotes/unity/typetrees/.json`, one file per Unity version (read by nnnotes.unity.scripttypes). +Each entry holds the typetree nodes as [level, type, name, meta flag] rows and the type hash the serialized file +records for the class; the file also records the game version it was generated from. The rows have the node layout Unity itself serializes for a MonoBehaviour (as in the typetrees of asset bundles built with them): root type `MonoBehaviour`; a `string` is `string / Array / int size / char data`; built-in @@ -34,7 +35,7 @@ not read back stops the script without writing anything. 3. When a new game version keeps the Unity version, the file is replaced; a new Unity version adds a file. Keep the older files while the game versions they describe are still in use. -4. When code starts reading another MonoBehaviour class with `PlayerData.mono`, add the class to CLASSES and +4. When code starts reading another MonoBehaviour class of these files, add the class to CLASSES and regenerate. """ @@ -47,7 +48,8 @@ from UnityPy.helpers.TypeTreeGenerator import TypeTreeGenerator -from nnnotes.unity.player import TYPETREE_FORMAT, PlayerData, dump_typetrees, typetree_key, typetree_node +from nnnotes.unity.player import PlayerData +from nnnotes.unity.scripttypes import TYPETREE_FORMAT, dump_typetrees, typetree_key, typetree_node # (assembly, namespace, class) as the MonoScript objects name them CLASSES = ( diff --git a/src/nnnotes/cli/assets.py b/src/nnnotes/cli/assets.py index 0c4eef3..6580554 100644 --- a/src/nnnotes/cli/assets.py +++ b/src/nnnotes/cli/assets.py @@ -19,7 +19,8 @@ catalogs {"main": [Input remote, Input apk]}: the catalog version's files in /catalogs/ index the catalog index document (nnnotes.catalog-index/1) bundles {stable bundle name: Input "bundle"} of the selected bundles and their dependencies (a bundle whose - bytes are not known yet: a Pending "fetch:") + bytes are not known yet: a Pending "fetch:"), and of the APK's built-in files + (`apk/data.unity3d`, `apk/Resources/unity default resources`: pipeline.builtins) selected the selected bundles' stable names, sorted (the subjects of the export stages) raw {stable raw file name: Input "raw"} of the selected raw files (only with a stage that reads them; export fetches them while the stages before run, and reading the fact waits for that) @@ -31,7 +32,8 @@ stable name up to `_assets_`, else up to its first `_`), `key:PREFIX` the bundles and raw files the catalog keys under PREFIX load (their dependency closure), `bundle:GLOB` bundles and raw files whose stable name matches. The census also covers the bundles the selected ones depend on (the dependencies of the keys they hold: a key's first -dependency is its own bundle), so script and atlas references resolve. +dependency is its own bundle), so script and atlas references resolve. With `[paths] apk` the census always covers +the APK's built-in files; everything and `bundle:GLOB` (`bundle:apk/*`) select them. Task cost estimates (scheduling only: never part of a key) are each stage's estimate scaled per stage by the costs measured in earlier runs (/costs/calibration.json, updated after every export). @@ -193,14 +195,16 @@ class Selection: raw: list # stable names of the selected raw files -def select(index: dict, selection: list[str]) -> Selection: - """The bundles and raw files a selection names in a catalog index (module documentation).""" +def select(index: dict, selection: list[str], builtin=()) -> Selection: + """The bundles and raw files a selection names in a catalog index, and of the built-in subjects `builtin` + (module documentation).""" from ..pipeline import catalogdb bundles = sorted(b["stable"] for b in index["bundles"]) raws = sorted(r["stable"] for r in index["rawFiles"]) + builtin = sorted(builtin) if selection == ["all"]: - return Selection(bundles, bundles, raws) + return Selection(sorted(bundles + builtin), sorted(bundles + builtin), raws) files = catalogdb.file_refs(index) reach = catalogdb.closures(index) chosen: set = set() @@ -223,8 +227,10 @@ def select(index: dict, selection: list[str]) -> Selection: and files.get(loc["dependencies"][0]) in chosen ): deps |= reach[loc["id"]] + globs = [v for k, _, v in (s.partition(":") for s in selection) if k == "bundle"] + own = [s for s in builtin if any(fnmatch.fnmatchcase(s, g) for g in globs)] pick = lambda s, kind: sorted(n for k, n in s if k == kind) # noqa: E731 - return Selection(pick(chosen, "bundle"), pick(deps, "bundle"), pick(chosen, "raw")) + return Selection(sorted(pick(chosen, "bundle") + own), sorted(pick(deps, "bundle") + builtin), pick(chosen, "raw")) # ---------------------------------------------------------------- settings @@ -669,11 +675,18 @@ def facts(self, pipeline: list[str], fetch: bool, background: bool = False) -> d """The planning facts (module documentation). `fetch`: fetch the files whose content is not known yet (else they are Pending); what cannot be read is in problems() (after wait()). `background`: the raw files are fetched while the caller goes on (the fact waits for them when it is read).""" + from ..pipeline import builtins + index = self.index() - sel = select(index, self.selection) + try: + builtin = builtins.inputs(self.depot, self.apk) + except (OSError, zipfile.BadZipFile): + raise ConfigError(f"setting paths.apk: {self.apk} is not a readable APK") from None + sel = select(index, self.selection, builtin) self._problems = [] bundles = {b["stable"]: b for b in index["bundles"]} - inputs = self._inputs("bundle", [bundles[s] for s in sel.census], fetch) + inputs = self._inputs("bundle", [bundles[s] for s in sel.census if s not in builtin], fetch) + inputs = inputs.plus(builtin) sel.selected = [s for s in sel.selected if s in inputs] # files that cannot be read: problems() facts = { "catalogs": self.catalogs_fact(), @@ -696,7 +709,7 @@ def facts(self, pipeline: list[str], fetch: bool, background: bool = False) -> d facts["raw"] = Later(self._later) else: facts["raw"] = job[0](*job[1:]) - facts["boot"] = boot_input(self.depot, self.apk) + facts["boot"] = boot_input(builtin) self.selected = sel self._facts, self._names = facts, {} return facts @@ -1048,6 +1061,10 @@ def waiting(self) -> list[str]: """The subjects not fetched yet.""" return [k for k, v in self._items.items() if isinstance(v, Pending)] + def plus(self, items: dict) -> Inputs: + """These inputs and `items` ({subject: Input}).""" + return Inputs({**self._items, **items}) + class Later(Mapping): """A mapping a background job makes (Future): reading it waits for the job.""" @@ -1138,7 +1155,7 @@ def _write_report(d: Path, name: str, doc) -> Path: def cmd_export(args, cfg): - from ..pipeline import layout + from ..pipeline import builtins, layout from ..pipeline.orchestrate import Orchestrator, failure try: @@ -1176,9 +1193,12 @@ def cmd_export(args, cfg): ws.run_index() facts = ws.facts(pipeline, fetch=True, background=True) sel = ws.selected + own = sum(1 for s in sel.census if builtins.is_builtin(s)) + picked = sum(1 for s in sel.selected if builtins.is_builtin(s)) ws.log( - f"{len(sel.selected)} bundles selected ({len(sel.census)} with their dependencies), " - f"{len(sel.raw)} raw files" + f"{len(sel.selected) - picked} bundles selected ({len(sel.census) - own} with their dependencies), " + + (f"{picked} of the APK's {own} built-in files, " if own else "") + + f"{len(sel.raw)} raw files" + (" (fetched while the bundle stages run)" if isinstance(facts["raw"], Later) else "") + f"; {sum(p['kind'] == 'bundle' for p in ws.problems())} bundles not readable" ) diff --git a/src/nnnotes/pipeline/builtins.py b/src/nnnotes/pipeline/builtins.py new file mode 100644 index 0000000..81edc98 --- /dev/null +++ b/src/nnnotes/pipeline/builtins.py @@ -0,0 +1,71 @@ +"""The game's own Unity files in its APK as object sources. + +The player reads files under `assets/bin/Data/` of the APK: `data.unity3d` (a bundle of the global managers, +globalgamemanagers.assets, the first scene level0, resources.assets, sharedassets0.assets and unity_builtin_extra), +`datapack.unity3d` in split builds, and the serialized file `Resources/unity default resources` (which bundles name +`Library/unity default resources`). Each of them the APK holds is a subject of unity.census and unity.export: +`apk/` (`apk/data.unity3d`, `apk/Resources/unity default resources`). Their objects +have the ids of any serialized file's (`unity_builtin_extra:12`, `unity default resources:10`): a serialized file +outside a bundle takes the last part of its path as its name (serialized_name), as references into it name it. + +inputs(depot, apk) gives them as Inputs (role "bundle"), each stored in the depot once; for an APK file the set is +recorded by the APK's content id (the APK is hashed once, depot.identify memo), so a later call reads nothing of it. +""" + +from __future__ import annotations + +from pathlib import Path + +from ..source.apkset import ApkSet +from . import contract +from .contract import Input + +__all__ = ["DATA", "DATA_DIR", "MEMBERS", "PREFIX", "inputs", "is_builtin", "serialized_name"] + +DATA_DIR = "assets/bin/Data/" +MEMBERS = ("data.unity3d", "datapack.unity3d", "Resources/unity default resources") +PREFIX = "apk/" +DATA = PREFIX + "data.unity3d" # the boot data (the CRI stages read the HCA keycode from it) + + +def is_builtin(subject: str) -> bool: + return subject.startswith(PREFIX) + + +def serialized_name(subject: str) -> str | None: + """The name a built-in subject's file is loaded under: the last part of its path (a serialized file outside a + bundle is named so); None for a catalog bundle.""" + return subject.rsplit("/", 1)[-1] if is_builtin(subject) else None + + +def _read(depot, apk: Path) -> dict: + """{member: {sha256, size}} of the built-in files an APK holds, their bytes stored in `depot`.""" + out = {} + with ApkSet(apk) as z: + names = set(z.namelist()) + for m in MEMBERS: + if DATA_DIR + m in names: + data = z.read(DATA_DIR + m) + out[m] = {"sha256": depot.put(data), "size": len(data)} + return out + + +def inputs(depot, apk) -> dict[str, Input]: + """{subject: Input "bundle"} of the built-in files of an APK (a file, an APK set or a directory; see the module + documentation), sorted; {} without an APK.""" + if apk is None: + return {} + apk = Path(apk) + if apk.is_dir(): + found = _read(depot, apk) + else: + name = f"builtins:{depot.identify(apk)[0]}" + known = depot.named("files", name) + found = contract.loads(depot.read(known[0])) if known is not None and depot.has(*known) else None + if found is None or not all(depot.has(r["sha256"], r["size"]) for r in found.values()): + found = _read(depot, apk) + depot.identify(depot.local(depot.put(contract.encode(found))), "files", name) + return { + PREFIX + m: Input("bundle", r["sha256"], int(r["size"]), m.rsplit("/", 1)[-1], ({"kind": "depot"},)) + for m, r in sorted(found.items()) + } diff --git a/src/nnnotes/pipeline/census.py b/src/nnnotes/pipeline/census.py index 23a75cb..7aa1d50 100644 --- a/src/nnnotes/pipeline/census.py +++ b/src/nnnotes/pipeline/census.py @@ -18,7 +18,8 @@ classes objects per class; `objects` the total errors [{object, message}] of the objects whose header could not be read -A census is a function of the bundle's bytes: it holds no file name, time or path. +A census is a function of the bundle's bytes: it holds no file name, time or path (a serialized file outside a +bundle, one of the APK's built-in files, has the name its subject gives: builtins.serialized_name). """ from __future__ import annotations @@ -26,7 +27,7 @@ import re from ..unity.atoms import impl_id -from . import contract +from . import builtins, contract from .contract import Cost, Input from .stages import Output, Pending, Stage @@ -249,11 +250,11 @@ def head(node, fields=HEAD_FIELDS): } -def census_file(path) -> dict: - """The census of a bundle file (decrypted UnityFS).""" +def census_file(path, name: str | None = None) -> dict: + """The census of a bundle file (decrypted UnityFS), or of a serialized file named `name`.""" from ..unity import reader as unity - env = unity.load(path) + env = unity.load(path, name) try: return census_env(env) finally: @@ -305,9 +306,10 @@ def bundle_subjects(index: dict) -> dict[str, dict]: # ---------------------------------------------------------------- stage class CensusStage(Stage): """unity.census: one bundle -> its census. Subjects and inputs come from the fact "bundles": {subject: Input - (role "bundle": the decrypted bundle's sha256, size, name, locators)}; a bundle not fetched yet is Pending - (raised by the fact, or given as a Pending / None). Artifact "#census". The census - reads the bundle with UnityPy: its atom is the reader's ("reader", the UnityPy version).""" + (role "bundle": the decrypted bundle's sha256, size, name, locators)}, the APK's built-in files included + (builtins); a bundle not fetched yet is Pending (raised by the fact, or given as a Pending / None). Artifact + "#census". The census reads the bundle with UnityPy: its atom is the reader's ("reader", the UnityPy + version).""" name = "unity.census" version = 2 @@ -331,7 +333,7 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: return Cost(0.002 + size / 40e6, (32 << 20) + 12 * size) def run(self, task, depot) -> Output: - doc = census_file(depot.open_input(task.input("bundle"))) + doc = census_file(depot.open_input(task.input("bundle")), builtins.serialized_name(task.subject)) content = depot.add(contract.encode(doc), "json") rec = contract.artifact( contract.artifact_id(task.id, "census"), diff --git a/src/nnnotes/pipeline/cristages.py b/src/nnnotes/pipeline/cristages.py index fbb0098..fd7f1b9 100644 --- a/src/nnnotes/pipeline/cristages.py +++ b/src/nnnotes/pipeline/cristages.py @@ -43,9 +43,8 @@ from pathlib import Path from ..formats.encoders import FLAC_LEVEL -from ..source.apkset import ApkSet from ..unity.atoms import impl_id -from . import contract +from . import builtins, contract from .contract import Cost, IncompatibleTask, Input from .stages import Output, Pending, Stage @@ -53,7 +52,6 @@ ACB_MAGIC, AWB_MAGIC, USM_MAGIC = b"@UTF", b"AFS2", b"CRID" ACB_KIND = "cri.acb" # semantics kind of the ACB artifacts of unity.export USM_KIND = "cri.usm" # semantics kind of the USM artifacts of unity.export -BOOT_IN_APK = "assets/bin/Data/data.unity3d" ACB_NAME = "sheet" # the stem of the ACB file vgmstream reads (names come from its cues) TOOL_REVISION = 1 FORMATS = ("mkv", "webm") @@ -70,30 +68,11 @@ # ---------------------------------------------------------------- inputs shared by the stages -def boot_input(depot, apk) -> Input | None: - """The input "boot" of the CRI stages: the boot data of an APK (`assets/bin/Data/data.unity3d`, where the HCA - keycode is read), stored in `depot` once; the APK is hashed once (depot.identify memo) and its boot data is - remembered by the APK's content id. A path that is not an .apk / .zip is the boot data itself. None without an - APK.""" - if apk is None: - return None - apk = Path(apk) - if not apk.is_dir() and apk.suffix.lower() not in (".apk", ".zip", ".apks", ".xapk"): - sha, size = depot.identify(apk) - return Input("boot", sha, size, apk.name, ({"kind": "file", "path": str(apk.resolve())},)) - if apk.is_dir(): - with ApkSet(apk) as z: - data = z.read(BOOT_IN_APK) - sha = depot.put(data) - return Input("boot", sha, len(data), "data.unity3d", ({"kind": "depot"},)) - apk_sha, _ = depot.identify(apk) - name = f"boot:{apk_sha}" - known = depot.named("files", name) - if known is None or not depot.has(*known): - with ApkSet(apk) as z: - data = z.read(BOOT_IN_APK) - known = depot.identify(depot.local(depot.put(data)), "files", name) - return Input("boot", known[0], known[1], "data.unity3d", ({"kind": "depot"},)) +def boot_input(builtin: dict) -> Input | None: + """The input "boot" of the CRI stages: the APK's `assets/bin/Data/data.unity3d`, where the HCA keycode is read, + of its built-in files (`builtin`, builtins.inputs); None when they lack it (no APK).""" + inp = builtin.get(builtins.DATA) + return None if inp is None else Input("boot", inp.sha256, inp.size, inp.name, inp.locators) def _boot(env) -> Input: diff --git a/src/nnnotes/pipeline/link.py b/src/nnnotes/pipeline/link.py index 9a6ea97..88363db 100644 --- a/src/nnnotes/pipeline/link.py +++ b/src/nnnotes/pipeline/link.py @@ -6,10 +6,11 @@ consumes. link.addresses (nnnotes.addresses/1), one task per catalog pair: the catalog index and the censuses of its bundles -(by stable bundle name) -> +(by stable bundle name) and of the APK's built-in files (subjects `apk/...`, pipeline.builtins) -> bundles {stable name: {name, files, assetBundleName, keys}}: the file name in this catalog version, its - serialized files, the AssetBundle name the census found and the keys located in it - files {serialized file: stable bundle name} (CAB -> bundle) + serialized files, the AssetBundle name the census found and the keys located in it; a built-in file + is listed by its subject, named by its path in the APK's `assets/bin/Data/`, without keys + files {serialized file: stable bundle name} (CAB -> bundle; `unity_builtin_extra` -> `apk/data.unity3d`) keys {primary key: [{location, type, internalId, bundle | raw, objects}]}: an asset location's bundle is the dependency whose container holds its internal id (compared ignoring case); `objects` the object ids the container path names (null when a dependency bundle has no census) @@ -22,7 +23,8 @@ its extension names, else the first by file and path id: the object the `original` layout puts at the path), "/[]" for another object under it with a name no other object under it has, else ":". In a bundle of several serialized files (their path ids may coincide) the path id form names -the file after the bundle: the part of the file name after its CAB- prefix ("scene_x.sharedAssets:12"). +the file after the bundle: the part of the file name after its CAB- prefix ("scene_x.sharedAssets:12"), or +"|" ("apk/data.unity3d|level0:3"). Objects that no container path names are not listed; `address_of` gives any object's address. link.artifacts (nnnotes.artifacts/1), one global task: the results of the stages that export objects (unity.export, @@ -41,7 +43,8 @@ import json import re -from . import catalogdb, census, contract +from ..unity import scripttypes +from . import builtins, catalogdb, census, contract from .contract import Cost, Input from .layout import EXT_CLASSES from .stages import Output, Pending, Stage @@ -55,11 +58,8 @@ # ---------------------------------------------------------------- scripts def script_entry(s: dict) -> str: - """ "assembly|namespace|class" of a census MonoScript record.""" - asm = s.get("assembly") or "" - if asm.endswith(".dll"): - asm = asm[:-4] - return f"{asm}|{s.get('namespace') or ''}|{s.get('class') or ''}" + """ "assembly|namespace|class" of a census MonoScript record (scripttypes.script_entry).""" + return scripttypes.script_entry(s.get("assembly"), s.get("namespace"), s.get("class")) def scripts(censuses) -> dict: @@ -81,12 +81,13 @@ def script_context(table: dict, refs) -> dict: # ---------------------------------------------------------------- addresses def file_tags(files) -> dict[str, str]: """{serialized file: the tag its path id addresses carry after the bundle name}: "" in a bundle of one file, - else the file name after its CAB- prefix ("" / ".sharedAssets"); "|" where those coincide.""" + else the file name after its CAB- prefix ("" / ".sharedAssets"); "|" where those coincide or a file + has no such prefix.""" files = list(files) if len(files) <= 1: return {f: "" for f in files} tags = {f: _CAB.sub("", f) for f in files} - if len(set(tags.values())) < len(tags): + if len(set(tags.values())) < len(tags) or not all(_CAB.match(f) for f in files): tags = {f: "|" + f for f in files} return tags @@ -128,9 +129,10 @@ def cls(e): def addresses(index: dict, censuses) -> dict: - """The address table (nnnotes.addresses/1) of a catalog index and the censuses of its bundles: {stable bundle - name: census document}, or (stable name, census document) pairs (read one at a time: only the container - entries and the objects they name are kept). Censuses of names the index does not list are ignored.""" + """The address table (nnnotes.addresses/1) of a catalog index and the censuses of its bundles and of the APK's + built-in files: {stable bundle name or built-in subject: census document}, or (name, census document) pairs + (read one at a time: only the container entries and the objects they name are kept). Censuses of other names + the index does not list are ignored.""" files_of = catalogdb.file_refs(index) listed = {b["stable"]: b for b in index["bundles"]} bundles, owners, info, groups, tags_of = {}, {}, {}, {}, {} @@ -138,6 +140,8 @@ def addresses(index: dict, censuses) -> dict: pairs = sorted(censuses.items()) if isinstance(censuses, dict) else censuses for stable, doc in pairs: b = listed.get(stable) + if b is None and builtins.is_builtin(stable): + b = {"name": stable[len(builtins.PREFIX) :]} if b is None: continue names = [f["name"] for f in doc["files"]] @@ -242,8 +246,12 @@ def reverse(doc: dict) -> dict[str, str]: # ---------------------------------------------------------------- stages +def _subject(tid: str) -> str: + return contract.parse_task_id(tid, census.CensusStage.name)[1] + + def _census_input(env, tid: str) -> Input: - return env.input_of(tid, "census", as_role="census:" + contract.parse_task_id(tid, census.CensusStage.name)[1]) + return env.input_of(tid, "census", as_role="census:" + _subject(tid)) def _read(depot, task, role: str) -> dict: @@ -289,9 +297,9 @@ def run(self, task, depot) -> Output: class AddressesStage(Stage): """link.addresses: a catalog index (input "index", the result of catalog.index:) and the censuses of its - bundles (inputs "census:") -> the address table. Subjects: those of catalog.index (the fact - "catalogs"). Waits for census tasks of its bundles that have not run; skips failed ones. Artifact - "link.addresses:#addresses".""" + bundles and of the APK's built-in files (inputs "census:") -> the address table. + Subjects: those of catalog.index (the fact "catalogs"). Waits for census tasks of its bundles that have not run; + skips failed ones. Artifact "link.addresses:#addresses".""" name = "link.addresses" version = 1 @@ -305,6 +313,7 @@ def inputs(self, subject: str, env) -> list[Input]: idx = env.input_of(tid, "index") stables = {b["stable"] for b in env.depot.document(idx.sha256)["bundles"]} wanted = {contract.task_id(census.CensusStage.name, s) for s in stables} + wanted |= {t for t in env.tasks(census.CensusStage.name) if builtins.is_builtin(_subject(t))} pending = [t for t in env.pending(census.CensusStage.name) if t in wanted] if pending: raise Pending(pending[0]) diff --git a/src/nnnotes/pipeline/objexport.py b/src/nnnotes/pipeline/objexport.py index 182e638..e06d5c8 100644 --- a/src/nnnotes/pipeline/objexport.py +++ b/src/nnnotes/pipeline/objexport.py @@ -24,7 +24,9 @@ import io from collections import defaultdict +from functools import cache +from ..unity import scripttypes from ..unity.atoms import ATOMS as ATOM_REGISTRY from ..unity.atoms import Impl, Unsupported, atom_id, resolve from ..unity.convert import ( @@ -45,9 +47,17 @@ prefab, textures, ) -from ..unity.convert.objects import BundleObjects, ReadError, encode_object, label, message, ref_object_id +from ..unity.convert.objects import ( + TYPETREE, + BundleObjects, + ReadError, + encode_object, + label, + message, + ref_object_id, +) +from . import builtins, contract, link from . import census as census_mod -from . import contract, link from .contract import Cost, IncompatibleTask, Input from .stages import Output, Pending, Stage @@ -81,6 +91,7 @@ def __init__(self, view, task, depot, fns: dict): blob_min=int(task.params["blobMin"]), png_level=int(task.params["png"]["level"]), emit=self._emit, + typetrees=task.context.get("typetrees", {}), ) classes = task.params.get("classes") self.selected = None if classes is None else frozenset(classes) @@ -190,14 +201,18 @@ def _fallback(self, info, code: str, why: str, status: str = "generic") -> None: self._commit(info, status, contract.reason(code, label(why))) def _generic(self, info) -> None: - """An object of a class without a specialized converter: its typetree JSON.""" + """An object of a class without a specialized converter: its typetree JSON (`generic` when it could be read + only in part).""" self._begin("generic.json") try: - generic.generic(self.ctx, info) + why = generic.generic(self.ctx, info) except Exception as e: self._failed(info, e) return - self._commit(info, "exported") + if why is None: + self._commit(info, "exported") + else: + self._commit(info, "generic", contract.reason(TYPETREE, label(why))) def _selected(self, cls: str) -> bool: return self.selected is None or cls in self.selected @@ -371,7 +386,9 @@ class ExportStage(Stage): "bundles" {subject: Input (role "bundle")}, as unity.census. It reads the census of its bundle (unity.census:) for the classes (the converters and atoms of its key) and the MonoScripts its objects reference, and the script table (link.scripts:all) for the entries of those: its context - {"scripts": {object id: "assembly|namespace|class"}}. Parameters: `blobMin` (bytes; byte arrays of this size or + {"scripts": {object id: "assembly|namespace|class"}}, and for the MonoBehaviours of files that store no type trees + (the APK's built-in files) the type trees nnnotes ships for their classes: "typetrees" {script entry: + {"typeHash", "nodes"}}. Parameters: `blobMin` (bytes; byte arrays of this size or more are blobs), `png.level` (zlib level of the PNG files). `reader`: another implementation of the reader atom (its id enters the keys).""" @@ -449,11 +466,16 @@ def uses(self, subject: str, env) -> dict: return atoms def context(self, subject: str, env) -> dict: - refs = census_mod.script_refs(self.census(subject, env)) + doc = self.census(subject, env) + refs = census_mod.script_refs(doc) if not refs: return {} table = self._docs.get(env.depot, env.input_of(SCRIPTS_TASK, "scripts").sha256)["scripts"] - return {"scripts": link.script_context(table, refs)} + out = {"scripts": link.script_context(table, refs)} + types = _script_types(doc, out["scripts"]) + if types: + out["typetrees"] = types + return out def depends(self, subject: str, env) -> list[str]: return [contract.task_id(CENSUS, subject), SCRIPTS_TASK] @@ -473,12 +495,34 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: def run(self, task, depot) -> Output: reader = self.reader if self.reader is not None else resolve("reader") _check_atoms(task, reader) - view = reader.fn(depot.input_bytes(task.input("bundle"))) + view = reader.fn(depot.input_bytes(task.input("bundle")), builtins.serialized_name(task.subject)) fns = {n: resolve(n).fn for n in task.atoms if n in ATOM_NAMES and n != "reader"} arts, items = BundleRun(view, task, depot, fns).run() return Output(arts, items) +@cache +def _embedded(unity_version: str) -> dict: + return scripttypes.script_types(unity_version) + + +def _script_types(doc: dict, scripts: dict) -> dict: + """{script entry: {"typeHash", "nodes"}} of the embedded type trees (nnnotes.unity.scripttypes, of each file's + Unity version) of the classes of the MonoBehaviours in the census' files that store no type trees ({} when it has + none).""" + out = {} + for f in doc["files"]: + if f["typetree"]: + continue + types = _embedded(f["unityVersion"]) + for o in f["objects"]: + s = o.get("script") if o["class"] == "MonoBehaviour" else None + entry = scripts.get(contract.object_id(s["file"], s["pathId"])) if s and s.get("file") else None + if entry in types and out.setdefault(entry, types[entry]) != types[entry]: + raise ValueError(f"{entry}: files of several Unity versions with different type trees of it") + return dict(sorted(out.items())) + + def _census_classes(env, subject: str) -> dict: """{class: objects} of a bundle's census ({} when its census has no result).""" try: diff --git a/src/nnnotes/unity/atoms/reader.py b/src/nnnotes/unity/atoms/reader.py index 0cb56ce..2f4cf9e 100644 --- a/src/nnnotes/unity/atoms/reader.py +++ b/src/nnnotes/unity/atoms/reader.py @@ -2,7 +2,8 @@ `BundleView` is what a stage reads a bundle through; `UnityBundle` is the reference implementation (UnityPy, the bundle loaded alone from its bytes: references into other bundles are not followed, resource files inside the -bundle -- `.resS` -- are read). Objects are named by `ObjectInfo` (serialized file name and path id), never by +bundle -- `.resS` -- are read). A serialized file outside a bundle (one of the APK's built-in files) is read the same +way, under the name it is given. Objects are named by `ObjectInfo` (serialized file name and path id), never by library objects, so another implementation can stand in. `texture_input` and `mesh_input` collect exactly what `texture.decode` and `mesh.arrays` need: a Texture2D's image @@ -69,6 +70,11 @@ def unity_version(self, file: str) -> str: """The Unity version a serialized file was written by.""" ... + def stores_typetrees(self, file: str) -> bool: + """Whether a serialized file stores the type trees of its objects (a player build's own files do not: its + script classes read only their header).""" + ... + def texture_input(self, obj: ObjectInfo) -> TextureInput: ... def mesh_input(self, obj: ObjectInfo) -> MeshInput: ... @@ -266,6 +272,9 @@ def unity_version(self, file: str) -> str: v = getattr(f, "unity_version", None) return v if isinstance(v, str) else ".".join(str(x) for x in f.version) + def stores_typetrees(self, file: str) -> bool: + return bool(getattr(self._files[file], "_enable_type_tree", True)) + def texture_input(self, obj: ObjectInfo) -> TextureInput: r = self._reader(obj) tex = r.read() @@ -306,11 +315,12 @@ def mesh_input(self, obj: ObjectInfo) -> MeshInput: ) -def open_bundle(data: bytes) -> UnityBundle: - """The BundleView of a bundle's bytes (decrypted UnityFS), loaded alone.""" +def open_bundle(data: bytes, name: str | None = None) -> UnityBundle: + """The BundleView of a bundle's bytes (decrypted UnityFS), loaded alone; of a serialized file's bytes with its + `name`.""" from ..reader import load_bytes - return UnityBundle(load_bytes(bytes(data))) + return UnityBundle(load_bytes(bytes(data), name)) def cost(facts: dict) -> dict: diff --git a/src/nnnotes/unity/convert/__init__.py b/src/nnnotes/unity/convert/__init__.py index 759d8ab..ce85c0c 100644 --- a/src/nnnotes/unity/convert/__init__.py +++ b/src/nnnotes/unity/convert/__init__.py @@ -38,6 +38,10 @@ - a string whose stored bytes are not UTF-8 is {"$hex": ""}; - NaN is {"$float": "nan"}, infinities 1e999 / -1e999. +A file of the player itself (the APK's built-in files) stores no type trees: its MonoBehaviours are read with the +type trees nnnotes ships for their classes (nnnotes.unity.scripttypes), and an object that can be read only in part +gives that part with `unsupported.typetree` (objects.py). + A converter's documented limit raises atoms.Unsupported with a reason code; the reason codes are listed in docs/stages.md. """ @@ -76,7 +80,7 @@ "scripts.json": 1, "bundle.json": 1, "shader.json": 1, - "generic.json": 1, + "generic.json": 2, "cri.usm": 1, } # class -> the converters its objects may use (every class may fall back to generic.json) @@ -118,6 +122,7 @@ "no_data.font", "unsupported.sprite.external_texture", "unsupported.cri.awb_external", + "unsupported.typetree", "failed.read", "failed.export", *(f"unsupported.class.{c}" for c in UNSUPPORTED_CLASSES), diff --git a/src/nnnotes/unity/convert/generic.py b/src/nnnotes/unity/convert/generic.py index 7199f98..c69e4b2 100644 --- a/src/nnnotes/unity/convert/generic.py +++ b/src/nnnotes/unity/convert/generic.py @@ -11,11 +11,13 @@ LOOSE = {"bundle.json": ("AssetBundle", "unity.bundle"), "scripts.json": ("MonoScript", "unity.scripts")} -def generic(ctx: BundleObjects, info) -> None: - """`json`: the object's typetree JSON.""" +def generic(ctx: BundleObjects, info) -> str | None: + """`json`: the object's typetree JSON. Returns why when the typetree could be read only in part (its part is + the JSON: BundleObjects.partial), else None.""" key = (info.file, info.path_id) - doc = ctx.tree(ctx.readers[key], ctx.tt(key)) + doc = ctx.tree(ctx.readers[key], ctx.tt(key, whole=False)) ctx.emit(key, "json", encode_object(doc), "json", "unity.object", "json", {}) + return ctx.partial.get(key) def loose_doc(ctx: BundleObjects, conv: str, entries: dict) -> dict: diff --git a/src/nnnotes/unity/convert/objects.py b/src/nnnotes/unity/convert/objects.py index b75a0d1..dc3a44b 100644 --- a/src/nnnotes/unity/convert/objects.py +++ b/src/nnnotes/unity/convert/objects.py @@ -7,6 +7,14 @@ Every object a document names by {"$ref"} is recorded in `seen` (by object id, other bundles' objects included): the objects an object's documents reference, which the caller clears before converting an object. + +A file that stores no type trees (a player build's own files) gives its MonoBehaviours only their header; they are +read with the task's type tree of their script class (`typetrees`, nnnotes.unity.scripttypes), checked against the +type hash the file stores and the object's size. One without such a tree is `unsupported.typetree` (TYPETREE): its +header is still read. Its other objects are read with the type trees of their classes UnityPy has for the file's +Unity version; for a class whose tree reads the stored layout right only up to a field (PREFIXES, by the type hash +the file stores) the typetree JSON is the fields up to it, also `unsupported.typetree`. The objects read in part are +in `partial`, with the reason. """ from __future__ import annotations @@ -17,10 +25,14 @@ from typing import cast from ...core.canon import encode +from ..atoms import Unsupported from ..ids import object_id +from ..scripttypes import typetree_node __all__ = [ "NO_SCRIPT", + "PREFIXES", + "TYPETREE", "BundleObjects", "Capture", "ReadError", @@ -33,6 +45,10 @@ ] NO_SCRIPT = "\0none" # the script entry of a MonoBehaviour whose m_Script is null +TYPETREE = "unsupported.typetree" # an object of a file without type trees read in part +# classes whose type tree (UnityPy's, for the file's Unity version) reads the stored layout right only up to a field, +# by the type hash the file stores: {(class, type hash): the last field read right} +PREFIXES = {("PlayerSettings", "936fd1913a4f0b5da2ce7cadb0ce9666"): "m_ActiveColorSpace"} _BYTE_TYPES = frozenset({"UInt8", "SInt8", "char"}) _SCALARS = (int, float, str, bool, type(None)) _ADDRESS = re.compile(r"0x[0-9a-fA-F]{6,}") @@ -135,6 +151,24 @@ def _ref_type_node(ref: dict, assets_file): raise ValueError(f"referenced type {asm}|{ns}|{cls} is not among the file's reference types") +def _cut(node, last: str): + """A type tree cut after its field `last` (None when it has no such field).""" + names = [c.m_Name for c in node.m_Children] + if last not in names: + return None + from UnityPy.helpers.TypeTreeNode import TypeTreeNode + + return TypeTreeNode( + node.m_Level, + node.m_Type, + node.m_Name, + node.m_ByteSize, + node.m_Version, + m_MetaFlag=node.m_MetaFlag, + m_Children=node.m_Children[: names.index(last) + 1], + ) + + class ReadError(Exception): """An object's typetree could not be read.""" @@ -156,17 +190,33 @@ class Capture: class BundleObjects: """The objects of one bundle (`view`, a reader atom's BundleView). `fns`: the atom functions by name; `scripts`: the script table entries the bundle's objects use ({MonoScript object id: "assembly|namespace|class"}); - `blob_min`: byte arrays of this size or more are blobs; `png_level`: the zlib level of PNG files.""" - - def __init__(self, view, fns: dict, *, scripts: dict, blob_min: int, png_level: int, emit: Emit): + `typetrees`: the type trees of script classes for the files that store none ({script entry: {"typeHash", + "nodes"}}); `blob_min`: byte arrays of this size or more are blobs; `png_level`: the zlib level of PNG files.""" + + def __init__( + self, + view, + fns: dict, + *, + scripts: dict, + blob_min: int, + png_level: int, + emit: Emit, + typetrees: dict | None = None, + ): self.view, self.fns, self.emit = view, fns, emit self.scripts, self.blob_min, self.png_level = dict(scripts), int(blob_min), int(png_level) + self.typetrees = dict(typetrees or {}) self.infos = view.objects() self.info = {(i.file, i.path_id): i for i in self.infos} self.readers = {k: view.reader(i) for k, i in self.info.items()} self.files = view.files() self.ext_files = {f: [_file_name(p) for p in view.externals(f)] for f in self.files} self.versions = {f: view.unity_version(f) for f in self.files} + self.stripped = frozenset(f for f in self.files if not view.stores_typetrees(f)) + self.partial: dict[tuple, str] = {} # objects read in part -> why + self._nodes: dict[tuple, object] = {} # objects read with a type tree of `typetrees` -> its node + self._script_nodes: dict[str, object] = {} self.containers: dict[tuple, str] = {} for path, file, pid in view.container(): k = (file, int(pid)) @@ -209,21 +259,7 @@ def _peek(self, r) -> str | None: node = r._get_typetree_node() hit = self._heads.get(id(node)) if hit is None or hit[0] is not node: - names = [c.m_Name for c in node.m_Children] - cut = None - if "m_Name" in names: - from UnityPy.helpers.TypeTreeNode import TypeTreeNode - - cut = TypeTreeNode( - node.m_Level, - node.m_Type, - node.m_Name, - node.m_ByteSize, - node.m_Version, - m_MetaFlag=node.m_MetaFlag, - m_Children=node.m_Children[: names.index("m_Name") + 1], - ) - hit = self._heads[id(node)] = (node, cut) + hit = self._heads[id(node)] = (node, _cut(node, "m_Name")) if hit[1] is None: return None name = r.read_typetree(hit[1], check_read=False).get("m_Name") @@ -241,16 +277,72 @@ def name_of(self, key: tuple) -> str | None: self.names[key] = name return self.names[key] - def tt(self, key: tuple) -> dict: - """The object's typetree (ReadError when it cannot be read).""" + def tt(self, key: tuple, *, whole: bool = True) -> dict: + """The object's typetree (ReadError when it cannot be read). An object of a file that stores no type trees + that can be read only in part (see the module documentation) raises Unsupported(TYPETREE); with `whole` False + it gives that part (and is in `partial`).""" + info = self.info[key] + if info.class_name == "MonoBehaviour" and info.file in self.stripped: + return self._script_tt(key, info, whole) try: - tt = self.view.typetree(self.info[key]) + tt = self.view.typetree(info) except Exception as e: - raise ReadError(message(e)) from None + last = PREFIXES.get((info.class_name, info.type_hash)) if info.file in self.stripped else None + if last is None: + raise ReadError(message(e)) from None + why = f"{info.class_name}: {message(e)}; read up to {last}, which its type tree reads right" + if whole: + raise Unsupported(TYPETREE, why) from None + tt = self._prefix(self.readers[key], last) + self.partial[key] = why name = tt.get("m_Name") self.names[key] = name if isinstance(name, str) else None return tt + def _script_tt(self, key: tuple, info, whole: bool) -> dict: + r = self.readers[key] + try: + head = r.read_typetree(check_read=False) # the class's own tree: the header + except Exception as e: + raise ReadError(message(e)) from None + entry = self.mono_script(key, head) + t = self.typetrees.get(entry) if entry is not None else None + if entry is None or entry == NO_SCRIPT: + s = head.get("m_Script") or {} + why = f"script {s.get('m_FileID')}:{s.get('m_PathID')} is unknown and the file stores no type trees" + elif t is None: + why = f"the file stores no type trees and the task has none for {entry}" + elif t["typeHash"] != info.type_hash: + why = f"{entry}: the file's type hash {info.type_hash} is not the type tree's {t['typeHash']}" + else: + node = self._script_nodes.get(entry) + if node is None: + node = self._script_nodes[entry] = typetree_node(t["nodes"]) + try: + tt = r.read_typetree(node, check_read=True) + except Exception as e: + why = f"{entry}: the type tree does not fit the object ({message(e)})" + else: + self._nodes[key] = node + self.partial.pop(key, None) + name = tt.get("m_Name") + self.names[key] = name if isinstance(name, str) else None + return tt + if whole: + raise Unsupported(TYPETREE, why) + self.partial[key] = why + return head + + def _prefix(self, r, last: str) -> dict: + """An object's typetree up to the field `last` (its class's tree cut after it).""" + cut = _cut(r._get_typetree_node(), last) + if cut is None: + raise ReadError(f"its type tree has no field {last}") + try: + return r.read_typetree(cut, check_read=False) + except Exception as e: + raise ReadError(message(e)) from None + def facts(self, key: tuple) -> dict: """The object facts of an artifact's provenance: file, path id, class, name, type hash, Unity version, container path.""" @@ -286,7 +378,7 @@ def mono_script(self, key: tuple, tt: dict) -> str | None: def script_class(self, key: tuple) -> str | None: """The class name of a MonoBehaviour from the script table (None when unknown).""" if key not in self._mono_scripts: - self.mono_script(key, self.tt(key)) + self.mono_script(key, self.tt(key, whole=False)) e = self._mono_scripts[key] return e.rsplit("|", 1)[-1] if e and e != NO_SCRIPT else None @@ -342,7 +434,8 @@ def tree(self, owner, tt: dict, *, captures: dict[str, Capture] | None = None, l name of an object of the prefab being walked (None for any other).""" self._captures, self._local = dict(captures or {}), local try: - doc = cast(dict, self._walk(owner, owner._get_typetree_node(), tt, "")) + node = self._nodes.get(self.key(owner)) or owner._get_typetree_node() + doc = cast(dict, self._walk(owner, node, tt, "")) left = self._captures finally: self._captures, self._local = {}, None diff --git a/src/nnnotes/unity/convert/prefab.py b/src/nnnotes/unity/convert/prefab.py index 845719e..1610952 100644 --- a/src/nnnotes/unity/convert/prefab.py +++ b/src/nnnotes/unity/convert/prefab.py @@ -10,7 +10,7 @@ from dataclasses import dataclass, field from types import SimpleNamespace -from .objects import NO_SCRIPT, BundleObjects, ReadError, encode_object, message +from .objects import NO_SCRIPT, TYPETREE, BundleObjects, ReadError, encode_object, message __all__ = ["Hierarchies", "Root", "hierarchies", "prefab"] @@ -132,7 +132,7 @@ def _subtree(ctx: BundleObjects, h: Hierarchies, f: str, g, root_tf: int): def _component(ctx: BundleObjects, h: Hierarchies, o, root: tuple, unresolved: list) -> dict: key = ctx.key(o) - tt = ctx.tt(key) + tt = ctx.tt(key, whole=False) out = {"type": o.type.name, "pathId": key[1]} if o.type.name == "MonoBehaviour": entry = ctx.mono_script(key, tt) @@ -184,10 +184,13 @@ def prefab(ctx: BundleObjects, info, h: Hierarchies, root: Root) -> None: if info.file in h.scene_of: doc["scene"] = h.scene_of[info.file] doc["nodes"] = nodes - issues = ( - [("script.unresolved", f"{len(unresolved)} MonoBehaviours whose script is not in the task's script table")] - if unresolved - else [] - ) + issues = [] + if unresolved: + issues.append( + ("script.unresolved", f"{len(unresolved)} MonoBehaviours whose script is not in the task's script table") + ) + partial = sum(1 for k in root.members if k in ctx.partial) + if partial: + issues.append((TYPETREE, f"{partial} components read in part (their file stores no type tree that fits)")) facts = {"nodes": len(nodes), "objects": len(root.members)} ctx.emit(key, "json", encode_object(doc), "json", "prefab", "json", facts, issues=issues) diff --git a/src/nnnotes/unity/player.py b/src/nnnotes/unity/player.py index 21c36ad..10638f7 100644 --- a/src/nnnotes/unity/player.py +++ b/src/nnnotes/unity/player.py @@ -3,10 +3,8 @@ `assets/bin/Data/data.unity3d` holds the global managers (graphics, quality, player settings) and `globalgamemanagers.assets` with the URP pipeline assets, renderer data, renderer features and post-process data. IL2CPP player builds -strip script typetrees there, so MonoBehaviours are read with typetrees that -ship with nnnotes: `typetrees/.json`, generated from the game's -managed assembly stubs by `scripts/gen_typetrees.py`. Each entry records the -type hash the serialized file stores for its class; a class without an entry, +strip script typetrees there, so MonoBehaviours are read with the typetrees +that ship with nnnotes (nnnotes.unity.scripttypes); a class without an entry, a different type hash or a read that does not consume exactly the object's serialized size raises UnsupportedVersion. `unity default resources` (built-in meshes) is read from the APK as well. @@ -16,10 +14,7 @@ import copy import io -import json -import re import struct -from importlib import resources from pathlib import Path from typing import cast @@ -30,63 +25,18 @@ from ..source.apkset import MANIFEST_IN_APK, ApkSet, manifest_version_name from ..source.config import ConfigError from .reader import DEFAULT_RESOURCES, deref, external_path, is_pptr +from .scripttypes import load_typetrees, typetree_key, typetree_node DATA_IN_APK = "assets/bin/Data/data.unity3d" DEFAULT_RESOURCES_IN_APK = "assets/bin/Data/Resources/unity default resources" COLOR_SPACE = {0: "Gamma", 1: "Linear"} HEADER = ("m_GameObject", "m_Script", "m_Enabled") -TYPETREE_FORMAT = 1 class UnsupportedVersion(ConfigError): """The APK's boot data has a MonoBehaviour class that nnnotes has no matching typetree for.""" -# ---------------------------------------------------------------- embedded typetrees -def typetree_key(assembly: str, namespace: str, cls: str) -> str: - """Key of a class in a typetree file: `assembly|namespace|class`, as its MonoScript names them.""" - return f"{assembly}|{namespace}|{cls}" - - -def typetree_file(unity_version: str): - """The embedded typetree file of a Unity version (an importlib.resources Traversable; it may not exist).""" - return resources.files("nnnotes.unity").joinpath("typetrees", f"{unity_version}.json") - - -def load_typetrees(unity_version: str) -> dict | None: - """The embedded typetrees of a Unity version, or None when nnnotes has none for it.""" - if not re.fullmatch(r"[0-9A-Za-z.]+", unity_version or ""): - return None - f = typetree_file(unity_version) - if not f.is_file(): - return None - doc = json.loads(f.read_text(encoding="utf-8")) - if doc.get("format") != TYPETREE_FORMAT or doc.get("unityVersion") != unity_version: - raise ValueError(f"typetree file {unity_version}.json: unexpected format or Unity version") - return doc - - -def typetree_node(rows: list) -> TypeTreeNode: - """TypeTreeNode from [level, type, name, meta flag] rows (byte size and version 0, as generated).""" - return TypeTreeNode.from_list([TypeTreeNode(lv, ty, nm, 0, 0, m_MetaFlag=mf) for lv, ty, nm, mf in rows]) - - -def dump_typetrees(doc: dict) -> str: - """Text of a typetree file: sorted keys, one class per line, LF.""" - head = {k: v for k, v in doc.items() if k != "classes"} - lines = [json.dumps(head, sort_keys=True, ensure_ascii=False, separators=(",", ":"))[:-1] + ',"classes":{'] - items = sorted(doc["classes"].items()) - for i, (k, v) in enumerate(items): - lines.append( - json.dumps(k, ensure_ascii=False) - + ":" - + json.dumps(v, sort_keys=True, ensure_ascii=False, separators=(",", ":")) - + ("," if i + 1 < len(items) else "") - ) - lines.append("}}") - return "\n".join(lines) + "\n" - - class PlayerData: def __init__(self, apk: Path): with ApkSet(apk) as z: diff --git a/src/nnnotes/unity/reader.py b/src/nnnotes/unity/reader.py index 2c640b8..fd600e4 100644 --- a/src/nnnotes/unity/reader.py +++ b/src/nnnotes/unity/reader.py @@ -92,12 +92,22 @@ def parse_blob(cls, reader, version): _install_blob_memo() -def load(path: str | Path) -> UnityPy.environment.Environment: - return UnityPy.load(str(path)) - - -def load_bytes(data: bytes) -> UnityPy.environment.Environment: - return UnityPy.load(io.BytesIO(data)) +def load(path: str | Path, name: str | None = None) -> UnityPy.environment.Environment: + """The environment of a bundle file or a serialized file (load_bytes).""" + if name is None: + return UnityPy.load(str(path)) + return load_bytes(Path(path).read_bytes(), name) + + +def load_bytes(data: bytes, name: str | None = None) -> UnityPy.environment.Environment: + """The environment of a bundle's bytes, or of a serialized file's: `name` names a serialized file outside a bundle + (`unity default resources`; references into it name it so), which UnityPy would name after the stream. A + bundle's files keep their own names.""" + if name is None: + return UnityPy.load(io.BytesIO(data)) + env = UnityPy.Environment() + env.load_file(io.BytesIO(data), name=name) + return env _closures = threading.local() diff --git a/src/nnnotes/unity/scripttypes.py b/src/nnnotes/unity/scripttypes.py new file mode 100644 index 0000000..2f7d7f3 --- /dev/null +++ b/src/nnnotes/unity/scripttypes.py @@ -0,0 +1,98 @@ +"""The type trees of game script classes that ship with nnnotes. + +IL2CPP player builds strip the type trees of script classes from the player's own files (`assets/bin/Data/` of the +APK: the global managers, the URP pipeline assets, renderer data, renderer features and post-process data), so a +MonoBehaviour there reads only its header. nnnotes ships the type trees of the classes it reads from those files in +`typetrees/.json`, generated from the game's managed assembly stubs by `scripts/gen_typetrees.py`: + + {"format": 1, "unityVersion", "gameVersion", "classes": {"assembly|namespace|class": {"typeHash", "nodes"}}} + +A class is keyed by the names its MonoScript stores (typetree_key); `nodes` are [level, type, name, meta flag] rows +of Unity's layout and `typeHash` the hex type hash the serialized file stores for the class. A tree fits an object +when the type hashes are equal and its read consumes exactly the object's serialized size. + +Nothing here imports UnityPy until typetree_node is called. +""" + +from __future__ import annotations + +import json +import re +from importlib import resources + +__all__ = [ + "TYPETREE_FORMAT", + "dump_typetrees", + "load_typetrees", + "script_entry", + "script_types", + "typetree_file", + "typetree_key", + "typetree_node", +] + +TYPETREE_FORMAT = 1 + + +def typetree_key(assembly: str, namespace: str, cls: str) -> str: + """Key of a class in a typetree file: `assembly|namespace|class`, as its MonoScript names them.""" + return f"{assembly}|{namespace}|{cls}" + + +def script_entry(assembly: str | None, namespace: str | None, cls: str | None) -> str: + """The script table entry of a class: "assembly|namespace|class", the assembly without `.dll`, an empty + namespace empty.""" + asm = assembly or "" + if asm.endswith(".dll"): + asm = asm[:-4] + return f"{asm}|{namespace or ''}|{cls or ''}" + + +def typetree_file(unity_version: str): + """The embedded typetree file of a Unity version (an importlib.resources Traversable; it may not exist).""" + return resources.files("nnnotes.unity").joinpath("typetrees", f"{unity_version}.json") + + +def load_typetrees(unity_version: str) -> dict | None: + """The embedded typetrees of a Unity version, or None when nnnotes has none for it.""" + if not re.fullmatch(r"[0-9A-Za-z.]+", unity_version or ""): + return None + f = typetree_file(unity_version) + if not f.is_file(): + return None + doc = json.loads(f.read_text(encoding="utf-8")) + if doc.get("format") != TYPETREE_FORMAT or doc.get("unityVersion") != unity_version: + raise ValueError(f"typetree file {unity_version}.json: unexpected format or Unity version") + return doc + + +def script_types(unity_version: str) -> dict[str, dict]: + """{script table entry (script_entry): {"typeHash", "nodes"}} of the embedded typetrees of a Unity version ({} + when nnnotes has none for it).""" + doc = load_typetrees(unity_version) + if doc is None: + return {} + return {script_entry(*k.split("|", 2)): v for k, v in sorted(doc["classes"].items())} + + +def typetree_node(rows: list): + """UnityPy TypeTreeNode from [level, type, name, meta flag] rows (byte size and version 0, as generated).""" + from UnityPy.helpers.TypeTreeNode import TypeTreeNode + + return TypeTreeNode.from_list([TypeTreeNode(lv, ty, nm, 0, 0, m_MetaFlag=mf) for lv, ty, nm, mf in rows]) + + +def dump_typetrees(doc: dict) -> str: + """Text of a typetree file: sorted keys, one class per line, LF.""" + head = {k: v for k, v in doc.items() if k != "classes"} + lines = [json.dumps(head, sort_keys=True, ensure_ascii=False, separators=(",", ":"))[:-1] + ',"classes":{'] + items = sorted(doc["classes"].items()) + for i, (k, v) in enumerate(items): + lines.append( + json.dumps(k, ensure_ascii=False) + + ":" + + json.dumps(v, sort_keys=True, ensure_ascii=False, separators=(",", ":")) + + ("," if i + 1 < len(items) else "") + ) + lines.append("}}") + return "\n".join(lines) + "\n" diff --git a/tests/cli/test_assets.py b/tests/cli/test_assets.py index 2e10549..85388f5 100644 --- a/tests/cli/test_assets.py +++ b/tests/cli/test_assets.py @@ -123,6 +123,25 @@ def bundle_bytes(doc: dict) -> bytes: return b"UnityFS\0" + contract.encode(doc) +def builtin_census() -> dict: + """The census of a player's data.unity3d: files without type trees and AssetBundle objects.""" + doc = cdoc("globalgamemanagers.assets", [obj(2, "Shader", "Blit")]) + head = dict(doc["files"][0], typetree=False) + files = [head, dict(head, name="unity_builtin_extra", objects=[obj(7, "Shader", "UI_Default")])] + return dict(doc, files=files, assetBundles=[], classes={"Shader": 2}, objects=2) + + +def builtin_apk(path, extra=()) -> Path: + """An APK holding the player's data.unity3d (builtin_census) and `extra` members {name: bytes}.""" + import zipfile + + with zipfile.ZipFile(path, "w") as z: + z.writestr("assets/bin/Data/data.unity3d", bundle_bytes(builtin_census())) + for n, data in dict(extra).items(): + z.writestr(n, data) + return path + + def catalog_bytes(extra=()) -> bytes: remote = {n: synth.remote(NAMES[n]) for n in NAMES} entries = [(NAMES[n], remote[n], []) for n in ("card_1", "card_2", "shared", "bg_x")] + list(extra) @@ -199,7 +218,7 @@ def run(self, task, depot): scripts = contract.loads(depot.input_bytes(task.input("scripts")))["scripts"] if any(o["class"] == "Crash" for f in doc["files"] for o in f["objects"]): raise ConfigError("setting paths.tool is not set") - container = {(e["file"], e["pathId"]): e["path"] for e in doc["assetBundles"][0]["container"]} + container = {(e["file"], e["pathId"]): e["path"] for b in doc["assetBundles"] for e in b["container"]} only = task.params["classes"] arts, items, first = [], [], None for f in doc["files"]: @@ -330,6 +349,60 @@ def test_select_adds_the_dependencies_to_the_census(): assert assets.select(idx, ["all"]).selected == sorted(stable.values()) +def test_select_always_census_the_built_in_files(): + from nnnotes.pipeline.catalogdb import index + + idx = index(catalog_bytes()) + stable = {n: contract.stable_bundle_name(NAMES[n]) for n in NAMES} + own = ["apk/data.unity3d", "apk/Resources/unity default resources"] + s = assets.select(idx, ["all"], own) + assert s.selected == s.census == sorted([*stable.values(), *own]) + s = assets.select(idx, ["group:bg"], own) + assert s.selected == [stable["bg_x"]] and s.census == sorted([stable["bg_x"], *own]) + s = assets.select(idx, ["bundle:apk/data*"], own) + assert s.selected == ["apk/data.unity3d"] and s.census == sorted(own) + + +def test_the_built_in_files_of_the_apk_are_exported(tmp_path, capsys, fake, monkeypatch): + monkeypatch.setattr("nnnotes.pipeline.catalogdb.apk_version", lambda apk: None) + d = setup_data(tmp_path, censuses(broken=False)) + apk = builtin_apk(tmp_path / "base.apk", {"assets/aa/catalog.bin": synth.CatalogWriter().build([])}) + base = [*flags(d), "--apk", apk, "depot", "plan", "--depot", tmp_path / "s", "--json"] + code, out, err = nn(capsys, *base, "--select", "group:bg") + ids = {n["id"] for n in json.loads(out)["nodes"]} + assert "unity.census:apk/data.unity3d" in ids and "unity.export:apk/data.unity3d" not in ids + code, out, err = nn( + capsys, + *flags(d), + "--apk", + apk, + "export", + "assets", + "-o", + tmp_path / "o", + "--depot", + tmp_path / "s", + "--workers", + "0", + ) + assert code == 0, err + root = tmp_path / "o" / "_bundles" / "apk" / "data.unity3d" + assert (root / "unity_builtin_extra" / "Shader" / "UI_Default~7.json").is_file() + assert (root / "globalgamemanagers.assets" / "Shader" / "Blit~2.json").is_file() + run = contract.loads((tmp_path / "o" / assets.REPORTS / "run.json").read_bytes()) + depot = Depot(tmp_path / "s") + key = {t["id"]: t["key"] for t in run["tasks"]}["link.addresses:main"] + rec = depot.result(key)["artifacts"][0] + table = contract.loads(depot.read(rec["content"]["sha256"])) + assert table["files"]["unity_builtin_extra"] == table["files"]["globalgamemanagers.assets"] == "apk/data.unity3d" + assert table["bundles"]["apk/data.unity3d"] == { + "name": "data.unity3d", + "files": ["globalgamemanagers.assets", "unity_builtin_extra"], + "assetBundleName": None, + "keys": [], + } + + # ---------------------------------------------------------------- export def test_export_writes_layouts_reports_and_hits_the_second_time(tmp_path, capsys, fake): d = setup_data(tmp_path, depots=("depot",)) @@ -1196,7 +1269,6 @@ def run(self, task, depot): def raw_apk(tmp_path, monkeypatch): """An APK with one raw file (a CRI sheet of the APK's catalog) and the CRI stage stand-in; -> (apk, name).""" - import zipfile monkeypatch.setattr( assets, @@ -1205,11 +1277,10 @@ def raw_apk(tmp_path, monkeypatch): ) monkeypatch.setattr("nnnotes.pipeline.catalogdb.apk_version", lambda apk: None) name = "cri_assets_embcri/sound/initialse_" + "ab" * 16 - apk = tmp_path / "base.apk" - with zipfile.ZipFile(apk, "w") as z: - z.writestr("assets/aa/catalog.bin", synth.CatalogWriter().build([("Cri/Initial/se", synth.local(name), [])])) - z.writestr("assets/aa/Android/" + name, b"@UTF sheet") - z.writestr("assets/bin/Data/data.unity3d", b"boot") + catalog = synth.CatalogWriter().build([("Cri/Initial/se", synth.local(name), [])]) + apk = builtin_apk( + tmp_path / "base.apk", {"assets/aa/catalog.bin": catalog, "assets/aa/Android/" + name: b"@UTF sheet"} + ) return apk, name diff --git a/tests/golden/unity.export.json b/tests/golden/unity.export.json index d310ee3..434f76a 100644 --- a/tests/golden/unity.export.json +++ b/tests/golden/unity.export.json @@ -7,7 +7,7 @@ "cri.usm": "cri.usm/1", "data": "data/2", "font": "font/2", - "generic.json": "generic.json/1", + "generic.json": "generic.json/2", "gltf.write": "numpy-2.4.6/1", "material.json": "material.json/1", "mesh.arrays": "numpy-2.4.6/1", @@ -26,68 +26,68 @@ }, "fixtures": { "all": { - "key": "10585d61586a4362e7e5a3c8857a9b583fac363abb5874e760d767fde9cb7d30", - "result": "3bd7b57ee922de132411cb8d027a00421fed5c5a595cdb3eef3e3cb5982b7374" + "key": "bfa791aaeaa8744197413dca21b95285355376a687dcb4a5b511410b4761bd28", + "result": "d9f811e5ca222529d22d1c5e2ec9cd89328d0229eac92b7e01d0cb59dbc56077" }, "bundle.json": { - "key": "fb99b80d44ff61bf4b90ea87e40a632c0f34372ddebd60dd00f74368c5f6b94d", - "result": "169b9bab48a8cb6423de878430b6291bf56999a5d640e9893dc2a1eeda9b93f9" + "key": "019bd61d3b8989c2f59289354d72d3857ed37cab1d90316359e6702ebab8a515", + "result": "f1516133579ff990af232a7b6d8b6303104c59997d20905f16aa280403d3a09e" }, "clip.json": { - "key": "535034c7ce7875a3ea19ebadbb4d12bca912621d38982cd47097cb1c66ff62c5", - "result": "fad3849534a5ad3892f968f567c88efa426088232df6e26a8f646757592fa135" + "key": "a9f5fb2a87fbcb197a5bc37f1c520b7b5a4a6fd5b5ad38b1198e9f8601dc223b", + "result": "6448026702f2239970f8041906335b3a2302a4463536d2f0394daa36906d9c79" }, "controller.json": { - "key": "522f6cefe4102041b197c0feda95de6668a7e88ab548b0d682ef26b492079fd1", - "result": "f17bc2432ad36b87f17e3fffec59eab429a803d23f0bc63db018ba2d99a847db" + "key": "472e6426b3af184e4fb17ccc0a147bd7b6fc9e5fc49f32d0ab2c2d79238417e1", + "result": "c43f0d2de335f10063d20f2d208b54cf7da37863860441a22781bce35f0cb1cc" }, "cri.usm": { - "key": "8d1d45652a6a631253b3bad7f096af988fcc3894831a8fe5e7ec90b939bc3427", - "result": "9a0ca28ce190fc8b5c03fc108b657825c1361a05b416894d2f9c55402cae62fd" + "key": "72cba01cb4fe8fa95d052a2fd17380eaacdd722dc0e5fc802bce8a301560e7a0", + "result": "9fecf00c65642c4ebb635839f79d9001cf4308b367711c2acaf82fa53326cff5" }, "data": { - "key": "98f3c28baedd007e88d9a48adccdad2a1970e63df6e487917cf1c6823feb651d", - "result": "a2c0972bc1c388038dedb9b70f4a8a6d67b58dfca0faac3c438610df125c7234" + "key": "1b44691876cbf5e28032ac2858d9616a8445e2357c543291e2806506c8c0a0b4", + "result": "cdaf30aeb760eef66ec41e029940a800af4d72b0ca0c4545c9c911b6c94fd331" }, "font": { - "key": "c1a50c8ae58a104313d93a7296464812f98c82d8c9aa516589b9d6e5cb89bffa", - "result": "d18d119de09ed000abca438357e72951a18d576c2b26b76932adf9621f67d2bd" + "key": "7d5549079bee9a0778d9d698545f21bfd0fea5466e0efa160661cfb527a83ea9", + "result": "d58abbc2a1b9990fadca993ac0502dc2d586b79e4947bfd1b6fb0a9f7bbc41b0" }, "generic.json": { - "key": "3f111bf015cc23af61fb457ad94be8c3479259d2e8ec2388390580161179afde", - "result": "a79ea2404c179705d54adb9c08f917acedd070bffccc01ba72da3f68af522cd4" + "key": "22deb0da7906ad444ca282fb2a8f993106268f748f2fab35357c049e5b020410", + "result": "a41f325a63c894bfe54aba013c1ee7aa6d674fddd49715c5e3898b7f40e6559c" }, "material.json": { - "key": "fd0603efb48c4582fa8664efcc44be7e238dc6168846dd5528ac6ec350732903", - "result": "373545644b49d9047bae346a78186d7bd0b1b5158d17a393b42050e505209e32" + "key": "b8b07211d9dbc578b3541a0e59653b01506a211426cc70b6e0aa2e82a948167e", + "result": "09085e48baebbda4a78a076e5dc66eeb19ae1f53710ec9f9b3553bb86dfdefde" }, "mesh.glb": { - "key": "d14c16a5c18145cc039a4568b4c8561b0120644c2de1e1abd1dd1be1f17917f0", - "result": "650489a1a9d4e52ae66bc44a101ad5c127e9aa836c8d69e4c08c662e70c73b47" + "key": "1c9b9a873c782e16650ef40cc0a41c17d37da55e05da6d44f47c488cc12d5027", + "result": "58304a1fd371109dbc38f99cd578ad8238453e769ae8c9d8562e1e187e2df1bc" }, "mono.json": { - "key": "5820ec6ca51a697d9ad2f3ed1ee5c8a849fd8f813b053f335b86b1b95082abd8", - "result": "cdabce4d56148f552c87738a9208fd36d0cc581590d7e5eb92f8e5b522de2266" + "key": "f2109d87be61d8cf5b0b527326d2cdc474785104091c55e770a18b31cd967b23", + "result": "02bc065f80513aad7e9afc21704d0f9591e0f110c9ff6fd818f357fe139686f8" }, "prefab.json": { - "key": "8994f1ef92388d443797f11c65a61a6346acf01ff0dc8bfbd36dc2fb2d2d23d2", - "result": "368986bcb1df65ea7fb906e01fe49b814c35e41212a079cdfcc5b70d5ed06559" + "key": "583e77337b68385457bb1f7ec789c943d69729706adc18b47d59354bd04fcf94", + "result": "3751c24516ea904f342e272234e57357b4c5d03ef6e90637c5c8bffbbca78d2a" }, "scripts.json": { - "key": "aafda4c0fe111458f3b48e85b8f60d0e9234dd250590591966a8ac3305d99ef9", - "result": "a2644c8fe5f8198b3771374f10ad01b91a9b111d74aff79b85a9b9b2e071e2c5" + "key": "6e6a37fb9135463601b78a8a5f699996221242585bcc31c63863158ca0a3f9dd", + "result": "9e9b03c721f1f163ed6e81c202b767a7af84be95e662d4d3b5a9b8d46bfe3392" }, "shader.json": { - "key": "a71aa982ce0da9929eaf721a1db903ac0565e1232c94a1d451cdf818f2c59f31", - "result": "fef75615b20f6dc056a4934606d1f094159e219b87482b5efae95c407d0f6d55" + "key": "f3c2c279ab2ed59a70932b19111c4f248593ec3ca9e0677b9402c38cb8505d03", + "result": "dac4f3d4f225dd3659ac78ab0b344daaa64e1a0a6f7dfe759f3ab49ffe3b3c43" }, "sprite.png": { - "key": "d42d38c45711b2de94fc4509b0dd8d919d01d123091f09d54e177df7e4cf751d", - "result": "5c6d4968cbc56a1eefb9c8c94b380487f9876f591e33acbef40bb735edbb70b9" + "key": "d35fe9220f41f10d16b8506f48f31774e9303a8ce6e7b2deba3eb2b26edc56d9", + "result": "ebe1ea68a2fb3289366624724ec4a9335ff6e42156ce4003c7669106e3981482" }, "tex.png": { - "key": "a5af40387bb2c9fe6ba31415af4e4028163eab23bd4114ad4384b57159ae1318", - "result": "54d91ffc00400f162f677f9e1525e34698b33dc439560b4372dffcd09a8dd73b" + "key": "1408f5b984cbcc1c74d23e3c6ef518288f2d112b6b2e4007aecb7f811ddfc809", + "result": "fec1a912d37045add008ea5c12941ea9c6e9eeb0cc0adb6b4d5be81c89ac26eb" } }, "libraries": { diff --git a/tests/helpers/fakeunity.py b/tests/helpers/fakeunity.py index 269cc1e..90fbbfd 100644 --- a/tests/helpers/fakeunity.py +++ b/tests/helpers/fakeunity.py @@ -62,6 +62,7 @@ "Animator": 95, "MonoBehaviour": 114, "MonoScript": 115, + "PlayerSettings": 129, "Font": 128, "AssetBundle": 142, "AnimatorController": 91, @@ -501,6 +502,9 @@ def node_of( raise TypeError(f"{name}: no node for {type(value).__name__}") +HEADER_FIELDS = ("m_GameObject", "m_Enabled", "m_Script", "m_Name") # a MonoBehaviour's own fields + + class FakeObjectReader: def __init__( self, @@ -515,8 +519,10 @@ def __init__( platform: int = ANDROID, node=None, bytes_fields=(), + short_read: bool = False, ): self.assets_file, self.path_id, self.tt, self.obj, self.raw = assets_file, path_id, tt, obj, raw + self.short_read = short_read # its class's tree reads fewer bytes than it has: a checked whole read fails self.class_id = CLASS_IDS[class_name] self.type = SimpleNamespace(name=class_name, value=self.class_id) self.byte_size = len(raw) @@ -525,18 +531,30 @@ def __init__( self.typetree_reads = 0 self._node, self._bytes_fields = node, tuple(bytes_fields) + def _stripped(self) -> bool: + """A MonoBehaviour of a file that stores no type trees: its class's own tree reads its header only.""" + return self.type.name == "MonoBehaviour" and not self.assets_file._enable_type_tree + def read_typetree(self, nodes=None, wrap: bool = False, check_read: bool = True): - """The typetree (a copy); with a cut node (UnityPy's head reads) only the fields the node names.""" + """The typetree (a copy); with a cut node (UnityPy's head reads) only the fields the node names. A stripped + MonoBehaviour read with a tree that does not name all its fields fails the size check.""" self.typetree_reads += 1 tt = copy.deepcopy(self.tt) + if nodes is None and self._stripped(): + nodes = self._get_typetree_node() + if nodes is None and check_read and self.short_read: + raise ValueError("Expected to read 996 bytes, but only read 992 bytes") if nodes is not None: names = [c.m_Name for c in nodes.m_Children] + if check_read and self._stripped() and set(tt) - set(names): + raise ValueError(f"Expected to read {len(self.raw) or 100} bytes, but only read 16 bytes") tt = {k: v for k, v in tt.items() if k in names} return tt def _get_typetree_node(self, node=None): if self._node is None: - self._node = node_of(self.tt, "Base", self.type.name, self._bytes_fields) + tt = {k: v for k, v in self.tt.items() if k in HEADER_FIELDS} if self._stripped() else self.tt + self._node = node_of(tt, "Base", self.type.name, self._bytes_fields) return self._node def peek_name(self): @@ -560,8 +578,11 @@ def read_bytes(self, n: int) -> bytes: class FakeSerializedFile: - def __init__(self, name: str, environment, externals=(), version=VERSION, unity_version="6000.3.12f1"): + def __init__( + self, name: str, environment, externals=(), version=VERSION, unity_version="6000.3.12f1", typetrees=True + ): self.name, self.environment, self.version, self.unity_version = name, environment, version, unity_version + self._enable_type_tree = typetrees self.objects: dict[int, FakeObjectReader] = {} self.externals = [SimpleNamespace(path=p) for p in externals] self.container = SimpleNamespace(container=[]) @@ -585,8 +606,8 @@ def __init__(self): self.resources: dict[str, bytes] = {} self.lookups: list[str] = [] - def file(self, name: str, externals=()) -> FakeSerializedFile: - f = FakeSerializedFile(name, self, externals) + def file(self, name: str, externals=(), typetrees: bool = True) -> FakeSerializedFile: + f = FakeSerializedFile(name, self, externals, typetrees=typetrees) self.assets.append(f) return f diff --git a/tests/pipeline/test_builtins.py b/tests/pipeline/test_builtins.py new file mode 100644 index 0000000..be9adec --- /dev/null +++ b/tests/pipeline/test_builtins.py @@ -0,0 +1,47 @@ +import zipfile +from unittest import mock + +from nnnotes.pipeline import builtins, contract +from nnnotes.pipeline.contract import Input +from nnnotes.pipeline.depot import Depot + +DATA = b"UnityFS data" +DEFAULTS = b"\0\0\0\0 a serialized file" + + +def apk(path, members): + with zipfile.ZipFile(path, "w") as z: + z.writestr("classes.dex", b"dex") + for m, data in members.items(): + z.writestr(builtins.DATA_DIR + m, data) + return path + + +def test_the_built_in_files_of_an_apk_are_subjects(tmp_path): + depot = Depot(tmp_path / "s") + a = apk(tmp_path / "base.apk", {"data.unity3d": DATA, "Resources/unity default resources": DEFAULTS}) + got = builtins.inputs(depot, a) + assert got == { + "apk/Resources/unity default resources": Input( + "bundle", contract.sha256(DEFAULTS), len(DEFAULTS), "unity default resources", ({"kind": "depot"},) + ), + "apk/data.unity3d": Input("bundle", contract.sha256(DATA), len(DATA), "data.unity3d", ({"kind": "depot"},)), + } + assert depot.read(contract.sha256(DEFAULTS)) == DEFAULTS + with mock.patch.object(builtins, "ApkSet", side_effect=AssertionError("read again")): + assert builtins.inputs(Depot(tmp_path / "s"), a) == got # remembered by the APK's content + assert builtins.inputs(depot, None) == {} + + +def test_an_apk_directory_and_its_members(tmp_path): + d = tmp_path / "apk" + d.mkdir() + apk(d / "base.apk", {"data.unity3d": DATA, "datapack.unity3d": b"UnityFS pack"}) + assert sorted(builtins.inputs(Depot(tmp_path / "s"), d)) == ["apk/data.unity3d", "apk/datapack.unity3d"] + + +def test_serialized_names(): + assert builtins.serialized_name("apk/Resources/unity default resources") == "unity default resources" + assert builtins.serialized_name("apk/data.unity3d") == "data.unity3d" + assert builtins.serialized_name("card_1") is None + assert builtins.is_builtin("apk/data.unity3d") and not builtins.is_builtin("card_1") diff --git a/tests/pipeline/test_census.py b/tests/pipeline/test_census.py index 933a34b..d215e85 100644 --- a/tests/pipeline/test_census.py +++ b/tests/pipeline/test_census.py @@ -295,8 +295,8 @@ def test_census_stage(tmp_path, monkeypatch): bundle.write_bytes(b"UnityFS\0 not really") loaded = [] - def load(path): - loaded.append(str(path)) + def load(path, name=None): + loaded.append((str(path), name)) return scene_bundle()[0] monkeypatch.setattr(unity, "load", load) @@ -316,10 +316,15 @@ def load(path): assert rec["id"] == "unity.census:x#census" and rec["semantics"]["facts"]["objects"] == 7 assert rec["provenance"]["inputs"] == [{"role": "bundle", "sha256": sha}] assert contract.loads(depot.read(rec["content"]["sha256"])) == census.census_env(scene_bundle()[0]) - assert loaded == [str(bundle)] + assert loaded == [(str(bundle), None)] # a catalog bundle: its files keep their names assert execute(task, depot, registry(stage)).status == "hit" and len(loaded) == 1 with pytest.raises(ValueError, match="expected 'bundle'"): describe(stage, "x", None, Env(depot, {"bundles": {"x": Input("other", sha, size)}})) for waiting in (None, Pending("fetch:x_1.bundle")): with pytest.raises(Pending, match="fetch:x"): describe(stage, "x", None, Env(depot, {"bundles": {"x": waiting}})) + # a serialized file of the APK outside a bundle is loaded under the name references into it use + own = "apk/Resources/unity default resources" + task = describe(stage, own, None, Env(depot, {"bundles": {own: inp}})) + assert execute(task, depot, registry(stage)).status == "ran" + assert loaded[-1] == (str(bundle), "unity default resources") diff --git a/tests/pipeline/test_cristages.py b/tests/pipeline/test_cristages.py index 4f2c418..97b24f5 100644 --- a/tests/pipeline/test_cristages.py +++ b/tests/pipeline/test_cristages.py @@ -15,7 +15,7 @@ from nnnotes.core import cache from nnnotes.formats import cri, crikey -from nnnotes.pipeline import contract, cristages +from nnnotes.pipeline import builtins, contract, cristages from nnnotes.pipeline.contract import IncompatibleTask, Input, Task from nnnotes.pipeline.cristages import AUDIO, AudioStage, MovieStage from nnnotes.pipeline.depot import Depot @@ -106,14 +106,19 @@ def catalog_index(keys: dict, bundles=()) -> dict: } +def boot_input(depot, tmp) -> Input: + apk = tmp / "base.apk" + with zipfile.ZipFile(apk, "w") as z: + z.writestr(builtins.DATA_DIR + "data.unity3d", b"UnityFS boot data") + return cristages.boot_input(builtins.inputs(depot, apk)) + + def planning(depot, tmp, files: dict, keys=None, boot=True, bundles=()) -> Env: raw = {s: file_input(depot, tmp, s, d) for s, d in files.items()} - b = tmp / "data.unity3d" - b.write_bytes(b"UnityFS boot data") facts = { "raw": raw, "index": catalog_index(keys or {}, bundles), - "boot": cristages.boot_input(depot, b) if boot else None, + "boot": boot_input(depot, tmp) if boot else None, } return Env(depot, facts) @@ -163,28 +168,17 @@ def load(depot, doc, aid): # ---------------------------------------------------------------- the key's source -def test_boot_input_from_an_apk_is_stored_once(tmp_path): +def test_boot_input_is_the_data_of_the_apk(tmp_path): depot = Depot(tmp_path / "s") - apk = tmp_path / "base.apk" - with zipfile.ZipFile(apk, "w") as z: - z.writestr(cristages.BOOT_IN_APK, b"UnityFS boot") - z.writestr("classes.dex", b"dex") - a = cristages.boot_input(depot, apk) + a = boot_input(depot, tmp_path) assert (a.role, a.sha256, a.size, a.name, a.locators) == ( "boot", - contract.sha256(b"UnityFS boot"), - 12, + contract.sha256(b"UnityFS boot data"), + 17, "data.unity3d", ({"kind": "depot"},), ) - assert depot.read(a.sha256) == b"UnityFS boot" - with mock.patch.object(zipfile, "ZipFile", side_effect=AssertionError("read again")): - assert cristages.boot_input(Depot(tmp_path / "s"), apk) == a # remembered by the APK's content - d = tmp_path / "data.unity3d" - d.write_bytes(b"UnityFS other") - b = cristages.boot_input(depot, d) - assert b.sha256 == contract.sha256(b"UnityFS other") and b.locators[0]["kind"] == "file" - assert cristages.boot_input(depot, None) is None + assert cristages.boot_input({}) is None def test_hca_key_is_read_once_per_content(tmp_path): diff --git a/tests/pipeline/test_objexport.py b/tests/pipeline/test_objexport.py index ddee38c..6689c94 100644 --- a/tests/pipeline/test_objexport.py +++ b/tests/pipeline/test_objexport.py @@ -560,10 +560,62 @@ def bundle_movie() -> FakeEnvironment: return env -BUNDLES = {"all": bundle_all, "atlas": bundle_atlas, "plain": bundle_plain, "movie": bundle_movie} +# a player's own file: no type trees; MonoBehaviours whose classes have a shipped type tree (one of another type +# hash), and none +PLAYER = "globalgamemanagers.assets" +PLAYER_HASH, OTHER_HASH = "a1" * 16, "b2" * 16 +SETTINGS, IMAGE, CHANGED = "Fwk|Fwk.Sound|SoundVolumeSettings", "UnityEngine.UI|UnityEngine.UI|Image", "Game|Game|Moved" +PLAYER_SCRIPTS = {f"{PLAYER}:50": SETTINGS, f"{PLAYER}:51": IMAGE, f"{PLAYER}:52": CHANGED} +SETTINGS_ROWS = [ + [0, "MonoBehaviour", "Base", 0], + [1, "PPtr", "m_GameObject", 0], + [2, "int", "m_FileID", 0], + [2, "SInt64", "m_PathID", 0], + [1, "UInt8", "m_Enabled", 0x4000], + [1, "PPtr", "m_Script", 0], + [2, "int", "m_FileID", 0], + [2, "SInt64", "m_PathID", 0], + [1, "string", "m_Name", 0], + [1, "PPtr", "target", 0], + [2, "int", "m_FileID", 0], + [2, "SInt64", "m_PathID", 0], + [1, "float", "volume", 0], +] +PLAYER_TYPETREES = { + SETTINGS: {"typeHash": PLAYER_HASH, "nodes": SETTINGS_ROWS}, + CHANGED: {"typeHash": OTHER_HASH, "nodes": SETTINGS_ROWS}, +} + + +def bundle_player() -> FakeEnvironment: + env = FakeEnvironment() + f = env.file(PLAYER, externals=["Library/unity default resources"], typetrees=False) + h = bytes.fromhex(PLAYER_HASH) + settings = {"productGUID": "00", "m_ActiveColorSpace": 1, "m_MTRendering": True} + f.add(1, "PlayerSettings", settings, type_hash=bytes.fromhex("936fd1913a4f0b5da2ce7cadb0ce9666"), short_read=True) + f.add(6, "PlayerSettings", settings, type_hash=h, short_read=True) + f.add(2, "MonoBehaviour", _mono(0, pptr(50), "volume", target=pptr(10, 1), volume=0.5), type_hash=h) + f.add(3, "MonoBehaviour", _mono(20, pptr(51), "", m_Sprite=pptr(10, 1)), type_hash=h) + f.add(4, "MonoBehaviour", _mono(0, pptr(52), "moved", target=pptr(0), volume=1.0), type_hash=h) + f.add(5, "MonoBehaviour", _mono(0, pptr(51), "image", m_Sprite=pptr(10, 1)), type_hash=h) + f.add(20, "GameObject", _go("Canvas", [21, 3])) + f.add(21, "Transform", _transform(20)) + for pid, entry in ((50, SETTINGS), (51, IMAGE), (52, CHANGED)): + asm, ns, cls = entry.split("|") + f.add(pid, "MonoScript", {"m_Name": cls, "m_ClassName": cls, "m_Namespace": ns, "m_AssemblyName": asm}) + return env -def open_fake(data: bytes) -> UnityBundle: +BUNDLES = { + "all": bundle_all, + "atlas": bundle_atlas, + "plain": bundle_plain, + "movie": bundle_movie, + "player": bundle_player, +} + + +def open_fake(data: bytes, name: str | None = None) -> UnityBundle: return UnityBundle(BUNDLES[bytes(data).decode().removeprefix("fake:")]()) @@ -574,7 +626,7 @@ def classes_of(name): return sorted({r.type.name for f in BUNDLES[name]().assets for r in f.objects.values()}) -def task_for(depot, name, classes=None, params=None, scripts=None): +def task_for(depot, name, classes=None, params=None, scripts=None, typetrees=None): data = f"fake:{name}".encode() sha = depot.put(data) stage = ExportStage(FAKE_READER) @@ -586,7 +638,7 @@ def task_for(depot, name, classes=None, params=None, scripts=None): p, export_atoms(classes_of(name), FAKE_READER.id), (Input("bundle", sha, len(data), f"{name}.bundle", ({"kind": "depot"},)),), - {"scripts": dict(SCRIPTS if scripts is None else scripts)}, + {"scripts": dict(SCRIPTS if scripts is None else scripts), **({"typetrees": typetrees} if typetrees else {})}, ) @@ -1268,7 +1320,7 @@ def census_doc(view: UnityBundle) -> dict: continue classes[o.class_name] = classes.get(o.class_name, 0) + 1 rec = {"pathId": o.path_id, "classId": o.class_id, "class": o.class_name} - tt = view.typetree(o) + tt = view.reader(o).read_typetree(check_read=False) # a header read, as the census' ext = view.externals(f) def ref(s): @@ -1286,7 +1338,9 @@ def ref(s): if bindings: rec["bindingScripts"] = [ref(b) for b in bindings] objs.append(rec) - files.append({"name": f, "objects": objs}) + files.append( + {"name": f, "unityVersion": view.unity_version(f), "typetree": view.stores_typetrees(f), "objects": objs} + ) return { "schema": contract.CENSUS, "files": files, @@ -1357,6 +1411,68 @@ def test_describe_reads_the_census_and_the_script_table(tmp_path): describe(stage, "all", None, fresh) +def test_files_without_type_trees_read_their_scripts_with_the_shipped_ones(tmp_path): + depot = Depot(tmp_path / "s") + doc = run(depot, task_for(depot, "player", scripts=PLAYER_SCRIPTS, typetrees=PLAYER_TYPETREES)) + items = by_object(doc) + + def p(pid): + return oid(pid, PLAYER) + + def script(pid, cls): + return {"$ref": {"file": PLAYER, "pathId": pid, "class": "MonoScript", "name": cls}} + + assert items[p(2)]["status"] == "exported" + assert obj(depot, doc, f"{p(2)}#json") == { + "$script": SETTINGS, + "m_GameObject": None, + "m_Enabled": 1, + "m_Script": script(50, "SoundVolumeSettings"), + "m_Name": "volume", + "target": {"$ref": {"file": "unity default resources", "pathId": 10}}, + "volume": 0.5, + } + assert items[p(2)]["refs"] == [p(50), "unity default resources:10"] + # a class without a shipped type tree, or with one of another type hash: its header, and the reason + for pid, why in ((5, f"has none for {IMAGE}"), (4, f"type hash {PLAYER_HASH} is not the type tree's")): + assert items[p(pid)]["status"] == "generic" + assert items[p(pid)]["reason"]["code"] == "unsupported.typetree" and why in items[p(pid)]["reason"]["message"] + assert obj(depot, doc, f"{p(5)}#json") == { + "m_GameObject": None, + "m_Enabled": 1, + "m_Script": script(51, "Image"), + "m_Name": "image", + } + # a hierarchy keeps its components' headers and says so + assert art(doc, f"{p(20)}#json")["semantics"]["issues"] == [ + { + "code": "unsupported.typetree", + "message": "1 components read in part (their file stores no type tree that fits)", + } + ] + # a class whose type tree reads the stored layout right up to a known field (by its type hash): those fields + assert items[p(1)]["status"] == "generic" and items[p(1)]["reason"]["code"] == "unsupported.typetree" + assert "read up to m_ActiveColorSpace" in items[p(1)]["reason"]["message"] + assert obj(depot, doc, f"{p(1)}#json") == {"productGUID": "00", "m_ActiveColorSpace": 1} + assert items[p(6)]["status"] == "failed" and items[p(6)]["reason"]["code"] == "failed.read" + (component,) = obj(depot, doc, f"{p(20)}#json")["nodes"][0]["components"] + assert component["script"] == IMAGE and set(component["fields"]) == {"m_Enabled", "m_Script", "m_Name"} + assert items[p(3)]["status"] == "contained" + + +def test_the_context_holds_the_shipped_type_trees_of_its_scripts(tmp_path): + from nnnotes.unity import scripttypes + + shipped = scripttypes.load_typetrees("6000.3.12f1")["classes"]["DOTween.dll|DG.Tweening.Core|DOTweenSettings"] + entry = "DOTween|DG.Tweening.Core|DOTweenSettings" # as the script table names it + env = planning_env(Depot(tmp_path / "s"), names=("player",), scripts={**PLAYER_SCRIPTS, f"{PLAYER}:50": entry}) + t = describe(ExportStage(FAKE_READER), "player", None, env) + assert t.context["typetrees"] == {entry: shipped} + assert ( + "typetrees" not in describe(ExportStage(FAKE_READER), "all", None, planning_env(Depot(tmp_path / "t"))).context + ) + + def test_class_selection_keeps_only_its_converters_in_the_key(tmp_path): depot = Depot(tmp_path / "s") env = planning_env(depot) diff --git a/tests/unity/test_typetrees.py b/tests/unity/test_typetrees.py index e62edc1..ce285e6 100644 --- a/tests/unity/test_typetrees.py +++ b/tests/unity/test_typetrees.py @@ -12,17 +12,8 @@ from nnnotes import cli from nnnotes.source.config import ConfigError -from nnnotes.unity.player import ( - HEADER, - TYPETREE_FORMAT, - PlayerData, - UnsupportedVersion, - dump_typetrees, - load_typetrees, - manifest_version_name, - typetree_key, - typetree_node, -) +from nnnotes.unity.player import HEADER, PlayerData, UnsupportedVersion, manifest_version_name +from nnnotes.unity.scripttypes import TYPETREE_FORMAT, dump_typetrees, load_typetrees, typetree_key, typetree_node from tests.helpers import synth TYPETREE_DIR = resources.files("nnnotes.unity").joinpath("typetrees")