diff --git a/docs/architecture.md b/docs/architecture.md index cd89197..c55d9c0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -12,7 +12,7 @@ subpackage of `nnnotes` and imports only the layers below it. | Products | `nnnotes.site`, `nnnotes.musicdata` | the ournotes-player site (charts, models, stories, indexes); the music data file and its deck statistics | | Pipeline | `nnnotes.pipeline` | stages, task descriptions and results, the content-addressed depot, plans, layouts, the catalog versions, the views of the master data | | Content | `nnnotes.content` | the game's subjects as exports: ADV episodes and stories, lives and scores, spots, rooms, the UI library | -| Unity objects | `nnnotes.unity` | bundle reading, object ids, the atoms (the pure functions objects are converted with: texture decoding, sprite images, mesh arrays, glTF, PNG, file sniffing), the converters (one per Unity class, `nnnotes.unity.convert`), the typetree exporter, the player data, Live2D prefabs, text styles and TextMesh Pro fonts | +| Unity objects | `nnnotes.unity` | bundle reading, object ids, the atoms (the pure functions objects are converted with: texture decoding, sprite images, mesh arrays, glTF, PNG, file sniffing), the converters (one per Unity class, `nnnotes.unity.convert`), the type trees shipped for the player's script classes (`nnnotes.unity.scripttypes`), the typetree exporter, the player data, Live2D prefabs, text styles and TextMesh Pro fonts | | File formats | `nnnotes.formats` | formats read from bytes: CRI ACB/AWB and HCA, USM movies, CRI Lips data, shaders, motions | | Sources | `nnnotes.source` | the configuration, the APK, Addressables catalogs and bundles, the game API, master data, languages, `Session` | | Core | `nnnotes.core` | the one JSON writer, canonical JSON and its hashes, media types, the process caches and the downloads cache | diff --git a/docs/configuration.md b/docs/configuration.md index 532e9b9..9aedab4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -103,7 +103,7 @@ exist when they are given. | `[paths] catalog` | `NNNOTES_PATHS_CATALOG` | `--catalog` | a catalog `.bin` file to read instead of discovering/downloading a catalog; explicit files stay usable offline | | `[paths] depot` | `NNNOTES_PATHS_DEPOT` | `--depot` | depot directory (created when missing): the downloaded game files, the results of the stages, catalog versions and runs ([commands.md](commands.md#downloads), [contracts.md](contracts.md#depot)) | | `[paths] master` | `NNNOTES_PATHS_MASTER` | `--master` | decoded master data directory, one `.json` per table; the flag overrides `[servers.] master` | -| `[paths] apk` | `NNNOTES_PATHS_APK` | `--apk` | the game's `base.apk` (the story export reads the CRI Lips library from it, or from `split_config.arm64_v8a.apk` next to it) | +| `[paths] apk` | `NNNOTES_PATHS_APK` | `--apk` | the game's `base.apk` (the story export reads the CRI Lips library from it, or from `split_config.arm64_v8a.apk` next to it; the asset export its built-in Unity files) | | `[paths] ffmpeg` | `NNNOTES_PATHS_FFMPEG` | `--ffmpeg` | `ffmpeg` executable; unset: `ffmpeg` on `PATH` | | `[paths] vgmstream` | `NNNOTES_PATHS_VGMSTREAM` | `--vgmstream` | `vgmstream-cli` executable; unset: `vgmstream-cli` on `PATH` | | `[paths] node` | `NNNOTES_PATHS_NODE` | `--node` | Node.js executable; unset: `node` on `PATH` | @@ -183,7 +183,7 @@ story text); it is not a setting. | `site build --live2d` / `--all-live2d` | catalog (bundles from the CDN of the site's first region), `[paths] apk`, `[paths] player`; not `node`; master data only for the model names (optional: without it `models.json` has no names) | | `site build --player-only` / `--reingest-json` | `[paths] player` | | `music-data` | catalog, `[master] key` + `iv` (not with `--decoded-master`); `--master-files` also `[catalog] region`; `--apk-master` also `[paths] apk`; `--decoded-master` also `[catalog] region` and master data with its `MasterManifest.json`; `[paths] apk` (optional otherwise) for the client version | -| `export assets`, `depot plan` | catalog (bundles are fetched into the depot); `[paths] apk` for the bundles inside the APK (without it they are reported as `source.absent`); master data for `--views`; with `--stored-catalog` a catalog version of the depot instead of the current catalog | +| `export assets`, `depot plan` | catalog (bundles are fetched into the depot); `[paths] apk` for the bundles inside the APK (without it they are reported as `source.absent`) and its built-in Unity files ([stages.md](stages.md#the-apks-built-in-files)); master data for `--views`; with `--stored-catalog` a catalog version of the depot instead of the current catalog | | `depot run` | the depot; `--fetch` also what fetching needs (region, `cdn`, bundle key, `[paths] apk`) | | `catalog versions` / `import` / `diff`, `depot verify` | the depot; `import` reads the APK's catalog from `[paths] apk` when it is set | | `catalog fetch` | the depot, `[catalog] region` and `language`, that region's `cdn`; its `api` (optional) for the resource version label | diff --git a/docs/stages.md b/docs/stages.md index f0473c6..c6746a8 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -19,15 +19,15 @@ gzip input and Japanese remote asset placeholders). | Stage | Subject | Inputs | Outputs | |---|---|---|---| | `catalog.index` | a catalog (remote + APK) | the catalog files | every location (with the bundle hash, crc and size), bundles and raw remote files | -| `unity.census` | a bundle | the bundle | serialized files, externals, objects (file, path id, class, size, type hash, name, script, the scripts an AnimationClip's bindings name), container, scene hashes, dependencies, script fields | +| `unity.census` | a bundle, or a built-in file of the APK | the bundle | serialized files, externals, objects (file, path id, class, size, type hash, name, script, the scripts an AnimationClip's bindings name), container, scene hashes, dependencies, script fields | | `link.scripts` | global | the censuses holding MonoScripts | serialized file and path id -> assembly, namespace, class | -| `unity.export` | a bundle | the bundle, the script entries its MonoBehaviours use | the objects' artifacts and one item per object | +| `unity.export` | a bundle, or a built-in file of the APK | the bundle, the script entries its MonoBehaviours use (and, for files without type trees, the type trees nnnotes ships for their classes) | the objects' artifacts and one item per object | | `sprite.crop` | a SpriteAtlas | the atlas' JSON, the images of its textures, the metas of the sprites of other bundles it packs | the sprites' images | | `cri.audio` | ACB content | `acb` (a raw file of the catalog, or the `acb` artifact of a cue sheet held in a bundle), `awb` (an AWB the catalog pairs with it), `boot` (the game's boot data: the key's source) | FLAC per stream, `cues.json`, `streams.json` | | `cri.movie` | USM content | `usm` (a raw file of the catalog, or the `usm` artifact of a movie held in a bundle), `boot` | every stream as stored (video, alpha, audio, subtitles), `movie.mkv` (or `movie.webm`) | | `live2d.model` | a Live2D model key (`Character/Live2D///model/`) whose bundle is selected | `bundle:`: the bundles of the key's dependency closure in the order the extractor loads them (the order is part of the key); context `internalId` (the key's location); atom `live2d.extract_model` (the extractor of `nnnotes export live2d`, run unchanged) | `live2d.model:#`: every file `nnnotes export live2d` writes for the model, `` relative to the model's directory; placed under `/` | | `spine.skeleton` | a SkeletonDataAsset of a selected bundle, by its stable address `:` (in a bundle of several serialized files the file's tag follows the bundle name, as in `link.addresses`) | `bundle:`: the bundles holding the serialized files the asset's file reaches through its externals; context `object` (the asset's object id); atom `spot.write_skeleton` (the Spine writer of `nnnotes export spot`) | `spine.skeleton:#`: the skeleton (`.json` or `.skel`), its atlases (`.atlas`) and their page images, the bytes `nnnotes export spot` writes into `spine/`; one item for the asset | -| `link.addresses` | global | catalog index, censuses | key -> objects, serialized file -> bundle, bundle -> keys | +| `link.addresses` | global | catalog index, censuses (the built-in files' too) | key -> objects, serialized file -> bundle or built-in file, bundle -> keys | | `link.artifacts` | global | the above, the results | object id -> artifact ids | | `view.` | a catalog (`main`) | `master:
` per master table it reads, `rules` (the view's rule and the resolvers it uses), `addresses` (the part of the address table it read: the keys looked up and the prefixes listed); no parameters, atoms or context | `view.:#view` (facts: rows, entries, gaps, unreferenced) | | `view.voices` | a catalog (`main`) | `master:
` per master table its rules read, `rules`, `addresses`, `sheets` (each cue sheet the `cri.audio` results are known by that a voice names or the rules' prefixes list: its task and ACB content id), `acb:` (the ACB of each decoded one), `episodes` (per story episode and part, the `unity.export` artifact holding the object its key names), `adv:` (those contents); context `audio` (the keys of those `cri.audio` results); runs after `unity.export` and `cri.audio` | `view.voices:#view` (facts: rows, entries, gaps, unreferenced streams) | @@ -61,7 +61,7 @@ with its reason code; any other error of a converter gives it with `generic.erro | Font | `font/2` | `font`: the font file, `json`: the typetree without `m_FontData` | `sniff` | | | MonoScript | `scripts.json/1` | `#scripts.json`: every MonoScript of the bundle | | | | AssetBundle | `bundle.json/1` | `#bundle.json`: serialized files, externals, the AssetBundle objects | | | -| any other class | `generic.json/1` | `json`: the typetree | `sniff` | | +| any other class | `generic.json/2` | `json`: the typetree | `sniff` | an object that can be read only in part (see below): that part, `unsupported.typetree` | Every object a converter takes has a `json` artifact or is contained in another object's. Each item lists in `refs` the objects its documents reference (by object id, other bundles' objects included, the objects of its own @@ -75,6 +75,32 @@ data, `atlas` for its SpriteAtlas' entry; `atlas`; and that render data's `textu of a position `p` is `((p.x * uvTransform.x + uvTransform.y) / width, (p.y * uvTransform.z + uvTransform.w) / height)` of the render data's texture. +### The APK's built-in files + +With `[paths] apk` set, the player's own Unity files under `assets/bin/Data/` of the APK are subjects of +`unity.census` and `unity.export` like the bundles, named `apk/`: + +| Subject | Holds | +|---|---| +| `apk/data.unity3d` | `globalgamemanagers` (player, graphics and quality settings, the resource table), `globalgamemanagers.assets` (render pipeline assets, renderer data and features, post-process data, shaders), `level0`, `resources.assets`, `sharedassets0.assets`, `unity_builtin_extra` (built-in shaders) | +| `apk/datapack.unity3d` | `resources.assets` in split builds | +| `apk/Resources/unity default resources` | the serialized file of the engine's default meshes, textures, shaders and font | + +Their objects have the ids of any serialized file's (`unity_builtin_extra:12`, `unity default resources:10`), so a +bundle's reference into them names an object these tasks export, and `link.addresses` maps their files to their +subjects. Census always covers them; everything and `--select bundle:apk/*` export them. + +These files store no type trees. Objects of the engine's classes are read with the type trees UnityPy has for the +file's Unity version; MonoBehaviours with the type trees nnnotes ships for the game's classes it reads there +(`nnnotes.unity.scripttypes`, generated by `scripts/gen_typetrees.py`): the render pipeline assets, renderer data, +renderer features, post-process data and the game's settings objects. A shipped type tree is used only when the type +hash the file stores is the one it was made for and its read consumes exactly the object's bytes; the task's +context holds the shipped type trees its MonoBehaviours use, so a new shipped tree runs those tasks again. A +MonoBehaviour without one is `unsupported.typetree` with its header (`m_GameObject`, `m_Enabled`, `m_Script`, +`m_Name`) as its typetree JSON; inside a hierarchy its component keeps that header and the prefab records the +issue. PlayerSettings, whose type tree reads the stored layout right up to `m_ActiveColorSpace` (by its stored +type hash), gives the fields up to it. + A task's key includes the atoms `reader` and `sniff`, the converter `generic.json`, and the converters (with their atoms) of the classes the bundle's census lists, so a converter's version bump runs again exactly the bundles holding its classes. `classes` (a list of class names) limits a task to the objects of those classes and the objects their @@ -153,7 +179,8 @@ and multi-audio paths are tested on synthetic movies only. The ids of the external tools are `-+sha256./` (the version as `vgmstream-cli -V` / `ffmpeg -version` report it): another build of a tool runs the tasks that use it again, and a task described for another build is refused as incompatible. The tools are `[paths] vgmstream` and `[paths] ffmpeg`, -else found on PATH. The `boot` input is the APK's `assets/bin/Data/data.unity3d` (`[paths] apk`), stored once. +else found on PATH. The `boot` input is the APK's `assets/bin/Data/data.unity3d` (`[paths] apk`), the input of +the `apk/data.unity3d` tasks. ## Atoms @@ -242,6 +269,7 @@ The codes of items that are `generic`, `unsupported` or `failed`, and of limits | `no_data.sprite` | a sprite whose render data names no texture, or whose texture has no image | | `unsupported.sprite.external_texture` | a sprite whose render data texture (not an atlas) is in another bundle | | `generic.error` | a converter failed on the object (the message says how); its typetree JSON instead | +| `unsupported.typetree` | an object of a file without type trees read in part: a MonoBehaviour without a shipped type tree of its class (its header), a class whose type tree fits only up to a field (those fields) | | `failed.read` | the object's typetree could not be read | | `failed.export` | the object's typetree JSON could not be made | | `source.absent` | a catalog entry whose file is not there (an APK entry without a file) | diff --git a/scripts/gen_typetrees.py b/scripts/gen_typetrees.py index 50e626e..6ce8f75 100644 --- a/scripts/gen_typetrees.py +++ b/scripts/gen_typetrees.py @@ -3,10 +3,11 @@ nnnotes reads a few MonoBehaviours of the APK's boot data (`assets/bin/Data/data.unity3d`): the URP pipeline assets, renderer data, renderer features and post-process data, DOTween and TextMeshPro settings, and the sound -and app-config defaults. IL2CPP player builds strip script typetrees from that data, so nnnotes ships the -typetrees of exactly these classes (CLASSES below) in `src/nnnotes/unity/typetrees/.json`, one file per -Unity version. Each entry holds the typetree nodes as [level, type, name, meta flag] rows and the type hash the -serialized file records for the class; the file also records the game version it was generated from. +and app-config defaults (the `apk/data.unity3d` tasks of `unity.export`, and PlayerData). IL2CPP player builds +strip script typetrees from that data, so nnnotes ships the typetrees of exactly these classes (CLASSES below) in +`src/nnnotes/unity/typetrees/.json`, one file per Unity version (read by nnnotes.unity.scripttypes). +Each entry holds the typetree nodes as [level, type, name, meta flag] rows and the type hash the serialized file +records for the class; the file also records the game version it was generated from. The rows have the node layout Unity itself serializes for a MonoBehaviour (as in the typetrees of asset bundles built with them): root type `MonoBehaviour`; a `string` is `string / Array / int size / char data`; built-in @@ -34,7 +35,7 @@ not read back stops the script without writing anything. 3. When a new game version keeps the Unity version, the file is replaced; a new Unity version adds a file. Keep the older files while the game versions they describe are still in use. -4. When code starts reading another MonoBehaviour class with `PlayerData.mono`, add the class to CLASSES and +4. When code starts reading another MonoBehaviour class of these files, add the class to CLASSES and regenerate. """ @@ -47,7 +48,8 @@ from UnityPy.helpers.TypeTreeGenerator import TypeTreeGenerator -from nnnotes.unity.player import TYPETREE_FORMAT, PlayerData, dump_typetrees, typetree_key, typetree_node +from nnnotes.unity.player import PlayerData +from nnnotes.unity.scripttypes import TYPETREE_FORMAT, dump_typetrees, typetree_key, typetree_node # (assembly, namespace, class) as the MonoScript objects name them CLASSES = ( diff --git a/src/nnnotes/cli/assets.py b/src/nnnotes/cli/assets.py index 0c4eef3..6580554 100644 --- a/src/nnnotes/cli/assets.py +++ b/src/nnnotes/cli/assets.py @@ -19,7 +19,8 @@ catalogs {"main": [Input remote, Input apk]}: the catalog version's files in /catalogs/ index the catalog index document (nnnotes.catalog-index/1) bundles {stable bundle name: Input "bundle"} of the selected bundles and their dependencies (a bundle whose - bytes are not known yet: a Pending "fetch:") + bytes are not known yet: a Pending "fetch:"), and of the APK's built-in files + (`apk/data.unity3d`, `apk/Resources/unity default resources`: pipeline.builtins) selected the selected bundles' stable names, sorted (the subjects of the export stages) raw {stable raw file name: Input "raw"} of the selected raw files (only with a stage that reads them; export fetches them while the stages before run, and reading the fact waits for that) @@ -31,7 +32,8 @@ stable name up to `_assets_`, else up to its first `_`), `key:PREFIX` the bundles and raw files the catalog keys under PREFIX load (their dependency closure), `bundle:GLOB` bundles and raw files whose stable name matches. The census also covers the bundles the selected ones depend on (the dependencies of the keys they hold: a key's first -dependency is its own bundle), so script and atlas references resolve. +dependency is its own bundle), so script and atlas references resolve. With `[paths] apk` the census always covers +the APK's built-in files; everything and `bundle:GLOB` (`bundle:apk/*`) select them. Task cost estimates (scheduling only: never part of a key) are each stage's estimate scaled per stage by the costs measured in earlier runs (/costs/calibration.json, updated after every export). @@ -193,14 +195,16 @@ class Selection: raw: list # stable names of the selected raw files -def select(index: dict, selection: list[str]) -> Selection: - """The bundles and raw files a selection names in a catalog index (module documentation).""" +def select(index: dict, selection: list[str], builtin=()) -> Selection: + """The bundles and raw files a selection names in a catalog index, and of the built-in subjects `builtin` + (module documentation).""" from ..pipeline import catalogdb bundles = sorted(b["stable"] for b in index["bundles"]) raws = sorted(r["stable"] for r in index["rawFiles"]) + builtin = sorted(builtin) if selection == ["all"]: - return Selection(bundles, bundles, raws) + return Selection(sorted(bundles + builtin), sorted(bundles + builtin), raws) files = catalogdb.file_refs(index) reach = catalogdb.closures(index) chosen: set = set() @@ -223,8 +227,10 @@ def select(index: dict, selection: list[str]) -> Selection: and files.get(loc["dependencies"][0]) in chosen ): deps |= reach[loc["id"]] + globs = [v for k, _, v in (s.partition(":") for s in selection) if k == "bundle"] + own = [s for s in builtin if any(fnmatch.fnmatchcase(s, g) for g in globs)] pick = lambda s, kind: sorted(n for k, n in s if k == kind) # noqa: E731 - return Selection(pick(chosen, "bundle"), pick(deps, "bundle"), pick(chosen, "raw")) + return Selection(sorted(pick(chosen, "bundle") + own), sorted(pick(deps, "bundle") + builtin), pick(chosen, "raw")) # ---------------------------------------------------------------- settings @@ -669,11 +675,18 @@ def facts(self, pipeline: list[str], fetch: bool, background: bool = False) -> d """The planning facts (module documentation). `fetch`: fetch the files whose content is not known yet (else they are Pending); what cannot be read is in problems() (after wait()). `background`: the raw files are fetched while the caller goes on (the fact waits for them when it is read).""" + from ..pipeline import builtins + index = self.index() - sel = select(index, self.selection) + try: + builtin = builtins.inputs(self.depot, self.apk) + except (OSError, zipfile.BadZipFile): + raise ConfigError(f"setting paths.apk: {self.apk} is not a readable APK") from None + sel = select(index, self.selection, builtin) self._problems = [] bundles = {b["stable"]: b for b in index["bundles"]} - inputs = self._inputs("bundle", [bundles[s] for s in sel.census], fetch) + inputs = self._inputs("bundle", [bundles[s] for s in sel.census if s not in builtin], fetch) + inputs = inputs.plus(builtin) sel.selected = [s for s in sel.selected if s in inputs] # files that cannot be read: problems() facts = { "catalogs": self.catalogs_fact(), @@ -696,7 +709,7 @@ def facts(self, pipeline: list[str], fetch: bool, background: bool = False) -> d facts["raw"] = Later(self._later) else: facts["raw"] = job[0](*job[1:]) - facts["boot"] = boot_input(self.depot, self.apk) + facts["boot"] = boot_input(builtin) self.selected = sel self._facts, self._names = facts, {} return facts @@ -1048,6 +1061,10 @@ def waiting(self) -> list[str]: """The subjects not fetched yet.""" return [k for k, v in self._items.items() if isinstance(v, Pending)] + def plus(self, items: dict) -> Inputs: + """These inputs and `items` ({subject: Input}).""" + return Inputs({**self._items, **items}) + class Later(Mapping): """A mapping a background job makes (Future): reading it waits for the job.""" @@ -1138,7 +1155,7 @@ def _write_report(d: Path, name: str, doc) -> Path: def cmd_export(args, cfg): - from ..pipeline import layout + from ..pipeline import builtins, layout from ..pipeline.orchestrate import Orchestrator, failure try: @@ -1176,9 +1193,12 @@ def cmd_export(args, cfg): ws.run_index() facts = ws.facts(pipeline, fetch=True, background=True) sel = ws.selected + own = sum(1 for s in sel.census if builtins.is_builtin(s)) + picked = sum(1 for s in sel.selected if builtins.is_builtin(s)) ws.log( - f"{len(sel.selected)} bundles selected ({len(sel.census)} with their dependencies), " - f"{len(sel.raw)} raw files" + f"{len(sel.selected) - picked} bundles selected ({len(sel.census) - own} with their dependencies), " + + (f"{picked} of the APK's {own} built-in files, " if own else "") + + f"{len(sel.raw)} raw files" + (" (fetched while the bundle stages run)" if isinstance(facts["raw"], Later) else "") + f"; {sum(p['kind'] == 'bundle' for p in ws.problems())} bundles not readable" ) diff --git a/src/nnnotes/pipeline/builtins.py b/src/nnnotes/pipeline/builtins.py new file mode 100644 index 0000000..81edc98 --- /dev/null +++ b/src/nnnotes/pipeline/builtins.py @@ -0,0 +1,71 @@ +"""The game's own Unity files in its APK as object sources. + +The player reads files under `assets/bin/Data/` of the APK: `data.unity3d` (a bundle of the global managers, +globalgamemanagers.assets, the first scene level0, resources.assets, sharedassets0.assets and unity_builtin_extra), +`datapack.unity3d` in split builds, and the serialized file `Resources/unity default resources` (which bundles name +`Library/unity default resources`). Each of them the APK holds is a subject of unity.census and unity.export: +`apk/` (`apk/data.unity3d`, `apk/Resources/unity default resources`). Their objects +have the ids of any serialized file's (`unity_builtin_extra:12`, `unity default resources:10`): a serialized file +outside a bundle takes the last part of its path as its name (serialized_name), as references into it name it. + +inputs(depot, apk) gives them as Inputs (role "bundle"), each stored in the depot once; for an APK file the set is +recorded by the APK's content id (the APK is hashed once, depot.identify memo), so a later call reads nothing of it. +""" + +from __future__ import annotations + +from pathlib import Path + +from ..source.apkset import ApkSet +from . import contract +from .contract import Input + +__all__ = ["DATA", "DATA_DIR", "MEMBERS", "PREFIX", "inputs", "is_builtin", "serialized_name"] + +DATA_DIR = "assets/bin/Data/" +MEMBERS = ("data.unity3d", "datapack.unity3d", "Resources/unity default resources") +PREFIX = "apk/" +DATA = PREFIX + "data.unity3d" # the boot data (the CRI stages read the HCA keycode from it) + + +def is_builtin(subject: str) -> bool: + return subject.startswith(PREFIX) + + +def serialized_name(subject: str) -> str | None: + """The name a built-in subject's file is loaded under: the last part of its path (a serialized file outside a + bundle is named so); None for a catalog bundle.""" + return subject.rsplit("/", 1)[-1] if is_builtin(subject) else None + + +def _read(depot, apk: Path) -> dict: + """{member: {sha256, size}} of the built-in files an APK holds, their bytes stored in `depot`.""" + out = {} + with ApkSet(apk) as z: + names = set(z.namelist()) + for m in MEMBERS: + if DATA_DIR + m in names: + data = z.read(DATA_DIR + m) + out[m] = {"sha256": depot.put(data), "size": len(data)} + return out + + +def inputs(depot, apk) -> dict[str, Input]: + """{subject: Input "bundle"} of the built-in files of an APK (a file, an APK set or a directory; see the module + documentation), sorted; {} without an APK.""" + if apk is None: + return {} + apk = Path(apk) + if apk.is_dir(): + found = _read(depot, apk) + else: + name = f"builtins:{depot.identify(apk)[0]}" + known = depot.named("files", name) + found = contract.loads(depot.read(known[0])) if known is not None and depot.has(*known) else None + if found is None or not all(depot.has(r["sha256"], r["size"]) for r in found.values()): + found = _read(depot, apk) + depot.identify(depot.local(depot.put(contract.encode(found))), "files", name) + return { + PREFIX + m: Input("bundle", r["sha256"], int(r["size"]), m.rsplit("/", 1)[-1], ({"kind": "depot"},)) + for m, r in sorted(found.items()) + } diff --git a/src/nnnotes/pipeline/census.py b/src/nnnotes/pipeline/census.py index 23a75cb..7aa1d50 100644 --- a/src/nnnotes/pipeline/census.py +++ b/src/nnnotes/pipeline/census.py @@ -18,7 +18,8 @@ classes objects per class; `objects` the total errors [{object, message}] of the objects whose header could not be read -A census is a function of the bundle's bytes: it holds no file name, time or path. +A census is a function of the bundle's bytes: it holds no file name, time or path (a serialized file outside a +bundle, one of the APK's built-in files, has the name its subject gives: builtins.serialized_name). """ from __future__ import annotations @@ -26,7 +27,7 @@ import re from ..unity.atoms import impl_id -from . import contract +from . import builtins, contract from .contract import Cost, Input from .stages import Output, Pending, Stage @@ -249,11 +250,11 @@ def head(node, fields=HEAD_FIELDS): } -def census_file(path) -> dict: - """The census of a bundle file (decrypted UnityFS).""" +def census_file(path, name: str | None = None) -> dict: + """The census of a bundle file (decrypted UnityFS), or of a serialized file named `name`.""" from ..unity import reader as unity - env = unity.load(path) + env = unity.load(path, name) try: return census_env(env) finally: @@ -305,9 +306,10 @@ def bundle_subjects(index: dict) -> dict[str, dict]: # ---------------------------------------------------------------- stage class CensusStage(Stage): """unity.census: one bundle -> its census. Subjects and inputs come from the fact "bundles": {subject: Input - (role "bundle": the decrypted bundle's sha256, size, name, locators)}; a bundle not fetched yet is Pending - (raised by the fact, or given as a Pending / None). Artifact "#census". The census - reads the bundle with UnityPy: its atom is the reader's ("reader", the UnityPy version).""" + (role "bundle": the decrypted bundle's sha256, size, name, locators)}, the APK's built-in files included + (builtins); a bundle not fetched yet is Pending (raised by the fact, or given as a Pending / None). Artifact + "#census". The census reads the bundle with UnityPy: its atom is the reader's ("reader", the UnityPy + version).""" name = "unity.census" version = 2 @@ -331,7 +333,7 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: return Cost(0.002 + size / 40e6, (32 << 20) + 12 * size) def run(self, task, depot) -> Output: - doc = census_file(depot.open_input(task.input("bundle"))) + doc = census_file(depot.open_input(task.input("bundle")), builtins.serialized_name(task.subject)) content = depot.add(contract.encode(doc), "json") rec = contract.artifact( contract.artifact_id(task.id, "census"), diff --git a/src/nnnotes/pipeline/cristages.py b/src/nnnotes/pipeline/cristages.py index fbb0098..fd7f1b9 100644 --- a/src/nnnotes/pipeline/cristages.py +++ b/src/nnnotes/pipeline/cristages.py @@ -43,9 +43,8 @@ from pathlib import Path from ..formats.encoders import FLAC_LEVEL -from ..source.apkset import ApkSet from ..unity.atoms import impl_id -from . import contract +from . import builtins, contract from .contract import Cost, IncompatibleTask, Input from .stages import Output, Pending, Stage @@ -53,7 +52,6 @@ ACB_MAGIC, AWB_MAGIC, USM_MAGIC = b"@UTF", b"AFS2", b"CRID" ACB_KIND = "cri.acb" # semantics kind of the ACB artifacts of unity.export USM_KIND = "cri.usm" # semantics kind of the USM artifacts of unity.export -BOOT_IN_APK = "assets/bin/Data/data.unity3d" ACB_NAME = "sheet" # the stem of the ACB file vgmstream reads (names come from its cues) TOOL_REVISION = 1 FORMATS = ("mkv", "webm") @@ -70,30 +68,11 @@ # ---------------------------------------------------------------- inputs shared by the stages -def boot_input(depot, apk) -> Input | None: - """The input "boot" of the CRI stages: the boot data of an APK (`assets/bin/Data/data.unity3d`, where the HCA - keycode is read), stored in `depot` once; the APK is hashed once (depot.identify memo) and its boot data is - remembered by the APK's content id. A path that is not an .apk / .zip is the boot data itself. None without an - APK.""" - if apk is None: - return None - apk = Path(apk) - if not apk.is_dir() and apk.suffix.lower() not in (".apk", ".zip", ".apks", ".xapk"): - sha, size = depot.identify(apk) - return Input("boot", sha, size, apk.name, ({"kind": "file", "path": str(apk.resolve())},)) - if apk.is_dir(): - with ApkSet(apk) as z: - data = z.read(BOOT_IN_APK) - sha = depot.put(data) - return Input("boot", sha, len(data), "data.unity3d", ({"kind": "depot"},)) - apk_sha, _ = depot.identify(apk) - name = f"boot:{apk_sha}" - known = depot.named("files", name) - if known is None or not depot.has(*known): - with ApkSet(apk) as z: - data = z.read(BOOT_IN_APK) - known = depot.identify(depot.local(depot.put(data)), "files", name) - return Input("boot", known[0], known[1], "data.unity3d", ({"kind": "depot"},)) +def boot_input(builtin: dict) -> Input | None: + """The input "boot" of the CRI stages: the APK's `assets/bin/Data/data.unity3d`, where the HCA keycode is read, + of its built-in files (`builtin`, builtins.inputs); None when they lack it (no APK).""" + inp = builtin.get(builtins.DATA) + return None if inp is None else Input("boot", inp.sha256, inp.size, inp.name, inp.locators) def _boot(env) -> Input: diff --git a/src/nnnotes/pipeline/link.py b/src/nnnotes/pipeline/link.py index 9a6ea97..88363db 100644 --- a/src/nnnotes/pipeline/link.py +++ b/src/nnnotes/pipeline/link.py @@ -6,10 +6,11 @@ consumes. link.addresses (nnnotes.addresses/1), one task per catalog pair: the catalog index and the censuses of its bundles -(by stable bundle name) -> +(by stable bundle name) and of the APK's built-in files (subjects `apk/...`, pipeline.builtins) -> bundles {stable name: {name, files, assetBundleName, keys}}: the file name in this catalog version, its - serialized files, the AssetBundle name the census found and the keys located in it - files {serialized file: stable bundle name} (CAB -> bundle) + serialized files, the AssetBundle name the census found and the keys located in it; a built-in file + is listed by its subject, named by its path in the APK's `assets/bin/Data/`, without keys + files {serialized file: stable bundle name} (CAB -> bundle; `unity_builtin_extra` -> `apk/data.unity3d`) keys {primary key: [{location, type, internalId, bundle | raw, objects}]}: an asset location's bundle is the dependency whose container holds its internal id (compared ignoring case); `objects` the object ids the container path names (null when a dependency bundle has no census) @@ -22,7 +23,8 @@ its extension names, else the first by file and path id: the object the `original` layout puts at the path), "/[]" for another object under it with a name no other object under it has, else ":". In a bundle of several serialized files (their path ids may coincide) the path id form names -the file after the bundle: the part of the file name after its CAB- prefix ("scene_x.sharedAssets:12"). +the file after the bundle: the part of the file name after its CAB- prefix ("scene_x.sharedAssets:12"), or +"|" ("apk/data.unity3d|level0:3"). Objects that no container path names are not listed; `address_of` gives any object's address. link.artifacts (nnnotes.artifacts/1), one global task: the results of the stages that export objects (unity.export, @@ -41,7 +43,8 @@ import json import re -from . import catalogdb, census, contract +from ..unity import scripttypes +from . import builtins, catalogdb, census, contract from .contract import Cost, Input from .layout import EXT_CLASSES from .stages import Output, Pending, Stage @@ -55,11 +58,8 @@ # ---------------------------------------------------------------- scripts def script_entry(s: dict) -> str: - """ "assembly|namespace|class" of a census MonoScript record.""" - asm = s.get("assembly") or "" - if asm.endswith(".dll"): - asm = asm[:-4] - return f"{asm}|{s.get('namespace') or ''}|{s.get('class') or ''}" + """ "assembly|namespace|class" of a census MonoScript record (scripttypes.script_entry).""" + return scripttypes.script_entry(s.get("assembly"), s.get("namespace"), s.get("class")) def scripts(censuses) -> dict: @@ -81,12 +81,13 @@ def script_context(table: dict, refs) -> dict: # ---------------------------------------------------------------- addresses def file_tags(files) -> dict[str, str]: """{serialized file: the tag its path id addresses carry after the bundle name}: "" in a bundle of one file, - else the file name after its CAB- prefix ("" / ".sharedAssets"); "|" where those coincide.""" + else the file name after its CAB- prefix ("" / ".sharedAssets"); "|" where those coincide or a file + has no such prefix.""" files = list(files) if len(files) <= 1: return {f: "" for f in files} tags = {f: _CAB.sub("", f) for f in files} - if len(set(tags.values())) < len(tags): + if len(set(tags.values())) < len(tags) or not all(_CAB.match(f) for f in files): tags = {f: "|" + f for f in files} return tags @@ -128,9 +129,10 @@ def cls(e): def addresses(index: dict, censuses) -> dict: - """The address table (nnnotes.addresses/1) of a catalog index and the censuses of its bundles: {stable bundle - name: census document}, or (stable name, census document) pairs (read one at a time: only the container - entries and the objects they name are kept). Censuses of names the index does not list are ignored.""" + """The address table (nnnotes.addresses/1) of a catalog index and the censuses of its bundles and of the APK's + built-in files: {stable bundle name or built-in subject: census document}, or (name, census document) pairs + (read one at a time: only the container entries and the objects they name are kept). Censuses of other names + the index does not list are ignored.""" files_of = catalogdb.file_refs(index) listed = {b["stable"]: b for b in index["bundles"]} bundles, owners, info, groups, tags_of = {}, {}, {}, {}, {} @@ -138,6 +140,8 @@ def addresses(index: dict, censuses) -> dict: pairs = sorted(censuses.items()) if isinstance(censuses, dict) else censuses for stable, doc in pairs: b = listed.get(stable) + if b is None and builtins.is_builtin(stable): + b = {"name": stable[len(builtins.PREFIX) :]} if b is None: continue names = [f["name"] for f in doc["files"]] @@ -242,8 +246,12 @@ def reverse(doc: dict) -> dict[str, str]: # ---------------------------------------------------------------- stages +def _subject(tid: str) -> str: + return contract.parse_task_id(tid, census.CensusStage.name)[1] + + def _census_input(env, tid: str) -> Input: - return env.input_of(tid, "census", as_role="census:" + contract.parse_task_id(tid, census.CensusStage.name)[1]) + return env.input_of(tid, "census", as_role="census:" + _subject(tid)) def _read(depot, task, role: str) -> dict: @@ -289,9 +297,9 @@ def run(self, task, depot) -> Output: class AddressesStage(Stage): """link.addresses: a catalog index (input "index", the result of catalog.index:) and the censuses of its - bundles (inputs "census:") -> the address table. Subjects: those of catalog.index (the fact - "catalogs"). Waits for census tasks of its bundles that have not run; skips failed ones. Artifact - "link.addresses:#addresses".""" + bundles and of the APK's built-in files (inputs "census:") -> the address table. + Subjects: those of catalog.index (the fact "catalogs"). Waits for census tasks of its bundles that have not run; + skips failed ones. Artifact "link.addresses:#addresses".""" name = "link.addresses" version = 1 @@ -305,6 +313,7 @@ def inputs(self, subject: str, env) -> list[Input]: idx = env.input_of(tid, "index") stables = {b["stable"] for b in env.depot.document(idx.sha256)["bundles"]} wanted = {contract.task_id(census.CensusStage.name, s) for s in stables} + wanted |= {t for t in env.tasks(census.CensusStage.name) if builtins.is_builtin(_subject(t))} pending = [t for t in env.pending(census.CensusStage.name) if t in wanted] if pending: raise Pending(pending[0]) diff --git a/src/nnnotes/pipeline/objexport.py b/src/nnnotes/pipeline/objexport.py index 182e638..e06d5c8 100644 --- a/src/nnnotes/pipeline/objexport.py +++ b/src/nnnotes/pipeline/objexport.py @@ -24,7 +24,9 @@ import io from collections import defaultdict +from functools import cache +from ..unity import scripttypes from ..unity.atoms import ATOMS as ATOM_REGISTRY from ..unity.atoms import Impl, Unsupported, atom_id, resolve from ..unity.convert import ( @@ -45,9 +47,17 @@ prefab, textures, ) -from ..unity.convert.objects import BundleObjects, ReadError, encode_object, label, message, ref_object_id +from ..unity.convert.objects import ( + TYPETREE, + BundleObjects, + ReadError, + encode_object, + label, + message, + ref_object_id, +) +from . import builtins, contract, link from . import census as census_mod -from . import contract, link from .contract import Cost, IncompatibleTask, Input from .stages import Output, Pending, Stage @@ -81,6 +91,7 @@ def __init__(self, view, task, depot, fns: dict): blob_min=int(task.params["blobMin"]), png_level=int(task.params["png"]["level"]), emit=self._emit, + typetrees=task.context.get("typetrees", {}), ) classes = task.params.get("classes") self.selected = None if classes is None else frozenset(classes) @@ -190,14 +201,18 @@ def _fallback(self, info, code: str, why: str, status: str = "generic") -> None: self._commit(info, status, contract.reason(code, label(why))) def _generic(self, info) -> None: - """An object of a class without a specialized converter: its typetree JSON.""" + """An object of a class without a specialized converter: its typetree JSON (`generic` when it could be read + only in part).""" self._begin("generic.json") try: - generic.generic(self.ctx, info) + why = generic.generic(self.ctx, info) except Exception as e: self._failed(info, e) return - self._commit(info, "exported") + if why is None: + self._commit(info, "exported") + else: + self._commit(info, "generic", contract.reason(TYPETREE, label(why))) def _selected(self, cls: str) -> bool: return self.selected is None or cls in self.selected @@ -371,7 +386,9 @@ class ExportStage(Stage): "bundles" {subject: Input (role "bundle")}, as unity.census. It reads the census of its bundle (unity.census:) for the classes (the converters and atoms of its key) and the MonoScripts its objects reference, and the script table (link.scripts:all) for the entries of those: its context - {"scripts": {object id: "assembly|namespace|class"}}. Parameters: `blobMin` (bytes; byte arrays of this size or + {"scripts": {object id: "assembly|namespace|class"}}, and for the MonoBehaviours of files that store no type trees + (the APK's built-in files) the type trees nnnotes ships for their classes: "typetrees" {script entry: + {"typeHash", "nodes"}}. Parameters: `blobMin` (bytes; byte arrays of this size or more are blobs), `png.level` (zlib level of the PNG files). `reader`: another implementation of the reader atom (its id enters the keys).""" @@ -449,11 +466,16 @@ def uses(self, subject: str, env) -> dict: return atoms def context(self, subject: str, env) -> dict: - refs = census_mod.script_refs(self.census(subject, env)) + doc = self.census(subject, env) + refs = census_mod.script_refs(doc) if not refs: return {} table = self._docs.get(env.depot, env.input_of(SCRIPTS_TASK, "scripts").sha256)["scripts"] - return {"scripts": link.script_context(table, refs)} + out = {"scripts": link.script_context(table, refs)} + types = _script_types(doc, out["scripts"]) + if types: + out["typetrees"] = types + return out def depends(self, subject: str, env) -> list[str]: return [contract.task_id(CENSUS, subject), SCRIPTS_TASK] @@ -473,12 +495,34 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: def run(self, task, depot) -> Output: reader = self.reader if self.reader is not None else resolve("reader") _check_atoms(task, reader) - view = reader.fn(depot.input_bytes(task.input("bundle"))) + view = reader.fn(depot.input_bytes(task.input("bundle")), builtins.serialized_name(task.subject)) fns = {n: resolve(n).fn for n in task.atoms if n in ATOM_NAMES and n != "reader"} arts, items = BundleRun(view, task, depot, fns).run() return Output(arts, items) +@cache +def _embedded(unity_version: str) -> dict: + return scripttypes.script_types(unity_version) + + +def _script_types(doc: dict, scripts: dict) -> dict: + """{script entry: {"typeHash", "nodes"}} of the embedded type trees (nnnotes.unity.scripttypes, of each file's + Unity version) of the classes of the MonoBehaviours in the census' files that store no type trees ({} when it has + none).""" + out = {} + for f in doc["files"]: + if f["typetree"]: + continue + types = _embedded(f["unityVersion"]) + for o in f["objects"]: + s = o.get("script") if o["class"] == "MonoBehaviour" else None + entry = scripts.get(contract.object_id(s["file"], s["pathId"])) if s and s.get("file") else None + if entry in types and out.setdefault(entry, types[entry]) != types[entry]: + raise ValueError(f"{entry}: files of several Unity versions with different type trees of it") + return dict(sorted(out.items())) + + def _census_classes(env, subject: str) -> dict: """{class: objects} of a bundle's census ({} when its census has no result).""" try: diff --git a/src/nnnotes/unity/atoms/reader.py b/src/nnnotes/unity/atoms/reader.py index 0cb56ce..2f4cf9e 100644 --- a/src/nnnotes/unity/atoms/reader.py +++ b/src/nnnotes/unity/atoms/reader.py @@ -2,7 +2,8 @@ `BundleView` is what a stage reads a bundle through; `UnityBundle` is the reference implementation (UnityPy, the bundle loaded alone from its bytes: references into other bundles are not followed, resource files inside the -bundle -- `.resS` -- are read). Objects are named by `ObjectInfo` (serialized file name and path id), never by +bundle -- `.resS` -- are read). A serialized file outside a bundle (one of the APK's built-in files) is read the same +way, under the name it is given. Objects are named by `ObjectInfo` (serialized file name and path id), never by library objects, so another implementation can stand in. `texture_input` and `mesh_input` collect exactly what `texture.decode` and `mesh.arrays` need: a Texture2D's image @@ -69,6 +70,11 @@ def unity_version(self, file: str) -> str: """The Unity version a serialized file was written by.""" ... + def stores_typetrees(self, file: str) -> bool: + """Whether a serialized file stores the type trees of its objects (a player build's own files do not: its + script classes read only their header).""" + ... + def texture_input(self, obj: ObjectInfo) -> TextureInput: ... def mesh_input(self, obj: ObjectInfo) -> MeshInput: ... @@ -266,6 +272,9 @@ def unity_version(self, file: str) -> str: v = getattr(f, "unity_version", None) return v if isinstance(v, str) else ".".join(str(x) for x in f.version) + def stores_typetrees(self, file: str) -> bool: + return bool(getattr(self._files[file], "_enable_type_tree", True)) + def texture_input(self, obj: ObjectInfo) -> TextureInput: r = self._reader(obj) tex = r.read() @@ -306,11 +315,12 @@ def mesh_input(self, obj: ObjectInfo) -> MeshInput: ) -def open_bundle(data: bytes) -> UnityBundle: - """The BundleView of a bundle's bytes (decrypted UnityFS), loaded alone.""" +def open_bundle(data: bytes, name: str | None = None) -> UnityBundle: + """The BundleView of a bundle's bytes (decrypted UnityFS), loaded alone; of a serialized file's bytes with its + `name`.""" from ..reader import load_bytes - return UnityBundle(load_bytes(bytes(data))) + return UnityBundle(load_bytes(bytes(data), name)) def cost(facts: dict) -> dict: diff --git a/src/nnnotes/unity/convert/__init__.py b/src/nnnotes/unity/convert/__init__.py index 759d8ab..ce85c0c 100644 --- a/src/nnnotes/unity/convert/__init__.py +++ b/src/nnnotes/unity/convert/__init__.py @@ -38,6 +38,10 @@ - a string whose stored bytes are not UTF-8 is {"$hex": ""}; - NaN is {"$float": "nan"}, infinities 1e999 / -1e999. +A file of the player itself (the APK's built-in files) stores no type trees: its MonoBehaviours are read with the +type trees nnnotes ships for their classes (nnnotes.unity.scripttypes), and an object that can be read only in part +gives that part with `unsupported.typetree` (objects.py). + A converter's documented limit raises atoms.Unsupported with a reason code; the reason codes are listed in docs/stages.md. """ @@ -76,7 +80,7 @@ "scripts.json": 1, "bundle.json": 1, "shader.json": 1, - "generic.json": 1, + "generic.json": 2, "cri.usm": 1, } # class -> the converters its objects may use (every class may fall back to generic.json) @@ -118,6 +122,7 @@ "no_data.font", "unsupported.sprite.external_texture", "unsupported.cri.awb_external", + "unsupported.typetree", "failed.read", "failed.export", *(f"unsupported.class.{c}" for c in UNSUPPORTED_CLASSES), diff --git a/src/nnnotes/unity/convert/generic.py b/src/nnnotes/unity/convert/generic.py index 7199f98..c69e4b2 100644 --- a/src/nnnotes/unity/convert/generic.py +++ b/src/nnnotes/unity/convert/generic.py @@ -11,11 +11,13 @@ LOOSE = {"bundle.json": ("AssetBundle", "unity.bundle"), "scripts.json": ("MonoScript", "unity.scripts")} -def generic(ctx: BundleObjects, info) -> None: - """`json`: the object's typetree JSON.""" +def generic(ctx: BundleObjects, info) -> str | None: + """`json`: the object's typetree JSON. Returns why when the typetree could be read only in part (its part is + the JSON: BundleObjects.partial), else None.""" key = (info.file, info.path_id) - doc = ctx.tree(ctx.readers[key], ctx.tt(key)) + doc = ctx.tree(ctx.readers[key], ctx.tt(key, whole=False)) ctx.emit(key, "json", encode_object(doc), "json", "unity.object", "json", {}) + return ctx.partial.get(key) def loose_doc(ctx: BundleObjects, conv: str, entries: dict) -> dict: diff --git a/src/nnnotes/unity/convert/objects.py b/src/nnnotes/unity/convert/objects.py index b75a0d1..dc3a44b 100644 --- a/src/nnnotes/unity/convert/objects.py +++ b/src/nnnotes/unity/convert/objects.py @@ -7,6 +7,14 @@ Every object a document names by {"$ref"} is recorded in `seen` (by object id, other bundles' objects included): the objects an object's documents reference, which the caller clears before converting an object. + +A file that stores no type trees (a player build's own files) gives its MonoBehaviours only their header; they are +read with the task's type tree of their script class (`typetrees`, nnnotes.unity.scripttypes), checked against the +type hash the file stores and the object's size. One without such a tree is `unsupported.typetree` (TYPETREE): its +header is still read. Its other objects are read with the type trees of their classes UnityPy has for the file's +Unity version; for a class whose tree reads the stored layout right only up to a field (PREFIXES, by the type hash +the file stores) the typetree JSON is the fields up to it, also `unsupported.typetree`. The objects read in part are +in `partial`, with the reason. """ from __future__ import annotations @@ -17,10 +25,14 @@ from typing import cast from ...core.canon import encode +from ..atoms import Unsupported from ..ids import object_id +from ..scripttypes import typetree_node __all__ = [ "NO_SCRIPT", + "PREFIXES", + "TYPETREE", "BundleObjects", "Capture", "ReadError", @@ -33,6 +45,10 @@ ] NO_SCRIPT = "\0none" # the script entry of a MonoBehaviour whose m_Script is null +TYPETREE = "unsupported.typetree" # an object of a file without type trees read in part +# classes whose type tree (UnityPy's, for the file's Unity version) reads the stored layout right only up to a field, +# by the type hash the file stores: {(class, type hash): the last field read right} +PREFIXES = {("PlayerSettings", "936fd1913a4f0b5da2ce7cadb0ce9666"): "m_ActiveColorSpace"} _BYTE_TYPES = frozenset({"UInt8", "SInt8", "char"}) _SCALARS = (int, float, str, bool, type(None)) _ADDRESS = re.compile(r"0x[0-9a-fA-F]{6,}") @@ -135,6 +151,24 @@ def _ref_type_node(ref: dict, assets_file): raise ValueError(f"referenced type {asm}|{ns}|{cls} is not among the file's reference types") +def _cut(node, last: str): + """A type tree cut after its field `last` (None when it has no such field).""" + names = [c.m_Name for c in node.m_Children] + if last not in names: + return None + from UnityPy.helpers.TypeTreeNode import TypeTreeNode + + return TypeTreeNode( + node.m_Level, + node.m_Type, + node.m_Name, + node.m_ByteSize, + node.m_Version, + m_MetaFlag=node.m_MetaFlag, + m_Children=node.m_Children[: names.index(last) + 1], + ) + + class ReadError(Exception): """An object's typetree could not be read.""" @@ -156,17 +190,33 @@ class Capture: class BundleObjects: """The objects of one bundle (`view`, a reader atom's BundleView). `fns`: the atom functions by name; `scripts`: the script table entries the bundle's objects use ({MonoScript object id: "assembly|namespace|class"}); - `blob_min`: byte arrays of this size or more are blobs; `png_level`: the zlib level of PNG files.""" - - def __init__(self, view, fns: dict, *, scripts: dict, blob_min: int, png_level: int, emit: Emit): + `typetrees`: the type trees of script classes for the files that store none ({script entry: {"typeHash", + "nodes"}}); `blob_min`: byte arrays of this size or more are blobs; `png_level`: the zlib level of PNG files.""" + + def __init__( + self, + view, + fns: dict, + *, + scripts: dict, + blob_min: int, + png_level: int, + emit: Emit, + typetrees: dict | None = None, + ): self.view, self.fns, self.emit = view, fns, emit self.scripts, self.blob_min, self.png_level = dict(scripts), int(blob_min), int(png_level) + self.typetrees = dict(typetrees or {}) self.infos = view.objects() self.info = {(i.file, i.path_id): i for i in self.infos} self.readers = {k: view.reader(i) for k, i in self.info.items()} self.files = view.files() self.ext_files = {f: [_file_name(p) for p in view.externals(f)] for f in self.files} self.versions = {f: view.unity_version(f) for f in self.files} + self.stripped = frozenset(f for f in self.files if not view.stores_typetrees(f)) + self.partial: dict[tuple, str] = {} # objects read in part -> why + self._nodes: dict[tuple, object] = {} # objects read with a type tree of `typetrees` -> its node + self._script_nodes: dict[str, object] = {} self.containers: dict[tuple, str] = {} for path, file, pid in view.container(): k = (file, int(pid)) @@ -209,21 +259,7 @@ def _peek(self, r) -> str | None: node = r._get_typetree_node() hit = self._heads.get(id(node)) if hit is None or hit[0] is not node: - names = [c.m_Name for c in node.m_Children] - cut = None - if "m_Name" in names: - from UnityPy.helpers.TypeTreeNode import TypeTreeNode - - cut = TypeTreeNode( - node.m_Level, - node.m_Type, - node.m_Name, - node.m_ByteSize, - node.m_Version, - m_MetaFlag=node.m_MetaFlag, - m_Children=node.m_Children[: names.index("m_Name") + 1], - ) - hit = self._heads[id(node)] = (node, cut) + hit = self._heads[id(node)] = (node, _cut(node, "m_Name")) if hit[1] is None: return None name = r.read_typetree(hit[1], check_read=False).get("m_Name") @@ -241,16 +277,72 @@ def name_of(self, key: tuple) -> str | None: self.names[key] = name return self.names[key] - def tt(self, key: tuple) -> dict: - """The object's typetree (ReadError when it cannot be read).""" + def tt(self, key: tuple, *, whole: bool = True) -> dict: + """The object's typetree (ReadError when it cannot be read). An object of a file that stores no type trees + that can be read only in part (see the module documentation) raises Unsupported(TYPETREE); with `whole` False + it gives that part (and is in `partial`).""" + info = self.info[key] + if info.class_name == "MonoBehaviour" and info.file in self.stripped: + return self._script_tt(key, info, whole) try: - tt = self.view.typetree(self.info[key]) + tt = self.view.typetree(info) except Exception as e: - raise ReadError(message(e)) from None + last = PREFIXES.get((info.class_name, info.type_hash)) if info.file in self.stripped else None + if last is None: + raise ReadError(message(e)) from None + why = f"{info.class_name}: {message(e)}; read up to {last}, which its type tree reads right" + if whole: + raise Unsupported(TYPETREE, why) from None + tt = self._prefix(self.readers[key], last) + self.partial[key] = why name = tt.get("m_Name") self.names[key] = name if isinstance(name, str) else None return tt + def _script_tt(self, key: tuple, info, whole: bool) -> dict: + r = self.readers[key] + try: + head = r.read_typetree(check_read=False) # the class's own tree: the header + except Exception as e: + raise ReadError(message(e)) from None + entry = self.mono_script(key, head) + t = self.typetrees.get(entry) if entry is not None else None + if entry is None or entry == NO_SCRIPT: + s = head.get("m_Script") or {} + why = f"script {s.get('m_FileID')}:{s.get('m_PathID')} is unknown and the file stores no type trees" + elif t is None: + why = f"the file stores no type trees and the task has none for {entry}" + elif t["typeHash"] != info.type_hash: + why = f"{entry}: the file's type hash {info.type_hash} is not the type tree's {t['typeHash']}" + else: + node = self._script_nodes.get(entry) + if node is None: + node = self._script_nodes[entry] = typetree_node(t["nodes"]) + try: + tt = r.read_typetree(node, check_read=True) + except Exception as e: + why = f"{entry}: the type tree does not fit the object ({message(e)})" + else: + self._nodes[key] = node + self.partial.pop(key, None) + name = tt.get("m_Name") + self.names[key] = name if isinstance(name, str) else None + return tt + if whole: + raise Unsupported(TYPETREE, why) + self.partial[key] = why + return head + + def _prefix(self, r, last: str) -> dict: + """An object's typetree up to the field `last` (its class's tree cut after it).""" + cut = _cut(r._get_typetree_node(), last) + if cut is None: + raise ReadError(f"its type tree has no field {last}") + try: + return r.read_typetree(cut, check_read=False) + except Exception as e: + raise ReadError(message(e)) from None + def facts(self, key: tuple) -> dict: """The object facts of an artifact's provenance: file, path id, class, name, type hash, Unity version, container path.""" @@ -286,7 +378,7 @@ def mono_script(self, key: tuple, tt: dict) -> str | None: def script_class(self, key: tuple) -> str | None: """The class name of a MonoBehaviour from the script table (None when unknown).""" if key not in self._mono_scripts: - self.mono_script(key, self.tt(key)) + self.mono_script(key, self.tt(key, whole=False)) e = self._mono_scripts[key] return e.rsplit("|", 1)[-1] if e and e != NO_SCRIPT else None @@ -342,7 +434,8 @@ def tree(self, owner, tt: dict, *, captures: dict[str, Capture] | None = None, l name of an object of the prefab being walked (None for any other).""" self._captures, self._local = dict(captures or {}), local try: - doc = cast(dict, self._walk(owner, owner._get_typetree_node(), tt, "")) + node = self._nodes.get(self.key(owner)) or owner._get_typetree_node() + doc = cast(dict, self._walk(owner, node, tt, "")) left = self._captures finally: self._captures, self._local = {}, None diff --git a/src/nnnotes/unity/convert/prefab.py b/src/nnnotes/unity/convert/prefab.py index 845719e..1610952 100644 --- a/src/nnnotes/unity/convert/prefab.py +++ b/src/nnnotes/unity/convert/prefab.py @@ -10,7 +10,7 @@ from dataclasses import dataclass, field from types import SimpleNamespace -from .objects import NO_SCRIPT, BundleObjects, ReadError, encode_object, message +from .objects import NO_SCRIPT, TYPETREE, BundleObjects, ReadError, encode_object, message __all__ = ["Hierarchies", "Root", "hierarchies", "prefab"] @@ -132,7 +132,7 @@ def _subtree(ctx: BundleObjects, h: Hierarchies, f: str, g, root_tf: int): def _component(ctx: BundleObjects, h: Hierarchies, o, root: tuple, unresolved: list) -> dict: key = ctx.key(o) - tt = ctx.tt(key) + tt = ctx.tt(key, whole=False) out = {"type": o.type.name, "pathId": key[1]} if o.type.name == "MonoBehaviour": entry = ctx.mono_script(key, tt) @@ -184,10 +184,13 @@ def prefab(ctx: BundleObjects, info, h: Hierarchies, root: Root) -> None: if info.file in h.scene_of: doc["scene"] = h.scene_of[info.file] doc["nodes"] = nodes - issues = ( - [("script.unresolved", f"{len(unresolved)} MonoBehaviours whose script is not in the task's script table")] - if unresolved - else [] - ) + issues = [] + if unresolved: + issues.append( + ("script.unresolved", f"{len(unresolved)} MonoBehaviours whose script is not in the task's script table") + ) + partial = sum(1 for k in root.members if k in ctx.partial) + if partial: + issues.append((TYPETREE, f"{partial} components read in part (their file stores no type tree that fits)")) facts = {"nodes": len(nodes), "objects": len(root.members)} ctx.emit(key, "json", encode_object(doc), "json", "prefab", "json", facts, issues=issues) diff --git a/src/nnnotes/unity/player.py b/src/nnnotes/unity/player.py index 21c36ad..10638f7 100644 --- a/src/nnnotes/unity/player.py +++ b/src/nnnotes/unity/player.py @@ -3,10 +3,8 @@ `assets/bin/Data/data.unity3d` holds the global managers (graphics, quality, player settings) and `globalgamemanagers.assets` with the URP pipeline assets, renderer data, renderer features and post-process data. IL2CPP player builds -strip script typetrees there, so MonoBehaviours are read with typetrees that -ship with nnnotes: `typetrees/.json`, generated from the game's -managed assembly stubs by `scripts/gen_typetrees.py`. Each entry records the -type hash the serialized file stores for its class; a class without an entry, +strip script typetrees there, so MonoBehaviours are read with the typetrees +that ship with nnnotes (nnnotes.unity.scripttypes); a class without an entry, a different type hash or a read that does not consume exactly the object's serialized size raises UnsupportedVersion. `unity default resources` (built-in meshes) is read from the APK as well. @@ -16,10 +14,7 @@ import copy import io -import json -import re import struct -from importlib import resources from pathlib import Path from typing import cast @@ -30,63 +25,18 @@ from ..source.apkset import MANIFEST_IN_APK, ApkSet, manifest_version_name from ..source.config import ConfigError from .reader import DEFAULT_RESOURCES, deref, external_path, is_pptr +from .scripttypes import load_typetrees, typetree_key, typetree_node DATA_IN_APK = "assets/bin/Data/data.unity3d" DEFAULT_RESOURCES_IN_APK = "assets/bin/Data/Resources/unity default resources" COLOR_SPACE = {0: "Gamma", 1: "Linear"} HEADER = ("m_GameObject", "m_Script", "m_Enabled") -TYPETREE_FORMAT = 1 class UnsupportedVersion(ConfigError): """The APK's boot data has a MonoBehaviour class that nnnotes has no matching typetree for.""" -# ---------------------------------------------------------------- embedded typetrees -def typetree_key(assembly: str, namespace: str, cls: str) -> str: - """Key of a class in a typetree file: `assembly|namespace|class`, as its MonoScript names them.""" - return f"{assembly}|{namespace}|{cls}" - - -def typetree_file(unity_version: str): - """The embedded typetree file of a Unity version (an importlib.resources Traversable; it may not exist).""" - return resources.files("nnnotes.unity").joinpath("typetrees", f"{unity_version}.json") - - -def load_typetrees(unity_version: str) -> dict | None: - """The embedded typetrees of a Unity version, or None when nnnotes has none for it.""" - if not re.fullmatch(r"[0-9A-Za-z.]+", unity_version or ""): - return None - f = typetree_file(unity_version) - if not f.is_file(): - return None - doc = json.loads(f.read_text(encoding="utf-8")) - if doc.get("format") != TYPETREE_FORMAT or doc.get("unityVersion") != unity_version: - raise ValueError(f"typetree file {unity_version}.json: unexpected format or Unity version") - return doc - - -def typetree_node(rows: list) -> TypeTreeNode: - """TypeTreeNode from [level, type, name, meta flag] rows (byte size and version 0, as generated).""" - return TypeTreeNode.from_list([TypeTreeNode(lv, ty, nm, 0, 0, m_MetaFlag=mf) for lv, ty, nm, mf in rows]) - - -def dump_typetrees(doc: dict) -> str: - """Text of a typetree file: sorted keys, one class per line, LF.""" - head = {k: v for k, v in doc.items() if k != "classes"} - lines = [json.dumps(head, sort_keys=True, ensure_ascii=False, separators=(",", ":"))[:-1] + ',"classes":{'] - items = sorted(doc["classes"].items()) - for i, (k, v) in enumerate(items): - lines.append( - json.dumps(k, ensure_ascii=False) - + ":" - + json.dumps(v, sort_keys=True, ensure_ascii=False, separators=(",", ":")) - + ("," if i + 1 < len(items) else "") - ) - lines.append("}}") - return "\n".join(lines) + "\n" - - class PlayerData: def __init__(self, apk: Path): with ApkSet(apk) as z: diff --git a/src/nnnotes/unity/reader.py b/src/nnnotes/unity/reader.py index 2c640b8..fd600e4 100644 --- a/src/nnnotes/unity/reader.py +++ b/src/nnnotes/unity/reader.py @@ -92,12 +92,22 @@ def parse_blob(cls, reader, version): _install_blob_memo() -def load(path: str | Path) -> UnityPy.environment.Environment: - return UnityPy.load(str(path)) - - -def load_bytes(data: bytes) -> UnityPy.environment.Environment: - return UnityPy.load(io.BytesIO(data)) +def load(path: str | Path, name: str | None = None) -> UnityPy.environment.Environment: + """The environment of a bundle file or a serialized file (load_bytes).""" + if name is None: + return UnityPy.load(str(path)) + return load_bytes(Path(path).read_bytes(), name) + + +def load_bytes(data: bytes, name: str | None = None) -> UnityPy.environment.Environment: + """The environment of a bundle's bytes, or of a serialized file's: `name` names a serialized file outside a bundle + (`unity default resources`; references into it name it so), which UnityPy would name after the stream. A + bundle's files keep their own names.""" + if name is None: + return UnityPy.load(io.BytesIO(data)) + env = UnityPy.Environment() + env.load_file(io.BytesIO(data), name=name) + return env _closures = threading.local() diff --git a/src/nnnotes/unity/scripttypes.py b/src/nnnotes/unity/scripttypes.py new file mode 100644 index 0000000..2f7d7f3 --- /dev/null +++ b/src/nnnotes/unity/scripttypes.py @@ -0,0 +1,98 @@ +"""The type trees of game script classes that ship with nnnotes. + +IL2CPP player builds strip the type trees of script classes from the player's own files (`assets/bin/Data/` of the +APK: the global managers, the URP pipeline assets, renderer data, renderer features and post-process data), so a +MonoBehaviour there reads only its header. nnnotes ships the type trees of the classes it reads from those files in +`typetrees/.json`, generated from the game's managed assembly stubs by `scripts/gen_typetrees.py`: + + {"format": 1, "unityVersion", "gameVersion", "classes": {"assembly|namespace|class": {"typeHash", "nodes"}}} + +A class is keyed by the names its MonoScript stores (typetree_key); `nodes` are [level, type, name, meta flag] rows +of Unity's layout and `typeHash` the hex type hash the serialized file stores for the class. A tree fits an object +when the type hashes are equal and its read consumes exactly the object's serialized size. + +Nothing here imports UnityPy until typetree_node is called. +""" + +from __future__ import annotations + +import json +import re +from importlib import resources + +__all__ = [ + "TYPETREE_FORMAT", + "dump_typetrees", + "load_typetrees", + "script_entry", + "script_types", + "typetree_file", + "typetree_key", + "typetree_node", +] + +TYPETREE_FORMAT = 1 + + +def typetree_key(assembly: str, namespace: str, cls: str) -> str: + """Key of a class in a typetree file: `assembly|namespace|class`, as its MonoScript names them.""" + return f"{assembly}|{namespace}|{cls}" + + +def script_entry(assembly: str | None, namespace: str | None, cls: str | None) -> str: + """The script table entry of a class: "assembly|namespace|class", the assembly without `.dll`, an empty + namespace empty.""" + asm = assembly or "" + if asm.endswith(".dll"): + asm = asm[:-4] + return f"{asm}|{namespace or ''}|{cls or ''}" + + +def typetree_file(unity_version: str): + """The embedded typetree file of a Unity version (an importlib.resources Traversable; it may not exist).""" + return resources.files("nnnotes.unity").joinpath("typetrees", f"{unity_version}.json") + + +def load_typetrees(unity_version: str) -> dict | None: + """The embedded typetrees of a Unity version, or None when nnnotes has none for it.""" + if not re.fullmatch(r"[0-9A-Za-z.]+", unity_version or ""): + return None + f = typetree_file(unity_version) + if not f.is_file(): + return None + doc = json.loads(f.read_text(encoding="utf-8")) + if doc.get("format") != TYPETREE_FORMAT or doc.get("unityVersion") != unity_version: + raise ValueError(f"typetree file {unity_version}.json: unexpected format or Unity version") + return doc + + +def script_types(unity_version: str) -> dict[str, dict]: + """{script table entry (script_entry): {"typeHash", "nodes"}} of the embedded typetrees of a Unity version ({} + when nnnotes has none for it).""" + doc = load_typetrees(unity_version) + if doc is None: + return {} + return {script_entry(*k.split("|", 2)): v for k, v in sorted(doc["classes"].items())} + + +def typetree_node(rows: list): + """UnityPy TypeTreeNode from [level, type, name, meta flag] rows (byte size and version 0, as generated).""" + from UnityPy.helpers.TypeTreeNode import TypeTreeNode + + return TypeTreeNode.from_list([TypeTreeNode(lv, ty, nm, 0, 0, m_MetaFlag=mf) for lv, ty, nm, mf in rows]) + + +def dump_typetrees(doc: dict) -> str: + """Text of a typetree file: sorted keys, one class per line, LF.""" + head = {k: v for k, v in doc.items() if k != "classes"} + lines = [json.dumps(head, sort_keys=True, ensure_ascii=False, separators=(",", ":"))[:-1] + ',"classes":{'] + items = sorted(doc["classes"].items()) + for i, (k, v) in enumerate(items): + lines.append( + json.dumps(k, ensure_ascii=False) + + ":" + + json.dumps(v, sort_keys=True, ensure_ascii=False, separators=(",", ":")) + + ("," if i + 1 < len(items) else "") + ) + lines.append("}}") + return "\n".join(lines) + "\n" diff --git a/tests/cli/test_assets.py b/tests/cli/test_assets.py index 2e10549..85388f5 100644 --- a/tests/cli/test_assets.py +++ b/tests/cli/test_assets.py @@ -123,6 +123,25 @@ def bundle_bytes(doc: dict) -> bytes: return b"UnityFS\0" + contract.encode(doc) +def builtin_census() -> dict: + """The census of a player's data.unity3d: files without type trees and AssetBundle objects.""" + doc = cdoc("globalgamemanagers.assets", [obj(2, "Shader", "Blit")]) + head = dict(doc["files"][0], typetree=False) + files = [head, dict(head, name="unity_builtin_extra", objects=[obj(7, "Shader", "UI_Default")])] + return dict(doc, files=files, assetBundles=[], classes={"Shader": 2}, objects=2) + + +def builtin_apk(path, extra=()) -> Path: + """An APK holding the player's data.unity3d (builtin_census) and `extra` members {name: bytes}.""" + import zipfile + + with zipfile.ZipFile(path, "w") as z: + z.writestr("assets/bin/Data/data.unity3d", bundle_bytes(builtin_census())) + for n, data in dict(extra).items(): + z.writestr(n, data) + return path + + def catalog_bytes(extra=()) -> bytes: remote = {n: synth.remote(NAMES[n]) for n in NAMES} entries = [(NAMES[n], remote[n], []) for n in ("card_1", "card_2", "shared", "bg_x")] + list(extra) @@ -199,7 +218,7 @@ def run(self, task, depot): scripts = contract.loads(depot.input_bytes(task.input("scripts")))["scripts"] if any(o["class"] == "Crash" for f in doc["files"] for o in f["objects"]): raise ConfigError("setting paths.tool is not set") - container = {(e["file"], e["pathId"]): e["path"] for e in doc["assetBundles"][0]["container"]} + container = {(e["file"], e["pathId"]): e["path"] for b in doc["assetBundles"] for e in b["container"]} only = task.params["classes"] arts, items, first = [], [], None for f in doc["files"]: @@ -330,6 +349,60 @@ def test_select_adds_the_dependencies_to_the_census(): assert assets.select(idx, ["all"]).selected == sorted(stable.values()) +def test_select_always_census_the_built_in_files(): + from nnnotes.pipeline.catalogdb import index + + idx = index(catalog_bytes()) + stable = {n: contract.stable_bundle_name(NAMES[n]) for n in NAMES} + own = ["apk/data.unity3d", "apk/Resources/unity default resources"] + s = assets.select(idx, ["all"], own) + assert s.selected == s.census == sorted([*stable.values(), *own]) + s = assets.select(idx, ["group:bg"], own) + assert s.selected == [stable["bg_x"]] and s.census == sorted([stable["bg_x"], *own]) + s = assets.select(idx, ["bundle:apk/data*"], own) + assert s.selected == ["apk/data.unity3d"] and s.census == sorted(own) + + +def test_the_built_in_files_of_the_apk_are_exported(tmp_path, capsys, fake, monkeypatch): + monkeypatch.setattr("nnnotes.pipeline.catalogdb.apk_version", lambda apk: None) + d = setup_data(tmp_path, censuses(broken=False)) + apk = builtin_apk(tmp_path / "base.apk", {"assets/aa/catalog.bin": synth.CatalogWriter().build([])}) + base = [*flags(d), "--apk", apk, "depot", "plan", "--depot", tmp_path / "s", "--json"] + code, out, err = nn(capsys, *base, "--select", "group:bg") + ids = {n["id"] for n in json.loads(out)["nodes"]} + assert "unity.census:apk/data.unity3d" in ids and "unity.export:apk/data.unity3d" not in ids + code, out, err = nn( + capsys, + *flags(d), + "--apk", + apk, + "export", + "assets", + "-o", + tmp_path / "o", + "--depot", + tmp_path / "s", + "--workers", + "0", + ) + assert code == 0, err + root = tmp_path / "o" / "_bundles" / "apk" / "data.unity3d" + assert (root / "unity_builtin_extra" / "Shader" / "UI_Default~7.json").is_file() + assert (root / "globalgamemanagers.assets" / "Shader" / "Blit~2.json").is_file() + run = contract.loads((tmp_path / "o" / assets.REPORTS / "run.json").read_bytes()) + depot = Depot(tmp_path / "s") + key = {t["id"]: t["key"] for t in run["tasks"]}["link.addresses:main"] + rec = depot.result(key)["artifacts"][0] + table = contract.loads(depot.read(rec["content"]["sha256"])) + assert table["files"]["unity_builtin_extra"] == table["files"]["globalgamemanagers.assets"] == "apk/data.unity3d" + assert table["bundles"]["apk/data.unity3d"] == { + "name": "data.unity3d", + "files": ["globalgamemanagers.assets", "unity_builtin_extra"], + "assetBundleName": None, + "keys": [], + } + + # ---------------------------------------------------------------- export def test_export_writes_layouts_reports_and_hits_the_second_time(tmp_path, capsys, fake): d = setup_data(tmp_path, depots=("depot",)) @@ -1196,7 +1269,6 @@ def run(self, task, depot): def raw_apk(tmp_path, monkeypatch): """An APK with one raw file (a CRI sheet of the APK's catalog) and the CRI stage stand-in; -> (apk, name).""" - import zipfile monkeypatch.setattr( assets, @@ -1205,11 +1277,10 @@ def raw_apk(tmp_path, monkeypatch): ) monkeypatch.setattr("nnnotes.pipeline.catalogdb.apk_version", lambda apk: None) name = "cri_assets_embcri/sound/initialse_" + "ab" * 16 - apk = tmp_path / "base.apk" - with zipfile.ZipFile(apk, "w") as z: - z.writestr("assets/aa/catalog.bin", synth.CatalogWriter().build([("Cri/Initial/se", synth.local(name), [])])) - z.writestr("assets/aa/Android/" + name, b"@UTF sheet") - z.writestr("assets/bin/Data/data.unity3d", b"boot") + catalog = synth.CatalogWriter().build([("Cri/Initial/se", synth.local(name), [])]) + apk = builtin_apk( + tmp_path / "base.apk", {"assets/aa/catalog.bin": catalog, "assets/aa/Android/" + name: b"@UTF sheet"} + ) return apk, name diff --git a/tests/golden/unity.export.json b/tests/golden/unity.export.json index d310ee3..434f76a 100644 --- a/tests/golden/unity.export.json +++ b/tests/golden/unity.export.json @@ -7,7 +7,7 @@ "cri.usm": "cri.usm/1", "data": "data/2", "font": "font/2", - "generic.json": "generic.json/1", + "generic.json": "generic.json/2", "gltf.write": "numpy-2.4.6/1", "material.json": "material.json/1", "mesh.arrays": "numpy-2.4.6/1", @@ -26,68 +26,68 @@ }, "fixtures": { "all": { - "key": "10585d61586a4362e7e5a3c8857a9b583fac363abb5874e760d767fde9cb7d30", - "result": "3bd7b57ee922de132411cb8d027a00421fed5c5a595cdb3eef3e3cb5982b7374" + "key": "bfa791aaeaa8744197413dca21b95285355376a687dcb4a5b511410b4761bd28", + "result": "d9f811e5ca222529d22d1c5e2ec9cd89328d0229eac92b7e01d0cb59dbc56077" }, "bundle.json": { - "key": "fb99b80d44ff61bf4b90ea87e40a632c0f34372ddebd60dd00f74368c5f6b94d", - "result": "169b9bab48a8cb6423de878430b6291bf56999a5d640e9893dc2a1eeda9b93f9" + "key": "019bd61d3b8989c2f59289354d72d3857ed37cab1d90316359e6702ebab8a515", + "result": "f1516133579ff990af232a7b6d8b6303104c59997d20905f16aa280403d3a09e" }, "clip.json": { - "key": "535034c7ce7875a3ea19ebadbb4d12bca912621d38982cd47097cb1c66ff62c5", - "result": "fad3849534a5ad3892f968f567c88efa426088232df6e26a8f646757592fa135" + "key": "a9f5fb2a87fbcb197a5bc37f1c520b7b5a4a6fd5b5ad38b1198e9f8601dc223b", + "result": "6448026702f2239970f8041906335b3a2302a4463536d2f0394daa36906d9c79" }, "controller.json": { - "key": "522f6cefe4102041b197c0feda95de6668a7e88ab548b0d682ef26b492079fd1", - "result": "f17bc2432ad36b87f17e3fffec59eab429a803d23f0bc63db018ba2d99a847db" + "key": "472e6426b3af184e4fb17ccc0a147bd7b6fc9e5fc49f32d0ab2c2d79238417e1", + "result": "c43f0d2de335f10063d20f2d208b54cf7da37863860441a22781bce35f0cb1cc" }, "cri.usm": { - "key": "8d1d45652a6a631253b3bad7f096af988fcc3894831a8fe5e7ec90b939bc3427", - "result": "9a0ca28ce190fc8b5c03fc108b657825c1361a05b416894d2f9c55402cae62fd" + "key": "72cba01cb4fe8fa95d052a2fd17380eaacdd722dc0e5fc802bce8a301560e7a0", + "result": "9fecf00c65642c4ebb635839f79d9001cf4308b367711c2acaf82fa53326cff5" }, "data": { - "key": "98f3c28baedd007e88d9a48adccdad2a1970e63df6e487917cf1c6823feb651d", - "result": "a2c0972bc1c388038dedb9b70f4a8a6d67b58dfca0faac3c438610df125c7234" + "key": "1b44691876cbf5e28032ac2858d9616a8445e2357c543291e2806506c8c0a0b4", + "result": "cdaf30aeb760eef66ec41e029940a800af4d72b0ca0c4545c9c911b6c94fd331" }, "font": { - "key": "c1a50c8ae58a104313d93a7296464812f98c82d8c9aa516589b9d6e5cb89bffa", - "result": "d18d119de09ed000abca438357e72951a18d576c2b26b76932adf9621f67d2bd" + "key": "7d5549079bee9a0778d9d698545f21bfd0fea5466e0efa160661cfb527a83ea9", + "result": "d58abbc2a1b9990fadca993ac0502dc2d586b79e4947bfd1b6fb0a9f7bbc41b0" }, "generic.json": { - "key": "3f111bf015cc23af61fb457ad94be8c3479259d2e8ec2388390580161179afde", - "result": "a79ea2404c179705d54adb9c08f917acedd070bffccc01ba72da3f68af522cd4" + "key": "22deb0da7906ad444ca282fb2a8f993106268f748f2fab35357c049e5b020410", + "result": "a41f325a63c894bfe54aba013c1ee7aa6d674fddd49715c5e3898b7f40e6559c" }, "material.json": { - "key": "fd0603efb48c4582fa8664efcc44be7e238dc6168846dd5528ac6ec350732903", - "result": "373545644b49d9047bae346a78186d7bd0b1b5158d17a393b42050e505209e32" + "key": "b8b07211d9dbc578b3541a0e59653b01506a211426cc70b6e0aa2e82a948167e", + "result": "09085e48baebbda4a78a076e5dc66eeb19ae1f53710ec9f9b3553bb86dfdefde" }, "mesh.glb": { - "key": "d14c16a5c18145cc039a4568b4c8561b0120644c2de1e1abd1dd1be1f17917f0", - "result": "650489a1a9d4e52ae66bc44a101ad5c127e9aa836c8d69e4c08c662e70c73b47" + "key": "1c9b9a873c782e16650ef40cc0a41c17d37da55e05da6d44f47c488cc12d5027", + "result": "58304a1fd371109dbc38f99cd578ad8238453e769ae8c9d8562e1e187e2df1bc" }, "mono.json": { - "key": "5820ec6ca51a697d9ad2f3ed1ee5c8a849fd8f813b053f335b86b1b95082abd8", - "result": "cdabce4d56148f552c87738a9208fd36d0cc581590d7e5eb92f8e5b522de2266" + "key": "f2109d87be61d8cf5b0b527326d2cdc474785104091c55e770a18b31cd967b23", + "result": "02bc065f80513aad7e9afc21704d0f9591e0f110c9ff6fd818f357fe139686f8" }, "prefab.json": { - "key": "8994f1ef92388d443797f11c65a61a6346acf01ff0dc8bfbd36dc2fb2d2d23d2", - "result": "368986bcb1df65ea7fb906e01fe49b814c35e41212a079cdfcc5b70d5ed06559" + "key": "583e77337b68385457bb1f7ec789c943d69729706adc18b47d59354bd04fcf94", + "result": "3751c24516ea904f342e272234e57357b4c5d03ef6e90637c5c8bffbbca78d2a" }, "scripts.json": { - "key": "aafda4c0fe111458f3b48e85b8f60d0e9234dd250590591966a8ac3305d99ef9", - "result": "a2644c8fe5f8198b3771374f10ad01b91a9b111d74aff79b85a9b9b2e071e2c5" + "key": "6e6a37fb9135463601b78a8a5f699996221242585bcc31c63863158ca0a3f9dd", + "result": "9e9b03c721f1f163ed6e81c202b767a7af84be95e662d4d3b5a9b8d46bfe3392" }, "shader.json": { - "key": "a71aa982ce0da9929eaf721a1db903ac0565e1232c94a1d451cdf818f2c59f31", - "result": "fef75615b20f6dc056a4934606d1f094159e219b87482b5efae95c407d0f6d55" + "key": "f3c2c279ab2ed59a70932b19111c4f248593ec3ca9e0677b9402c38cb8505d03", + "result": "dac4f3d4f225dd3659ac78ab0b344daaa64e1a0a6f7dfe759f3ab49ffe3b3c43" }, "sprite.png": { - "key": "d42d38c45711b2de94fc4509b0dd8d919d01d123091f09d54e177df7e4cf751d", - "result": "5c6d4968cbc56a1eefb9c8c94b380487f9876f591e33acbef40bb735edbb70b9" + "key": "d35fe9220f41f10d16b8506f48f31774e9303a8ce6e7b2deba3eb2b26edc56d9", + "result": "ebe1ea68a2fb3289366624724ec4a9335ff6e42156ce4003c7669106e3981482" }, "tex.png": { - "key": "a5af40387bb2c9fe6ba31415af4e4028163eab23bd4114ad4384b57159ae1318", - "result": "54d91ffc00400f162f677f9e1525e34698b33dc439560b4372dffcd09a8dd73b" + "key": "1408f5b984cbcc1c74d23e3c6ef518288f2d112b6b2e4007aecb7f811ddfc809", + "result": "fec1a912d37045add008ea5c12941ea9c6e9eeb0cc0adb6b4d5be81c89ac26eb" } }, "libraries": { diff --git a/tests/helpers/fakeunity.py b/tests/helpers/fakeunity.py index 269cc1e..90fbbfd 100644 --- a/tests/helpers/fakeunity.py +++ b/tests/helpers/fakeunity.py @@ -62,6 +62,7 @@ "Animator": 95, "MonoBehaviour": 114, "MonoScript": 115, + "PlayerSettings": 129, "Font": 128, "AssetBundle": 142, "AnimatorController": 91, @@ -501,6 +502,9 @@ def node_of( raise TypeError(f"{name}: no node for {type(value).__name__}") +HEADER_FIELDS = ("m_GameObject", "m_Enabled", "m_Script", "m_Name") # a MonoBehaviour's own fields + + class FakeObjectReader: def __init__( self, @@ -515,8 +519,10 @@ def __init__( platform: int = ANDROID, node=None, bytes_fields=(), + short_read: bool = False, ): self.assets_file, self.path_id, self.tt, self.obj, self.raw = assets_file, path_id, tt, obj, raw + self.short_read = short_read # its class's tree reads fewer bytes than it has: a checked whole read fails self.class_id = CLASS_IDS[class_name] self.type = SimpleNamespace(name=class_name, value=self.class_id) self.byte_size = len(raw) @@ -525,18 +531,30 @@ def __init__( self.typetree_reads = 0 self._node, self._bytes_fields = node, tuple(bytes_fields) + def _stripped(self) -> bool: + """A MonoBehaviour of a file that stores no type trees: its class's own tree reads its header only.""" + return self.type.name == "MonoBehaviour" and not self.assets_file._enable_type_tree + def read_typetree(self, nodes=None, wrap: bool = False, check_read: bool = True): - """The typetree (a copy); with a cut node (UnityPy's head reads) only the fields the node names.""" + """The typetree (a copy); with a cut node (UnityPy's head reads) only the fields the node names. A stripped + MonoBehaviour read with a tree that does not name all its fields fails the size check.""" self.typetree_reads += 1 tt = copy.deepcopy(self.tt) + if nodes is None and self._stripped(): + nodes = self._get_typetree_node() + if nodes is None and check_read and self.short_read: + raise ValueError("Expected to read 996 bytes, but only read 992 bytes") if nodes is not None: names = [c.m_Name for c in nodes.m_Children] + if check_read and self._stripped() and set(tt) - set(names): + raise ValueError(f"Expected to read {len(self.raw) or 100} bytes, but only read 16 bytes") tt = {k: v for k, v in tt.items() if k in names} return tt def _get_typetree_node(self, node=None): if self._node is None: - self._node = node_of(self.tt, "Base", self.type.name, self._bytes_fields) + tt = {k: v for k, v in self.tt.items() if k in HEADER_FIELDS} if self._stripped() else self.tt + self._node = node_of(tt, "Base", self.type.name, self._bytes_fields) return self._node def peek_name(self): @@ -560,8 +578,11 @@ def read_bytes(self, n: int) -> bytes: class FakeSerializedFile: - def __init__(self, name: str, environment, externals=(), version=VERSION, unity_version="6000.3.12f1"): + def __init__( + self, name: str, environment, externals=(), version=VERSION, unity_version="6000.3.12f1", typetrees=True + ): self.name, self.environment, self.version, self.unity_version = name, environment, version, unity_version + self._enable_type_tree = typetrees self.objects: dict[int, FakeObjectReader] = {} self.externals = [SimpleNamespace(path=p) for p in externals] self.container = SimpleNamespace(container=[]) @@ -585,8 +606,8 @@ def __init__(self): self.resources: dict[str, bytes] = {} self.lookups: list[str] = [] - def file(self, name: str, externals=()) -> FakeSerializedFile: - f = FakeSerializedFile(name, self, externals) + def file(self, name: str, externals=(), typetrees: bool = True) -> FakeSerializedFile: + f = FakeSerializedFile(name, self, externals, typetrees=typetrees) self.assets.append(f) return f diff --git a/tests/pipeline/test_builtins.py b/tests/pipeline/test_builtins.py new file mode 100644 index 0000000..be9adec --- /dev/null +++ b/tests/pipeline/test_builtins.py @@ -0,0 +1,47 @@ +import zipfile +from unittest import mock + +from nnnotes.pipeline import builtins, contract +from nnnotes.pipeline.contract import Input +from nnnotes.pipeline.depot import Depot + +DATA = b"UnityFS data" +DEFAULTS = b"\0\0\0\0 a serialized file" + + +def apk(path, members): + with zipfile.ZipFile(path, "w") as z: + z.writestr("classes.dex", b"dex") + for m, data in members.items(): + z.writestr(builtins.DATA_DIR + m, data) + return path + + +def test_the_built_in_files_of_an_apk_are_subjects(tmp_path): + depot = Depot(tmp_path / "s") + a = apk(tmp_path / "base.apk", {"data.unity3d": DATA, "Resources/unity default resources": DEFAULTS}) + got = builtins.inputs(depot, a) + assert got == { + "apk/Resources/unity default resources": Input( + "bundle", contract.sha256(DEFAULTS), len(DEFAULTS), "unity default resources", ({"kind": "depot"},) + ), + "apk/data.unity3d": Input("bundle", contract.sha256(DATA), len(DATA), "data.unity3d", ({"kind": "depot"},)), + } + assert depot.read(contract.sha256(DEFAULTS)) == DEFAULTS + with mock.patch.object(builtins, "ApkSet", side_effect=AssertionError("read again")): + assert builtins.inputs(Depot(tmp_path / "s"), a) == got # remembered by the APK's content + assert builtins.inputs(depot, None) == {} + + +def test_an_apk_directory_and_its_members(tmp_path): + d = tmp_path / "apk" + d.mkdir() + apk(d / "base.apk", {"data.unity3d": DATA, "datapack.unity3d": b"UnityFS pack"}) + assert sorted(builtins.inputs(Depot(tmp_path / "s"), d)) == ["apk/data.unity3d", "apk/datapack.unity3d"] + + +def test_serialized_names(): + assert builtins.serialized_name("apk/Resources/unity default resources") == "unity default resources" + assert builtins.serialized_name("apk/data.unity3d") == "data.unity3d" + assert builtins.serialized_name("card_1") is None + assert builtins.is_builtin("apk/data.unity3d") and not builtins.is_builtin("card_1") diff --git a/tests/pipeline/test_census.py b/tests/pipeline/test_census.py index 933a34b..d215e85 100644 --- a/tests/pipeline/test_census.py +++ b/tests/pipeline/test_census.py @@ -295,8 +295,8 @@ def test_census_stage(tmp_path, monkeypatch): bundle.write_bytes(b"UnityFS\0 not really") loaded = [] - def load(path): - loaded.append(str(path)) + def load(path, name=None): + loaded.append((str(path), name)) return scene_bundle()[0] monkeypatch.setattr(unity, "load", load) @@ -316,10 +316,15 @@ def load(path): assert rec["id"] == "unity.census:x#census" and rec["semantics"]["facts"]["objects"] == 7 assert rec["provenance"]["inputs"] == [{"role": "bundle", "sha256": sha}] assert contract.loads(depot.read(rec["content"]["sha256"])) == census.census_env(scene_bundle()[0]) - assert loaded == [str(bundle)] + assert loaded == [(str(bundle), None)] # a catalog bundle: its files keep their names assert execute(task, depot, registry(stage)).status == "hit" and len(loaded) == 1 with pytest.raises(ValueError, match="expected 'bundle'"): describe(stage, "x", None, Env(depot, {"bundles": {"x": Input("other", sha, size)}})) for waiting in (None, Pending("fetch:x_1.bundle")): with pytest.raises(Pending, match="fetch:x"): describe(stage, "x", None, Env(depot, {"bundles": {"x": waiting}})) + # a serialized file of the APK outside a bundle is loaded under the name references into it use + own = "apk/Resources/unity default resources" + task = describe(stage, own, None, Env(depot, {"bundles": {own: inp}})) + assert execute(task, depot, registry(stage)).status == "ran" + assert loaded[-1] == (str(bundle), "unity default resources") diff --git a/tests/pipeline/test_cristages.py b/tests/pipeline/test_cristages.py index 4f2c418..97b24f5 100644 --- a/tests/pipeline/test_cristages.py +++ b/tests/pipeline/test_cristages.py @@ -15,7 +15,7 @@ from nnnotes.core import cache from nnnotes.formats import cri, crikey -from nnnotes.pipeline import contract, cristages +from nnnotes.pipeline import builtins, contract, cristages from nnnotes.pipeline.contract import IncompatibleTask, Input, Task from nnnotes.pipeline.cristages import AUDIO, AudioStage, MovieStage from nnnotes.pipeline.depot import Depot @@ -106,14 +106,19 @@ def catalog_index(keys: dict, bundles=()) -> dict: } +def boot_input(depot, tmp) -> Input: + apk = tmp / "base.apk" + with zipfile.ZipFile(apk, "w") as z: + z.writestr(builtins.DATA_DIR + "data.unity3d", b"UnityFS boot data") + return cristages.boot_input(builtins.inputs(depot, apk)) + + def planning(depot, tmp, files: dict, keys=None, boot=True, bundles=()) -> Env: raw = {s: file_input(depot, tmp, s, d) for s, d in files.items()} - b = tmp / "data.unity3d" - b.write_bytes(b"UnityFS boot data") facts = { "raw": raw, "index": catalog_index(keys or {}, bundles), - "boot": cristages.boot_input(depot, b) if boot else None, + "boot": boot_input(depot, tmp) if boot else None, } return Env(depot, facts) @@ -163,28 +168,17 @@ def load(depot, doc, aid): # ---------------------------------------------------------------- the key's source -def test_boot_input_from_an_apk_is_stored_once(tmp_path): +def test_boot_input_is_the_data_of_the_apk(tmp_path): depot = Depot(tmp_path / "s") - apk = tmp_path / "base.apk" - with zipfile.ZipFile(apk, "w") as z: - z.writestr(cristages.BOOT_IN_APK, b"UnityFS boot") - z.writestr("classes.dex", b"dex") - a = cristages.boot_input(depot, apk) + a = boot_input(depot, tmp_path) assert (a.role, a.sha256, a.size, a.name, a.locators) == ( "boot", - contract.sha256(b"UnityFS boot"), - 12, + contract.sha256(b"UnityFS boot data"), + 17, "data.unity3d", ({"kind": "depot"},), ) - assert depot.read(a.sha256) == b"UnityFS boot" - with mock.patch.object(zipfile, "ZipFile", side_effect=AssertionError("read again")): - assert cristages.boot_input(Depot(tmp_path / "s"), apk) == a # remembered by the APK's content - d = tmp_path / "data.unity3d" - d.write_bytes(b"UnityFS other") - b = cristages.boot_input(depot, d) - assert b.sha256 == contract.sha256(b"UnityFS other") and b.locators[0]["kind"] == "file" - assert cristages.boot_input(depot, None) is None + assert cristages.boot_input({}) is None def test_hca_key_is_read_once_per_content(tmp_path): diff --git a/tests/pipeline/test_objexport.py b/tests/pipeline/test_objexport.py index ddee38c..6689c94 100644 --- a/tests/pipeline/test_objexport.py +++ b/tests/pipeline/test_objexport.py @@ -560,10 +560,62 @@ def bundle_movie() -> FakeEnvironment: return env -BUNDLES = {"all": bundle_all, "atlas": bundle_atlas, "plain": bundle_plain, "movie": bundle_movie} +# a player's own file: no type trees; MonoBehaviours whose classes have a shipped type tree (one of another type +# hash), and none +PLAYER = "globalgamemanagers.assets" +PLAYER_HASH, OTHER_HASH = "a1" * 16, "b2" * 16 +SETTINGS, IMAGE, CHANGED = "Fwk|Fwk.Sound|SoundVolumeSettings", "UnityEngine.UI|UnityEngine.UI|Image", "Game|Game|Moved" +PLAYER_SCRIPTS = {f"{PLAYER}:50": SETTINGS, f"{PLAYER}:51": IMAGE, f"{PLAYER}:52": CHANGED} +SETTINGS_ROWS = [ + [0, "MonoBehaviour", "Base", 0], + [1, "PPtr", "m_GameObject", 0], + [2, "int", "m_FileID", 0], + [2, "SInt64", "m_PathID", 0], + [1, "UInt8", "m_Enabled", 0x4000], + [1, "PPtr", "m_Script", 0], + [2, "int", "m_FileID", 0], + [2, "SInt64", "m_PathID", 0], + [1, "string", "m_Name", 0], + [1, "PPtr", "target", 0], + [2, "int", "m_FileID", 0], + [2, "SInt64", "m_PathID", 0], + [1, "float", "volume", 0], +] +PLAYER_TYPETREES = { + SETTINGS: {"typeHash": PLAYER_HASH, "nodes": SETTINGS_ROWS}, + CHANGED: {"typeHash": OTHER_HASH, "nodes": SETTINGS_ROWS}, +} + + +def bundle_player() -> FakeEnvironment: + env = FakeEnvironment() + f = env.file(PLAYER, externals=["Library/unity default resources"], typetrees=False) + h = bytes.fromhex(PLAYER_HASH) + settings = {"productGUID": "00", "m_ActiveColorSpace": 1, "m_MTRendering": True} + f.add(1, "PlayerSettings", settings, type_hash=bytes.fromhex("936fd1913a4f0b5da2ce7cadb0ce9666"), short_read=True) + f.add(6, "PlayerSettings", settings, type_hash=h, short_read=True) + f.add(2, "MonoBehaviour", _mono(0, pptr(50), "volume", target=pptr(10, 1), volume=0.5), type_hash=h) + f.add(3, "MonoBehaviour", _mono(20, pptr(51), "", m_Sprite=pptr(10, 1)), type_hash=h) + f.add(4, "MonoBehaviour", _mono(0, pptr(52), "moved", target=pptr(0), volume=1.0), type_hash=h) + f.add(5, "MonoBehaviour", _mono(0, pptr(51), "image", m_Sprite=pptr(10, 1)), type_hash=h) + f.add(20, "GameObject", _go("Canvas", [21, 3])) + f.add(21, "Transform", _transform(20)) + for pid, entry in ((50, SETTINGS), (51, IMAGE), (52, CHANGED)): + asm, ns, cls = entry.split("|") + f.add(pid, "MonoScript", {"m_Name": cls, "m_ClassName": cls, "m_Namespace": ns, "m_AssemblyName": asm}) + return env -def open_fake(data: bytes) -> UnityBundle: +BUNDLES = { + "all": bundle_all, + "atlas": bundle_atlas, + "plain": bundle_plain, + "movie": bundle_movie, + "player": bundle_player, +} + + +def open_fake(data: bytes, name: str | None = None) -> UnityBundle: return UnityBundle(BUNDLES[bytes(data).decode().removeprefix("fake:")]()) @@ -574,7 +626,7 @@ def classes_of(name): return sorted({r.type.name for f in BUNDLES[name]().assets for r in f.objects.values()}) -def task_for(depot, name, classes=None, params=None, scripts=None): +def task_for(depot, name, classes=None, params=None, scripts=None, typetrees=None): data = f"fake:{name}".encode() sha = depot.put(data) stage = ExportStage(FAKE_READER) @@ -586,7 +638,7 @@ def task_for(depot, name, classes=None, params=None, scripts=None): p, export_atoms(classes_of(name), FAKE_READER.id), (Input("bundle", sha, len(data), f"{name}.bundle", ({"kind": "depot"},)),), - {"scripts": dict(SCRIPTS if scripts is None else scripts)}, + {"scripts": dict(SCRIPTS if scripts is None else scripts), **({"typetrees": typetrees} if typetrees else {})}, ) @@ -1268,7 +1320,7 @@ def census_doc(view: UnityBundle) -> dict: continue classes[o.class_name] = classes.get(o.class_name, 0) + 1 rec = {"pathId": o.path_id, "classId": o.class_id, "class": o.class_name} - tt = view.typetree(o) + tt = view.reader(o).read_typetree(check_read=False) # a header read, as the census' ext = view.externals(f) def ref(s): @@ -1286,7 +1338,9 @@ def ref(s): if bindings: rec["bindingScripts"] = [ref(b) for b in bindings] objs.append(rec) - files.append({"name": f, "objects": objs}) + files.append( + {"name": f, "unityVersion": view.unity_version(f), "typetree": view.stores_typetrees(f), "objects": objs} + ) return { "schema": contract.CENSUS, "files": files, @@ -1357,6 +1411,68 @@ def test_describe_reads_the_census_and_the_script_table(tmp_path): describe(stage, "all", None, fresh) +def test_files_without_type_trees_read_their_scripts_with_the_shipped_ones(tmp_path): + depot = Depot(tmp_path / "s") + doc = run(depot, task_for(depot, "player", scripts=PLAYER_SCRIPTS, typetrees=PLAYER_TYPETREES)) + items = by_object(doc) + + def p(pid): + return oid(pid, PLAYER) + + def script(pid, cls): + return {"$ref": {"file": PLAYER, "pathId": pid, "class": "MonoScript", "name": cls}} + + assert items[p(2)]["status"] == "exported" + assert obj(depot, doc, f"{p(2)}#json") == { + "$script": SETTINGS, + "m_GameObject": None, + "m_Enabled": 1, + "m_Script": script(50, "SoundVolumeSettings"), + "m_Name": "volume", + "target": {"$ref": {"file": "unity default resources", "pathId": 10}}, + "volume": 0.5, + } + assert items[p(2)]["refs"] == [p(50), "unity default resources:10"] + # a class without a shipped type tree, or with one of another type hash: its header, and the reason + for pid, why in ((5, f"has none for {IMAGE}"), (4, f"type hash {PLAYER_HASH} is not the type tree's")): + assert items[p(pid)]["status"] == "generic" + assert items[p(pid)]["reason"]["code"] == "unsupported.typetree" and why in items[p(pid)]["reason"]["message"] + assert obj(depot, doc, f"{p(5)}#json") == { + "m_GameObject": None, + "m_Enabled": 1, + "m_Script": script(51, "Image"), + "m_Name": "image", + } + # a hierarchy keeps its components' headers and says so + assert art(doc, f"{p(20)}#json")["semantics"]["issues"] == [ + { + "code": "unsupported.typetree", + "message": "1 components read in part (their file stores no type tree that fits)", + } + ] + # a class whose type tree reads the stored layout right up to a known field (by its type hash): those fields + assert items[p(1)]["status"] == "generic" and items[p(1)]["reason"]["code"] == "unsupported.typetree" + assert "read up to m_ActiveColorSpace" in items[p(1)]["reason"]["message"] + assert obj(depot, doc, f"{p(1)}#json") == {"productGUID": "00", "m_ActiveColorSpace": 1} + assert items[p(6)]["status"] == "failed" and items[p(6)]["reason"]["code"] == "failed.read" + (component,) = obj(depot, doc, f"{p(20)}#json")["nodes"][0]["components"] + assert component["script"] == IMAGE and set(component["fields"]) == {"m_Enabled", "m_Script", "m_Name"} + assert items[p(3)]["status"] == "contained" + + +def test_the_context_holds_the_shipped_type_trees_of_its_scripts(tmp_path): + from nnnotes.unity import scripttypes + + shipped = scripttypes.load_typetrees("6000.3.12f1")["classes"]["DOTween.dll|DG.Tweening.Core|DOTweenSettings"] + entry = "DOTween|DG.Tweening.Core|DOTweenSettings" # as the script table names it + env = planning_env(Depot(tmp_path / "s"), names=("player",), scripts={**PLAYER_SCRIPTS, f"{PLAYER}:50": entry}) + t = describe(ExportStage(FAKE_READER), "player", None, env) + assert t.context["typetrees"] == {entry: shipped} + assert ( + "typetrees" not in describe(ExportStage(FAKE_READER), "all", None, planning_env(Depot(tmp_path / "t"))).context + ) + + def test_class_selection_keeps_only_its_converters_in_the_key(tmp_path): depot = Depot(tmp_path / "s") env = planning_env(depot) diff --git a/tests/unity/test_typetrees.py b/tests/unity/test_typetrees.py index e62edc1..ce285e6 100644 --- a/tests/unity/test_typetrees.py +++ b/tests/unity/test_typetrees.py @@ -12,17 +12,8 @@ from nnnotes import cli from nnnotes.source.config import ConfigError -from nnnotes.unity.player import ( - HEADER, - TYPETREE_FORMAT, - PlayerData, - UnsupportedVersion, - dump_typetrees, - load_typetrees, - manifest_version_name, - typetree_key, - typetree_node, -) +from nnnotes.unity.player import HEADER, PlayerData, UnsupportedVersion, manifest_version_name +from nnnotes.unity.scripttypes import TYPETREE_FORMAT, dump_typetrees, load_typetrees, typetree_key, typetree_node from tests.helpers import synth TYPETREE_DIR = resources.files("nnnotes.unity").joinpath("typetrees")