From 23cf493b68ed530b037d6a2b9d4d5eda673952b6 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Thu, 15 Oct 2026 02:50:48 +0800 Subject: [PATCH] feat(census)!: list what every object references and the hierarchy holding it The census (document nnnotes.census/2, stage version 3) reads every object's typetree and records two new fields per object: `refs`, the object ids its references name, and `unit`, the root GameObject of the hierarchy whose prefab holds it. The objects reachable from a request therefore follow from censuses alone, before anything is converted; they are a superset of the references in the unity.export documents. Shipped MonoBehaviour type trees are now selected by the type hash the file stores for an object. The census of a built-in file of the APK reads its MonoBehaviours with them and names them in its key (atom `typetrees`). BREAKING CHANGE: census documents are nnnotes.census/2, and the unity.export context keys shipped type trees by type hash. Co-Authored-By: Claude Opus 5.5 --- docs/contracts.md | 2 +- docs/schemas.md | 2 +- docs/stages.md | 20 +- src/nnnotes/pipeline/census.py | 203 ++++++++------ src/nnnotes/pipeline/contract.py | 2 +- src/nnnotes/pipeline/objexport.py | 36 +-- src/nnnotes/schemas/census.schema.json | 26 +- src/nnnotes/unity/atoms/reader.py | 4 + src/nnnotes/unity/convert/objects.py | 82 ++++-- src/nnnotes/unity/scripttypes.py | 36 ++- tests/helpers/fakeunity.py | 21 +- tests/pipeline/test_census.py | 366 ++++++++++++------------- tests/pipeline/test_objexport.py | 37 +-- 13 files changed, 480 insertions(+), 357 deletions(-) diff --git a/docs/contracts.md b/docs/contracts.md index 389a28b..5c5569b 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -218,7 +218,7 @@ Documents written by particular stages (artifacts of no object: their ids are `< | Document | Stage | Schema | |---|---|---| | `nnnotes.catalog-index/1` | `catalog.index` | [catalog-index.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/catalog-index.schema.json) | -| `nnnotes.census/1` | `unity.census` | [census.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/census.schema.json) | +| `nnnotes.census/2` | `unity.census` | [census.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/census.schema.json) | | `nnnotes.scripts/1` | `link.scripts` | [scripts.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/scripts.schema.json) | | `nnnotes.addresses/1` | `link.addresses` | [addresses.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/addresses.schema.json) | | `nnnotes.artifacts/1` | `link.artifacts` | [artifacts.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/artifacts.schema.json) | diff --git a/docs/schemas.md b/docs/schemas.md index 04a7d89..f0930e7 100644 --- a/docs/schemas.md +++ b/docs/schemas.md @@ -20,7 +20,7 @@ schemas.load("plan") # the schema as a dict | `catalog-diff` | `nnnotes.catalog-diff/1` | the difference between two catalog versions (`catalog diff`) | | `catalog-index` | `nnnotes.catalog-index/1` | the index of a catalog pair: remote and APK catalogs, every location in catalog order | | `catalogs` | `nnnotes.catalogs/1` | the catalog versions of a depot (`catalog versions`) | -| `census` | `nnnotes.census/1` | the census of one bundle: serialized files, type trees and objects, read without decoding | +| `census` | `nnnotes.census/2` | the census of one bundle: serialized files, type trees, objects and what they reference, read without decoding | | `common` | | definitions shared by the other schemas | | `cost` | `nnnotes.cost/1` | the measured cost of a task's last run | | `coverage` | `nnnotes.coverage/1` | the coverage of a run per class and per reason code | diff --git a/docs/stages.md b/docs/stages.md index c6746a8..a350faf 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -12,16 +12,17 @@ The asset export is built from three layers: ## Stages -Every stage is at version 1, except `unity.census` (version 2: it lists the scripts animation clip bindings name) +Every stage is at version 1, except `unity.census` (version 3: it lists what every object references and the +hierarchy holding it) and `cri.movie` (version 2: every stream kind, codec and channel of a USM), and `catalog.index` (version 2: gzip input and Japanese remote asset placeholders). | Stage | Subject | Inputs | Outputs | |---|---|---|---| | `catalog.index` | a catalog (remote + APK) | the catalog files | every location (with the bundle hash, crc and size), bundles and raw remote files | -| `unity.census` | a bundle, or a built-in file of the APK | the bundle | serialized files, externals, objects (file, path id, class, size, type hash, name, script, the scripts an AnimationClip's bindings name), container, scene hashes, dependencies, script fields | +| `unity.census` | a bundle, or a built-in file of the APK | the bundle | serialized files, externals, objects (file, path id, class, size, type hash, name, script, the scripts an AnimationClip's bindings name, the objects it references, the hierarchy holding it), container, scene hashes, dependencies, script fields | | `link.scripts` | global | the censuses holding MonoScripts | serialized file and path id -> assembly, namespace, class | -| `unity.export` | a bundle, or a built-in file of the APK | the bundle, the script entries its MonoBehaviours use (and, for files without type trees, the type trees nnnotes ships for their classes) | the objects' artifacts and one item per object | +| `unity.export` | a bundle, or a built-in file of the APK | the bundle, the script entries its MonoBehaviours use (and, for files without type trees, the type trees nnnotes ships for their type hashes) | the objects' artifacts and one item per object | | `sprite.crop` | a SpriteAtlas | the atlas' JSON, the images of its textures, the metas of the sprites of other bundles it packs | the sprites' images | | `cri.audio` | ACB content | `acb` (a raw file of the catalog, or the `acb` artifact of a cue sheet held in a bundle), `awb` (an AWB the catalog pairs with it), `boot` (the game's boot data: the key's source) | FLAC per stream, `cues.json`, `streams.json` | | `cri.movie` | USM content | `usm` (a raw file of the catalog, or the `usm` artifact of a movie held in a bundle), `boot` | every stream as stored (video, alpha, audio, subtitles), `movie.mkv` (or `movie.webm`) | @@ -65,7 +66,9 @@ with its reason code; any other error of a converter gives it with `generic.erro Every object a converter takes has a `json` artifact or is contained in another object's. Each item lists in `refs` the objects its documents reference (by object id, other bundles' objects included, the objects of its own -prefab left out): the objects reachable from an object follow from the items alone. +prefab left out): the objects reachable from an object follow from the items alone. The census of a bundle lists the +same before anything is converted, as a superset: per object `refs`, every object its typetree references, and +`unit`, the root of the hierarchy whose prefab holds it. A sprite's `json` holds `sprite` (its typetree), `mesh` (its geometry decoded: `positions` in the sprite's units, `submeshes` of triangles, `uv0` when stored), `renderData` (`source`: `renderData` for the sprite's own render @@ -93,10 +96,11 @@ subjects. Census always covers them; everything and `--select bundle:apk/*` expo These files store no type trees. Objects of the engine's classes are read with the type trees UnityPy has for the file's Unity version; MonoBehaviours with the type trees nnnotes ships for the game's classes it reads there (`nnnotes.unity.scripttypes`, generated by `scripts/gen_typetrees.py`): the render pipeline assets, renderer data, -renderer features, post-process data and the game's settings objects. A shipped type tree is used only when the type -hash the file stores is the one it was made for and its read consumes exactly the object's bytes; the task's -context holds the shipped type trees its MonoBehaviours use, so a new shipped tree runs those tasks again. A -MonoBehaviour without one is `unsupported.typetree` with its header (`m_GameObject`, `m_Enabled`, `m_Script`, +renderer features, post-process data and the game's settings objects. A MonoBehaviour takes the shipped type tree +made for the type hash the file stores for it (the hash of its class's serialized layout), and the read must consume +exactly the object's bytes. The export task's context holds the shipped type trees its MonoBehaviours use, and the +census of a built-in file has the shipped type trees' id among its atoms, so a new shipped tree runs those tasks +again. A MonoBehaviour without one is `unsupported.typetree` with its header (`m_GameObject`, `m_Enabled`, `m_Script`, `m_Name`) as its typetree JSON; inside a hierarchy its component keeps that header and the prefab records the issue. PlayerSettings, whose type tree reads the stored layout right up to `m_ActiveColorSpace` (by its stored type hash), gives the fields up to it. diff --git a/src/nnnotes/pipeline/census.py b/src/nnnotes/pipeline/census.py index 7aa1d50..c7b43c3 100644 --- a/src/nnnotes/pipeline/census.py +++ b/src/nnnotes/pipeline/census.py @@ -1,22 +1,31 @@ -"""The census of a bundle (stage unity.census, document nnnotes.census/1): what a bundle holds, read without -decoding any object. +"""The census of a bundle (stage unity.census, document nnnotes.census/2): what a bundle holds and what its +objects reference, read without decoding any object. files the serialized files, sorted by name, each with its Unity version, platform, whether it embeds type trees, its externals ({file, path, guid, type}; `file` is the name an object id uses: the last component of the path, e.g. CAB-... or "unity default resources") and its objects sorted by - path id: {pathId, classId, class, byteSize, typeHash, name, script}. `name` is the object's m_Name - when its type has one (read without the fields after it), `script` the {file, pathId} of a + path id: {pathId, classId, class, byteSize, typeHash, name, script, bindingScripts, refs, unit}. + `name` is the object's m_Name when its type has one, `script` the {file, pathId} of a MonoBehaviour's m_Script (null for a null reference), `bindingScripts` the {file, pathId} of the MonoScripts an AnimationClip's bindings name (MonoBehaviour fields it animates; sorted, only when - there are any). Objects are grouped per file: the files of one bundle may use the same path ids - (a scene bundle's scene and its shared assets). + there are any). `refs` the object ids every reference in its typetree names (sorted, itself left + out; only when there are any), `unit` the object id of the root GameObject of the hierarchy whose + prefab holds it (unity.export converts the objects of a hierarchy together; only for the objects + of a hierarchy other than its root). Objects are grouped per file: the files of one bundle may use + the same path ids (a scene bundle's scene and its shared assets). resources the other files of the bundle (.resS / .resource): {name, size} assetBundles the AssetBundle objects: {file, pathId, name, assetBundleName, dependencies, isStreamedSceneAssetBundle, sceneHashes [[scene, hash]], container [{path, file, pathId, preloadIndex, preloadSize}] (stored order)} scripts the MonoScript objects: {file, pathId, assembly, namespace, class, name} classes objects per class; `objects` the total - errors [{object, message}] of the objects whose header could not be read + errors [{object, message}] of the objects whose typetree could not be read (their record has no name, + script or refs) + +The objects reachable from an object follow from `refs` and `unit` across censuses: they are a superset of what the +documents of unity.export reference (their items' `refs`), so a request's closure computed from censuses holds every +object its products name. A file without type trees (the APK's built-in files) gives a MonoBehaviour's fields only +with a type tree nnnotes ships for its type hash (`shipped`), else its header. A census is a function of the bundle's bytes: it holds no file name, time or path (a serialized file outside a bundle, one of the APK's built-in files, has the name its subject gives: builtins.serialized_name). @@ -26,14 +35,13 @@ import re +from ..unity import scripttypes from ..unity.atoms import impl_id from . import builtins, contract -from .contract import Cost, Input +from .contract import Cost, IncompatibleTask, Input from .stages import Output, Pending, Stage _ADDRESS = re.compile(r"0x[0-9a-fA-F]{6,}") -HEAD_FIELDS = ("m_Name", "m_Script") -CLIP_FIELDS = ("m_Name", "m_ClipBindingConstant") # an AnimationClip is read up to its bindings def _ext_file(path: str) -> str: @@ -46,7 +54,7 @@ def _message(e: BaseException) -> str: def _serialized(env) -> tuple[list, list]: - """(serialized files, [name, size] of the other files) of a loaded environment, walking nested bundles.""" + """(serialized files, {name, size} of the other files) of a loaded environment, walking nested bundles.""" files, resources = [], [] def walk(container): @@ -70,11 +78,10 @@ def walk(container): class _Refs: - """PPtr resolution within one serialized file.""" + """PPtr resolution within one serialized file: its name and the names of its externals.""" - def __init__(self, sf): - self.name = sf.name - self.externals = [_ext_file(x.path) for x in sf.externals] + def __init__(self, name: str, externals: list[str]): + self.name, self.externals = name, list(externals) def file(self, file_id: int) -> str | None: if file_id == 0: @@ -143,89 +150,80 @@ def _binding_scripts(d: dict, refs: _Refs) -> list: return [out[k] for k in sorted(out)] -def census_env(env) -> dict: - """The census document of a loaded UnityPy environment holding one bundle.""" - from UnityPy.helpers.TypeTreeNode import TypeTreeNode - - heads: dict[tuple, tuple] = {} - - def head(node, fields=HEAD_FIELDS): - """The type tree cut after the last of its `fields` (None when it has none of them).""" - hit = heads.get((id(node), fields)) - if hit is not None and hit[0] is node: - return hit[1] - names = [c.m_Name for c in node.m_Children] - last = max((names.index(n) for n in fields if n in names), default=None) - cut = None - if last is not None: - cut = TypeTreeNode( - node.m_Level, - node.m_Type, - node.m_Name, - node.m_ByteSize, - node.m_Version, - m_MetaFlag=node.m_MetaFlag, - m_Children=node.m_Children[: last + 1], - ) - heads[(id(node), fields)] = (node, cut) - return cut - - sfiles, resources = _serialized(env) +def census_env(env, shipped: bool = False) -> dict: + """The census document of a loaded UnityPy environment holding one bundle (or one serialized file). `shipped`: + the MonoBehaviours of files without type trees are read with the type trees nnnotes ships + (nnnotes.unity.scripttypes) for their type hashes.""" + from ..unity.atoms.reader import UnityBundle + from ..unity.convert import prefab + from ..unity.convert.objects import BundleObjects, ReadError + + _, resources = _serialized(env) + view = UnityBundle(env) + trees: dict = {} + if shipped: + for f in view.files(): + if not view.stores_typetrees(f): + trees.update(scripttypes.script_types(view.unity_version(f))) + ctx = BundleObjects(view, {}, scripts={}, blob_min=1, png_level=0, emit=_no_emit, typetrees=trees) + units = _units(ctx, prefab.hierarchies(ctx)) + by_file: dict[str, list] = {} + for i in ctx.infos: + by_file.setdefault(i.file, []).append(i) files, bundles, scripts, errors, classes = [], [], [], [], {} - for sf in sfiles: - refs = _Refs(sf) + for name in sorted(by_file): + sf = view.serialized_file(name) + refs = _Refs(name, ctx.ext_files[name]) objects = [] - for pid in sorted(sf.objects): - o = sf.objects[pid] - try: - cls = o.type.name - except Exception: - cls = f"Class{o.class_id}" - st = getattr(o, "serialized_type", None) - th = getattr(st, "old_type_hash", None) if st is not None else None + for i in sorted(by_file[name], key=lambda i: i.path_id): + key = (i.file, i.path_id) rec = { - "pathId": int(pid), - "classId": int(o.class_id), - "class": cls, - "byteSize": int(o.byte_size), - "typeHash": th.hex() if th else None, + "pathId": i.path_id, + "classId": i.class_id, + "class": i.class_name, + "byteSize": i.byte_size, + "typeHash": i.type_hash, } - classes[cls] = classes.get(cls, 0) + 1 + classes[i.class_name] = classes.get(i.class_name, 0) + 1 try: - node = o._get_typetree_node() - cut = head(node, CLIP_FIELDS if cls == "AnimationClip" else HEAD_FIELDS) - d = o.read_typetree(cut, check_read=False) if cut is not None else {} - if "m_Name" in d: + d = ctx.tt(key, whole=False) + found = ctx.references(key, d) + except Exception as e: + why = str(e) if isinstance(e, ReadError) else _message(e) + errors.append({"object": contract.object_id(*key), "message": why}) + else: + if isinstance(d.get("m_Name"), str): rec["name"] = d["m_Name"] if "m_Script" in d: rec["script"] = refs.ref(d["m_Script"]) - if cls == "AnimationClip": + if i.class_name == "AnimationClip": scripts_ = _binding_scripts(d, refs) if scripts_: rec["bindingScripts"] = scripts_ - if cls == "AssetBundle": - bundles.append(_asset_bundle(o.read_typetree(), refs, (sf.name, int(pid)))) - elif cls == "MonoScript": - m = o.read_typetree() + if found: + rec["refs"] = sorted(contract.object_id(*k) for k in found) + if i.class_name == "AssetBundle": + bundles.append(_asset_bundle(d, refs, key)) + elif i.class_name == "MonoScript": scripts.append( { - "file": sf.name, - "pathId": int(pid), - "assembly": m.get("m_AssemblyName"), - "namespace": m.get("m_Namespace"), - "class": m.get("m_ClassName"), - "name": m.get("m_Name"), + "file": name, + "pathId": i.path_id, + "assembly": d.get("m_AssemblyName"), + "namespace": d.get("m_Namespace"), + "class": d.get("m_ClassName"), + "name": d.get("m_Name"), } ) - except Exception as e: - errors.append({"object": contract.object_id(sf.name, pid), "message": _message(e)}) + if key in units: + rec["unit"] = units[key] objects.append(rec) files.append( { - "name": sf.name, - "unityVersion": getattr(sf, "unity_version", None), + "name": name, + "unityVersion": view.unity_version(name), "platform": int(getattr(sf, "_m_target_platform", getattr(sf, "target_platform", 0)) or 0), - "typetree": bool(getattr(sf, "_enable_type_tree", True)), + "typetree": view.stores_typetrees(name), "externals": [ { "file": _ext_file(x.path), @@ -250,13 +248,34 @@ def head(node, fields=HEAD_FIELDS): } -def census_file(path, name: str | None = None) -> dict: - """The census of a bundle file (decrypted UnityFS), or of a serialized file named `name`.""" +def _no_emit(*args, **kw): + raise RuntimeError("a census writes no artifact") + + +def _units(ctx, h) -> dict[tuple, str]: + """{object key: the object id of the root GameObject of the hierarchy it belongs to} of the objects of each + hierarchy the converters export as one prefab (prefab.hierarchies: walked without error, rooted at a GameObject), + the root left out.""" + out = {} + for root in h.roots: + go = (root.file, root.game_object) + info = ctx.info.get(go) + if root.error is not None or info is None or info.class_name != "GameObject": + continue + oid = contract.object_id(*go) + for k in root.members: + if k != go: + out[k] = oid + return out + + +def census_file(path, name: str | None = None, shipped: bool = False) -> dict: + """The census of a bundle file (decrypted UnityFS), or of a serialized file named `name` (census_env).""" from ..unity import reader as unity env = unity.load(path, name) try: - return census_env(env) + return census_env(env, shipped) finally: del env @@ -309,10 +328,11 @@ class CensusStage(Stage): (role "bundle": the decrypted bundle's sha256, size, name, locators)}, the APK's built-in files included (builtins); a bundle not fetched yet is Pending (raised by the fact, or given as a Pending / None). Artifact "#census". The census reads the bundle with UnityPy: its atom is the reader's ("reader", the UnityPy - version).""" + version); a built-in file's also the shipped type trees ("typetrees", scripttypes.types_id) it reads its + MonoBehaviours with.""" name = "unity.census" - version = 2 + version = 3 ATOMS = {"reader": impl_id(("UnityPy",), 1)} def subjects(self, env) -> list[str]: @@ -328,12 +348,23 @@ def inputs(self, subject: str, env) -> list[Input]: raise ValueError(f"unity.census {subject}: input role {inp.role!r}, expected 'bundle'") return [inp] + def uses(self, subject: str, env) -> dict: + if builtins.is_builtin(subject): + return {**self.ATOMS, "typetrees": scripttypes.types_id()} + return dict(self.ATOMS) + def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: size = sum(i.size for i in inputs) - return Cost(0.002 + size / 40e6, (32 << 20) + 12 * size) + return Cost(0.002 + size / 20e6, (32 << 20) + 14 * size) def run(self, task, depot) -> Output: - doc = census_file(depot.open_input(task.input("bundle")), builtins.serialized_name(task.subject)) + if task.atoms.get("typetrees", scripttypes.types_id()) != scripttypes.types_id(): + raise IncompatibleTask(f"task {task.id}: typetrees are {scripttypes.types_id()} here") + doc = census_file( + depot.open_input(task.input("bundle")), + builtins.serialized_name(task.subject), + shipped="typetrees" in task.atoms, + ) content = depot.add(contract.encode(doc), "json") rec = contract.artifact( contract.artifact_id(task.id, "census"), diff --git a/src/nnnotes/pipeline/contract.py b/src/nnnotes/pipeline/contract.py index 26fa343..97ff7f6 100644 --- a/src/nnnotes/pipeline/contract.py +++ b/src/nnnotes/pipeline/contract.py @@ -53,7 +53,7 @@ PLAN = "nnnotes.plan/1" COST = "nnnotes.cost/1" CATALOG_INDEX = "nnnotes.catalog-index/1" -CENSUS = "nnnotes.census/1" +CENSUS = "nnnotes.census/2" VIEW = "nnnotes.view/1" # item statuses of a result: every object a task covers has exactly one diff --git a/src/nnnotes/pipeline/objexport.py b/src/nnnotes/pipeline/objexport.py index e06d5c8..0c40d9f 100644 --- a/src/nnnotes/pipeline/objexport.py +++ b/src/nnnotes/pipeline/objexport.py @@ -24,7 +24,6 @@ import io from collections import defaultdict -from functools import cache from ..unity import scripttypes from ..unity.atoms import ATOMS as ATOM_REGISTRY @@ -387,8 +386,8 @@ class ExportStage(Stage): (unity.census:) for the classes (the converters and atoms of its key) and the MonoScripts its objects reference, and the script table (link.scripts:all) for the entries of those: its context {"scripts": {object id: "assembly|namespace|class"}}, and for the MonoBehaviours of files that store no type trees - (the APK's built-in files) the type trees nnnotes ships for their classes: "typetrees" {script entry: - {"typeHash", "nodes"}}. Parameters: `blobMin` (bytes; byte arrays of this size or + (the APK's built-in files) the type trees nnnotes ships for their type hashes: "typetrees" {type hash: nodes}. + Parameters: `blobMin` (bytes; byte arrays of this size or more are blobs), `png.level` (zlib level of the PNG files). `reader`: another implementation of the reader atom (its id enters the keys).""" @@ -467,12 +466,12 @@ def uses(self, subject: str, env) -> dict: def context(self, subject: str, env) -> dict: doc = self.census(subject, env) + out = {} refs = census_mod.script_refs(doc) - if not refs: - return {} - table = self._docs.get(env.depot, env.input_of(SCRIPTS_TASK, "scripts").sha256)["scripts"] - out = {"scripts": link.script_context(table, refs)} - types = _script_types(doc, out["scripts"]) + if refs: + table = self._docs.get(env.depot, env.input_of(SCRIPTS_TASK, "scripts").sha256)["scripts"] + out["scripts"] = link.script_context(table, refs) + types = _script_types(doc) if types: out["typetrees"] = types return out @@ -501,25 +500,18 @@ def run(self, task, depot) -> Output: return Output(arts, items) -@cache -def _embedded(unity_version: str) -> dict: - return scripttypes.script_types(unity_version) - - -def _script_types(doc: dict, scripts: dict) -> dict: - """{script entry: {"typeHash", "nodes"}} of the embedded type trees (nnnotes.unity.scripttypes, of each file's - Unity version) of the classes of the MonoBehaviours in the census' files that store no type trees ({} when it has - none).""" +def _script_types(doc: dict) -> dict: + """{type hash: nodes} of the embedded type trees (nnnotes.unity.scripttypes, of each file's Unity version) of the + type hashes of the MonoBehaviours in the census' files that store no type trees ({} when it has none).""" out = {} for f in doc["files"]: if f["typetree"]: continue - types = _embedded(f["unityVersion"]) + types = scripttypes.script_types(f["unityVersion"] or "") for o in f["objects"]: - s = o.get("script") if o["class"] == "MonoBehaviour" else None - entry = scripts.get(contract.object_id(s["file"], s["pathId"])) if s and s.get("file") else None - if entry in types and out.setdefault(entry, types[entry]) != types[entry]: - raise ValueError(f"{entry}: files of several Unity versions with different type trees of it") + h = o["typeHash"] if o["class"] == "MonoBehaviour" else None + if h in types: + out[h] = types[h] return dict(sorted(out.items())) diff --git a/src/nnnotes/schemas/census.schema.json b/src/nnnotes/schemas/census.schema.json index f195134..a1c0354 100644 --- a/src/nnnotes/schemas/census.schema.json +++ b/src/nnnotes/schemas/census.schema.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "urn:nnnotes:schema:census", - "title": "nnnotes.census/1", - "description": "The census of one bundle (stage unity.census), read without decoding any object: its serialized files by name (Unity version, platform, type trees, externals, objects by path id with class, size, type hash, m_Name and m_Script), the other files of the bundle, its AssetBundle objects (container in stored order, dependencies, scene hashes), its MonoScripts, objects per class and the objects whose header could not be read. A function of the bundle's bytes.", + "title": "nnnotes.census/2", + "description": "The census of one bundle (stage unity.census), read without decoding any object: its serialized files by name (Unity version, platform, type trees, externals, objects by path id with class, size, type hash, m_Name, m_Script, the objects they reference and the hierarchy holding them), the other files of the bundle, its AssetBundle objects (container in stored order, dependencies, scene hashes), its MonoScripts, objects per class and the objects whose typetree could not be read. A function of the bundle's bytes.", "type": "object", "required": [ "schema", @@ -16,7 +16,7 @@ ], "properties": { "schema": { - "const": "nnnotes.census/1" + "const": "nnnotes.census/2" }, "files": { "type": "array", @@ -151,7 +151,7 @@ ] }, "bindingScripts": { - "description": "An AnimationClip's MonoScripts named by its bindings (the MonoBehaviour fields it animates), sorted; present only when there are any (stage unity.census version 2).", + "description": "An AnimationClip's MonoScripts named by its bindings (the MonoBehaviour fields it animates), sorted; present only when there are any.", "type": "array", "items": { "anyOf": [ @@ -188,6 +188,21 @@ ] }, "minItems": 1 + }, + "refs": { + "description": "The object ids every reference in the object's typetree names, sorted, itself left out; present only when there are any.", + "type": "array", + "items": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + }, + "minItems": 1, + "uniqueItems": true + }, + "unit": { + "description": "The object id of the root GameObject of the hierarchy whose prefab holds the object (the objects of a hierarchy are converted together); present only for the objects of a hierarchy other than its root.", + "type": "string", + "pattern": "^.+:-?[0-9]+$" } } } @@ -387,7 +402,8 @@ "type": "string" } } - } + }, + "description": "The objects whose typetree could not be read: their records have no name, script or refs." } } } diff --git a/src/nnnotes/unity/atoms/reader.py b/src/nnnotes/unity/atoms/reader.py index 2f4cf9e..4d4f8e2 100644 --- a/src/nnnotes/unity/atoms/reader.py +++ b/src/nnnotes/unity/atoms/reader.py @@ -242,6 +242,10 @@ def reader(self, obj: ObjectInfo): """The library's object reader of `obj` (UnityPy's ObjectReader), for code that works on library objects.""" return self._readers[(obj.file, obj.path_id)] + def serialized_file(self, file: str): + """The library's serialized file (UnityPy's SerializedFile) named `file`.""" + return self._files[file] + def files(self) -> list[str]: return list(self._files) diff --git a/src/nnnotes/unity/convert/objects.py b/src/nnnotes/unity/convert/objects.py index dc3a44b..d6383cf 100644 --- a/src/nnnotes/unity/convert/objects.py +++ b/src/nnnotes/unity/convert/objects.py @@ -6,11 +6,12 @@ task-level name `owner`) and returns (artifact id, content record). Every object a document names by {"$ref"} is recorded in `seen` (by object id, other bundles' objects included): -the objects an object's documents reference, which the caller clears before converting an object. +the objects an object's documents reference, which the caller clears before converting an object. `references(key)` +gives the objects an object's typetree references, read as tt(key, whole=False) reads it. A file that stores no type trees (a player build's own files) gives its MonoBehaviours only their header; they are -read with the task's type tree of their script class (`typetrees`, nnnotes.unity.scripttypes), checked against the -type hash the file stores and the object's size. One without such a tree is `unsupported.typetree` (TYPETREE): its +read with the given type tree of the type hash the file stores for them (`typetrees`, nnnotes.unity.scripttypes), +which must consume exactly the object's bytes. One without such a tree is `unsupported.typetree` (TYPETREE): its header is still read. Its other objects are read with the type trees of their classes UnityPy has for the file's Unity version; for a class whose tree reads the stored layout right only up to a field (PREFIXES, by the type hash the file stores) the typetree JSON is the fields up to it, also `unsupported.typetree`. The objects read in part are @@ -190,8 +191,8 @@ class Capture: class BundleObjects: """The objects of one bundle (`view`, a reader atom's BundleView). `fns`: the atom functions by name; `scripts`: the script table entries the bundle's objects use ({MonoScript object id: "assembly|namespace|class"}); - `typetrees`: the type trees of script classes for the files that store none ({script entry: {"typeHash", - "nodes"}}); `blob_min`: byte arrays of this size or more are blobs; `png_level`: the zlib level of PNG files.""" + `typetrees`: the type trees of script classes for the files that store none ({type hash: nodes}); `blob_min`: + byte arrays of this size or more are blobs; `png_level`: the zlib level of PNG files.""" def __init__( self, @@ -216,7 +217,7 @@ def __init__( self.stripped = frozenset(f for f in self.files if not view.stores_typetrees(f)) self.partial: dict[tuple, str] = {} # objects read in part -> why self._nodes: dict[tuple, object] = {} # objects read with a type tree of `typetrees` -> its node - self._script_nodes: dict[str, object] = {} + self._script_nodes: dict[str, object] = {} # type hash -> node self.containers: dict[tuple, str] = {} for path, file, pid in view.container(): k = (file, int(pid)) @@ -306,22 +307,18 @@ def _script_tt(self, key: tuple, info, whole: bool) -> dict: except Exception as e: raise ReadError(message(e)) from None entry = self.mono_script(key, head) - t = self.typetrees.get(entry) if entry is not None else None - if entry is None or entry == NO_SCRIPT: - s = head.get("m_Script") or {} - why = f"script {s.get('m_FileID')}:{s.get('m_PathID')} is unknown and the file stores no type trees" - elif t is None: - why = f"the file stores no type trees and the task has none for {entry}" - elif t["typeHash"] != info.type_hash: - why = f"{entry}: the file's type hash {info.type_hash} is not the type tree's {t['typeHash']}" + what = entry if entry and entry != NO_SCRIPT else "its script" + rows = self.typetrees.get(info.type_hash) if info.type_hash else None + if rows is None: + why = f"the file stores no type trees and none is given for {what} (type hash {info.type_hash})" else: - node = self._script_nodes.get(entry) + node = self._script_nodes.get(info.type_hash) if node is None: - node = self._script_nodes[entry] = typetree_node(t["nodes"]) + node = self._script_nodes[info.type_hash] = typetree_node(rows) try: tt = r.read_typetree(node, check_read=True) except Exception as e: - why = f"{entry}: the type tree does not fit the object ({message(e)})" + why = f"the type tree of {what} does not fit the object ({message(e)})" else: self._nodes[key] = node self.partial.pop(key, None) @@ -333,6 +330,57 @@ def _script_tt(self, key: tuple, info, whole: bool) -> dict: self.partial[key] = why return head + def references(self, key: tuple, tt: dict | None = None) -> set[tuple]: + """The (file, pathId) of every object the object's typetree references (`tt`, as tt(key, whole=False) reads + it; read here when not given), itself and references with an unknown file index left out (ReadError when it + cannot be read).""" + r = self.readers[key] + tt = self.tt(key, whole=False) if tt is None else tt + out: set[tuple] = set() + self._pptrs(r, self._nodes.get(key) or r._get_typetree_node(), tt, out) + out.discard(key) + return out + + def _pptrs(self, owner, node, v, out: set) -> None: + """Collect the targets of the references in `v` (read with `node`), as _walk finds them.""" + if not self._special(node): + return + t = node.m_Type + if t.startswith("PPtr<"): + if isinstance(v, dict): + k = self.target(owner, v) + if k is not None: + out.add(k) + return + if t == "TypelessData" or not isinstance(v, (dict, list, tuple)): + return + ch = node.m_Children + if t == "ReferencedObject" and isinstance(v, dict): + for k, x in v.items(): + c = self._child(node, k) + if c is not None and c.m_Type == "ReferencedObjectData": + rn = _ref_type_node(v, owner.assets_file) + if rn is not None: + self._pptrs(owner, rn, x, out) + elif c is not None: + self._pptrs(owner, c, x, out) + return + if t != "string" and ch and ch[0].m_Type == "Array": + sub = ch[0].m_Children[1] + if sub.m_Type not in _BYTE_TYPES: + for x in v: + self._pptrs(owner, sub, x, out) + return + if t == "pair": + self._pptrs(owner, ch[0], v[0], out) + self._pptrs(owner, ch[1], v[1], out) + return + if isinstance(v, dict): + for k, x in v.items(): + c = self._child(node, k) + if c is not None: + self._pptrs(owner, c, x, out) + def _prefix(self, r, last: str) -> dict: """An object's typetree up to the field `last` (its class's tree cut after it).""" cut = _cut(r._get_typetree_node(), last) diff --git a/src/nnnotes/unity/scripttypes.py b/src/nnnotes/unity/scripttypes.py index 2f7d7f3..a2bf469 100644 --- a/src/nnnotes/unity/scripttypes.py +++ b/src/nnnotes/unity/scripttypes.py @@ -8,16 +8,20 @@ {"format": 1, "unityVersion", "gameVersion", "classes": {"assembly|namespace|class": {"typeHash", "nodes"}}} A class is keyed by the names its MonoScript stores (typetree_key); `nodes` are [level, type, name, meta flag] rows -of Unity's layout and `typeHash` the hex type hash the serialized file stores for the class. A tree fits an object -when the type hashes are equal and its read consumes exactly the object's serialized size. +of Unity's layout and `typeHash` the hex type hash the serialized file stores for the class. The type hash is the +hash of the class's serialized layout, so a stored object picks its tree by it alone (script_types); a tree fits the +object when its read consumes exactly the object's serialized size. `types_id()` names the shipped trees (the files' +contents), for the keys of the tasks that read with them. Nothing here imports UnityPy until typetree_node is called. """ from __future__ import annotations +import hashlib import json import re +from functools import cache from importlib import resources __all__ = [ @@ -26,6 +30,7 @@ "load_typetrees", "script_entry", "script_types", + "types_id", "typetree_file", "typetree_key", "typetree_node", @@ -66,13 +71,28 @@ def load_typetrees(unity_version: str) -> dict | None: return doc -def script_types(unity_version: str) -> dict[str, dict]: - """{script table entry (script_entry): {"typeHash", "nodes"}} of the embedded typetrees of a Unity version ({} - when nnnotes has none for it).""" +@cache +def script_types(unity_version: str) -> dict[str, list]: + """{type hash: nodes} of the embedded typetrees of a Unity version ({} when nnnotes has none for it). Shared: + callers must not change it.""" doc = load_typetrees(unity_version) - if doc is None: - return {} - return {script_entry(*k.split("|", 2)): v for k, v in sorted(doc["classes"].items())} + out: dict[str, list] = {} + for k, v in sorted((doc or {"classes": {}})["classes"].items()): + if out.setdefault(v["typeHash"], v["nodes"]) != v["nodes"]: + raise ValueError(f"typetree file {unity_version}.json: {k} and another class share a type hash") + return out + + +@cache +def types_id() -> str: + """The id of the shipped typetrees: the digest of every typetree file's name and bytes.""" + h = hashlib.sha256() + folder = resources.files("nnnotes.unity").joinpath("typetrees") + for f in sorted((f for f in folder.iterdir() if f.name.endswith(".json")), key=lambda f: f.name): + data = f.read_bytes() + h.update(f"{f.name}/{len(data)}/".encode()) + h.update(data) + return f"typetrees-{h.hexdigest()[:16]}/1" def typetree_node(rows: list): diff --git a/tests/helpers/fakeunity.py b/tests/helpers/fakeunity.py index 90fbbfd..49089c1 100644 --- a/tests/helpers/fakeunity.py +++ b/tests/helpers/fakeunity.py @@ -520,12 +520,15 @@ def __init__( node=None, bytes_fields=(), short_read: bool = False, + broken: bool = False, + byte_size: int | None = None, ): self.assets_file, self.path_id, self.tt, self.obj, self.raw = assets_file, path_id, tt, obj, raw self.short_read = short_read # its class's tree reads fewer bytes than it has: a checked whole read fails + self.broken = broken # no read succeeds self.class_id = CLASS_IDS[class_name] self.type = SimpleNamespace(name=class_name, value=self.class_id) - self.byte_size = len(raw) + self.byte_size = len(raw) if byte_size is None else byte_size self.serialized_type = SimpleNamespace(old_type_hash=type_hash) self.version, self.platform = version, platform self.typetree_reads = 0 @@ -539,6 +542,8 @@ def read_typetree(self, nodes=None, wrap: bool = False, check_read: bool = True) """The typetree (a copy); with a cut node (UnityPy's head reads) only the fields the node names. A stripped MonoBehaviour read with a tree that does not name all its fields fails the size check.""" self.typetree_reads += 1 + if self.broken: + raise ValueError(f"cannot read ") tt = copy.deepcopy(self.tt) if nodes is None and self._stripped(): nodes = self._get_typetree_node() @@ -583,8 +588,13 @@ def __init__( ): self.name, self.environment, self.version, self.unity_version = name, environment, version, unity_version self._enable_type_tree = typetrees + self._m_target_platform = ANDROID self.objects: dict[int, FakeObjectReader] = {} - self.externals = [SimpleNamespace(path=p) for p in externals] + # an external: its path, or (path, guid bytes, type) + self.externals = [ + SimpleNamespace(path=p) if isinstance(p, str) else SimpleNamespace(path=p[0], guid=p[1], type=p[2]) + for p in externals + ] self.container = SimpleNamespace(container=[]) self.ref_types: list = [] @@ -614,3 +624,10 @@ def file(self, name: str, externals=(), typetrees: bool = True) -> FakeSerialize def get_cab(self, name: str): self.lookups.append(name) return FakeResource(self.resources[name]) if name in self.resources else None + + @property + def files(self) -> dict: + """The loaded files as UnityPy lists them: one bundle holding the serialized files and resource files.""" + inner: dict = {f.name: f for f in self.assets} + inner.update({n: SimpleNamespace(Length=len(d)) for n, d in self.resources.items()}) + return {"fake.bundle": SimpleNamespace(files=inner)} diff --git a/tests/pipeline/test_census.py b/tests/pipeline/test_census.py index d215e85..8a6b73a 100644 --- a/tests/pipeline/test_census.py +++ b/tests/pipeline/test_census.py @@ -1,191 +1,148 @@ -from types import SimpleNamespace - import pytest -from UnityPy.helpers.TypeTreeNode import TypeTreeNode from nnnotes.pipeline import census, contract from nnnotes.pipeline.contract import Input from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.stages import Env, Pending, describe, execute, registry from nnnotes.unity import reader as unity +from nnnotes.unity import scripttypes +from tests.helpers.fakeunity import FakeEnvironment, pptr SCRIPTS = "CAB-5c1e0000000000000000000000000000" +MAIN = "CAB-aaaa0000000000000000000000000000" +SHARED = MAIN + ".sharedAssets" +HASH = bytes.fromhex("01" * 16) +# a MonoBehaviour's tree in Unity's layout: its header, a texture reference and a float +ROWS = [ + [0, "MonoBehaviour", "Base", 0], + [1, "PPtr", "m_GameObject", 0], + [2, "int", "m_FileID", 0], + [2, "SInt64", "m_PathID", 0], + [1, "UInt8", "m_Enabled", 0x4000], + [1, "PPtr", "m_Script", 0], + [2, "int", "m_FileID", 0], + [2, "SInt64", "m_PathID", 0], + [1, "string", "m_Name", 0], + [1, "PPtr", "target", 0], + [2, "int", "m_FileID", 0], + [2, "SInt64", "m_PathID", 0], + [1, "float", "volume", 0], +] + + +def tf(go, children=(), father=0): + return { + "m_GameObject": pptr(go), + "m_LocalRotation": {"x": 0.0, "y": 0.0, "z": 0.0, "w": 1.0}, + "m_LocalPosition": {"x": 0.0, "y": 0.0, "z": 0.0}, + "m_LocalScale": {"x": 1.0, "y": 1.0, "z": 1.0}, + "m_Children": [pptr(c) for c in children], + "m_Father": pptr(father), + } -def node(cls: str, *fields: str) -> TypeTreeNode: - return TypeTreeNode(0, cls, "Base", -1, 1, m_Children=[TypeTreeNode(1, "x", f, -1, 0) for f in fields]) - - -class Obj: - """An object as UnityPy reads it: header reads get the fields of the node they are given.""" +def go(name, comps): + return {"m_Component": [{"component": pptr(c)} for c in comps], "m_Layer": 0, "m_Name": name, "m_IsActive": 1} - def __init__(self, pid, class_id, cls, fields: dict, *, size=100, type_hash=b"\x01" * 16, broken=False): - self.path_id, self.class_id, self.byte_size = pid, class_id, size - self.type = SimpleNamespace(name=cls) - self.serialized_type = SimpleNamespace(old_type_hash=type_hash) - self.fields, self.broken, self.reads = fields, broken, [] - self.node = node(cls, *fields) - def _get_typetree_node(self): - return self.node +def mono(go_pid, script, name="", **fields): + return {"m_GameObject": pptr(go_pid), "m_Enabled": 1, "m_Script": script, "m_Name": name, **fields} - def read_typetree(self, nodes=None, wrap=False, check_read=True): - if self.broken: - raise ValueError(f"cannot read ") - names = [c.m_Name for c in (nodes or self.node).m_Children] - self.reads.append(names) - return {k: self.fields[k] for k in names} +def asset_bundle(): + def entry(index, size, asset): + return {"preloadIndex": index, "preloadSize": size, "asset": asset} -def ext(path, guid=b"", typ=0): - return SimpleNamespace(path=path, guid=guid, type=typ) + return { + "m_Name": "scene_x", + "m_PreloadTable": [], + "m_Container": [ + ("Assets/Scenes/X.unity", entry(0, 0, pptr(0))), + ("Assets/Tex/tex.png", entry(0, 1, pptr(-5))), + ("Assets/Tex/tex.png", entry(1, 1, pptr(3, 2))), + ], + "m_MainAsset": {"preloadIndex": 0, "preloadSize": 0, "asset": pptr(0)}, + "m_RuntimeCompatibility": 1, + "m_AssetBundleName": "0123_abcd.bundle", + "m_Dependencies": ["dep_1.bundle"], + "m_IsStreamedSceneAssetBundle": True, + "m_SceneHashes": [("Assets/Scenes/X.unity", "ffee")], + } -def sfile(name, objs, externals=()): - return SimpleNamespace( - name=name, - objects={o.path_id: o for o in objs}, - externals=list(externals), - unity_version="6000.0.0f1", - _m_target_platform=13, - _enable_type_tree=True, +def scene_bundle() -> FakeEnvironment: + """A scene bundle: two serialized files sharing path ids, a resource file, a hierarchy (a root GameObject with a + child and MonoBehaviours, one of them without a script), a texture and the AssetBundle object, an unreadable + object.""" + env = FakeEnvironment() + scene = env.file(MAIN, externals=["archive:/" + SCRIPTS + "/" + SCRIPTS, "archive:/" + SHARED + "/" + SHARED]) + scene.add(1, "GameObject", go("root", [2, 7])) + scene.add(2, "Transform", tf(1, [4])) + scene.add(3, "GameObject", go("child", [4, 8])) + scene.add(4, "Transform", tf(3, father=2)) + scene.add(7, "MonoBehaviour", mono(1, pptr(42, 1), "", tex=pptr(-5, 2), big=[0] * 10), type_hash=HASH) + scene.add(8, "MonoBehaviour", mono(3, pptr(0), "n", other=pptr(9, 7)), type_hash=HASH) + scene.add(11, "Material", {"m_Name": "x"}, broken=True) + shared = env.file( + SHARED, + externals=[ + ("archive:/" + SCRIPTS + "/" + SCRIPTS, b"", 0), + ("Library/unity default resources", b"\x0f" * 16, 0), + ], ) - - -def ptr(fid, pid): - return {"m_FileID": fid, "m_PathID": pid} - - -def scene_bundle(): - """A bundle with two serialized files sharing path ids, a resource file, MonoBehaviours with and without a - script, an AssetBundle object and an unreadable object.""" - main = "CAB-aaaa0000000000000000000000000000" - mb = Obj( - 7, - 114, - "MonoBehaviour", - {"m_GameObject": ptr(0, 3), "m_Enabled": 1, "m_Script": ptr(1, 42), "m_Name": "", "big": [0] * 10}, - ) - null_mb = Obj( - 8, 114, "MonoBehaviour", {"m_GameObject": ptr(0, 3), "m_Enabled": 1, "m_Script": ptr(0, 0), "m_Name": "n"} - ) - tex = Obj(-5, 28, "Texture2D", {"m_Name": "tex", "m_Width": 4, "image data": b"\0" * 64}, size=500) - ab = Obj( - 1, - 142, - "AssetBundle", - { - "m_Name": "scene_x", - "m_PreloadTable": [], - "m_Container": [ - ("Assets/Scenes/X.unity", {"preloadIndex": 0, "preloadSize": 0, "asset": ptr(0, 0)}), - ("Assets/Tex/tex.png", {"preloadIndex": 0, "preloadSize": 1, "asset": ptr(0, -5)}), - ("Assets/Tex/tex.png", {"preloadIndex": 1, "preloadSize": 1, "asset": ptr(2, 3)}), - ], - "m_MainAsset": {}, - "m_RuntimeCompatibility": 1, - "m_AssetBundleName": "0123_abcd.bundle", - "m_Dependencies": ["dep_1.bundle"], - "m_IsStreamedSceneAssetBundle": True, - "m_SceneHashes": [("Assets/Scenes/X.unity", "ffee")], - }, - ) - tf = Obj(3, 4, "Transform", {"m_GameObject": ptr(0, 9), "m_LocalPosition": {}}, type_hash=None) - bad = Obj(11, 1, "GameObject", {"m_Component": [], "m_Name": "x"}, broken=True) - shared = sfile( - main + ".sharedAssets", - [tex, ab, Obj(3, 1, "GameObject", {"m_Component": [], "m_Layer": 0, "m_Name": "go", "m_IsActive": 1})], - [ext("archive:/" + SCRIPTS + "/" + SCRIPTS), ext("Library/unity default resources", b"\x0f" * 16)], - ) - scene = sfile(main, [tf, mb, null_mb, bad], [ext("archive:/" + SCRIPTS + "/" + SCRIPTS)]) - res = SimpleNamespace(Length=1234) - bundle = SimpleNamespace(files={scene.name: scene, main + ".resS": res, shared.name: shared}) - return SimpleNamespace(files={"x.bundle": bundle}), {"mb": mb, "tex": tex, "tf": tf} - - -def scripts_bundle(): - s = [ - Obj( - 42, - 115, - "MonoScript", - { - "m_Name": "Widget", - "m_ExecutionOrder": 0, - "m_ClassName": "Widget", - "m_Namespace": "Game.UI", - "m_AssemblyName": "Game.dll", - }, - ), - Obj( - 43, - 115, - "MonoScript", - { - "m_Name": "Top", - "m_ExecutionOrder": 0, - "m_ClassName": "Top", - "m_Namespace": "", - "m_AssemblyName": "Assembly-CSharp.dll", - }, - ), - ] - return SimpleNamespace(files={"s.bundle": SimpleNamespace(files={SCRIPTS: sfile(SCRIPTS, s)})}) + shared.add(-5, "Texture2D", {"m_Name": "tex", "m_Width": 4, "image data": b"\0" * 64}, byte_size=500) + shared.add(1, "AssetBundle", asset_bundle()) + shared.add(3, "GameObject", go("go", [])) + env.resources[MAIN + ".resS"] = b"\0" * 1234 + return env def test_census_of_a_scene_bundle(): - env, objs = scene_bundle() - doc = census.census_env(env) - main = "CAB-aaaa0000000000000000000000000000" - assert doc["schema"] == contract.CENSUS - assert [f["name"] for f in doc["files"]] == [main, main + ".sharedAssets"] - assert doc["resources"] == [{"name": main + ".resS", "size": 1234}] + doc = census.census_env(scene_bundle()) + assert doc["schema"] == contract.CENSUS == "nnnotes.census/2" + assert [f["name"] for f in doc["files"]] == [MAIN, SHARED] + assert doc["resources"] == [{"name": MAIN + ".resS", "size": 1234}] scene, shared = doc["files"] - assert [o["pathId"] for o in scene["objects"]] == [3, 7, 8, 11] + assert [o["pathId"] for o in scene["objects"]] == [1, 2, 3, 4, 7, 8, 11] assert [o["pathId"] for o in shared["objects"]] == [-5, 1, 3] # path id 3 in both files - assert scene["externals"] == [ - {"file": SCRIPTS, "path": "archive:/" + SCRIPTS + "/" + SCRIPTS, "guid": None, "type": 0} - ] + assert scene["externals"][0] == { + "file": SCRIPTS, + "path": "archive:/" + SCRIPTS + "/" + SCRIPTS, + "guid": None, + "type": None, + } assert shared["externals"][1] == { "file": "unity default resources", "path": "Library/unity default resources", "guid": "0f" * 16, "type": 0, } - assert (scene["unityVersion"], scene["platform"], scene["typetree"]) == ("6000.0.0f1", 13, True) - tf, mb, null_mb, bad = scene["objects"] - assert tf == {"pathId": 3, "classId": 4, "class": "Transform", "byteSize": 100, "typeHash": None} - assert mb == { + assert (scene["unityVersion"], scene["platform"], scene["typetree"]) == ("6000.3.12f1", 13, True) + objs = {o["pathId"]: o for o in scene["objects"]} + assert objs[7] == { "pathId": 7, "classId": 114, "class": "MonoBehaviour", - "byteSize": 100, + "byteSize": 0, "typeHash": "01" * 16, "name": "", "script": {"file": SCRIPTS, "pathId": 42}, + "refs": sorted([f"{MAIN}:1", f"{SHARED}:-5", f"{SCRIPTS}:42"]), # its GameObject, the texture, its script + "unit": f"{MAIN}:1", } - assert null_mb["script"] is None and null_mb["name"] == "n" - assert "name" not in bad and doc["errors"] == [ - {"object": main + ":11", "message": "ValueError: cannot read "} - ] + assert objs[8]["script"] is None and objs[8]["name"] == "n" + assert objs[8]["refs"] == [f"{MAIN}:3"] # a reference with an unknown file index is left out + assert "name" not in objs[11] and "refs" not in objs[11] + assert doc["errors"] == [{"object": MAIN + ":11", "message": "ValueError: cannot read "}] tex = shared["objects"][0] - assert tex["name"] == "tex" and tex["byteSize"] == 500 - # headers only: a MonoBehaviour is read up to m_Name, a texture up to m_Name, a transform not at all - assert objs["mb"].reads == [["m_GameObject", "m_Enabled", "m_Script", "m_Name"]] - assert objs["tex"].reads == [["m_Name"]] and objs["tf"].reads == [] + assert tex["name"] == "tex" and tex["byteSize"] == 500 and "refs" not in tex (ab,) = doc["assetBundles"] - assert ab["file"] == main + ".sharedAssets" and ab["pathId"] == 1 and ab["assetBundleName"] == "0123_abcd.bundle" + assert ab["file"] == SHARED and ab["pathId"] == 1 and ab["assetBundleName"] == "0123_abcd.bundle" assert ab["dependencies"] == ["dep_1.bundle"] and ab["isStreamedSceneAssetBundle"] is True assert ab["sceneHashes"] == [["Assets/Scenes/X.unity", "ffee"]] assert ab["container"] == [ {"path": "Assets/Scenes/X.unity", "file": None, "pathId": None, "preloadIndex": 0, "preloadSize": 0}, - { - "path": "Assets/Tex/tex.png", - "file": main + ".sharedAssets", - "pathId": -5, - "preloadIndex": 0, - "preloadSize": 1, - }, + {"path": "Assets/Tex/tex.png", "file": SHARED, "pathId": -5, "preloadIndex": 0, "preloadSize": 1}, { "path": "Assets/Tex/tex.png", "file": "unity default resources", @@ -194,21 +151,38 @@ def test_census_of_a_scene_bundle(): "preloadSize": 1, }, ] + assert shared["objects"][1]["refs"] == [f"{SHARED}:-5", "unity default resources:3"] assert census.container(doc) == ab["container"] - assert doc["classes"] == {"AssetBundle": 1, "GameObject": 2, "MonoBehaviour": 2, "Texture2D": 1, "Transform": 1} - assert doc["objects"] == 7 and doc["scripts"] == [] + assert doc["classes"] == { + "AssetBundle": 1, + "GameObject": 3, + "Material": 1, + "MonoBehaviour": 2, + "Texture2D": 1, + "Transform": 2, + } + assert doc["objects"] == 10 and doc["scripts"] == [] assert census.script_refs(doc) == [SCRIPTS + ":42"] - assert [(f, o["pathId"]) for f, o in census.objects(doc)][:2] == [(main, 3), (main, 7)] + assert [(f, o["pathId"]) for f, o in census.objects(doc)][:2] == [(MAIN, 1), (MAIN, 2)] assert census.facts(doc) == { "files": 2, - "objects": 7, + "objects": 10, "classes": doc["classes"], "scripts": 0, "scriptRefs": 1, "errors": 1, } - again, _ = scene_bundle() - assert contract.encode(census.census_env(again)) == contract.encode(doc) # deterministic + assert contract.encode(census.census_env(scene_bundle())) == contract.encode(doc) # deterministic + + +def test_the_objects_of_a_hierarchy_name_its_root(): + doc = census.census_env(scene_bundle()) + objs = {o["pathId"]: o for o in doc["files"][0]["objects"]} + assert "unit" not in objs[1] # the root GameObject is its own unit + assert {pid: objs[pid].get("unit") for pid in (2, 3, 4, 7, 8)} == dict.fromkeys((2, 3, 4, 7, 8), f"{MAIN}:1") + assert objs[2]["refs"] == [f"{MAIN}:1", f"{MAIN}:4"] # its GameObject and its child + assert objs[1]["refs"] == [f"{MAIN}:2", f"{MAIN}:7"] + assert "unit" not in objs[11] and "unit" not in doc["files"][1]["objects"][2] # outside any hierarchy def test_scripts_named_by_clip_bindings(): @@ -217,51 +191,43 @@ def test_scripts_named_by_clip_bindings(): def binding(tid, script): return {"path": 1, "attribute": 2, "script": script, "typeID": tid, "customType": 0, "isPPtrCurve": 0} - clip = Obj( + env = FakeEnvironment() + f = env.file(main, externals=["archive:/" + SCRIPTS + "/" + SCRIPTS]) + bindings = [ + binding(114, pptr(42, 1)), + binding(4, pptr(0)), + binding(114, pptr(42, 1)), + binding(114, pptr(6)), + binding(114, pptr(1, 5)), + ] + f.add( 5, - 74, "AnimationClip", - { - "m_Name": "fade", - "m_Legacy": False, - "m_MuscleClip": {}, - "m_ClipBindingConstant": { - "genericBindings": [ - binding(114, ptr(1, 42)), - binding(4, ptr(0, 0)), - binding(114, ptr(1, 42)), - binding(114, ptr(0, 6)), - binding(114, ptr(5, 1)), - ], - "pptrCurveMapping": [], - }, - "m_Events": [], - }, - ) - plain = Obj( - 6, - 74, - "AnimationClip", - {"m_Name": "move", "m_ClipBindingConstant": {"genericBindings": [binding(4, ptr(0, 0))]}, "m_Events": []}, + {"m_Name": "fade", "m_ClipBindingConstant": {"genericBindings": bindings, "pptrCurveMapping": []}}, ) - local = Obj(7, 115, "MonoScript", {"m_Name": "L", "m_ClassName": "L", "m_Namespace": "", "m_AssemblyName": "A.dll"}) - f = sfile(main, [clip, plain, local], [ext("archive:/" + SCRIPTS + "/" + SCRIPTS)]) - doc = census.census_env(SimpleNamespace(files={"c.bundle": SimpleNamespace(files={main: f})})) + f.add(6, "AnimationClip", {"m_Name": "move", "m_ClipBindingConstant": {"genericBindings": [binding(4, pptr(0))]}}) + f.add(7, "MonoScript", {"m_Name": "L", "m_ClassName": "L", "m_Namespace": "", "m_AssemblyName": "A.dll"}) + doc = census.census_env(env) rec, other, _ = doc["files"][0]["objects"] assert rec["bindingScripts"] == [ {"file": SCRIPTS, "pathId": 42}, {"file": main, "pathId": 6}, {"fileId": 5, "pathId": 1}, ] + assert rec["refs"] == [f"{SCRIPTS}:42", f"{main}:6"] assert rec["name"] == "fade" and "bindingScripts" not in other and "script" not in rec - assert clip.reads == [["m_Name", "m_Legacy", "m_MuscleClip", "m_ClipBindingConstant"]] # not m_Events assert census.script_refs(doc) == [SCRIPTS + ":42", main + ":6"] assert census.facts(doc)["scriptRefs"] == 2 - assert census.CensusStage.version == 2 def test_monoscripts_and_unknown_file_index(): - doc = census.census_env(scripts_bundle()) + env = FakeEnvironment() + f = env.file(SCRIPTS) + widget = {"m_Name": "Widget", "m_ClassName": "Widget", "m_Namespace": "Game.UI", "m_AssemblyName": "Game.dll"} + f.add(42, "MonoScript", widget) + top = {"m_Name": "Top", "m_ClassName": "Top", "m_Namespace": "", "m_AssemblyName": "Assembly-CSharp.dll"} + f.add(43, "MonoScript", top) + doc = census.census_env(env) assert doc["scripts"] == [ { "file": SCRIPTS, @@ -280,9 +246,29 @@ def test_monoscripts_and_unknown_file_index(): "name": "Top", }, ] - refs = census._Refs(sfile("CAB-1", [])) - assert refs.ref(ptr(3, 9)) == {"fileId": 3, "pathId": 9} and refs.ref(ptr(0, 0)) is None - assert refs.ref(None) is None and refs.ref(ptr(0, 5)) == {"file": "CAB-1", "pathId": 5} + refs = census._Refs("CAB-1", []) + assert refs.ref(pptr(9, 3)) == {"fileId": 3, "pathId": 9} and refs.ref(pptr(0)) is None + assert refs.ref(None) is None and refs.ref(pptr(5)) == {"file": "CAB-1", "pathId": 5} + + +def player_file() -> FakeEnvironment: + """A player's own file without type trees: a MonoBehaviour that references a built-in texture.""" + env = FakeEnvironment() + f = env.file("globalgamemanagers.assets", externals=["Library/unity default resources"], typetrees=False) + f.add(50, "MonoScript", {"m_Name": "S", "m_ClassName": "S", "m_Namespace": "", "m_AssemblyName": "A.dll"}) + f.add(2, "MonoBehaviour", mono(0, pptr(50), "volume", target=pptr(10, 1), volume=0.5), type_hash=HASH) + return env + + +def test_files_without_type_trees_read_with_the_shipped_ones(monkeypatch): + header = ["globalgamemanagers.assets:50"] + for trees, refs in (({HASH.hex(): ROWS}, [*header, "unity default resources:10"]), ({}, header)): + monkeypatch.setattr(scripttypes, "script_types", lambda version, trees=trees: trees) + rec = census.census_env(player_file(), shipped=True)["files"][0]["objects"][0] + assert rec["name"] == "volume" and rec["script"] == {"file": "globalgamemanagers.assets", "pathId": 50} + assert rec["refs"] == refs # without a shipped tree, the header's + monkeypatch.setattr(scripttypes, "script_types", lambda version: {HASH.hex(): ROWS}) + assert census.census_env(player_file())["files"][0]["objects"][0]["refs"] == header # not asked to ship def test_bundle_subjects(): @@ -297,7 +283,7 @@ def test_census_stage(tmp_path, monkeypatch): def load(path, name=None): loaded.append((str(path), name)) - return scene_bundle()[0] + return scene_bundle() monkeypatch.setattr(unity, "load", load) depot = Depot(tmp_path / "depot") @@ -307,15 +293,15 @@ def load(path, name=None): env = Env(depot, {"bundles": {"x": inp}}) assert stage.subjects(env) == ["x"] task = describe(stage, "x", None, env) - assert task.id == "unity.census:x" and task.atoms == census.CensusStage.ATOMS + assert task.id == "unity.census:x" and task.atoms == census.CensusStage.ATOMS and task.version == 3 assert task.atoms["reader"].startswith("unitypy-") assert task.cost.peak_bytes > size assert execute(task, depot, registry(stage)).status == "ran" doc = depot.result(task.key) (rec,) = doc["artifacts"] - assert rec["id"] == "unity.census:x#census" and rec["semantics"]["facts"]["objects"] == 7 + assert rec["id"] == "unity.census:x#census" and rec["semantics"]["facts"]["objects"] == 10 assert rec["provenance"]["inputs"] == [{"role": "bundle", "sha256": sha}] - assert contract.loads(depot.read(rec["content"]["sha256"])) == census.census_env(scene_bundle()[0]) + assert contract.loads(depot.read(rec["content"]["sha256"])) == census.census_env(scene_bundle()) assert loaded == [(str(bundle), None)] # a catalog bundle: its files keep their names assert execute(task, depot, registry(stage)).status == "hit" and len(loaded) == 1 with pytest.raises(ValueError, match="expected 'bundle'"): @@ -323,8 +309,10 @@ def load(path, name=None): for waiting in (None, Pending("fetch:x_1.bundle")): with pytest.raises(Pending, match="fetch:x"): describe(stage, "x", None, Env(depot, {"bundles": {"x": waiting}})) - # a serialized file of the APK outside a bundle is loaded under the name references into it use + # a serialized file of the APK outside a bundle is loaded under the name references into it use, and read with the + # shipped type trees, which its key names own = "apk/Resources/unity default resources" task = describe(stage, own, None, Env(depot, {"bundles": {own: inp}})) + assert task.atoms == {**census.CensusStage.ATOMS, "typetrees": scripttypes.types_id()} assert execute(task, depot, registry(stage)).status == "ran" assert loaded[-1] == (str(bundle), "unity default resources") diff --git a/tests/pipeline/test_objexport.py b/tests/pipeline/test_objexport.py index 6689c94..f2a1879 100644 --- a/tests/pipeline/test_objexport.py +++ b/tests/pipeline/test_objexport.py @@ -560,10 +560,10 @@ def bundle_movie() -> FakeEnvironment: return env -# a player's own file: no type trees; MonoBehaviours whose classes have a shipped type tree (one of another type -# hash), and none +# a player's own file: no type trees; MonoBehaviours whose type hashes have a shipped type tree (one that does not +# fit the object), and none PLAYER = "globalgamemanagers.assets" -PLAYER_HASH, OTHER_HASH = "a1" * 16, "b2" * 16 +PLAYER_HASH, OTHER_HASH, IMAGE_HASH = "a1" * 16, "b2" * 16, "c3" * 16 SETTINGS, IMAGE, CHANGED = "Fwk|Fwk.Sound|SoundVolumeSettings", "UnityEngine.UI|UnityEngine.UI|Image", "Game|Game|Moved" PLAYER_SCRIPTS = {f"{PLAYER}:50": SETTINGS, f"{PLAYER}:51": IMAGE, f"{PLAYER}:52": CHANGED} SETTINGS_ROWS = [ @@ -581,10 +581,7 @@ def bundle_movie() -> FakeEnvironment: [2, "SInt64", "m_PathID", 0], [1, "float", "volume", 0], ] -PLAYER_TYPETREES = { - SETTINGS: {"typeHash": PLAYER_HASH, "nodes": SETTINGS_ROWS}, - CHANGED: {"typeHash": OTHER_HASH, "nodes": SETTINGS_ROWS}, -} +PLAYER_TYPETREES = {PLAYER_HASH: SETTINGS_ROWS, OTHER_HASH: SETTINGS_ROWS[:-1]} # the second lacks a field def bundle_player() -> FakeEnvironment: @@ -595,9 +592,10 @@ def bundle_player() -> FakeEnvironment: f.add(1, "PlayerSettings", settings, type_hash=bytes.fromhex("936fd1913a4f0b5da2ce7cadb0ce9666"), short_read=True) f.add(6, "PlayerSettings", settings, type_hash=h, short_read=True) f.add(2, "MonoBehaviour", _mono(0, pptr(50), "volume", target=pptr(10, 1), volume=0.5), type_hash=h) - f.add(3, "MonoBehaviour", _mono(20, pptr(51), "", m_Sprite=pptr(10, 1)), type_hash=h) - f.add(4, "MonoBehaviour", _mono(0, pptr(52), "moved", target=pptr(0), volume=1.0), type_hash=h) - f.add(5, "MonoBehaviour", _mono(0, pptr(51), "image", m_Sprite=pptr(10, 1)), type_hash=h) + image, other = bytes.fromhex(IMAGE_HASH), bytes.fromhex(OTHER_HASH) + f.add(3, "MonoBehaviour", _mono(20, pptr(51), "", m_Sprite=pptr(10, 1)), type_hash=image) + f.add(4, "MonoBehaviour", _mono(0, pptr(52), "moved", target=pptr(0), volume=1.0), type_hash=other) + f.add(5, "MonoBehaviour", _mono(0, pptr(51), "image", m_Sprite=pptr(10, 1)), type_hash=image) f.add(20, "GameObject", _go("Canvas", [21, 3])) f.add(21, "Transform", _transform(20)) for pid, entry in ((50, SETTINGS), (51, IMAGE), (52, CHANGED)): @@ -1319,7 +1317,7 @@ def census_doc(view: UnityBundle) -> dict: if o.file != f: continue classes[o.class_name] = classes.get(o.class_name, 0) + 1 - rec = {"pathId": o.path_id, "classId": o.class_id, "class": o.class_name} + rec = {"pathId": o.path_id, "classId": o.class_id, "class": o.class_name, "typeHash": o.type_hash} tt = view.reader(o).read_typetree(check_read=False) # a header read, as the census' ext = view.externals(f) @@ -1433,8 +1431,11 @@ def script(pid, cls): "volume": 0.5, } assert items[p(2)]["refs"] == [p(50), "unity default resources:10"] - # a class without a shipped type tree, or with one of another type hash: its header, and the reason - for pid, why in ((5, f"has none for {IMAGE}"), (4, f"type hash {PLAYER_HASH} is not the type tree's")): + # a type hash without a shipped type tree, or with one that does not fit the object: its header, and the reason + for pid, why in ( + (5, f"none is given for {IMAGE} (type hash {IMAGE_HASH})"), + (4, f"type tree of {CHANGED} does not fit"), + ): assert items[p(pid)]["status"] == "generic" assert items[p(pid)]["reason"]["code"] == "unsupported.typetree" and why in items[p(pid)]["reason"]["message"] assert obj(depot, doc, f"{p(5)}#json") == { @@ -1460,14 +1461,16 @@ def script(pid, cls): assert items[p(3)]["status"] == "contained" -def test_the_context_holds_the_shipped_type_trees_of_its_scripts(tmp_path): +def test_the_context_holds_the_shipped_type_trees_of_its_scripts(tmp_path, monkeypatch): from nnnotes.unity import scripttypes shipped = scripttypes.load_typetrees("6000.3.12f1")["classes"]["DOTween.dll|DG.Tweening.Core|DOTweenSettings"] - entry = "DOTween|DG.Tweening.Core|DOTweenSettings" # as the script table names it - env = planning_env(Depot(tmp_path / "s"), names=("player",), scripts={**PLAYER_SCRIPTS, f"{PLAYER}:50": entry}) + assert scripttypes.script_types("6000.3.12f1")[shipped["typeHash"]] == shipped["nodes"] + trees = {**PLAYER_TYPETREES, shipped["typeHash"]: shipped["nodes"]} + monkeypatch.setattr(scripttypes, "script_types", lambda version: trees) + env = planning_env(Depot(tmp_path / "s"), names=("player",), scripts=PLAYER_SCRIPTS) t = describe(ExportStage(FAKE_READER), "player", None, env) - assert t.context["typetrees"] == {entry: shipped} + assert t.context["typetrees"] == PLAYER_TYPETREES # the trees of its MonoBehaviours' type hashes assert ( "typetrees" not in describe(ExportStage(FAKE_READER), "all", None, planning_env(Depot(tmp_path / "t"))).context )