From aa886f29674dcbec30d78607d0faa4239f31cf2b Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Thu, 15 Oct 2026 10:29:34 +0800 Subject: [PATCH] feat(link)!: find objects by their Resources.Load path The census lists the container of a ResourceManager (`resourcePaths`: the paths Resources.Load finds objects at, in stored order); unity.census is at version 4. The address table (nnnotes.addresses/2, link.addresses version 2) adds `resources`, each path with the object ids it names, and gives those objects stable addresses as container paths of the built-in file holding the ResourceManager. AddressIndex.resource looks a path up ignoring case. A request's roots and a product stage take resource paths besides catalog keys and object ids (`resources`), so a product can start from the objects a runtime loads by name, such as the materials of a model's masks. BREAKING CHANGE: address tables are nnnotes.addresses/2 with a required `resources` table, and census documents carry `resourcePaths`. Co-Authored-By: Claude Opus 5.5 --- docs/contracts.md | 2 +- docs/products.md | 8 +++-- docs/schemas.md | 2 +- docs/stages.md | 16 +++++---- src/nnnotes/pipeline/assemble.py | 31 +++++++++++------ src/nnnotes/pipeline/census.py | 18 ++++++++-- src/nnnotes/pipeline/link.py | 25 +++++++++----- src/nnnotes/pipeline/units.py | 21 +++++++----- src/nnnotes/pipeline/views.py | 35 ++++++++++++++----- src/nnnotes/schemas/addresses.schema.json | 18 ++++++++-- src/nnnotes/schemas/census.schema.json | 32 +++++++++++++++++- tests/cli/test_assets.py | 7 ++-- tests/helpers/fakeunity.py | 1 + tests/helpers/units.py | 1 + tests/pipeline/test_assemble.py | 41 ++++++++++++++++++++--- tests/pipeline/test_census.py | 22 +++++++++++- tests/pipeline/test_link.py | 41 +++++++++++++++++++++-- tests/pipeline/test_modelstages.py | 1 + tests/pipeline/test_views.py | 14 +++++--- 19 files changed, 266 insertions(+), 70 deletions(-) diff --git a/docs/contracts.md b/docs/contracts.md index dcd8fe2..7c63b3c 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -229,7 +229,7 @@ Documents written by particular stages (artifacts of no object: their ids are `< | `nnnotes.catalog-index/1` | `catalog.index` | [catalog-index.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/catalog-index.schema.json) | | `nnnotes.census/2` | `unity.census` | [census.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/census.schema.json) | | `nnnotes.scripts/1` | `link.scripts` | [scripts.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/scripts.schema.json) | -| `nnnotes.addresses/1` | `link.addresses` | [addresses.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/addresses.schema.json) | +| `nnnotes.addresses/2` | `link.addresses` | [addresses.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/addresses.schema.json) | | `nnnotes.view/1` | `view.` | [view.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/view.schema.json) | | `nnnotes.objects/1` | `pipeline.assemble` (a product's object table) | [objects.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/objects.schema.json) | diff --git a/docs/products.md b/docs/products.md index 8606ccd..002ce50 100644 --- a/docs/products.md +++ b/docs/products.md @@ -12,10 +12,12 @@ objects: - the objects of the catalog keys the stage gives for the subject (`keys`), through the address table of `link.addresses`; +- the objects of the Resources.Load paths it gives (`resources`): paths of the ResourceManager in the APK's built-in + files, such as the materials a runtime loads by name, through the same table and compared ignoring case; - the object ids the stage names besides (`root_objects`). -Both are read from the planning facts alone, so a command can ask for what its products need before anything runs: -`assemble.request(stages, products, env)` gives the fact `roots` (`{"keys", "objects"}`), and the request of +All are read from the planning facts alone, so a command can ask for what its products need before anything runs: +`assemble.request(stages, products, env)` gives the fact `roots` (`{"keys", "resources", "objects"}`), and the request of `unity.export` then holds the units of those objects and every unit reachable from them through the census `refs` ([units](stages.md#units)), and nothing else of their bundles. @@ -27,7 +29,7 @@ object that a reached object references and no census holds is listed as `missin | Part of the task | | |---|---| | inputs | `objects`: the object table of what it reaches (`nnnotes.objects/1`); the stage's own inputs (master tables, data its parameters name) | -| depends | `link.addresses` (when it has keys), the `unity.export` and `sprite.crop` tasks whose results it read | +| depends | `link.addresses` (when it has keys or resource paths), the `unity.export` and `sprite.crop` tasks whose results it read | | artifacts | `#`: the product's own files; `#objects`: the object table of the objects its files use (the whole table unless the stage narrows it) | | item | one for the subject: `exported`, or `unsupported` with a reason code | diff --git a/docs/schemas.md b/docs/schemas.md index fd0a6b7..6ffa241 100644 --- a/docs/schemas.md +++ b/docs/schemas.md @@ -14,7 +14,7 @@ schemas.load("plan") # the schema as a dict | Schema | Format | Document | |---|---|---| -| `addresses` | `nnnotes.addresses/1` | the address table of a catalog version and the censuses of its bundles | +| `addresses` | `nnnotes.addresses/2` | the address table of a catalog version and the censuses of its bundles | | `artifact` | `nnnotes.artifact/1` | the content, provenance and semantics of one output file | | `catalog-diff` | `nnnotes.catalog-diff/1` | the difference between two catalog versions (`catalog diff`) | | `catalog-index` | `nnnotes.catalog-index/1` | the index of a catalog pair: remote and APK catalogs, every location in catalog order | diff --git a/docs/stages.md b/docs/stages.md index bc7eb16..9e8e106 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -12,14 +12,15 @@ The asset export is built from three layers: ## Stages -Every stage is at version 1, except `unity.census` (version 3: it lists what every object references and the -hierarchy holding it), `unity.export` (version 2: one task per unit), `cri.movie` (version 2: every stream kind, +Every stage is at version 1, except `unity.census` (version 4: it lists what every object references, the +hierarchy holding it and the Resources.Load paths of a ResourceManager), `link.addresses` (version 2: the +Resources.Load paths), `unity.export` (version 2: one task per unit), `cri.movie` (version 2: every stream kind, codec and channel of a USM) and `catalog.index` (version 2: gzip input and Japanese remote asset placeholders). | Stage | Subject | Inputs | Outputs | |---|---|---|---| | `catalog.index` | a catalog (remote + APK) | the catalog files | every location (with the bundle hash, crc and size), bundles and raw remote files | -| `unity.census` | a bundle, or a built-in file of the APK | the bundle | serialized files, externals, objects (file, path id, class, size, type hash, name, script, the scripts an AnimationClip's bindings name, the objects it references, the hierarchy holding it), container, scene hashes, dependencies, script fields | +| `unity.census` | a bundle, or a built-in file of the APK | the bundle | serialized files, externals, objects (file, path id, class, size, type hash, name, script, the scripts an AnimationClip's bindings name, the objects it references, the hierarchy holding it), container, scene hashes, dependencies, script fields, the ResourceManager's paths (Resources.Load) | | `link.scripts` | global | the censuses holding MonoScripts | serialized file and path id -> assembly, namespace, class | | `unity.export` | a unit of a bundle or of a built-in file of the APK (below), by the stable address of its root object | the bundle; context `unit` (the root's object id), `bundle`, the script entries its objects use and, for files without type trees, the type trees nnnotes ships for their type hashes | its objects' artifacts and one item per object | | `sprite.crop` | a SpriteAtlas | the atlas' JSON, the images of its textures, the metas of the sprites of other bundles it packs | the sprites' images | @@ -27,7 +28,7 @@ codec and channel of a USM) and `catalog.index` (version 2: gzip input and Japan | `cri.movie` | USM content | `usm` (a raw file of the catalog, or the `usm` artifact of a movie held in a bundle), `boot` | every stream as stored (video, alpha, audio, subtitles), `movie.mkv` (or `movie.webm`) | | `live2d.model` | a Live2D model key (`Character/Live2D///model/`) whose bundle is selected | `bundle:`: the bundles of the key's dependency closure in the order the extractor loads them (the order is part of the key); context `internalId` (the key's location); atom `live2d.extract_model` (the extractor of `nnnotes export live2d`, run unchanged) | `live2d.model:#`: every file `nnnotes export live2d` writes for the model, `` relative to the model's directory; placed under `/` | | `spine.skeleton` | a SkeletonDataAsset of a selected bundle, by its stable address `:` (in a bundle of several serialized files the file's tag follows the bundle name, as in `link.addresses`) | `bundle:`: the bundles holding the serialized files the asset's file reaches through its externals; context `object` (the asset's object id); atom `spot.write_skeleton` (the Spine writer of `nnnotes export spot`) | `spine.skeleton:#`: the skeleton (`.json` or `.skel`), its atlases (`.atlas`) and their page images, the bytes `nnnotes export spot` writes into `spine/`; one item for the asset | -| `link.addresses` | global | catalog index, censuses (the built-in files' too) | key -> objects, serialized file -> bundle or built-in file, bundle -> keys | +| `link.addresses` | global | catalog index, censuses (the built-in files' too) | key -> objects, Resources.Load path -> objects, serialized file -> bundle or built-in file, bundle -> keys | | `view.` | a catalog (`main`) | `master:` per master table it reads, `rules` (the view's rule and the resolvers it uses), `addresses` (the part of the address table it read: the keys looked up and the prefixes listed); no parameters, atoms or context | `view.:#view` (facts: rows, entries, gaps, unreferenced) | | `view.voices` | a catalog (`main`) | `master:
` per master table its rules read, `rules`, `addresses`, `sheets` (each cue sheet the `cri.audio` results are known by that a voice names or the rules' prefixes list: its task and ACB content id), `acb:` (the ACB of each decoded one), `episodes` (per story episode and part, the `unity.export` artifact holding the object its key names), `adv:` (those contents); context `audio` (the keys of those `cri.audio` results); runs after `unity.export` and `cri.audio` | `view.voices:#view` (facts: rows, entries, gaps, unreferenced streams) | @@ -110,9 +111,10 @@ several serialized files the file's tag follows the bundle name, as in `link.add bundle's content changes, so a unit keeps its task across catalog versions. A run exports the units holding an object of the selected bundles, the units of the objects the fact "roots" names -(a product's roots: [products](products.md)) and every unit reachable from those through what their objects reference -(the census `refs`, across the censuses of the run); with `--only-class` only the units whose root object is of those -classes. The selection is not part of any key: a unit's task is the same whichever request reaches it. +(catalog keys, Resources.Load paths and object ids: a product's roots, [products](products.md)) and every unit reachable +from those through what their objects reference (the census `refs`, across the censuses of the run); with `--only-class` +only the units whose root object is of those classes. The selection is not part of any key: a unit's task is the same +whichever request reaches it. A task's key includes the atoms `reader` and `sniff`, the converter `generic.json`, the converters (with their atoms) of the classes of the unit's objects, its bundle, its root and the script entries and shipped type trees its diff --git a/src/nnnotes/pipeline/assemble.py b/src/nnnotes/pipeline/assemble.py index 1c2e9a1..4fc3020 100644 --- a/src/nnnotes/pipeline/assemble.py +++ b/src/nnnotes/pipeline/assemble.py @@ -1,8 +1,9 @@ """Product stages: products assembled from the unity.export results of the objects they reach, never from a bundle. A product (a Live2D model, a story, a live, ...) is a stage whose task per subject starts from root objects: the -objects of the catalog keys `keys(subject)` gives and the objects `root_objects(subject)` names (both read from the -planning facts alone, so a command can ask for the units its products need before anything runs: `request`). The +objects of the catalog keys `keys(subject)` gives, of the Resources.Load paths `resources(subject)` gives and the +objects `root_objects(subject)` names (all read from the planning facts alone, so a command can ask for the units its +products need before anything runs: `request`). The objects it reaches are those of the units holding its roots and every unit an item of a reached unit references (contract.item `refs`), through the unity.export results; a sprite whose atlas is in another bundle has its image from sprite.crop. A reached unit that the request did not export is an error of the request; a referenced object @@ -34,15 +35,16 @@ def request(stages: dict, products: dict, env) -> dict: - """The fact "roots" (pipeline.units) of a request for `products` {stage name: subjects}: the catalog keys and the - root objects of every subject, sorted.""" - keys, objects = set(), set() + """The fact "roots" (pipeline.units) of a request for `products` {stage name: subjects}: the catalog keys, the + resource paths and the root objects of every subject, sorted.""" + keys, resources, objects = set(), set(), set() for name, subjects in sorted(products.items()): stage = stages[name] for s in subjects: keys.update(stage.keys(s, env)) + resources.update(stage.resources(s, env)) objects.update(stage.root_objects(s, env)) - return {"keys": sorted(keys), "objects": sorted(objects)} + return {"keys": sorted(keys), "resources": sorted(resources), "objects": sorted(objects)} def _entry(item: dict, arts: dict) -> dict: @@ -211,7 +213,8 @@ class Assembly: class ProductStage(Stage): """Base of the product stages (module documentation). Subclasses set `name`, `version` and `KIND`, and define - `keys` and/or `root_objects` and `assemble`; `subjects` defaults to the subjects of the fact "products".""" + `keys`, `resources` and/or `root_objects` and `assemble`; `subjects` defaults to the subjects of the fact + "products".""" after = (ADDRESSES, EXPORT, CROP) KIND = "product" @@ -235,8 +238,13 @@ def keys(self, subject: str, env) -> list[str]: """The catalog keys whose objects are roots of `subject` (from the facts alone).""" return [] + def resources(self, subject: str, env) -> list[str]: + """The Resources.Load paths whose objects are roots of `subject` (from the facts alone).""" + return [] + def root_objects(self, subject: str, env) -> list[str]: - """Object ids that are roots of `subject` besides those of its keys (from the facts alone).""" + """Object ids that are roots of `subject` besides those of its keys and resource paths (from the facts + alone).""" return [] def more_inputs(self, subject: str, env) -> list[Input]: @@ -253,7 +261,9 @@ def manifest(self, subject: str) -> str: # ---------------------------------------------------------------- planning def reach(self, subject: str, env) -> Reach: """What `subject` reaches (kept while the results it read are the same).""" - roots = units.root_objects(env, self.keys(subject, env), self.root_objects(subject, env)) + roots = units.root_objects( + env, self.keys(subject, env), self.root_objects(subject, env), self.resources(subject, env) + ) sig = (tuple(roots), id(units.table(env))) memo = self._memo.get(subject) if memo is not None and memo[0] == sig and all(env.key(t) == k for t, k in memo[1]): @@ -268,7 +278,8 @@ def inputs(self, subject: str, env) -> list[Input]: return [table, *self.more_inputs(subject, env)] def depends(self, subject: str, env) -> list[str]: - own = [contract.task_id(ADDRESSES, s) for s in sorted(env.fact("catalogs"))] if self.keys(subject, env) else [] + named = self.keys(subject, env) or self.resources(subject, env) + own = [contract.task_id(ADDRESSES, s) for s in sorted(env.fact("catalogs"))] if named else [] return own + self.reach(subject, env).read def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: diff --git a/src/nnnotes/pipeline/census.py b/src/nnnotes/pipeline/census.py index c7b43c3..a660cdc 100644 --- a/src/nnnotes/pipeline/census.py +++ b/src/nnnotes/pipeline/census.py @@ -17,6 +17,9 @@ assetBundles the AssetBundle objects: {file, pathId, name, assetBundleName, dependencies, isStreamedSceneAssetBundle, sceneHashes [[scene, hash]], container [{path, file, pathId, preloadIndex, preloadSize}] (stored order)} + resourcePaths the container of the ResourceManager objects (the paths Resources.Load finds objects at, in the + APK's global managers): [{path, file, pathId}] in stored order; a path names one object per + class it was saved as scripts the MonoScript objects: {file, pathId, assembly, namespace, class, name} classes objects per class; `objects` the total errors [{object, message}] of the objects whose typetree could not be read (their record has no name, @@ -140,6 +143,14 @@ def _asset_bundle(d: dict, refs: _Refs, oid: tuple[str, int]) -> dict: } +def _resource_paths(d: dict, refs: _Refs) -> list: + out = [] + for path, p in _pairs(d.get("m_Container")): + target = refs.ref(p) or {} + out.append({"path": path, "file": target.get("file"), "pathId": target.get("pathId")}) + return out + + def _binding_scripts(d: dict, refs: _Refs) -> list: """The MonoScripts an AnimationClip's generic bindings name, {file, pathId} (or {fileId, pathId}), sorted.""" out = {} @@ -170,7 +181,7 @@ def census_env(env, shipped: bool = False) -> dict: by_file: dict[str, list] = {} for i in ctx.infos: by_file.setdefault(i.file, []).append(i) - files, bundles, scripts, errors, classes = [], [], [], [], {} + files, bundles, paths, scripts, errors, classes = [], [], [], [], [], {} for name in sorted(by_file): sf = view.serialized_file(name) refs = _Refs(name, ctx.ext_files[name]) @@ -204,6 +215,8 @@ def census_env(env, shipped: bool = False) -> dict: rec["refs"] = sorted(contract.object_id(*k) for k in found) if i.class_name == "AssetBundle": bundles.append(_asset_bundle(d, refs, key)) + elif i.class_name == "ResourceManager": + paths.extend(_resource_paths(d, refs)) elif i.class_name == "MonoScript": scripts.append( { @@ -241,6 +254,7 @@ def census_env(env, shipped: bool = False) -> dict: "files": files, "resources": resources, "assetBundles": sorted(bundles, key=lambda b: (b["file"], b["pathId"])), + "resourcePaths": paths, "scripts": sorted(scripts, key=lambda s: (s["file"], s["pathId"])), "classes": dict(sorted(classes.items())), "objects": sum(classes.values()), @@ -332,7 +346,7 @@ class CensusStage(Stage): MonoBehaviours with.""" name = "unity.census" - version = 3 + version = 4 ATOMS = {"reader": impl_id(("UnityPy",), 1)} def subjects(self, env) -> list[str]: diff --git a/src/nnnotes/pipeline/link.py b/src/nnnotes/pipeline/link.py index bf41aa9..968454c 100644 --- a/src/nnnotes/pipeline/link.py +++ b/src/nnnotes/pipeline/link.py @@ -5,7 +5,7 @@ stays empty). A MonoBehaviour's class is the entry of its m_Script; `script_context` picks the entries a task consumes. -link.addresses (nnnotes.addresses/1), one task per catalog pair: the catalog index and the censuses of its bundles +link.addresses (nnnotes.addresses/2), one task per catalog pair: the catalog index and the censuses of its bundles (by stable bundle name) and of the APK's built-in files (subjects `apk/...`, pipeline.builtins) -> bundles {stable name: {name, files, assetBundleName, keys}}: the file name in this catalog version, its serialized files, the AssetBundle name the census found and the keys located in it; a built-in file @@ -14,7 +14,10 @@ keys {primary key: [{location, type, internalId, bundle | raw, objects}]}: an asset location's bundle is the dependency whose container holds its internal id (compared ignoring case); `objects` the object ids the container path names (null when a dependency bundle has no census) - objects {stable address: {object, class, name, byteSize, typeHash}} of the objects a container path names + resources {resource path: [object ids]}: the paths of the ResourceManager container of the built-in files (what + Resources.Load finds, as stored), the objects each names in stored order + objects {stable address: {object, class, name, byteSize, typeHash}} of the objects a container path or a + resource path names problems files named by two bundles, bundles whose census AssetBundle name is not the catalog's, keys whose internal id no container holds @@ -24,8 +27,9 @@ "/[]" for another object under it with a name no other object under it has, else ":". In a bundle of several serialized files (their path ids may coincide) the path id form names the file after the bundle: the part of the file name after its CAB- prefix ("scene_x.sharedAssets:12"), or -"|" ("apk/data.unity3d|level0:3"). -Objects that no container path names are not listed; `address_of` gives any object's address. +"|" ("apk/data.unity3d|level0:3"). A resource path counts as a container path of the built-in file holding the +ResourceManager ("apk/data.unity3d/live2d/cubism/materials/mask"). +Objects that no container or resource path names are not listed; `address_of` gives any object's address. """ @@ -40,7 +44,7 @@ from .stages import Output, Pending, Stage SCRIPTS = "nnnotes.scripts/1" -ADDRESSES = "nnnotes.addresses/1" +ADDRESSES = "nnnotes.addresses/2" _CAB = re.compile(r"^CAB-[0-9a-f]+") @@ -125,7 +129,7 @@ def addresses(index: dict, censuses) -> dict: the index does not list are ignored.""" files_of = catalogdb.file_refs(index) listed = {b["stable"]: b for b in index["bundles"]} - bundles, owners, info, groups, tags_of = {}, {}, {}, {}, {} + bundles, owners, info, groups, tags_of, resources = {}, {}, {}, {}, {}, {} problems = {"files": [], "bundleNames": [], "keys": []} pairs = sorted(censuses.items()) if isinstance(censuses, dict) else censuses for stable, doc in pairs: @@ -141,8 +145,11 @@ def addresses(index: dict, censuses) -> dict: if b.get("bundleName") and found and b["bundleName"] + ".bundle" not in found: problems["bundleNames"].append({"bundle": stable, "catalog": b["bundleName"] + ".bundle", "census": found}) g: dict[str, list[dict]] = {} - for e in census.container(doc): + for e in census.container(doc) + doc["resourcePaths"]: g.setdefault(e["path"].casefold(), []).append(e) + for e in doc["resourcePaths"]: + if e["file"] is not None: + resources.setdefault(e["path"], []).append(contract.object_id(e["file"], e["pathId"])) wanted = {(e["file"], e["pathId"]) for es in g.values() for e in es if e["file"] is not None} for file, o in census.objects(doc): if (file, o["pathId"]) in wanted: @@ -208,6 +215,7 @@ def addresses(index: dict, censuses) -> dict: "bundles": dict(sorted(bundles.items())), "files": dict(sorted(cab.items())), "keys": dict(sorted(keys.items())), + "resources": dict(sorted(resources.items())), "objects": dict(sorted(objects.items())), "problems": problems, } @@ -292,7 +300,7 @@ class AddressesStage(Stage): skips failed ones. Artifact "link.addresses:#addresses".""" name = "link.addresses" - version = 1 + version = 2 after = ("catalog.index", "unity.census") def subjects(self, env) -> list[str]: @@ -323,6 +331,7 @@ def run(self, task, depot) -> Output: facts = { "bundles": len(doc["bundles"]), "keys": len(doc["keys"]), + "resources": len(doc["resources"]), "objects": len(doc["objects"]), "problems": {k: len(v) for k, v in doc["problems"].items()}, } diff --git a/src/nnnotes/pipeline/units.py b/src/nnnotes/pipeline/units.py index a82dff1..788caf7 100644 --- a/src/nnnotes/pipeline/units.py +++ b/src/nnnotes/pipeline/units.py @@ -11,9 +11,10 @@ every class), only the units whose root object is of those classes. A bundle whose census failed has no units; a reference into a file no census holds is left out. -The fact "roots" (absent: none) is {"keys": catalog keys, "objects": object ids}: the objects of each key (all those -the address table of link.addresses lists for it; a key without objects adds none) and the objects named. The -products of a request start from them (pipeline.assemble). +The fact "roots" (absent: none) is {"keys": catalog keys, "resources": resource paths, "objects": object ids}, each +list optional: the objects of each key and of each resource path (all those the address tables of link.addresses +list for it: a resource path is a Resources.Load path of the APK's built-in files, compared ignoring case; one +without objects adds none) and the objects named. The products of a request start from them (pipeline.assemble). """ from __future__ import annotations @@ -186,22 +187,24 @@ def address_index(env, subject: str) -> views.AddressIndex: return index -def root_objects(env, keys=(), objects=()) -> list[str]: - """The object ids of the catalog `keys` (through the address tables of the fact "catalogs") and the `objects`, - sorted.""" +def root_objects(env, keys=(), objects=(), resources=()) -> list[str]: + """The object ids of the catalog `keys` and the resource paths `resources` (through the address tables of the + fact "catalogs") and the `objects`, sorted.""" out = set(objects) - if keys: + if keys or resources: for subject in sorted(env.fact("catalogs")): index = address_index(env, subject) for k in keys: out.update(o.id for o in index.objects(k) or ()) + for p in resources: + out.update(o.id for o in index.resource(p) or ()) return sorted(out) def table(env) -> Units: """The Units of the request of a planning environment: the censuses of the fact "bundles" (Pending while one has not run; a failed one is skipped), the facts "selected", "roots" (Pending while link.addresses has not run, - when it names keys) and "classes". Kept for the planning state it was made in.""" + when it names keys or resource paths) and "classes". Kept for the planning state it was made in.""" bundles = sorted(env.fact("bundles")) keys = [] for b in bundles: @@ -213,7 +216,7 @@ def table(env) -> Units: raise classes = env.facts.get("classes") request = env.facts.get("roots") or {} - roots = root_objects(env, request.get("keys", ()), request.get("objects", ())) + roots = root_objects(env, request.get("keys", ()), request.get("objects", ()), request.get("resources", ())) sig = ( id(env.depot), tuple(keys), diff --git a/src/nnnotes/pipeline/views.py b/src/nnnotes/pipeline/views.py index 18c7012..a8c3683 100644 --- a/src/nnnotes/pipeline/views.py +++ b/src/nnnotes/pipeline/views.py @@ -121,11 +121,17 @@ def objects(self, key: str) -> list[Obj] | None: class AddressIndex: - """Addresses from {catalog key: objects in stored order} (a key mapped to None or [] has no census object).""" + """Addresses from {catalog key: objects in stored order} (a key mapped to None or [] has no census object), and + the objects of the resource paths of the APK's built-in files ({path: objects in stored order}, `resource`).""" - def __init__(self, entries: Mapping[str, Iterable[Obj] | None]): + def __init__( + self, entries: Mapping[str, Iterable[Obj] | None], resources: Mapping[str, Iterable[Obj]] | None = None + ): self._objs = {k: list(v or ()) for k, v in entries.items()} self._keys = sorted(self._objs) + self._resources: dict[str, list[Obj]] = {} + for path, objs in (resources or {}).items(): + self._resources.setdefault(path.casefold(), []).extend(objs) def keys(self, prefix: str = "") -> list[str]: i = bisect.bisect_left(self._keys, prefix) @@ -140,6 +146,12 @@ def objects(self, key: str) -> list[Obj] | None: objs = self._objs.get(key) return None if objs is None else list(objs) + def resource(self, path: str) -> list[Obj] | None: + """The objects a Resources.Load `path` finds (compared ignoring case, as Unity does), in stored order; None + for a path no ResourceManager lists.""" + objs = self._resources.get(path.casefold()) + return None if objs is None else list(objs) + @classmethod def build(cls, locations: Mapping[str, str], containers: Mapping[str, Iterable[Obj]]) -> AddressIndex: """The index of a catalog and its censuses: `locations` {key: internal id} (the asset path a key loads), @@ -152,22 +164,27 @@ def build(cls, locations: Mapping[str, str], containers: Mapping[str, Iterable[O @classmethod def from_addresses(cls, doc: dict) -> AddressIndex: - """The index of an address table (nnnotes.addresses/1, the link.addresses stage): a key's objects are those - of all its locations (in the table's order, the order of the bundle's container), with class, name and - stable address from the table's `objects`; a location whose bundle has no census yet contributes none.""" + """The index of an address table (nnnotes.addresses/2, the link.addresses stage): a key's objects are those + of all its locations (in the table's order, the order of the bundle's container), a resource path's those + its table lists, with class, name and stable address from the table's `objects`; a location whose bundle has + no census yet contributes none.""" where: dict[str, str] = {} for addr, info in doc["objects"].items(): where.setdefault(info["object"], addr) + + def obj(oid: str) -> Obj: + addr = where.get(oid) + info = doc["objects"][addr] if addr is not None else {} + return Obj(oid, info.get("class") or "", info.get("name") or "", addr) + entries = {} for key, locs in doc["keys"].items(): objs = {} for loc in locs: for oid in loc.get("objects") or (): - addr = where.get(oid) - info = doc["objects"][addr] if addr is not None else {} - objs[oid] = Obj(oid, info.get("class") or "", info.get("name") or "", addr) + objs[oid] = obj(oid) entries[key] = list(objs.values()) - return cls(entries) + return cls(entries, {path: [obj(o) for o in oids] for path, oids in doc["resources"].items()}) def to_json(self) -> dict: return {"keys": {k: [o.to_json() for o in self._objs[k]] for k in self._keys}} diff --git a/src/nnnotes/schemas/addresses.schema.json b/src/nnnotes/schemas/addresses.schema.json index 320e4e3..d503b25 100644 --- a/src/nnnotes/schemas/addresses.schema.json +++ b/src/nnnotes/schemas/addresses.schema.json @@ -1,8 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "urn:nnnotes:schema:addresses", - "title": "nnnotes.addresses/1", - "description": "The address table of a catalog version and the censuses of its bundles (stage link.addresses): per stable bundle name its file name, serialized files, AssetBundle name and keys; serialized file -> bundle; per catalog key its locations with their bundle (or raw files) and the objects the container path names; per stable address the object it names; the problems found.", + "title": "nnnotes.addresses/2", + "description": "The address table of a catalog version and the censuses of its bundles (stage link.addresses): per stable bundle name its file name, serialized files, AssetBundle name and keys; serialized file -> bundle; per catalog key its locations with their bundle (or raw files) and the objects the container path names; per resource path (Resources.Load, the APK's built-in files) the objects it names; per stable address the object it names; the problems found.", "type": "object", "required": [ "schema", @@ -10,12 +10,13 @@ "bundles", "files", "keys", + "resources", "objects", "problems" ], "properties": { "schema": { - "const": "nnnotes.addresses/1" + "const": "nnnotes.addresses/2" }, "catalog": { "type": "object", @@ -148,6 +149,17 @@ } } }, + "resources": { + "description": "Per path of a ResourceManager container (the paths Resources.Load finds objects at, as stored) the object ids it names, in stored order.", + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "string", + "pattern": "^.+:-?[0-9]+$" + } + } + }, "objects": { "type": "object", "additionalProperties": { diff --git a/src/nnnotes/schemas/census.schema.json b/src/nnnotes/schemas/census.schema.json index a1c0354..4cecd0e 100644 --- a/src/nnnotes/schemas/census.schema.json +++ b/src/nnnotes/schemas/census.schema.json @@ -2,13 +2,14 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "urn:nnnotes:schema:census", "title": "nnnotes.census/2", - "description": "The census of one bundle (stage unity.census), read without decoding any object: its serialized files by name (Unity version, platform, type trees, externals, objects by path id with class, size, type hash, m_Name, m_Script, the objects they reference and the hierarchy holding them), the other files of the bundle, its AssetBundle objects (container in stored order, dependencies, scene hashes), its MonoScripts, objects per class and the objects whose typetree could not be read. A function of the bundle's bytes.", + "description": "The census of one bundle (stage unity.census), read without decoding any object: its serialized files by name (Unity version, platform, type trees, externals, objects by path id with class, size, type hash, m_Name, m_Script, the objects they reference and the hierarchy holding them), the other files of the bundle, its AssetBundle objects (container in stored order, dependencies, scene hashes), the container of its ResourceManager (the paths Resources.Load finds objects at), its MonoScripts, objects per class and the objects whose typetree could not be read. A function of the bundle's bytes.", "type": "object", "required": [ "schema", "files", "resources", "assetBundles", + "resourcePaths", "scripts", "classes", "objects", @@ -329,6 +330,35 @@ } } }, + "resourcePaths": { + "description": "The container of the ResourceManager objects (in the APK's global managers): the paths Resources.Load finds objects at, in stored order; a path names one object per class it was saved as.", + "type": "array", + "items": { + "type": "object", + "required": [ + "path", + "file", + "pathId" + ], + "properties": { + "path": { + "type": "string" + }, + "file": { + "type": [ + "string", + "null" + ] + }, + "pathId": { + "type": [ + "integer", + "null" + ] + } + } + } + }, "scripts": { "type": "array", "items": { diff --git a/tests/cli/test_assets.py b/tests/cli/test_assets.py index 9a1eca6..e52104d 100644 --- a/tests/cli/test_assets.py +++ b/tests/cli/test_assets.py @@ -16,7 +16,7 @@ from nnnotes import cli from nnnotes.cli import assets, common from nnnotes.cli import catalog as catalog_cli -from nnnotes.pipeline import contract, units +from nnnotes.pipeline import contract, link, units from nnnotes.pipeline.census import CensusStage from nnnotes.pipeline.contract import Cost from nnnotes.pipeline.depot import Depot @@ -75,6 +75,7 @@ def cdoc(file: str, objs: list, container=(), scripts=(), bundle_name=None) -> d ], "resources": [], "assetBundles": [ab], + "resourcePaths": [], "scripts": list(scripts), "classes": classes, "objects": len(objs), @@ -915,8 +916,8 @@ def test_views_are_placed_with_paths_and_every_document_validates(tmp_path, caps contract.CATALOG_INDEX, contract.CENSUS, contract.VIEW, - "nnnotes.scripts/1", - "nnnotes.addresses/1", + link.SCRIPTS, + link.ADDRESSES, }, checked code, out, _ = nn( capsys, diff --git a/tests/helpers/fakeunity.py b/tests/helpers/fakeunity.py index 49089c1..ead4cb3 100644 --- a/tests/helpers/fakeunity.py +++ b/tests/helpers/fakeunity.py @@ -65,6 +65,7 @@ "PlayerSettings": 129, "Font": 128, "AssetBundle": 142, + "ResourceManager": 147, "AnimatorController": 91, "AnimatorOverrideController": 221, "RectTransform": 224, diff --git a/tests/helpers/units.py b/tests/helpers/units.py index bf36993..87c86b1 100644 --- a/tests/helpers/units.py +++ b/tests/helpers/units.py @@ -32,6 +32,7 @@ def census_doc(file: str, objects) -> dict: ], "resources": [], "assetBundles": [], + "resourcePaths": [], "scripts": [], "classes": dict(sorted(Counter(o["class"] for o in objs).items())), "objects": len(objs), diff --git a/tests/pipeline/test_assemble.py b/tests/pipeline/test_assemble.py index 547d509..b14dc93 100644 --- a/tests/pipeline/test_assemble.py +++ b/tests/pipeline/test_assemble.py @@ -15,12 +15,14 @@ from tests.pipeline.test_objexport import ATLAS_CAB, FAKE_READER, oid, planning_env -def addresses(depot, env, keys: dict) -> None: - """A committed link.addresses result whose table maps `keys` {catalog key: [object ids]}.""" +def addresses(depot, env, keys: dict, resources=None) -> None: + """A committed link.addresses result whose table maps `keys` {catalog key: [object ids]} and `resources` + {resource path: [object ids]}.""" t = Task("link.addresses", 1, "main", {}, {}, ()) doc = { "schema": link.ADDRESSES, "keys": {k: [{"objects": ids}] for k, ids in sorted(keys.items())}, + "resources": dict(sorted((resources or {}).items())), "objects": {}, "files": {}, "bundles": {}, @@ -75,11 +77,25 @@ def root_objects(self, subject, env): return self.objects -def env_for(depot, keys=None, names=("all", "atlas"), roots_objects=(), selected=None): +class Loaded(Toy): + """A Toy rooted at the objects of Resources.Load paths alone.""" + + def __init__(self, paths): + super().__init__() + self.paths = list(paths) + + def keys(self, subject, env): + return [] + + def resources(self, subject, env): + return self.paths + + +def env_for(depot, keys=None, names=("all", "atlas"), roots_objects=(), selected=None, resources=None): """A planning environment over the fake bundles with a committed address table and the facts of a request.""" env = planning_env(depot, names, selected=selected) env.facts["catalogs"] = {"main": []} - addresses(depot, env, keys if keys is not None else {}) + addresses(depot, env, keys if keys is not None else {}, resources) subjects = sorted(keys if keys is not None else {}) env.facts["products"] = {"toy.thing": subjects} env.facts["roots"] = {"keys": subjects, "objects": list(roots_objects)} @@ -117,7 +133,8 @@ def test_a_products_key_is_asked_for_before_anything_runs(tmp_path): stage = Toy({"m/a": "key/a", "m/b": "key/b"}) stages = {"toy.thing": stage} env = Env(depot, {"keys": {}}) - assert assemble.request(stages, {"toy.thing": ["m/b", "m/a"]}, env) == {"keys": ["key/a", "key/b"], "objects": []} + roots = assemble.request(stages, {"toy.thing": ["m/b", "m/a"]}, env) + assert roots == {"keys": ["key/a", "key/b"], "resources": [], "objects": []} assert stage.subjects(Env(depot, {"products": {"toy.thing": ["m/b"]}})) == ["m/b"] @@ -175,6 +192,20 @@ def test_the_request_roots_select_the_units_and_nothing_else(tmp_path): assert subjects == ["all:10", "all:60"] # the prefab and the texture it references +def test_a_resources_load_path_is_a_root(tmp_path): + """A product's Resources.Load paths name its roots through the address table, ignoring case.""" + depot = Depot(tmp_path / "s") + env = env_for(depot, {}, names=("all",), selected=[], resources={"shaders/thing": [oid(60)]}) + stage = Loaded(["Shaders/Thing"]) + env.facts["roots"] = assemble.request({"toy.thing": stage}, {"toy.thing": ["m/a"]}, env) + assert env.facts["roots"] == {"keys": [], "resources": ["Shaders/Thing"], "objects": []} + assert ExportStage(FAKE_READER).subjects(env) == ["all:10", "all:60"] + exported(depot, env) + assert stage.depends("m/a", env) == ["link.addresses:main", "unity.export:all:10", "unity.export:all:60"] + _, doc = product(depot, env, "m/a", stage=stage) + assert table_of(depot, doc)["roots"] == [oid(60)] + + def test_sprite_images_come_from_the_crop_of_their_atlas(tmp_path): depot = Depot(tmp_path / "s") env = env_for(depot, {"key/a": [oid(13)]}, names=("all", "atlas"), roots_objects=[f"{ATLAS_CAB}:99"]) diff --git a/tests/pipeline/test_census.py b/tests/pipeline/test_census.py index 8a6b73a..705e86a 100644 --- a/tests/pipeline/test_census.py +++ b/tests/pipeline/test_census.py @@ -251,6 +251,26 @@ def test_monoscripts_and_unknown_file_index(): assert refs.ref(None) is None and refs.ref(pptr(5)) == {"file": "CAB-1", "pathId": 5} +def test_the_resource_manager_lists_the_resources_load_paths(): + env = FakeEnvironment() + f = env.file("globalgamemanagers.assets", externals=["resources.assets"]) + container = [ + ("live2d/cubism/materials/mask", pptr(4, 1)), + ("live2d/cubism/materials/mask", pptr(5, 1)), + ("missing", pptr(0)), + ] + f.add(3, "ResourceManager", {"m_Container": container, "m_DependentAssets": []}) + doc = census.census_env(env) + assert doc["resourcePaths"] == [ + {"path": "live2d/cubism/materials/mask", "file": "resources.assets", "pathId": 4}, + {"path": "live2d/cubism/materials/mask", "file": "resources.assets", "pathId": 5}, + {"path": "missing", "file": None, "pathId": None}, + ] + (rm,) = doc["files"][0]["objects"] + assert rm["refs"] == ["resources.assets:4", "resources.assets:5"] + assert doc["assetBundles"] == [] + + def player_file() -> FakeEnvironment: """A player's own file without type trees: a MonoBehaviour that references a built-in texture.""" env = FakeEnvironment() @@ -293,7 +313,7 @@ def load(path, name=None): env = Env(depot, {"bundles": {"x": inp}}) assert stage.subjects(env) == ["x"] task = describe(stage, "x", None, env) - assert task.id == "unity.census:x" and task.atoms == census.CensusStage.ATOMS and task.version == 3 + assert task.id == "unity.census:x" and task.atoms == census.CensusStage.ATOMS and task.version == 4 assert task.atoms["reader"].startswith("unitypy-") assert task.cost.peak_bytes > size assert execute(task, depot, registry(stage)).status == "ran" diff --git a/tests/pipeline/test_link.py b/tests/pipeline/test_link.py index eee3d1a..67b9333 100644 --- a/tests/pipeline/test_link.py +++ b/tests/pipeline/test_link.py @@ -1,6 +1,6 @@ import pytest -from nnnotes.pipeline import catalogdb, census, contract, link +from nnnotes.pipeline import catalogdb, census, contract, link, views from nnnotes.pipeline.catalogdb import CatalogDB, IndexStage, index from nnnotes.pipeline.contract import Input from nnnotes.pipeline.depot import Depot @@ -20,8 +20,9 @@ def obj(pid, cls, name=None, size=10, script=None): return o -def cdoc(files: dict, container=(), name=None, scripts=()): - """A census document: files {file: [objects]}, container [(path, file, pathId)] on the first file's AssetBundle.""" +def cdoc(files: dict, container=(), name=None, scripts=(), resource_paths=()): + """A census document: files {file: [objects]}, container [(path, file, pathId)] on the first file's AssetBundle, + resource_paths [(path, file, pathId)] of a ResourceManager.""" fs = [ { "name": f, @@ -55,6 +56,7 @@ def cdoc(files: dict, container=(), name=None, scripts=()): "files": fs, "resources": [], "assetBundles": [ab], + "resourcePaths": [{"path": p, "file": f, "pathId": i} for p, f, i in resource_paths], "scripts": list(scripts), "classes": classes, "objects": sum(classes.values()), @@ -133,6 +135,39 @@ def censuses(): return {"card_a": card, "shared": shared, "scene_x": scene, "scripts": scripts} +def test_resource_paths_name_objects_of_the_builtin_files(): + """A built-in file's ResourceManager paths: the resources table, and addresses like container paths.""" + idx = index(synth.CatalogWriter().build(catalog_entries())) + data = cdoc( + { + "globalgamemanagers.assets": [obj(1, "ResourceManager")], + "resources.assets": [obj(4, "Material", "Mask"), obj(5, "Texture2D", "Mask"), obj(6, "Shader", "S")], + }, + resource_paths=[ + ("live2d/cubism/materials/mask", "resources.assets", 4), + ("live2d/cubism/materials/mask", "resources.assets", 5), + ("shaders/s", "resources.assets", 6), + ("gone", None, None), + ], + ) + doc = link.addresses(idx, {**censuses(), "apk/data.unity3d": data}) + assert doc["resources"] == { + "live2d/cubism/materials/mask": ["resources.assets:4", "resources.assets:5"], + "shaders/s": ["resources.assets:6"], + } + objs = doc["objects"] + assert objs["apk/data.unity3d/live2d/cubism/materials/mask"]["object"] == "resources.assets:4" + assert objs["apk/data.unity3d/live2d/cubism/materials/mask[Mask]"]["object"] == "resources.assets:5" + assert objs["apk/data.unity3d/shaders/s"]["class"] == "Shader" + assert doc["bundles"]["apk/data.unity3d"]["keys"] == [] and doc["problems"]["keys"] == ["Card/Gone"] + ix = views.AddressIndex.from_addresses(doc) + mask = ix.resource("Live2D/Cubism/Materials/Mask") + assert [(o.id, o.cls) for o in mask] == [("resources.assets:4", "Material"), ("resources.assets:5", "Texture2D")] + assert mask[0].stable == "apk/data.unity3d/live2d/cubism/materials/mask" + assert ix.resource("live2d/cubism/materials/maskculling") is None + assert link.addresses(idx, censuses())["resources"] == {} + + def test_scripts_table_and_context(): docs = censuses() table = link.scripts(docs.values()) diff --git a/tests/pipeline/test_modelstages.py b/tests/pipeline/test_modelstages.py index 49afcd8..bb59984 100644 --- a/tests/pipeline/test_modelstages.py +++ b/tests/pipeline/test_modelstages.py @@ -299,6 +299,7 @@ def cdoc(files, scripts=()): "files": fs, "resources": [], "assetBundles": [], + "resourcePaths": [], "scripts": list(scripts), "classes": classes, "objects": sum(classes.values()), diff --git a/tests/pipeline/test_views.py b/tests/pipeline/test_views.py index dd9dda9..29c3865 100644 --- a/tests/pipeline/test_views.py +++ b/tests/pipeline/test_views.py @@ -10,7 +10,7 @@ import pytest -from nnnotes.pipeline import contract, views +from nnnotes.pipeline import contract, link, views from nnnotes.pipeline.contract import Input from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.orchestrate import Orchestrator @@ -116,6 +116,7 @@ def test_stored_order_decides_not_ids(): } ] }, + "resources": {}, "objects": {f"b/s.png[{x.name}]": {"object": x.id, "class": x.cls, "name": x.name} for x in sheet}, } assert [x.id for x in AddressIndex.from_addresses(doc).objects("S")] == [x.id for x in sheet] @@ -162,7 +163,7 @@ def test_address_index_build_and_prefixes(): def test_address_index_of_an_address_table(): doc = { - "schema": "nnnotes.addresses/1", + "schema": link.ADDRESSES, "keys": { "MemberCard/1/member_full": [ { @@ -191,7 +192,9 @@ def test_address_index_of_an_address_table(): ], "Cri/Sound/bgm": [{"location": 4, "type": "System.Object", "raw": "bgm.acb", "internalId": "y"}], }, + "resources": {"live2d/mask": ["resources.assets:3"]}, "objects": { + "apk/data.unity3d/live2d/mask": {"object": "resources.assets:3", "class": "Material", "name": "Mask"}, "membercard_1/assets/membercard/1/member_full.png": { "object": "CAB-a:1", "class": "Texture2D", @@ -212,6 +215,8 @@ def test_address_index_of_an_address_table(): assert ix.objects("MemberCard/2/member_full") == [] and ix.objects("Cri/Sound/bgm") == [] r = views.resolve(ix, "MemberCard/1/member_full[member_full]", "Sprite") assert (r["object"], r["stable"]) == ("CAB-a:2", "membercard_1/assets/membercard/1/member_full.png[member_full]") + assert ix.resource("Live2D/Mask") == [Obj("resources.assets:3", "Material", "Mask", "apk/data.unity3d/live2d/mask")] + assert ix.resource("Live2D/Other") is None and ix.objects("live2d/mask") is None # paths are no catalog keys # ---------------------------------------------------------------- rules @@ -712,7 +717,7 @@ def test_master_tables_from_a_directory(tmp_path): # ---------------------------------------------------------------- the view stages def addresses_doc(entries: dict) -> dict: - """A synthetic address table (nnnotes.addresses/1) of {catalog key: objects}: one location per key in bundle + """A synthetic address table (nnnotes.addresses/2) of {catalog key: objects}: one location per key in bundle `b`, every object listed under a stable address of its own.""" keys, objects = {}, {} for n, (key, objs) in enumerate(sorted(entries.items())): @@ -734,11 +739,12 @@ def addresses_doc(entries: dict) -> dict: "typeHash": None, } return { - "schema": "nnnotes.addresses/1", + "schema": link.ADDRESSES, "catalog": {"remote": None, "apk": None}, "bundles": {}, "files": {}, "keys": keys, + "resources": {}, "objects": objects, "problems": {"files": [], "bundleNames": [], "keys": []}, }