From 414a23115864266509355867f49c7bb4e94c3ca9 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Thu, 15 Oct 2026 04:33:51 +0800 Subject: [PATCH] feat(assemble): add product stages assembled from the exported objects A product stage (nnnotes.pipeline.assemble.ProductStage) makes one task per subject from root objects: the objects of the catalog keys it gives for the subject, through the address table, and the object ids it names. Its task reaches the units holding its roots and every unit an item of a reached unit references, through the unity.export results; a sprite whose atlas is in another bundle takes its image from sprite.crop. - The task's input is the object table of what it reaches (nnnotes.objects/1): each object's class, name, item status and files, the object holding a contained one, and the referenced objects no census holds. Its key therefore follows exactly the files it can read. - The result holds the product's own files as `#`, its object table as `#objects` and one item for the subject. - The unit request takes the fact `roots` ({keys, objects}): `assemble.request` gives it from the products asked for, so a run exports the units their roots reach and nothing else of their bundles. - layout.product_entries and layout.product_manifest place products for a player: every file at `assets/.` and a manifest (nnnotes.product/1) per product listing its files and its objects' files. - view.voices reads the address table through the same cached reader. Co-Authored-By: Claude Opus 5.5 --- docs/contracts.md | 8 + docs/products.md | 83 ++++++ docs/schemas.md | 2 + docs/stages.md | 8 +- mkdocs.yml | 1 + src/nnnotes/pipeline/assemble.py | 304 +++++++++++++++++++++ src/nnnotes/pipeline/contract.py | 2 + src/nnnotes/pipeline/layout.py | 57 ++++ src/nnnotes/pipeline/units.py | 64 ++++- src/nnnotes/pipeline/voices.py | 16 +- src/nnnotes/schemas/objects.schema.json | 89 ++++++ src/nnnotes/schemas/product.schema.json | 86 ++++++ tests/pipeline/test_assemble.py | 343 ++++++++++++++++++++++++ 13 files changed, 1031 insertions(+), 32 deletions(-) create mode 100644 docs/products.md create mode 100644 src/nnnotes/pipeline/assemble.py create mode 100644 src/nnnotes/schemas/objects.schema.json create mode 100644 src/nnnotes/schemas/product.schema.json create mode 100644 tests/pipeline/test_assemble.py diff --git a/docs/contracts.md b/docs/contracts.md index 369222e..dcd8fe2 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -202,6 +202,13 @@ bytes nor keys; the file names of `original` and `cas` both follow from the reco `original` manifest alone. Documents derived for `original` from records and paths (the views with the files of their objects) are the exception: `original` has the derived documents, `cas` the artifacts they are made from. +## Product manifest: nnnotes.product/1 + +A product layout writes, for each product of `pipeline.assemble`, a manifest +([product.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/product.schema.json)): the product's +files and the files of the objects it uses, each a content-addressed `assets/.`. The product's object +table is `nnnotes.objects/1`. See [products](products.md). + ## Reports - Coverage, `nnnotes.coverage/1` ([coverage.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/coverage.schema.json)): per class the census @@ -224,6 +231,7 @@ Documents written by particular stages (artifacts of no object: their ids are `< | `nnnotes.scripts/1` | `link.scripts` | [scripts.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/scripts.schema.json) | | `nnnotes.addresses/1` | `link.addresses` | [addresses.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/addresses.schema.json) | | `nnnotes.view/1` | `view.` | [view.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/view.schema.json) | +| `nnnotes.objects/1` | `pipeline.assemble` (a product's object table) | [objects.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/objects.schema.json) | The catalog versions of a depot, `nnnotes.catalogs/1` ([catalogs.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/catalogs.schema.json), `/catalogs/index.json`), and the difference between two of them, `nnnotes.catalog-diff/1` diff --git a/docs/products.md b/docs/products.md new file mode 100644 index 0000000..8606ccd --- /dev/null +++ b/docs/products.md @@ -0,0 +1,83 @@ +# Products + +A product is something a player or a site reads as a whole: a Live2D model, a story episode, a live. Products are +assembled from the results of `unity.export` ([stages](stages.md)), never from a bundle: by the time a product is +made, every object it uses has been converted once, and the product combines those conversions with what it adds +itself. + +## Product stages + +A product stage (`nnnotes.pipeline.assemble.ProductStage`) has one task per subject. Its task starts from root +objects: + +- the objects of the catalog keys the stage gives for the subject (`keys`), through the address table of + `link.addresses`; +- the object ids the stage names besides (`root_objects`). + +Both are read from the planning facts alone, so a command can ask for what its products need before anything runs: +`assemble.request(stages, products, env)` gives the fact `roots` (`{"keys", "objects"}`), and the request of +`unity.export` then holds the units of those objects and every unit reachable from them through the census `refs` +([units](stages.md#units)), and nothing else of their bundles. + +What a task reaches: the units holding its roots, then every unit that an item of a reached unit references (the +item's `refs`), through the `unity.export` results. A sprite whose atlas is in another bundle has its `image` from the +`sprite.crop` result of that atlas. A reached unit that the request did not export is an error of the request; an +object that a reached object references and no census holds is listed as `missing`. + +| Part of the task | | +|---|---| +| inputs | `objects`: the object table of what it reaches (`nnnotes.objects/1`); the stage's own inputs (master tables, data its parameters name) | +| depends | `link.addresses` (when it has keys), the `unity.export` and `sprite.crop` tasks whose results it read | +| artifacts | `#`: the product's own files; `#objects`: the object table of the objects its files use (the whole table unless the stage narrows it) | +| item | one for the subject: `exported`, or `unsupported` with a reason code | + +The key holds the object table, so a product is assembled again exactly when a file of an object it reaches +changes, or what it reaches changes. + +## Object table: nnnotes.objects/1 + +([objects.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/objects.schema.json)) + +```json +{"schema": "nnnotes.objects/1", + "roots": ["CAB-…:-4153…"], + "objects": { + "CAB-…:-4153…": {"status": "exported", "class": "GameObject", "name": "model", + "files": {"json": {"sha256": "…", "size": 5120, "ext": "json"}}}, + "CAB-…:12": {"status": "contained", "class": "Transform", "in": "CAB-…:-4153…"}, + "CAB-…:40": {"status": "exported", "class": "Texture2D", "name": "texture_00", + "files": {"image": {"sha256": "…", "size": 812345, "ext": "png"}, + "json": {"sha256": "…", "size": 640, "ext": "json"}}}}, + "missing": []} +``` + +Each object has the status of its item (the most severe when two stages made items for it), its class and name +where known, its files by role (the artifacts of its items), the object whose file holds it (`in`, a contained +object), or the reason code of a generic, unsupported or failed object. + +## Product layout + +A product layout places products for a player: + +- every file of a product and every file of the objects of its object table at `assets/.` (a file + several products or objects share is written once); +- each product's manifest at `/.json` (the stage's `manifest(subject)`, by default + `/.json` with the subject escaped as one path segment). + +## Product manifest: nnnotes.product/1 + +([product.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/product.schema.json)) + +```json +{"schema": "nnnotes.product/1", "product": "live2d.model", "subject": "…", + "files": {"model.json": {"asset": "assets/.json", "size": 2048}}, + "objects": { + "CAB-…:-4153…": {"class": "GameObject", "name": "model", + "files": {"json": {"asset": "assets/.json", "size": 5120}}}, + "CAB-…:12": {"class": "Transform", "in": "CAB-…:-4153…"}}} +``` + +`files` are the product's own files by path; `objects` are the objects of its object table that have files, or are +contained in an object that has one. An asset path is relative to the directory above the manifest's directory, as +in the chart manifests of ournotes-player. A reference in an object's JSON (`{"$ref": {"file", "pathId"}}`) names +the object `:` of `objects`. diff --git a/docs/schemas.md b/docs/schemas.md index 0fa3405..fd0a6b7 100644 --- a/docs/schemas.md +++ b/docs/schemas.md @@ -26,7 +26,9 @@ schemas.load("plan") # the schema as a dict | `failures` | `nnnotes.failures/1` | the failed tasks and items of a run | | `layout` | `nnnotes.layout/1` | the files a layout owns in its output directory | | `music-data` | `nnnotes.music-data/2` | every song and chart of one master data version ([music data](music-data.md)) | +| `objects` | `nnnotes.objects/1` | the objects a product reaches: their class, item status and files ([products](products.md)) | | `plan` | `nnnotes.plan/1` | what a run would do and why (`depot plan`) | +| `product` | `nnnotes.product/1` | a product's manifest: its files and the files of the objects it uses, as content-addressed assets ([products](products.md)) | | `result` | `nnnotes.result/1` | the result of a task, a function of its key alone | | `run` | `nnnotes.run/1` | a run manifest: context, selection, tasks with keys and statuses, layouts | | `scripts` | `nnnotes.scripts/1` | every MonoScript of the censuses, by object id | diff --git a/docs/stages.md b/docs/stages.md index 7d46629..bc7eb16 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -109,10 +109,10 @@ hierarchy. Its subject is the stable address of its root object, `#", the +table of the objects they use as "#objects", and one item for the subject. A product layout +(layout.product_entries) places the files and the objects' files and writes the product's manifest. +""" + +from __future__ import annotations + +from collections import OrderedDict +from dataclasses import dataclass, field + +from ..unity.atoms import Unsupported +from . import contract, units +from .contract import Cost, Input +from .layout import PRODUCT_TABLE, escape_segment +from .orchestrate import SEVERITY +from .stages import Output, Stage + +EXPORT, CROP = "unity.export", "sprite.crop" +ADDRESSES = units.ADDRESSES +TABLE_KIND = "product.objects" + + +def request(stages: dict, products: dict, env) -> dict: + """The fact "roots" (pipeline.units) of a request for `products` {stage name: subjects}: the catalog keys and the + root objects of every subject, sorted.""" + keys, objects = set(), set() + for name, subjects in sorted(products.items()): + stage = stages[name] + for s in subjects: + keys.update(stage.keys(s, env)) + objects.update(stage.root_objects(s, env)) + return {"keys": sorted(keys), "objects": sorted(objects)} + + +def _entry(item: dict, arts: dict) -> dict: + """The object table entry of a unity.export or sprite.crop item.""" + e: dict = {"status": item["status"]} + cls, name, files = item.get("class"), None, {} + for aid in item.get("artifacts", ()): + a = arts[aid] + c = a["content"] + files[contract.parse_artifact_id(aid)[1]] = {"sha256": c["sha256"], "size": c["size"], "ext": c["ext"]} + obj = (a.get("provenance") or {}).get("object") or {} + cls, name = cls or obj.get("class"), name or obj.get("name") + if cls: + e["class"] = cls + if name: + e["name"] = name + if files: + e["files"] = files + if "in" in item: + e["in"] = contract.parse_artifact_id(item["in"])[0] + if "reason" in item: + e["reason"] = item["reason"]["code"] + return e + + +def _merge(e: dict, other: dict) -> None: + """Add the files of another item of the same object; the status becomes the more severe.""" + e.setdefault("files", {}).update(other.get("files", {})) + if SEVERITY.index(other["status"]) < SEVERITY.index(e["status"]): + e["status"] = other["status"] + if "reason" in other: + e["reason"] = other["reason"] + + +@dataclass +class Reach: + """What a product reaches: its object table (nnnotes.objects/1), the tasks whose results it read, the bytes of + its objects' files.""" + + table: dict + read: list = field(default_factory=list) + size: int = 0 + + +class _Results: + """The items and artifact records of the results read, by result key (a result never changes).""" + + def __init__(self, size: int = 4096): + self.size, self._docs = size, OrderedDict() + + def get(self, env, tid: str) -> tuple[list, dict]: + key = env.key(tid) + doc = self._docs.get(key) + if doc is None: + res = env.result(tid) + doc = self._docs[key] = (res["items"], {a["id"]: a for a in res["artifacts"]}) + while len(self._docs) > self.size: + self._docs.popitem(last=False) + else: + self._docs.move_to_end(key) + return doc + + +def reach(env, roots, results: _Results | None = None) -> Reach: + """The objects reachable from the object ids `roots` (module documentation).""" + results = results or _Results() + table = units.table(env) + todo, seen, missing = [], set(), set() + for o in roots: + s = table.subject_of(o) + if s is None: + missing.add(o) + else: + todo.append(s) + objects: dict[str, dict] = {} + read = [] + crops: dict[str, list] = {} # atlas object id -> sprite object ids + while todo: + s = todo.pop() + if s in seen: + continue + seen.add(s) + table.unit(s) # ValueError: a unit the request does not export + tid = contract.task_id(EXPORT, s) + items, arts = results.get(env, tid) + read.append(tid) + for it in items: + oid = it["object"] + objects[oid] = _entry(it, arts) + for aid in it.get("artifacts", ()): + facts = arts[aid]["semantics"].get("facts") or {} + if facts.get("crop") == CROP: + crops.setdefault(facts["atlas"], []).append(oid) + for r in it.get("refs", ()): + t = table.subject_of(r) + if t is None: + missing.add(r) + elif t not in seen: + todo.append(t) + for atlas, sprites in sorted(crops.items()): + s = table.subject_of(atlas) + tid = contract.task_id(CROP, s) if s is not None else None + if tid is None or not env.known(tid): # the atlas is not exported: its sprites have no image + missing.add(atlas) + continue + items, arts = results.get(env, tid) + read.append(tid) + made = {it["object"]: it for it in items} + for oid in sprites: + if oid in made: + _merge(objects[oid], _entry(made[oid], arts)) + doc = { + "schema": contract.OBJECTS, + "roots": sorted(set(roots)), + "objects": dict(sorted(objects.items())), + "missing": sorted(missing), + } + size = sum(f["size"] for e in objects.values() for f in e.get("files", {}).values()) + return Reach(doc, sorted(read), size) + + +class Objects: + """The object table of a product task and the bytes of its objects' files (from the depot).""" + + def __init__(self, doc: dict, depot): + self.doc, self.depot = doc, depot + self.roots: list[str] = doc["roots"] + self._objects: dict = doc["objects"] + self._json: dict = {} + + def __contains__(self, oid: str) -> bool: + return oid in self._objects + + def ids(self, cls: str | None = None) -> list[str]: + """The object ids of the table (of class `cls`), sorted.""" + return [o for o, e in self._objects.items() if cls is None or e.get("class") == cls] + + def info(self, oid: str) -> dict: + try: + return self._objects[oid] + except KeyError: + raise KeyError(f"object {oid} is not in the product's object table") from None + + def bytes(self, oid: str, role: str) -> bytes: + files = self.info(oid).get("files") or {} + if role not in files: + raise KeyError(f"object {oid} has no {role!r} file") + return self.depot.read(files[role]["sha256"]) + + def json(self, oid: str, role: str = "json"): + """The decoded JSON of a file of an object (kept for the task).""" + k = (oid, role) + if k not in self._json: + self._json[k] = contract.loads(self.bytes(oid, role)) + return self._json[k] + + +@dataclass +class Assembly: + """What `assemble` makes: the product's files {path: (bytes, extension, kind)} and the object ids its files use + (None: every object of the table).""" + + files: dict + objects: set | None = None + + +class ProductStage(Stage): + """Base of the product stages (module documentation). Subclasses set `name`, `version` and `KIND`, and define + `keys` and/or `root_objects` and `assemble`; `subjects` defaults to the subjects of the fact "products".""" + + after = (ADDRESSES, EXPORT, CROP) + KIND = "product" + + def __init__(self): + self._memo: dict = {} + self._results = _Results() + + def __getstate__(self): + return {k: v for k, v in self.__dict__.items() if k not in ("_memo", "_results")} + + def __setstate__(self, state): + self.__dict__.update(state) + self._memo, self._results = {}, _Results() + + # ---------------------------------------------------------------- what a subclass defines + def subjects(self, env) -> list[str]: + return sorted((env.facts.get("products") or {}).get(self.name, ())) + + def keys(self, subject: str, env) -> list[str]: + """The catalog keys whose objects are roots of `subject` (from the facts alone).""" + return [] + + def root_objects(self, subject: str, env) -> list[str]: + """Object ids that are roots of `subject` besides those of its keys (from the facts alone).""" + return [] + + def more_inputs(self, subject: str, env) -> list[Input]: + """Inputs besides the object table.""" + return [] + + def assemble(self, task, objects: Objects) -> Assembly: + raise NotImplementedError + + def manifest(self, subject: str) -> str: + """The path of the subject's manifest in a product layout: one directory, then a file.""" + return f"{self.name}/{escape_segment(subject)}.json" + + # ---------------------------------------------------------------- planning + def reach(self, subject: str, env) -> Reach: + """What `subject` reaches (kept while the results it read are the same).""" + roots = units.root_objects(env, self.keys(subject, env), self.root_objects(subject, env)) + sig = (tuple(roots), id(units.table(env))) + memo = self._memo.get(subject) + if memo is not None and memo[0] == sig and all(env.key(t) == k for t, k in memo[1]): + return memo[2] + r = reach(env, roots, self._results) + self._memo[subject] = (sig, [(t, env.key(t)) for t in r.read], r) + return r + + def inputs(self, subject: str, env) -> list[Input]: + data = contract.encode(self.reach(subject, env).table) + table = Input("objects", env.depot.put(data), len(data), None, ({"kind": "depot"},)) + return [table, *self.more_inputs(subject, env)] + + def depends(self, subject: str, env) -> list[str]: + own = [contract.task_id(ADDRESSES, s) for s in sorted(env.fact("catalogs"))] if self.keys(subject, env) else [] + return own + self.reach(subject, env).read + + def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: + """From the bytes of the objects' files: a product reads each of them about once.""" + size = self.reach(subject, env).size + sum(i.size for i in inputs) + return Cost(0.02 + size / 100e6, (64 << 20) + 2 * size) + + # ---------------------------------------------------------------- running + def run(self, task, depot) -> Output: + data = depot.input_bytes(task.input("objects")) + doc = contract.loads(data) + try: + made = self.assemble(task, Objects(doc, depot)) + except Unsupported as e: + why = contract.reason(e.code, e.message) + return Output([], [contract.item(task.id, "unsupported", why=why, cls=self.KIND)]) + prov = contract.provenance(task) + arts = [] + for path, (body, ext, kind) in sorted(made.files.items()): + if path == PRODUCT_TABLE: + raise ValueError(f"{task.id}: a product file cannot be named {PRODUCT_TABLE!r}") + aid = contract.artifact_id(task.id, path) + arts.append(contract.artifact(aid, depot.add(body, ext), prov, {"kind": kind})) + if made.objects is not None: + unknown = sorted(set(made.objects) - set(doc["objects"])) + if unknown: + raise ValueError(f"{task.id}: objects not in the table: {', '.join(unknown[:5])}") + doc = dict(doc, objects={o: e for o, e in doc["objects"].items() if o in made.objects}) + data = contract.encode(doc) + aid = contract.artifact_id(task.id, PRODUCT_TABLE) + arts.append(contract.artifact(aid, depot.add(data, "json"), prov, {"kind": TABLE_KIND})) + item = contract.item(task.id, "exported", artifacts=[a["id"] for a in arts], cls=self.KIND) + return Output(arts, [item]) diff --git a/src/nnnotes/pipeline/contract.py b/src/nnnotes/pipeline/contract.py index 97ff7f6..bf25f02 100644 --- a/src/nnnotes/pipeline/contract.py +++ b/src/nnnotes/pipeline/contract.py @@ -55,6 +55,8 @@ CATALOG_INDEX = "nnnotes.catalog-index/1" CENSUS = "nnnotes.census/2" VIEW = "nnnotes.view/1" +OBJECTS = "nnnotes.objects/1" +PRODUCT = "nnnotes.product/1" # item statuses of a result: every object a task covers has exactly one ITEM_STATUSES = ("exported", "contained", "generic", "unsupported", "failed") diff --git a/src/nnnotes/pipeline/layout.py b/src/nnnotes/pipeline/layout.py index 3fbb13d..b20eb4d 100644 --- a/src/nnnotes/pipeline/layout.py +++ b/src/nnnotes/pipeline/layout.py @@ -27,6 +27,9 @@ - paths that coincide ignoring case (or a file named like another path's directory) keep the first by path and id; each other gets `~` before its extension and is reported; artifacts with the same path and the same content share the file +`product` (product_entries): the products of pipeline.assemble. Every file of a product and of the objects of its +object table is at `assets/.`; each product has a manifest (nnnotes.product/1, product_manifest) at +`/.json` (ProductStage.manifest), whose asset paths are relative to the directory above its own. So every path ends with its artifact's extension, and `cas` follows from the `original` manifest alone; `original` follows from `cas` plus the results (the records). Documents a stage derives for `original` from its records and the paths (the views with the files of their objects) are the exception: `original` has the derived documents, @@ -63,6 +66,7 @@ MANIFEST = "manifest.json" JOURNAL = ".nnnotes-layout-journal.json" LAYOUTS = ("original", "cas") +PRODUCT_TABLE = "objects" # the role of a product's object table (nnnotes.objects/1) among its artifacts PLACEMENTS = ("clone", "hard", "copy") # how a file is made from its byte source LINKS = ("auto", *PLACEMENTS) # what a run may ask for: auto probes FICLONE = 0x40049409 # Linux ioctl: make a file share another's blocks @@ -353,6 +357,59 @@ def _resolve(placed: list) -> tuple[list[dict], dict]: } +def _asset(c: dict) -> dict: + return {"asset": f"assets/{c['sha256']}.{c['ext']}", "size": c["size"]} + + +def product_manifest(task: str, result: dict, table: dict) -> dict: + """The manifest (nnnotes.product/1) of a product: its files {path: {asset, size}} and the objects of its object + table `table` that have files ({class, name, files: {role: {asset, size}}}) or are contained in one ({class, + in}).""" + stage, subject = contract.parse_task_id(task) + files = {} + for a in result["artifacts"]: + role = contract.parse_artifact_id(a["id"])[1] + if role != PRODUCT_TABLE: + files[role] = _asset(a["content"]) + objects = {} + for oid, o in table["objects"].items(): + e = {k: o[k] for k in ("class", "name") if k in o} + if o.get("files"): + e["files"] = {r: _asset(f) for r, f in sorted(o["files"].items())} + elif "in" in o: + e["in"] = o["in"] + else: + continue + objects[oid] = e + return {"schema": contract.PRODUCT, "product": stage, "subject": subject, "files": files, "objects": objects} + + +def product_entries(products) -> list[dict]: + """The entries of the `product` layout: `products` [(manifest path, the manifest's record {id, sha256, size}, + result, object table)].""" + out: dict[tuple, dict] = {} + for path, manifest, result, table in products: + segs = path.split("/") + if len(segs) != 2 or not all(segs) or not segs[1].endswith(".json"): + raise ValueError(f"product manifest path {path!r}: expected /.json") + out[(path, manifest["id"])] = { + "path": path, + "sha256": manifest["sha256"], + "size": manifest["size"], + "id": manifest["id"], + } + for a in result["artifacts"]: + if contract.parse_artifact_id(a["id"])[1] != PRODUCT_TABLE: + e = _entry(_asset(a["content"])["asset"], a) + out[(e["path"], e["id"])] = e + for oid, o in table["objects"].items(): + for role, f in (o.get("files") or {}).items(): + aid = contract.artifact_id(oid, role) + p = f"assets/{f['sha256']}.{f['ext']}" + out[(p, aid)] = {"path": p, "sha256": f["sha256"], "size": f["size"], "id": aid} + return [out[k] for k in sorted(out)] + + def entries(name: str, sources, **kw) -> tuple[list[dict], dict]: """The entries and report of layout `name`.""" if name == "cas": diff --git a/src/nnnotes/pipeline/units.py b/src/nnnotes/pipeline/units.py index bfce293..a82dff1 100644 --- a/src/nnnotes/pipeline/units.py +++ b/src/nnnotes/pipeline/units.py @@ -5,11 +5,15 @@ path-id stable address of its root object (link.path_address: ":", with the file after the bundle in a bundle of several serialized files), which stays when the bundle's content changes; its object ids do not. -The request (`table`): every unit holding an object of the selected bundles (fact "selected"), and every unit -reachable from those through what their objects reference (census `refs`) across the censuses of the bundles of -the fact "bundles"; with the fact "classes" (a list of class names; absent or None: every class), only the units -whose root object is of those classes. A bundle whose census failed has no units; a reference into a file no -census holds is left out. +The request (`table`): every unit holding an object of the selected bundles (fact "selected") or a root object of the +fact "roots", and every unit reachable from those through what their objects reference (census `refs`) across the +censuses of the bundles of the fact "bundles"; with the fact "classes" (a list of class names; absent or None: +every class), only the units whose root object is of those classes. A bundle whose census failed has no units; a +reference into a file no census holds is left out. + +The fact "roots" (absent: none) is {"keys": catalog keys, "objects": object ids}: the objects of each key (all those +the address table of link.addresses lists for it; a key without objects adds none) and the objects named. The +products of a request start from them (pipeline.assemble). """ from __future__ import annotations @@ -17,11 +21,12 @@ from collections import Counter from dataclasses import dataclass -from . import contract, link +from . import contract, link, views from .census import CensusStage from .stages import Pending CENSUS = CensusStage.name +ADDRESSES = views.ADDRESSES_STAGE @dataclass(frozen=True) @@ -131,9 +136,9 @@ def _census_units(stable: str, doc: dict, need_refs: bool): return units, of, {f["name"]: stable for f in doc["files"]}, refs -def build(censuses, bundles, selected, classes=None) -> Units: - """The Units of a request over `censuses` [(stable bundle name, census document)] of the bundles `bundles` - (module documentation).""" +def build(censuses, bundles, selected, classes=None, roots=()) -> Units: + """The Units of a request over `censuses` [(stable bundle name, census document)] of the bundles `bundles`, from + the units of the bundles `selected` and of the objects `roots` (module documentation).""" selected = set(selected) every = selected >= set(bundles) # no reference can add a unit: none is read units: dict[str, Unit] = {} @@ -152,6 +157,7 @@ def build(censuses, bundles, selected, classes=None) -> Units: want = set(units) else: want = {s for s, u in units.items() if u.bundle in selected} + want.update(of[o] for o in roots if o in of) todo = list(want) while todo: for oid in refs.get(todo.pop(), ()): @@ -166,12 +172,36 @@ def build(censuses, bundles, selected, classes=None) -> Units: _MEMO: list = [] # [(signature, Units)] of the last table +_ADDRESSES: dict = {} # {sha256: AddressIndex} of the last address table read + + +def address_index(env, subject: str) -> views.AddressIndex: + """The AddressIndex of the result of link.addresses for the catalog `subject` (Pending while it has not run).""" + sha = env.artifact(contract.task_id(ADDRESSES, subject), "addresses")["content"]["sha256"] + index = _ADDRESSES.get(sha) + if index is None: + index = views.AddressIndex.from_addresses(contract.loads(env.depot.read(sha))) + _ADDRESSES.clear() + _ADDRESSES[sha] = index + return index + + +def root_objects(env, keys=(), objects=()) -> list[str]: + """The object ids of the catalog `keys` (through the address tables of the fact "catalogs") and the `objects`, + sorted.""" + out = set(objects) + if keys: + for subject in sorted(env.fact("catalogs")): + index = address_index(env, subject) + for k in keys: + out.update(o.id for o in index.objects(k) or ()) + return sorted(out) def table(env) -> Units: """The Units of the request of a planning environment: the censuses of the fact "bundles" (Pending while one - has not run; a failed one is skipped), the facts "selected" and "classes". Kept for the planning state it was - made in.""" + has not run; a failed one is skipped), the facts "selected", "roots" (Pending while link.addresses has not run, + when it names keys) and "classes". Kept for the planning state it was made in.""" bundles = sorted(env.fact("bundles")) keys = [] for b in bundles: @@ -182,7 +212,15 @@ def table(env) -> Units: if not p.failed: raise classes = env.facts.get("classes") - sig = (id(env.depot), tuple(keys), tuple(env.fact("selected")), None if classes is None else tuple(classes)) + request = env.facts.get("roots") or {} + roots = root_objects(env, request.get("keys", ()), request.get("objects", ())) + sig = ( + id(env.depot), + tuple(keys), + tuple(env.fact("selected")), + None if classes is None else tuple(classes), + tuple(roots), + ) if _MEMO and _MEMO[0][0] == sig: return _MEMO[0][1] @@ -193,6 +231,6 @@ def censuses(): rec = next(a for a in env.depot.committed(key, tid)["artifacts"] if a["id"] == aid) yield b, contract.loads(env.depot.read(rec["content"]["sha256"])) - t = build(censuses(), bundles, env.fact("selected"), classes) + t = build(censuses(), bundles, env.fact("selected"), classes, roots) _MEMO[:] = [(sig, t)] return t diff --git a/src/nnnotes/pipeline/voices.py b/src/nnnotes/pipeline/voices.py index 147b060..d6c519c 100644 --- a/src/nnnotes/pipeline/voices.py +++ b/src/nnnotes/pipeline/voices.py @@ -820,20 +820,6 @@ def _sheet_name(name: str) -> str | None: return name if "/" not in name else None -_address_cache: dict = {} # content id of an address table -> its AddressIndex, the latest one - - -def _address_index(env, tid: str): - """The AddressIndex of the result of the link.addresses task `tid`.""" - sha = env.artifact(tid, "addresses")["content"]["sha256"] - hit = _address_cache.get(sha) - if hit is None: - hit = views.AddressIndex.from_addresses(contract.loads(env.depot.read(sha))) - _address_cache.clear() - _address_cache[sha] = hit - return hit - - class VoiceStage(Stage): """view.voices (module documentation). Artifact "#view" (nnnotes.voices/1); facts rows, entries (rows per status), gaps, unreferenced (streams of the decoded sheets no row reaches). No items.""" @@ -929,7 +915,7 @@ def _plan(self, subject: str, env) -> tuple: ) if self._memo is not None and self._memo[0] == sig: return self._memo[1] - index = _address_index(env, link) + index = units.address_index(env, subject) tables = { i.role[len(views.MASTER_ROLE) :]: views._cached_rows(i.sha256, Path(i.locators[0]["path"]).read_bytes) for i in masters diff --git a/src/nnnotes/schemas/objects.schema.json b/src/nnnotes/schemas/objects.schema.json new file mode 100644 index 0000000..b9d006e --- /dev/null +++ b/src/nnnotes/schemas/objects.schema.json @@ -0,0 +1,89 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:nnnotes:schema:objects", + "title": "nnnotes.objects/1", + "description": "The object table of a product (pipeline.assemble): the objects reachable from its roots through the unity.export results, with their item status and files.", + "type": "object", + "required": [ + "schema", + "roots", + "objects", + "missing" + ], + "properties": { + "schema": { + "const": "nnnotes.objects/1" + }, + "roots": { + "type": "array", + "items": { + "type": "string" + } + }, + "objects": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "status" + ], + "properties": { + "status": { + "enum": [ + "exported", + "contained", + "generic", + "unsupported", + "failed" + ] + }, + "class": { + "type": "string" + }, + "name": { + "type": "string" + }, + "files": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "sha256", + "size", + "ext" + ], + "properties": { + "sha256": { + "$ref": "urn:nnnotes:schema:common#/$defs/sha256" + }, + "size": { + "type": "integer", + "minimum": 0 + }, + "ext": { + "type": "string", + "pattern": "^[0-9a-z]+$" + } + } + } + }, + "in": { + "type": "string", + "description": "The object whose file holds a contained object." + }, + "reason": { + "type": "string", + "description": "The reason code of a generic, unsupported or failed object." + } + } + } + }, + "missing": { + "type": "array", + "description": "Objects a reached object references that no census holds.", + "items": { + "type": "string" + } + } + } +} diff --git a/src/nnnotes/schemas/product.schema.json b/src/nnnotes/schemas/product.schema.json new file mode 100644 index 0000000..bb01211 --- /dev/null +++ b/src/nnnotes/schemas/product.schema.json @@ -0,0 +1,86 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:nnnotes:schema:product", + "title": "nnnotes.product/1", + "description": "The manifest of a product in a product layout: its files and the files of the objects it uses, each a content-addressed asset whose path is relative to the directory above the manifest's.", + "type": "object", + "required": [ + "schema", + "product", + "subject", + "files", + "objects" + ], + "properties": { + "schema": { + "const": "nnnotes.product/1" + }, + "product": { + "$ref": "urn:nnnotes:schema:common#/$defs/stageName" + }, + "subject": { + "type": "string" + }, + "files": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/asset" + } + }, + "objects": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "class": { + "type": "string" + }, + "name": { + "type": "string" + }, + "files": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/asset" + } + }, + "in": { + "type": "string" + } + }, + "oneOf": [ + { + "required": [ + "files" + ] + }, + { + "required": [ + "in" + ] + } + ] + } + } + }, + "$defs": { + "asset": { + "type": "object", + "required": [ + "asset", + "size" + ], + "additionalProperties": false, + "properties": { + "asset": { + "type": "string", + "pattern": "^assets/[0-9a-f]{64}\\.[0-9a-z]+$" + }, + "size": { + "type": "integer", + "minimum": 0 + } + } + } + } +} diff --git a/tests/pipeline/test_assemble.py b/tests/pipeline/test_assemble.py new file mode 100644 index 0000000..547d509 --- /dev/null +++ b/tests/pipeline/test_assemble.py @@ -0,0 +1,343 @@ +"""Product stages (pipeline.assemble) over the fake bundles of test_objexport: a toy product rooted at a prefab and +at an atlas sprite, its object table, its result and the product layout.""" + +from __future__ import annotations + +import json + +import pytest + +from nnnotes.pipeline import assemble, contract, layout, link +from nnnotes.pipeline.contract import Task +from nnnotes.pipeline.depot import Depot +from nnnotes.pipeline.objexport import ExportStage, SpriteCropStage +from nnnotes.pipeline.stages import Env, Pending, describe, execute, registry +from tests.pipeline.test_objexport import ATLAS_CAB, FAKE_READER, oid, planning_env + + +def addresses(depot, env, keys: dict) -> None: + """A committed link.addresses result whose table maps `keys` {catalog key: [object ids]}.""" + t = Task("link.addresses", 1, "main", {}, {}, ()) + doc = { + "schema": link.ADDRESSES, + "keys": {k: [{"objects": ids}] for k, ids in sorted(keys.items())}, + "objects": {}, + "files": {}, + "bundles": {}, + } + rec = contract.artifact( + contract.artifact_id(t.id, "addresses"), + depot.add(contract.encode(doc), "json"), + contract.provenance(t), + {"kind": "link.addresses", "format": "json"}, + ) + depot.commit(contract.result(t, [rec], [])) + env.done(t.id, t.key) + + +class Toy(assemble.ProductStage): + """A product whose roots are the objects of the keys of its subjects: `json` with the objects it uses and a + `data` file of theirs.""" + + name = "toy.thing" + version = 1 + KIND = "Toy" + KEYS: dict = {} + + def __init__(self, keys=None): + super().__init__() + self.KEYS = dict(Toy.KEYS if keys is None else keys) + + def keys(self, subject, env): + return [self.KEYS.get(subject, subject)] + + def assemble(self, task, objects): + files = { + "json": ( + contract.encode({"subject": task.subject, "objects": objects.ids()}), + "json", + "toy.json", + ) + } + for i, o in enumerate(sorted(objects.ids("TextAsset"))): + files[f"data/{i}"] = (objects.bytes(o, "data"), "txt", "toy.data") + return assemble.Assembly(files) + + +class Rooted(Toy): + """A Toy with root objects besides those of its keys.""" + + def __init__(self, keys, objects): + super().__init__(keys) + self.objects = list(objects) + + def root_objects(self, subject, env): + return self.objects + + +def env_for(depot, keys=None, names=("all", "atlas"), roots_objects=(), selected=None): + """A planning environment over the fake bundles with a committed address table and the facts of a request.""" + env = planning_env(depot, names, selected=selected) + env.facts["catalogs"] = {"main": []} + addresses(depot, env, keys if keys is not None else {}) + subjects = sorted(keys if keys is not None else {}) + env.facts["products"] = {"toy.thing": subjects} + env.facts["roots"] = {"keys": subjects, "objects": list(roots_objects)} + return env + + +def exported(depot, env) -> dict: + """Every unity.export task of the request, then every sprite.crop task: {stage: {subject: result}}.""" + out = {"unity.export": {}, "sprite.crop": {}} + for stage in (ExportStage(FAKE_READER), SpriteCropStage()): + for s in stage.subjects(env): + t = describe(stage, s, None, env) + execute(t, depot, registry(stage), force=True) + env.done(t.id, t.key) + out[stage.name][s] = depot.result(t.key) + return out + + +def product(depot, env, subject="card", keys=None, stage=None): + stage = stage or Toy(keys) + t = describe(stage, subject, None, env) + execute(t, depot, registry(stage), force=True) + return t, depot.result(t.key) + + +def table_of(depot, doc) -> dict: + rec = next(a for a in doc["artifacts"] if a["id"].endswith("#objects")) + return contract.loads(depot.read(rec["content"]["sha256"])) + + +# ---------------------------------------------------------------- the request and the table +def test_a_products_key_is_asked_for_before_anything_runs(tmp_path): + """request(): the roots fact of the products, from the keys of their subjects alone.""" + depot = Depot(tmp_path / "s") + stage = Toy({"m/a": "key/a", "m/b": "key/b"}) + stages = {"toy.thing": stage} + env = Env(depot, {"keys": {}}) + assert assemble.request(stages, {"toy.thing": ["m/b", "m/a"]}, env) == {"keys": ["key/a", "key/b"], "objects": []} + assert stage.subjects(Env(depot, {"products": {"toy.thing": ["m/b"]}})) == ["m/b"] + + +def test_the_object_table_is_what_the_root_reaches(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + t, doc = product(depot, env, "m/a", {"m/a": "key/a"}) + table = table_of(depot, doc) + assert table["roots"] == [oid(60)] + assert table["missing"] and all(m.startswith("CAB-scripts:") for m in table["missing"]) # no census of it + # the hierarchy: the root exported, its members contained in its prefab + assert table["objects"][oid(60)] == { + "status": "exported", + "class": "GameObject", + "name": "root", + "files": {"json": oid_file(table, oid(60), "json")}, + } + assert table["objects"][oid(61)]["status"] == "contained" and table["objects"][oid(61)]["in"] == oid(60) + assert table["objects"][oid(65)]["class"] == "MonoBehaviour" + # what the prefab references across bundles: the texture and the material + assert {"json", "image"} <= set(table["objects"][oid(10)]["files"]) + assert oid(70) not in table["objects"] # not referenced + item = doc["items"][0] + assert (item["object"], item["status"], item["class"]) == (t.id, "exported", "Toy") + assert item["artifacts"] == sorted(f"{t.id}#{p}" for p in ("json", "objects")) # no TextAsset is reached + + +def oid_file(table, o, role): + return table["objects"][o]["files"][role] + + +def test_a_reached_object_that_no_census_holds_is_missing(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}, roots_objects=["CAB-none:1"]) + exported(depot, env) + _, doc = product(depot, env, "m/a", {"m/a": "key/a"}, stage=Rooted({"m/a": "key/a"}, ["CAB-none:1"])) + table = table_of(depot, doc) + assert "CAB-none:1" in table["missing"] and "CAB-none:1" in table["roots"] + + +def test_a_root_of_a_unit_the_request_leaves_out_is_an_error(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}, selected=["atlas"]) + env.facts["roots"] = {"keys": [], "objects": []} # the request does not start from the product's key + with pytest.raises(ValueError, match="no requested unit"): + describe(Toy({"m/a": "key/a"}), "m/a", None, env) + + +def test_the_request_roots_select_the_units_and_nothing_else(tmp_path): + """A request for one prefab exports its units, not every unit of its bundle.""" + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}, names=("all",), selected=[]) + subjects = ExportStage(FAKE_READER).subjects(env) + assert subjects == ["all:10", "all:60"] # the prefab and the texture it references + + +def test_sprite_images_come_from_the_crop_of_their_atlas(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(13)]}, names=("all", "atlas"), roots_objects=[f"{ATLAS_CAB}:99"]) + exported(depot, env) + t, doc = product(depot, env, "m/a", {"m/a": "key/a"}) + table = table_of(depot, doc) + sprite = table["objects"][oid(13)] + assert set(sprite["files"]) == {"json", "image"} # the image from sprite.crop + assert table["objects"][f"{ATLAS_CAB}:99"]["class"] == "SpriteAtlas" + assert table["objects"][f"{ATLAS_CAB}:97"]["status"] == "exported" # the atlas references its own sprite + crop = [c for c in [*doc["artifacts"]] if c["semantics"]["kind"] == "sprite.meta"] + assert crop == [] # the sprite's meta belongs to its export result, not to the product + assert f"{t.id}#objects" in {a["id"] for a in doc["artifacts"]} + + +def test_depends_names_the_units_it_reads(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + deps = Toy({"m/a": "key/a"}).depends("m/a", env) + assert deps[0] == "link.addresses:main" + assert deps[1:] == ["unity.export:all:10", "unity.export:all:60"] # what it reads, not every unit + + +def test_inputs_and_the_key_follow_the_objects(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + stage = Toy({"m/a": "key/a"}) + t = describe(stage, "m/a", None, env) + assert [(i.role, i.size) for i in t.inputs] == [("objects", len(contract.encode(stage.reach("m/a", env).table)))] + assert t.context == {} and t.atoms == {} and t.params == {} + # another object of the same unit is not in the table: the key does not follow it + t2 = describe(Toy({"m/a": "key/a"}), "m/a", None, env) + assert t2.key == t.key and t2.cost.cpu_seconds > 0 + + +def test_a_product_waits_for_its_units(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + with pytest.raises(Pending, match="unity.export"): + describe(Toy({"m/a": "key/a"}), "m/a", None, env) + + +# ---------------------------------------------------------------- the result and the layout +def test_a_product_result_is_the_files_and_the_table(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + t, doc = product(depot, env, "m/a", {"m/a": "key/a"}) + body = contract.loads( + depot.read(next(a for a in doc["artifacts"] if a["id"] == f"{t.id}#json")["content"]["sha256"]) + ) + assert body["subject"] == "m/a" and body["objects"] + assert doc["status"] == "ok" + again = execute(t, depot, registry(Toy({"m/a": "key/a"})), force=True) + assert again.status == "ran" and contract.encode(depot.result(t.key)) == contract.encode(doc) + + +def test_an_unsupported_product_is_an_item(tmp_path): + class Picky(Toy): + def assemble(self, task, objects): + raise assemble.Unsupported("unsupported.toy", "nothing to make") + + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + t, doc = product(depot, env, "m/a", {"m/a": "key/a"}, stage=Picky({"m/a": "key/a"})) + assert doc["artifacts"] == [] + assert doc["items"] == [ + { + "object": t.id, + "status": "unsupported", + "class": "Toy", + "reason": {"code": "unsupported.toy", "message": "nothing to make"}, + } + ] + assert doc["status"] == "ok" # an unsupported product is a reported outcome, not a failure + + +def test_the_table_can_be_pruned_to_the_objects_the_product_uses(tmp_path): + class Pruned(Toy): + def assemble(self, task, objects): + made = super().assemble(task, objects) + return assemble.Assembly(made.files, {oid(60)}) + + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + _, doc = product(depot, env, "m/a", {"m/a": "key/a"}, stage=Pruned({"m/a": "key/a"})) + assert list(table_of(depot, doc)["objects"]) == [oid(60)] + + +def test_a_product_names_an_object_the_table_lacks(tmp_path): + class Wrong(Toy): + def assemble(self, task, objects): + made = super().assemble(task, objects) + return assemble.Assembly(made.files, {"CAB-none:1"}) + + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + t = describe(Wrong({"m/a": "key/a"}), "m/a", None, env) + with pytest.raises(ValueError, match="not in the table"): + execute(t, depot, registry(Wrong({"m/a": "key/a"})), force=True) + + +# ---------------------------------------------------------------- the layout +def manifest_of(depot, t, doc): + """(entries, the manifest) of a product layout for one result.""" + table = table_of(depot, doc) + stage = Toy({"m/a": "key/a"}) + path = stage.manifest("m/a") + body = contract.encode(layout.product_manifest(t.id, doc, table)) + sha = depot.put(body) + rec = {"id": f"{t.id}#manifest", "sha256": sha, "size": len(body)} + return layout.product_entries([(path, rec, doc, table)]), contract.loads(body) + + +def test_the_product_layout_places_the_files_and_the_objects(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + t, doc = product(depot, env, "m/a", {"m/a": "key/a"}) + entries, manifest = manifest_of(depot, t, doc) + place = {e["path"]: e["id"] for e in entries} + assert place["toy.thing/m%2Fa.json"] == f"{t.id}#manifest" + json_rec = next(a for a in doc["artifacts"] if a["id"] == f"{t.id}#json") + assert place[f"assets/{json_rec['content']['sha256']}.json"] == f"{t.id}#json" + assert place[f"assets/{oid_file(table_of(depot, doc), oid(10), 'image')['sha256']}.png"] == f"{oid(10)}#image" + assert manifest["files"]["json"] == { + "asset": f"assets/{json_rec['content']['sha256']}.json", + "size": json_rec["content"]["size"], + } + assert manifest["objects"][oid(10)]["class"] == "Texture2D" + assert manifest["objects"][oid(61)]["in"] == oid(60) + assert "assets" in manifest["objects"][oid(10)]["files"]["image"]["asset"] + # the manifest path is one directory and one file + assert len(entries) == len({(e["path"], e["id"]) for e in entries}) + + +def test_the_product_layout_refuses_a_nested_manifest_path(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + t, doc = product(depot, env, "m/a", {"m/a": "key/a"}) + table = table_of(depot, doc) + body = contract.encode(layout.product_manifest(t.id, doc, table)) + rec = {"id": f"{t.id}#manifest", "sha256": depot.put(body), "size": len(body)} + with pytest.raises(ValueError, match="manifest path"): + layout.product_entries([("a/b/c.json", rec, doc, table)]) + + +def test_the_product_layout_writes_and_hits(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + t, doc = product(depot, env, "m/a", {"m/a": "key/a"}) + entries, _ = manifest_of(depot, t, doc) + w = layout.materialize(tmp_path / "out", "product", {}, entries, layout.from_depot(depot)) + assert w["write"] == len(entries) and w["remove"] == 0 + assert (tmp_path / "out" / "toy.thing" / "m%2Fa.json").is_file() + assert layout.diff(tmp_path / "out", entries) == {"write": 0, "remove": 0, "keep": len(entries)} + assert json.loads((tmp_path / "out" / "toy.thing" / "m%2Fa.json").read_text(encoding="utf-8"))["schema"] == ( + contract.PRODUCT + )