diff --git a/docs/contracts.md b/docs/contracts.md index 369222e..dcd8fe2 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -202,6 +202,13 @@ bytes nor keys; the file names of `original` and `cas` both follow from the reco `original` manifest alone. Documents derived for `original` from records and paths (the views with the files of their objects) are the exception: `original` has the derived documents, `cas` the artifacts they are made from. +## Product manifest: nnnotes.product/1 + +A product layout writes, for each product of `pipeline.assemble`, a manifest +([product.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/product.schema.json)): the product's +files and the files of the objects it uses, each a content-addressed `assets/.`. The product's object +table is `nnnotes.objects/1`. See [products](products.md). + ## Reports - Coverage, `nnnotes.coverage/1` ([coverage.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/coverage.schema.json)): per class the census @@ -224,6 +231,7 @@ Documents written by particular stages (artifacts of no object: their ids are `< | `nnnotes.scripts/1` | `link.scripts` | [scripts.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/scripts.schema.json) | | `nnnotes.addresses/1` | `link.addresses` | [addresses.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/addresses.schema.json) | | `nnnotes.view/1` | `view.` | [view.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/view.schema.json) | +| `nnnotes.objects/1` | `pipeline.assemble` (a product's object table) | [objects.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/objects.schema.json) | The catalog versions of a depot, `nnnotes.catalogs/1` ([catalogs.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/catalogs.schema.json), `/catalogs/index.json`), and the difference between two of them, `nnnotes.catalog-diff/1` diff --git a/docs/products.md b/docs/products.md new file mode 100644 index 0000000..8606ccd --- /dev/null +++ b/docs/products.md @@ -0,0 +1,83 @@ +# Products + +A product is something a player or a site reads as a whole: a Live2D model, a story episode, a live. Products are +assembled from the results of `unity.export` ([stages](stages.md)), never from a bundle: by the time a product is +made, every object it uses has been converted once, and the product combines those conversions with what it adds +itself. + +## Product stages + +A product stage (`nnnotes.pipeline.assemble.ProductStage`) has one task per subject. Its task starts from root +objects: + +- the objects of the catalog keys the stage gives for the subject (`keys`), through the address table of + `link.addresses`; +- the object ids the stage names besides (`root_objects`). + +Both are read from the planning facts alone, so a command can ask for what its products need before anything runs: +`assemble.request(stages, products, env)` gives the fact `roots` (`{"keys", "objects"}`), and the request of +`unity.export` then holds the units of those objects and every unit reachable from them through the census `refs` +([units](stages.md#units)), and nothing else of their bundles. + +What a task reaches: the units holding its roots, then every unit that an item of a reached unit references (the +item's `refs`), through the `unity.export` results. A sprite whose atlas is in another bundle has its `image` from the +`sprite.crop` result of that atlas. A reached unit that the request did not export is an error of the request; an +object that a reached object references and no census holds is listed as `missing`. + +| Part of the task | | +|---|---| +| inputs | `objects`: the object table of what it reaches (`nnnotes.objects/1`); the stage's own inputs (master tables, data its parameters name) | +| depends | `link.addresses` (when it has keys), the `unity.export` and `sprite.crop` tasks whose results it read | +| artifacts | `#`: the product's own files; `#objects`: the object table of the objects its files use (the whole table unless the stage narrows it) | +| item | one for the subject: `exported`, or `unsupported` with a reason code | + +The key holds the object table, so a product is assembled again exactly when a file of an object it reaches +changes, or what it reaches changes. + +## Object table: nnnotes.objects/1 + +([objects.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/objects.schema.json)) + +```json +{"schema": "nnnotes.objects/1", + "roots": ["CAB-…:-4153…"], + "objects": { + "CAB-…:-4153…": {"status": "exported", "class": "GameObject", "name": "model", + "files": {"json": {"sha256": "…", "size": 5120, "ext": "json"}}}, + "CAB-…:12": {"status": "contained", "class": "Transform", "in": "CAB-…:-4153…"}, + "CAB-…:40": {"status": "exported", "class": "Texture2D", "name": "texture_00", + "files": {"image": {"sha256": "…", "size": 812345, "ext": "png"}, + "json": {"sha256": "…", "size": 640, "ext": "json"}}}}, + "missing": []} +``` + +Each object has the status of its item (the most severe when two stages made items for it), its class and name +where known, its files by role (the artifacts of its items), the object whose file holds it (`in`, a contained +object), or the reason code of a generic, unsupported or failed object. + +## Product layout + +A product layout places products for a player: + +- every file of a product and every file of the objects of its object table at `assets/.` (a file + several products or objects share is written once); +- each product's manifest at `/.json` (the stage's `manifest(subject)`, by default + `/.json` with the subject escaped as one path segment). + +## Product manifest: nnnotes.product/1 + +([product.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/product.schema.json)) + +```json +{"schema": "nnnotes.product/1", "product": "live2d.model", "subject": "…", + "files": {"model.json": {"asset": "assets/.json", "size": 2048}}, + "objects": { + "CAB-…:-4153…": {"class": "GameObject", "name": "model", + "files": {"json": {"asset": "assets/.json", "size": 5120}}}, + "CAB-…:12": {"class": "Transform", "in": "CAB-…:-4153…"}}} +``` + +`files` are the product's own files by path; `objects` are the objects of its object table that have files, or are +contained in an object that has one. An asset path is relative to the directory above the manifest's directory, as +in the chart manifests of ournotes-player. A reference in an object's JSON (`{"$ref": {"file", "pathId"}}`) names +the object `:` of `objects`. diff --git a/docs/schemas.md b/docs/schemas.md index 0fa3405..fd0a6b7 100644 --- a/docs/schemas.md +++ b/docs/schemas.md @@ -26,7 +26,9 @@ schemas.load("plan") # the schema as a dict | `failures` | `nnnotes.failures/1` | the failed tasks and items of a run | | `layout` | `nnnotes.layout/1` | the files a layout owns in its output directory | | `music-data` | `nnnotes.music-data/2` | every song and chart of one master data version ([music data](music-data.md)) | +| `objects` | `nnnotes.objects/1` | the objects a product reaches: their class, item status and files ([products](products.md)) | | `plan` | `nnnotes.plan/1` | what a run would do and why (`depot plan`) | +| `product` | `nnnotes.product/1` | a product's manifest: its files and the files of the objects it uses, as content-addressed assets ([products](products.md)) | | `result` | `nnnotes.result/1` | the result of a task, a function of its key alone | | `run` | `nnnotes.run/1` | a run manifest: context, selection, tasks with keys and statuses, layouts | | `scripts` | `nnnotes.scripts/1` | every MonoScript of the censuses, by object id | diff --git a/docs/stages.md b/docs/stages.md index 7d46629..bc7eb16 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -109,10 +109,10 @@ hierarchy. Its subject is the stable address of its root object, `#", the +table of the objects they use as "#objects", and one item for the subject. A product layout +(layout.product_entries) places the files and the objects' files and writes the product's manifest. +""" + +from __future__ import annotations + +from collections import OrderedDict +from dataclasses import dataclass, field + +from ..unity.atoms import Unsupported +from . import contract, units +from .contract import Cost, Input +from .layout import PRODUCT_TABLE, escape_segment +from .orchestrate import SEVERITY +from .stages import Output, Stage + +EXPORT, CROP = "unity.export", "sprite.crop" +ADDRESSES = units.ADDRESSES +TABLE_KIND = "product.objects" + + +def request(stages: dict, products: dict, env) -> dict: + """The fact "roots" (pipeline.units) of a request for `products` {stage name: subjects}: the catalog keys and the + root objects of every subject, sorted.""" + keys, objects = set(), set() + for name, subjects in sorted(products.items()): + stage = stages[name] + for s in subjects: + keys.update(stage.keys(s, env)) + objects.update(stage.root_objects(s, env)) + return {"keys": sorted(keys), "objects": sorted(objects)} + + +def _entry(item: dict, arts: dict) -> dict: + """The object table entry of a unity.export or sprite.crop item.""" + e: dict = {"status": item["status"]} + cls, name, files = item.get("class"), None, {} + for aid in item.get("artifacts", ()): + a = arts[aid] + c = a["content"] + files[contract.parse_artifact_id(aid)[1]] = {"sha256": c["sha256"], "size": c["size"], "ext": c["ext"]} + obj = (a.get("provenance") or {}).get("object") or {} + cls, name = cls or obj.get("class"), name or obj.get("name") + if cls: + e["class"] = cls + if name: + e["name"] = name + if files: + e["files"] = files + if "in" in item: + e["in"] = contract.parse_artifact_id(item["in"])[0] + if "reason" in item: + e["reason"] = item["reason"]["code"] + return e + + +def _merge(e: dict, other: dict) -> None: + """Add the files of another item of the same object; the status becomes the more severe.""" + e.setdefault("files", {}).update(other.get("files", {})) + if SEVERITY.index(other["status"]) < SEVERITY.index(e["status"]): + e["status"] = other["status"] + if "reason" in other: + e["reason"] = other["reason"] + + +@dataclass +class Reach: + """What a product reaches: its object table (nnnotes.objects/1), the tasks whose results it read, the bytes of + its objects' files.""" + + table: dict + read: list = field(default_factory=list) + size: int = 0 + + +class _Results: + """The items and artifact records of the results read, by result key (a result never changes).""" + + def __init__(self, size: int = 4096): + self.size, self._docs = size, OrderedDict() + + def get(self, env, tid: str) -> tuple[list, dict]: + key = env.key(tid) + doc = self._docs.get(key) + if doc is None: + res = env.result(tid) + doc = self._docs[key] = (res["items"], {a["id"]: a for a in res["artifacts"]}) + while len(self._docs) > self.size: + self._docs.popitem(last=False) + else: + self._docs.move_to_end(key) + return doc + + +def reach(env, roots, results: _Results | None = None) -> Reach: + """The objects reachable from the object ids `roots` (module documentation).""" + results = results or _Results() + table = units.table(env) + todo, seen, missing = [], set(), set() + for o in roots: + s = table.subject_of(o) + if s is None: + missing.add(o) + else: + todo.append(s) + objects: dict[str, dict] = {} + read = [] + crops: dict[str, list] = {} # atlas object id -> sprite object ids + while todo: + s = todo.pop() + if s in seen: + continue + seen.add(s) + table.unit(s) # ValueError: a unit the request does not export + tid = contract.task_id(EXPORT, s) + items, arts = results.get(env, tid) + read.append(tid) + for it in items: + oid = it["object"] + objects[oid] = _entry(it, arts) + for aid in it.get("artifacts", ()): + facts = arts[aid]["semantics"].get("facts") or {} + if facts.get("crop") == CROP: + crops.setdefault(facts["atlas"], []).append(oid) + for r in it.get("refs", ()): + t = table.subject_of(r) + if t is None: + missing.add(r) + elif t not in seen: + todo.append(t) + for atlas, sprites in sorted(crops.items()): + s = table.subject_of(atlas) + tid = contract.task_id(CROP, s) if s is not None else None + if tid is None or not env.known(tid): # the atlas is not exported: its sprites have no image + missing.add(atlas) + continue + items, arts = results.get(env, tid) + read.append(tid) + made = {it["object"]: it for it in items} + for oid in sprites: + if oid in made: + _merge(objects[oid], _entry(made[oid], arts)) + doc = { + "schema": contract.OBJECTS, + "roots": sorted(set(roots)), + "objects": dict(sorted(objects.items())), + "missing": sorted(missing), + } + size = sum(f["size"] for e in objects.values() for f in e.get("files", {}).values()) + return Reach(doc, sorted(read), size) + + +class Objects: + """The object table of a product task and the bytes of its objects' files (from the depot).""" + + def __init__(self, doc: dict, depot): + self.doc, self.depot = doc, depot + self.roots: list[str] = doc["roots"] + self._objects: dict = doc["objects"] + self._json: dict = {} + + def __contains__(self, oid: str) -> bool: + return oid in self._objects + + def ids(self, cls: str | None = None) -> list[str]: + """The object ids of the table (of class `cls`), sorted.""" + return [o for o, e in self._objects.items() if cls is None or e.get("class") == cls] + + def info(self, oid: str) -> dict: + try: + return self._objects[oid] + except KeyError: + raise KeyError(f"object {oid} is not in the product's object table") from None + + def bytes(self, oid: str, role: str) -> bytes: + files = self.info(oid).get("files") or {} + if role not in files: + raise KeyError(f"object {oid} has no {role!r} file") + return self.depot.read(files[role]["sha256"]) + + def json(self, oid: str, role: str = "json"): + """The decoded JSON of a file of an object (kept for the task).""" + k = (oid, role) + if k not in self._json: + self._json[k] = contract.loads(self.bytes(oid, role)) + return self._json[k] + + +@dataclass +class Assembly: + """What `assemble` makes: the product's files {path: (bytes, extension, kind)} and the object ids its files use + (None: every object of the table).""" + + files: dict + objects: set | None = None + + +class ProductStage(Stage): + """Base of the product stages (module documentation). Subclasses set `name`, `version` and `KIND`, and define + `keys` and/or `root_objects` and `assemble`; `subjects` defaults to the subjects of the fact "products".""" + + after = (ADDRESSES, EXPORT, CROP) + KIND = "product" + + def __init__(self): + self._memo: dict = {} + self._results = _Results() + + def __getstate__(self): + return {k: v for k, v in self.__dict__.items() if k not in ("_memo", "_results")} + + def __setstate__(self, state): + self.__dict__.update(state) + self._memo, self._results = {}, _Results() + + # ---------------------------------------------------------------- what a subclass defines + def subjects(self, env) -> list[str]: + return sorted((env.facts.get("products") or {}).get(self.name, ())) + + def keys(self, subject: str, env) -> list[str]: + """The catalog keys whose objects are roots of `subject` (from the facts alone).""" + return [] + + def root_objects(self, subject: str, env) -> list[str]: + """Object ids that are roots of `subject` besides those of its keys (from the facts alone).""" + return [] + + def more_inputs(self, subject: str, env) -> list[Input]: + """Inputs besides the object table.""" + return [] + + def assemble(self, task, objects: Objects) -> Assembly: + raise NotImplementedError + + def manifest(self, subject: str) -> str: + """The path of the subject's manifest in a product layout: one directory, then a file.""" + return f"{self.name}/{escape_segment(subject)}.json" + + # ---------------------------------------------------------------- planning + def reach(self, subject: str, env) -> Reach: + """What `subject` reaches (kept while the results it read are the same).""" + roots = units.root_objects(env, self.keys(subject, env), self.root_objects(subject, env)) + sig = (tuple(roots), id(units.table(env))) + memo = self._memo.get(subject) + if memo is not None and memo[0] == sig and all(env.key(t) == k for t, k in memo[1]): + return memo[2] + r = reach(env, roots, self._results) + self._memo[subject] = (sig, [(t, env.key(t)) for t in r.read], r) + return r + + def inputs(self, subject: str, env) -> list[Input]: + data = contract.encode(self.reach(subject, env).table) + table = Input("objects", env.depot.put(data), len(data), None, ({"kind": "depot"},)) + return [table, *self.more_inputs(subject, env)] + + def depends(self, subject: str, env) -> list[str]: + own = [contract.task_id(ADDRESSES, s) for s in sorted(env.fact("catalogs"))] if self.keys(subject, env) else [] + return own + self.reach(subject, env).read + + def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: + """From the bytes of the objects' files: a product reads each of them about once.""" + size = self.reach(subject, env).size + sum(i.size for i in inputs) + return Cost(0.02 + size / 100e6, (64 << 20) + 2 * size) + + # ---------------------------------------------------------------- running + def run(self, task, depot) -> Output: + data = depot.input_bytes(task.input("objects")) + doc = contract.loads(data) + try: + made = self.assemble(task, Objects(doc, depot)) + except Unsupported as e: + why = contract.reason(e.code, e.message) + return Output([], [contract.item(task.id, "unsupported", why=why, cls=self.KIND)]) + prov = contract.provenance(task) + arts = [] + for path, (body, ext, kind) in sorted(made.files.items()): + if path == PRODUCT_TABLE: + raise ValueError(f"{task.id}: a product file cannot be named {PRODUCT_TABLE!r}") + aid = contract.artifact_id(task.id, path) + arts.append(contract.artifact(aid, depot.add(body, ext), prov, {"kind": kind})) + if made.objects is not None: + unknown = sorted(set(made.objects) - set(doc["objects"])) + if unknown: + raise ValueError(f"{task.id}: objects not in the table: {', '.join(unknown[:5])}") + doc = dict(doc, objects={o: e for o, e in doc["objects"].items() if o in made.objects}) + data = contract.encode(doc) + aid = contract.artifact_id(task.id, PRODUCT_TABLE) + arts.append(contract.artifact(aid, depot.add(data, "json"), prov, {"kind": TABLE_KIND})) + item = contract.item(task.id, "exported", artifacts=[a["id"] for a in arts], cls=self.KIND) + return Output(arts, [item]) diff --git a/src/nnnotes/pipeline/contract.py b/src/nnnotes/pipeline/contract.py index 97ff7f6..bf25f02 100644 --- a/src/nnnotes/pipeline/contract.py +++ b/src/nnnotes/pipeline/contract.py @@ -55,6 +55,8 @@ CATALOG_INDEX = "nnnotes.catalog-index/1" CENSUS = "nnnotes.census/2" VIEW = "nnnotes.view/1" +OBJECTS = "nnnotes.objects/1" +PRODUCT = "nnnotes.product/1" # item statuses of a result: every object a task covers has exactly one ITEM_STATUSES = ("exported", "contained", "generic", "unsupported", "failed") diff --git a/src/nnnotes/pipeline/layout.py b/src/nnnotes/pipeline/layout.py index 3fbb13d..b20eb4d 100644 --- a/src/nnnotes/pipeline/layout.py +++ b/src/nnnotes/pipeline/layout.py @@ -27,6 +27,9 @@ - paths that coincide ignoring case (or a file named like another path's directory) keep the first by path and id; each other gets `~` before its extension and is reported; artifacts with the same path and the same content share the file +`product` (product_entries): the products of pipeline.assemble. Every file of a product and of the objects of its +object table is at `assets/.`; each product has a manifest (nnnotes.product/1, product_manifest) at +`/.json` (ProductStage.manifest), whose asset paths are relative to the directory above its own. So every path ends with its artifact's extension, and `cas` follows from the `original` manifest alone; `original` follows from `cas` plus the results (the records). Documents a stage derives for `original` from its records and the paths (the views with the files of their objects) are the exception: `original` has the derived documents, @@ -63,6 +66,7 @@ MANIFEST = "manifest.json" JOURNAL = ".nnnotes-layout-journal.json" LAYOUTS = ("original", "cas") +PRODUCT_TABLE = "objects" # the role of a product's object table (nnnotes.objects/1) among its artifacts PLACEMENTS = ("clone", "hard", "copy") # how a file is made from its byte source LINKS = ("auto", *PLACEMENTS) # what a run may ask for: auto probes FICLONE = 0x40049409 # Linux ioctl: make a file share another's blocks @@ -353,6 +357,59 @@ def _resolve(placed: list) -> tuple[list[dict], dict]: } +def _asset(c: dict) -> dict: + return {"asset": f"assets/{c['sha256']}.{c['ext']}", "size": c["size"]} + + +def product_manifest(task: str, result: dict, table: dict) -> dict: + """The manifest (nnnotes.product/1) of a product: its files {path: {asset, size}} and the objects of its object + table `table` that have files ({class, name, files: {role: {asset, size}}}) or are contained in one ({class, + in}).""" + stage, subject = contract.parse_task_id(task) + files = {} + for a in result["artifacts"]: + role = contract.parse_artifact_id(a["id"])[1] + if role != PRODUCT_TABLE: + files[role] = _asset(a["content"]) + objects = {} + for oid, o in table["objects"].items(): + e = {k: o[k] for k in ("class", "name") if k in o} + if o.get("files"): + e["files"] = {r: _asset(f) for r, f in sorted(o["files"].items())} + elif "in" in o: + e["in"] = o["in"] + else: + continue + objects[oid] = e + return {"schema": contract.PRODUCT, "product": stage, "subject": subject, "files": files, "objects": objects} + + +def product_entries(products) -> list[dict]: + """The entries of the `product` layout: `products` [(manifest path, the manifest's record {id, sha256, size}, + result, object table)].""" + out: dict[tuple, dict] = {} + for path, manifest, result, table in products: + segs = path.split("/") + if len(segs) != 2 or not all(segs) or not segs[1].endswith(".json"): + raise ValueError(f"product manifest path {path!r}: expected /.json") + out[(path, manifest["id"])] = { + "path": path, + "sha256": manifest["sha256"], + "size": manifest["size"], + "id": manifest["id"], + } + for a in result["artifacts"]: + if contract.parse_artifact_id(a["id"])[1] != PRODUCT_TABLE: + e = _entry(_asset(a["content"])["asset"], a) + out[(e["path"], e["id"])] = e + for oid, o in table["objects"].items(): + for role, f in (o.get("files") or {}).items(): + aid = contract.artifact_id(oid, role) + p = f"assets/{f['sha256']}.{f['ext']}" + out[(p, aid)] = {"path": p, "sha256": f["sha256"], "size": f["size"], "id": aid} + return [out[k] for k in sorted(out)] + + def entries(name: str, sources, **kw) -> tuple[list[dict], dict]: """The entries and report of layout `name`.""" if name == "cas": diff --git a/src/nnnotes/pipeline/units.py b/src/nnnotes/pipeline/units.py index bfce293..a82dff1 100644 --- a/src/nnnotes/pipeline/units.py +++ b/src/nnnotes/pipeline/units.py @@ -5,11 +5,15 @@ path-id stable address of its root object (link.path_address: ":", with the file after the bundle in a bundle of several serialized files), which stays when the bundle's content changes; its object ids do not. -The request (`table`): every unit holding an object of the selected bundles (fact "selected"), and every unit -reachable from those through what their objects reference (census `refs`) across the censuses of the bundles of -the fact "bundles"; with the fact "classes" (a list of class names; absent or None: every class), only the units -whose root object is of those classes. A bundle whose census failed has no units; a reference into a file no -census holds is left out. +The request (`table`): every unit holding an object of the selected bundles (fact "selected") or a root object of the +fact "roots", and every unit reachable from those through what their objects reference (census `refs`) across the +censuses of the bundles of the fact "bundles"; with the fact "classes" (a list of class names; absent or None: +every class), only the units whose root object is of those classes. A bundle whose census failed has no units; a +reference into a file no census holds is left out. + +The fact "roots" (absent: none) is {"keys": catalog keys, "objects": object ids}: the objects of each key (all those +the address table of link.addresses lists for it; a key without objects adds none) and the objects named. The +products of a request start from them (pipeline.assemble). """ from __future__ import annotations @@ -17,11 +21,12 @@ from collections import Counter from dataclasses import dataclass -from . import contract, link +from . import contract, link, views from .census import CensusStage from .stages import Pending CENSUS = CensusStage.name +ADDRESSES = views.ADDRESSES_STAGE @dataclass(frozen=True) @@ -131,9 +136,9 @@ def _census_units(stable: str, doc: dict, need_refs: bool): return units, of, {f["name"]: stable for f in doc["files"]}, refs -def build(censuses, bundles, selected, classes=None) -> Units: - """The Units of a request over `censuses` [(stable bundle name, census document)] of the bundles `bundles` - (module documentation).""" +def build(censuses, bundles, selected, classes=None, roots=()) -> Units: + """The Units of a request over `censuses` [(stable bundle name, census document)] of the bundles `bundles`, from + the units of the bundles `selected` and of the objects `roots` (module documentation).""" selected = set(selected) every = selected >= set(bundles) # no reference can add a unit: none is read units: dict[str, Unit] = {} @@ -152,6 +157,7 @@ def build(censuses, bundles, selected, classes=None) -> Units: want = set(units) else: want = {s for s, u in units.items() if u.bundle in selected} + want.update(of[o] for o in roots if o in of) todo = list(want) while todo: for oid in refs.get(todo.pop(), ()): @@ -166,12 +172,36 @@ def build(censuses, bundles, selected, classes=None) -> Units: _MEMO: list = [] # [(signature, Units)] of the last table +_ADDRESSES: dict = {} # {sha256: AddressIndex} of the last address table read + + +def address_index(env, subject: str) -> views.AddressIndex: + """The AddressIndex of the result of link.addresses for the catalog `subject` (Pending while it has not run).""" + sha = env.artifact(contract.task_id(ADDRESSES, subject), "addresses")["content"]["sha256"] + index = _ADDRESSES.get(sha) + if index is None: + index = views.AddressIndex.from_addresses(contract.loads(env.depot.read(sha))) + _ADDRESSES.clear() + _ADDRESSES[sha] = index + return index + + +def root_objects(env, keys=(), objects=()) -> list[str]: + """The object ids of the catalog `keys` (through the address tables of the fact "catalogs") and the `objects`, + sorted.""" + out = set(objects) + if keys: + for subject in sorted(env.fact("catalogs")): + index = address_index(env, subject) + for k in keys: + out.update(o.id for o in index.objects(k) or ()) + return sorted(out) def table(env) -> Units: """The Units of the request of a planning environment: the censuses of the fact "bundles" (Pending while one - has not run; a failed one is skipped), the facts "selected" and "classes". Kept for the planning state it was - made in.""" + has not run; a failed one is skipped), the facts "selected", "roots" (Pending while link.addresses has not run, + when it names keys) and "classes". Kept for the planning state it was made in.""" bundles = sorted(env.fact("bundles")) keys = [] for b in bundles: @@ -182,7 +212,15 @@ def table(env) -> Units: if not p.failed: raise classes = env.facts.get("classes") - sig = (id(env.depot), tuple(keys), tuple(env.fact("selected")), None if classes is None else tuple(classes)) + request = env.facts.get("roots") or {} + roots = root_objects(env, request.get("keys", ()), request.get("objects", ())) + sig = ( + id(env.depot), + tuple(keys), + tuple(env.fact("selected")), + None if classes is None else tuple(classes), + tuple(roots), + ) if _MEMO and _MEMO[0][0] == sig: return _MEMO[0][1] @@ -193,6 +231,6 @@ def censuses(): rec = next(a for a in env.depot.committed(key, tid)["artifacts"] if a["id"] == aid) yield b, contract.loads(env.depot.read(rec["content"]["sha256"])) - t = build(censuses(), bundles, env.fact("selected"), classes) + t = build(censuses(), bundles, env.fact("selected"), classes, roots) _MEMO[:] = [(sig, t)] return t diff --git a/src/nnnotes/pipeline/voices.py b/src/nnnotes/pipeline/voices.py index 147b060..d6c519c 100644 --- a/src/nnnotes/pipeline/voices.py +++ b/src/nnnotes/pipeline/voices.py @@ -820,20 +820,6 @@ def _sheet_name(name: str) -> str | None: return name if "/" not in name else None -_address_cache: dict = {} # content id of an address table -> its AddressIndex, the latest one - - -def _address_index(env, tid: str): - """The AddressIndex of the result of the link.addresses task `tid`.""" - sha = env.artifact(tid, "addresses")["content"]["sha256"] - hit = _address_cache.get(sha) - if hit is None: - hit = views.AddressIndex.from_addresses(contract.loads(env.depot.read(sha))) - _address_cache.clear() - _address_cache[sha] = hit - return hit - - class VoiceStage(Stage): """view.voices (module documentation). Artifact "#view" (nnnotes.voices/1); facts rows, entries (rows per status), gaps, unreferenced (streams of the decoded sheets no row reaches). No items.""" @@ -929,7 +915,7 @@ def _plan(self, subject: str, env) -> tuple: ) if self._memo is not None and self._memo[0] == sig: return self._memo[1] - index = _address_index(env, link) + index = units.address_index(env, subject) tables = { i.role[len(views.MASTER_ROLE) :]: views._cached_rows(i.sha256, Path(i.locators[0]["path"]).read_bytes) for i in masters diff --git a/src/nnnotes/schemas/objects.schema.json b/src/nnnotes/schemas/objects.schema.json new file mode 100644 index 0000000..b9d006e --- /dev/null +++ b/src/nnnotes/schemas/objects.schema.json @@ -0,0 +1,89 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:nnnotes:schema:objects", + "title": "nnnotes.objects/1", + "description": "The object table of a product (pipeline.assemble): the objects reachable from its roots through the unity.export results, with their item status and files.", + "type": "object", + "required": [ + "schema", + "roots", + "objects", + "missing" + ], + "properties": { + "schema": { + "const": "nnnotes.objects/1" + }, + "roots": { + "type": "array", + "items": { + "type": "string" + } + }, + "objects": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "status" + ], + "properties": { + "status": { + "enum": [ + "exported", + "contained", + "generic", + "unsupported", + "failed" + ] + }, + "class": { + "type": "string" + }, + "name": { + "type": "string" + }, + "files": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "sha256", + "size", + "ext" + ], + "properties": { + "sha256": { + "$ref": "urn:nnnotes:schema:common#/$defs/sha256" + }, + "size": { + "type": "integer", + "minimum": 0 + }, + "ext": { + "type": "string", + "pattern": "^[0-9a-z]+$" + } + } + } + }, + "in": { + "type": "string", + "description": "The object whose file holds a contained object." + }, + "reason": { + "type": "string", + "description": "The reason code of a generic, unsupported or failed object." + } + } + } + }, + "missing": { + "type": "array", + "description": "Objects a reached object references that no census holds.", + "items": { + "type": "string" + } + } + } +} diff --git a/src/nnnotes/schemas/product.schema.json b/src/nnnotes/schemas/product.schema.json new file mode 100644 index 0000000..bb01211 --- /dev/null +++ b/src/nnnotes/schemas/product.schema.json @@ -0,0 +1,86 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:nnnotes:schema:product", + "title": "nnnotes.product/1", + "description": "The manifest of a product in a product layout: its files and the files of the objects it uses, each a content-addressed asset whose path is relative to the directory above the manifest's.", + "type": "object", + "required": [ + "schema", + "product", + "subject", + "files", + "objects" + ], + "properties": { + "schema": { + "const": "nnnotes.product/1" + }, + "product": { + "$ref": "urn:nnnotes:schema:common#/$defs/stageName" + }, + "subject": { + "type": "string" + }, + "files": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/asset" + } + }, + "objects": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "class": { + "type": "string" + }, + "name": { + "type": "string" + }, + "files": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/asset" + } + }, + "in": { + "type": "string" + } + }, + "oneOf": [ + { + "required": [ + "files" + ] + }, + { + "required": [ + "in" + ] + } + ] + } + } + }, + "$defs": { + "asset": { + "type": "object", + "required": [ + "asset", + "size" + ], + "additionalProperties": false, + "properties": { + "asset": { + "type": "string", + "pattern": "^assets/[0-9a-f]{64}\\.[0-9a-z]+$" + }, + "size": { + "type": "integer", + "minimum": 0 + } + } + } + } +} diff --git a/tests/pipeline/test_assemble.py b/tests/pipeline/test_assemble.py new file mode 100644 index 0000000..547d509 --- /dev/null +++ b/tests/pipeline/test_assemble.py @@ -0,0 +1,343 @@ +"""Product stages (pipeline.assemble) over the fake bundles of test_objexport: a toy product rooted at a prefab and +at an atlas sprite, its object table, its result and the product layout.""" + +from __future__ import annotations + +import json + +import pytest + +from nnnotes.pipeline import assemble, contract, layout, link +from nnnotes.pipeline.contract import Task +from nnnotes.pipeline.depot import Depot +from nnnotes.pipeline.objexport import ExportStage, SpriteCropStage +from nnnotes.pipeline.stages import Env, Pending, describe, execute, registry +from tests.pipeline.test_objexport import ATLAS_CAB, FAKE_READER, oid, planning_env + + +def addresses(depot, env, keys: dict) -> None: + """A committed link.addresses result whose table maps `keys` {catalog key: [object ids]}.""" + t = Task("link.addresses", 1, "main", {}, {}, ()) + doc = { + "schema": link.ADDRESSES, + "keys": {k: [{"objects": ids}] for k, ids in sorted(keys.items())}, + "objects": {}, + "files": {}, + "bundles": {}, + } + rec = contract.artifact( + contract.artifact_id(t.id, "addresses"), + depot.add(contract.encode(doc), "json"), + contract.provenance(t), + {"kind": "link.addresses", "format": "json"}, + ) + depot.commit(contract.result(t, [rec], [])) + env.done(t.id, t.key) + + +class Toy(assemble.ProductStage): + """A product whose roots are the objects of the keys of its subjects: `json` with the objects it uses and a + `data` file of theirs.""" + + name = "toy.thing" + version = 1 + KIND = "Toy" + KEYS: dict = {} + + def __init__(self, keys=None): + super().__init__() + self.KEYS = dict(Toy.KEYS if keys is None else keys) + + def keys(self, subject, env): + return [self.KEYS.get(subject, subject)] + + def assemble(self, task, objects): + files = { + "json": ( + contract.encode({"subject": task.subject, "objects": objects.ids()}), + "json", + "toy.json", + ) + } + for i, o in enumerate(sorted(objects.ids("TextAsset"))): + files[f"data/{i}"] = (objects.bytes(o, "data"), "txt", "toy.data") + return assemble.Assembly(files) + + +class Rooted(Toy): + """A Toy with root objects besides those of its keys.""" + + def __init__(self, keys, objects): + super().__init__(keys) + self.objects = list(objects) + + def root_objects(self, subject, env): + return self.objects + + +def env_for(depot, keys=None, names=("all", "atlas"), roots_objects=(), selected=None): + """A planning environment over the fake bundles with a committed address table and the facts of a request.""" + env = planning_env(depot, names, selected=selected) + env.facts["catalogs"] = {"main": []} + addresses(depot, env, keys if keys is not None else {}) + subjects = sorted(keys if keys is not None else {}) + env.facts["products"] = {"toy.thing": subjects} + env.facts["roots"] = {"keys": subjects, "objects": list(roots_objects)} + return env + + +def exported(depot, env) -> dict: + """Every unity.export task of the request, then every sprite.crop task: {stage: {subject: result}}.""" + out = {"unity.export": {}, "sprite.crop": {}} + for stage in (ExportStage(FAKE_READER), SpriteCropStage()): + for s in stage.subjects(env): + t = describe(stage, s, None, env) + execute(t, depot, registry(stage), force=True) + env.done(t.id, t.key) + out[stage.name][s] = depot.result(t.key) + return out + + +def product(depot, env, subject="card", keys=None, stage=None): + stage = stage or Toy(keys) + t = describe(stage, subject, None, env) + execute(t, depot, registry(stage), force=True) + return t, depot.result(t.key) + + +def table_of(depot, doc) -> dict: + rec = next(a for a in doc["artifacts"] if a["id"].endswith("#objects")) + return contract.loads(depot.read(rec["content"]["sha256"])) + + +# ---------------------------------------------------------------- the request and the table +def test_a_products_key_is_asked_for_before_anything_runs(tmp_path): + """request(): the roots fact of the products, from the keys of their subjects alone.""" + depot = Depot(tmp_path / "s") + stage = Toy({"m/a": "key/a", "m/b": "key/b"}) + stages = {"toy.thing": stage} + env = Env(depot, {"keys": {}}) + assert assemble.request(stages, {"toy.thing": ["m/b", "m/a"]}, env) == {"keys": ["key/a", "key/b"], "objects": []} + assert stage.subjects(Env(depot, {"products": {"toy.thing": ["m/b"]}})) == ["m/b"] + + +def test_the_object_table_is_what_the_root_reaches(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + t, doc = product(depot, env, "m/a", {"m/a": "key/a"}) + table = table_of(depot, doc) + assert table["roots"] == [oid(60)] + assert table["missing"] and all(m.startswith("CAB-scripts:") for m in table["missing"]) # no census of it + # the hierarchy: the root exported, its members contained in its prefab + assert table["objects"][oid(60)] == { + "status": "exported", + "class": "GameObject", + "name": "root", + "files": {"json": oid_file(table, oid(60), "json")}, + } + assert table["objects"][oid(61)]["status"] == "contained" and table["objects"][oid(61)]["in"] == oid(60) + assert table["objects"][oid(65)]["class"] == "MonoBehaviour" + # what the prefab references across bundles: the texture and the material + assert {"json", "image"} <= set(table["objects"][oid(10)]["files"]) + assert oid(70) not in table["objects"] # not referenced + item = doc["items"][0] + assert (item["object"], item["status"], item["class"]) == (t.id, "exported", "Toy") + assert item["artifacts"] == sorted(f"{t.id}#{p}" for p in ("json", "objects")) # no TextAsset is reached + + +def oid_file(table, o, role): + return table["objects"][o]["files"][role] + + +def test_a_reached_object_that_no_census_holds_is_missing(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}, roots_objects=["CAB-none:1"]) + exported(depot, env) + _, doc = product(depot, env, "m/a", {"m/a": "key/a"}, stage=Rooted({"m/a": "key/a"}, ["CAB-none:1"])) + table = table_of(depot, doc) + assert "CAB-none:1" in table["missing"] and "CAB-none:1" in table["roots"] + + +def test_a_root_of_a_unit_the_request_leaves_out_is_an_error(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}, selected=["atlas"]) + env.facts["roots"] = {"keys": [], "objects": []} # the request does not start from the product's key + with pytest.raises(ValueError, match="no requested unit"): + describe(Toy({"m/a": "key/a"}), "m/a", None, env) + + +def test_the_request_roots_select_the_units_and_nothing_else(tmp_path): + """A request for one prefab exports its units, not every unit of its bundle.""" + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}, names=("all",), selected=[]) + subjects = ExportStage(FAKE_READER).subjects(env) + assert subjects == ["all:10", "all:60"] # the prefab and the texture it references + + +def test_sprite_images_come_from_the_crop_of_their_atlas(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(13)]}, names=("all", "atlas"), roots_objects=[f"{ATLAS_CAB}:99"]) + exported(depot, env) + t, doc = product(depot, env, "m/a", {"m/a": "key/a"}) + table = table_of(depot, doc) + sprite = table["objects"][oid(13)] + assert set(sprite["files"]) == {"json", "image"} # the image from sprite.crop + assert table["objects"][f"{ATLAS_CAB}:99"]["class"] == "SpriteAtlas" + assert table["objects"][f"{ATLAS_CAB}:97"]["status"] == "exported" # the atlas references its own sprite + crop = [c for c in [*doc["artifacts"]] if c["semantics"]["kind"] == "sprite.meta"] + assert crop == [] # the sprite's meta belongs to its export result, not to the product + assert f"{t.id}#objects" in {a["id"] for a in doc["artifacts"]} + + +def test_depends_names_the_units_it_reads(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + deps = Toy({"m/a": "key/a"}).depends("m/a", env) + assert deps[0] == "link.addresses:main" + assert deps[1:] == ["unity.export:all:10", "unity.export:all:60"] # what it reads, not every unit + + +def test_inputs_and_the_key_follow_the_objects(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + stage = Toy({"m/a": "key/a"}) + t = describe(stage, "m/a", None, env) + assert [(i.role, i.size) for i in t.inputs] == [("objects", len(contract.encode(stage.reach("m/a", env).table)))] + assert t.context == {} and t.atoms == {} and t.params == {} + # another object of the same unit is not in the table: the key does not follow it + t2 = describe(Toy({"m/a": "key/a"}), "m/a", None, env) + assert t2.key == t.key and t2.cost.cpu_seconds > 0 + + +def test_a_product_waits_for_its_units(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + with pytest.raises(Pending, match="unity.export"): + describe(Toy({"m/a": "key/a"}), "m/a", None, env) + + +# ---------------------------------------------------------------- the result and the layout +def test_a_product_result_is_the_files_and_the_table(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + t, doc = product(depot, env, "m/a", {"m/a": "key/a"}) + body = contract.loads( + depot.read(next(a for a in doc["artifacts"] if a["id"] == f"{t.id}#json")["content"]["sha256"]) + ) + assert body["subject"] == "m/a" and body["objects"] + assert doc["status"] == "ok" + again = execute(t, depot, registry(Toy({"m/a": "key/a"})), force=True) + assert again.status == "ran" and contract.encode(depot.result(t.key)) == contract.encode(doc) + + +def test_an_unsupported_product_is_an_item(tmp_path): + class Picky(Toy): + def assemble(self, task, objects): + raise assemble.Unsupported("unsupported.toy", "nothing to make") + + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + t, doc = product(depot, env, "m/a", {"m/a": "key/a"}, stage=Picky({"m/a": "key/a"})) + assert doc["artifacts"] == [] + assert doc["items"] == [ + { + "object": t.id, + "status": "unsupported", + "class": "Toy", + "reason": {"code": "unsupported.toy", "message": "nothing to make"}, + } + ] + assert doc["status"] == "ok" # an unsupported product is a reported outcome, not a failure + + +def test_the_table_can_be_pruned_to_the_objects_the_product_uses(tmp_path): + class Pruned(Toy): + def assemble(self, task, objects): + made = super().assemble(task, objects) + return assemble.Assembly(made.files, {oid(60)}) + + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + _, doc = product(depot, env, "m/a", {"m/a": "key/a"}, stage=Pruned({"m/a": "key/a"})) + assert list(table_of(depot, doc)["objects"]) == [oid(60)] + + +def test_a_product_names_an_object_the_table_lacks(tmp_path): + class Wrong(Toy): + def assemble(self, task, objects): + made = super().assemble(task, objects) + return assemble.Assembly(made.files, {"CAB-none:1"}) + + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + t = describe(Wrong({"m/a": "key/a"}), "m/a", None, env) + with pytest.raises(ValueError, match="not in the table"): + execute(t, depot, registry(Wrong({"m/a": "key/a"})), force=True) + + +# ---------------------------------------------------------------- the layout +def manifest_of(depot, t, doc): + """(entries, the manifest) of a product layout for one result.""" + table = table_of(depot, doc) + stage = Toy({"m/a": "key/a"}) + path = stage.manifest("m/a") + body = contract.encode(layout.product_manifest(t.id, doc, table)) + sha = depot.put(body) + rec = {"id": f"{t.id}#manifest", "sha256": sha, "size": len(body)} + return layout.product_entries([(path, rec, doc, table)]), contract.loads(body) + + +def test_the_product_layout_places_the_files_and_the_objects(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + t, doc = product(depot, env, "m/a", {"m/a": "key/a"}) + entries, manifest = manifest_of(depot, t, doc) + place = {e["path"]: e["id"] for e in entries} + assert place["toy.thing/m%2Fa.json"] == f"{t.id}#manifest" + json_rec = next(a for a in doc["artifacts"] if a["id"] == f"{t.id}#json") + assert place[f"assets/{json_rec['content']['sha256']}.json"] == f"{t.id}#json" + assert place[f"assets/{oid_file(table_of(depot, doc), oid(10), 'image')['sha256']}.png"] == f"{oid(10)}#image" + assert manifest["files"]["json"] == { + "asset": f"assets/{json_rec['content']['sha256']}.json", + "size": json_rec["content"]["size"], + } + assert manifest["objects"][oid(10)]["class"] == "Texture2D" + assert manifest["objects"][oid(61)]["in"] == oid(60) + assert "assets" in manifest["objects"][oid(10)]["files"]["image"]["asset"] + # the manifest path is one directory and one file + assert len(entries) == len({(e["path"], e["id"]) for e in entries}) + + +def test_the_product_layout_refuses_a_nested_manifest_path(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + t, doc = product(depot, env, "m/a", {"m/a": "key/a"}) + table = table_of(depot, doc) + body = contract.encode(layout.product_manifest(t.id, doc, table)) + rec = {"id": f"{t.id}#manifest", "sha256": depot.put(body), "size": len(body)} + with pytest.raises(ValueError, match="manifest path"): + layout.product_entries([("a/b/c.json", rec, doc, table)]) + + +def test_the_product_layout_writes_and_hits(tmp_path): + depot = Depot(tmp_path / "s") + env = env_for(depot, {"key/a": [oid(60)]}) + exported(depot, env) + t, doc = product(depot, env, "m/a", {"m/a": "key/a"}) + entries, _ = manifest_of(depot, t, doc) + w = layout.materialize(tmp_path / "out", "product", {}, entries, layout.from_depot(depot)) + assert w["write"] == len(entries) and w["remove"] == 0 + assert (tmp_path / "out" / "toy.thing" / "m%2Fa.json").is_file() + assert layout.diff(tmp_path / "out", entries) == {"write": 0, "remove": 0, "keep": len(entries)} + assert json.loads((tmp_path / "out" / "toy.thing" / "m%2Fa.json").read_text(encoding="utf-8"))["schema"] == ( + contract.PRODUCT + )