From 3220b05cdd994e539121018261372976967fe31c Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Sun, 18 Oct 2026 13:50:11 +0800 Subject: [PATCH] feat(site): assemble immutable product snapshots --- docs/products.md | 25 +++ docs/stages.md | 10 ++ src/nnnotes/cli/assets.py | 2 + src/nnnotes/pipeline/layout.py | 20 ++- src/nnnotes/pipeline/siteproducts.py | 115 ++++++++++++++ src/nnnotes/schemas/product.schema.json | 131 +++++++++++++++- src/nnnotes/site/products.py | 193 ++++++++++++++++++++++++ src/nnnotes/site/publish.py | 38 +++++ tests/site/test_products.py | 182 ++++++++++++++++++++++ tests/site/test_publish.py | 89 +++++++++++ 10 files changed, 796 insertions(+), 9 deletions(-) create mode 100644 src/nnnotes/pipeline/siteproducts.py create mode 100644 src/nnnotes/site/products.py create mode 100644 src/nnnotes/site/publish.py create mode 100644 tests/site/test_products.py create mode 100644 tests/site/test_publish.py diff --git a/docs/products.md b/docs/products.md index 4583062..6a76c50 100644 --- a/docs/products.md +++ b/docs/products.md @@ -542,3 +542,28 @@ every episode whose key the selection holds. `ui.pack` assembles one catalog key and its reachable objects. Its `ui.json` (`nnnotes.ui-pack/1`) names root prefabs, controllers, animation bindings and shaders. The product layout writes `index.json` (`nnnotes.ui-library/1`) over the selected UI products; all referenced data lives in their object tables. See [UI libraries](ui.md). A stage with `INCLUDE_FONTS = True` requests traversal through its referenced font objects. The request records these roots in `font_roots`; unit selection and assembly follow the same rule. Other products keep their own font selection behavior. + +## Site snapshots + +`nnnotes.site.products.Snapshot` takes a depot and product-layout manifests keyed by path. It builds each product's +dependencies first, using `site.references` to rewrite the declared `parts` fields, `site.asset` for the files and +`site.manifest` for their manifest. Every transformation is recorded as a depot task in `snapshot.run`. + +The immutable manifest path is `products/.json`, where the content id is that of the manifest itself. +References in a live, story or home host point at those immutable paths. A dependency change gives its parents +new manifest contents while unrelated products keep their identities. Object ids and file roles stay unchanged. +Only declared reference fields are rewritten; text and serialized component values retain their values. + +Stories retain the selected language groups in their part index. With no explicit language selection, the +snapshot uses the languages whose UI and font products are available. An explicitly requested missing language +or a missing required dependency is an error. Public aliases may use region directories; their `root` points +back to the site root. + +Product file entries accept whole files, compressed files and split JSON objects, including object JSON roles. +A split entry has `{parts, size}`, with `[key, asset, size]` rows, or `[key, asset, size, stored]` for compressed +parts. Rejoining the key names and value texts reconstructs the exact minified document and its byte count. + +`nnnotes.site.publish.materialize` places a complete snapshot and its listing files from the depot. It writes +assets, immutable manifests, public aliases, then indexes. A failed replacement stops publication before file +collection and keeps the ownership journal for retry. After every replacement succeeds, files owned by the +preceding layout that are outside the supplied complete reference closure are removed. Unowned files are kept. diff --git a/docs/stages.md b/docs/stages.md index cf8be03..07732b2 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -343,3 +343,13 @@ The result's `entry.json` is a product file entry, either `{asset, size, stored? use the decoded content id; compressed bytes have their own depot content id. The result records both byte forms, and layout entries select the stored form. Replaying a task reuses its result; changing a parameter or encoder version gives a different task key. + +## site.references and site.manifest + +Both document stages take `source` and `spec` JSON inputs and emit `document.json`. `site.references` applies an +explicit path-to-manifest map to declared reference fields and selects the requested language groups. +`site.manifest` applies transformed file entries to a product while checking that its logical paths, object ids +and file roles agree with the source. It also records the relative site root and optional default language. + +The snapshot assembler consumes these task results to build immutable product graphs; see +[site snapshots](products.md#site-snapshots). diff --git a/src/nnnotes/cli/assets.py b/src/nnnotes/cli/assets.py index 884e26c..3815644 100644 --- a/src/nnnotes/cli/assets.py +++ b/src/nnnotes/cli/assets.py @@ -141,6 +141,8 @@ def matches(self, stage: str) -> bool: StageSource("story", "nnnotes.pipeline.story:StoryStage", output=True, product=True), StageSource("ui.pack", "nnnotes.pipeline.ui:UiStage", output=True, product=True), StageSource("site.asset", "nnnotes.pipeline.siteassets:AssetStage"), + StageSource("site.references", "nnnotes.pipeline.siteproducts:ReferencesStage"), + StageSource("site.manifest", "nnnotes.pipeline.siteproducts:ManifestStage"), StageSource("view.", "nnnotes.pipeline.views:stages", output=True), StageSource("view.voices", "nnnotes.pipeline.voices:VoiceStage", output=True), ) diff --git a/src/nnnotes/pipeline/layout.py b/src/nnnotes/pipeline/layout.py index 1b961aa..aa1364f 100644 --- a/src/nnnotes/pipeline/layout.py +++ b/src/nnnotes/pipeline/layout.py @@ -499,11 +499,23 @@ def diff(out_dir, entries_: list[dict]) -> dict: return {"write": write, "remove": remove, "keep": len(files) - write} -def materialize(out_dir, name: str, params: dict, entries_: list[dict], source, *, link: str = "copy") -> dict: +def materialize( + out_dir, + name: str, + params: dict, + entries_: list[dict], + source, + *, + link: str = "copy", + write_order=None, + fail_fast: bool = False, +) -> dict: """Write layout `name` into `out_dir` (see the module documentation). `source(sha256) -> Path` gives the bytes (from_depot, from_layout); `link`: the placement (PLACEMENTS). -> {"manifest": the layout document (without the files not placed), "sha256": of its file, "write", "remove", "keep", "cloned", "linked", "copied", - "failed": [{path, message}] of the files not placed}.""" + "failed": [{path, message}] of the files not placed}. `write_order(path)` orders replacements; `fail_fast` + raises on a failed replacement before removing owned files or updating the manifest, keeping the journal + for a subsequent run.""" if link not in PLACEMENTS: raise ValueError(f"unknown placement {link!r}") out = Path(out_dir) @@ -528,13 +540,15 @@ def restore(p): # a replaced or removed hard link's depot object: read-only aga except (KeyError, OSError): pass - for p in write: + for p in sorted(write, key=write_order) if write_order else write: sha, size = files[p] dst = out / p if not (p not in mine and _present(dst, size) and _same(dst, files[p])): try: _place(Path(source(sha)), dst, link, stats, lambda p=p: restore(p)) except OSError as e: + if fail_fast: + raise LayoutError(f"could not publish {p}: {type(e).__name__}: {e.strerror or e}") from e failed.append({"path": p, "message": f"{type(e).__name__}: {e.strerror or e}"}) for p in remove: _remove(out, p, lambda p=p: restore(p)) diff --git a/src/nnnotes/pipeline/siteproducts.py b/src/nnnotes/pipeline/siteproducts.py new file mode 100644 index 0000000..26dadea --- /dev/null +++ b/src/nnnotes/pipeline/siteproducts.py @@ -0,0 +1,115 @@ +"""Task-backed product documents with immutable references for a static site.""" + +from __future__ import annotations + +import copy +import json + +from ..core import jsonio +from . import contract +from .stages import Output, Stage + +REFERENCES, MANIFEST = "site.references", "site.manifest" + + +def at(document, path): + value = document + for key in path: + if not isinstance(value, dict) or key not in value: + return None + value = value[key] + return value + + +def select_languages(document, paths, languages): + document = copy.deepcopy(document) + for path in paths: + values = at(document, path) + if values is None: + continue + if not isinstance(values, dict): + raise ValueError(f"{'.'.join(path)}: language group must be an object") + missing = set(languages) - set(values) + if missing: + raise ValueError(f"{'.'.join(path)}: missing languages {', '.join(sorted(missing))}") + parent = at(document, path[:-1]) + if not isinstance(parent, dict): + raise ValueError("language group parent must be an object") + parent[path[-1]] = {code: values[code] for code in languages} + return document + + +class _DocumentStage(Stage): + def subjects(self, env): + return sorted(env.facts.get(self.name, {})) + + def inputs(self, subject, env): + return env.fact(self.name)[subject] + + def run(self, task, depot): + source = json.loads(depot.input_bytes(task.input("source"))) + spec = json.loads(depot.input_bytes(task.input("spec"))) + doc = self.transform(source, spec) + content = depot.add( + jsonio.dumps(doc, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode(), "json" + ) + art = contract.artifact( + contract.artifact_id(task.id, "document.json"), content, contract.provenance(task), {"kind": self.name} + ) + return Output([art]) + + def transform(self, source, spec): + raise NotImplementedError + + +class ReferencesStage(_DocumentStage): + """Rewrite only declared product-reference fields and language groups.""" + + name, version = REFERENCES, 1 + + def transform(self, source, spec): + doc = select_languages(source, spec["languagePaths"], spec["languages"]) + + def rewrite(value): + if isinstance(value, str): + if value not in spec["routes"]: + raise ValueError(f"product reference has no manifest: {value}") + return spec["routes"][value] + if isinstance(value, dict): + return {key: rewrite(v) for key, v in value.items()} + if isinstance(value, list): + return [rewrite(v) for v in value] + if value is not None: + raise ValueError("product reference must be a path, group or null") + return None + + for path in spec["paths"]: + value = at(doc, path) + if value is not None: + parent = at(doc, path[:-1]) + if not isinstance(parent, dict): + raise ValueError("product reference parent must be an object") + parent[path[-1]] = rewrite(value) + return doc + + +class ManifestStage(_DocumentStage): + """Apply transformed file entries while preserving product and object identity.""" + + name, version = MANIFEST, 1 + + def transform(self, source, spec): + if source.get("schema") != contract.PRODUCT: + raise ValueError("site manifest requires a product") + doc = copy.deepcopy(source) + if set(doc["files"]) != set(spec["files"]) or set(doc["objects"]) != set(spec["objects"]): + raise ValueError("site manifest file or object identity differs") + doc["files"] = spec["files"] + for oid, entry in doc["objects"].items(): + if set(entry["files"]) != set(spec["objects"][oid]): + raise ValueError(f"site manifest object {oid}: file roles differ") + entry["files"] = spec["objects"][oid] + doc["root"] = spec.get("root", "../") + if spec.get("language"): + doc["language"] = spec["language"] + return doc diff --git a/src/nnnotes/schemas/product.schema.json b/src/nnnotes/schemas/product.schema.json index b9377c0..ce2eaba 100644 --- a/src/nnnotes/schemas/product.schema.json +++ b/src/nnnotes/schemas/product.schema.json @@ -24,7 +24,7 @@ "files": { "type": "object", "additionalProperties": { - "$ref": "#/$defs/asset" + "$ref": "#/$defs/file" } }, "objects": { @@ -41,7 +41,7 @@ "files": { "type": "object", "additionalProperties": { - "$ref": "#/$defs/asset" + "$ref": "#/$defs/file" } } }, @@ -50,10 +50,55 @@ ], "additionalProperties": false } + }, + "root": { + "type": "string", + "minLength": 1 + }, + "language": { + "enum": [ + "ja", + "en", + "zh-Hant", + "zh-Hans", + "ko" + ] } }, "$defs": { + "file": { + "anyOf": [ + { + "$ref": "#/$defs/whole" + }, + { + "$ref": "#/$defs/encoded" + }, + { + "$ref": "#/$defs/parts" + } + ] + }, "asset": { + "type": "string", + "pattern": "^assets/[0-9a-f]{64}\\.[a-z0-9]+$" + }, + "encodedAsset": { + "description": "An asset stored gzip- or brotli-encoded: the SHA-256 of its decoded bytes, the file's extension, then .gz or .br.", + "type": "string", + "pattern": "^assets/[0-9a-f]{64}\\.[a-z0-9]+\\.(gz|br)$" + }, + "size": { + "description": "The byte count of the file (decoded).", + "type": "integer", + "minimum": 0 + }, + "stored": { + "description": "The byte count of an encoded asset as stored (less than size).", + "type": "integer", + "minimum": 0 + }, + "whole": { "type": "object", "required": [ "asset", @@ -62,12 +107,86 @@ "additionalProperties": false, "properties": { "asset": { - "type": "string", - "pattern": "^assets/[0-9a-f]{64}\\.[0-9a-z]+$" + "$ref": "#/$defs/asset" + }, + "size": { + "$ref": "#/$defs/size" + } + } + }, + "encoded": { + "type": "object", + "required": [ + "asset", + "size", + "stored" + ], + "additionalProperties": false, + "properties": { + "asset": { + "$ref": "#/$defs/encodedAsset" + }, + "size": { + "$ref": "#/$defs/size" + }, + "stored": { + "$ref": "#/$defs/stored" + } + } + }, + "parts": { + "type": "object", + "required": [ + "parts", + "size" + ], + "additionalProperties": false, + "properties": { + "parts": { + "type": "array", + "minItems": 1, + "items": { + "anyOf": [ + { + "type": "array", + "prefixItems": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/asset" + }, + { + "$ref": "#/$defs/size" + } + ], + "minItems": 3, + "maxItems": 3 + }, + { + "type": "array", + "prefixItems": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/encodedAsset" + }, + { + "$ref": "#/$defs/size" + }, + { + "$ref": "#/$defs/stored" + } + ], + "minItems": 4, + "maxItems": 4 + } + ] + } }, "size": { - "type": "integer", - "minimum": 0 + "$ref": "#/$defs/size" } } } diff --git a/src/nnnotes/site/products.py b/src/nnnotes/site/products.py new file mode 100644 index 0000000..28fa089 --- /dev/null +++ b/src/nnnotes/site/products.py @@ -0,0 +1,193 @@ +"""Immutable product snapshots assembled through depot tasks. + +Input manifests use product-layout paths. A snapshot rewrites declared references to the content-addressed +manifests it produces and collects the stored asset placements. Construction writes only to the depot; callers +choose the public aliases and publish the complete layout after all requested products have succeeded. +""" + +from __future__ import annotations + +import json +import re + +from ..pipeline import contract, siteassets, siteproducts +from ..pipeline.contract import Input +from ..pipeline.orchestrate import Run +from ..pipeline.stages import Env, describe, execute + +_ASSET = re.compile(r"assets/([0-9a-f]{64})\.([a-z0-9]+)") +REFERENCE_FIELDS = { + "live": {"live.json": [["parts"]]}, + "story": {"story.json": [["parts"]]}, + "story.host": {"host.json": [["home", "parts"]]}, +} +LANGUAGE_FIELDS = { + "live": [["parts", "ui"]], + "story": [["parts", "ui"], ["parts", "fonts"], ["parts", "simpleui"]], +} + + +def _strings(value): + if isinstance(value, str): + yield value + elif isinstance(value, dict): + for v in value.values(): + yield from _strings(v) + elif isinstance(value, list): + for v in value: + yield from _strings(v) + + +class Snapshot: + """One coherent group of products; equal inputs share tasks across snapshots and regions.""" + + def __init__(self, depot, manifests, *, languages=None, encoding="gzip", split=siteassets.SPLIT_MIN_BYTES): + self.depot, self.sources = depot, manifests + self.languages = None if languages is None else sorted(set(languages)) + self.asset_params = siteassets.AssetStage().normalize({"encoding": encoding, "split": split}) + self.run = Run(depot, {}, [], {siteassets.ASSET: self.asset_params}, {}) + self.paths, self.documents, self.records, self.entries = {}, {}, {}, {} + self._visiting = set() + + def _input(self, role, body): + return Input(role, self.depot.put(body), len(body), None, ({"kind": "depot"},)) + + def _task(self, stage, subject, inputs, params=None): + env = Env(self.depot, {stage.name: {subject: inputs}}) + if stage.name == siteassets.ASSET: + env = Env(self.depot, {"site_assets": {subject: inputs[0]}}) + task = describe(stage, subject, params, env) + if task.id in self.run.tasks: + if self.run.tasks[task.id]["key"] != task.key: + raise ValueError(f"site task {task.id}: conflicting inputs") + return self.depot.committed(task.key, task.id) + outcome = execute(task, self.depot, {stage.name: stage}) + self.run.stages[stage.name] = stage.version + self.run.params[stage.name] = task.params + self.run.tasks[task.id] = {"id": task.id, "key": task.key, "status": outcome.status} + if outcome.cost: + self.run.log.append({"task": task.id, **outcome.cost}) + return self.depot.committed(task.key, task.id) + + def _document(self, stage, source, spec): + inputs = [self._input("source", source), self._input("spec", contract.encode(spec))] + key = contract.digest([[i.role, i.sha256] for i in inputs]) + result = self._task(stage, key, inputs) + return result["artifacts"][0] + + def _source(self, entry): + match = _ASSET.fullmatch(entry.get("asset", "")) + if not match or "stored" in entry or "parts" in entry: + raise ValueError("site snapshot requires a decoded product-layout file entry") + return Input("source", match[1], entry["size"], None, ({"kind": "depot"},)), match[2] + + def _asset(self, inp, ext): + result = self._task(siteassets.AssetStage(), siteassets.subject(inp, ext), [inp], self.asset_params) + for placed in siteassets.placements(result): + self.entries[placed["path"]] = placed + return siteassets.entry(result, self.depot) + + def _languages(self, product, doc): + fields = [p for p in LANGUAGE_FIELDS.get(product, []) if siteproducts.at(doc, p) is not None] + if not fields: + return [], [] + if self.languages is not None: + wanted = self.languages + else: + groups = [] + for path in fields: + group = siteproducts.at(doc, path) + if not isinstance(group, dict): + raise ValueError("product language group must be an object") + groups.append(group) + wanted = sorted(set.intersection(*[{k for k, v in group.items() if v in self.sources} for group in groups])) + if not wanted: + raise ValueError(f"{product}: no available language products") + return fields, wanted + + def product(self, path): + if path in self.paths: + return self.paths[path] + if path in self._visiting: + raise ValueError(f"cyclic product references: {path}") + if path not in self.sources: + raise ValueError(f"missing product manifest: {path}") + self._visiting.add(path) + try: + source = self.sources[path] + if source.get("schema") != contract.PRODUCT: + raise ValueError(f"{path}: expected {contract.PRODUCT}") + files, objects, language = {}, {}, None + for logical, entry in sorted(source["files"].items()): + inp, ext = self._source(entry) + paths = REFERENCE_FIELDS.get(source["product"], {}).get(logical, []) + if paths: + body = self.depot.input_bytes(inp) + doc = json.loads(body) + language_paths, wanted = self._languages(source["product"], doc) + selected = siteproducts.select_languages(doc, language_paths, wanted) + dependencies = sorted({v for p in paths for v in _strings(siteproducts.at(selected, p))}) + routes = {p: self.product(p) for p in dependencies} + art = self._document( + siteproducts.ReferencesStage(), + body, + { + "paths": paths, + "routes": routes, + "languagePaths": language_paths, + "languages": wanted, + }, + ) + c = art["content"] + inp = Input("source", c["sha256"], c["size"], None, ({"kind": "depot"},)) + if wanted: + language = source.get("language") if source.get("language") in wanted else wanted[0] + files[logical] = self._asset(inp, ext) + for oid, obj in sorted(source["objects"].items()): + objects[oid] = {role: self._asset(*self._source(entry)) for role, entry in sorted(obj["files"].items())} + art = self._document( + siteproducts.ManifestStage(), + contract.encode(source), + { + "files": files, + "objects": objects, + "language": language, + }, + ) + c = art["content"] + target = f"products/{c['sha256']}.json" + self.paths[path] = target + self.documents[path] = json.loads(self.depot.read(c["sha256"])) + self.records[path] = art + self.entries[target] = {"path": target, "sha256": c["sha256"], "size": c["size"], "id": art["id"]} + return target + finally: + self._visiting.remove(path) + + def alias(self, source, path): + """Add a public entry point after its immutable product snapshot has been constructed.""" + target = self.product(source) + segments = path.split("/") + if ( + len(segments) < 2 + or any(not s or s in {".", ".."} for s in segments) + or re.search(r"[\\:?#\x00]", path) + or not path.endswith(".json") + ): + raise ValueError("site product alias requires a relative manifest path") + if len(segments) == 2: + self.entries[path] = {**self.entries[target], "path": path} + else: + doc = self.documents[source] + art = self._document( + siteproducts.ManifestStage(), + contract.encode(doc), + { + "files": doc["files"], + "objects": {oid: obj["files"] for oid, obj in doc["objects"].items()}, + "root": "../" * (len(segments) - 1), + }, + ) + c = art["content"] + self.entries[path] = {"path": path, "sha256": c["sha256"], "size": c["size"], "id": art["id"]} + return path diff --git a/src/nnnotes/site/publish.py b/src/nnnotes/site/publish.py new file mode 100644 index 0000000..c4eab02 --- /dev/null +++ b/src/nnnotes/site/publish.py @@ -0,0 +1,38 @@ +"""Publish a complete site layout in dependency order, then collect retired owned files.""" + +from __future__ import annotations + +import re + +from ..pipeline import layout + +INDEXES = frozenset({"charts.json", "models.json", "stories.json", "index.json"}) + + +def order(path: str) -> tuple[int, str]: + if path in INDEXES: + return 3, path + if path.startswith("assets/"): + return 0, path + if path.startswith("products/"): + return 1, path + return 2, path + + +def materialize(directory, entries, depot, *, link="copy", params=None): + """Place all files before replacing indexes. Failure keeps the ownership manifest and defers collection. + + Content-addressed products keep every replaced public manifest's dependencies coherent. The caller supplies + the complete reference closure of selected and retained public entries. Unowned files are preserved. + """ + seen = {} + for entry in entries: + path = entry["path"] + if re.search(r"[\\:?#\x00]", path) or any(p in {"", ".", ".."} for p in path.split("/")): + raise layout.LayoutError(f"site file is not a relative path: {path}") + old = seen.setdefault(path.casefold(), path) + if old != path: + raise layout.LayoutError(f"site paths differ only by case: {old}, {path}") + return layout.materialize( + directory, "site", params or {}, entries, layout.from_depot(depot), link=link, write_order=order, fail_fast=True + ) diff --git a/tests/site/test_products.py b/tests/site/test_products.py new file mode 100644 index 0000000..53c45ea --- /dev/null +++ b/tests/site/test_products.py @@ -0,0 +1,182 @@ +"""Immutable product graphs and task replay with synthetic data.""" + +import copy +import json + +import pytest + +from nnnotes.formats import compression +from nnnotes.pipeline import contract +from nnnotes.pipeline.depot import Depot +from nnnotes.site.products import Snapshot + + +def file(depot, doc): + data = json.dumps(doc, ensure_ascii=False, indent=2).encode() + return {"asset": f"assets/{depot.put(data)}.json", "size": len(data)} + + +def product(depot, kind, subject, files, objects=None): + return { + "schema": contract.PRODUCT, + "product": kind, + "subject": subject, + "files": {k: file(depot, v) for k, v in files.items()}, + "objects": objects or {}, + } + + +def graph(depot): + return { + "stories/1.json": product( + depot, + "story", + "1", + { + "story.json": { + "format": 1, + "advId": 1, + "parts": { + "scene": "storyscenes/default.json", + "ui": {"ja": "storyui/ja.json", "en": "storyui/en.json"}, + "fonts": {"ja": "storyfonts/1_ja.json", "en": "storyfonts/1_en.json"}, + }, + }, + "episode.json": {"advId": 1, "text": "storyscenes/default.json"}, + }, + ), + "storyscenes/default.json": product(depot, "story.scene", "default", {"scene.json": {"field": 1}}), + "storyui/ja.json": product(depot, "story.ui", "ja", {"ui.json": {"language": "ja"}}), + "storyfonts/1_ja.json": product(depot, "story.fonts", "1/ja", {"fonts.json": {"language": "ja"}}), + } + + +def read_file(snapshot, entry): + def read(path): + data = snapshot.depot.read(snapshot.entries[path]["sha256"]) + return compression.decode(path, data) + + if "parts" not in entry: + return json.loads(read(entry["asset"])) + return {k: json.loads(read(p)) for k, p, *_ in entry["parts"]} + + +def test_snapshot_routes_exact_products_and_only_available_languages(tmp_path): + depot = Depot(tmp_path) + manifests = graph(depot) + before = copy.deepcopy(manifests) + snapshot = Snapshot(depot, manifests, split=1) + root = snapshot.product("stories/1.json") + assert root.startswith("products/") + man = snapshot.documents["stories/1.json"] + story = read_file(snapshot, man["files"]["story.json"]) + assert story["parts"]["scene"] == snapshot.paths["storyscenes/default.json"] + assert story["parts"]["ui"] == {"ja": snapshot.paths["storyui/ja.json"]} + assert story["parts"]["fonts"] == {"ja": snapshot.paths["storyfonts/1_ja.json"]} + assert man["language"] == "ja" + assert read_file(snapshot, man["files"]["episode.json"])["text"] == "storyscenes/default.json" + assert manifests == before + assert snapshot.run.summary()["ran"] == len(snapshot.run.tasks) + assert not snapshot.run.failures() + from tests.pipeline.test_contract import validators + + schemas = validators() + assert not list(schemas[contract.RUN].iter_errors(snapshot.run.manifest())) + for _, result in snapshot.run.results(): + assert not list(schemas[contract.RESULT].iter_errors(result)) + + +def test_shared_objects_and_product_snapshots_reuse_tasks(tmp_path): + depot = Depot(tmp_path) + manifests = graph(depot) + shared = {"class": "MonoBehaviour", "name": "Shared", "files": {"json": file(depot, {"x": 1})}} + manifests["storyui/ja.json"]["objects"] = {"example:1": shared} + manifests["storyscenes/default.json"]["objects"] = {"example:1": shared} + first = Snapshot(depot, manifests) + old = first.product("stories/1.json") + count = len(first.run.tasks) + assert first.product("stories/1.json") == old and len(first.run.tasks) == count + a = first.documents["storyui/ja.json"]["objects"]["example:1"]["files"]["json"] + b = first.documents["storyscenes/default.json"]["objects"]["example:1"]["files"]["json"] + assert a == b + second = Snapshot(depot, manifests) + assert second.product("stories/1.json") == old + assert second.entries == first.entries + assert second.run.summary()["hit"] == count + assert second.run.summary()["ran"] == 0 + + +def test_dependency_change_updates_parent_reference_without_changing_other_products(tmp_path): + depot = Depot(tmp_path) + manifests = graph(depot) + old = Snapshot(depot, manifests) + old.product("stories/1.json") + manifests["storyscenes/default.json"]["files"]["scene.json"] = file(depot, {"field": 2}) + new = Snapshot(depot, manifests) + new.product("stories/1.json") + assert new.paths["stories/1.json"] != old.paths["stories/1.json"] + assert new.paths["storyscenes/default.json"] != old.paths["storyscenes/default.json"] + assert new.paths["storyui/ja.json"] == old.paths["storyui/ja.json"] + + +def test_aliases_include_their_relative_site_root(tmp_path): + depot = Depot(tmp_path) + snapshot = Snapshot(depot, graph(depot)) + snapshot.alias("stories/1.json", "stories/1.json") + snapshot.alias("stories/1.json", "stories/region/1.json") + for path, root in [("stories/1.json", "../"), ("stories/region/1.json", "../../")]: + man = json.loads(depot.read(snapshot.entries[path]["sha256"])) + assert man["root"] == root + assert read_file(snapshot, man["files"]["story.json"])["parts"]["scene"] in snapshot.entries + with pytest.raises(ValueError, match="relative manifest"): + snapshot.alias("stories/1.json", "../outside.json") + + +def test_requested_missing_language_or_product_fails_without_a_public_layout(tmp_path): + depot = Depot(tmp_path) + manifests = graph(depot) + with pytest.raises(ValueError, match="missing product"): + Snapshot(depot, manifests, languages=["en"]).product("stories/1.json") + with pytest.raises(ValueError, match="missing languages"): + Snapshot(depot, manifests, languages=["ko"]).product("stories/1.json") + manifests.pop("storyscenes/default.json") + with pytest.raises(ValueError, match="missing product"): + Snapshot(depot, manifests).product("stories/1.json") + + +def test_cycles_are_reported(tmp_path): + depot = Depot(tmp_path) + manifests = { + path: product(depot, "live", path, {"live.json": {"parts": {"scene": other}}}) + for path, other in [("lives/a.json", "lives/b.json"), ("lives/b.json", "lives/a.json")] + } + with pytest.raises(ValueError, match="cyclic"): + Snapshot(depot, manifests).product("lives/a.json") + + +def test_home_host_and_story_share_the_same_room_and_spot_snapshots(tmp_path): + depot = Depot(tmp_path) + manifests = graph(depot) + root = manifests["stories/1.json"] + story = json.loads(depot.read(root["files"]["story.json"]["asset"].split("/")[1].split(".")[0])) + story["parts"].update( + host={"home/1": "storyhosts/home_1.json"}, rooms={"room": "rooms/room.json"}, spots={"spot": "spots/spot.json"} + ) + root["files"]["story.json"] = file(depot, story) + manifests["storyhosts/home_1.json"] = product( + depot, + "story.host", + "home/1", + { + "host.json": {"home": {"parts": {"room": "rooms/room.json", "spot": "spots/spot.json"}}}, + }, + ) + manifests["rooms/room.json"] = product(depot, "room.gltf", "room", {"room.json": {"name": "room"}}) + manifests["spots/spot.json"] = product(depot, "spot.files", "spot", {"spot.json": {"name": "spot"}}) + snapshot = Snapshot(depot, manifests) + snapshot.product("stories/1.json") + host = read_file(snapshot, snapshot.documents["storyhosts/home_1.json"]["files"]["host.json"]) + assert host["home"]["parts"] == { + "room": snapshot.paths["rooms/room.json"], + "spot": snapshot.paths["spots/spot.json"], + } diff --git a/tests/site/test_publish.py b/tests/site/test_publish.py new file mode 100644 index 0000000..2853fc7 --- /dev/null +++ b/tests/site/test_publish.py @@ -0,0 +1,89 @@ +"""Site publication order, interruption recovery and ownership over synthetic files.""" + +import json + +import pytest + +from nnnotes.pipeline import layout +from nnnotes.pipeline.depot import Depot +from nnnotes.site import publish + + +def entries(depot, files): + return [{"path": path, "sha256": depot.put(data), "size": len(data), "id": path} for path, data in files.items()] + + +def test_publish_orders_assets_and_manifests_before_indexes(tmp_path, monkeypatch): + depot, out = Depot(tmp_path / "depot"), tmp_path / "site" + files = { + "charts.json": b"index", + "charts/1.json": b"alias", + "products/one.json": b"product", + "assets/one.json": b"asset", + } + calls, place = [], layout._place + + def record(src, dst, *args): + calls.append(dst.relative_to(out).as_posix()) + return place(src, dst, *args) + + monkeypatch.setattr(layout, "_place", record) + result = publish.materialize(out, entries(depot, files), depot) + assert not result["failed"] + assert calls == ["assets/one.json", "products/one.json", "charts/1.json", "charts.json"] + + +@pytest.mark.parametrize("fail_at", ["assets/new.json", "products/new.json", "charts/1.json"]) +def test_failed_publish_keeps_indexes_and_retired_assets_then_recovers(tmp_path, monkeypatch, fail_at): + depot, out = Depot(tmp_path / "depot"), tmp_path / "site" + old = {"assets/old.json": b"old", "products/old.json": b"old", "charts/1.json": b"old", "charts.json": b"old"} + first = publish.materialize(out, entries(depot, old), depot) + (out / "custom.txt").write_text("user file") + new = {"assets/new.json": b"new", "products/new.json": b"new", "charts/1.json": b"new", "charts.json": b"new"} + place = layout._place + + def fail(src, dst, *args): + if dst.relative_to(out).as_posix() == fail_at: + raise OSError("write failed") + return place(src, dst, *args) + + monkeypatch.setattr(layout, "_place", fail) + with pytest.raises(layout.LayoutError, match="could not publish"): + publish.materialize(out, entries(depot, new), depot) + assert (out / "charts.json").read_bytes() == b"old" + assert (out / "assets/old.json").read_bytes() == b"old" + assert layout.read_manifest(out) == first["manifest"] + assert (out / layout.JOURNAL).is_file() + monkeypatch.setattr(layout, "_place", place) + result = publish.materialize(out, entries(depot, new), depot) + assert not result["failed"] + assert (out / "charts.json").read_bytes() == b"new" + assert not (out / "assets/old.json").exists() + assert not (out / layout.JOURNAL).exists() + assert (out / "custom.txt").read_text() == "user file" + + +def test_new_snapshot_collects_unreferenced_files_from_an_interrupted_attempt(tmp_path, monkeypatch): + depot, out = Depot(tmp_path / "depot"), tmp_path / "site" + place = layout._place + + def fail(src, dst, *args): + if dst.name == "charts.json": + raise OSError("index locked") + return place(src, dst, *args) + + monkeypatch.setattr(layout, "_place", fail) + with pytest.raises(layout.LayoutError): + publish.materialize(out, entries(depot, {"assets/abandoned.json": b"data", "charts.json": b"index"}), depot) + monkeypatch.setattr(layout, "_place", place) + publish.materialize(out, entries(depot, {"charts.json": b"{}"}), depot) + assert not (out / "assets/abandoned.json").exists() + assert json.loads((out / "charts.json").read_bytes()) == {} + + +@pytest.mark.parametrize("path", ["../outside", "/absolute", "a/../outside", "a\\outside", "C:/outside"]) +def test_site_paths_are_confined_to_the_output(tmp_path, path): + depot = Depot(tmp_path / "depot") + with pytest.raises(layout.LayoutError, match="relative path"): + publish.materialize(tmp_path / "site", entries(depot, {path: b"x"}), depot) + assert not (tmp_path / "site").exists()