From 06f2df1cd1654bf939e02ba9b111f0de54f70d0f Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Wed, 14 Oct 2026 23:42:35 +0800 Subject: [PATCH] refactor: move the atoms to the unity layer The atoms (texture decoding, PNG and ASTC encoding, sprite images, mesh arrays, glTF, file sniffing and the bundle reader) are the pure functions Unity objects are converted with, and live in nnnotes.unity.atoms. Object ids (nnnotes.unity.ids) and media types (nnnotes.core.media) sit below them; nnnotes.pipeline.contract keeps object_id, parse_object_id, media_type and MEDIA_TYPES among its names. Atom ids, and with them task keys, are unchanged. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 4 +- docs/stages.md | 4 +- scripts/atom_eval.py | 18 ++++---- scripts/atom_eval_pillow.py | 6 +-- scripts/atom_eval_unityrs.py | 18 ++++---- src/nnnotes/core/media.py | 34 +++++++++++++++ src/nnnotes/pipeline/census.py | 2 +- src/nnnotes/pipeline/contract.py | 43 ++----------------- src/nnnotes/pipeline/cristages.py | 2 +- src/nnnotes/pipeline/modelstages.py | 2 +- src/nnnotes/pipeline/objexport.py | 14 +++--- .../{pipeline => unity}/atoms/__init__.py | 0 src/nnnotes/{pipeline => unity}/atoms/astc.py | 0 src/nnnotes/{pipeline => unity}/atoms/gltf.py | 0 src/nnnotes/{pipeline => unity}/atoms/mesh.py | 0 src/nnnotes/{pipeline => unity}/atoms/png.py | 0 .../{pipeline => unity}/atoms/reader.py | 6 +-- .../{pipeline => unity}/atoms/sniff.py | 4 +- .../{pipeline => unity}/atoms/sprite.py | 0 .../{pipeline => unity}/atoms/texture.py | 0 src/nnnotes/unity/ids.py | 17 ++++++++ tests/pipeline/test_atom_eval.py | 6 +-- tests/pipeline/test_cristages.py | 2 +- tests/pipeline/test_objexport.py | 12 +++--- tests/{pipeline => unity}/test_atoms_mesh.py | 6 +-- .../{pipeline => unity}/test_atoms_sprite.py | 8 ++-- .../{pipeline => unity}/test_atoms_texture.py | 16 +++---- tests/{pipeline => unity}/test_sniff.py | 4 +- 28 files changed, 122 insertions(+), 106 deletions(-) create mode 100644 src/nnnotes/core/media.py rename src/nnnotes/{pipeline => unity}/atoms/__init__.py (100%) rename src/nnnotes/{pipeline => unity}/atoms/astc.py (100%) rename src/nnnotes/{pipeline => unity}/atoms/gltf.py (100%) rename src/nnnotes/{pipeline => unity}/atoms/mesh.py (100%) rename src/nnnotes/{pipeline => unity}/atoms/png.py (100%) rename src/nnnotes/{pipeline => unity}/atoms/reader.py (98%) rename src/nnnotes/{pipeline => unity}/atoms/sniff.py (99%) rename src/nnnotes/{pipeline => unity}/atoms/sprite.py (100%) rename src/nnnotes/{pipeline => unity}/atoms/texture.py (100%) create mode 100644 src/nnnotes/unity/ids.py rename tests/{pipeline => unity}/test_atoms_mesh.py (98%) rename tests/{pipeline => unity}/test_atoms_sprite.py (97%) rename tests/{pipeline => unity}/test_atoms_texture.py (94%) rename tests/{pipeline => unity}/test_sniff.py (97%) diff --git a/docs/architecture.md b/docs/architecture.md index 8b5646b..1aeb4a7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -12,10 +12,10 @@ subpackage of `nnnotes` and imports only the layers below it. | Products | `nnnotes.site`, `nnnotes.musicdata` | the ournotes-player site (charts, models, stories, indexes); the music data file and its deck statistics | | Pipeline | `nnnotes.pipeline` | stages, task descriptions and results, the content-addressed depot, plans, layouts, the catalog versions, the views of the master data | | Content | `nnnotes.content` | the game's subjects as exports: ADV episodes and stories, lives and scores, spots, rooms, the UI library | -| Unity objects | `nnnotes.unity` | bundle reading, the typetree exporter, the player data, Live2D prefabs, text styles and TextMesh Pro fonts | +| Unity objects | `nnnotes.unity` | bundle reading, object ids, the atoms (the pure functions objects are converted with: texture decoding, sprite images, mesh arrays, glTF, PNG, file sniffing), the typetree exporter, the player data, Live2D prefabs, text styles and TextMesh Pro fonts | | File formats | `nnnotes.formats` | formats read from bytes: CRI ACB/AWB and HCA, USM movies, CRI Lips data, shaders, motions | | Sources | `nnnotes.source` | the configuration, the APK, Addressables catalogs and bundles, the game API, master data, languages, `Session` | -| Core | `nnnotes.core` | the one JSON writer and the process caches | +| Core | `nnnotes.core` | the one JSON writer, media types, the process caches and the downloads cache | The order is a contract: the `layers` contract of [import-linter](https://import-linter.readthedocs.io/) in `pyproject.toml` fails the build when a module imports a layer above its own, and `nnnotes.site` and diff --git a/docs/stages.md b/docs/stages.md index 16d9e9f..fadf601 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -128,14 +128,14 @@ else found on PATH. The `boot` input is the APK's `assets/bin/Data/data.unity3d` ## Atoms -`nnnotes.pipeline.atoms.ATOMS[name]` is the reference implementation of an atom: its id, its function and its cost estimate. +`nnnotes.unity.atoms.ATOMS[name]` is the reference implementation of an atom: its id, its function and its cost estimate. The id names the libraries whose behaviour the output depends on, with their installed versions, and the revision of the nnnotes code (`pillow-12.1.1/1`); `nnnotes/` when no library's behaviour is involved. Task keys include the ids of the atoms a task uses, so upgrading a library or changing an atom re-runs exactly the tasks that use it. | Atom | Reference implementation | Id | Does | |---|---|---|---| -| `reader` | `nnnotes.pipeline.atoms.reader.open_bundle` (UnityPy) | `unitypy-/1` | one bundle loaded alone from its bytes: serialized files, objects, raw bytes, typetrees, container, externals, and the inputs of `texture.decode` and `mesh.arrays` (resource files are read from inside the bundle only) | +| `reader` | `nnnotes.unity.atoms.reader.open_bundle` (UnityPy) | `unitypy-/1` | one bundle loaded alone from its bytes: serialized files, objects, raw bytes, typetrees, container, externals, and the inputs of `texture.decode` and `mesh.arrays` (resource files are read from inside the bundle only) | | `texture.decode` | UnityPy `Texture2DConverter.parse_image_data` | `unitypy-+texture2ddecoder-+astc-encoder-py-+pillow-/1` | the first mip level, top row first, as UnityPy's `Texture2D.image` | | `png.encode` | Pillow | `pillow-/1` | PNG bytes; level 6 (the default) gives Pillow's default bytes, the ones the other commands write | | `astc.container` | nnnotes | `nnnotes/1` | the standard `.astc` file (16-byte header, first mip level's blocks) | diff --git a/scripts/atom_eval.py b/scripts/atom_eval.py index ece713e..1e23446 100644 --- a/scripts/atom_eval.py +++ b/scripts/atom_eval.py @@ -2,7 +2,7 @@ """Measure candidate implementations of the export atoms against the reference implementations (maintainer tool; not installed, not needed to use nnnotes). -`nnnotes.pipeline.atoms.ATOMS` holds one reference implementation per atom; `NNNOTES_ATOM_=module:attribute` names +`nnnotes.unity.atoms.ATOMS` holds one reference implementation per atom; `NNNOTES_ATOM_=module:attribute` names another one for evaluation (docs/stages.md, "Atoms"). This script gives the numbers a switch is decided on: sample DIR -o FILE a seeded stratified sample of a bundle directory: one bundle of every @@ -42,7 +42,7 @@ A candidate view is the reader candidate's result; the optional methods census(), text_bytes(obj), font_bytes(obj), sprite_image(obj) and the attribute typetree_form serve the checks that have no atom of their own. A candidate -raising NotImplementedError has no such facet ("absent"); `nnnotes.pipeline.atoms.Unsupported` is a refusal. +raising NotImplementedError has no such facet ("absent"); `nnnotes.unity.atoms.Unsupported` is a refusal. Statuses: equal, differ, refused (the candidate raised where the reference gave a result), refused.both, extra (a result where the reference refused), absent, reference-error. Output: one JSON document (-o); m1 and m3 also write @@ -133,7 +133,7 @@ def check_clean_env() -> None: def load_impl(spec: str): """The Impl named by module:attribute (an Impl, or a callable returning one).""" - from nnnotes.pipeline.atoms import Impl + from nnnotes.unity.atoms import Impl module, sep, attr = spec.partition(":") if not sep or not module or not attr: @@ -146,7 +146,7 @@ def load_impl(spec: str): def parse_atoms(items) -> list[tuple[str, str]]: - from nnnotes.pipeline.atoms import ATOMS + from nnnotes.unity.atoms import ATOMS out = [] for it in items or (): @@ -449,7 +449,7 @@ def cmd_sample(a) -> dict: # ---------------------------------------------------------------- m1 def m1_init(cfg: dict) -> None: - from nnnotes.pipeline.atoms import resolve + from nnnotes.unity.atoms import resolve G.clear() G.update(cfg) @@ -477,7 +477,7 @@ def _key(info) -> tuple: def _call(fn, *args): """(result, None) or (None, refusal) where refusal is the Unsupported code or the error's message.""" - from nnnotes.pipeline.atoms import Unsupported + from nnnotes.unity.atoms import Unsupported try: return fn(*args), None @@ -700,7 +700,7 @@ def hdr_reference(inp): import numpy as np from PIL import Image - from nnnotes.pipeline.atoms.texture import ASTC_BLOCKS + from nnnotes.unity.atoms.texture import ASTC_BLOCKS bx, by = ASTC_BLOCKS[int(inp.format)] w, h = int(inp.width), int(inp.height) @@ -723,7 +723,7 @@ def _hdr_library_image(inp): def check_textures(T: Tally, ref, cv, infos, cmap, checks) -> None: - from nnnotes.pipeline.atoms.texture import format_name, is_hdr_astc + from nnnotes.unity.atoms.texture import format_name, is_hdr_astc ref_dec, ref_png = G["ref"]["texture.decode"], G["ref"]["png.encode"] decs, pngs = G["cand"].get("texture.decode", []), G["cand"].get("png.encode", []) @@ -1173,7 +1173,7 @@ def _m3_worker(task) -> dict: def _apply_atoms(atoms) -> None: - from nnnotes.pipeline.atoms import env_name, resolve + from nnnotes.unity.atoms import env_name, resolve for name, spec in atoms: os.environ[env_name(name)] = spec diff --git a/scripts/atom_eval_pillow.py b/scripts/atom_eval_pillow.py index ec5d9f4..b5741b4 100644 --- a/scripts/atom_eval_pillow.py +++ b/scripts/atom_eval_pillow.py @@ -1,6 +1,6 @@ """Candidate `png.encode` implementations on Pillow, for scripts/atom_eval.py only. -The reference (`nnnotes.pipeline.atoms.png.encode`) writes at the level a task's `png.level` parameter gives (6 by +The reference (`nnnotes.unity.atoms.png.encode`) writes at the level a task's `png.level` parameter gives (6 by default). These fix the setting instead, so several can be measured side by side in one `atom_eval.py m1` run: level1, level3, level6, level9 Pillow at that zlib level @@ -15,8 +15,8 @@ from functools import cache from importlib import metadata -from nnnotes.pipeline.atoms import Impl -from nnnotes.pipeline.atoms import png as png_atom +from nnnotes.unity.atoms import Impl +from nnnotes.unity.atoms import png as png_atom def _pillow() -> str: diff --git a/scripts/atom_eval_unityrs.py b/scripts/atom_eval_unityrs.py index d403c35..e544f1a 100644 --- a/scripts/atom_eval_unityrs.py +++ b/scripts/atom_eval_unityrs.py @@ -27,13 +27,13 @@ from functools import cache from importlib import metadata -from nnnotes.pipeline.atoms import Impl, Unsupported -from nnnotes.pipeline.atoms import png as png_atom -from nnnotes.pipeline.atoms import reader as reader_atom -from nnnotes.pipeline.atoms import texture as texture_atom -from nnnotes.pipeline.atoms.mesh import MeshInput -from nnnotes.pipeline.atoms.reader import ObjectInfo, UnityBundle -from nnnotes.pipeline.atoms.texture import TextureInput +from nnnotes.unity.atoms import Impl, Unsupported +from nnnotes.unity.atoms import png as png_atom +from nnnotes.unity.atoms import reader as reader_atom +from nnnotes.unity.atoms import texture as texture_atom +from nnnotes.unity.atoms.mesh import MeshInput +from nnnotes.unity.atoms.reader import ObjectInfo, UnityBundle +from nnnotes.unity.atoms.texture import TextureInput FALLBACKS = {"png": 0} _KEEP = "unity_rs.image" @@ -390,7 +390,7 @@ def png_encode_fast() -> Impl: def _mesh(inp: MeshInput): import numpy as np - from nnnotes.pipeline.atoms.mesh import Channel, MeshArrays, Primitive + from nnnotes.unity.atoms.mesh import Channel, MeshArrays, Primitive src = getattr(inp, "source", None) if src is None: @@ -429,6 +429,6 @@ def _mesh(inp: MeshInput): @cache def mesh_arrays() -> Impl: - from nnnotes.pipeline.atoms import mesh as mesh_atom + from nnnotes.unity.atoms import mesh as mesh_atom return Impl(_id(_rs(), "obj"), _mesh, mesh_atom.cost) diff --git a/src/nnnotes/core/media.py b/src/nnnotes/core/media.py new file mode 100644 index 0000000..5e96322 --- /dev/null +++ b/src/nnnotes/core/media.py @@ -0,0 +1,34 @@ +"""Media types of file extensions.""" + +from __future__ import annotations + +__all__ = ["MEDIA_TYPES", "media_type"] + +MEDIA_TYPES = { + "png": "image/png", + "astc": "image/astc", + "jpg": "image/jpeg", + "json": "application/json", + "glb": "model/gltf-binary", + "flac": "audio/flac", + "wav": "audio/wav", + "ogg": "audio/ogg", + "mkv": "video/x-matroska", + "ivf": "video/x-ivf", + "adx": "audio/x-adx", + "ttf": "font/ttf", + "otf": "font/otf", + "ttc": "font/collection", + "woff": "font/woff", + "woff2": "font/woff2", + "txt": "text/plain", + "xml": "application/xml", + "csv": "text/csv", + "bin": "application/octet-stream", + "bytes": "application/octet-stream", +} + + +def media_type(ext: str) -> str: + """The media type of a file extension (case-insensitive); application/octet-stream when unknown.""" + return MEDIA_TYPES.get(ext.lower(), "application/octet-stream") diff --git a/src/nnnotes/pipeline/census.py b/src/nnnotes/pipeline/census.py index 2875e36..23a75cb 100644 --- a/src/nnnotes/pipeline/census.py +++ b/src/nnnotes/pipeline/census.py @@ -25,8 +25,8 @@ import re +from ..unity.atoms import impl_id from . import contract -from .atoms import impl_id from .contract import Cost, Input from .stages import Output, Pending, Stage diff --git a/src/nnnotes/pipeline/contract.py b/src/nnnotes/pipeline/contract.py index 8c958c8..68d6f26 100644 --- a/src/nnnotes/pipeline/contract.py +++ b/src/nnnotes/pipeline/contract.py @@ -35,6 +35,10 @@ from functools import cached_property from ..core import jsonio +from ..core.media import MEDIA_TYPES as MEDIA_TYPES +from ..core.media import media_type as media_type +from ..unity.ids import object_id as object_id +from ..unity.ids import parse_object_id as parse_object_id CANON = "canon/1" TASK = "nnnotes.task/1" @@ -59,29 +63,6 @@ LOCATOR_KINDS = ("depot", "download", "file", "catalog") # an object's primary artifact is the one whose role comes first here; its other artifacts are secondary PRIMARY_ROLES = ("image", "mesh", "audio", "video", "font", "data", "prefab", "json") -MEDIA_TYPES = { - "png": "image/png", - "astc": "image/astc", - "jpg": "image/jpeg", - "json": "application/json", - "glb": "model/gltf-binary", - "flac": "audio/flac", - "wav": "audio/wav", - "ogg": "audio/ogg", - "mkv": "video/x-matroska", - "ivf": "video/x-ivf", - "adx": "audio/x-adx", - "ttf": "font/ttf", - "otf": "font/otf", - "ttc": "font/collection", - "woff": "font/woff", - "woff2": "font/woff2", - "txt": "text/plain", - "xml": "application/xml", - "csv": "text/csv", - "bin": "application/octet-stream", - "bytes": "application/octet-stream", -} _SHA256 = re.compile(r"[0-9a-f]{64}") _EXT = re.compile(r"[0-9a-z]+") _BUNDLE_SUFFIX = re.compile(r"(_[0-9a-f]{32})?\.bundle$") @@ -181,18 +162,6 @@ def is_sha256(s) -> bool: # ---------------------------------------------------------------- identities -def object_id(file: str, path_id: int) -> str: - return f"{file}:{int(path_id)}" - - -def parse_object_id(oid: str) -> tuple[str, int]: - """(serialized file, pathId) of an object id.""" - file, sep, pid = oid.rpartition(":") - if not sep or not file: - raise ValueError(f"not an object id: {oid!r}") - return file, int(pid) - - def artifact_id(owner: str, role: str) -> str: """The id of the artifact `role` of `owner` (an object id, or a task id for artifacts of no object).""" if "#" in owner: @@ -262,10 +231,6 @@ def stable_address( return f"{bundle}:{int(path_id)}" -def media_type(ext: str) -> str: - return MEDIA_TYPES.get(ext.lower(), "application/octet-stream") - - # ---------------------------------------------------------------- tasks @dataclass(frozen=True) class Cost: diff --git a/src/nnnotes/pipeline/cristages.py b/src/nnnotes/pipeline/cristages.py index c80c897..fbb0098 100644 --- a/src/nnnotes/pipeline/cristages.py +++ b/src/nnnotes/pipeline/cristages.py @@ -44,8 +44,8 @@ from ..formats.encoders import FLAC_LEVEL from ..source.apkset import ApkSet +from ..unity.atoms import impl_id from . import contract -from .atoms import impl_id from .contract import Cost, IncompatibleTask, Input from .stages import Output, Pending, Stage diff --git a/src/nnnotes/pipeline/modelstages.py b/src/nnnotes/pipeline/modelstages.py index 1fadad3..290feda 100644 --- a/src/nnnotes/pipeline/modelstages.py +++ b/src/nnnotes/pipeline/modelstages.py @@ -35,8 +35,8 @@ from pathlib import Path from ..source.catalog import Bundle +from ..unity.atoms import impl_id from . import catalogdb, contract, link -from .atoms import impl_id from .contract import Cost, IncompatibleTask, Input from .stages import Output, Pending, Stage diff --git a/src/nnnotes/pipeline/objexport.py b/src/nnnotes/pipeline/objexport.py index a1fbb0d..c82db05 100644 --- a/src/nnnotes/pipeline/objexport.py +++ b/src/nnnotes/pipeline/objexport.py @@ -69,10 +69,10 @@ from types import SimpleNamespace from typing import TYPE_CHECKING, cast +from ..unity.atoms import ATOMS as ATOM_REGISTRY +from ..unity.atoms import Impl, Unsupported, atom_id, resolve from . import census as census_mod from . import contract, link -from .atoms import ATOMS as ATOM_REGISTRY -from .atoms import Impl, Unsupported, atom_id, resolve from .contract import Cost, IncompatibleTask, Input from .stages import Output, Pending, Stage @@ -948,7 +948,7 @@ def _texture_key(self, owner, pptr: dict) -> tuple: return k def conv_texture(self, info, holder: dict) -> list: - from .atoms.texture import ASTC_BLOCKS, format_name, is_hdr_astc + from ..unity.atoms.texture import ASTC_BLOCKS, format_name, is_hdr_astc key = (info.file, info.path_id) try: @@ -983,7 +983,7 @@ def conv_texture(self, info, holder: dict) -> list: def _decode_into(self, info, holder: dict) -> None: """The texture's image for its sprites (a task whose `classes` leave Texture2D out).""" - from .atoms.texture import format_name, is_hdr_astc + from ..unity.atoms.texture import format_name, is_hdr_astc try: inp = self.view.texture_input(info) @@ -997,7 +997,7 @@ def _decoded(self, key: tuple): return self.fns["texture.decode"](self.view.texture_input(self.info[key])) def conv_sprite(self, info, plan: dict, holder: dict) -> list: - from .atoms import sprite as sprite_atom + from ..unity.atoms import sprite as sprite_atom err = holder.get("error") if err is not None: @@ -1060,7 +1060,7 @@ def conv_sprite(self, info, plan: dict, holder: dict) -> list: return [image_id, meta_id] def conv_sprite_external(self, info, plan: dict) -> list: - from .atoms import sprite as sprite_atom + from ..unity.atoms import sprite as sprite_atom key = (info.file, info.path_id) m = self.fns["mesh.arrays"](self.view.mesh_input(info)) @@ -1700,7 +1700,7 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: def run(self, task, depot) -> Output: from PIL import Image - from .atoms import sprite as sprite_atom + from ..unity.atoms import sprite as sprite_atom _check_atoms(task) crop, encode = resolve("sprite.crop").fn, resolve("png.encode").fn diff --git a/src/nnnotes/pipeline/atoms/__init__.py b/src/nnnotes/unity/atoms/__init__.py similarity index 100% rename from src/nnnotes/pipeline/atoms/__init__.py rename to src/nnnotes/unity/atoms/__init__.py diff --git a/src/nnnotes/pipeline/atoms/astc.py b/src/nnnotes/unity/atoms/astc.py similarity index 100% rename from src/nnnotes/pipeline/atoms/astc.py rename to src/nnnotes/unity/atoms/astc.py diff --git a/src/nnnotes/pipeline/atoms/gltf.py b/src/nnnotes/unity/atoms/gltf.py similarity index 100% rename from src/nnnotes/pipeline/atoms/gltf.py rename to src/nnnotes/unity/atoms/gltf.py diff --git a/src/nnnotes/pipeline/atoms/mesh.py b/src/nnnotes/unity/atoms/mesh.py similarity index 100% rename from src/nnnotes/pipeline/atoms/mesh.py rename to src/nnnotes/unity/atoms/mesh.py diff --git a/src/nnnotes/pipeline/atoms/png.py b/src/nnnotes/unity/atoms/png.py similarity index 100% rename from src/nnnotes/pipeline/atoms/png.py rename to src/nnnotes/unity/atoms/png.py diff --git a/src/nnnotes/pipeline/atoms/reader.py b/src/nnnotes/unity/atoms/reader.py similarity index 98% rename from src/nnnotes/pipeline/atoms/reader.py rename to src/nnnotes/unity/atoms/reader.py index 8e55ffc..0cb56ce 100644 --- a/src/nnnotes/pipeline/atoms/reader.py +++ b/src/nnnotes/unity/atoms/reader.py @@ -17,7 +17,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Protocol -from .. import contract +from ..ids import object_id from . import estimate from .mesh import CHANNELS, RAW_LAYOUT_SINCE, MeshInput, Submesh from .texture import TextureInput @@ -37,7 +37,7 @@ class ObjectInfo: @property def id(self) -> str: - return contract.object_id(self.file, self.path_id) + return object_id(self.file, self.path_id) class BundleView(Protocol): @@ -308,7 +308,7 @@ def mesh_input(self, obj: ObjectInfo) -> MeshInput: def open_bundle(data: bytes) -> UnityBundle: """The BundleView of a bundle's bytes (decrypted UnityFS), loaded alone.""" - from ...unity.reader import load_bytes + from ..reader import load_bytes return UnityBundle(load_bytes(bytes(data))) diff --git a/src/nnnotes/pipeline/atoms/sniff.py b/src/nnnotes/unity/atoms/sniff.py similarity index 99% rename from src/nnnotes/pipeline/atoms/sniff.py rename to src/nnnotes/unity/atoms/sniff.py index 79289f4..f43f7d6 100644 --- a/src/nnnotes/pipeline/atoms/sniff.py +++ b/src/nnnotes/unity/atoms/sniff.py @@ -13,7 +13,7 @@ import re from dataclasses import dataclass -from .. import contract +from ...core.media import media_type from . import estimate @@ -99,7 +99,7 @@ class Sniffed: def _media(ext: str, fallback: str = "application/octet-stream") -> str: - m = contract.media_type(ext) + m = media_type(ext) return fallback if m == "application/octet-stream" else m diff --git a/src/nnnotes/pipeline/atoms/sprite.py b/src/nnnotes/unity/atoms/sprite.py similarity index 100% rename from src/nnnotes/pipeline/atoms/sprite.py rename to src/nnnotes/unity/atoms/sprite.py diff --git a/src/nnnotes/pipeline/atoms/texture.py b/src/nnnotes/unity/atoms/texture.py similarity index 100% rename from src/nnnotes/pipeline/atoms/texture.py rename to src/nnnotes/unity/atoms/texture.py diff --git a/src/nnnotes/unity/ids.py b/src/nnnotes/unity/ids.py new file mode 100644 index 0000000..ad9c3e5 --- /dev/null +++ b/src/nnnotes/unity/ids.py @@ -0,0 +1,17 @@ +"""Object ids: an object of a serialized file is `:`.""" + +from __future__ import annotations + +__all__ = ["object_id", "parse_object_id"] + + +def object_id(file: str, path_id: int) -> str: + return f"{file}:{int(path_id)}" + + +def parse_object_id(oid: str) -> tuple[str, int]: + """(serialized file, pathId) of an object id.""" + file, sep, pid = oid.rpartition(":") + if not sep or not file: + raise ValueError(f"not an object id: {oid!r}") + return file, int(pid) diff --git a/tests/pipeline/test_atom_eval.py b/tests/pipeline/test_atom_eval.py index 7537316..3e075f4 100644 --- a/tests/pipeline/test_atom_eval.py +++ b/tests/pipeline/test_atom_eval.py @@ -17,9 +17,9 @@ import atom_eval # noqa: E402 -from nnnotes.pipeline.atoms import Impl, resolve # noqa: E402 -from nnnotes.pipeline.atoms.mesh import Channel, MeshArrays, Primitive # noqa: E402 -from nnnotes.pipeline.atoms.reader import UnityBundle # noqa: E402 +from nnnotes.unity.atoms import Impl, resolve # noqa: E402 +from nnnotes.unity.atoms.mesh import Channel, MeshArrays, Primitive # noqa: E402 +from nnnotes.unity.atoms.reader import UnityBundle # noqa: E402 from tests.helpers.fakeunity import FakeEnvironment, FakeTexture2D, rgba_bytes # noqa: E402 RGBA32, RGB24 = 4, 3 diff --git a/tests/pipeline/test_cristages.py b/tests/pipeline/test_cristages.py index 6881be6..4f2c418 100644 --- a/tests/pipeline/test_cristages.py +++ b/tests/pipeline/test_cristages.py @@ -631,7 +631,7 @@ def run(self, task, depot): class GoldenMovie(MovieStage): @property def ATOMS(self) -> dict: - from nnnotes.pipeline.atoms import impl_id + from nnnotes.unity.atoms import impl_id return { "usm.demux": impl_id(("numpy",), cristages.USM_DEMUX_REVISION), diff --git a/tests/pipeline/test_objexport.py b/tests/pipeline/test_objexport.py index d0019dc..264ac09 100644 --- a/tests/pipeline/test_objexport.py +++ b/tests/pipeline/test_objexport.py @@ -16,12 +16,12 @@ from nnnotes.pipeline import census as census_mod from nnnotes.pipeline import contract, objexport -from nnnotes.pipeline.atoms import Impl, astc, estimate, gltf -from nnnotes.pipeline.atoms.reader import UnityBundle from nnnotes.pipeline.contract import IncompatibleTask, Input, Task from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.objexport import CONVERTERS, CROP, EXPORT, ExportStage, SpriteCropStage, export_atoms from nnnotes.pipeline.stages import Env, describe, execute, registry +from nnnotes.unity.atoms import Impl, astc, estimate, gltf +from nnnotes.unity.atoms.reader import UnityBundle from tests.helpers.fakeunity import ( F32, NORMAL, @@ -734,10 +734,10 @@ def test_textures(exported): def test_sprites_same_bundle(exported): depot, _task, doc = exported - from nnnotes.pipeline.atoms import png, sprite - from nnnotes.pipeline.atoms.mesh import arrays - from nnnotes.pipeline.atoms.reader import mesh_input_from_typetree - from nnnotes.pipeline.atoms.texture import TextureInput, decode + from nnnotes.unity.atoms import png, sprite + from nnnotes.unity.atoms.mesh import arrays + from nnnotes.unity.atoms.reader import mesh_input_from_typetree + from nnnotes.unity.atoms.texture import TextureInput, decode env = bundle_all() f = env.assets[0] diff --git a/tests/pipeline/test_atoms_mesh.py b/tests/unity/test_atoms_mesh.py similarity index 98% rename from tests/pipeline/test_atoms_mesh.py rename to tests/unity/test_atoms_mesh.py index 2e89bd2..f00fbf4 100644 --- a/tests/pipeline/test_atoms_mesh.py +++ b/tests/unity/test_atoms_mesh.py @@ -8,10 +8,10 @@ import pytest from UnityPy.helpers.MeshHelper import MeshHandler -from nnnotes.pipeline.atoms import ATOMS, Unsupported, gltf -from nnnotes.pipeline.atoms.mesh import MeshInput, arrays -from nnnotes.pipeline.atoms.reader import UnityBundle, handler_input, mesh_input_from_typetree from nnnotes.unity import reader as unity +from nnnotes.unity.atoms import ATOMS, Unsupported, gltf +from nnnotes.unity.atoms.mesh import MeshInput, arrays +from nnnotes.unity.atoms.reader import UnityBundle, handler_input, mesh_input_from_typetree from tests.helpers.fakeunity import ( BLENDINDICES, BLENDWEIGHT, diff --git a/tests/pipeline/test_atoms_sprite.py b/tests/unity/test_atoms_sprite.py similarity index 97% rename from tests/pipeline/test_atoms_sprite.py rename to tests/unity/test_atoms_sprite.py index 7adc803..e06f8e9 100644 --- a/tests/pipeline/test_atoms_sprite.py +++ b/tests/unity/test_atoms_sprite.py @@ -7,10 +7,10 @@ import pytest from UnityPy.export.SpriteHelper import get_image_from_sprite -from nnnotes.pipeline.atoms import ATOMS, Unsupported, sprite -from nnnotes.pipeline.atoms.mesh import arrays -from nnnotes.pipeline.atoms.reader import mesh_input_from_typetree -from nnnotes.pipeline.atoms.texture import TextureInput, decode +from nnnotes.unity.atoms import ATOMS, Unsupported, sprite +from nnnotes.unity.atoms.mesh import arrays +from nnnotes.unity.atoms.reader import mesh_input_from_typetree +from nnnotes.unity.atoms.texture import TextureInput, decode from tests.helpers.fakeunity import ( ANDROID, VERSION, diff --git a/tests/pipeline/test_atoms_texture.py b/tests/unity/test_atoms_texture.py similarity index 94% rename from tests/pipeline/test_atoms_texture.py rename to tests/unity/test_atoms_texture.py index 6d4f6dc..a87d90b 100644 --- a/tests/pipeline/test_atoms_texture.py +++ b/tests/unity/test_atoms_texture.py @@ -13,10 +13,10 @@ from PIL import Image from UnityPy.export.Texture2DConverter import parse_image_data -from nnnotes.pipeline.atoms import ATOMS, Impl, Unsupported, astc, env_name, ids, impl_id, png, resolve -from nnnotes.pipeline.atoms.reader import ObjectInfo, UnityBundle -from nnnotes.pipeline.atoms.texture import ASTC_BLOCKS, TextureInput, decode, is_hdr_astc from nnnotes.unity import export +from nnnotes.unity.atoms import ATOMS, Impl, Unsupported, astc, env_name, ids, impl_id, png, resolve +from nnnotes.unity.atoms.reader import ObjectInfo, UnityBundle +from nnnotes.unity.atoms.texture import ASTC_BLOCKS, TextureInput, decode, is_hdr_astc from tests.helpers.fakeunity import ANDROID, VERSION, FakeEnvironment, FakeTexture2D, rgba_bytes RGBA32, RGB24, ALPHA8, ARGB4444, YUY2 = 4, 3, 1, 2, 21 @@ -255,7 +255,7 @@ def test_every_atom_has_one_reference_with_versioned_id(): def test_evaluation_override(monkeypatch): assert env_name("png.encode") == "NNNOTES_ATOM_PNG_ENCODE" - monkeypatch.setenv("NNNOTES_ATOM_PNG_ENCODE", "tests.pipeline.test_atoms_texture:REPLACEMENT") + monkeypatch.setenv("NNNOTES_ATOM_PNG_ENCODE", "tests.unity.test_atoms_texture:REPLACEMENT") assert resolve("png.encode") is REPLACEMENT and ATOMS["png.encode"].fn(None) == b"replaced" assert ids("png.encode") == {"png.encode": "pillow-test/9"} # keys change with the override monkeypatch.setenv("NNNOTES_ATOM_PNG_ENCODE", "no-colon") @@ -268,17 +268,17 @@ def test_evaluation_override(monkeypatch): def test_override_with_the_reference_id_is_refused(monkeypatch): - import tests.pipeline.test_atoms_texture as me + import tests.unity.test_atoms_texture as me monkeypatch.setattr(me, "SAME", Impl(ATOMS["png.encode"].id, png.encode, png.cost), raising=False) - monkeypatch.setenv("NNNOTES_ATOM_PNG_ENCODE", "tests.pipeline.test_atoms_texture:SAME") + monkeypatch.setenv("NNNOTES_ATOM_PNG_ENCODE", "tests.unity.test_atoms_texture:SAME") with pytest.raises(ValueError, match="reference's id"): resolve("png.encode") def test_listing_the_registry_loads_no_image_or_unity_library(): code = ( - "import sys, nnnotes.pipeline.atoms as a; a.ids(*a.ATOMS); " + "import sys, nnnotes.unity.atoms as a; a.ids(*a.ATOMS); " "print(sorted(m for m in ('UnityPy', 'PIL') if m in sys.modules))" ) out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True).stdout @@ -296,7 +296,7 @@ def test_every_atom_and_reason_code_is_documented(): assert documented == set(ATOMS) codes = set(re.findall(r"^\| `([a-zA-Z_.]+)` \|", doc.split("## Reason codes", 1)[1], re.M)) emitted = set() - for src in (root / "src" / "nnnotes" / "pipeline" / "atoms").glob("*.py"): + for src in (root / "src" / "nnnotes" / "unity" / "atoms").glob("*.py"): emitted |= set( re.findall( r"\"((?:empty|unsupported|partial|generic|no_data)\.[a-zA-Z_.]+)\"", src.read_text(encoding="utf-8") diff --git a/tests/pipeline/test_sniff.py b/tests/unity/test_sniff.py similarity index 97% rename from tests/pipeline/test_sniff.py rename to tests/unity/test_sniff.py index bc3df9b..1c9d70b 100644 --- a/tests/pipeline/test_sniff.py +++ b/tests/unity/test_sniff.py @@ -9,8 +9,8 @@ import pytest from nnnotes.pipeline import contract -from nnnotes.pipeline.atoms import ATOMS -from nnnotes.pipeline.atoms.sniff import MAGIC, Sniffed, sniff, spine_binary_version +from nnnotes.unity.atoms import ATOMS +from nnnotes.unity.atoms.sniff import MAGIC, Sniffed, sniff, spine_binary_version def sfnt(tag: bytes, tables: int = 3) -> bytes: