diff --git a/docs/architecture.md b/docs/architecture.md index 8b5646b..1aeb4a7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -12,10 +12,10 @@ subpackage of `nnnotes` and imports only the layers below it. | Products | `nnnotes.site`, `nnnotes.musicdata` | the ournotes-player site (charts, models, stories, indexes); the music data file and its deck statistics | | Pipeline | `nnnotes.pipeline` | stages, task descriptions and results, the content-addressed depot, plans, layouts, the catalog versions, the views of the master data | | Content | `nnnotes.content` | the game's subjects as exports: ADV episodes and stories, lives and scores, spots, rooms, the UI library | -| Unity objects | `nnnotes.unity` | bundle reading, the typetree exporter, the player data, Live2D prefabs, text styles and TextMesh Pro fonts | +| Unity objects | `nnnotes.unity` | bundle reading, object ids, the atoms (the pure functions objects are converted with: texture decoding, sprite images, mesh arrays, glTF, PNG, file sniffing), the typetree exporter, the player data, Live2D prefabs, text styles and TextMesh Pro fonts | | File formats | `nnnotes.formats` | formats read from bytes: CRI ACB/AWB and HCA, USM movies, CRI Lips data, shaders, motions | | Sources | `nnnotes.source` | the configuration, the APK, Addressables catalogs and bundles, the game API, master data, languages, `Session` | -| Core | `nnnotes.core` | the one JSON writer and the process caches | +| Core | `nnnotes.core` | the one JSON writer, media types, the process caches and the downloads cache | The order is a contract: the `layers` contract of [import-linter](https://import-linter.readthedocs.io/) in `pyproject.toml` fails the build when a module imports a layer above its own, and `nnnotes.site` and diff --git a/docs/stages.md b/docs/stages.md index 16d9e9f..fadf601 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -128,14 +128,14 @@ else found on PATH. The `boot` input is the APK's `assets/bin/Data/data.unity3d` ## Atoms -`nnnotes.pipeline.atoms.ATOMS[name]` is the reference implementation of an atom: its id, its function and its cost estimate. +`nnnotes.unity.atoms.ATOMS[name]` is the reference implementation of an atom: its id, its function and its cost estimate. The id names the libraries whose behaviour the output depends on, with their installed versions, and the revision of the nnnotes code (`pillow-12.1.1/1`); `nnnotes/` when no library's behaviour is involved. Task keys include the ids of the atoms a task uses, so upgrading a library or changing an atom re-runs exactly the tasks that use it. | Atom | Reference implementation | Id | Does | |---|---|---|---| -| `reader` | `nnnotes.pipeline.atoms.reader.open_bundle` (UnityPy) | `unitypy-/1` | one bundle loaded alone from its bytes: serialized files, objects, raw bytes, typetrees, container, externals, and the inputs of `texture.decode` and `mesh.arrays` (resource files are read from inside the bundle only) | +| `reader` | `nnnotes.unity.atoms.reader.open_bundle` (UnityPy) | `unitypy-/1` | one bundle loaded alone from its bytes: serialized files, objects, raw bytes, typetrees, container, externals, and the inputs of `texture.decode` and `mesh.arrays` (resource files are read from inside the bundle only) | | `texture.decode` | UnityPy `Texture2DConverter.parse_image_data` | `unitypy-+texture2ddecoder-+astc-encoder-py-+pillow-/1` | the first mip level, top row first, as UnityPy's `Texture2D.image` | | `png.encode` | Pillow | `pillow-/1` | PNG bytes; level 6 (the default) gives Pillow's default bytes, the ones the other commands write | | `astc.container` | nnnotes | `nnnotes/1` | the standard `.astc` file (16-byte header, first mip level's blocks) | diff --git a/scripts/atom_eval.py b/scripts/atom_eval.py index ece713e..1e23446 100644 --- a/scripts/atom_eval.py +++ b/scripts/atom_eval.py @@ -2,7 +2,7 @@ """Measure candidate implementations of the export atoms against the reference implementations (maintainer tool; not installed, not needed to use nnnotes). -`nnnotes.pipeline.atoms.ATOMS` holds one reference implementation per atom; `NNNOTES_ATOM_=module:attribute` names +`nnnotes.unity.atoms.ATOMS` holds one reference implementation per atom; `NNNOTES_ATOM_=module:attribute` names another one for evaluation (docs/stages.md, "Atoms"). This script gives the numbers a switch is decided on: sample DIR -o FILE a seeded stratified sample of a bundle directory: one bundle of every @@ -42,7 +42,7 @@ A candidate view is the reader candidate's result; the optional methods census(), text_bytes(obj), font_bytes(obj), sprite_image(obj) and the attribute typetree_form serve the checks that have no atom of their own. A candidate -raising NotImplementedError has no such facet ("absent"); `nnnotes.pipeline.atoms.Unsupported` is a refusal. +raising NotImplementedError has no such facet ("absent"); `nnnotes.unity.atoms.Unsupported` is a refusal. Statuses: equal, differ, refused (the candidate raised where the reference gave a result), refused.both, extra (a result where the reference refused), absent, reference-error. Output: one JSON document (-o); m1 and m3 also write @@ -133,7 +133,7 @@ def check_clean_env() -> None: def load_impl(spec: str): """The Impl named by module:attribute (an Impl, or a callable returning one).""" - from nnnotes.pipeline.atoms import Impl + from nnnotes.unity.atoms import Impl module, sep, attr = spec.partition(":") if not sep or not module or not attr: @@ -146,7 +146,7 @@ def load_impl(spec: str): def parse_atoms(items) -> list[tuple[str, str]]: - from nnnotes.pipeline.atoms import ATOMS + from nnnotes.unity.atoms import ATOMS out = [] for it in items or (): @@ -449,7 +449,7 @@ def cmd_sample(a) -> dict: # ---------------------------------------------------------------- m1 def m1_init(cfg: dict) -> None: - from nnnotes.pipeline.atoms import resolve + from nnnotes.unity.atoms import resolve G.clear() G.update(cfg) @@ -477,7 +477,7 @@ def _key(info) -> tuple: def _call(fn, *args): """(result, None) or (None, refusal) where refusal is the Unsupported code or the error's message.""" - from nnnotes.pipeline.atoms import Unsupported + from nnnotes.unity.atoms import Unsupported try: return fn(*args), None @@ -700,7 +700,7 @@ def hdr_reference(inp): import numpy as np from PIL import Image - from nnnotes.pipeline.atoms.texture import ASTC_BLOCKS + from nnnotes.unity.atoms.texture import ASTC_BLOCKS bx, by = ASTC_BLOCKS[int(inp.format)] w, h = int(inp.width), int(inp.height) @@ -723,7 +723,7 @@ def _hdr_library_image(inp): def check_textures(T: Tally, ref, cv, infos, cmap, checks) -> None: - from nnnotes.pipeline.atoms.texture import format_name, is_hdr_astc + from nnnotes.unity.atoms.texture import format_name, is_hdr_astc ref_dec, ref_png = G["ref"]["texture.decode"], G["ref"]["png.encode"] decs, pngs = G["cand"].get("texture.decode", []), G["cand"].get("png.encode", []) @@ -1173,7 +1173,7 @@ def _m3_worker(task) -> dict: def _apply_atoms(atoms) -> None: - from nnnotes.pipeline.atoms import env_name, resolve + from nnnotes.unity.atoms import env_name, resolve for name, spec in atoms: os.environ[env_name(name)] = spec diff --git a/scripts/atom_eval_pillow.py b/scripts/atom_eval_pillow.py index ec5d9f4..b5741b4 100644 --- a/scripts/atom_eval_pillow.py +++ b/scripts/atom_eval_pillow.py @@ -1,6 +1,6 @@ """Candidate `png.encode` implementations on Pillow, for scripts/atom_eval.py only. -The reference (`nnnotes.pipeline.atoms.png.encode`) writes at the level a task's `png.level` parameter gives (6 by +The reference (`nnnotes.unity.atoms.png.encode`) writes at the level a task's `png.level` parameter gives (6 by default). These fix the setting instead, so several can be measured side by side in one `atom_eval.py m1` run: level1, level3, level6, level9 Pillow at that zlib level @@ -15,8 +15,8 @@ from functools import cache from importlib import metadata -from nnnotes.pipeline.atoms import Impl -from nnnotes.pipeline.atoms import png as png_atom +from nnnotes.unity.atoms import Impl +from nnnotes.unity.atoms import png as png_atom def _pillow() -> str: diff --git a/scripts/atom_eval_unityrs.py b/scripts/atom_eval_unityrs.py index d403c35..e544f1a 100644 --- a/scripts/atom_eval_unityrs.py +++ b/scripts/atom_eval_unityrs.py @@ -27,13 +27,13 @@ from functools import cache from importlib import metadata -from nnnotes.pipeline.atoms import Impl, Unsupported -from nnnotes.pipeline.atoms import png as png_atom -from nnnotes.pipeline.atoms import reader as reader_atom -from nnnotes.pipeline.atoms import texture as texture_atom -from nnnotes.pipeline.atoms.mesh import MeshInput -from nnnotes.pipeline.atoms.reader import ObjectInfo, UnityBundle -from nnnotes.pipeline.atoms.texture import TextureInput +from nnnotes.unity.atoms import Impl, Unsupported +from nnnotes.unity.atoms import png as png_atom +from nnnotes.unity.atoms import reader as reader_atom +from nnnotes.unity.atoms import texture as texture_atom +from nnnotes.unity.atoms.mesh import MeshInput +from nnnotes.unity.atoms.reader import ObjectInfo, UnityBundle +from nnnotes.unity.atoms.texture import TextureInput FALLBACKS = {"png": 0} _KEEP = "unity_rs.image" @@ -390,7 +390,7 @@ def png_encode_fast() -> Impl: def _mesh(inp: MeshInput): import numpy as np - from nnnotes.pipeline.atoms.mesh import Channel, MeshArrays, Primitive + from nnnotes.unity.atoms.mesh import Channel, MeshArrays, Primitive src = getattr(inp, "source", None) if src is None: @@ -429,6 +429,6 @@ def _mesh(inp: MeshInput): @cache def mesh_arrays() -> Impl: - from nnnotes.pipeline.atoms import mesh as mesh_atom + from nnnotes.unity.atoms import mesh as mesh_atom return Impl(_id(_rs(), "obj"), _mesh, mesh_atom.cost) diff --git a/src/nnnotes/core/media.py b/src/nnnotes/core/media.py new file mode 100644 index 0000000..5e96322 --- /dev/null +++ b/src/nnnotes/core/media.py @@ -0,0 +1,34 @@ +"""Media types of file extensions.""" + +from __future__ import annotations + +__all__ = ["MEDIA_TYPES", "media_type"] + +MEDIA_TYPES = { + "png": "image/png", + "astc": "image/astc", + "jpg": "image/jpeg", + "json": "application/json", + "glb": "model/gltf-binary", + "flac": "audio/flac", + "wav": "audio/wav", + "ogg": "audio/ogg", + "mkv": "video/x-matroska", + "ivf": "video/x-ivf", + "adx": "audio/x-adx", + "ttf": "font/ttf", + "otf": "font/otf", + "ttc": "font/collection", + "woff": "font/woff", + "woff2": "font/woff2", + "txt": "text/plain", + "xml": "application/xml", + "csv": "text/csv", + "bin": "application/octet-stream", + "bytes": "application/octet-stream", +} + + +def media_type(ext: str) -> str: + """The media type of a file extension (case-insensitive); application/octet-stream when unknown.""" + return MEDIA_TYPES.get(ext.lower(), "application/octet-stream") diff --git a/src/nnnotes/pipeline/census.py b/src/nnnotes/pipeline/census.py index 2875e36..23a75cb 100644 --- a/src/nnnotes/pipeline/census.py +++ b/src/nnnotes/pipeline/census.py @@ -25,8 +25,8 @@ import re +from ..unity.atoms import impl_id from . import contract -from .atoms import impl_id from .contract import Cost, Input from .stages import Output, Pending, Stage diff --git a/src/nnnotes/pipeline/contract.py b/src/nnnotes/pipeline/contract.py index 8c958c8..68d6f26 100644 --- a/src/nnnotes/pipeline/contract.py +++ b/src/nnnotes/pipeline/contract.py @@ -35,6 +35,10 @@ from functools import cached_property from ..core import jsonio +from ..core.media import MEDIA_TYPES as MEDIA_TYPES +from ..core.media import media_type as media_type +from ..unity.ids import object_id as object_id +from ..unity.ids import parse_object_id as parse_object_id CANON = "canon/1" TASK = "nnnotes.task/1" @@ -59,29 +63,6 @@ LOCATOR_KINDS = ("depot", "download", "file", "catalog") # an object's primary artifact is the one whose role comes first here; its other artifacts are secondary PRIMARY_ROLES = ("image", "mesh", "audio", "video", "font", "data", "prefab", "json") -MEDIA_TYPES = { - "png": "image/png", - "astc": "image/astc", - "jpg": "image/jpeg", - "json": "application/json", - "glb": "model/gltf-binary", - "flac": "audio/flac", - "wav": "audio/wav", - "ogg": "audio/ogg", - "mkv": "video/x-matroska", - "ivf": "video/x-ivf", - "adx": "audio/x-adx", - "ttf": "font/ttf", - "otf": "font/otf", - "ttc": "font/collection", - "woff": "font/woff", - "woff2": "font/woff2", - "txt": "text/plain", - "xml": "application/xml", - "csv": "text/csv", - "bin": "application/octet-stream", - "bytes": "application/octet-stream", -} _SHA256 = re.compile(r"[0-9a-f]{64}") _EXT = re.compile(r"[0-9a-z]+") _BUNDLE_SUFFIX = re.compile(r"(_[0-9a-f]{32})?\.bundle$") @@ -181,18 +162,6 @@ def is_sha256(s) -> bool: # ---------------------------------------------------------------- identities -def object_id(file: str, path_id: int) -> str: - return f"{file}:{int(path_id)}" - - -def parse_object_id(oid: str) -> tuple[str, int]: - """(serialized file, pathId) of an object id.""" - file, sep, pid = oid.rpartition(":") - if not sep or not file: - raise ValueError(f"not an object id: {oid!r}") - return file, int(pid) - - def artifact_id(owner: str, role: str) -> str: """The id of the artifact `role` of `owner` (an object id, or a task id for artifacts of no object).""" if "#" in owner: @@ -262,10 +231,6 @@ def stable_address( return f"{bundle}:{int(path_id)}" -def media_type(ext: str) -> str: - return MEDIA_TYPES.get(ext.lower(), "application/octet-stream") - - # ---------------------------------------------------------------- tasks @dataclass(frozen=True) class Cost: diff --git a/src/nnnotes/pipeline/cristages.py b/src/nnnotes/pipeline/cristages.py index c80c897..fbb0098 100644 --- a/src/nnnotes/pipeline/cristages.py +++ b/src/nnnotes/pipeline/cristages.py @@ -44,8 +44,8 @@ from ..formats.encoders import FLAC_LEVEL from ..source.apkset import ApkSet +from ..unity.atoms import impl_id from . import contract -from .atoms import impl_id from .contract import Cost, IncompatibleTask, Input from .stages import Output, Pending, Stage diff --git a/src/nnnotes/pipeline/modelstages.py b/src/nnnotes/pipeline/modelstages.py index 1fadad3..290feda 100644 --- a/src/nnnotes/pipeline/modelstages.py +++ b/src/nnnotes/pipeline/modelstages.py @@ -35,8 +35,8 @@ from pathlib import Path from ..source.catalog import Bundle +from ..unity.atoms import impl_id from . import catalogdb, contract, link -from .atoms import impl_id from .contract import Cost, IncompatibleTask, Input from .stages import Output, Pending, Stage diff --git a/src/nnnotes/pipeline/objexport.py b/src/nnnotes/pipeline/objexport.py index a1fbb0d..c82db05 100644 --- a/src/nnnotes/pipeline/objexport.py +++ b/src/nnnotes/pipeline/objexport.py @@ -69,10 +69,10 @@ from types import SimpleNamespace from typing import TYPE_CHECKING, cast +from ..unity.atoms import ATOMS as ATOM_REGISTRY +from ..unity.atoms import Impl, Unsupported, atom_id, resolve from . import census as census_mod from . import contract, link -from .atoms import ATOMS as ATOM_REGISTRY -from .atoms import Impl, Unsupported, atom_id, resolve from .contract import Cost, IncompatibleTask, Input from .stages import Output, Pending, Stage @@ -948,7 +948,7 @@ def _texture_key(self, owner, pptr: dict) -> tuple: return k def conv_texture(self, info, holder: dict) -> list: - from .atoms.texture import ASTC_BLOCKS, format_name, is_hdr_astc + from ..unity.atoms.texture import ASTC_BLOCKS, format_name, is_hdr_astc key = (info.file, info.path_id) try: @@ -983,7 +983,7 @@ def conv_texture(self, info, holder: dict) -> list: def _decode_into(self, info, holder: dict) -> None: """The texture's image for its sprites (a task whose `classes` leave Texture2D out).""" - from .atoms.texture import format_name, is_hdr_astc + from ..unity.atoms.texture import format_name, is_hdr_astc try: inp = self.view.texture_input(info) @@ -997,7 +997,7 @@ def _decoded(self, key: tuple): return self.fns["texture.decode"](self.view.texture_input(self.info[key])) def conv_sprite(self, info, plan: dict, holder: dict) -> list: - from .atoms import sprite as sprite_atom + from ..unity.atoms import sprite as sprite_atom err = holder.get("error") if err is not None: @@ -1060,7 +1060,7 @@ def conv_sprite(self, info, plan: dict, holder: dict) -> list: return [image_id, meta_id] def conv_sprite_external(self, info, plan: dict) -> list: - from .atoms import sprite as sprite_atom + from ..unity.atoms import sprite as sprite_atom key = (info.file, info.path_id) m = self.fns["mesh.arrays"](self.view.mesh_input(info)) @@ -1700,7 +1700,7 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: def run(self, task, depot) -> Output: from PIL import Image - from .atoms import sprite as sprite_atom + from ..unity.atoms import sprite as sprite_atom _check_atoms(task) crop, encode = resolve("sprite.crop").fn, resolve("png.encode").fn diff --git a/src/nnnotes/pipeline/atoms/__init__.py b/src/nnnotes/unity/atoms/__init__.py similarity index 100% rename from src/nnnotes/pipeline/atoms/__init__.py rename to src/nnnotes/unity/atoms/__init__.py diff --git a/src/nnnotes/pipeline/atoms/astc.py b/src/nnnotes/unity/atoms/astc.py similarity index 100% rename from src/nnnotes/pipeline/atoms/astc.py rename to src/nnnotes/unity/atoms/astc.py diff --git a/src/nnnotes/pipeline/atoms/gltf.py b/src/nnnotes/unity/atoms/gltf.py similarity index 100% rename from src/nnnotes/pipeline/atoms/gltf.py rename to src/nnnotes/unity/atoms/gltf.py diff --git a/src/nnnotes/pipeline/atoms/mesh.py b/src/nnnotes/unity/atoms/mesh.py similarity index 100% rename from src/nnnotes/pipeline/atoms/mesh.py rename to src/nnnotes/unity/atoms/mesh.py diff --git a/src/nnnotes/pipeline/atoms/png.py b/src/nnnotes/unity/atoms/png.py similarity index 100% rename from src/nnnotes/pipeline/atoms/png.py rename to src/nnnotes/unity/atoms/png.py diff --git a/src/nnnotes/pipeline/atoms/reader.py b/src/nnnotes/unity/atoms/reader.py similarity index 98% rename from src/nnnotes/pipeline/atoms/reader.py rename to src/nnnotes/unity/atoms/reader.py index 8e55ffc..0cb56ce 100644 --- a/src/nnnotes/pipeline/atoms/reader.py +++ b/src/nnnotes/unity/atoms/reader.py @@ -17,7 +17,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Protocol -from .. import contract +from ..ids import object_id from . import estimate from .mesh import CHANNELS, RAW_LAYOUT_SINCE, MeshInput, Submesh from .texture import TextureInput @@ -37,7 +37,7 @@ class ObjectInfo: @property def id(self) -> str: - return contract.object_id(self.file, self.path_id) + return object_id(self.file, self.path_id) class BundleView(Protocol): @@ -308,7 +308,7 @@ def mesh_input(self, obj: ObjectInfo) -> MeshInput: def open_bundle(data: bytes) -> UnityBundle: """The BundleView of a bundle's bytes (decrypted UnityFS), loaded alone.""" - from ...unity.reader import load_bytes + from ..reader import load_bytes return UnityBundle(load_bytes(bytes(data))) diff --git a/src/nnnotes/pipeline/atoms/sniff.py b/src/nnnotes/unity/atoms/sniff.py similarity index 99% rename from src/nnnotes/pipeline/atoms/sniff.py rename to src/nnnotes/unity/atoms/sniff.py index 79289f4..f43f7d6 100644 --- a/src/nnnotes/pipeline/atoms/sniff.py +++ b/src/nnnotes/unity/atoms/sniff.py @@ -13,7 +13,7 @@ import re from dataclasses import dataclass -from .. import contract +from ...core.media import media_type from . import estimate @@ -99,7 +99,7 @@ class Sniffed: def _media(ext: str, fallback: str = "application/octet-stream") -> str: - m = contract.media_type(ext) + m = media_type(ext) return fallback if m == "application/octet-stream" else m diff --git a/src/nnnotes/pipeline/atoms/sprite.py b/src/nnnotes/unity/atoms/sprite.py similarity index 100% rename from src/nnnotes/pipeline/atoms/sprite.py rename to src/nnnotes/unity/atoms/sprite.py diff --git a/src/nnnotes/pipeline/atoms/texture.py b/src/nnnotes/unity/atoms/texture.py similarity index 100% rename from src/nnnotes/pipeline/atoms/texture.py rename to src/nnnotes/unity/atoms/texture.py diff --git a/src/nnnotes/unity/ids.py b/src/nnnotes/unity/ids.py new file mode 100644 index 0000000..ad9c3e5 --- /dev/null +++ b/src/nnnotes/unity/ids.py @@ -0,0 +1,17 @@ +"""Object ids: an object of a serialized file is `:`.""" + +from __future__ import annotations + +__all__ = ["object_id", "parse_object_id"] + + +def object_id(file: str, path_id: int) -> str: + return f"{file}:{int(path_id)}" + + +def parse_object_id(oid: str) -> tuple[str, int]: + """(serialized file, pathId) of an object id.""" + file, sep, pid = oid.rpartition(":") + if not sep or not file: + raise ValueError(f"not an object id: {oid!r}") + return file, int(pid) diff --git a/tests/pipeline/test_atom_eval.py b/tests/pipeline/test_atom_eval.py index 7537316..3e075f4 100644 --- a/tests/pipeline/test_atom_eval.py +++ b/tests/pipeline/test_atom_eval.py @@ -17,9 +17,9 @@ import atom_eval # noqa: E402 -from nnnotes.pipeline.atoms import Impl, resolve # noqa: E402 -from nnnotes.pipeline.atoms.mesh import Channel, MeshArrays, Primitive # noqa: E402 -from nnnotes.pipeline.atoms.reader import UnityBundle # noqa: E402 +from nnnotes.unity.atoms import Impl, resolve # noqa: E402 +from nnnotes.unity.atoms.mesh import Channel, MeshArrays, Primitive # noqa: E402 +from nnnotes.unity.atoms.reader import UnityBundle # noqa: E402 from tests.helpers.fakeunity import FakeEnvironment, FakeTexture2D, rgba_bytes # noqa: E402 RGBA32, RGB24 = 4, 3 diff --git a/tests/pipeline/test_cristages.py b/tests/pipeline/test_cristages.py index 6881be6..4f2c418 100644 --- a/tests/pipeline/test_cristages.py +++ b/tests/pipeline/test_cristages.py @@ -631,7 +631,7 @@ def run(self, task, depot): class GoldenMovie(MovieStage): @property def ATOMS(self) -> dict: - from nnnotes.pipeline.atoms import impl_id + from nnnotes.unity.atoms import impl_id return { "usm.demux": impl_id(("numpy",), cristages.USM_DEMUX_REVISION), diff --git a/tests/pipeline/test_objexport.py b/tests/pipeline/test_objexport.py index d0019dc..264ac09 100644 --- a/tests/pipeline/test_objexport.py +++ b/tests/pipeline/test_objexport.py @@ -16,12 +16,12 @@ from nnnotes.pipeline import census as census_mod from nnnotes.pipeline import contract, objexport -from nnnotes.pipeline.atoms import Impl, astc, estimate, gltf -from nnnotes.pipeline.atoms.reader import UnityBundle from nnnotes.pipeline.contract import IncompatibleTask, Input, Task from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.objexport import CONVERTERS, CROP, EXPORT, ExportStage, SpriteCropStage, export_atoms from nnnotes.pipeline.stages import Env, describe, execute, registry +from nnnotes.unity.atoms import Impl, astc, estimate, gltf +from nnnotes.unity.atoms.reader import UnityBundle from tests.helpers.fakeunity import ( F32, NORMAL, @@ -734,10 +734,10 @@ def test_textures(exported): def test_sprites_same_bundle(exported): depot, _task, doc = exported - from nnnotes.pipeline.atoms import png, sprite - from nnnotes.pipeline.atoms.mesh import arrays - from nnnotes.pipeline.atoms.reader import mesh_input_from_typetree - from nnnotes.pipeline.atoms.texture import TextureInput, decode + from nnnotes.unity.atoms import png, sprite + from nnnotes.unity.atoms.mesh import arrays + from nnnotes.unity.atoms.reader import mesh_input_from_typetree + from nnnotes.unity.atoms.texture import TextureInput, decode env = bundle_all() f = env.assets[0] diff --git a/tests/pipeline/test_atoms_mesh.py b/tests/unity/test_atoms_mesh.py similarity index 98% rename from tests/pipeline/test_atoms_mesh.py rename to tests/unity/test_atoms_mesh.py index 2e89bd2..f00fbf4 100644 --- a/tests/pipeline/test_atoms_mesh.py +++ b/tests/unity/test_atoms_mesh.py @@ -8,10 +8,10 @@ import pytest from UnityPy.helpers.MeshHelper import MeshHandler -from nnnotes.pipeline.atoms import ATOMS, Unsupported, gltf -from nnnotes.pipeline.atoms.mesh import MeshInput, arrays -from nnnotes.pipeline.atoms.reader import UnityBundle, handler_input, mesh_input_from_typetree from nnnotes.unity import reader as unity +from nnnotes.unity.atoms import ATOMS, Unsupported, gltf +from nnnotes.unity.atoms.mesh import MeshInput, arrays +from nnnotes.unity.atoms.reader import UnityBundle, handler_input, mesh_input_from_typetree from tests.helpers.fakeunity import ( BLENDINDICES, BLENDWEIGHT, diff --git a/tests/pipeline/test_atoms_sprite.py b/tests/unity/test_atoms_sprite.py similarity index 97% rename from tests/pipeline/test_atoms_sprite.py rename to tests/unity/test_atoms_sprite.py index 7adc803..e06f8e9 100644 --- a/tests/pipeline/test_atoms_sprite.py +++ b/tests/unity/test_atoms_sprite.py @@ -7,10 +7,10 @@ import pytest from UnityPy.export.SpriteHelper import get_image_from_sprite -from nnnotes.pipeline.atoms import ATOMS, Unsupported, sprite -from nnnotes.pipeline.atoms.mesh import arrays -from nnnotes.pipeline.atoms.reader import mesh_input_from_typetree -from nnnotes.pipeline.atoms.texture import TextureInput, decode +from nnnotes.unity.atoms import ATOMS, Unsupported, sprite +from nnnotes.unity.atoms.mesh import arrays +from nnnotes.unity.atoms.reader import mesh_input_from_typetree +from nnnotes.unity.atoms.texture import TextureInput, decode from tests.helpers.fakeunity import ( ANDROID, VERSION, diff --git a/tests/pipeline/test_atoms_texture.py b/tests/unity/test_atoms_texture.py similarity index 94% rename from tests/pipeline/test_atoms_texture.py rename to tests/unity/test_atoms_texture.py index 6d4f6dc..a87d90b 100644 --- a/tests/pipeline/test_atoms_texture.py +++ b/tests/unity/test_atoms_texture.py @@ -13,10 +13,10 @@ from PIL import Image from UnityPy.export.Texture2DConverter import parse_image_data -from nnnotes.pipeline.atoms import ATOMS, Impl, Unsupported, astc, env_name, ids, impl_id, png, resolve -from nnnotes.pipeline.atoms.reader import ObjectInfo, UnityBundle -from nnnotes.pipeline.atoms.texture import ASTC_BLOCKS, TextureInput, decode, is_hdr_astc from nnnotes.unity import export +from nnnotes.unity.atoms import ATOMS, Impl, Unsupported, astc, env_name, ids, impl_id, png, resolve +from nnnotes.unity.atoms.reader import ObjectInfo, UnityBundle +from nnnotes.unity.atoms.texture import ASTC_BLOCKS, TextureInput, decode, is_hdr_astc from tests.helpers.fakeunity import ANDROID, VERSION, FakeEnvironment, FakeTexture2D, rgba_bytes RGBA32, RGB24, ALPHA8, ARGB4444, YUY2 = 4, 3, 1, 2, 21 @@ -255,7 +255,7 @@ def test_every_atom_has_one_reference_with_versioned_id(): def test_evaluation_override(monkeypatch): assert env_name("png.encode") == "NNNOTES_ATOM_PNG_ENCODE" - monkeypatch.setenv("NNNOTES_ATOM_PNG_ENCODE", "tests.pipeline.test_atoms_texture:REPLACEMENT") + monkeypatch.setenv("NNNOTES_ATOM_PNG_ENCODE", "tests.unity.test_atoms_texture:REPLACEMENT") assert resolve("png.encode") is REPLACEMENT and ATOMS["png.encode"].fn(None) == b"replaced" assert ids("png.encode") == {"png.encode": "pillow-test/9"} # keys change with the override monkeypatch.setenv("NNNOTES_ATOM_PNG_ENCODE", "no-colon") @@ -268,17 +268,17 @@ def test_evaluation_override(monkeypatch): def test_override_with_the_reference_id_is_refused(monkeypatch): - import tests.pipeline.test_atoms_texture as me + import tests.unity.test_atoms_texture as me monkeypatch.setattr(me, "SAME", Impl(ATOMS["png.encode"].id, png.encode, png.cost), raising=False) - monkeypatch.setenv("NNNOTES_ATOM_PNG_ENCODE", "tests.pipeline.test_atoms_texture:SAME") + monkeypatch.setenv("NNNOTES_ATOM_PNG_ENCODE", "tests.unity.test_atoms_texture:SAME") with pytest.raises(ValueError, match="reference's id"): resolve("png.encode") def test_listing_the_registry_loads_no_image_or_unity_library(): code = ( - "import sys, nnnotes.pipeline.atoms as a; a.ids(*a.ATOMS); " + "import sys, nnnotes.unity.atoms as a; a.ids(*a.ATOMS); " "print(sorted(m for m in ('UnityPy', 'PIL') if m in sys.modules))" ) out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True).stdout @@ -296,7 +296,7 @@ def test_every_atom_and_reason_code_is_documented(): assert documented == set(ATOMS) codes = set(re.findall(r"^\| `([a-zA-Z_.]+)` \|", doc.split("## Reason codes", 1)[1], re.M)) emitted = set() - for src in (root / "src" / "nnnotes" / "pipeline" / "atoms").glob("*.py"): + for src in (root / "src" / "nnnotes" / "unity" / "atoms").glob("*.py"): emitted |= set( re.findall( r"\"((?:empty|unsupported|partial|generic|no_data)\.[a-zA-Z_.]+)\"", src.read_text(encoding="utf-8") diff --git a/tests/pipeline/test_sniff.py b/tests/unity/test_sniff.py similarity index 97% rename from tests/pipeline/test_sniff.py rename to tests/unity/test_sniff.py index bc3df9b..1c9d70b 100644 --- a/tests/pipeline/test_sniff.py +++ b/tests/unity/test_sniff.py @@ -9,8 +9,8 @@ import pytest from nnnotes.pipeline import contract -from nnnotes.pipeline.atoms import ATOMS -from nnnotes.pipeline.atoms.sniff import MAGIC, Sniffed, sniff, spine_binary_version +from nnnotes.unity.atoms import ATOMS +from nnnotes.unity.atoms.sniff import MAGIC, Sniffed, sniff, spine_binary_version def sfnt(tag: bytes, tables: int = 3) -> bytes: