diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e94723..6c50675 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,7 @@ jobs: run: | python -m ruff check . python -m ruff format --check . + lint-imports - name: Command line run: python -m nnnotes --help - name: Test @@ -62,4 +63,4 @@ jobs: - name: Golden records env: GOLDENS_STRICT: "1" - run: python -m pytest tests/test_goldens.py tests/test_objexport.py + run: python -m pytest tests/pipeline/test_goldens.py tests/pipeline/test_objexport.py diff --git a/README.en.md b/README.en.md index 59ed006..787ee4c 100644 --- a/README.en.md +++ b/README.en.md @@ -115,7 +115,7 @@ overriding earlier ones: `--vgmstream`, `--node` go before the command name; `--player` is an option of `web` `nnnotes config init` writes the config template (every setting empty; it ships with the package, in the repository -[`src/nnnotes/nnnotes.example.toml`](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/nnnotes.example.toml)), +[`src/nnnotes/source/nnnotes.example.toml`](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/source/nnnotes.example.toml)), asking for each value in a terminal; `config set` / `config unset` change one value and keep the file's comments. The settings are: the bundle key and nonce seed, the master data key and IV, the region in use (`[catalog] region`) and the catalog language diff --git a/README.md b/README.md index c27afea..0f86f2d 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ nnnotes config check # 每项设置的来源和格式是否有效， 2. 环境变量：`NNNOTES_<节>_<键>`（如 `NNNOTES_BUNDLE_KEY`、`NNNOTES_SERVERS_TW_CDN`） 3. 命令行参数：`--region`、`--language`、`--catalog`、`--cache`、`--master`、`--apk`、`--ffmpeg`、`--vgmstream`、`--node` 写在命令名之前；`--player` 是 `web` 的参数 -`nnnotes config init` 写出配置模板（每项设置为空，模板随包安装，仓库中为 [`src/nnnotes/nnnotes.example.toml`](src/nnnotes/nnnotes.example.toml)），在终端中会逐项询问；`config set` / `config unset` 修改单项并保留文件中的注释。需要的设置包括：资源包解密密钥与 nonce 种子、masterdata 的密钥与 IV、使用的区服 `[catalog] region` 与 catalog 语言 `[catalog] language`、每个区服的 CDN 地址与 API 根地址（`[servers.<区服>]` 的 `cdn`、`api`）、客户端版本 `[client] version`（未设置时读取 APK 的 versionName）、可选的引导 API 根地址 `[bootstrap] api`（`servers` 使用），以及缓存目录、APK、masterdata 目录、ournotes-player 和 vgmstream / FFmpeg / Node.js 的路径（三个工具未设置时在 `PATH` 中查找）。这些值都来自使用者自己的游戏客户端。 +`nnnotes config init` 写出配置模板（每项设置为空，模板随包安装，仓库中为 [`src/nnnotes/source/nnnotes.example.toml`](src/nnnotes/source/nnnotes.example.toml)），在终端中会逐项询问；`config set` / `config unset` 修改单项并保留文件中的注释。需要的设置包括：资源包解密密钥与 nonce 种子、masterdata 的密钥与 IV、使用的区服 `[catalog] region` 与 catalog 语言 `[catalog] language`、每个区服的 CDN 地址与 API 根地址（`[servers.<区服>]` 的 `cdn`、`api`）、客户端版本 `[client] version`（未设置时读取 APK 的 versionName）、可选的引导 API 根地址 `[bootstrap] api`（`servers` 使用），以及缓存目录、APK、masterdata 目录、ournotes-player 和 vgmstream / FFmpeg / Node.js 的路径（三个工具未设置时在 `PATH` 中查找）。这些值都来自使用者自己的游戏客户端。 缺少或格式错误的设置会让命令以退出码 2 结束，并用一行说明对应的配置键、环境变量和命令行参数，不会输出任何设置值。`nnnotes.toml` 已在 `.gitignore` 中，请勿提交。 diff --git a/docs/commands.md b/docs/commands.md index 8ebfd0a..a59a96c 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -617,7 +617,7 @@ root), and `stories.json` counts their files in `size.models`. The format is des window of an Overlay story is handled the same way. Each generated asset takes the point size, padding, style settings and render mode of the game font asset the texts use in that language, so the game's text materials apply unchanged; those values, the game's fallback chains, the characters the game's assets - have and the text materials come from the table of them shipped with nnnotes (`src/nnnotes/fontsettings.json`, + have and the text materials come from the table of them shipped with nnnotes (`src/nnnotes/unity/fontsettings.json`, `fontsettings.FORMAT`), so an open-font export reads no game font asset, material or font bundle; its face info and glyph metrics come from the font file (FreeType, no hinting), its distance field from the generator `--fonts game` uses for runtime glyphs (supersampled render modes diff --git a/docs/configuration.md b/docs/configuration.md index 2532938..7da2092 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -20,7 +20,7 @@ Lowest to highest: 3. **Command-line flags**, for the settings that have one (table below). An empty value (`""`, `[]`, an empty environment variable, an empty flag) counts as unset, so a lower source still -applies. The template [`nnnotes.example.toml`](../src/nnnotes/nnnotes.example.toml) ships with the package and lists +applies. The template [`nnnotes.example.toml`](../src/nnnotes/source/nnnotes.example.toml) ships with the package and lists every setting with an empty value; `nnnotes config init` writes it (below). `nnnotes.toml` and `*.local.toml` are in `.gitignore`; do not commit a filled-in copy. diff --git a/docs/stages.md b/docs/stages.md index 948f0d1..153f1e9 100644 --- a/docs/stages.md +++ b/docs/stages.md @@ -32,8 +32,8 @@ gzip input and Japanese remote asset placeholders). | `view.` | a catalog (`main`) | `master:` per master table it reads, `rules` (the view's rule and the resolvers it uses), `addresses` (the part of the address table it read: the keys looked up and the prefixes listed); no parameters, atoms or context | `view.:#view` (facts: rows, entries, gaps, unreferenced) | | `view.voices` | a catalog (`main`) | `master:
` per master table its rules read, `rules`, `addresses`, `sheets` (each cue sheet the `cri.audio` results are known by that a voice names or the rules' prefixes list: its task and ACB content id), `acb:` (the ACB of each decoded one), `episodes` (per story episode and part, the `unity.export` artifact holding the object its key names), `adv:` (those contents); context `audio` (the keys of those `cri.audio` results); runs after `unity.export` and `cri.audio` | `view.voices:#view` (facts: rows, entries, gaps, unreferenced streams) | -`view.voices` is a stage of its own (`nnnotes.voices:VoiceStage`), not one of the master-only views of -`nnnotes.views:stages`: besides master tables and the address table it reads the story episodes of the +`view.voices` is a stage of its own (`nnnotes.pipeline.voices:VoiceStage`), not one of the master-only views of +`nnnotes.pipeline.views:stages`: besides master tables and the address table it reads the story episodes of the `unity.export` results, `cri.audio` results and ACB tables ([views.md](views.md#voices)). ## Converters (`unity.export`) @@ -128,14 +128,14 @@ else found on PATH. The `boot` input is the APK's `assets/bin/Data/data.unity3d` ## Atoms -`nnnotes.atoms.ATOMS[name]` is the reference implementation of an atom: its id, its function and its cost estimate. +`nnnotes.pipeline.atoms.ATOMS[name]` is the reference implementation of an atom: its id, its function and its cost estimate. The id names the libraries whose behaviour the output depends on, with their installed versions, and the revision of the nnnotes code (`pillow-12.1.1/1`); `nnnotes/` when no library's behaviour is involved. Task keys include the ids of the atoms a task uses, so upgrading a library or changing an atom re-runs exactly the tasks that use it. | Atom | Reference implementation | Id | Does | |---|---|---|---| -| `reader` | `nnnotes.atoms.reader.open_bundle` (UnityPy) | `unitypy-/1` | one bundle loaded alone from its bytes: serialized files, objects, raw bytes, typetrees, container, externals, and the inputs of `texture.decode` and `mesh.arrays` (resource files are read from inside the bundle only) | +| `reader` | `nnnotes.pipeline.atoms.reader.open_bundle` (UnityPy) | `unitypy-/1` | one bundle loaded alone from its bytes: serialized files, objects, raw bytes, typetrees, container, externals, and the inputs of `texture.decode` and `mesh.arrays` (resource files are read from inside the bundle only) | | `texture.decode` | UnityPy `Texture2DConverter.parse_image_data` | `unitypy-+texture2ddecoder-+astc-encoder-py-+pillow-/1` | the first mip level, top row first, as UnityPy's `Texture2D.image` | | `png.encode` | Pillow | `pillow-/1` | PNG bytes; level 6 (the default) gives Pillow's default bytes, the ones the other commands write | | `astc.container` | nnnotes | `nnnotes/1` | the standard `.astc` file (16-byte header, first mip level's blocks) | diff --git a/docs/views.md b/docs/views.md index 5af2093..8a90ffe 100644 --- a/docs/views.md +++ b/docs/views.md @@ -2,8 +2,8 @@ A view answers, for every row of a master table, which objects of the catalog the game shows for it: the full illustration of a member card and its sub-sprites, the icon of a stamp, the jacket of a song, the banner of a story -episode, the icon of a reward. The rules are data (`src/nnnotes/viewrules.json`, package data) read by -`nnnotes.views`; a view document lists object ids, so it depends on the master data and the catalog, never on how +episode, the icon of a reward. The rules are data (`src/nnnotes/pipeline/viewrules.json`, package data) read by +`nnnotes.pipeline.views`; a view document lists object ids, so it depends on the master data and the catalog, never on how the objects are encoded. ## Rules @@ -108,7 +108,7 @@ resolver's name). ## Stages -`nnnotes.views:stages` gives one stage per view, `view.` (docs/stages.md), run by `nnnotes export --views` +`nnnotes.pipeline.views:stages` gives one stage per view, `view.` (docs/stages.md), run by `nnnotes export --views` and planned by `nnnotes plan`: - Subjects: the catalog subjects of `link.addresses` (the export command has one, `main`), when master data is set @@ -474,7 +474,7 @@ icon: `not-applicable`): ## Voices -`view.voices` (`nnnotes.voices`) indexes the character voices the master data names and those of the story +`view.voices` (`nnnotes.pipeline.voices`) indexes the character voices the master data names and those of the story episodes: per voice its source row, characters, category, text in the five languages and conditions, its cue sheet and cue, and the decoded streams of the export that play it. It is a stage of its own rather than a view of the rules above: besides master tables and the address table it reads the results of `unity.export` (the episodes) and @@ -579,7 +579,7 @@ A voice of a home or after-live episode is often a cue that a row of another sou ### Cues and streams The sound's `_soundCueSheetID` names the cue sheet (MasterSoundCueSheet `_cueSheetName`, catalog key -`Cri/Sound/`) and `_cueName` the cue. The cue's streams come from the sheet's ACB tables (`nnnotes.acb`): a +`Cri/Sound/`) and `_cueName` the cue. The cue's streams come from the sheet's ACB tables (`nnnotes.formats.acb`): a cue references a waveform, a synth, a sequence or a block sequence; synths reference waveforms, synths and sequences; the tracks of sequences and blocks note on synths and sequences. A waveform of the memory AWB is the stream at its position in the AWB, the numbering of the sheet's `streams.json`. A cue may play several streams (a diff --git a/pyproject.toml b/pyproject.toml index 2150fc6..7304b63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ dynamic = ["version"] [project.optional-dependencies] -test = ["pytest>=8", "ruff>=0.15", "jsonschema>=4.18"] +test = ["pytest>=8", "ruff>=0.15", "jsonschema>=4.18", "import-linter>=2.1"] fonts = ["fonttools>=4.50", "freetype-py>=2.5", "scipy>=1.13"] # the deck model of music-data: one nnnotes version carries one ournotes-deck release (.github/workflows/deck.yml moves # the pin when ournotes-deck publishes a newer one) @@ -65,4 +65,22 @@ select = ["E", "W", "F", "I", "UP", "B"] ignore = ["B023", "B905"] [tool.ruff.lint.flake8-bugbear] -extend-immutable-calls = ["nnnotes.liveoptions.LiveOptions"] +extend-immutable-calls = ["nnnotes.content.liveoptions.LiveOptions"] + +[tool.importlinter] +root_package = "nnnotes" + +[[tool.importlinter.contracts]] +name = "Layers: each subpackage imports only the layers below it" +type = "layers" +layers = [ + "cli", + "site | musicdata", + "pipeline", + "content", + "unity", + "formats", + "source", + "core", +] +containers = ["nnnotes"] diff --git a/scripts/atom_eval.py b/scripts/atom_eval.py index e76bc95..e3543ce 100644 --- a/scripts/atom_eval.py +++ b/scripts/atom_eval.py @@ -2,7 +2,7 @@ """Measure candidate implementations of the export atoms against the reference implementations (maintainer tool; not installed, not needed to use nnnotes). -`nnnotes.atoms.ATOMS` holds one reference implementation per atom; `NNNOTES_ATOM_=module:attribute` names +`nnnotes.pipeline.atoms.ATOMS` holds one reference implementation per atom; `NNNOTES_ATOM_=module:attribute` names another one for evaluation (docs/stages.md, "Atoms"). This script gives the numbers a switch is decided on: sample DIR -o FILE a seeded stratified sample of a bundle directory: one bundle of every @@ -42,7 +42,7 @@ A candidate view is the reader candidate's result; the optional methods census(), text_bytes(obj), font_bytes(obj), sprite_image(obj) and the attribute typetree_form serve the checks that have no atom of their own. A candidate -raising NotImplementedError has no such facet ("absent"); `nnnotes.atoms.Unsupported` is a refusal. +raising NotImplementedError has no such facet ("absent"); `nnnotes.pipeline.atoms.Unsupported` is a refusal. Statuses: equal, differ, refused (the candidate raised where the reference gave a result), refused.both, extra (a result where the reference refused), absent, reference-error. Output: one JSON document (-o); m1 and m3 also write @@ -133,7 +133,7 @@ def check_clean_env() -> None: def load_impl(spec: str): """The Impl named by module:attribute (an Impl, or a callable returning one).""" - from nnnotes.atoms import Impl + from nnnotes.pipeline.atoms import Impl module, sep, attr = spec.partition(":") if not sep or not module or not attr: @@ -146,7 +146,7 @@ def load_impl(spec: str): def parse_atoms(items) -> list[tuple[str, str]]: - from nnnotes.atoms import ATOMS + from nnnotes.pipeline.atoms import ATOMS out = [] for it in items or (): @@ -412,8 +412,8 @@ def merge(total: dict, rec: dict) -> None: # ---------------------------------------------------------------- sample def cmd_sample(a) -> dict: - from nnnotes import contract - from nnnotes.cli_assets import bundle_group + from nnnotes.cli.cli_assets import bundle_group + from nnnotes.pipeline import contract root = Path(a.dir) items = bundle_list(root, a.bundles) @@ -449,7 +449,7 @@ def cmd_sample(a) -> dict: # ---------------------------------------------------------------- m1 def m1_init(cfg: dict) -> None: - from nnnotes.atoms import resolve + from nnnotes.pipeline.atoms import resolve G.clear() G.update(cfg) @@ -477,7 +477,7 @@ def _key(info) -> tuple: def _call(fn, *args): """(result, None) or (None, refusal) where refusal is the Unsupported code or the error's message.""" - from nnnotes.atoms import Unsupported + from nnnotes.pipeline.atoms import Unsupported try: return fn(*args), None @@ -578,8 +578,8 @@ def _census_facets(T: Tally, section: str, ra: dict, rb: dict | None, oid: str, def check_census(T: Tally, ref, cv, bundle: str, data: bytes) -> None: - from nnnotes import census - from nnnotes.unity import load_bytes + from nnnotes.pipeline import census + from nnnotes.unity.reader import load_bytes if not hasattr(cv, "census"): T.add("census", "document", "absent", bundle) @@ -700,7 +700,7 @@ def hdr_reference(inp): import numpy as np from PIL import Image - from nnnotes.atoms.texture import ASTC_BLOCKS + from nnnotes.pipeline.atoms.texture import ASTC_BLOCKS bx, by = ASTC_BLOCKS[int(inp.format)] w, h = int(inp.width), int(inp.height) @@ -723,7 +723,7 @@ def _hdr_library_image(inp): def check_textures(T: Tally, ref, cv, infos, cmap, checks) -> None: - from nnnotes.atoms.texture import format_name, is_hdr_astc + from nnnotes.pipeline.atoms.texture import format_name, is_hdr_astc ref_dec, ref_png = G["ref"]["texture.decode"], G["ref"]["png.encode"] decs, pngs = G["cand"].get("texture.decode", []), G["cand"].get("png.encode", []) @@ -1038,7 +1038,7 @@ def versions() -> dict: # ---------------------------------------------------------------- prepare (base store) def _identify(job): - from nnnotes.store import Store + from nnnotes.pipeline.store import Store root, store, name = job s, size = Store(store).identify(Path(root) / name) @@ -1046,18 +1046,18 @@ def _identify(job): def _census_task(task): - from nnnotes import census - from nnnotes.stages import execute + from nnnotes.pipeline import census + from nnnotes.pipeline.stages import execute ex = execute(task, G["store"], {"unity.census": census.CensusStage()}) return task.subject, task.key, ex.status def cmd_prepare(a) -> dict: - from nnnotes import census, contract, link - from nnnotes.contract import Input - from nnnotes.stages import Env, describe, execute - from nnnotes.store import Store + from nnnotes.pipeline import census, contract, link + from nnnotes.pipeline.contract import Input + from nnnotes.pipeline.stages import Env, describe, execute + from nnnotes.pipeline.store import Store check_clean_env() root, base = Path(a.dir).resolve(), Path(a.base) @@ -1100,9 +1100,9 @@ def cmd_prepare(a) -> dict: # ---------------------------------------------------------------- m3 (stage cost) def _prepared_env(base: Path): - from nnnotes.contract import Input - from nnnotes.stages import Env - from nnnotes.store import Store + from nnnotes.pipeline.contract import Input + from nnnotes.pipeline.stages import Env + from nnnotes.pipeline.store import Store prep = json.loads((base / PREPARED).read_text(encoding="utf-8")) root = Path(prep["dir"]) @@ -1125,7 +1125,7 @@ def _prepared_env(base: Path): def _new_store(base: Path, path: Path): """A run store that knows the base store's input identities (so inputs are not hashed again).""" - from nnnotes.store import Store + from nnnotes.pipeline.store import Store path.mkdir(parents=True, exist_ok=True) src = base / "inputs" @@ -1138,14 +1138,14 @@ def _new_store(base: Path, path: Path): def _stage(name: str): - from nnnotes import census - from nnnotes.objexport import ExportStage + from nnnotes.pipeline import census + from nnnotes.pipeline.objexport import ExportStage return {"unity.export": ExportStage, "unity.census": census.CensusStage}[name]() def _run_task(task, store, stage_name: str) -> dict: - from nnnotes.stages import execute + from nnnotes.pipeline.stages import execute hwm = reset_hwm() u0, s0 = cpu_now() @@ -1173,7 +1173,7 @@ def _m3_worker(task) -> dict: def _apply_atoms(atoms) -> None: - from nnnotes.atoms import env_name, resolve + from nnnotes.pipeline.atoms import env_name, resolve for name, spec in atoms: os.environ[env_name(name)] = spec @@ -1184,13 +1184,13 @@ def _apply_atoms(atoms) -> None: def _import_stage_modules() -> dict: """Import what a unity.export task needs, timed (wall, CPU).""" t0, c0 = time.perf_counter(), time.process_time() - for m in ("numpy", "PIL.Image", "nnnotes.objexport", "nnnotes.export"): + for m in ("numpy", "PIL.Image", "nnnotes.pipeline.objexport", "nnnotes.unity.export"): importlib.import_module(m) return {"wall": round(time.perf_counter() - t0, 4), "cpu": round(time.process_time() - c0, 4)} def _describe_all(stage_name: str, subjects, params, env): - from nnnotes.stages import describe + from nnnotes.pipeline.stages import describe stage = _stage(stage_name) return [describe(stage, s, params, env) for s in subjects] @@ -1400,7 +1400,7 @@ def cmd_m3_one(a) -> None: prep, env = _prepared_env(base) (task,) = _describe_all(a.stage, [a.subject], parse_params(a.param) or None, env) describe_cost = {"wall": round(time.perf_counter() - t0, 4), "cpu": round(time.process_time() - cd, 4)} - from nnnotes.store import Store + from nnnotes.pipeline.store import Store rec = _run_task(task, Store(a.store), a.stage) rec.update(imports=imp, describe=describe_cost, processCpuInside=round(time.process_time() - c0, 4)) @@ -1409,7 +1409,7 @@ def cmd_m3_one(a) -> None: # ---------------------------------------------------------------- imports def cmd_imports(a) -> dict: - sets = [m.split(",") for m in (a.module or ["numpy,PIL.Image,nnnotes.objexport,nnnotes.export"])] + sets = [m.split(",") for m in (a.module or ["numpy,PIL.Image,nnnotes.pipeline.objexport,nnnotes.unity.export"])] code = ( "import resource, sys, time, json\n" "t = time.perf_counter(); c = time.process_time()\n" @@ -1525,7 +1525,7 @@ def _compare_job(job) -> dict: def cmd_compare(a) -> dict: - from nnnotes.store import Store + from nnnotes.pipeline.store import Store ra, rb = (json.loads(Path(p).read_text(encoding="utf-8")) for p in (a.run_a, a.run_b)) G.update(sa=Store(ra["store"]), sb=Store(rb["store"])) diff --git a/scripts/atom_eval_pillow.py b/scripts/atom_eval_pillow.py index 7e0dacd..ec5d9f4 100644 --- a/scripts/atom_eval_pillow.py +++ b/scripts/atom_eval_pillow.py @@ -1,6 +1,6 @@ """Candidate `png.encode` implementations on Pillow, for scripts/atom_eval.py only. -The reference (`nnnotes.atoms.png.encode`) writes at the level a task's `png.level` parameter gives (6 by +The reference (`nnnotes.pipeline.atoms.png.encode`) writes at the level a task's `png.level` parameter gives (6 by default). These fix the setting instead, so several can be measured side by side in one `atom_eval.py m1` run: level1, level3, level6, level9 Pillow at that zlib level @@ -15,8 +15,8 @@ from functools import cache from importlib import metadata -from nnnotes.atoms import Impl -from nnnotes.atoms import png as png_atom +from nnnotes.pipeline.atoms import Impl +from nnnotes.pipeline.atoms import png as png_atom def _pillow() -> str: diff --git a/scripts/atom_eval_unityrs.py b/scripts/atom_eval_unityrs.py index de026dd..d403c35 100644 --- a/scripts/atom_eval_unityrs.py +++ b/scripts/atom_eval_unityrs.py @@ -27,13 +27,13 @@ from functools import cache from importlib import metadata -from nnnotes.atoms import Impl, Unsupported -from nnnotes.atoms import png as png_atom -from nnnotes.atoms import reader as reader_atom -from nnnotes.atoms import texture as texture_atom -from nnnotes.atoms.mesh import MeshInput -from nnnotes.atoms.reader import ObjectInfo, UnityBundle -from nnnotes.atoms.texture import TextureInput +from nnnotes.pipeline.atoms import Impl, Unsupported +from nnnotes.pipeline.atoms import png as png_atom +from nnnotes.pipeline.atoms import reader as reader_atom +from nnnotes.pipeline.atoms import texture as texture_atom +from nnnotes.pipeline.atoms.mesh import MeshInput +from nnnotes.pipeline.atoms.reader import ObjectInfo, UnityBundle +from nnnotes.pipeline.atoms.texture import TextureInput FALLBACKS = {"png": 0} _KEEP = "unity_rs.image" @@ -133,7 +133,7 @@ def texture_input(self, obj: ObjectInfo) -> TextureInput: def open_hybrid(data: bytes) -> HybridBundle: - from nnnotes.unity import load_bytes + from nnnotes.unity.reader import load_bytes data = bytes(data) return HybridBundle(load_bytes(data), data) @@ -390,7 +390,7 @@ def png_encode_fast() -> Impl: def _mesh(inp: MeshInput): import numpy as np - from nnnotes.atoms.mesh import Channel, MeshArrays, Primitive + from nnnotes.pipeline.atoms.mesh import Channel, MeshArrays, Primitive src = getattr(inp, "source", None) if src is None: @@ -429,6 +429,6 @@ def _mesh(inp: MeshInput): @cache def mesh_arrays() -> Impl: - from nnnotes.atoms import mesh as mesh_atom + from nnnotes.pipeline.atoms import mesh as mesh_atom return Impl(_id(_rs(), "obj"), _mesh, mesh_atom.cost) diff --git a/scripts/gen_font_settings.py b/scripts/gen_font_settings.py index 2c82771..428ef86 100644 --- a/scripts/gen_font_settings.py +++ b/scripts/gen_font_settings.py @@ -3,7 +3,7 @@ With `--fonts open` (the default) the story, start canvas and talk window exports give each localized game font asset an open font asset with that asset's point size, padding, render mode, style settings, line metrics, fallback -chain and text materials. nnnotes ships these values in `src/nnnotes/fontsettings.json` (fontsettings.FORMAT), so +chain and text materials. nnnotes ships these values in `src/nnnotes/unity/fontsettings.json` (fontsettings.FORMAT), so those exports read no game font asset; this script reads them from the game's font assets and writes the file. Supporting a new game version @@ -27,14 +27,16 @@ import tempfile from pathlib import Path -from nnnotes import fontsettings, tmpfont -from nnnotes.config import Config, use -from nnnotes.export import Exporter -from nnnotes.web import open_data +from nnnotes.site.web import open_data +from nnnotes.source.config import Config, use +from nnnotes.unity import fontsettings, tmpfont +from nnnotes.unity.export import Exporter def main(argv=None) -> None: - p = argparse.ArgumentParser(description="regenerate src/nnnotes/fontsettings.json from the game's font assets") + p = argparse.ArgumentParser( + description="regenerate src/nnnotes/unity/fontsettings.json from the game's font assets" + ) p.add_argument("--region", help="region of the catalog (default: [catalog] region)") p.add_argument("--game-version", help="game version to record (default: the APK manifest's versionName)") p.add_argument("-o", "--out", type=Path, default=fontsettings.FILE, help="output file") diff --git a/scripts/gen_typetrees.py b/scripts/gen_typetrees.py index 4401f25..50e626e 100644 --- a/scripts/gen_typetrees.py +++ b/scripts/gen_typetrees.py @@ -4,7 +4,7 @@ nnnotes reads a few MonoBehaviours of the APK's boot data (`assets/bin/Data/data.unity3d`): the URP pipeline assets, renderer data, renderer features and post-process data, DOTween and TextMeshPro settings, and the sound and app-config defaults. IL2CPP player builds strip script typetrees from that data, so nnnotes ships the -typetrees of exactly these classes (CLASSES below) in `src/nnnotes/typetrees/.json`, one file per +typetrees of exactly these classes (CLASSES below) in `src/nnnotes/unity/typetrees/.json`, one file per Unity version. Each entry holds the typetree nodes as [level, type, name, meta flag] rows and the type hash the serialized file records for the class; the file also records the game version it was generated from. @@ -28,7 +28,7 @@ python scripts/gen_typetrees.py --dll It reads every object of each class in CLASSES with the generated typetree (the read must consume exactly the - object's serialized size), then writes `src/nnnotes/typetrees/.json` (`-o` for another + object's serialized size), then writes `src/nnnotes/unity/typetrees/.json` (`-o` for another directory; `--game-version` when the APK's manifest has no versionName). A class that is not in the boot data, has a field Unity's layout is not known for here (a `[SerializeReference]` field, another array form), or does not read back stops the script without writing anything. @@ -47,7 +47,7 @@ from UnityPy.helpers.TypeTreeGenerator import TypeTreeGenerator -from nnnotes.player import TYPETREE_FORMAT, PlayerData, dump_typetrees, typetree_key, typetree_node +from nnnotes.unity.player import TYPETREE_FORMAT, PlayerData, dump_typetrees, typetree_key, typetree_node # (assembly, namespace, class) as the MonoScript objects name them CLASSES = ( @@ -69,7 +69,7 @@ ("Unity.TextMeshPro", "TMPro", "TMP_Settings"), ("Unity.TextMeshPro", "TMPro", "TMP_StyleSheet"), ) -DEFAULT_OUT = Path(__file__).resolve().parents[1] / "src" / "nnnotes" / "typetrees" +DEFAULT_OUT = Path(__file__).resolve().parents[1] / "src" / "nnnotes" / "unity" / "typetrees" ALIGN = 0x4000 # TransferMetaFlags.kAlignBytesFlag # the MonoBehaviour header: field -> meta flag @@ -241,7 +241,7 @@ def generate(player: PlayerData, gen: TypeTreeGenerator, game_version: str) -> d def main(argv=None) -> None: - p = argparse.ArgumentParser(description="regenerate src/nnnotes/typetrees/.json from an APK") + p = argparse.ArgumentParser(description="regenerate src/nnnotes/unity/typetrees/.json from an APK") p.add_argument("apk", type=Path, help="base.apk of the game version") g = p.add_mutually_exclusive_group(required=True) g.add_argument("--dll", type=Path, help="folder of managed assembly stubs (e.g. Il2CppDumper DummyDll)") diff --git a/src/nnnotes/cli.py b/src/nnnotes/cli/__init__.py similarity index 96% rename from src/nnnotes/cli.py rename to src/nnnotes/cli/__init__.py index b937cd8..b8958e8 100644 --- a/src/nnnotes/cli.py +++ b/src/nnnotes/cli/__init__.py @@ -59,14 +59,16 @@ # inherit it. os.environ.setdefault("OPENBLAS_NUM_THREADS", "1") -from . import __version__, cli_assets, voices -from .catalog import Catalog -from .compress import DEFAULT_ENCODING, ENCODINGS -from .config import DEFAULT_FILE, ENV_CONFIG, Config, ConfigError, config_files, find_file, use, user_file -from .gameapi import GameApiError -from .jsonio import dumps, write_json -from .session import Session -from .webaudio import DEFAULT_AUDIO_FORMAT, WEB_AUDIO +from .. import __version__ +from ..core.jsonio import dumps, write_json +from ..pipeline import voices +from ..site.compress import DEFAULT_ENCODING, ENCODINGS +from ..site.webaudio import DEFAULT_AUDIO_FORMAT, WEB_AUDIO +from ..source.catalog import Catalog +from ..source.config import DEFAULT_FILE, ENV_CONFIG, Config, ConfigError, config_files, find_file, use, user_file +from ..source.gameapi import GameApiError +from ..source.session import Session +from . import cli_assets DIFFICULTY_CHOICES = ("easy", "normal", "hard", "expert") AUDIO_CHOICES = ("flac", "ogg", "wav") @@ -109,7 +111,7 @@ def known_keys(args, cat: Catalog, keys) -> None: def known_row(args, md: Path, table: str, row_id: int, what: str) -> None: """A usage error (exit 2) naming `row_id` when the master table `table` has no row with that `_id`.""" - from .master import has_row + from ..source.master import has_row try: found = has_row(md, table, row_id) @@ -164,7 +166,7 @@ def cmd_pull(args, cfg): def cmd_master_decode(args, cfg): - from . import master + from ..source import master key = Session(cfg).master_key() files = master.input_files(args.inputs) @@ -208,7 +210,7 @@ def _config_target(args, default: Path | None) -> Path: def _config_value(args, name: str, text: str, stdin_used: list): """(section, key, value) of `name` = `text` (`-`: read from standard input, once), checked.""" - from . import configfile + from ..source import configfile setting, section = configfile.resolve(name) if text == "-": @@ -237,7 +239,7 @@ def _config_note(target: Path) -> None: def cmd_config_init(args, cfg): - from . import configfile + from ..source import configfile named = next((f for n, f in config_files(args.config)[:2] if f is not None), None) target = _config_target(args, named or Path(DEFAULT_FILE)) @@ -269,7 +271,7 @@ def cmd_config_init(args, cfg): def _config_edit(args, value_text: str | None) -> None: - from . import configfile + from ..source import configfile target = _config_target(args, None) text = target.read_text(encoding="utf-8") if target.exists() else configfile.template().decode("utf-8") @@ -292,7 +294,7 @@ def cmd_config_unset(args, cfg): def cmd_config_check(args, cfg): - from . import configfile + from ..source import configfile r = configfile.check(cfg) if args.json: @@ -309,7 +311,7 @@ def cmd_config_check(args, cfg): def cmd_config_path(args, cfg): - from . import configfile + from ..source import configfile files, read = [], None for name, file in config_files(args.config): @@ -334,7 +336,7 @@ def cmd_config_path(args, cfg): def cmd_servers(args, cfg): - from . import gameapi + from ..source import gameapi try: servers = gameapi.server_list(cfg) @@ -345,7 +347,7 @@ def cmd_servers(args, cfg): def cmd_master_version(args, cfg): - from . import gameapi + from ..source import gameapi region = cfg.region() try: @@ -359,13 +361,13 @@ def cmd_master_version(args, cfg): def cmd_master_download(args, cfg): - from . import gameapi, master + from ..source import gameapi, master region = cfg.region() cdn = cfg.cdn(region) try: if cfg.provider(region) == "jp": - from .jp import Client, master_version + from ..source.jp import Client, master_version client = Client(cfg, region) observation = client.observe() @@ -389,7 +391,7 @@ def cmd_master_download(args, cfg): def cmd_adv(args, cfg): - from . import adv + from ..content import adv md = Session(cfg).master_dir() known_row(args, md, "MasterAdv", args.adv_id, "episode") @@ -402,7 +404,8 @@ def cmd_adv(args, cfg): def cmd_live2d(args, cfg): - from . import live2d, webmodel + from ..site import webmodel + from ..unity import live2d cfg.require_path("paths", "apk") # the Cubism component classes are read from the APK cat = Session(cfg).catalog() @@ -415,7 +418,8 @@ def cmd_live2d(args, cfg): def cmd_spot(args, cfg): - from . import room, shader, spot + from ..content import room, spot + from ..formats import shader cfg.require_path("paths", "apk") # the Spot / Spine component classes are read from the APK md = Session(cfg).master_dir() @@ -433,7 +437,7 @@ def cmd_spot(args, cfg): def cmd_room(args, cfg): - from . import room + from ..content import room cat = Session(cfg).catalog() known_keys(args, cat, [args.key]) @@ -442,7 +446,7 @@ def cmd_room(args, cfg): def cmd_shader(args, cfg): - from . import shader + from ..formats import shader cat = Session(cfg).catalog() if args.key: @@ -464,7 +468,7 @@ def cmd_shader(args, cfg): def cmd_audio(args, cfg): - from . import cri + from ..formats import cri flac = _flac_options(args) cfg.require_path("paths", "apk") # the HCA keycode is read from the APK @@ -477,7 +481,7 @@ def cmd_audio(args, cfg): def cmd_crikey(args, cfg): - from . import crikey + from ..formats import crikey cfg.require_path("paths", "apk") k = crikey.find_key(Session(cfg).path("paths", "apk")) @@ -489,7 +493,7 @@ def cmd_crikey(args, cfg): def cmd_player(args, cfg): - from .player import PlayerData + from ..unity.player import PlayerData g = PlayerData(Session(cfg).apk()).graphics() out = Path(args.out) @@ -499,11 +503,13 @@ def cmd_player(args, cfg): def cmd_story(args, cfg): - from . import languages, story, webmodel - from .player import PlayerData + from ..content import story + from ..site import webmodel + from ..source import languages + from ..unity.player import PlayerData if args.fonts == "game": - from .tmpfont import require_extra + from ..unity.tmpfont import require_extra require_extra("--fonts game") flac = _flac_options(args) @@ -553,14 +559,14 @@ def _flac_options(args) -> dict: def _fonts_extra(fonts: str) -> None: """`--fonts game` needs the optional `fonts` dependencies.""" if fonts == "game": - from .tmpfont import require_extra + from ..unity.tmpfont import require_extra require_extra("--fonts game") def _live_options(args) -> dict: """The --live-option specs as a request (liveoptions.parse_specs); a malformed spec is a usage error.""" - from .liveoptions import OptionSpecError, parse_specs + from ..content.liveoptions import OptionSpecError, parse_specs try: return parse_specs(args.live_option) @@ -569,9 +575,10 @@ def _live_options(args) -> dict: def cmd_live(args, cfg): - from . import languages, live, liveoptions - from .player import PlayerData - from .web import all_pairs + from ..content import live, liveoptions + from ..site.web import all_pairs + from ..source import languages + from ..unity.player import PlayerData _fonts_extra(args.fonts) request = _live_options(args) @@ -609,7 +616,8 @@ def cmd_live(args, cfg): def cmd_web(args, cfg): - from . import liveoptions, sitefiles, siteindex, storysite, web, webmodel + from ..content import liveoptions + from ..site import sitefiles, siteindex, storysite, web, webmodel live_options = _live_options(args) charts, models = bool(args.pair or args.all), bool(args.live2d or args.all_live2d) @@ -699,7 +707,7 @@ def cmd_web(args, cfg): ) ) if stories: - from .tmpfont import require_extra + from ..unity.tmpfont import require_extra require_extra("--story / --all-stories") # the font assets of the story text (open and game) cfg.require_path("paths", "apk") # the ADV settings and UI are embedded content of the APK @@ -739,7 +747,8 @@ def cmd_web(args, cfg): def cmd_music_data(args, cfg): - from . import deckdata, musicdata + from .. import musicdata + from ..musicdata import deckdata if args.apk_master: cfg.require_path("paths", "apk") # the master data files ship in the APK @@ -791,7 +800,7 @@ def parse_pair(s: str) -> tuple[int, str]: def parse_languages(s: str) -> list[str]: - from .languages import LANGUAGES + from ..source.languages import LANGUAGES codes = [c.strip() for c in s.split(",") if c.strip()] if not codes or any(c not in LANGUAGES for c in codes): @@ -803,7 +812,7 @@ def parse_languages(s: str) -> list[str]: def parse_font(s: str) -> tuple[str, str]: - from .languages import LANGUAGES + from ..source.languages import LANGUAGES lang, sep, path = s.partition("=") if not sep or (lang not in LANGUAGES and lang != EMOJI_FONT) or not path: @@ -862,7 +871,7 @@ def _out(c, what: str, required: bool = True) -> None: def cmd_ui(args, cfg): - from . import ui + from ..content import ui ui.command(args, cfg) diff --git a/src/nnnotes/browse.py b/src/nnnotes/cli/browse.py similarity index 96% rename from src/nnnotes/browse.py rename to src/nnnotes/cli/browse.py index 98e492d..ec17ec2 100644 --- a/src/nnnotes/browse.py +++ b/src/nnnotes/cli/browse.py @@ -10,13 +10,13 @@ from urllib.parse import quote, unquote, urlsplit from urllib.request import urlopen -from . import __version__ -from .addressables import BundleKey, decrypt, remote_path -from .catalog import Bundle, Catalog -from .config import ConfigError -from .gameapi import GameApiError -from .jp import open_catalog -from .session import Session +from .. import __version__ +from ..source.addressables import BundleKey, decrypt, remote_path +from ..source.catalog import Bundle, Catalog +from ..source.config import ConfigError +from ..source.gameapi import GameApiError +from ..source.jp import open_catalog +from ..source.session import Session def browse(entries: list[dict]) -> tuple[dict, dict]: diff --git a/src/nnnotes/cli_assets.py b/src/nnnotes/cli/cli_assets.py similarity index 95% rename from src/nnnotes/cli_assets.py rename to src/nnnotes/cli/cli_assets.py index 0f43d7e..121570b 100644 --- a/src/nnnotes/cli_assets.py +++ b/src/nnnotes/cli/cli_assets.py @@ -61,12 +61,12 @@ from pathlib import Path from types import SimpleNamespace -from . import contract -from .apkset import ApkSet -from .config import ConfigError, check_catalog_version, usable_cpus, use -from .config import describe as describe_setting -from .contract import Cost, IncompatibleTask, Input, Task -from .stages import Env, Pending, Stage, describe, execute +from ..pipeline import contract +from ..pipeline.contract import Cost, IncompatibleTask, Input, Task +from ..pipeline.stages import Env, Pending, Stage, describe, execute +from ..source.apkset import ApkSet +from ..source.config import ConfigError, check_catalog_version, usable_cpus, use +from ..source.config import describe as describe_setting GIB = 1 << 30 LAYOUT_NAMES = ("original", "cas") @@ -100,19 +100,19 @@ def matches(self, stage: str) -> bool: STAGE_SOURCES = ( - StageSource("catalog.index", "nnnotes.catalogdb:IndexStage"), - StageSource("unity.census", "nnnotes.census:CensusStage"), - StageSource("link.scripts", "nnnotes.link:ScriptsStage"), - StageSource("link.addresses", "nnnotes.link:AddressesStage"), - StageSource("unity.export", "nnnotes.objexport:ExportStage", output=True, root="_bundles/{subject}"), - StageSource("sprite.crop", "nnnotes.objexport:SpriteCropStage", output=True), - StageSource("cri.audio", "nnnotes.cristages:AudioStage", output=True), - StageSource("cri.movie", "nnnotes.cristages:MovieStage", output=True), - StageSource("live2d.model", "nnnotes.modelstages:Live2DStage", output=True), - StageSource("spine.skeleton", "nnnotes.modelstages:SpineStage", output=True), - StageSource("link.artifacts", "nnnotes.link:ArtifactsStage"), - StageSource("view.", "nnnotes.views:stages", output=True), - StageSource("view.voices", "nnnotes.voices:VoiceStage", output=True), + StageSource("catalog.index", "nnnotes.pipeline.catalogdb:IndexStage"), + StageSource("unity.census", "nnnotes.pipeline.census:CensusStage"), + StageSource("link.scripts", "nnnotes.pipeline.link:ScriptsStage"), + StageSource("link.addresses", "nnnotes.pipeline.link:AddressesStage"), + StageSource("unity.export", "nnnotes.pipeline.objexport:ExportStage", output=True, root="_bundles/{subject}"), + StageSource("sprite.crop", "nnnotes.pipeline.objexport:SpriteCropStage", output=True), + StageSource("cri.audio", "nnnotes.pipeline.cristages:AudioStage", output=True), + StageSource("cri.movie", "nnnotes.pipeline.cristages:MovieStage", output=True), + StageSource("live2d.model", "nnnotes.pipeline.modelstages:Live2DStage", output=True), + StageSource("spine.skeleton", "nnnotes.pipeline.modelstages:SpineStage", output=True), + StageSource("link.artifacts", "nnnotes.pipeline.link:ArtifactsStage"), + StageSource("view.", "nnnotes.pipeline.views:stages", output=True), + StageSource("view.voices", "nnnotes.pipeline.voices:VoiceStage", output=True), ) @@ -197,7 +197,7 @@ class Selection: def select(index: dict, selection: list[str]) -> Selection: """The bundles and raw files a selection names in a catalog index (module documentation).""" - from . import catalogdb + from ..pipeline import catalogdb bundles = sorted(b["stable"] for b in index["bundles"]) raws = sorted(r["stable"] for r in index["rawFiles"]) @@ -269,7 +269,7 @@ def recycle_rss(memory: int | None, workers: int) -> int: def _bundle_key(cfg): - from .addressables import BundleKey + from ..source.addressables import BundleKey return BundleKey(cfg.hex("bundle", "key", 16), cfg.hex("bundle", "nonce_seed")) @@ -292,8 +292,8 @@ def __setstate__(self, state): def catalog(self, vid: str): """(Catalog, {location id: location}) of a stored catalog version.""" - from . import catalogdb - from .catalog import Catalog + from ..pipeline import catalogdb + from ..source.catalog import Catalog with self._lock: hit = self._catalogs.get(vid) @@ -311,7 +311,7 @@ def catalog(self, vid: str): cfg = cfg.for_region(region) source, client, cache = None, None, self.cache if v.get("source") is not None: - from .jp import Client, Source + from ..source.jp import Client, Source source = Source.from_dict(v["source"]) client = Client(cfg, cfg.region()) @@ -331,8 +331,8 @@ def catalog(self, vid: str): return hit def fetch_location(self, vid: str, lid: str) -> Path: - from .addressables import remote_path - from .catalog import Bundle, file_name + from ..source.addressables import remote_path + from ..source.catalog import Bundle, file_name cat, locs = self.catalog(vid) loc = locs.get(lid) @@ -347,8 +347,8 @@ def fetch_location(self, vid: str, lid: str) -> Path: def _apk_file(self, internal_id: str, cat) -> Path: """A raw APK file through its snapshot's cache; only a cache miss needs the matching APK.""" - from .cache import write_atomic - from .catalog import APK_AA_DIR + from ..core.cache import write_atomic + from ..source.catalog import APK_AA_DIR rel = apk_rel(internal_id) dst = cat.local_cache_dir() / "raw" / rel @@ -368,7 +368,7 @@ def __call__(self, inp: Input, loc: dict) -> Path: def apk_rel(internal_id: str) -> str: """The path of an APK-local location's file below the APK's Addressables directory (catalog.APK_AA_DIR).""" - from .catalog import LOCAL_PREFIX + from ..source.catalog import LOCAL_PREFIX return internal_id[len(LOCAL_PREFIX) :].lstrip("/") if internal_id.startswith(LOCAL_PREFIX) else internal_id @@ -379,7 +379,7 @@ def _worker_init(store, cfg) -> None: def _log(message: str) -> None: - from .orchestrate import _log as log + from ..pipeline.orchestrate import _log as log log(message) @@ -461,7 +461,7 @@ def calibrate(log: list[dict], factors: dict, worker_bytes: int) -> dict: def write_calibration(root: Path, factors: dict) -> None: - from .store import write_file + from ..pipeline.store import write_file write_file(Path(root) / "costs" / CALIBRATION, contract.encode({"stages": factors})) @@ -472,8 +472,8 @@ class Workspace: parameters, the facts.""" def __init__(self, args, cfg, common, log=_log): - from .catalogdb import CatalogDB - from .store import Store + from ..pipeline.catalogdb import CatalogDB + from ..pipeline.store import Store self.args, self.cfg, self.common, self.log = args, cfg, common, log self.root = store_root(args, cfg) @@ -504,7 +504,7 @@ def __init__(self, args, cfg, common, log=_log): # ---------------------------------------------------------------- catalog def _catalog_version(self): - from . import catalogdb + from ..pipeline import catalogdb ref = getattr(self.args, "catalog_version", None) if ref: @@ -545,7 +545,7 @@ def index_task(self) -> Task: def index(self) -> dict: """The catalog index: from the store when catalog.index has run, else computed here.""" if self._index is None: - from . import catalogdb + from ..pipeline import catalogdb t = self.index_task() doc = self.store.result(t.key) @@ -578,7 +578,7 @@ def context(self) -> dict: def master_context(self, results) -> dict | None: """The master data a run read: {"version": the `version` of /MasterManifest.json (None without one), "tables": {table: sha256} of the tables its view tasks read}; None when no view task has a result.""" - from .views import MASTER_ROLE + from ..pipeline.views import MASTER_ROLE tables, any_view = {}, False for tid, doc in results: @@ -700,7 +700,7 @@ def facts(self, pipeline: list[str], fetch: bool, background: bool = False) -> d "raw": Inputs({}), } if any(n in pipeline for n in RAW_STAGES): - from .cristages import boot_input + from ..pipeline.cristages import boot_input raws = {r["stable"]: r for r in index["rawFiles"]} job = (self._inputs, "raw", [raws[s] for s in sel.raw], fetch) @@ -730,11 +730,11 @@ def problems(self) -> list[dict]: return sorted(self._problems, key=lambda p: (p["kind"], p["stable"])) def _cache_rel(self, kind: str, entry: dict, loc: dict) -> str: - from .addressables import remote_path + from ..source.addressables import remote_path prefix = "" if self.version.get("source"): - from .jp import Source + from ..source.jp import Source prefix = Source.from_dict(self.version["source"]).cache_dir(Path(".")).as_posix() + "/" if not entry.get("remote", True) and self.version.get("apk"): @@ -747,7 +747,7 @@ def _cache_rel(self, kind: str, entry: dict, loc: dict) -> str: return prefix + "raw/" + rel.lstrip("/") def _inputs(self, kind: str, entries: list[dict], fetch: bool) -> dict: - from .catalogdb import by_id + from ..pipeline.catalogdb import by_id locs = by_id(self.index()) memo_kind = "bundles" if kind == "bundle" else "raw" @@ -805,7 +805,7 @@ def one(job): def _in_apk(self, internal_id: str) -> bool: """Whether the APK holds the file of a location (read from its directory, not fetched).""" - from .catalog import APK_AA_DIR + from ..source.catalog import APK_AA_DIR if self._apk_names is None: try: @@ -863,7 +863,7 @@ def previous(self, since: str | None) -> dict | None: return doc def plan(self, pipeline: list[str], params: dict, *, since=None, out=None, layouts=()): - from . import plan as plan_mod + from ..pipeline import plan as plan_mod facts = self.facts(pipeline, fetch=False) p = plan_mod.plan( @@ -877,7 +877,7 @@ def plan(self, pipeline: list[str], params: dict, *, since=None, out=None, layou memory=self.memory(), ) if out is not None and layouts: - from . import layout + from ..pipeline import layout self.done({n["id"]: n["key"] for n in p.doc["nodes"] if n["status"] == "hit"}) hits = [(n["id"], n["key"]) for n in p.doc["nodes"] if n["status"] == "hit" and self._output(n["stage"])] @@ -907,7 +907,7 @@ def layout_input(self, tasks: dict) -> LayoutInput: def layout_entries(self, name: str, placed: LayoutInput, *, store_derived: bool = True) -> tuple[list[dict], dict]: """The entries of layout `name` for the results of the output stages given to `placed` (derived documents of the stages in `original`) and its report.""" - from . import layout + from ..pipeline import layout stages = self.installed.stages sources, named = [], [] @@ -962,7 +962,7 @@ def names(self, stage_name: str) -> dict: return self._names[stage_name] def _derived(self, sources, entries: list[dict], store_derived: bool, objects: dict) -> list[dict]: - from . import layout + from ..pipeline import layout stages = self.installed.stages makers = list( @@ -1116,7 +1116,7 @@ def parse_layouts(spec: str) -> list[str]: def _apk_version(apk: Path | None) -> str | None: if apk is None or not apk.is_file(): return None - from .gameapi import apk_version_name + from ..source.gameapi import apk_version_name return apk_version_name(apk) @@ -1153,7 +1153,7 @@ def view_gaps(results) -> int: def _write_report(d: Path, name: str, doc) -> Path: - from .store import write_file + from ..pipeline.store import write_file p = d / name write_file(p, contract.encode(doc)) @@ -1161,8 +1161,8 @@ def _write_report(d: Path, name: str, doc) -> Path: def cmd_export(args, cfg, common): - from . import layout - from .orchestrate import Orchestrator, failure + from ..pipeline import layout + from ..pipeline.orchestrate import Orchestrator, failure try: names = parse_layouts(args.layout) @@ -1307,7 +1307,7 @@ def visit(tid, doc): def link_mode(args, cfg) -> str: """How export places the layout files (layout.LINKS): --link, else [export] link, else auto.""" - from .layout import LINKS + from ..pipeline.layout import LINKS v = getattr(args, "link", None) if v: @@ -1320,7 +1320,7 @@ def link_mode(args, cfg) -> str: # ---------------------------------------------------------------- plan def cmd_plan(args, cfg, common): - from .orchestrate import Orchestrator + from ..pipeline.orchestrate import Orchestrator layouts = [] if args.out: @@ -1401,8 +1401,8 @@ def _task_files(items) -> list[Path]: def cmd_run_stage(args, cfg, common): - from .orchestrate import run_one - from .store import Store + from ..pipeline.orchestrate import run_one + from ..pipeline.store import Store problems, tasks = [], [] for f in _task_files(args.tasks): @@ -1451,7 +1451,7 @@ def cmd_run_stage(args, cfg, common): # ---------------------------------------------------------------- catalogs def _db(args, cfg): - from .catalogdb import CatalogDB + from ..pipeline.catalogdb import CatalogDB return CatalogDB(store_root(args, cfg)) @@ -1466,7 +1466,7 @@ def _version_line(v: dict) -> str: def cmd_catalogs_list(args, cfg, common): - from .catalogdb import CATALOGS + from ..pipeline.catalogdb import CATALOGS vs = _db(args, cfg).versions() if args.json: @@ -1478,7 +1478,7 @@ def cmd_catalogs_list(args, cfg, common): def _apk_catalog(args, cfg) -> bytes | None: - from .catalogdb import apk_catalog + from ..pipeline.catalogdb import apk_catalog if getattr(args, "apk_catalog", None): return Path(args.apk_catalog).read_bytes() @@ -1522,7 +1522,7 @@ def cmd_catalogs_import(args, cfg, common): try: source = None if cfg.provider() == "jp": - from .jp import read_source + from ..source.jp import read_source source = read_source(args.file, remote).to_dict() resource = _import_resource_version(args.resource_version, Path(args.file), source) @@ -1542,12 +1542,12 @@ def cmd_catalogs_import(args, cfg, common): def cmd_catalogs_fetch(args, cfg, common): - from . import catalogdb - from .catalog import Catalog + from ..pipeline import catalogdb + from ..source.catalog import Catalog region = cfg.region() if cfg.provider(region) == "jp": - from .jp import open_catalog + from ..source.jp import open_catalog cat = open_catalog(cfg, region, apk=cfg.path("paths", "apk")) src = cat.sources() @@ -1617,8 +1617,8 @@ def addresses_of(store, vid: str) -> dict | None: def cmd_catalogs_diff(args, cfg, common): - from . import catalogdb - from .store import Store + from ..pipeline import catalogdb + from ..pipeline.store import Store db = _db(args, cfg) vs = [] @@ -1639,7 +1639,7 @@ def cmd_catalogs_diff(args, cfg, common): # ---------------------------------------------------------------- store def cmd_store_verify(args, cfg, common): - from .store import Store + from ..pipeline.store import Store r = Store(store_root(args, cfg)).verify(quick=args.quick, workers=args.workers or FETCH_WORKERS) common.print_json(r) @@ -1650,8 +1650,8 @@ def cmd_store_verify(args, cfg, common): # ---------------------------------------------------------------- parser def default_common(): """The helpers of the command line the handlers use (session: the data sources of a Config; print_json).""" - from . import cli - from .session import Session + from .. import cli + from ..source.session import Session return SimpleNamespace(session=Session, print_json=cli._print_json) @@ -1790,7 +1790,7 @@ def bind(fn): def main(argv=None) -> None: """The asset commands alone, with the global flags of the command line (tests; `nnnotes` has them too).""" - from .cli import load_config + from . import load_config p = argparse.ArgumentParser(prog="nnnotes") for flag in ( diff --git a/src/nnnotes/content/__init__.py b/src/nnnotes/content/__init__.py new file mode 100644 index 0000000..6a12f66 --- /dev/null +++ b/src/nnnotes/content/__init__.py @@ -0,0 +1 @@ +"""The game's content as exports: stories, lives, scores, spots, rooms and UI.""" diff --git a/src/nnnotes/adv.py b/src/nnnotes/content/adv.py similarity index 98% rename from src/nnnotes/adv.py rename to src/nnnotes/content/adv.py index 4c254e4..8dc863e 100644 --- a/src/nnnotes/adv.py +++ b/src/nnnotes/content/adv.py @@ -14,10 +14,11 @@ from dataclasses import dataclass from pathlib import Path -from . import master, unity +from ..source import master +from ..source.catalog import Catalog +from ..source.config import apk_missing +from ..unity import reader as unity from .advcommand import COMMAND -from .catalog import Catalog -from .config import apk_missing # AdvEpisodeResourceLoader.AddEpisodeLoadingTask, per command: (resource kind, catalog-key prefix) of the asset named # by TargetAssetName. The address is prefix + name; a blank name loads nothing (GetXxxAssetAddress). Costume rows load diff --git a/src/nnnotes/advcommand.py b/src/nnnotes/content/advcommand.py similarity index 100% rename from src/nnnotes/advcommand.py rename to src/nnnotes/content/advcommand.py diff --git a/src/nnnotes/advmedia.py b/src/nnnotes/content/advmedia.py similarity index 97% rename from src/nnnotes/advmedia.py rename to src/nnnotes/content/advmedia.py index 32195fb..3fe5848 100644 --- a/src/nnnotes/advmedia.py +++ b/src/nnnotes/content/advmedia.py @@ -36,12 +36,14 @@ from pathlib import Path -from . import adv, fontsettings, unity -from . import master as master_mod -from .catalog import Catalog -from .export import HEADER, Exporter -from .jsonio import write_json -from .player import PlayerData +from ..core.jsonio import write_json +from ..source import master as master_mod +from ..source.catalog import Catalog +from ..unity import fontsettings +from ..unity import reader as unity +from ..unity.export import HEADER, Exporter +from ..unity.player import PlayerData +from . import adv TMP_SHADER_PREFIX = "TextMeshPro/" TMP_ASSETS = ("TMP_FontAsset", "TMP_SpriteAsset", "TMP_StyleSheet") diff --git a/src/nnnotes/advscene.py b/src/nnnotes/content/advscene.py similarity index 96% rename from src/nnnotes/advscene.py rename to src/nnnotes/content/advscene.py index cbdbdaa..26ac2a1 100644 --- a/src/nnnotes/advscene.py +++ b/src/nnnotes/content/advscene.py @@ -27,11 +27,11 @@ from pathlib import Path -from . import shader as shader_mod -from .catalog import Catalog -from .export import Exporter -from .jsonio import write_json -from .player import PlayerData +from ..core.jsonio import write_json +from ..formats import shader as shader_mod +from ..source.catalog import Catalog +from ..unity.export import Exporter +from ..unity.player import PlayerData ADV_KEYS = { "cameraManager": "EmbCommon/Prefab/CameraManager", diff --git a/src/nnnotes/advui.py b/src/nnnotes/content/advui.py similarity index 99% rename from src/nnnotes/advui.py rename to src/nnnotes/content/advui.py index 31ed71e..0b5afa0 100644 --- a/src/nnnotes/advui.py +++ b/src/nnnotes/content/advui.py @@ -80,12 +80,14 @@ import re from pathlib import Path -from . import adv, advscene, fontsettings, languages, textstyle, tmpfont -from .catalog import Catalog -from .export import Exporter, TexelPacker, _safe -from .jsonio import write_json -from .player import PlayerData -from .textstyle import LANGUAGE_LINE_SPACING +from ..core.jsonio import write_json +from ..source import languages +from ..source.catalog import Catalog +from ..unity import fontsettings, textstyle, tmpfont +from ..unity.export import Exporter, TexelPacker, _safe +from ..unity.player import PlayerData +from ..unity.textstyle import LANGUAGE_LINE_SPACING +from . import adv, advscene WIDGET_KEY = "EmbUI/Prefab/UIAdvWidget" # UIManager: UIManager._cameraController -> UICameraController._uiCamera / _uiCameraData @@ -1264,7 +1266,7 @@ def mono(owner, pptr, cls: str, what: str): def master_texts(master: Path, ids, field: str) -> dict[str, str]: """{MasterText id: its text in the text field `field`} of `ids`; a missing id raises.""" - from . import master as master_mod + from ..source import master as master_mod ids = set(ids) rows = {r["_id"]: r for r in master_mod.table(Path(master), "MasterText") if r["_id"] in ids} diff --git a/src/nnnotes/advvideo.py b/src/nnnotes/content/advvideo.py similarity index 93% rename from src/nnnotes/advvideo.py rename to src/nnnotes/content/advvideo.py index b59aca6..0fb7754 100644 --- a/src/nnnotes/advvideo.py +++ b/src/nnnotes/content/advvideo.py @@ -12,11 +12,11 @@ from pathlib import Path from typing import TYPE_CHECKING -from .jsonio import write_json -from .usm import AUDIO_BITRATE, VP9_ENCODE, adx_info, demux, hca_info, webm_reencoded, webm_streams, write_webm +from ..core.jsonio import write_json +from ..formats.usm import AUDIO_BITRATE, VP9_ENCODE, adx_info, demux, hca_info, webm_reencoded, webm_streams, write_webm if TYPE_CHECKING: - from .catalog import Catalog + from ..source.catalog import Catalog VIDEO_DIR = "videos" INDEX = "videos/videos.json" @@ -28,7 +28,7 @@ def usm_asset(cat: Catalog, key: str) -> tuple[dict, Path]: """(the movie's CriWare.Assets MonoBehaviour typetree, the USM file) of a `Cri/Video/...` key.""" import UnityPy - from .addressables import remote_path + from ..source.addressables import remote_path info, raw = None, None for e in cat.dependencies(key): @@ -53,7 +53,7 @@ def export(cat: Catalog, key: str, dst: Path, cri_key: int, work: Path) -> dict: """One USM -> dst (.webm, write_webm: VP9 copied, or re-encoded when the movie's video is not VP9 or has an alpha stream; the first audio stream as Opus). Returns its record (movie info, the audio's source; `encode` when the video was re-encoded).""" - from .config import tool + from ..source.config import tool info, raw = usm_asset(cat, key) mi = info["movieInfo"] @@ -96,7 +96,8 @@ def extract(cat: Catalog, episode: dict, out_dir: Path, cri_key: int | None = No """Every video of the episode's closure -> videos/.webm; videos/videos.json maps each VideoID of the Movie / Clip rows to its file, its -Video row (`master`: size, autoStop, hasAudio) and the stream facts. Returns the index, or None when the episode has no video.""" - from . import adv, cri + from ..formats import cri + from . import adv keys = sorted(r["address"] for r in episode["resources"] if r["kind"] == "video") if not keys: diff --git a/src/nnnotes/live.py b/src/nnnotes/content/live.py similarity index 98% rename from src/nnnotes/live.py rename to src/nnnotes/content/live.py index c9fc9d3..9058fab 100644 --- a/src/nnnotes/live.py +++ b/src/nnnotes/content/live.py @@ -27,12 +27,12 @@ from pathlib import Path +from ..core.jsonio import write_json +from ..formats.encoders import FLAC_LEVEL +from ..source.catalog import Catalog +from ..unity.player import PlayerData from . import liveaudio, livenotes, livescene, liveui, score -from .catalog import Catalog -from .encoders import FLAC_LEVEL -from .jsonio import write_json from .liveoptions import LiveOptions -from .player import PlayerData from .score import master_table # Band of the preview when no deck centre is given: the band of the music's first vocal character diff --git a/src/nnnotes/liveaudio.py b/src/nnnotes/content/liveaudio.py similarity index 97% rename from src/nnnotes/liveaudio.py rename to src/nnnotes/content/liveaudio.py index f037ca3..07e7e32 100644 --- a/src/nnnotes/liveaudio.py +++ b/src/nnnotes/content/liveaudio.py @@ -6,7 +6,8 @@ sounds MasterSound id -> {row, sheet, cue, categories, volume, busSends, layers[{file, sampleRate, samples, loopStart, loopEnd, loopFlag, volume, busSends}], lengthMs}; the cue structure (sequence -> tracks -> - synth -> waveform) is read from the ACB (nnnotes.acb; command 65 categories, 146 volume, 111 bus send) + synth -> waveform) is read from the ACB (nnnotes.formats.acb; command 65 categories, 146 volume, 111 bus + send) categories CRI category volumes for a fresh profile (SoundVolumeSettings x AppConfigDefaultData) react the ACF's REACT (ducking) entries between live categories music the BGM sound id and PlayMusic volume @@ -28,16 +29,16 @@ import struct from pathlib import Path -from . import cri -from .acb import commands as _commands -from .acb import tables as _tables -from .acb import u16s as _u16s -from .apkset import ApkSet -from .catalog import Catalog -from .encoders import FLAC_LEVEL -from .jsonio import write_json +from ..core.jsonio import write_json +from ..formats import cri +from ..formats.acb import commands as _commands +from ..formats.acb import tables as _tables +from ..formats.acb import u16s as _u16s +from ..formats.encoders import FLAC_LEVEL +from ..source.apkset import ApkSet +from ..source.catalog import Catalog +from ..unity.player import PlayerData from .liveoptions import LiveOptions -from .player import PlayerData from .score import master_table ACF_IN_APK = "assets/Cri/Sound/Sirius.acf" @@ -442,7 +443,7 @@ def extract( acf = acf_info(cat.apk) doc = { "musicId": music_id, - "spec": "nnnotes.liveaudio", + "spec": "nnnotes.content.liveaudio", "sounds": entries, "categories": category_volumes(player), "acf": {"name": acf["name"], "buses": acf["buses"], "categories": acf["categories"]}, diff --git a/src/nnnotes/livenotes.py b/src/nnnotes/content/livenotes.py similarity index 99% rename from src/nnnotes/livenotes.py rename to src/nnnotes/content/livenotes.py index ce8c85b..3ffdabc 100644 --- a/src/nnnotes/livenotes.py +++ b/src/nnnotes/content/livenotes.py @@ -32,12 +32,12 @@ import numpy as np -from .catalog import Catalog -from .export import Exporter -from .jsonio import write_json +from ..core.jsonio import write_json +from ..source.catalog import Catalog +from ..unity.export import Exporter +from ..unity.player import PlayerData from .liveoptions import OPTION_ITEM_TYPES, PLAYER_ITEMS, LiveOptions from .livescene import LANE_COUNT, LANE_VIEW -from .player import PlayerData from .score import master_table # --- keys ------------------------------------------------------------------ diff --git a/src/nnnotes/liveoptions.py b/src/nnnotes/content/liveoptions.py similarity index 100% rename from src/nnnotes/liveoptions.py rename to src/nnnotes/content/liveoptions.py diff --git a/src/nnnotes/livescene.py b/src/nnnotes/content/livescene.py similarity index 99% rename from src/nnnotes/livescene.py rename to src/nnnotes/content/livescene.py index 52dbe23..1fc5a4e 100644 --- a/src/nnnotes/livescene.py +++ b/src/nnnotes/content/livescene.py @@ -36,8 +36,8 @@ import numpy as np -from .export import Exporter -from .jsonio import write_json +from ..core.jsonio import write_json +from ..unity.export import Exporter from .liveoptions import OPTION_ITEM_TYPES from .score import master_table @@ -512,7 +512,7 @@ def music_part( already in /livescene): the document extract writes. Raises SharedSceneMismatch when the jacket's bundle closure holds shaders or a hierarchy (extract would dump or resolve against them) or the jacket texture is one of the scene's (extract would keep it).""" - from .export import Exporter as _Exporter + from ..unity.export import Exporter as _Exporter out_dir = Path(out_dir) band = part["band"] diff --git a/src/nnnotes/liveui.py b/src/nnnotes/content/liveui.py similarity index 98% rename from src/nnnotes/liveui.py rename to src/nnnotes/content/liveui.py index 44e6c8b..9438237 100644 --- a/src/nnnotes/liveui.py +++ b/src/nnnotes/content/liveui.py @@ -27,13 +27,14 @@ import json from pathlib import Path -from . import fontsettings, languages, textstyle, tmpfont -from .catalog import Catalog -from .export import Exporter, TexelPacker, _safe -from .jsonio import write_json -from .player import PlayerData +from ..core.jsonio import write_json +from ..source import languages +from ..source.catalog import Catalog +from ..unity import fontsettings, textstyle, tmpfont +from ..unity.export import Exporter, TexelPacker, _safe +from ..unity.player import PlayerData +from ..unity.textstyle import LANGUAGE_LINE_SPACING from .score import master_table -from .textstyle import LANGUAGE_LINE_SPACING SCENE_KEY = "EmbScene/Live" CANVAS = "Live/UILiveStartCanvas" diff --git a/src/nnnotes/room.py b/src/nnnotes/content/room.py similarity index 99% rename from src/nnnotes/room.py rename to src/nnnotes/content/room.py index c9ea6a3..d3e0ef7 100644 --- a/src/nnnotes/room.py +++ b/src/nnnotes/content/room.py @@ -40,9 +40,9 @@ import numpy as np import UnityPy -from .catalog import Catalog -from .jsonio import dumps, write_json -from .unity import SceneGraph, deref, mesh_arrays, texture_image +from ..core.jsonio import dumps, write_json +from ..source.catalog import Catalog +from ..unity.reader import SceneGraph, deref, mesh_arrays, texture_image # UnityEngine.Rendering.BlendMode BLEND_ONE, BLEND_ZERO, BLEND_SRC_ALPHA, BLEND_ONE_MINUS_SRC_ALPHA = 1, 0, 5, 10 diff --git a/src/nnnotes/score.py b/src/nnnotes/content/score.py similarity index 99% rename from src/nnnotes/score.py rename to src/nnnotes/content/score.py index ee949e4..80bce23 100644 --- a/src/nnnotes/score.py +++ b/src/nnnotes/content/score.py @@ -30,9 +30,10 @@ import numpy as np -from . import cri, languages -from .encoders import FLAC_LEVEL -from .jsonio import write_json +from ..core.jsonio import write_json +from ..formats import cri +from ..formats.encoders import FLAC_LEVEL +from ..source import languages f32 = np.float32 @@ -1215,7 +1216,7 @@ def chart_key(file_name: str) -> str: def fetch_chart(cat, file_name: str) -> bytes: """Raw TextAsset bytes of a chart (still gzip, as shipped).""" - from .unity import load_closure + from ..unity.reader import load_closure key = chart_key(file_name) name = file_name.rsplit("/", 1)[-1] diff --git a/src/nnnotes/spot.py b/src/nnnotes/content/spot.py similarity index 97% rename from src/nnnotes/spot.py rename to src/nnnotes/content/spot.py index 4daec94..92b1280 100644 --- a/src/nnnotes/spot.py +++ b/src/nnnotes/content/spot.py @@ -24,10 +24,10 @@ import UnityPy -from .catalog import Catalog -from .config import apk_missing -from .jsonio import write_json -from .unity import SceneGraph, script_class, strip_pptrs, texture_image +from ..core.jsonio import write_json +from ..source.catalog import Catalog +from ..source.config import apk_missing +from ..unity.reader import SceneGraph, script_class, strip_pptrs, texture_image LANGS = ("_japanese", "_english", "_traditionalChinese", "_simplifiedChinese", "_korean") diff --git a/src/nnnotes/story.py b/src/nnnotes/content/story.py similarity index 97% rename from src/nnnotes/story.py rename to src/nnnotes/content/story.py index ba543da..1209e69 100644 --- a/src/nnnotes/story.py +++ b/src/nnnotes/content/story.py @@ -30,10 +30,11 @@ import json from pathlib import Path -from . import adv, advmedia, advscene, advui, advvideo, cri, crilips -from .catalog import Catalog -from .jsonio import write_json -from .player import PlayerData +from ..core.jsonio import write_json +from ..formats import cri, crilips +from ..source.catalog import Catalog +from ..unity.player import PlayerData +from . import adv, advmedia, advscene, advui, advvideo # resource kind -> where the story directory holds it KIND_OUTPUT = { diff --git a/src/nnnotes/storyfonts.py b/src/nnnotes/content/storyfonts.py similarity index 99% rename from src/nnnotes/storyfonts.py rename to src/nnnotes/content/storyfonts.py index 526877e..3d944e8 100644 --- a/src/nnnotes/storyfonts.py +++ b/src/nnnotes/content/storyfonts.py @@ -56,10 +56,13 @@ import numpy as np -from . import adv, advui, cache, fontsettings, jsonio, languages, master, textstyle, tmpfont -from . import shader as shader_mod -from .config import ConfigError -from .export import Exporter, _safe, packed_png, texture_png +from ..core import cache, jsonio +from ..formats import shader as shader_mod +from ..source import languages, master +from ..source.config import ConfigError +from ..unity import fontsettings, textstyle, tmpfont +from ..unity.export import Exporter, _safe, packed_png, texture_png +from . import adv, advui FONTS_FORMAT = "ournotes.story-fonts/1" LANGUAGE_FORMAT = "ournotes.story-language/1" diff --git a/src/nnnotes/storyhost.py b/src/nnnotes/content/storyhost.py similarity index 99% rename from src/nnnotes/storyhost.py rename to src/nnnotes/content/storyhost.py index 0680d65..51d283d 100644 --- a/src/nnnotes/storyhost.py +++ b/src/nnnotes/content/storyhost.py @@ -35,11 +35,13 @@ import numpy as np import UnityPy -from . import advui, fontsettings, languages, master, room, spot, storyfonts, textstyle, tmpfont -from . import shader as shader_mod -from .export import RECT_PROPS, Exporter, TexelPacker, streamed_frames -from .jsonio import dumps, write_json -from .unity import SceneGraph, deref, mesh_arrays, script_class +from ..core.jsonio import dumps, write_json +from ..formats import shader as shader_mod +from ..source import languages, master +from ..unity import fontsettings, textstyle, tmpfont +from ..unity.export import RECT_PROPS, Exporter, TexelPacker, streamed_frames +from ..unity.reader import SceneGraph, deref, mesh_arrays, script_class +from . import advui, room, spot, storyfonts FORMAT = "ournotes.story-host/1" OVERLAY = 1 # MasterAdv._playbackMode of the episodes SimpleAdvPlayer plays diff --git a/src/nnnotes/storyvoices.py b/src/nnnotes/content/storyvoices.py similarity index 99% rename from src/nnnotes/storyvoices.py rename to src/nnnotes/content/storyvoices.py index 959003d..5533170 100644 --- a/src/nnnotes/storyvoices.py +++ b/src/nnnotes/content/storyvoices.py @@ -83,7 +83,7 @@ def episode(script=None, text=None, sound=None, sheets=None, *, sheet_column: st def load_episode(cat, keys: dict) -> dict: """The parts of one episode read from the catalog (`keys`: {part: catalog key}, PARTS; a key the catalog lacks gives no part), as `episode` takes them.""" - from . import unity + from ..unity import reader as unity got: dict = {} for part, key in keys.items(): diff --git a/src/nnnotes/ui.py b/src/nnnotes/content/ui.py similarity index 97% rename from src/nnnotes/ui.py rename to src/nnnotes/content/ui.py index 0b8e639..a732f98 100644 --- a/src/nnnotes/ui.py +++ b/src/nnnotes/content/ui.py @@ -11,9 +11,9 @@ from collections import Counter from pathlib import Path -from .cache import write_atomic -from .export import Exporter -from .jsonio import dumps, write_json +from ..core.cache import write_atomic +from ..core.jsonio import dumps, write_json +from ..unity.export import Exporter def identity(value: str) -> str: @@ -375,11 +375,11 @@ def save_index(): def command(args, cfg): - from .apkset import MANIFEST_IN_APK, ApkSet, manifest_version_code, manifest_version_name - from .catalog import APK_CATALOG, Catalog - from .config import ConfigError - from .player import PlayerData - from .session import Session + from ..source.apkset import MANIFEST_IN_APK, ApkSet, manifest_version_code, manifest_version_name + from ..source.catalog import APK_CATALOG, Catalog + from ..source.config import ConfigError + from ..source.session import Session + from ..unity.player import PlayerData if args.limit is not None and args.limit < 1: args.usage("--limit must be positive") diff --git a/src/nnnotes/core/__init__.py b/src/nnnotes/core/__init__.py new file mode 100644 index 0000000..2753cd7 --- /dev/null +++ b/src/nnnotes/core/__init__.py @@ -0,0 +1 @@ +"""Foundations every layer uses: JSON text and the process caches.""" diff --git a/src/nnnotes/cache.py b/src/nnnotes/core/cache.py similarity index 99% rename from src/nnnotes/cache.py rename to src/nnnotes/core/cache.py index db9ca1e..7844665 100644 --- a/src/nnnotes/cache.py +++ b/src/nnnotes/core/cache.py @@ -7,7 +7,7 @@ reuses them. Keys of the disk layer are salted with the source of the module that produces the value and the versions of the libraries it calls, so a code or library change never reads an older result. - from nnnotes import cache + from nnnotes.core import cache cache.configure(enabled=False) # every get misses, nothing is stored cache.configure(memory_mb=512, directory="work/cache") PNG = cache.bucket("png", salt=cache.source_salt(__file__)) diff --git a/src/nnnotes/jsonio.py b/src/nnnotes/core/jsonio.py similarity index 100% rename from src/nnnotes/jsonio.py rename to src/nnnotes/core/jsonio.py diff --git a/src/nnnotes/formats/__init__.py b/src/nnnotes/formats/__init__.py new file mode 100644 index 0000000..388f3b4 --- /dev/null +++ b/src/nnnotes/formats/__init__.py @@ -0,0 +1 @@ +"""File formats read from bytes: CRI audio, movies and lip sync, shaders, motions.""" diff --git a/src/nnnotes/acb.py b/src/nnnotes/formats/acb.py similarity index 100% rename from src/nnnotes/acb.py rename to src/nnnotes/formats/acb.py diff --git a/src/nnnotes/cri.py b/src/nnnotes/formats/cri.py similarity index 98% rename from src/nnnotes/cri.py rename to src/nnnotes/formats/cri.py index 9bf6c8c..195d05c 100644 --- a/src/nnnotes/cri.py +++ b/src/nnnotes/formats/cri.py @@ -46,12 +46,13 @@ import numpy as np -from . import cache, crikey -from .addressables import remote_path -from .catalog import Catalog -from .config import tool, usable_cpus +from ..core import cache +from ..core.jsonio import dumps, write_json +from ..source.addressables import remote_path +from ..source.catalog import Catalog +from ..source.config import tool, usable_cpus +from . import crikey from .encoders import FLAC_LEVEL, encode_args -from .jsonio import dumps, write_json DECODED = cache.bucket("cuesheets", salt=cache.source_salt(__file__), disk=True) SHEETS = cache.bucket("acb", max_item=128 << 20) # (cue sheet, bundle closure) -> ACB / AWB bytes, layout diff --git a/src/nnnotes/crikey.py b/src/nnnotes/formats/crikey.py similarity index 98% rename from src/nnnotes/crikey.py rename to src/nnnotes/formats/crikey.py index eefac01..23f59e8 100644 --- a/src/nnnotes/crikey.py +++ b/src/nnnotes/formats/crikey.py @@ -16,7 +16,7 @@ import struct from pathlib import Path -from .apkset import ApkSet +from ..source.apkset import ApkSet DATA_IN_APK = "assets/bin/Data/data.unity3d" diff --git a/src/nnnotes/crilips.py b/src/nnnotes/formats/crilips.py similarity index 99% rename from src/nnnotes/crilips.py rename to src/nnnotes/formats/crilips.py index 984464a..2093340 100644 --- a/src/nnnotes/crilips.py +++ b/src/nnnotes/formats/crilips.py @@ -25,8 +25,8 @@ from dataclasses import dataclass, field from pathlib import Path -from .apkset import ApkSet -from .jsonio import write_json +from ..core.jsonio import write_json +from ..source.apkset import ApkSet LIB_ENTRY_SUFFIX = "lib/arm64-v8a/libcri_lips_unity.so" # the arm64-v8a split of an installed app (adb pull of its directory), or of an .xapk package diff --git a/src/nnnotes/encoders.py b/src/nnnotes/formats/encoders.py similarity index 100% rename from src/nnnotes/encoders.py rename to src/nnnotes/formats/encoders.py diff --git a/src/nnnotes/motion.py b/src/nnnotes/formats/motion.py similarity index 99% rename from src/nnnotes/motion.py rename to src/nnnotes/formats/motion.py index dfa59b2..82f140e 100644 --- a/src/nnnotes/motion.py +++ b/src/nnnotes/formats/motion.py @@ -17,7 +17,7 @@ import math from pathlib import Path -from .jsonio import write_json +from ..core.jsonio import write_json SEG_LINEAR, SEG_BEZIER, SEG_STEPPED = 0, 1, 2 WEIGHT_IN, WEIGHT_OUT = 1, 2 # UnityEngine.WeightedMode flags diff --git a/src/nnnotes/shader.py b/src/nnnotes/formats/shader.py similarity index 99% rename from src/nnnotes/shader.py rename to src/nnnotes/formats/shader.py index 5b7b20f..aac32f8 100644 --- a/src/nnnotes/shader.py +++ b/src/nnnotes/formats/shader.py @@ -33,8 +33,8 @@ from UnityPy.enums import ShaderGpuProgramType from UnityPy.export.ShaderConverter import CompressionHelper -from . import cache -from .jsonio import dumps, write_json +from ..core import cache +from ..core.jsonio import dumps, write_json DUMPS = cache.bucket("shaders", salt=cache.source_salt(__file__, "UnityPy"), disk=True) diff --git a/src/nnnotes/usm.py b/src/nnnotes/formats/usm.py similarity index 100% rename from src/nnnotes/usm.py rename to src/nnnotes/formats/usm.py diff --git a/src/nnnotes/musicdata.py b/src/nnnotes/musicdata/__init__.py similarity index 99% rename from src/nnnotes/musicdata.py rename to src/nnnotes/musicdata/__init__.py index 2153cc5..2ed65ff 100644 --- a/src/nnnotes/musicdata.py +++ b/src/nnnotes/musicdata/__init__.py @@ -33,8 +33,8 @@ from decimal import Decimal, localcontext from pathlib import Path +from ..source.languages import LANGUAGES from . import deckdata -from .languages import LANGUAGES FORMAT = "nnnotes.music-data/2" DIFFICULTIES = ("easy", "normal", "hard", "expert") @@ -126,7 +126,7 @@ def bpm_facts(bpm_events: list, first_ms: int, last_ms: int) -> dict: def chart_facts(score_row: dict, key: str, raw: bytes) -> dict: """The facts of one chart: level, note counts, BPM, chart times, skill events and fevers.""" - from . import score + from ..content import score try: root = score.load_bytes(raw) @@ -173,7 +173,7 @@ def chart_facts(score_row: dict, key: str, raw: bytes) -> dict: def cue_length(acb: bytes, cue: str, where: str) -> dict: """{lengthMs, samples, sampleRate, durationMs} of a cue of an ACB: the CueTable `Length` and the first stream's sample count and rate (durationMs = samples * 1000 // sampleRate).""" - from . import acb as acbmod + from ..formats import acb as acbmod try: cues = acbmod.cue_streams(acb) @@ -194,7 +194,7 @@ def cue_length(acb: bytes, cue: str, where: str) -> dict: def catalog_bgm(cat) -> Callable[[str, str], dict]: """bgm(cue sheet, cue) for a catalog: the cue sheet's ACB (cri.acb_data) -> cue_length.""" - from . import cri + from ..formats import cri def bgm(sheet: str, cue: str) -> dict: try: @@ -234,7 +234,7 @@ def catalog_jacket(cat, size: int = JACKET_SIZE) -> Callable[[str], bytes]: """jacket(name) for a catalog: the WebP bytes (jacket_webp) of the Texture2D `Image/Jacket/`.""" import tempfile - from .export import Exporter + from ..unity.export import Exporter ex = Exporter(cat, Path(tempfile.gettempdir()), textures="deferred") @@ -1212,7 +1212,7 @@ def build( deck model measuring the songs' charts (and their Gekisou aptitude), None to leave every chart's `deck` null; `full`: add the deck input (`master`, `charts`: deckdata.TABLES and every chart's runtime notes). `tables`: tables_of(deck, full).""" - from . import __version__ + from .. import __version__ raws: dict[str, bytes] = {} @@ -1431,7 +1431,7 @@ def export( """Read the master data, every chart and every BGM cue sheet, measure the charts with `deck`, then write the file `out` (gzip when it ends in `.gz`) through a temporary file and a rename; with `jacket` and `jackets_dir`, first every song's jacket as `/.webp`. Returns the summary.""" - from .cache import write_atomic + from ..core.cache import write_atomic out = Path(out) try: diff --git a/src/nnnotes/deckdata.py b/src/nnnotes/musicdata/deckdata.py similarity index 98% rename from src/nnnotes/deckdata.py rename to src/nnnotes/musicdata/deckdata.py index d1a462e..5643b43 100644 --- a/src/nnnotes/deckdata.py +++ b/src/nnnotes/musicdata/deckdata.py @@ -29,7 +29,7 @@ from dataclasses import dataclass from pathlib import Path -from .apkset import ApkSet +from ..source.apkset import ApkSet DECK_FORMAT = "nnnotes.deck-data/1" # the deck model's input format (ournotes-sim data::FORMAT) CHART_FORMAT = "nnnotes.live-score/1" # score.convert's format: the converter the notes come from @@ -256,7 +256,7 @@ def read(names): def read_master(src: MasterSource, key, tables=None) -> tuple[dict[str, list[dict]], dict[str, str]]: """The rows (`_allData`) of every table of `tables` (default: TABLES) and the SHA-256 of each file as served. `key`: a master.MasterKey (unused for decoded master data: None).""" - from . import master + from ..source import master names = {t: f"{t}.bin" for t in (tables if tables is not None else (t for t, _ in TABLES))} unlisted = [t for t, n in names.items() if n not in src.hashes] @@ -356,7 +356,7 @@ def chart_key(file_name: str) -> str: def chart_record(score_id: int, key: str, raw: bytes) -> dict: """The record of one chart: the TextAsset bytes as shipped -> runtime notes in enumeration order, skill events in chart order, fever ranges sorted by start.""" - from . import score + from ..content import score try: root = score.load_bytes(raw) @@ -410,7 +410,7 @@ def charts(score_rows: list[dict], fetch: Callable[[str], bytes]) -> list[dict]: def catalog_fetch(cat) -> Callable[[str], bytes]: """fetch() for charts from a catalog: score.fetch_chart, KeyError for a key the catalog does not have.""" - from . import score + from ..content import score def fetch(file_name: str) -> bytes: if not cat.has(chart_key(file_name)): @@ -423,7 +423,7 @@ def fetch(file_name: str) -> bytes: # ---------------------------------------------------------------- provenance def apk_client(apk) -> dict: """{versionName, versionCode} of an APK's AndroidManifest.xml (None when absent).""" - from .apkset import MANIFEST_IN_APK, manifest_version_code, manifest_version_name + from ..source.apkset import MANIFEST_IN_APK, manifest_version_code, manifest_version_name try: with ApkSet(apk) as z: @@ -453,7 +453,7 @@ def resource_version(store_root, remote_sha: str) -> str | None: (the latest import that has one), None when none is recorded. Read only.""" if store_root is None: return None - from .catalogdb import CatalogDB + from ..pipeline.catalogdb import CatalogDB try: versions = CatalogDB(store_root).versions() diff --git a/src/nnnotes/replaydata.py b/src/nnnotes/musicdata/replaydata.py similarity index 99% rename from src/nnnotes/replaydata.py rename to src/nnnotes/musicdata/replaydata.py index f3ac2a0..218c161 100644 --- a/src/nnnotes/replaydata.py +++ b/src/nnnotes/musicdata/replaydata.py @@ -8,8 +8,8 @@ import tarfile from pathlib import Path +from ..core.cache import write_atomic from . import deckdata -from .cache import write_atomic FORMAT = "nnnotes.replay-manifest/1" CHART_FORMAT = "nnnotes.replay-chart/1" diff --git a/src/nnnotes/pipeline/__init__.py b/src/nnnotes/pipeline/__init__.py new file mode 100644 index 0000000..261d5d6 --- /dev/null +++ b/src/nnnotes/pipeline/__init__.py @@ -0,0 +1 @@ +"""The content-addressed export pipeline: stages, tasks, the store, layouts and the atoms.""" diff --git a/src/nnnotes/atoms/__init__.py b/src/nnnotes/pipeline/atoms/__init__.py similarity index 100% rename from src/nnnotes/atoms/__init__.py rename to src/nnnotes/pipeline/atoms/__init__.py diff --git a/src/nnnotes/atoms/astc.py b/src/nnnotes/pipeline/atoms/astc.py similarity index 100% rename from src/nnnotes/atoms/astc.py rename to src/nnnotes/pipeline/atoms/astc.py diff --git a/src/nnnotes/atoms/gltf.py b/src/nnnotes/pipeline/atoms/gltf.py similarity index 99% rename from src/nnnotes/atoms/gltf.py rename to src/nnnotes/pipeline/atoms/gltf.py index d323453..65a9611 100644 --- a/src/nnnotes/atoms/gltf.py +++ b/src/nnnotes/pipeline/atoms/gltf.py @@ -27,7 +27,7 @@ import numpy as np -from .. import jsonio +from ...core import jsonio from . import Unsupported, estimate from .mesh import Channel, MeshArrays, require_vertices diff --git a/src/nnnotes/atoms/mesh.py b/src/nnnotes/pipeline/atoms/mesh.py similarity index 100% rename from src/nnnotes/atoms/mesh.py rename to src/nnnotes/pipeline/atoms/mesh.py diff --git a/src/nnnotes/atoms/png.py b/src/nnnotes/pipeline/atoms/png.py similarity index 100% rename from src/nnnotes/atoms/png.py rename to src/nnnotes/pipeline/atoms/png.py diff --git a/src/nnnotes/atoms/reader.py b/src/nnnotes/pipeline/atoms/reader.py similarity index 99% rename from src/nnnotes/atoms/reader.py rename to src/nnnotes/pipeline/atoms/reader.py index dac1f20..dc74dd8 100644 --- a/src/nnnotes/atoms/reader.py +++ b/src/nnnotes/pipeline/atoms/reader.py @@ -298,7 +298,7 @@ def mesh_input(self, obj: ObjectInfo) -> MeshInput: def open_bundle(data: bytes) -> UnityBundle: """The BundleView of a bundle's bytes (decrypted UnityFS), loaded alone.""" - from ..unity import load_bytes + from ...unity.reader import load_bytes return UnityBundle(load_bytes(bytes(data))) diff --git a/src/nnnotes/atoms/sniff.py b/src/nnnotes/pipeline/atoms/sniff.py similarity index 100% rename from src/nnnotes/atoms/sniff.py rename to src/nnnotes/pipeline/atoms/sniff.py diff --git a/src/nnnotes/atoms/sprite.py b/src/nnnotes/pipeline/atoms/sprite.py similarity index 100% rename from src/nnnotes/atoms/sprite.py rename to src/nnnotes/pipeline/atoms/sprite.py diff --git a/src/nnnotes/atoms/texture.py b/src/nnnotes/pipeline/atoms/texture.py similarity index 100% rename from src/nnnotes/atoms/texture.py rename to src/nnnotes/pipeline/atoms/texture.py diff --git a/src/nnnotes/catalogdb.py b/src/nnnotes/pipeline/catalogdb.py similarity index 99% rename from src/nnnotes/catalogdb.py rename to src/nnnotes/pipeline/catalogdb.py index bc17b14..7b5388f 100644 --- a/src/nnnotes/catalogdb.py +++ b/src/nnnotes/pipeline/catalogdb.py @@ -30,10 +30,10 @@ import urllib.request from pathlib import Path +from ..source.addressables import parse_header, parse_keys, parse_locations, remote_path +from ..source.apkset import ApkSet +from ..source.catalog import APK_CATALOG, Catalog, file_name, location_kind from . import contract -from .addressables import parse_header, parse_keys, parse_locations, remote_path -from .apkset import ApkSet -from .catalog import APK_CATALOG, Catalog, file_name, location_kind from .contract import Cost, Input from .stages import Output, Stage from .store import write_file @@ -370,7 +370,7 @@ def version_id(remote_sha: str, apk_sha: str | None, source: dict | None = None) """The id of a catalog version: the key hash of its two catalogs' content ids.""" identity = {"remote": remote_sha, "apk": apk_sha} if source is not None: - from .jp import Source + from ..source.jp import Source identity["source"] = Source.from_dict(source).to_dict() return contract.digest(identity) diff --git a/src/nnnotes/census.py b/src/nnnotes/pipeline/census.py similarity index 99% rename from src/nnnotes/census.py rename to src/nnnotes/pipeline/census.py index b288627..4eb71e2 100644 --- a/src/nnnotes/census.py +++ b/src/nnnotes/pipeline/census.py @@ -251,7 +251,7 @@ def head(node, fields=HEAD_FIELDS): def census_file(path) -> dict: """The census of a bundle file (decrypted UnityFS).""" - from . import unity + from ..unity import reader as unity env = unity.load(path) try: diff --git a/src/nnnotes/contract.py b/src/nnnotes/pipeline/contract.py similarity index 99% rename from src/nnnotes/contract.py rename to src/nnnotes/pipeline/contract.py index d984925..00fb993 100644 --- a/src/nnnotes/contract.py +++ b/src/nnnotes/pipeline/contract.py @@ -34,7 +34,7 @@ from dataclasses import dataclass, field from functools import cached_property -from . import jsonio +from ..core import jsonio CANON = "canon/1" TASK = "nnnotes.task/1" diff --git a/src/nnnotes/cristages.py b/src/nnnotes/pipeline/cristages.py similarity index 98% rename from src/nnnotes/cristages.py rename to src/nnnotes/pipeline/cristages.py index b378ff0..87f314d 100644 --- a/src/nnnotes/cristages.py +++ b/src/nnnotes/pipeline/cristages.py @@ -42,11 +42,11 @@ from collections import defaultdict from pathlib import Path +from ..formats.encoders import FLAC_LEVEL +from ..source.apkset import ApkSet from . import contract -from .apkset import ApkSet from .atoms import impl_id from .contract import Cost, IncompatibleTask, Input -from .encoders import FLAC_LEVEL from .stages import Output, Pending, Stage AUDIO, MOVIE, EXPORT = "cri.audio", "cri.movie", "unity.export" @@ -108,7 +108,7 @@ def hca_key(store, inp: Input) -> int: with _lock: if inp.sha256 in _keys: return _keys[inp.sha256] - from . import crikey + from ..formats import crikey key = crikey.find_key(store.open_input(inp)) with _lock: @@ -148,7 +148,7 @@ def tool_id(which: str, exe: str) -> str: def _tool(which: str) -> str: - from .config import tool + from ..source.config import tool return tool(which, "vgmstream-cli" if which == "vgmstream" else "ffmpeg") @@ -379,7 +379,7 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: return Cost(0.3 + size * 2.0e-6, (64 << 20) + 8 * size) def run(self, task, store) -> Output: - from . import cri + from ..formats import cri vgm, ff = _tool("vgmstream"), _tool("ffmpeg") self._check_atoms(task, {"hca.decode": tool_id("vgmstream", vgm), "flac.encode": tool_id("ffmpeg", ff)}) @@ -543,7 +543,7 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: return Cost(0.3 + size * 1.1e-7, (96 << 20) + 12 * size) def run(self, task, store) -> Output: - from . import usm + from ..formats import usm ff, vgm = _tool("ffmpeg"), _tool("vgmstream") self._check_atoms(task, movie_atoms(ff, vgm)) @@ -598,8 +598,8 @@ def run(self, task, store) -> Output: def _subtitle(self, task, store, s) -> tuple[list[dict], str | None]: """The artifacts of a subtitle channel and why it is not a track (None when it is): the records as JSON, and as SubRip and WebVTT when every text is UTF-8; the bytes as stored (`.sbt`) when they are not records.""" - from . import usm - from .jsonio import dumps + from ..core.jsonio import dumps + from ..formats import usm try: records = usm.subtitle_records(s.data) diff --git a/src/nnnotes/layout.py b/src/nnnotes/pipeline/layout.py similarity index 99% rename from src/nnnotes/layout.py rename to src/nnnotes/pipeline/layout.py index 3f8241f..9659b15 100644 --- a/src/nnnotes/layout.py +++ b/src/nnnotes/pipeline/layout.py @@ -55,8 +55,8 @@ from dataclasses import dataclass from pathlib import Path +from ..core.cache import temp_path from . import contract -from .cache import temp_path from .store import write_doc, write_file MANIFEST = "manifest.json" diff --git a/src/nnnotes/link.py b/src/nnnotes/pipeline/link.py similarity index 100% rename from src/nnnotes/link.py rename to src/nnnotes/pipeline/link.py diff --git a/src/nnnotes/modelstages.py b/src/nnnotes/pipeline/modelstages.py similarity index 99% rename from src/nnnotes/modelstages.py rename to src/nnnotes/pipeline/modelstages.py index 2737319..d476618 100644 --- a/src/nnnotes/modelstages.py +++ b/src/nnnotes/pipeline/modelstages.py @@ -34,9 +34,9 @@ import tempfile from pathlib import Path +from ..source.catalog import Bundle from . import catalogdb, contract, link from .atoms import impl_id -from .catalog import Bundle from .contract import Cost, IncompatibleTask, Input from .stages import Output, Pending, Stage @@ -269,7 +269,7 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: def run(self, task, store) -> Output: _check_atoms(task, self.ATOMS) - from . import live2d + from ..unity import live2d cat = TaskCatalog.from_task(task, store) arts = [] @@ -412,7 +412,8 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: def run(self, task, store) -> Output: _check_atoms(task, self.ATOMS) - from . import spot, unity + from ..content import spot + from ..unity import reader as unity oid = task.context["object"] paths = [store.open_input(i) for i in task.inputs if i.role.startswith("bundle:")] diff --git a/src/nnnotes/objexport.py b/src/nnnotes/pipeline/objexport.py similarity index 99% rename from src/nnnotes/objexport.py rename to src/nnnotes/pipeline/objexport.py index 89718c6..733bb22 100644 --- a/src/nnnotes/objexport.py +++ b/src/nnnotes/pipeline/objexport.py @@ -566,7 +566,7 @@ def _bytes_value(self, owner, v, path: str, signed: bool = False) -> dict: else: data = bytes(v) if self.acb_field is not None and self.acb_field[:2] == (self._key(owner), path): - from . import cri + from ..formats import cri if data[:4] != cri.ACB_SIGNATURE: raise ValueError(f"{path}: the cue sheet data is not an ACB (@UTF): {data[:4].hex()}") @@ -760,7 +760,7 @@ def run_bundle(self) -> tuple[list, list]: def _index(self) -> None: """Hierarchies per serialized file (their roots, the objects each root's prefab holds), scene files.""" - from .unity import SceneGraph + from ..unity.reader import SceneGraph by_file = defaultdict(list) for i in self.infos: @@ -1116,7 +1116,7 @@ def conv_material(self, info) -> list: def conv_shader(self, info) -> list: """The parsed form (the bytes `nnnotes shader` writes as .json), every compiled sub-program as stored (`program:/sp

__`), the variant index and the typetree JSON.""" - from . import shader + from ..formats import shader key = (info.file, info.path_id) r, tt = self.readers[key], self._tt(key) @@ -1207,7 +1207,7 @@ def conv_mono(self, info) -> list: def _split_acb(self, key: tuple, acb: dict) -> str: """The `acb` artifact of a SplitAcbData: its chunk TextAssets (of this bundle) joined and unmasked.""" - from . import cri + from ..formats import cri owner, parts = self.readers[key], [] for c in acb["chunks"]: @@ -1385,7 +1385,7 @@ def _loose(self, cls: str, conv: str) -> None: @cache def exporter_type(): """RefExporter combined with export.Exporter (imported here: it loads UnityPy).""" - from .export import Exporter + from ..unity.export import Exporter return type("ObjectExporter", (RefExporter, Exporter), {}) diff --git a/src/nnnotes/orchestrate.py b/src/nnnotes/pipeline/orchestrate.py similarity index 99% rename from src/nnnotes/orchestrate.py rename to src/nnnotes/pipeline/orchestrate.py index be5ffa0..bb534cf 100644 --- a/src/nnnotes/orchestrate.py +++ b/src/nnnotes/pipeline/orchestrate.py @@ -32,8 +32,8 @@ from dataclasses import dataclass, field from multiprocessing.connection import wait +from ..source.config import ConfigError from . import contract -from .config import ConfigError from .contract import Task from .stages import Env, Pending, Stage, describe, execute diff --git a/src/nnnotes/plan.py b/src/nnnotes/pipeline/plan.py similarity index 100% rename from src/nnnotes/plan.py rename to src/nnnotes/pipeline/plan.py diff --git a/src/nnnotes/stages.py b/src/nnnotes/pipeline/stages.py similarity index 100% rename from src/nnnotes/stages.py rename to src/nnnotes/pipeline/stages.py diff --git a/src/nnnotes/store.py b/src/nnnotes/pipeline/store.py similarity index 99% rename from src/nnnotes/store.py rename to src/nnnotes/pipeline/store.py index 4f75040..a3218aa 100644 --- a/src/nnnotes/store.py +++ b/src/nnnotes/pipeline/store.py @@ -28,8 +28,8 @@ import time from pathlib import Path +from ..core.cache import temp_path from . import contract -from .cache import temp_path from .contract import Input INPUT_KINDS = ("bundles", "raw", "files") diff --git a/src/nnnotes/viewrules.json b/src/nnnotes/pipeline/viewrules.json similarity index 100% rename from src/nnnotes/viewrules.json rename to src/nnnotes/pipeline/viewrules.json diff --git a/src/nnnotes/views.py b/src/nnnotes/pipeline/views.py similarity index 99% rename from src/nnnotes/views.py rename to src/nnnotes/pipeline/views.py index a27b17f..061b25f 100644 --- a/src/nnnotes/views.py +++ b/src/nnnotes/pipeline/views.py @@ -49,7 +49,8 @@ from pathlib import Path from typing import Protocol -from . import contract, languages +from ..source import languages +from . import contract from .contract import Cost, Input from .stages import Output, Stage diff --git a/src/nnnotes/voices.py b/src/nnnotes/pipeline/voices.py similarity index 99% rename from src/nnnotes/voices.py rename to src/nnnotes/pipeline/voices.py index 426c731..5ac9315 100644 --- a/src/nnnotes/voices.py +++ b/src/nnnotes/pipeline/voices.py @@ -60,7 +60,10 @@ import unicodedata from pathlib import Path -from . import acb, advcommand, contract, languages, storyvoices, views +from ..content import advcommand, storyvoices +from ..formats import acb +from ..source import languages +from . import contract, views from .contract import Cost, Input from .stages import Output, Pending, Stage @@ -1253,7 +1256,7 @@ def _out(text: str) -> None: def _print_json(obj) -> None: - from .jsonio import dumps + from ..core.jsonio import dumps _out(dumps(obj, ensure_ascii=False, indent=1) + "\n") @@ -1402,7 +1405,7 @@ def cmd_summary(args, cfg, common) -> None: def _decode_one(cfg, common, sheet: str, cue: str, fmt: str, level: int, dst: Path) -> list[Path]: """The files of a cue's streams, decoded from its sheet alone (cri.decode, as the export decodes a sheet).""" - from . import cri + from ..formats import cri cfg.require_path("paths", "apk") cat = common.session(cfg).catalog() @@ -1444,14 +1447,14 @@ def cmd_get(args, cfg, common) -> None: print(f"nnnotes: {r['id']}: {r['status']}" + (f" ({r['detail']})" if r.get("detail") else ""), file=sys.stderr) sys.exit(1) else: - from .encoders import FLAC_LEVEL + from ..formats.encoders import FLAC_LEVEL try: written = _decode_one(cfg, common, sheet, cue, args.format, FLAC_LEVEL, dst) except LookupError as e: print(f"nnnotes: {r['id']}: {e}", file=sys.stderr) sys.exit(1) - from .jsonio import dumps + from ..core.jsonio import dumps (dst / "voice.json").write_text( dumps({"rows": rows, "files": [p.name for p in written]}, ensure_ascii=False, indent=1) + "\n", diff --git a/src/nnnotes/site/__init__.py b/src/nnnotes/site/__init__.py new file mode 100644 index 0000000..f343a9b --- /dev/null +++ b/src/nnnotes/site/__init__.py @@ -0,0 +1 @@ +"""The ournotes-player site: charts, models, stories, the shared site files and indexes.""" diff --git a/src/nnnotes/compress.py b/src/nnnotes/site/compress.py similarity index 100% rename from src/nnnotes/compress.py rename to src/nnnotes/site/compress.py diff --git a/src/nnnotes/sitefiles.py b/src/nnnotes/site/sitefiles.py similarity index 99% rename from src/nnnotes/sitefiles.py rename to src/nnnotes/site/sitefiles.py index 1bc8021..5622854 100644 --- a/src/nnnotes/sitefiles.py +++ b/src/nnnotes/site/sitefiles.py @@ -25,9 +25,10 @@ import time from pathlib import Path -from . import cache, compress, jsonio +from ..core import cache, jsonio +from ..source.config import Config, ConfigError +from . import compress from .compress import DEFAULT_ENCODING -from .config import Config, ConfigError SITE_FORMAT = 2 # charts.json "format" MANIFEST_FORMAT = 3 # chart manifest "format" (3: entries may be stored encoded) @@ -271,8 +272,8 @@ def entry_assets(e: dict) -> list[str]: def open_data(cfg: Config, region: str | None = None): """Catalog (fetching from the CDN of `region`), the master dir of `region` and PlayerData from the settings (as the command line opens them; `region` None: [catalog] region).""" - from .player import PlayerData - from .session import Session + from ..source.session import Session + from ..unity.player import PlayerData session = Session(cfg) return session.catalog(region=region), session.master_dir(region), PlayerData(session.apk(region)) @@ -317,7 +318,7 @@ def site_regions(cfg: Config, regions=None, all_regions: bool = False) -> list[s def region_masters(cfg: Config, regions: list[str]) -> dict[str, Path]: """{region: decoded master dir} (Session.master_dir). With more than one region the --master flag is refused and at most one region may use [paths] master (the others need their own `[servers.] master`).""" - from .session import Session + from ..source.session import Session if len(regions) > 1 and cfg.origin("paths", "master") == "flag": raise ConfigError( diff --git a/src/nnnotes/siteindex.py b/src/nnnotes/site/siteindex.py similarity index 98% rename from src/nnnotes/siteindex.py rename to src/nnnotes/site/siteindex.py index aeaffbe..df2d611 100644 --- a/src/nnnotes/siteindex.py +++ b/src/nnnotes/site/siteindex.py @@ -6,8 +6,9 @@ import json from pathlib import Path -from . import compress, languages -from .score import DIFFICULTIES +from ..content.score import DIFFICULTIES +from ..source import languages +from . import compress from .sitefiles import ( CHARTS_INDEX, MODELS_DIR, diff --git a/src/nnnotes/storysite.py b/src/nnnotes/site/storysite.py similarity index 99% rename from src/nnnotes/storysite.py rename to src/nnnotes/site/storysite.py index 0f89925..e166319 100644 --- a/src/nnnotes/storysite.py +++ b/src/nnnotes/site/storysite.py @@ -49,8 +49,11 @@ import uuid from pathlib import Path -from . import adv, advui, jsonio, languages, master, story, storyfonts, storyhost, webmodel -from .config import Config, ConfigError, usable_cpus, use +from ..content import adv, advui, story, storyfonts, storyhost +from ..core import jsonio +from ..source import languages, master +from ..source.config import Config, ConfigError, usable_cpus, use +from . import webmodel from .sitefiles import ( MODELS_DIR, SPLIT_KEY, @@ -731,7 +734,7 @@ def build( `encoding`: the stored encoding of the assets (sitefiles.site_store). The models the stories to build load are built first (module docstring; bundles from the CDN of the first region, names as webmodel.build gives them); the summary's `storyModels` holds that build's model lists.""" - from .tmpfont import require_extra + from ..unity.tmpfont import require_extra player_dir = check_player(player_dir) if audio_format not in WEB_AUDIO: diff --git a/src/nnnotes/web.py b/src/nnnotes/site/web.py similarity index 99% rename from src/nnnotes/web.py rename to src/nnnotes/site/web.py index 6362487..c580b1f 100644 --- a/src/nnnotes/web.py +++ b/src/nnnotes/site/web.py @@ -79,10 +79,14 @@ from pathlib import Path from types import SimpleNamespace -from . import cache, compress, cri, jsonio, languages, live, livenotes, liveoptions, livescene +from ..content import live, livenotes, liveoptions, livescene +from ..content.score import DIFFICULTIES, master_table +from ..core import cache, jsonio +from ..formats import cri +from ..source import languages +from ..source.config import Config, ConfigError, tool, usable_cpus, use +from . import compress from .compress import DEFAULT_ENCODING -from .config import Config, ConfigError, tool, usable_cpus, use -from .score import DIFFICULTIES, master_table from .sitefiles import ( MANIFEST_FORMAT, READ_SET_SCRIPT, @@ -388,7 +392,7 @@ def build_music_dirs( from `livenotes_dir` (livenotes.extract reads no music input). `scene`: (band part, its files) of the music's band (livescene.band_part), else the scene is exported whole. `band` / `leader_card` / `language` / `fonts` / `options`: as for live.build (livenotes_dir made with the same options). Returns {difficulty: (dir, summary)}.""" - from . import liveaudio, liveui, score + from ..content import liveaudio, liveui, score base = root / "base" base.mkdir(parents=True) diff --git a/src/nnnotes/webaudio.py b/src/nnnotes/site/webaudio.py similarity index 100% rename from src/nnnotes/webaudio.py rename to src/nnnotes/site/webaudio.py diff --git a/src/nnnotes/webmodel.py b/src/nnnotes/site/webmodel.py similarity index 98% rename from src/nnnotes/webmodel.py rename to src/nnnotes/site/webmodel.py index ca54a8d..f5910df 100644 --- a/src/nnnotes/webmodel.py +++ b/src/nnnotes/site/webmodel.py @@ -54,10 +54,12 @@ import traceback from pathlib import Path -from . import jsonio, languages, live2d, master -from . import shader as shader_mod -from .config import Config, usable_cpus, use -from .export import Exporter +from ..core import jsonio +from ..formats import shader as shader_mod +from ..source import languages, master +from ..source.config import Config, usable_cpus, use +from ..unity import live2d +from ..unity.export import Exporter from .sitefiles import ( MANIFEST_FORMAT, MODELS_DIR, @@ -181,7 +183,7 @@ def master_names(cfg: Config, region: str | None = None, log=None) -> dict[str, section, key = cfg.master(region or cfg.get("catalog", "region")) if cfg.path(section, key) is None: return None - from .session import Session + from ..source.session import Session code = cfg.get("catalog", "language") try: @@ -501,8 +503,8 @@ def outdated(site: Path, mid: str) -> str | None: def _open(cfg: Config, region: str | None = None): - from .player import PlayerData - from .session import Session + from ..source.session import Session + from ..unity.player import PlayerData session = Session(cfg) return session.catalog(region=region), PlayerData(session.apk(region)) @@ -569,7 +571,7 @@ def build( log = log or _log t0 = time.time() if models is None: - from .session import Session + from ..source.session import Session models = catalog_models(Session(cfg).catalog(bundles=False, region=region)) todo, skipped, rebuilt = [], [], [] diff --git a/src/nnnotes/source/__init__.py b/src/nnnotes/source/__init__.py new file mode 100644 index 0000000..39b4cf8 --- /dev/null +++ b/src/nnnotes/source/__init__.py @@ -0,0 +1 @@ +"""The data sources: settings, the APK, Addressables catalogs and bundles, the game API, master data.""" diff --git a/src/nnnotes/addressables.py b/src/nnnotes/source/addressables.py similarity index 100% rename from src/nnnotes/addressables.py rename to src/nnnotes/source/addressables.py diff --git a/src/nnnotes/apkset.py b/src/nnnotes/source/apkset.py similarity index 99% rename from src/nnnotes/apkset.py rename to src/nnnotes/source/apkset.py index c2ccc19..f5e5c8f 100644 --- a/src/nnnotes/apkset.py +++ b/src/nnnotes/source/apkset.py @@ -16,7 +16,7 @@ from functools import lru_cache from pathlib import Path -from .cache import file_id +from ..core.cache import file_id _EXPAND_LOCK = threading.Lock() MANIFEST_IN_APK = "AndroidManifest.xml" diff --git a/src/nnnotes/catalog.py b/src/nnnotes/source/catalog.py similarity index 99% rename from src/nnnotes/catalog.py rename to src/nnnotes/source/catalog.py index 2f2c60a..732e89b 100644 --- a/src/nnnotes/catalog.py +++ b/src/nnnotes/source/catalog.py @@ -16,9 +16,9 @@ from dataclasses import dataclass from pathlib import Path +from ..core.cache import write_atomic as _write_atomic from .addressables import REMOTE_PREFIX, BundleKey, decrypt, parse, parse_locations, relative_path, remote_path from .apkset import ApkSet -from .cache import write_atomic as _write_atomic from .config import ConfigError, apk_missing, check_catalog_version LOCAL_PREFIX = "{UnityEngine.AddressableAssets.Addressables.RuntimePath}" diff --git a/src/nnnotes/config.py b/src/nnnotes/source/config.py similarity index 100% rename from src/nnnotes/config.py rename to src/nnnotes/source/config.py diff --git a/src/nnnotes/configfile.py b/src/nnnotes/source/configfile.py similarity index 100% rename from src/nnnotes/configfile.py rename to src/nnnotes/source/configfile.py diff --git a/src/nnnotes/gameapi.py b/src/nnnotes/source/gameapi.py similarity index 100% rename from src/nnnotes/gameapi.py rename to src/nnnotes/source/gameapi.py diff --git a/src/nnnotes/jp.py b/src/nnnotes/source/jp.py similarity index 99% rename from src/nnnotes/jp.py rename to src/nnnotes/source/jp.py index 6153d49..d59a146 100644 --- a/src/nnnotes/jp.py +++ b/src/nnnotes/source/jp.py @@ -17,9 +17,9 @@ from pathlib import Path from urllib.parse import urlsplit +from ..core.cache import write_atomic from . import gameapi from .addressables import parse_header, relative_path -from .cache import write_atomic from .catalog import Catalog from .config import ConfigError @@ -278,7 +278,7 @@ def read_source(path, data): def write_source(path, data, source): - from .cache import write_atomic + from ..core.cache import write_atomic doc = {"catalog_sha256": hashlib.sha256(data).hexdigest(), "source": source.to_dict()} write_atomic(sidecar(path), (json.dumps(doc, sort_keys=True) + "\n").encode()) diff --git a/src/nnnotes/languages.py b/src/nnnotes/source/languages.py similarity index 100% rename from src/nnnotes/languages.py rename to src/nnnotes/source/languages.py diff --git a/src/nnnotes/master.py b/src/nnnotes/source/master.py similarity index 99% rename from src/nnnotes/master.py rename to src/nnnotes/source/master.py index a2959b7..4574a79 100644 --- a/src/nnnotes/master.py +++ b/src/nnnotes/source/master.py @@ -29,7 +29,7 @@ import numpy as np -from . import jsonio +from ..core import jsonio PREFIX = 64 # bytes before the ciphertext BLOCK = 32 # 256-bit block @@ -261,7 +261,7 @@ def download(cdn: str, version: str, out_dir: Path, workers: int = 16, *, get=No seen.add(name) else: # Preserve the original partial-download receipt even if a worker raises. - from .cache import write_atomic + from ..core.cache import write_atomic write_atomic(out_dir / "MasterManifest.json", manifest_raw) @@ -290,7 +290,7 @@ def one(f: dict): with ThreadPoolExecutor(max(1, workers)) as ex: results = list(ex.map(one, files)) if not strict or not any(isinstance(r, dict) for r in results): - from .cache import write_atomic + from ..core.cache import write_atomic write_atomic(out_dir / "MasterManifest.json", manifest_raw) return { diff --git a/src/nnnotes/nnnotes.example.toml b/src/nnnotes/source/nnnotes.example.toml similarity index 100% rename from src/nnnotes/nnnotes.example.toml rename to src/nnnotes/source/nnnotes.example.toml diff --git a/src/nnnotes/session.py b/src/nnnotes/source/session.py similarity index 100% rename from src/nnnotes/session.py rename to src/nnnotes/source/session.py diff --git a/src/nnnotes/unity/__init__.py b/src/nnnotes/unity/__init__.py new file mode 100644 index 0000000..681c98e --- /dev/null +++ b/src/nnnotes/unity/__init__.py @@ -0,0 +1 @@ +"""Unity objects: bundle reading, the typetree exporter, the player data and the TextMeshPro fonts.""" diff --git a/src/nnnotes/export.py b/src/nnnotes/unity/export.py similarity index 99% rename from src/nnnotes/export.py rename to src/nnnotes/unity/export.py index 6d84d78..0aa99e8 100644 --- a/src/nnnotes/export.py +++ b/src/nnnotes/unity/export.py @@ -34,9 +34,9 @@ from PIL import Image from UnityPy.helpers import MeshHelper -from . import cache -from .catalog import Catalog -from .unity import ( +from ..core import cache +from ..source.catalog import Catalog +from .reader import ( DEFAULT_RESOURCES, SceneGraph, check_texture, @@ -1797,7 +1797,7 @@ def key_sprite(self, key: str) -> dict: def dump_shaders(self, out_dir: Path, objects: dict | None = None) -> tuple[list, dict]: """Dump shaders (name -> Shader object; default every registered one) in name order with shader.py's layout into `out_dir`; returns (index records, write_index summary).""" - from . import shader as shader_mod + from ..formats import shader as shader_mod objects = self.shaders if objects is None else objects index: list = [] diff --git a/src/nnnotes/fontsettings.json b/src/nnnotes/unity/fontsettings.json similarity index 100% rename from src/nnnotes/fontsettings.json rename to src/nnnotes/unity/fontsettings.json diff --git a/src/nnnotes/fontsettings.py b/src/nnnotes/unity/fontsettings.py similarity index 100% rename from src/nnnotes/fontsettings.py rename to src/nnnotes/unity/fontsettings.py diff --git a/src/nnnotes/live2d.py b/src/nnnotes/unity/live2d.py similarity index 98% rename from src/nnnotes/live2d.py rename to src/nnnotes/unity/live2d.py index 5ab9d51..0529e04 100644 --- a/src/nnnotes/live2d.py +++ b/src/nnnotes/unity/live2d.py @@ -38,12 +38,12 @@ from dataclasses import dataclass from pathlib import Path -from . import motion as motion_mod -from .catalog import Catalog -from .config import apk_missing +from ..core.jsonio import write_json +from ..formats import motion as motion_mod +from ..source.catalog import Catalog +from ..source.config import apk_missing from .export import Exporter -from .jsonio import write_json -from .unity import SceneGraph, script_class +from .reader import SceneGraph, script_class # Live2D.Cubism.Core.CubismParameterBlendMode -> exp3.json "Blend" EXP_BLEND = {0: "Overwrite", 1: "Add", 2: "Multiply"} diff --git a/src/nnnotes/player.py b/src/nnnotes/unity/player.py similarity index 98% rename from src/nnnotes/player.py rename to src/nnnotes/unity/player.py index 94c0408..b9de4b5 100644 --- a/src/nnnotes/player.py +++ b/src/nnnotes/unity/player.py @@ -25,9 +25,9 @@ import UnityPy from UnityPy.helpers.TypeTreeNode import TypeTreeNode -from .apkset import MANIFEST_IN_APK, ApkSet, manifest_version_name -from .config import ConfigError -from .unity import DEFAULT_RESOURCES, deref, external_path, is_pptr +from ..source.apkset import MANIFEST_IN_APK, ApkSet, manifest_version_name +from ..source.config import ConfigError +from .reader import DEFAULT_RESOURCES, deref, external_path, is_pptr DATA_IN_APK = "assets/bin/Data/data.unity3d" DEFAULT_RESOURCES_IN_APK = "assets/bin/Data/Resources/unity default resources" diff --git a/src/nnnotes/unity.py b/src/nnnotes/unity/reader.py similarity index 99% rename from src/nnnotes/unity.py rename to src/nnnotes/unity/reader.py index 0e22f79..5887a57 100644 --- a/src/nnnotes/unity.py +++ b/src/nnnotes/unity/reader.py @@ -17,7 +17,7 @@ from UnityPy.helpers import MeshHelper from UnityPy.helpers.TypeTreeNode import TypeTreeNode -from . import cache +from ..core import cache # ---- typetree blobs --------------------------------------------------------- # A serialized file lists the typetree of each of its classes as a blob, and UnityPy parses every blob in Python diff --git a/src/nnnotes/textstyle.py b/src/nnnotes/unity/textstyle.py similarity index 99% rename from src/nnnotes/textstyle.py rename to src/nnnotes/unity/textstyle.py index fd75519..a49893e 100644 --- a/src/nnnotes/textstyle.py +++ b/src/nnnotes/unity/textstyle.py @@ -21,8 +21,8 @@ import struct +from ..source.languages import LANGUAGES from .export import Exporter -from .languages import LANGUAGES from .player import PlayerData # Fwk.Localization.LanguageMode / language field names of the text tables. diff --git a/src/nnnotes/tmpfont.py b/src/nnnotes/unity/tmpfont.py similarity index 99% rename from src/nnnotes/tmpfont.py rename to src/nnnotes/unity/tmpfont.py index 7a5a962..f05d53c 100644 --- a/src/nnnotes/tmpfont.py +++ b/src/nnnotes/unity/tmpfont.py @@ -21,7 +21,7 @@ import numpy as np -from .config import ConfigError +from ..source.config import ConfigError from .export import Exporter, TexelPacker, _key, grow_box from .fontsettings import ( END_OF_TEXT, diff --git a/src/nnnotes/typetrees/6000.3.12f1.json b/src/nnnotes/unity/typetrees/6000.3.12f1.json similarity index 100% rename from src/nnnotes/typetrees/6000.3.12f1.json rename to src/nnnotes/unity/typetrees/6000.3.12f1.json diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cli/__init__.py b/tests/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_cli.py b/tests/cli/test_cli.py similarity index 97% rename from tests/test_cli.py rename to tests/cli/test_cli.py index 7870801..72b28a8 100644 --- a/tests/test_cli.py +++ b/tests/cli/test_cli.py @@ -7,9 +7,9 @@ from pathlib import Path import pytest -import synth from nnnotes import cli +from tests.helpers import synth KEY_HEX = synth.BUNDLE_KEY.hex() SEED_HEX = synth.BUNDLE_SEED.hex() @@ -85,7 +85,7 @@ def find_spec(self, name, path=None, target=None): print(os.environ.get("OPENBLAS_NUM_THREADS")) sys.meta_path.insert(0, Probe()) import nnnotes.cli -import nnnotes.unity +import nnnotes.unity.reader """ @@ -289,7 +289,7 @@ def test_apk_is_a_required_setting(argv, catalog_file, tmp_path, cdn, capsys, mo def test_crikey_write_creates_the_directory(tmp_path, capsys, monkeypatch): - from nnnotes import crikey + from nnnotes.formats import crikey apk = tmp_path / "base.apk" apk.write_bytes(b"not read") @@ -375,7 +375,7 @@ def test_unknown_key_or_id_is_a_usage_error(data, argv, message, capsys): def test_known_id_reaches_the_extractor(data, tmp_path, capsys, monkeypatch): - from nnnotes import adv + from nnnotes.content import adv seen = [] monkeypatch.setattr(adv, "extract", lambda cat, md, adv_id: seen.append(adv_id) or "x") @@ -399,7 +399,7 @@ def test_story_needs_the_language(data, capsys): def test_the_command_line_starts_without_the_exporters(): probe = ( "import sys, nnnotes.cli as c; c.build_parser(); " - "print([m for m in ('UnityPy', 'nnnotes.web') if m in sys.modules])" + "print([m for m in ('UnityPy', 'nnnotes.site.web') if m in sys.modules])" ) r = subprocess.run([sys.executable, "-c", probe], capture_output=True, text=True, check=True) assert r.stdout.split() == ["[]"] @@ -424,8 +424,10 @@ def test_flac_level_is_checked(data, argv, message, capsys): def test_flac_level_reaches_the_decode(data, tmp_path, capsys, monkeypatch): from types import SimpleNamespace - from nnnotes import cri, live, player, story - from nnnotes.session import Session + from nnnotes.content import live, story + from nnnotes.formats import cri + from nnnotes.source.session import Session + from nnnotes.unity import player seen = [] monkeypatch.setattr(Session, "catalog", lambda self: SimpleNamespace(has=lambda key: True)) @@ -443,7 +445,8 @@ def test_flac_level_reaches_the_decode(data, tmp_path, capsys, monkeypatch): def test_live_passes_the_flac_level_to_its_decodes(tmp_path, monkeypatch): - from nnnotes import cri, live, liveaudio + from nnnotes.content import live, liveaudio + from nnnotes.formats import cri class Stop(Exception): pass diff --git a/tests/test_cli_assets.py b/tests/cli/test_cli_assets.py similarity index 97% rename from tests/test_cli_assets.py rename to tests/cli/test_cli_assets.py index d233bb2..96a05d0 100644 --- a/tests/test_cli_assets.py +++ b/tests/cli/test_cli_assets.py @@ -12,14 +12,16 @@ from pathlib import Path import pytest -import synth -from nnnotes import cli, cli_assets, contract -from nnnotes.census import CensusStage -from nnnotes.config import ConfigError -from nnnotes.contract import Cost -from nnnotes.stages import Output, Stage -from nnnotes.store import Store +from nnnotes import cli +from nnnotes.cli import cli_assets +from nnnotes.pipeline import contract +from nnnotes.pipeline.census import CensusStage +from nnnotes.pipeline.contract import Cost +from nnnotes.pipeline.stages import Output, Stage +from nnnotes.pipeline.store import Store +from nnnotes.source.config import ConfigError +from tests.helpers import synth HASHES = {n: hashlib.md5(n.encode()).hexdigest() for n in ("card_1", "card_2", "shared", "bg_x")} NAMES = { @@ -27,7 +29,7 @@ for n, h in HASHES.items() } CABS = {n: f"CAB-{hashlib.md5(('cab' + n).encode()).hexdigest()}" for n in HASHES} -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[2] # ---------------------------------------------------------------- synthetic data @@ -247,10 +249,12 @@ def run(self, task, store): REAL = {s.name: s for s in cli_assets.STAGE_SOURCES} FAKE_SOURCES = ( REAL["catalog.index"], - cli_assets.StageSource("unity.census", "test_cli_assets:FakeCensus"), + cli_assets.StageSource("unity.census", "tests.cli.test_cli_assets:FakeCensus"), REAL["link.scripts"], REAL["link.addresses"], - cli_assets.StageSource("unity.export", "test_cli_assets:FakeExport", output=True, root="_bundles/{subject}"), + cli_assets.StageSource( + "unity.export", "tests.cli.test_cli_assets:FakeExport", output=True, root="_bundles/{subject}" + ), REAL["link.artifacts"], ) @@ -299,7 +303,7 @@ def test_selection_items_and_groups(): def test_select_adds_the_dependencies_to_the_census(): - from nnnotes.catalogdb import index + from nnnotes.pipeline.catalogdb import index idx = index(catalog_bytes()) stable = {n: contract.stable_bundle_name(NAMES[n]) for n in NAMES} @@ -460,7 +464,7 @@ def test_the_flac_level_is_a_parameter_of_the_stages_that_encode_flac(tmp_path, def test_the_placement_is_probed_once_and_never_changes_the_files(tmp_path, capsys, fake, monkeypatch): import errno - from nnnotes import layout + from nnnotes.pipeline import layout def clone_by_copy(src, dst): with open(dst, "xb") as f: @@ -553,10 +557,10 @@ def test_plan_and_run_stage_do_not_import_unitypy(tmp_path): d = setup_data(tmp_path) script = f""" import sys -from nnnotes import cli_assets +from nnnotes.cli import cli_assets args = {[str(a) for a in flags(d)]!r} from pathlib import Path -from nnnotes import contract +from nnnotes.pipeline import contract def main(argv): try: cli_assets.main(argv) @@ -709,7 +713,7 @@ def test_derived_documents_read_the_paths_of_the_objects_they_name(tmp_path, cap monkeypatch.setattr( cli_assets, "STAGE_SOURCES", - FAKE_SOURCES + (cli_assets.StageSource("fake.paths", "test_cli_assets:ObjectPaths", output=True),), + FAKE_SOURCES + (cli_assets.StageSource("fake.paths", "tests.cli.test_cli_assets:ObjectPaths", output=True),), ) d = setup_data(tmp_path) code, _, err = export(capsys, tmp_path, d, "s", "o") @@ -726,9 +730,8 @@ def test_derived_documents_read_the_paths_of_the_objects_they_name(tmp_path, cap # ---------------------------------------------------------------- views and the documents of a run def test_views_are_placed_with_paths_and_every_document_validates(tmp_path, capsys, monkeypatch): - from test_contract import validators - - from nnnotes import views + from nnnotes.pipeline import views + from tests.pipeline.test_contract import validators monkeypatch.setattr(cli_assets, "STAGE_SOURCES", FAKE_SOURCES + (REAL["view."],)) rules = views.load_rules() @@ -939,7 +942,7 @@ def subjects(self, env): return ["s"] def uses(self, subject, env): - from nnnotes.config import active + from nnnotes.source.config import active cfg = active() return {"tool": str(cfg.path("paths", "ffmpeg")) if cfg is not None else "unset"} @@ -949,12 +952,12 @@ def run(self, task, store): def test_named_contents_are_placed_under_each_name(tmp_path, capsys, monkeypatch): - from nnnotes import layout + from nnnotes.pipeline import layout monkeypatch.setattr( cli_assets, "STAGE_SOURCES", - FAKE_SOURCES + (cli_assets.StageSource("toy.named", "test_cli_assets:Named", output=True),), + FAKE_SOURCES + (cli_assets.StageSource("toy.named", "tests.cli.test_cli_assets:Named", output=True),), ) d = setup_data(tmp_path, censuses(broken=False)) code, out, err = export(capsys, tmp_path, d, "s", "o", "--layout", "original,cas") @@ -978,7 +981,9 @@ def test_named_contents_are_placed_under_each_name(tmp_path, capsys, monkeypatch def test_plan_and_export_describe_with_the_same_settings(tmp_path, capsys, monkeypatch): monkeypatch.setattr( - cli_assets, "STAGE_SOURCES", FAKE_SOURCES + (cli_assets.StageSource("toy.tool", "test_cli_assets:Tooled"),) + cli_assets, + "STAGE_SOURCES", + FAKE_SOURCES + (cli_assets.StageSource("toy.tool", "tests.cli.test_cli_assets:Tooled"),), ) d = setup_data(tmp_path, censuses(broken=False)) tool = tmp_path / "ffmpeg-bin" @@ -1045,7 +1050,8 @@ def test_names_see_the_tasks_that_ran(tmp_path, capsys, monkeypatch): monkeypatch.setattr( cli_assets, "STAGE_SOURCES", - FAKE_SOURCES + (cli_assets.StageSource("toy.byexport", "test_cli_assets:NamedByExport", output=True),), + FAKE_SOURCES + + (cli_assets.StageSource("toy.byexport", "tests.cli.test_cli_assets:NamedByExport", output=True),), ) d = setup_data(tmp_path, censuses(broken=False)) code, out, err = export(capsys, tmp_path, d, "s", "o") @@ -1074,7 +1080,7 @@ def test_calibration_keeps_the_cpu_factor_without_child_measurements(): def test_census_documents_with_binding_scripts_validate(): - from test_contract import validators + from tests.pipeline.test_contract import validators doc = cdoc(CABS["card_1"], [obj(1, "AssetBundle", "ab"), obj(9, "AnimationClip", "clip")]) doc["files"][0]["objects"][1]["bindingScripts"] = [ @@ -1115,7 +1121,9 @@ def raw_apk(tmp_path, monkeypatch): import zipfile monkeypatch.setattr( - cli_assets, "STAGE_SOURCES", FAKE_SOURCES + (cli_assets.StageSource("cri.audio", "test_cli_assets:RawCopy"),) + cli_assets, + "STAGE_SOURCES", + FAKE_SOURCES + (cli_assets.StageSource("cri.audio", "tests.cli.test_cli_assets:RawCopy"),), ) monkeypatch.setattr(cli_assets, "_apk_version", lambda apk: None) name = "cri_assets_embcri/sound/initialse_" + "ab" * 16 @@ -1207,7 +1215,7 @@ def inputs(self, kind, entries, fetch): def test_an_interrupt_starts_no_more_raw_fetches(tmp_path, capsys, monkeypatch): import threading - from nnnotes import orchestrate + from nnnotes.pipeline import orchestrate apk, name = raw_apk(tmp_path, monkeypatch) ended = threading.Event() @@ -1238,7 +1246,8 @@ def test_worker_counts_follow_the_cpus_this_process_may_use(monkeypatch): import os from types import SimpleNamespace - from nnnotes import config, cri + from nnnotes.formats import cri + from nnnotes.source import config monkeypatch.setattr(os, "cpu_count", lambda: 64) monkeypatch.setattr(os, "process_cpu_count", lambda: 6, raising=False) diff --git a/tests/conftest.py b/tests/conftest.py index 432799a..1343c8d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,8 @@ import pytest -from nnnotes import cache, config +from nnnotes.core import cache +from nnnotes.source import config CACHE_DEFAULTS = cache.settings() diff --git a/tests/content/__init__.py b/tests/content/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_live_options.py b/tests/content/test_live_options.py similarity index 98% rename from tests/test_live_options.py rename to tests/content/test_live_options.py index f2e8ab2..019a42d 100644 --- a/tests/test_live_options.py +++ b/tests/content/test_live_options.py @@ -4,8 +4,8 @@ import pytest -from nnnotes import live, liveaudio, livenotes, liveoptions, livescene, score -from nnnotes.liveoptions import LiveOptions, OptionSpecError, parse_specs, resolve +from nnnotes.content import live, liveaudio, livenotes, liveoptions, livescene, score +from nnnotes.content.liveoptions import LiveOptions, OptionSpecError, parse_specs, resolve SWITCHES = { 4, diff --git a/tests/test_room.py b/tests/content/test_room.py similarity index 99% rename from tests/test_room.py rename to tests/content/test_room.py index 16840e3..b01a938 100644 --- a/tests/test_room.py +++ b/tests/content/test_room.py @@ -9,12 +9,12 @@ import numpy as np import pytest -from fakeunity import F32, POSITION, UV0, mesh_tt, submesh, unitypy_mesh from PIL import Image from UnityPy.classes import PPtr as UnityPPtr -from nnnotes import room -from nnnotes.unity import DEFAULT_RESOURCES +from nnnotes.content import room +from nnnotes.unity.reader import DEFAULT_RESOURCES +from tests.helpers.fakeunity import F32, POSITION, UV0, mesh_tt, submesh, unitypy_mesh FIXED = "" diff --git a/tests/test_spot.py b/tests/content/test_spot.py similarity index 99% rename from tests/test_spot.py rename to tests/content/test_spot.py index 98e1aef..3048981 100644 --- a/tests/test_spot.py +++ b/tests/content/test_spot.py @@ -6,7 +6,7 @@ import pytest from PIL import Image -from nnnotes import spot +from nnnotes.content import spot class Ref: diff --git a/tests/test_story_closure.py b/tests/content/test_story_closure.py similarity index 98% rename from tests/test_story_closure.py rename to tests/content/test_story_closure.py index 4bc2b13..71d114b 100644 --- a/tests/test_story_closure.py +++ b/tests/content/test_story_closure.py @@ -2,8 +2,8 @@ import pytest -from nnnotes import adv -from nnnotes.config import ConfigError +from nnnotes.content import adv +from nnnotes.source.config import ConfigError CMD = {v: k for k, v in adv.COMMAND.items()} diff --git a/tests/test_story_fonts.py b/tests/content/test_story_fonts.py similarity index 99% rename from tests/test_story_fonts.py rename to tests/content/test_story_fonts.py index 6e39f34..f4608c4 100644 --- a/tests/test_story_fonts.py +++ b/tests/content/test_story_fonts.py @@ -12,7 +12,9 @@ pytest.importorskip("scipy") pytest.importorskip("fontTools") -from nnnotes import advui, cache, fontsettings, storyfonts, tmpfont # noqa: E402 +from nnnotes.content import advui, storyfonts +from nnnotes.core import cache +from nnnotes.unity import fontsettings, tmpfont # noqa: E402 UPEM = 1000 diff --git a/tests/test_story_host.py b/tests/content/test_story_host.py similarity index 99% rename from tests/test_story_host.py rename to tests/content/test_story_host.py index 6bf83d2..f1b01d2 100644 --- a/tests/test_story_host.py +++ b/tests/content/test_story_host.py @@ -8,10 +8,10 @@ import numpy as np import pytest -from test_room import ROOM_KEY, File, RoomCatalog, load_room, two_file_room -from nnnotes import advui, room, storyhost -from nnnotes.unity import SceneGraph +from nnnotes.content import advui, room, storyhost +from nnnotes.unity.reader import SceneGraph +from tests.content.test_room import ROOM_KEY, File, RoomCatalog, load_room, two_file_room V0 = {"x": 0.0, "y": 0.0, "z": 0.0} V1 = {"x": 1.0, "y": 1.0, "z": 1.0} @@ -792,7 +792,7 @@ def test_clip_view_keys_child_bindings_discrete_dense_and_lost_paths(): def test_ui_only_characters(): """The characters of a UI alone: storyfonts.shown_characters over the empty episode storyhost passes.""" - from nnnotes import storyfonts + from nnnotes.content import storyfonts ui_doc = {"nodes": [], "masterIdTexts": {"a": {"text": "Ab"}}} assert storyfonts.shown_characters(storyhost.NO_EPISODE, ui_doc, None, "japanese") == [ord("A"), ord("b")] diff --git a/tests/test_story_kinds.py b/tests/content/test_story_kinds.py similarity index 98% rename from tests/test_story_kinds.py rename to tests/content/test_story_kinds.py index 7516d56..0e8023f 100644 --- a/tests/test_story_kinds.py +++ b/tests/content/test_story_kinds.py @@ -2,7 +2,8 @@ import pytest -from nnnotes import adv, advmedia, cli, story +from nnnotes import cli +from nnnotes.content import adv, advmedia, story def res(kind, address, present=True): diff --git a/tests/test_story_ui.py b/tests/content/test_story_ui.py similarity index 99% rename from tests/test_story_ui.py rename to tests/content/test_story_ui.py index ae969ae..84c3ddb 100644 --- a/tests/test_story_ui.py +++ b/tests/content/test_story_ui.py @@ -6,7 +6,8 @@ import pytest -from nnnotes import advui, fontsettings +from nnnotes.content import advui +from nnnotes.unity import fontsettings WHITE = {"spriteRef": "cab:1", "name": "WhiteRect"} diff --git a/tests/test_ui.py b/tests/content/test_ui.py similarity index 97% rename from tests/test_ui.py rename to tests/content/test_ui.py index a35496e..66fd105 100644 --- a/tests/test_ui.py +++ b/tests/content/test_ui.py @@ -7,9 +7,11 @@ import pytest from PIL import Image -from nnnotes import cli, player, ui -from nnnotes.config import Config -from nnnotes.session import Session +from nnnotes import cli +from nnnotes.content import ui +from nnnotes.source.config import Config +from nnnotes.source.session import Session +from nnnotes.unity import player KEY = "EmbUI/Example/Box" @@ -159,7 +161,7 @@ def test_failure_is_an_explicit_index_entry(tmp_path, exporter): def test_cli_selects_the_jp_split_apk_and_keeps_public_provenance(tmp_path, exporter, monkeypatch, capsys): - from nnnotes import catalog + from nnnotes.source import catalog apk = tmp_path / "jp" apk.mkdir() diff --git a/tests/core/__init__.py b/tests/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_cache.py b/tests/core/test_cache.py similarity index 99% rename from tests/test_cache.py rename to tests/core/test_cache.py index 5a2ea04..314ebc2 100644 --- a/tests/test_cache.py +++ b/tests/core/test_cache.py @@ -2,7 +2,7 @@ import pytest -from nnnotes import cache +from nnnotes.core import cache @pytest.fixture(autouse=True) diff --git a/tests/test_jsonio.py b/tests/core/test_jsonio.py similarity index 96% rename from tests/test_jsonio.py rename to tests/core/test_jsonio.py index 536903a..d8b051d 100644 --- a/tests/test_jsonio.py +++ b/tests/core/test_jsonio.py @@ -3,7 +3,7 @@ import pytest -from nnnotes.jsonio import dumps, write_json +from nnnotes.core.jsonio import dumps, write_json def test_infinity_literals(): diff --git a/tests/formats/__init__.py b/tests/formats/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_crilips.py b/tests/formats/test_crilips.py similarity index 96% rename from tests/test_crilips.py rename to tests/formats/test_crilips.py index faf56c6..b9771f9 100644 --- a/tests/test_crilips.py +++ b/tests/formats/test_crilips.py @@ -1,4 +1,4 @@ -"""Tests for nnnotes.crilips using only synthetic ELF fixtures and a synthetic version table. +"""Tests for nnnotes.formats.crilips using only synthetic ELF fixtures and a synthetic version table. No game data is used: we build a tiny ELF64 whose .rodata holds random blocks, register a synthetic Version table keyed by their sha256, and check that extraction locates, orders and verifies them. @@ -11,8 +11,8 @@ import pytest -from nnnotes import crilips -from nnnotes.crilips import Block, Const, CriLipsError, Version +from nnnotes.formats import crilips +from nnnotes.formats.crilips import Block, Const, CriLipsError, Version def _sha16(b: bytes) -> str: @@ -193,7 +193,7 @@ def _prefab(*classes): def test_has_motion_sync_needs_both_components_on_the_root(): - from nnnotes import webmodel + from nnnotes.site import webmodel assert webmodel.has_motion_sync( _prefab("CubismModel", "CubismMotionSyncController", "Live2DMotionSyncCriAudioInput") @@ -207,7 +207,7 @@ def _talk(names, voices, **kw): def test_needs_cri_lips(): - from nnnotes import story + from nnnotes.content import story def ep(*rows): return {"commands": list(rows)} diff --git a/tests/test_usm.py b/tests/formats/test_usm.py similarity index 99% rename from tests/test_usm.py rename to tests/formats/test_usm.py index 1895d1d..fb9a22c 100644 --- a/tests/test_usm.py +++ b/tests/formats/test_usm.py @@ -10,8 +10,9 @@ import pytest -from nnnotes import advvideo, usm -from nnnotes.usm import Stream +from nnnotes.content import advvideo +from nnnotes.formats import usm +from nnnotes.formats.usm import Stream KEY = 0x0123456789ABCDEF # made up M1, M2, AM = usm.masks(KEY) @@ -692,7 +693,7 @@ def test_story_export_keeps_its_command(tmp_path, monkeypatch): build([(b"@SFV", 0, None, VP9, "video"), (b"@SFA", 0, None, [adx(samples=100) + rand(4 * 36, 1)], "audio")]) ) monkeypatch.setattr(advvideo, "usm_asset", lambda cat, key: (info, movie)) - monkeypatch.setattr("nnnotes.config.tool", lambda name, exe: name) + monkeypatch.setattr("nnnotes.source.config.tool", lambda name, exe: name) calls = [] def fake_run(args, **kw): diff --git a/tests/golden/cri.audio.json b/tests/golden/cri.audio.json index c685196..f24e876 100644 --- a/tests/golden/cri.audio.json +++ b/tests/golden/cri.audio.json @@ -14,7 +14,7 @@ } }, "libraries": {}, - "provider": "test_cristages:audio_golden", + "provider": "tests.pipeline.test_cristages:audio_golden", "stage": "cri.audio", "version": 1 } diff --git a/tests/golden/cri.movie.json b/tests/golden/cri.movie.json index 7edca82..bf26c25 100644 --- a/tests/golden/cri.movie.json +++ b/tests/golden/cri.movie.json @@ -33,7 +33,7 @@ "libraries": { "numpy": "2.4.6" }, - "provider": "test_cristages:movie_golden", + "provider": "tests.pipeline.test_cristages:movie_golden", "stage": "cri.movie", "version": 2 } diff --git a/tests/golden/sprite.crop.json b/tests/golden/sprite.crop.json index 6422b01..6cc0b0b 100644 --- a/tests/golden/sprite.crop.json +++ b/tests/golden/sprite.crop.json @@ -16,7 +16,7 @@ "numpy": "2.4.6", "texture2ddecoder": "1.0.6" }, - "provider": "test_objexport:crop_golden", + "provider": "tests.pipeline.test_objexport:crop_golden", "stage": "sprite.crop", "version": 1 } diff --git a/tests/golden/unity.export.json b/tests/golden/unity.export.json index d7ff5b7..b1bb240 100644 --- a/tests/golden/unity.export.json +++ b/tests/golden/unity.export.json @@ -97,7 +97,7 @@ "numpy": "2.4.6", "texture2ddecoder": "1.0.6" }, - "provider": "test_objexport:export_golden", + "provider": "tests.pipeline.test_objexport:export_golden", "stage": "unity.export", "version": 1 } diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/fakeunity.py b/tests/helpers/fakeunity.py similarity index 100% rename from tests/fakeunity.py rename to tests/helpers/fakeunity.py diff --git a/tests/synth.py b/tests/helpers/synth.py similarity index 100% rename from tests/synth.py rename to tests/helpers/synth.py diff --git a/tests/musicdata/__init__.py b/tests/musicdata/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_deckdata.py b/tests/musicdata/test_deckdata.py similarity index 98% rename from tests/test_deckdata.py rename to tests/musicdata/test_deckdata.py index 53020a2..807262a 100644 --- a/tests/test_deckdata.py +++ b/tests/musicdata/test_deckdata.py @@ -11,11 +11,15 @@ import numpy as np import pytest -import synth -from nnnotes import apkset, cli, deckdata, jsonio, master, score -from nnnotes.catalogdb import CatalogDB -from nnnotes.master import MasterKey +from nnnotes import cli +from nnnotes.content import score +from nnnotes.core import jsonio +from nnnotes.musicdata import deckdata +from nnnotes.pipeline.catalogdb import CatalogDB +from nnnotes.source import apkset, master +from nnnotes.source.master import MasterKey +from tests.helpers import synth KEY = MasterKey(synth.MASTER_KEY, synth.MASTER_IV) @@ -619,7 +623,7 @@ def run(argv, capsys): def test_the_tables_are_documented(): from pathlib import Path - text = (Path(__file__).resolve().parents[1] / "docs" / "music-data.md").read_text(encoding="utf-8") + text = (Path(__file__).resolve().parents[2] / "docs" / "music-data.md").read_text(encoding="utf-8") documented = [] for line in text.splitlines(): if line.startswith("| `Master"): diff --git a/tests/test_musicdata.py b/tests/musicdata/test_musicdata.py similarity index 99% rename from tests/test_musicdata.py rename to tests/musicdata/test_musicdata.py index 35247f2..8fe8bc3 100644 --- a/tests/test_musicdata.py +++ b/tests/musicdata/test_musicdata.py @@ -13,16 +13,18 @@ from pathlib import Path import pytest -import synth -from test_deckdata import CHART, SMALL, FakeCatalog, rows_of, run -from test_voices import simple_acb -from nnnotes import cli, deckdata, musicdata, schemas, score -from nnnotes.master import MasterKey -from nnnotes.session import Session +from nnnotes import cli, musicdata, schemas +from nnnotes.content import score +from nnnotes.musicdata import deckdata +from nnnotes.source.master import MasterKey +from nnnotes.source.session import Session +from tests.helpers import synth +from tests.musicdata.test_deckdata import CHART, SMALL, FakeCatalog, rows_of, run +from tests.pipeline.test_voices import simple_acb KEY = MasterKey(synth.MASTER_KEY, synth.MASTER_IV) -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[2] def text(tid, stem): @@ -1283,7 +1285,7 @@ def test_command(tmp_path, capsys, monkeypatch): def test_command_decoded_master(tmp_path, capsys, monkeypatch): - from test_deckdata import decoded_dir + from tests.musicdata.test_deckdata import decoded_dir d = master_dir(tmp_path) dec = decoded_dir(tmp_path, d) diff --git a/tests/test_replaydata.py b/tests/musicdata/test_replaydata.py similarity index 97% rename from tests/test_replaydata.py rename to tests/musicdata/test_replaydata.py index 9435b8e..35fe605 100644 --- a/tests/test_replaydata.py +++ b/tests/musicdata/test_replaydata.py @@ -5,9 +5,10 @@ import tarfile import pytest -from test_musicdata import CHARTS, KEY, PROV, FakeDeck, bgm, export -from nnnotes import deckdata, musicdata, replaydata +from nnnotes import musicdata +from nnnotes.musicdata import deckdata, replaydata +from tests.musicdata.test_musicdata import CHARTS, KEY, PROV, FakeDeck, bgm, export def test_export_replay_keeps_main_file_small_and_preserves_runtime_input(tmp_path): @@ -54,9 +55,8 @@ def test_replay_requires_actual_audio_length_and_contains_no_guess(tmp_path): def test_replay_label_binding_rejects_other_snapshots(tmp_path, change): import copy - from test_musicdata import TABLE_ROWS - - from nnnotes import deckdata + from nnnotes.musicdata import deckdata + from tests.musicdata.test_musicdata import TABLE_ROWS hashes = {name: hashlib.sha256(name.encode()).hexdigest() for name in replaydata.LABEL_TABLES} provenance = { @@ -85,7 +85,7 @@ def test_replay_label_binding_rejects_other_snapshots(tmp_path, change): def test_raw_label_rows_preserve_names_levels_and_bind_artwork(tmp_path): import copy - from test_musicdata import TABLE_ROWS + from tests.musicdata.test_musicdata import TABLE_ROWS rows = {name: copy.deepcopy(TABLE_ROWS.get(name, [])) for name in replaydata.LABEL_TABLES} rows["MasterSupportSkill"] = [{"_id": 1, "_nameTextID": "first", "_descriptionTextFormatID": "duration"}] diff --git a/tests/test_statscache.py b/tests/musicdata/test_statscache.py similarity index 97% rename from tests/test_statscache.py rename to tests/musicdata/test_statscache.py index c518921..903e2cb 100644 --- a/tests/test_statscache.py +++ b/tests/musicdata/test_statscache.py @@ -10,9 +10,10 @@ from pathlib import Path import pytest -from test_musicdata import CHARTS, KEY, PROV, FakeDeck, all_rows, bgm, master_dir, run -from nnnotes import deckdata, musicdata +from nnnotes import musicdata +from nnnotes.musicdata import deckdata +from tests.musicdata.test_musicdata import CHARTS, KEY, PROV, FakeDeck, all_rows, bgm, master_dir, run @pytest.fixture diff --git a/tests/pipeline/__init__.py b/tests/pipeline/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_atom_eval.py b/tests/pipeline/test_atom_eval.py similarity index 91% rename from tests/test_atom_eval.py rename to tests/pipeline/test_atom_eval.py index 2e368d1..7537316 100644 --- a/tests/test_atom_eval.py +++ b/tests/pipeline/test_atom_eval.py @@ -12,15 +12,15 @@ import pytest from PIL import Image -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / "scripts")) import atom_eval # noqa: E402 -from fakeunity import FakeEnvironment, FakeTexture2D, rgba_bytes # noqa: E402 -from nnnotes.atoms import Impl, resolve # noqa: E402 -from nnnotes.atoms.mesh import Channel, MeshArrays, Primitive # noqa: E402 -from nnnotes.atoms.reader import UnityBundle # noqa: E402 +from nnnotes.pipeline.atoms import Impl, resolve # noqa: E402 +from nnnotes.pipeline.atoms.mesh import Channel, MeshArrays, Primitive # noqa: E402 +from nnnotes.pipeline.atoms.reader import UnityBundle # noqa: E402 +from tests.helpers.fakeunity import FakeEnvironment, FakeTexture2D, rgba_bytes # noqa: E402 RGBA32, RGB24 = 4, 3 @@ -84,7 +84,7 @@ def _m1(d: Path, atoms, checks) -> dict: "atoms": atoms, "checks": checks, "threads": 4, - "reference_reader": "test_atom_eval:fake_reader", + "reference_reader": "tests.pipeline.test_atom_eval:fake_reader", } return atom_eval.run_m1(cfg, atom_eval.bundle_list(d, None), workers=1) @@ -95,9 +95,9 @@ def test_m1_texture_decoders_per_format(tmp_path): out = _m1( d, [ - ("reader", "test_atom_eval:fake_reader"), - ("texture.decode", "test_atom_eval:same_decode"), - ("texture.decode", "test_atom_eval:flipped_decode"), + ("reader", "tests.pipeline.test_atom_eval:fake_reader"), + ("texture.decode", "tests.pipeline.test_atom_eval:same_decode"), + ("texture.decode", "tests.pipeline.test_atom_eval:flipped_decode"), ], ("texture",), ) @@ -117,7 +117,7 @@ def test_m1_png_round_trip_and_bytes(tmp_path): [ ("png.encode", "atom_eval_pillow:optimize"), ("png.encode", "atom_eval_pillow:level6"), - ("png.encode", "test_atom_eval:rgba_png"), + ("png.encode", "tests.pipeline.test_atom_eval:rgba_png"), ], ("png",), ) @@ -134,7 +134,9 @@ def test_m1_png_round_trip_and_bytes(tmp_path): def test_m1_determinism_check(tmp_path): d = _bundles(tmp_path) out = _m1( - d, [("texture.decode", "test_atom_eval:same_decode"), ("png.encode", "atom_eval_pillow:level1")], ("det",) + d, + [("texture.decode", "tests.pipeline.test_atom_eval:same_decode"), ("png.encode", "atom_eval_pillow:level1")], + ("det",), ) det = out["counts"]["det"] assert det["same/1"] == {"equal": 2} @@ -149,7 +151,7 @@ def test_m1_determinism_of_an_encoder_alone(tmp_path): def test_m1_objects_against_itself(tmp_path): d = _bundles(tmp_path) - out = _m1(d, [("reader", "test_atom_eval:fake_reader")], ("objects",)) + out = _m1(d, [("reader", "tests.pipeline.test_atom_eval:fake_reader")], ("objects",)) c = out["counts"]["objects"] assert c["set"] == {"equal": 2} and c["classId"] == {"equal": 4} and c["raw"] == {"equal": 4} assert c["externals"] == {"equal": 2} and c["container"] == {"equal": 2} diff --git a/tests/test_atoms_mesh.py b/tests/pipeline/test_atoms_mesh.py similarity index 98% rename from tests/test_atoms_mesh.py rename to tests/pipeline/test_atoms_mesh.py index 4414a0b..2e89bd2 100644 --- a/tests/test_atoms_mesh.py +++ b/tests/pipeline/test_atoms_mesh.py @@ -6,7 +6,13 @@ import numpy as np import pytest -from fakeunity import ( +from UnityPy.helpers.MeshHelper import MeshHandler + +from nnnotes.pipeline.atoms import ATOMS, Unsupported, gltf +from nnnotes.pipeline.atoms.mesh import MeshInput, arrays +from nnnotes.pipeline.atoms.reader import UnityBundle, handler_input, mesh_input_from_typetree +from nnnotes.unity import reader as unity +from tests.helpers.fakeunity import ( BLENDINDICES, BLENDWEIGHT, COLOR, @@ -28,12 +34,6 @@ submesh, unitypy_mesh, ) -from UnityPy.helpers.MeshHelper import MeshHandler - -from nnnotes import unity -from nnnotes.atoms import ATOMS, Unsupported, gltf -from nnnotes.atoms.mesh import MeshInput, arrays -from nnnotes.atoms.reader import UnityBundle, handler_input, mesh_input_from_typetree rng = np.random.default_rng(7) @@ -391,7 +391,7 @@ def test_streamed_vertex_data_is_read_from_the_bundle(): def test_sprite_render_mesh_input(): - from fakeunity import pptr, quad, rect, render_data, sprite_tt + from tests.helpers.fakeunity import pptr, quad, rect, render_data, sprite_tt r = rect(0, 0, 40, 20) pos, idx = quad(r) diff --git a/tests/test_atoms_sprite.py b/tests/pipeline/test_atoms_sprite.py similarity index 97% rename from tests/test_atoms_sprite.py rename to tests/pipeline/test_atoms_sprite.py index 40c5b64..7adc803 100644 --- a/tests/test_atoms_sprite.py +++ b/tests/pipeline/test_atoms_sprite.py @@ -5,7 +5,13 @@ import numpy as np import pytest -from fakeunity import ( +from UnityPy.export.SpriteHelper import get_image_from_sprite + +from nnnotes.pipeline.atoms import ATOMS, Unsupported, sprite +from nnnotes.pipeline.atoms.mesh import arrays +from nnnotes.pipeline.atoms.reader import mesh_input_from_typetree +from nnnotes.pipeline.atoms.texture import TextureInput, decode +from tests.helpers.fakeunity import ( ANDROID, VERSION, FakeTexture2D, @@ -17,12 +23,6 @@ sprite_tt, unitypy_sprite, ) -from UnityPy.export.SpriteHelper import get_image_from_sprite - -from nnnotes.atoms import ATOMS, Unsupported, sprite -from nnnotes.atoms.mesh import arrays -from nnnotes.atoms.reader import mesh_input_from_typetree -from nnnotes.atoms.texture import TextureInput, decode W, H = 64, 48 TEX = 3 diff --git a/tests/test_atoms_texture.py b/tests/pipeline/test_atoms_texture.py similarity index 93% rename from tests/test_atoms_texture.py rename to tests/pipeline/test_atoms_texture.py index 5c63174..6d4f6dc 100644 --- a/tests/test_atoms_texture.py +++ b/tests/pipeline/test_atoms_texture.py @@ -10,14 +10,14 @@ import numpy as np import pytest -from fakeunity import ANDROID, VERSION, FakeEnvironment, FakeTexture2D, rgba_bytes from PIL import Image from UnityPy.export.Texture2DConverter import parse_image_data -from nnnotes import export -from nnnotes.atoms import ATOMS, Impl, Unsupported, astc, env_name, ids, impl_id, png, resolve -from nnnotes.atoms.reader import ObjectInfo, UnityBundle -from nnnotes.atoms.texture import ASTC_BLOCKS, TextureInput, decode, is_hdr_astc +from nnnotes.pipeline.atoms import ATOMS, Impl, Unsupported, astc, env_name, ids, impl_id, png, resolve +from nnnotes.pipeline.atoms.reader import ObjectInfo, UnityBundle +from nnnotes.pipeline.atoms.texture import ASTC_BLOCKS, TextureInput, decode, is_hdr_astc +from nnnotes.unity import export +from tests.helpers.fakeunity import ANDROID, VERSION, FakeEnvironment, FakeTexture2D, rgba_bytes RGBA32, RGB24, ALPHA8, ARGB4444, YUY2 = 4, 3, 1, 2, 21 ASTC_4x4, ASTC_6x6, ASTC_HDR_6x6 = 54, 50, 68 @@ -255,7 +255,7 @@ def test_every_atom_has_one_reference_with_versioned_id(): def test_evaluation_override(monkeypatch): assert env_name("png.encode") == "NNNOTES_ATOM_PNG_ENCODE" - monkeypatch.setenv("NNNOTES_ATOM_PNG_ENCODE", "test_atoms_texture:REPLACEMENT") + monkeypatch.setenv("NNNOTES_ATOM_PNG_ENCODE", "tests.pipeline.test_atoms_texture:REPLACEMENT") assert resolve("png.encode") is REPLACEMENT and ATOMS["png.encode"].fn(None) == b"replaced" assert ids("png.encode") == {"png.encode": "pillow-test/9"} # keys change with the override monkeypatch.setenv("NNNOTES_ATOM_PNG_ENCODE", "no-colon") @@ -268,17 +268,17 @@ def test_evaluation_override(monkeypatch): def test_override_with_the_reference_id_is_refused(monkeypatch): - import test_atoms_texture as me + import tests.pipeline.test_atoms_texture as me monkeypatch.setattr(me, "SAME", Impl(ATOMS["png.encode"].id, png.encode, png.cost), raising=False) - monkeypatch.setenv("NNNOTES_ATOM_PNG_ENCODE", "test_atoms_texture:SAME") + monkeypatch.setenv("NNNOTES_ATOM_PNG_ENCODE", "tests.pipeline.test_atoms_texture:SAME") with pytest.raises(ValueError, match="reference's id"): resolve("png.encode") def test_listing_the_registry_loads_no_image_or_unity_library(): code = ( - "import sys, nnnotes.atoms as a; a.ids(*a.ATOMS); " + "import sys, nnnotes.pipeline.atoms as a; a.ids(*a.ATOMS); " "print(sorted(m for m in ('UnityPy', 'PIL') if m in sys.modules))" ) out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True).stdout @@ -289,14 +289,14 @@ def test_listing_the_registry_loads_no_image_or_unity_library(): def test_every_atom_and_reason_code_is_documented(): from pathlib import Path - root = Path(__file__).resolve().parents[1] + root = Path(__file__).resolve().parents[2] doc = (root / "docs" / "stages.md").read_text(encoding="utf-8") atoms_table = doc.split("## Atoms", 1)[1].split("## Reason codes", 1)[0] documented = set(re.findall(r"^\| `([a-z.]+)` \|", atoms_table, re.M)) assert documented == set(ATOMS) codes = set(re.findall(r"^\| `([a-zA-Z_.]+)` \|", doc.split("## Reason codes", 1)[1], re.M)) emitted = set() - for src in (root / "src" / "nnnotes" / "atoms").glob("*.py"): + for src in (root / "src" / "nnnotes" / "pipeline" / "atoms").glob("*.py"): emitted |= set( re.findall( r"\"((?:empty|unsupported|partial|generic|no_data)\.[a-zA-Z_.]+)\"", src.read_text(encoding="utf-8") diff --git a/tests/test_catalogdb.py b/tests/pipeline/test_catalogdb.py similarity index 97% rename from tests/test_catalogdb.py rename to tests/pipeline/test_catalogdb.py index 9f1bad2..1b40ba2 100644 --- a/tests/test_catalogdb.py +++ b/tests/pipeline/test_catalogdb.py @@ -2,12 +2,9 @@ import zipfile import pytest -import synth -from nnnotes import contract -from nnnotes.addressables import parse, parse_header, parse_keys, parse_locations -from nnnotes.catalog import APK_OFFSET_BASE, Catalog, file_name, location_kind -from nnnotes.catalogdb import ( +from nnnotes.pipeline import contract +from nnnotes.pipeline.catalogdb import ( CatalogDB, IndexStage, default_label, @@ -18,8 +15,11 @@ stable_raw_name, version_id, ) -from nnnotes.stages import Env, describe, execute, registry -from nnnotes.store import Store +from nnnotes.pipeline.stages import Env, describe, execute, registry +from nnnotes.pipeline.store import Store +from nnnotes.source.addressables import parse, parse_header, parse_keys, parse_locations +from nnnotes.source.catalog import APK_OFFSET_BASE, Catalog, file_name, location_kind +from tests.helpers import synth H1, H2, H3, H4 = ("1" * 32, "2" * 32, "3" * 32, "4" * 32) RAW = "cri_x/sound/bgm_a_" + "a" * 32 diff --git a/tests/test_census.py b/tests/pipeline/test_census.py similarity index 98% rename from tests/test_census.py rename to tests/pipeline/test_census.py index 9a36f6f..40647a5 100644 --- a/tests/test_census.py +++ b/tests/pipeline/test_census.py @@ -3,10 +3,11 @@ import pytest from UnityPy.helpers.TypeTreeNode import TypeTreeNode -from nnnotes import census, contract, unity -from nnnotes.contract import Input -from nnnotes.stages import Env, Pending, describe, execute, registry -from nnnotes.store import Store +from nnnotes.pipeline import census, contract +from nnnotes.pipeline.contract import Input +from nnnotes.pipeline.stages import Env, Pending, describe, execute, registry +from nnnotes.pipeline.store import Store +from nnnotes.unity import reader as unity SCRIPTS = "CAB-5c1e0000000000000000000000000000" diff --git a/tests/test_contract.py b/tests/pipeline/test_contract.py similarity index 97% rename from tests/test_contract.py rename to tests/pipeline/test_contract.py index 627336f..aaaa396 100644 --- a/tests/test_contract.py +++ b/tests/pipeline/test_contract.py @@ -7,15 +7,15 @@ from pathlib import Path import pytest -from test_stages import PIPELINE, toy_files, toy_stages -from nnnotes import contract, layout -from nnnotes import plan as plan_mod -from nnnotes.contract import Cost, IncompatibleTask, Input, Task -from nnnotes.orchestrate import FAILURE_CODES, Orchestrator -from nnnotes.store import Store +from nnnotes.pipeline import contract, layout +from nnnotes.pipeline import plan as plan_mod +from nnnotes.pipeline.contract import Cost, IncompatibleTask, Input, Task +from nnnotes.pipeline.orchestrate import FAILURE_CODES, Orchestrator +from nnnotes.pipeline.store import Store +from tests.pipeline.test_stages import PIPELINE, toy_files, toy_stages -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[2] SCHEMAS = ROOT / "src" / "nnnotes" / "schemas" SHA = "0" * 64 diff --git a/tests/test_cristages.py b/tests/pipeline/test_cristages.py similarity index 96% rename from tests/test_cristages.py rename to tests/pipeline/test_cristages.py index 2bb7361..86d21aa 100644 --- a/tests/test_cristages.py +++ b/tests/pipeline/test_cristages.py @@ -12,16 +12,18 @@ from unittest import mock import pytest -from test_speed_exact import FakeTools -from test_usm import ADX, HCA, KEY, MPEG1, VP9, audio_header, build, chunk, rand, sbt, video_header -from nnnotes import cache, contract, cri, crikey, cristages -from nnnotes.contract import IncompatibleTask, Input, Task -from nnnotes.cristages import AUDIO, AudioStage, MovieStage -from nnnotes.stages import Env, Pending, describe, execute, registry -from nnnotes.store import Store - -ROOT = Path(__file__).resolve().parents[1] +from nnnotes.core import cache +from nnnotes.formats import cri, crikey +from nnnotes.pipeline import contract, cristages +from nnnotes.pipeline.contract import IncompatibleTask, Input, Task +from nnnotes.pipeline.cristages import AUDIO, AudioStage, MovieStage +from nnnotes.pipeline.stages import Env, Pending, describe, execute, registry +from nnnotes.pipeline.store import Store +from tests.formats.test_usm import ADX, HCA, KEY, MPEG1, VP9, audio_header, build, chunk, rand, sbt, video_header +from tests.unity.test_speed_exact import FakeTools + +ROOT = Path(__file__).resolve().parents[2] ACB = b"@UTF" + b"cue sheet bytes" AWB = b"AFS2" + b"streamed waveforms" NAMES = ["a", "b", "a", "c", "c"] # stream names (layered waveforms repeat a name) @@ -244,7 +246,7 @@ def test_audio_subjects_by_content(tmp_path, tools): def test_audio_waits_for_raw_files_not_fetched(tmp_path, tools): - from nnnotes.cli_assets import Inputs + from nnnotes.cli.cli_assets import Inputs store = Store(tmp_path / "s") env = planning(store, tmp_path, {"cri_assets_cri/sound/a": ACB}) @@ -348,7 +350,7 @@ def test_a_task_of_other_tools_is_incompatible(tmp_path, tools): # ---------------------------------------------------------------- cri.movie def test_movie_streams_as_stored_and_the_mkv(tmp_path, tools): - from nnnotes import usm + from nnnotes.formats import usm store = Store(tmp_path / "s") data, frames = usm_file() @@ -416,7 +418,7 @@ def test_movie_without_audio(tmp_path, tools): def test_movie_every_stream_kind(tmp_path, tools): """MPEG-1 video and alpha, two ADX channels, an HCA channel (decoded by vgmstream with the key) and two subtitle channels: every stream as stored, the subtitles as JSON / SubRip / WebVTT, every track in the Matroska file.""" - from nnnotes import usm + from nnnotes.formats import usm store = Store(tmp_path / "s") data = build(EVERY) @@ -590,9 +592,9 @@ def test_movie_parameters(): # ---------------------------------------------------------------- laziness and documentation def test_stage_modules_import_nothing_heavy(): code = ( - "import sys; import nnnotes.cristages as c; c.AudioStage().normalize(None); " - "print(sorted(m for m in ('UnityPy', 'numpy', 'PIL', 'nnnotes.cri', 'nnnotes.usm', 'nnnotes.export')" - " if m in sys.modules))" + "import sys; import nnnotes.pipeline.cristages as c; c.AudioStage().normalize(None); " + "heavy = ('UnityPy', 'numpy', 'PIL', 'nnnotes.formats.cri', 'nnnotes.formats.usm', 'nnnotes.unity.export'); " + "print(sorted(m for m in heavy if m in sys.modules))" ) out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, cwd=ROOT, check=True) assert out.stdout.strip() == "[]" @@ -601,7 +603,8 @@ def test_stage_modules_import_nothing_heavy(): def test_reason_codes_and_stages_are_documented(): doc = (ROOT / "docs" / "stages.md").read_text(encoding="utf-8") documented = set(re.findall(r"^\| `([a-z_]+(?:\.[A-Za-z_]+)+)` \|", doc, re.M)) - text = "".join((ROOT / "src" / "nnnotes" / f).read_text(encoding="utf-8") for f in ("cristages.py", "usm.py")) + sources = ("pipeline/cristages.py", "formats/usm.py") + text = "".join((ROOT / "src" / "nnnotes" / f).read_text(encoding="utf-8") for f in sources) raised = set(cristages.REASONS) | set(re.findall(r'"(unsupported\.[a-z_.]+)"', text)) assert raised - documented == set() and raised == set(cristages.REASONS) usm_codes = {c for c in documented if c.startswith("unsupported.usm.")} @@ -628,7 +631,7 @@ def run(self, task, store): class GoldenMovie(MovieStage): @property def ATOMS(self) -> dict: - from nnnotes.atoms import impl_id + from nnnotes.pipeline.atoms import impl_id return { "usm.demux": impl_id(("numpy",), cristages.USM_DEMUX_REVISION), @@ -698,7 +701,7 @@ def movie_golden(): def test_golden_providers_run_twice_alike(tmp_path): - from nnnotes.stages import golden, golden_problems + from nnnotes.pipeline.stages import golden, golden_problems for provider in (audio_golden, movie_golden): p = provider() diff --git a/tests/test_goldens.py b/tests/pipeline/test_goldens.py similarity index 87% rename from tests/test_goldens.py rename to tests/pipeline/test_goldens.py index c1fd6f4..c2e0d1d 100644 --- a/tests/test_goldens.py +++ b/tests/pipeline/test_goldens.py @@ -18,13 +18,13 @@ from pathlib import Path import pytest -from test_stages import Src -from nnnotes import contract -from nnnotes.stages import Output, golden, golden_problems, library_versions -from nnnotes.store import Store +from nnnotes.pipeline import contract +from nnnotes.pipeline.stages import Output, golden, golden_problems, library_versions +from nnnotes.pipeline.store import Store +from tests.pipeline.test_stages import Src -GOLDEN_DIR = Path(__file__).resolve().parent / "golden" +GOLDEN_DIR = Path(__file__).resolve().parents[1] / "golden" def provided(spec: str) -> dict: @@ -57,7 +57,7 @@ def test_golden_record(path, tmp_path): # ---------------------------------------------------------------- the machinery itself (toy stage) def text_fixture(text: str): def build(store): - from nnnotes.contract import Input, Task + from nnnotes.pipeline.contract import Input, Task data = text.encode() sha = store.put(data) @@ -82,8 +82,8 @@ def toy_provider(): def test_a_regenerated_record_matches(tmp_path): - a = record("test_goldens:toy_provider", Store(tmp_path / "a")) - b = record("test_goldens:toy_provider", Store(tmp_path / "b")) + a = record("tests.pipeline.test_goldens:toy_provider", Store(tmp_path / "a")) + b = record("tests.pipeline.test_goldens:toy_provider", Store(tmp_path / "b")) assert a == b and golden_problems(a, b, Src()) == [] assert a["libraries"] == library_versions(["pytest"]) and set(a["fixtures"]) == {"plain", "items"} assert a["atoms"] == Src.ATOMS and a["version"] == 1 @@ -99,7 +99,7 @@ def run(self, task, store): def test_an_output_change_without_a_version_bump_fails(tmp_path): - expected = record("test_goldens:toy_provider", Store(tmp_path / "a")) + expected = record("tests.pipeline.test_goldens:toy_provider", Store(tmp_path / "a")) fixtures = toy_provider()["fixtures"] actual = golden(Changed(), fixtures, Store(tmp_path / "b"), ["pytest"]) assert golden_problems(expected, actual) == ["toy.src: fixture items: output changed without a version bump"] @@ -123,14 +123,14 @@ def test_every_declared_atom_needs_a_fixture(tmp_path): class More(Src): ATOMS = {**Src.ATOMS, "mesh.glb": "toy-glb/1"} - doc = record("test_goldens:toy_provider", Store(tmp_path / "a")) + doc = record("tests.pipeline.test_goldens:toy_provider", Store(tmp_path / "a")) assert golden_problems(doc, doc, More()) == ["toy.src: no fixture uses mesh.glb"] missing = dict(doc, fixtures={"plain": doc["fixtures"]["plain"]}) assert golden_problems(missing, doc) == ["toy.src: fixture items has no golden entry"] def test_records_are_canonical_documents(tmp_path): - doc = record("test_goldens:toy_provider", Store(tmp_path / "a")) + doc = record("tests.pipeline.test_goldens:toy_provider", Store(tmp_path / "a")) path = tmp_path / "toy.src.json" path.write_bytes(contract.encode(doc)) assert contract.loads(path.read_bytes()) == doc diff --git a/tests/test_layout.py b/tests/pipeline/test_layout.py similarity index 98% rename from tests/test_layout.py rename to tests/pipeline/test_layout.py index 3c74a04..924d992 100644 --- a/tests/test_layout.py +++ b/tests/pipeline/test_layout.py @@ -6,10 +6,10 @@ import pytest -from nnnotes import contract, layout -from nnnotes.contract import Task -from nnnotes.layout import LayoutError, Source, escape_segment, unescape_path -from nnnotes.store import Store +from nnnotes.pipeline import contract, layout +from nnnotes.pipeline.contract import Task +from nnnotes.pipeline.layout import LayoutError, Source, escape_segment, unescape_path +from nnnotes.pipeline.store import Store def bundle(store, subject, objects, loose=(), root=True): diff --git a/tests/test_link.py b/tests/pipeline/test_link.py similarity index 98% rename from tests/test_link.py rename to tests/pipeline/test_link.py index 237f375..f6d5f3f 100644 --- a/tests/test_link.py +++ b/tests/pipeline/test_link.py @@ -1,11 +1,11 @@ import pytest -import synth -from nnnotes import catalogdb, census, contract, link -from nnnotes.catalogdb import CatalogDB, IndexStage, index -from nnnotes.contract import Input -from nnnotes.stages import Env, Pending, describe, execute, registry -from nnnotes.store import Store +from nnnotes.pipeline import catalogdb, census, contract, link +from nnnotes.pipeline.catalogdb import CatalogDB, IndexStage, index +from nnnotes.pipeline.contract import Input +from nnnotes.pipeline.stages import Env, Pending, describe, execute, registry +from nnnotes.pipeline.store import Store +from tests.helpers import synth HA, HB, HC = "a" * 32, "b" * 32, "c" * 32 CAB_A, CAB_B, CAB_S, CAB_M = (f"CAB-{c * 32}" for c in "1234") diff --git a/tests/test_modelstages.py b/tests/pipeline/test_modelstages.py similarity index 96% rename from tests/test_modelstages.py rename to tests/pipeline/test_modelstages.py index ae40b01..0df3759 100644 --- a/tests/test_modelstages.py +++ b/tests/pipeline/test_modelstages.py @@ -7,16 +7,16 @@ from pathlib import Path import pytest -import synth -from nnnotes import census, contract, link, modelstages -from nnnotes.catalog import Catalog -from nnnotes.catalogdb import index -from nnnotes.census import CensusStage -from nnnotes.contract import IncompatibleTask, Input, Task -from nnnotes.modelstages import Closures, Live2DStage, SpineStage, TaskCatalog -from nnnotes.stages import Env, Pending, describe, execute, registry -from nnnotes.store import Store +from nnnotes.pipeline import census, contract, link, modelstages +from nnnotes.pipeline.catalogdb import index +from nnnotes.pipeline.census import CensusStage +from nnnotes.pipeline.contract import IncompatibleTask, Input, Task +from nnnotes.pipeline.modelstages import Closures, Live2DStage, SpineStage, TaskCatalog +from nnnotes.pipeline.stages import Env, Pending, describe, execute, registry +from nnnotes.pipeline.store import Store +from nnnotes.source.catalog import Catalog +from tests.helpers import synth H = [f"{i:x}" * 32 for i in range(1, 10)] M1 = "Character/Live2D/g1/m1/model/m1" @@ -200,7 +200,7 @@ def extract_model(cat, key, out_dir): def test_live2d_run_stores_every_file_of_the_extractor(tmp_path, monkeypatch): - from nnnotes import live2d + from nnnotes.unity import live2d calls = [] monkeypatch.setattr(live2d, "extract_model", fake_extract(calls)) @@ -248,7 +248,8 @@ def test_live2d_run_stores_every_file_of_the_extractor(tmp_path, monkeypatch): def test_the_model_key_and_libraries_are_those_of_the_extractors(): - from nnnotes import export, webmodel + from nnnotes.site import webmodel + from nnnotes.unity import export assert modelstages.MODEL_KEY.pattern == webmodel.MODEL_KEY.pattern assert modelstages.LIBRARIES == export._LIBS @@ -256,7 +257,7 @@ def test_the_model_key_and_libraries_are_those_of_the_extractors(): def test_importing_the_stages_does_not_import_unitypy(): code = ( - "import sys, nnnotes.modelstages as m; m.Live2DStage(); m.SpineStage(); " + "import sys, nnnotes.pipeline.modelstages as m; m.Live2DStage(); m.SpineStage(); " "assert 'UnityPy' not in sys.modules, 'UnityPy imported'" ) r = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True) @@ -487,7 +488,7 @@ def fake_skeleton(pages=1): def test_spine_run_stores_the_files_of_the_skeleton(tmp_path, monkeypatch): - from nnnotes import unity + from nnnotes.unity import reader as unity stage, store = SpineStage(), Store(tmp_path / "s") env = spine_env(store) diff --git a/tests/test_objexport.py b/tests/pipeline/test_objexport.py similarity index 97% rename from tests/test_objexport.py rename to tests/pipeline/test_objexport.py index fb69d19..f4e9ad2 100644 --- a/tests/test_objexport.py +++ b/tests/pipeline/test_objexport.py @@ -12,7 +12,17 @@ import numpy as np import pytest -from fakeunity import ( +from PIL import Image + +from nnnotes.pipeline import census as census_mod +from nnnotes.pipeline import contract, objexport +from nnnotes.pipeline.atoms import Impl, astc, estimate, gltf +from nnnotes.pipeline.atoms.reader import UnityBundle +from nnnotes.pipeline.contract import IncompatibleTask, Input, Task +from nnnotes.pipeline.objexport import CONVERTERS, CROP, EXPORT, ExportStage, SpriteCropStage, export_atoms +from nnnotes.pipeline.stages import Env, describe, execute, registry +from nnnotes.pipeline.store import Store +from tests.helpers.fakeunity import ( F32, NORMAL, POSITION, @@ -28,18 +38,8 @@ sprite_tt, submesh, ) -from PIL import Image - -from nnnotes import census as census_mod -from nnnotes import contract, objexport -from nnnotes.atoms import Impl, astc, estimate, gltf -from nnnotes.atoms.reader import UnityBundle -from nnnotes.contract import IncompatibleTask, Input, Task -from nnnotes.objexport import CONVERTERS, CROP, EXPORT, ExportStage, SpriteCropStage, export_atoms -from nnnotes.stages import Env, describe, execute, registry -from nnnotes.store import Store -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[2] CAB, ATLAS_CAB = "CAB-a", "CAB-atlas" SCRIPTS = { f"{CAB}:80": "A|Game|Local", @@ -177,7 +177,7 @@ def _embedded_acb_asset(f, pid: int, name: str, acb: bytes, script: int = 11, ** """A CriWare.Assets asset (script `script`) whose `implementation` managed reference is a CriSerializedBytesAssetImpl holding `acb`, with UnityPy's managed reference nodes; `fields` after `implementation` (by default an ACB asset's `awb`).""" - from fakeunity import node_of + from tests.helpers.fakeunity import node_of tt = _mono( 0, @@ -712,7 +712,7 @@ def test_textures(exported): assert img.size == (W, H) and img.mode == "RGBA" expected = Image.frombytes("RGBA", (W, H), TEX_PIXELS).transpose(Image.Transpose.FLIP_TOP_BOTTOM) assert img.tobytes() == expected.tobytes() - from nnnotes.export import texture_png + from nnnotes.unity.export import texture_png tex = bundle_all().assets[0].objects[10].obj assert png == texture_png(tex) # the other commands' bytes @@ -734,10 +734,10 @@ def test_textures(exported): def test_sprites_same_bundle(exported): store, _task, doc = exported - from nnnotes.atoms import png, sprite - from nnnotes.atoms.mesh import arrays - from nnnotes.atoms.reader import mesh_input_from_typetree - from nnnotes.atoms.texture import TextureInput, decode + from nnnotes.pipeline.atoms import png, sprite + from nnnotes.pipeline.atoms.mesh import arrays + from nnnotes.pipeline.atoms.reader import mesh_input_from_typetree + from nnnotes.pipeline.atoms.texture import TextureInput, decode env = bundle_all() f = env.assets[0] @@ -844,7 +844,7 @@ def test_scriptable_objects_blobs_and_strings(exported): def test_cue_sheets_held_in_the_bundle(exported): store, _task, doc = exported - from nnnotes import cri + from nnnotes.formats import cri a = art(doc, f"{oid(73)}#acb") assert load(store, doc, a["id"]) == SPLIT_ACB == cri.split_acb(SPLIT_CHUNKS) @@ -913,7 +913,7 @@ def test_the_movie_converter_is_keyed_by_the_script_context(tmp_path): def test_acb_layouts(tmp_path): - from nnnotes import cri + from nnnotes.formats import cri assert objexport.acb_asset({"_cueSheetName": "x", "_chunks": []})["layout"] == "split" t = { @@ -966,7 +966,7 @@ def test_a_split_sheet_with_a_missing_chunk_gives_the_typetree(tmp_path, monkeyp def test_shader(exported): store, _task, doc = exported - from nnnotes import shader + from nnnotes.formats import shader env = bundle_all() r = env.assets[0].objects[51] @@ -1257,7 +1257,7 @@ def test_describe_reads_the_census_and_the_script_table(tmp_path): assert a.context == {} and "mesh.glb" not in a.atoms assert t.cost.peak_bytes >= 72 << 20 fresh = Env(store, env.facts) - from nnnotes.stages import Pending + from nnnotes.pipeline.stages import Pending fresh.waiting("unity.census:all") with pytest.raises(Pending): @@ -1318,7 +1318,7 @@ def test_sprite_crop_waits_for_exports(tmp_path): store = Store(tmp_path / "s") env = planning_env(store) env.waiting("unity.export:all") - from nnnotes.stages import Pending + from nnnotes.pipeline.stages import Pending with pytest.raises(Pending): SpriteCropStage().subjects(env) @@ -1326,8 +1326,8 @@ def test_sprite_crop_waits_for_exports(tmp_path): def test_modules_import_without_unitypy(): code = ( - "import sys; import nnnotes.objexport as o; o.ExportStage().ATOMS; o.SpriteCropStage().ATOMS; " - "print(sorted(m for m in ('UnityPy', 'PIL', 'nnnotes.export') if m in sys.modules))" + "import sys; import nnnotes.pipeline.objexport as o; o.ExportStage().ATOMS; o.SpriteCropStage().ATOMS; " + "print(sorted(m for m in ('UnityPy', 'PIL', 'nnnotes.unity.export') if m in sys.modules))" ) out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, cwd=ROOT, check=True) assert out.stdout.strip() == "[]" @@ -1337,7 +1337,8 @@ def test_modules_import_without_unitypy(): def test_every_reason_code_and_converter_is_documented(): doc = (ROOT / "docs" / "stages.md").read_text(encoding="utf-8") documented = set(re.findall(r"^\| `([a-z_]+(?:\.[A-Za-z_]+)+)` \|", doc, re.M)) - sources = [ROOT / "src" / "nnnotes" / "objexport.py", *sorted((ROOT / "src" / "nnnotes" / "atoms").glob("*.py"))] + pipeline = ROOT / "src" / "nnnotes" / "pipeline" + sources = [pipeline / "objexport.py", *sorted((pipeline / "atoms").glob("*.py"))] raised = set(objexport.REASONS) for p in sources: text = p.read_text(encoding="utf-8") @@ -1401,7 +1402,7 @@ def test_golden_providers_cover_every_converter(): def test_a_converter_change_without_a_version_bump_fails_its_golden(tmp_path, monkeypatch): - from nnnotes.stages import golden, golden_problems, library_versions + from nnnotes.pipeline.stages import golden, golden_problems, library_versions expected = contract.loads((ROOT / "tests" / "golden" / "unity.export.json").read_bytes()) if library_versions(expected["libraries"]) != expected["libraries"]: diff --git a/tests/test_orchestrate.py b/tests/pipeline/test_orchestrate.py similarity index 96% rename from tests/test_orchestrate.py rename to tests/pipeline/test_orchestrate.py index 5e92306..b00abd2 100644 --- a/tests/test_orchestrate.py +++ b/tests/pipeline/test_orchestrate.py @@ -4,14 +4,14 @@ from pathlib import Path import pytest -from test_stages import PIPELINE, toy_files, toy_stages -from nnnotes import contract -from nnnotes.contract import Cost, Task -from nnnotes.orchestrate import Orchestrator, Scheduler, coverage -from nnnotes.plan import plan -from nnnotes.stages import execute -from nnnotes.store import Store +from nnnotes.pipeline import contract +from nnnotes.pipeline.contract import Cost, Task +from nnnotes.pipeline.orchestrate import Orchestrator, Scheduler, coverage +from nnnotes.pipeline.plan import plan +from nnnotes.pipeline.stages import execute +from nnnotes.pipeline.store import Store +from tests.pipeline.test_stages import PIPELINE, toy_files, toy_stages TEXTS = { "a": "alpha\nbeta\n\nFAIL here\n?what", @@ -214,9 +214,8 @@ def test_unknown_or_misordered_stages_are_refused(tmp_path): def test_a_config_error_stops_the_run(tmp_path): - from test_stages import Src - - from nnnotes.config import ConfigError + from nnnotes.source.config import ConfigError + from tests.pipeline.test_stages import Src class NeedsTool(Src): def run(self, task, store): @@ -230,7 +229,7 @@ def run(self, task, store): def test_a_stage_whose_subjects_need_a_failed_task_fails_once(tmp_path): - from nnnotes.stages import Pending, Stage + from nnnotes.pipeline.stages import Pending, Stage class AfterAll(Stage): name, version, after = "toy.after", 1, ("toy.src",) diff --git a/tests/test_plan.py b/tests/pipeline/test_plan.py similarity index 96% rename from tests/test_plan.py rename to tests/pipeline/test_plan.py index e116a33..177e017 100644 --- a/tests/test_plan.py +++ b/tests/pipeline/test_plan.py @@ -2,13 +2,12 @@ from pathlib import Path -from test_stages import PIPELINE, toy_files, toy_stages - -from nnnotes import contract -from nnnotes.contract import Task -from nnnotes.orchestrate import Orchestrator -from nnnotes.plan import plan, why -from nnnotes.store import Store +from nnnotes.pipeline import contract +from nnnotes.pipeline.contract import Task +from nnnotes.pipeline.orchestrate import Orchestrator +from nnnotes.pipeline.plan import plan, why +from nnnotes.pipeline.store import Store +from tests.pipeline.test_stages import PIPELINE, toy_files, toy_stages TEXTS = {"a": "alpha\nbeta", "b": "gamma", "c": "delta\n?x"} diff --git a/tests/test_sniff.py b/tests/pipeline/test_sniff.py similarity index 97% rename from tests/test_sniff.py rename to tests/pipeline/test_sniff.py index 9988699..bc3df9b 100644 --- a/tests/test_sniff.py +++ b/tests/pipeline/test_sniff.py @@ -8,9 +8,9 @@ import pytest -from nnnotes import contract -from nnnotes.atoms import ATOMS -from nnnotes.atoms.sniff import MAGIC, Sniffed, sniff, spine_binary_version +from nnnotes.pipeline import contract +from nnnotes.pipeline.atoms import ATOMS +from nnnotes.pipeline.atoms.sniff import MAGIC, Sniffed, sniff, spine_binary_version def sfnt(tag: bytes, tables: int = 3) -> bytes: diff --git a/tests/test_stages.py b/tests/pipeline/test_stages.py similarity index 96% rename from tests/test_stages.py rename to tests/pipeline/test_stages.py index dc3d7bb..3ebd6c4 100644 --- a/tests/test_stages.py +++ b/tests/pipeline/test_stages.py @@ -12,10 +12,10 @@ import pytest -from nnnotes import contract -from nnnotes.contract import Cost, IncompatibleTask, Input -from nnnotes.stages import Env, Output, Pending, Stage, describe, execute, registry -from nnnotes.store import InputMissing, Store, StoreConflict +from nnnotes.pipeline import contract +from nnnotes.pipeline.contract import Cost, IncompatibleTask, Input +from nnnotes.pipeline.stages import Env, Output, Pending, Stage, describe, execute, registry +from nnnotes.pipeline.store import InputMissing, Store, StoreConflict class Src(Stage): @@ -261,7 +261,7 @@ def test_registry_refuses_repeated_names(): def test_describe_gives_the_stage_its_parameters(tmp_path): - from nnnotes.contract import Task + from nnnotes.pipeline.contract import Task class Selective(Stage): name, version = "toy.sel", 1 @@ -298,8 +298,8 @@ def run(self, task, store): def test_the_cost_counts_the_child_processes(tmp_path): - from nnnotes.contract import Task - from nnnotes.stages import children_cpu + from nnnotes.pipeline.contract import Task + from nnnotes.pipeline.stages import children_cpu store = Store(tmp_path / "s") ex = execute(Task("toy.spawn", 1, "s"), store, registry(_Spawner())) diff --git a/tests/test_store.py b/tests/pipeline/test_store.py similarity index 97% rename from tests/test_store.py rename to tests/pipeline/test_store.py index 826c207..56c596b 100644 --- a/tests/test_store.py +++ b/tests/pipeline/test_store.py @@ -7,10 +7,10 @@ import pytest -from nnnotes import contract -from nnnotes import store as store_mod -from nnnotes.contract import Input, Task -from nnnotes.store import InputMissing, Store, StoreConflict +from nnnotes.pipeline import contract +from nnnotes.pipeline import store as store_mod +from nnnotes.pipeline.contract import Input, Task +from nnnotes.pipeline.store import InputMissing, Store, StoreConflict def a_result(s: Store, task: Task, payload: bytes) -> dict: @@ -200,7 +200,7 @@ def test_a_store_crosses_processes_as_its_settings(tmp_path): def test_write_doc_writes_the_canonical_bytes_as_it_encodes(tmp_path): - from nnnotes.store import write_doc + from nnnotes.pipeline.store import write_doc docs = [ {"b": [1, 2, {"é": "x\n"}], "a": {}, "c": [], "d": None, "e": 1.5, "f": "立ち絵" * 1000}, diff --git a/tests/test_views.py b/tests/pipeline/test_views.py similarity index 98% rename from tests/test_views.py rename to tests/pipeline/test_views.py index 07af065..3137335 100644 --- a/tests/test_views.py +++ b/tests/pipeline/test_views.py @@ -10,15 +10,16 @@ import pytest -from nnnotes import contract, languages, views -from nnnotes.contract import Input -from nnnotes.orchestrate import Orchestrator -from nnnotes.plan import plan -from nnnotes.stages import Env, Output, Pending, Stage, describe, execute, registry -from nnnotes.store import Store -from nnnotes.views import AddressIndex, Obj, Recorder, ViewError, ViewStage +from nnnotes.pipeline import contract, views +from nnnotes.pipeline.contract import Input +from nnnotes.pipeline.orchestrate import Orchestrator +from nnnotes.pipeline.plan import plan +from nnnotes.pipeline.stages import Env, Output, Pending, Stage, describe, execute, registry +from nnnotes.pipeline.store import Store +from nnnotes.pipeline.views import AddressIndex, Obj, Recorder, ViewError, ViewStage +from nnnotes.source import languages -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[2] def o(n, cls, name, stable=None): @@ -987,8 +988,8 @@ def test_views_plan_and_run_after_the_address_table(tmp_path): def test_views_import_neither_unitypy_nor_numpy(): code = ( - "import sys, nnnotes.views; " - "print(sorted(m for m in ('UnityPy', 'numpy', 'PIL', 'nnnotes.export') if m in sys.modules))" + "import sys, nnnotes.pipeline.views; " + "print(sorted(m for m in ('UnityPy', 'numpy', 'PIL', 'nnnotes.unity.export') if m in sys.modules))" ) out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, cwd=ROOT, check=True).stdout assert out.strip() == "[]" diff --git a/tests/test_voices.py b/tests/pipeline/test_voices.py similarity index 98% rename from tests/test_voices.py rename to tests/pipeline/test_voices.py index f6cd8ce..7c78353 100644 --- a/tests/test_voices.py +++ b/tests/pipeline/test_voices.py @@ -13,15 +13,17 @@ from types import SimpleNamespace import pytest -from test_views import FakeAddresses, addresses_doc, write_master -from nnnotes import acb, contract, languages, voices -from nnnotes.contract import Input, Task -from nnnotes.stages import Env, Pending, describe, execute, registry -from nnnotes.store import Store -from nnnotes.views import AddressIndex, ViewError +from nnnotes.formats import acb +from nnnotes.pipeline import contract, voices +from nnnotes.pipeline.contract import Input, Task +from nnnotes.pipeline.stages import Env, Pending, describe, execute, registry +from nnnotes.pipeline.store import Store +from nnnotes.pipeline.views import AddressIndex, ViewError +from nnnotes.source import languages +from tests.pipeline.test_views import FakeAddresses, addresses_doc, write_master -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[2] RATE = 48000 @@ -958,7 +960,7 @@ def test_get_copies_the_exported_files(tmp_path, capsysbinary): def test_get_without_an_export_decodes_the_one_sheet(tmp_path, capsysbinary, monkeypatch): - from nnnotes import cri + from nnnotes.formats import cri mdir = write_master(tmp_path / "m", TABLES) calls = [] @@ -989,8 +991,9 @@ def decode(cat, sheet, out_dir, key=None, fmt="flac", **kw): # ---------------------------------------------------------------- laziness and documentation def test_voices_import_nothing_heavy(): code = ( - "import sys, nnnotes.voices, nnnotes.acb, nnnotes.storyvoices, nnnotes.advcommand; " - "print(sorted(m for m in ('UnityPy', 'numpy', 'PIL', 'nnnotes.cri') if m in sys.modules))" + "import sys, nnnotes.pipeline.voices, nnnotes.formats.acb, nnnotes.content.storyvoices, " + "nnnotes.content.advcommand; " + "print(sorted(m for m in ('UnityPy', 'numpy', 'PIL', 'nnnotes.formats.cri') if m in sys.modules))" ) out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, cwd=ROOT, check=True).stdout assert out.strip() == "[]" diff --git a/tests/test_voices_story.py b/tests/pipeline/test_voices_story.py similarity index 97% rename from tests/test_voices_story.py rename to tests/pipeline/test_voices_story.py index be80d9c..257c030 100644 --- a/tests/test_voices_story.py +++ b/tests/pipeline/test_voices_story.py @@ -9,14 +9,16 @@ from types import SimpleNamespace import pytest -from test_views import addresses_doc, write_master -from test_voices import KEYS, SHEETS, TABLES, Cfg, Paths, audio, audio_task, command, planning, row, text -from nnnotes import advcommand, contract, languages, storyvoices, voices -from nnnotes.contract import Task -from nnnotes.stages import Pending, describe, execute, registry -from nnnotes.store import Store -from nnnotes.views import AddressIndex, Obj, ViewError +from nnnotes.content import advcommand, storyvoices +from nnnotes.pipeline import contract, voices +from nnnotes.pipeline.contract import Task +from nnnotes.pipeline.stages import Pending, describe, execute, registry +from nnnotes.pipeline.store import Store +from nnnotes.pipeline.views import AddressIndex, Obj, ViewError +from nnnotes.source import languages +from tests.pipeline.test_views import addresses_doc, write_master +from tests.pipeline.test_voices import KEYS, SHEETS, TABLES, Cfg, Paths, audio, audio_task, command, planning, row, text RULES = voices.load_rules() STORY_SHEET = "adv_voice_x_01" @@ -651,9 +653,8 @@ def keys(self, prefix=""): def test_command_without_an_export_reads_one_episode(tmp_path, capsysbinary, monkeypatch): - from test_voices import sheet_acb, stream_files - - from nnnotes import cri + from nnnotes.formats import cri + from tests.pipeline.test_voices import sheet_acb, stream_files mdir = write_master(tmp_path / "m", tables()) loaded = [] diff --git a/tests/site/__init__.py b/tests/site/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_compress.py b/tests/site/test_compress.py similarity index 98% rename from tests/test_compress.py rename to tests/site/test_compress.py index a4d7b91..c702bb3 100644 --- a/tests/test_compress.py +++ b/tests/site/test_compress.py @@ -6,7 +6,7 @@ import pytest -from nnnotes import compress +from nnnotes.site import compress TEXT = json.dumps({str(i): [i, i * i, f"v{i % 7}"] for i in range(500)}).encode() NOISE = random.Random(7).randbytes(4096) diff --git a/tests/test_live_options_web.py b/tests/site/test_live_options_web.py similarity index 95% rename from tests/test_live_options_web.py rename to tests/site/test_live_options_web.py index 97f5351..8b29ece 100644 --- a/tests/test_live_options_web.py +++ b/tests/site/test_live_options_web.py @@ -7,9 +7,9 @@ import pytest -from nnnotes import sitefiles, web -from nnnotes.config import ConfigError -from nnnotes.liveoptions import LiveOptions, parse_specs, resolve +from nnnotes.content.liveoptions import LiveOptions, parse_specs, resolve +from nnnotes.site import sitefiles, web +from nnnotes.source.config import ConfigError # ---------------------------------------------------------------- read sets with settings @@ -29,7 +29,7 @@ def run(args, **kw): def test_the_read_set_key_includes_the_settings(tmp_path, monkeypatch): - from nnnotes import cache + from nnnotes.core import cache cache.configure(enabled=True) live = tmp_path / "live-settings" @@ -144,7 +144,7 @@ def test_region_manifests_with_other_options_stay(tmp_path): def test_manifest_options(tmp_path): - from test_live_options import write_master + from tests.content.test_live_options import write_master m = write_master(tmp_path / "m") assert resolve(parse_specs(["MirrorChart", "NoteDesignId"]), m).manifest_options() is None @@ -154,10 +154,9 @@ def test_manifest_options(tmp_path): # ---------------------------------------------------------------- the flag def test_live_option_flag(tmp_path, capsys, monkeypatch): - from test_live_options import write_master as option_tables - from test_regions import fake_player, write_master - from nnnotes import cli + from tests.content.test_live_options import write_master as option_tables + from tests.site.test_regions import fake_player, write_master master = option_tables(write_master(tmp_path / "m")) monkeypatch.setenv("NNNOTES_SERVERS_ZZ_CDN", (tmp_path / "cdn").as_uri()) diff --git a/tests/test_pipeline.py b/tests/site/test_pipeline.py similarity index 97% rename from tests/test_pipeline.py rename to tests/site/test_pipeline.py index 9711f62..fd8ea51 100644 --- a/tests/test_pipeline.py +++ b/tests/site/test_pipeline.py @@ -7,7 +7,8 @@ import pytest -from nnnotes import cache, sitefiles, web +from nnnotes.core import cache +from nnnotes.site import sitefiles, web def run(musics, extract, read, ingest, window=2, workers=2, readers=3): @@ -145,9 +146,8 @@ def test_web_audio_takes_the_encoding_made_with_the_decode(tmp_path, monkeypatch def test_catalog_cached(tmp_path): - import synth - - from nnnotes.catalog import Bundle, Catalog + from nnnotes.source.catalog import Bundle, Catalog + from tests.helpers import synth cat = Catalog(synth.CatalogWriter().build([("k", synth.remote("b_1.bundle"), [])]), tmp_path) b = Bundle(1, synth.remote("b_1.bundle"), "b_1.bundle", True) diff --git a/tests/test_readsets.py b/tests/site/test_readsets.py similarity index 98% rename from tests/test_readsets.py rename to tests/site/test_readsets.py index 661e369..89dd8c2 100644 --- a/tests/test_readsets.py +++ b/tests/site/test_readsets.py @@ -8,8 +8,8 @@ import pytest -from nnnotes import sitefiles, web -from nnnotes.config import ConfigError +from nnnotes.site import sitefiles, web +from nnnotes.source.config import ConfigError # ---------------------------------------------------------------- the player's modes @@ -226,7 +226,7 @@ def test_a_first_chart_the_plan_cannot_list_is_simulated(player, monkeypatch, tm def test_the_read_set_key_includes_the_mode(tmp_path, monkeypatch): - from nnnotes import cache + from nnnotes.core import cache cache.configure(enabled=True) live = tmp_path / "live" @@ -241,7 +241,7 @@ def test_the_read_set_key_includes_the_mode(tmp_path, monkeypatch): def test_a_read_set_run_by_a_server_is_stored_under_the_same_key(tmp_path, monkeypatch): - from nnnotes import cache + from nnnotes.core import cache cache.configure(enabled=True) live = tmp_path / "live-served" @@ -435,9 +435,8 @@ def test_web_read_workers_flag_reaches_the_build(tmp_path, capsys, monkeypatch): def test_web_pair_that_no_region_has_is_a_usage_error(tmp_path, capsys, monkeypatch): - from test_regions import fake_player, write_master - from nnnotes import cli + from tests.site.test_regions import fake_player, write_master master = write_master(tmp_path / "m") # music 1: easy and expert monkeypatch.setenv("NNNOTES_SERVERS_ZZ_CDN", (tmp_path / "cdn").as_uri()) @@ -474,7 +473,7 @@ def test_web_pair_that_no_region_has_is_a_usage_error(tmp_path, capsys, monkeypa def test_unknown_pairs_over_the_regions_of_the_site(tmp_path): - from test_regions import config, write_master + from tests.site.test_regions import config, write_master tw = write_master(tmp_path / "tw") en = write_master(tmp_path / "en", extra_music={"_hardID": 11}) # en also has 1:hard diff --git a/tests/test_regions.py b/tests/site/test_regions.py similarity index 97% rename from tests/test_regions.py rename to tests/site/test_regions.py index 90fbeb6..99f376b 100644 --- a/tests/test_regions.py +++ b/tests/site/test_regions.py @@ -6,9 +6,11 @@ import pytest -from nnnotes import languages, liveui, score, sitefiles, siteindex, web -from nnnotes.config import Config, ConfigError -from nnnotes.session import Session +from nnnotes.content import liveui, score +from nnnotes.site import sitefiles, siteindex, web +from nnnotes.source import languages +from nnnotes.source.config import Config, ConfigError +from nnnotes.source.session import Session LANGS = list(languages.LANGUAGES) @@ -165,10 +167,10 @@ def test_region_groups_follow_the_chart_tables(tmp_path): def test_live_tables_cover_the_chart_build(): - src = Path(web.__file__).parent + content = Path(web.__file__).parents[1] / "content" named = set() for mod in ("live", "liveaudio", "livescene", "livenotes", "liveui", "score"): - named |= set(re.findall(r'"(Master[A-Za-z]+)"', (src / f"{mod}.py").read_text(encoding="utf-8"))) + named |= set(re.findall(r'"(Master[A-Za-z]+)"', (content / f"{mod}.py").read_text(encoding="utf-8"))) assert named <= set(web.LIVE_TABLES), sorted(named - set(web.LIVE_TABLES)) diff --git a/tests/test_story_site.py b/tests/site/test_story_site.py similarity index 99% rename from tests/test_story_site.py rename to tests/site/test_story_site.py index f17ef14..c1640a7 100644 --- a/tests/test_story_site.py +++ b/tests/site/test_story_site.py @@ -8,8 +8,10 @@ import pytest -from nnnotes import jsonio, sitefiles, siteindex, storysite, tmpfont, webmodel -from nnnotes.config import Config, ConfigError +from nnnotes.core import jsonio +from nnnotes.site import sitefiles, siteindex, storysite, webmodel +from nnnotes.source.config import Config, ConfigError +from nnnotes.unity import tmpfont def box(kind: bytes, payload: bytes) -> bytes: diff --git a/tests/test_web.py b/tests/site/test_web.py similarity index 99% rename from tests/test_web.py rename to tests/site/test_web.py index ee8859e..f718453 100644 --- a/tests/test_web.py +++ b/tests/site/test_web.py @@ -4,8 +4,8 @@ import pytest -from nnnotes import compress, sitefiles, siteindex, web -from nnnotes.config import ConfigError +from nnnotes.site import compress, sitefiles, siteindex, web +from nnnotes.source.config import ConfigError DOC = sitefiles._minify({"rows": [{"id": i, "name": f"row {i}", "tags": ["a", "b"]} for i in range(200)]}).encode() diff --git a/tests/test_webmodel.py b/tests/site/test_webmodel.py similarity index 99% rename from tests/test_webmodel.py rename to tests/site/test_webmodel.py index 5e29be3..e037c59 100644 --- a/tests/test_webmodel.py +++ b/tests/site/test_webmodel.py @@ -6,10 +6,13 @@ import pytest -from nnnotes import cli, jsonio, live2d, sitefiles, siteindex, web, webmodel -from nnnotes import player as nnnotes_player -from nnnotes.config import Config, ConfigError -from nnnotes.session import Session +from nnnotes import cli +from nnnotes.core import jsonio +from nnnotes.site import sitefiles, siteindex, web, webmodel +from nnnotes.source.config import Config, ConfigError +from nnnotes.source.session import Session +from nnnotes.unity import live2d +from nnnotes.unity import player as nnnotes_player KEY_A = "Character/Live2D/001_adv/adv_model_a/model/adv_model_a" KEY_B = "Character/Live2D/sub_x/adv_model_b/model/adv_model_b" @@ -896,7 +899,7 @@ def exists(p): def test_story_command_exports_its_models_next_to_the_story(tmp_path, capsys, monkeypatch): - from nnnotes import story + from nnnotes.content import story player = fake_export(monkeypatch) monkeypatch.setattr(Session, "catalog", lambda self, bundles=True, region=None: None) diff --git a/tests/source/__init__.py b/tests/source/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_apkset.py b/tests/source/test_apkset.py similarity index 96% rename from tests/test_apkset.py rename to tests/source/test_apkset.py index c8c2cb2..8405555 100644 --- a/tests/test_apkset.py +++ b/tests/source/test_apkset.py @@ -6,10 +6,10 @@ from unittest.mock import Mock import pytest -from test_jp import cat_bytes, zip_bytes -from nnnotes import apkset -from nnnotes.catalog import APK_AA_DIR, APK_CATALOG, Catalog +from nnnotes.source import apkset +from nnnotes.source.catalog import APK_AA_DIR, APK_CATALOG, Catalog +from tests.source.test_jp import cat_bytes, zip_bytes @pytest.fixture(autouse=True) diff --git a/tests/test_catalog.py b/tests/source/test_catalog.py similarity index 97% rename from tests/test_catalog.py rename to tests/source/test_catalog.py index 0ca942d..795ce92 100644 --- a/tests/test_catalog.py +++ b/tests/source/test_catalog.py @@ -1,11 +1,11 @@ import zipfile import pytest -import synth -from nnnotes.addressables import BundleKey, decrypt, parse, remote_path -from nnnotes.browse import browse -from nnnotes.catalog import APK_OFFSET_BASE, Catalog +from nnnotes.cli.browse import browse +from nnnotes.source.addressables import BundleKey, decrypt, parse, remote_path +from nnnotes.source.catalog import APK_OFFSET_BASE, Catalog +from tests.helpers import synth KEY = BundleKey(synth.BUNDLE_KEY, synth.BUNDLE_SEED) @@ -162,7 +162,7 @@ def test_downloads_keep_one_connection_per_thread(monkeypatch): import http.server import threading - from nnnotes import catalog + from nnnotes.source import catalog for v in ("http_proxy", "HTTP_PROXY", "all_proxy", "ALL_PROXY"): monkeypatch.delenv(v, raising=False) diff --git a/tests/test_catalog_lazy.py b/tests/source/test_catalog_lazy.py similarity index 96% rename from tests/test_catalog_lazy.py rename to tests/source/test_catalog_lazy.py index d6c1362..36a3ba9 100644 --- a/tests/test_catalog_lazy.py +++ b/tests/source/test_catalog_lazy.py @@ -5,12 +5,13 @@ from pathlib import Path import pytest -import synth -from nnnotes import catalog, cli -from nnnotes.addressables import BundleKey -from nnnotes.catalog import Catalog -from nnnotes.config import ConfigError +from nnnotes import cli +from nnnotes.source import catalog +from nnnotes.source.addressables import BundleKey +from nnnotes.source.catalog import Catalog +from nnnotes.source.config import ConfigError +from tests.helpers import synth KEY = BundleKey(synth.BUNDLE_KEY, synth.BUNDLE_SEED) ENTRIES = [ @@ -131,7 +132,7 @@ def test_load_reads_the_cdn_only_for_an_uncached_catalog(tmp_path): def test_write_atomic_temporary_name_is_short_and_unique(tmp_path, monkeypatch): - from nnnotes import cache + from nnnotes.core import cache long_name = "b" * 244 + ".bundle" # 251 characters: name + pid + thread id + .part > 255 dst = tmp_path / long_name @@ -155,7 +156,7 @@ def test_write_atomic_temporary_name_is_short_and_unique(tmp_path, monkeypatch): def test_write_atomic_removes_the_temporary_file_on_failure(tmp_path, monkeypatch): - from nnnotes import cache + from nnnotes.core import cache def refuse(a, b): raise PermissionError("in use") @@ -167,7 +168,7 @@ def refuse(a, b): def test_disk_cache_entries_use_the_short_temporary_name(tmp_path): - from nnnotes import cache + from nnnotes.core import cache p = tmp_path / "ab" / ("c" * 64) assert cache._disk_write(p, b"payload") and cache._disk_read(p) == b"payload" diff --git a/tests/test_catalog_versions.py b/tests/source/test_catalog_versions.py similarity index 94% rename from tests/test_catalog_versions.py rename to tests/source/test_catalog_versions.py index d3768cc..8ce0b59 100644 --- a/tests/test_catalog_versions.py +++ b/tests/source/test_catalog_versions.py @@ -8,11 +8,15 @@ from types import SimpleNamespace import pytest -import synth -from nnnotes import browse, catalog, catalogdb, cli, configfile, deckdata, gameapi -from nnnotes.config import Config, ConfigError -from nnnotes.session import Session +from nnnotes import cli +from nnnotes.cli import browse +from nnnotes.musicdata import deckdata +from nnnotes.pipeline import catalogdb +from nnnotes.source import catalog, configfile, gameapi +from nnnotes.source.config import Config, ConfigError +from nnnotes.source.session import Session +from tests.helpers import synth MODEL = "Character/Live2D/test/new/model/new" EPISODE = "Adv/Episode/event_new/event_new" @@ -36,7 +40,7 @@ def config(tmp_path, *, api=True, version=None, region="tw"): def test_auto_discovery_ignores_stale_main_and_advances_resource_version(tmp_path, monkeypatch): - from nnnotes.webmodel import catalog_models + from nnnotes.site.webmodel import catalog_models cfg = config(tmp_path) cache = cfg.require_path("paths", "cache") @@ -158,8 +162,8 @@ def test_build_workers_inherit_the_version_pin(tmp_path, monkeypatch): command = [ sys.executable, "-c", - "from nnnotes.config import Config; from nnnotes.session import Session; " - "from nnnotes.webmodel import catalog_models; c=Session(Config.load()).catalog(bundles=False); " + "from nnnotes.source.config import Config; from nnnotes.source.session import Session; " + "from nnnotes.site.webmodel import catalog_models; c=Session(Config.load()).catalog(bundles=False); " "assert 'new' in catalog_models(c); print(c.resource_version)", ] child = subprocess.run(command, env=os.environ.copy(), text=True, capture_output=True, check=True) diff --git a/tests/test_config.py b/tests/source/test_config.py similarity index 98% rename from tests/test_config.py rename to tests/source/test_config.py index 6ffed0b..5472dd1 100644 --- a/tests/test_config.py +++ b/tests/source/test_config.py @@ -3,8 +3,8 @@ import pytest -from nnnotes import config -from nnnotes.config import Config, ConfigError, env_name +from nnnotes.source import config +from nnnotes.source.config import Config, ConfigError, env_name KEY_HEX = bytes(range(16)).hex() diff --git a/tests/test_configfile.py b/tests/source/test_configfile.py similarity index 98% rename from tests/test_configfile.py rename to tests/source/test_configfile.py index 7a8bf4b..79c5b12 100644 --- a/tests/test_configfile.py +++ b/tests/source/test_configfile.py @@ -7,10 +7,11 @@ import pytest -from nnnotes import cli, configfile -from nnnotes.config import ConfigError, template +from nnnotes import cli +from nnnotes.source import configfile +from nnnotes.source.config import ConfigError, template -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[2] KEY = bytes(range(16)).hex() MKEY = bytes(range(32)).hex() diff --git a/tests/test_gameapi.py b/tests/source/test_gameapi.py similarity index 99% rename from tests/test_gameapi.py rename to tests/source/test_gameapi.py index 51bebab..de0349a 100644 --- a/tests/test_gameapi.py +++ b/tests/source/test_gameapi.py @@ -8,11 +8,12 @@ import grpc import pytest -import synth -from nnnotes import cli, gameapi -from nnnotes.config import Config, ConfigError -from nnnotes.session import Session +from nnnotes import cli +from nnnotes.source import gameapi +from nnnotes.source.config import Config, ConfigError +from nnnotes.source.session import Session +from tests.helpers import synth API = "https://api.example.test" CDN_A, CDN_B = "https://cdn-a.example.test/prod/a1", "https://cdn-b.example.test/prod/a1" diff --git a/tests/test_jp.py b/tests/source/test_jp.py similarity index 94% rename from tests/test_jp.py rename to tests/source/test_jp.py index cee4e42..0d96c92 100644 --- a/tests/test_jp.py +++ b/tests/source/test_jp.py @@ -10,16 +10,19 @@ from types import SimpleNamespace import pytest -import synth -from test_gameapi import server as grpc_server -from test_gameapi import version_response -from nnnotes import addressables, catalogdb, cli, configfile, deckdata, gameapi, jp -from nnnotes.apkset import ApkSet -from nnnotes.catalog import Catalog, file_name, location_kind -from nnnotes.cli_assets import CatalogFetcher -from nnnotes.config import Config, ConfigError -from nnnotes.session import Session +from nnnotes import cli +from nnnotes.cli.cli_assets import CatalogFetcher +from nnnotes.musicdata import deckdata +from nnnotes.pipeline import catalogdb +from nnnotes.source import addressables, configfile, gameapi, jp +from nnnotes.source.apkset import ApkSet +from nnnotes.source.catalog import Catalog, file_name, location_kind +from nnnotes.source.config import Config, ConfigError +from nnnotes.source.session import Session +from tests.helpers import synth +from tests.source.test_gameapi import server as grpc_server +from tests.source.test_gameapi import version_response HASH = "a" * 32 MASTER = "1.0.0.100/" + "b" * 32 @@ -279,14 +282,14 @@ def test_catalogdb_source_identity_and_fetcher(upstream, tmp_path): def test_apk_update_with_same_asset_snapshot_does_not_reuse_embedded_cache(tmp_path): - from nnnotes.catalog import APK_AA_DIR, APK_CATALOG + from nnnotes.source.catalog import APK_AA_DIR, APK_CATALOG source = jp.Source("jp", "1.0", HASH, "https://cdn.invalid") apk = tmp_path / "base.apk" cache = tmp_path / "cache" db = catalogdb.CatalogDB(tmp_path / "store") cfg = Config({"catalog": {"region": "jp"}, "paths": {"apk": str(apk)}}, environ={}) - from nnnotes.cli_assets import Workspace + from nnnotes.cli.cli_assets import Workspace for revision in (1, 2): local = synth.CatalogWriter().build( @@ -322,7 +325,7 @@ def local_catalog(revision=1): @pytest.mark.parametrize("apk_state", ["unset", "missing", "different", "broken"]) @pytest.mark.parametrize("kind", ["bundles", "rawFiles"]) def test_remote_replay_does_not_open_the_apk(upstream, tmp_path, apk_state, kind): - from nnnotes.catalog import APK_CATALOG + from nnnotes.source.catalog import APK_CATALOG cfg = upstream.cfg cfg._data["bundle"] = {"key": "00" * 16, "nonce_seed": "11"} @@ -347,7 +350,7 @@ def test_remote_replay_does_not_open_the_apk(upstream, tmp_path, apk_state, kind @pytest.mark.parametrize("kind", ["bundles", "rawFiles"]) @pytest.mark.parametrize("state", ["matching", "different", "no-catalog", "cached"]) def test_local_replay_validates_only_on_cache_miss(tmp_path, kind, state): - from nnnotes.catalog import APK_AA_DIR, APK_CATALOG + from nnnotes.source.catalog import APK_AA_DIR, APK_CATALOG source = jp.Source("jp", "1.0", HASH, "https://cdn.invalid") apk = tmp_path / "base.apk" @@ -431,7 +434,8 @@ def test_per_region_apk_and_client_settings(tmp_path): @pytest.mark.parametrize("kind", ["--all-live2d", "--all-stories"]) @pytest.mark.parametrize("region", ["jp", "tw"]) def test_web_preflight_uses_the_selected_regions_apk(tmp_path, monkeypatch, kind, region): - from nnnotes import sitefiles, storysite, tmpfont, webmodel + from nnnotes.site import sitefiles, storysite, webmodel + from nnnotes.unity import tmpfont conf = tmp_path / "nnnotes.toml" default = "jp" if region == "tw" else "tw" @@ -460,7 +464,7 @@ def build(out, selection, cfg, *args, **kwargs): def test_datapack_is_loaded_in_the_boot_environment(tmp_path, monkeypatch): - from nnnotes import player + from nnnotes.unity import player boot = SimpleNamespace( objects=[ @@ -487,7 +491,7 @@ def test_datapack_is_loaded_in_the_boot_environment(tmp_path, monkeypatch): def test_jp_master_manifest_checks_before_writing(tmp_path): - from nnnotes import master + from nnnotes.source import master manifest = {"version": MASTER, "files": [{"name": "../secret.bin", "hash": "a" * 64, "size": 128}]} seen = [] @@ -503,7 +507,7 @@ def get(url): @pytest.mark.parametrize("files", [None, [], {}, [None]]) def test_jp_master_rejects_missing_or_malformed_file_lists(tmp_path, files): - from nnnotes import master + from nnnotes.source import master with pytest.raises(master.DownloadError, match="manifest"): master.download( @@ -517,7 +521,7 @@ def test_jp_master_rejects_missing_or_malformed_file_lists(tmp_path, files): def test_export_cache_locator_uses_source_namespace(tmp_path): - from nnnotes.cli_assets import Workspace + from nnnotes.cli.cli_assets import Workspace source = jp.Source("jp", "1.0.0.300", HASH, "https://cdn.invalid") workspace = Workspace.__new__(Workspace) @@ -529,9 +533,8 @@ def test_export_cache_locator_uses_source_namespace(tmp_path): def test_music_data_preserves_asset_hash(tmp_path): - from test_musicdata import CHARTS, KEY, PROV, bgm, master_dir - from nnnotes import musicdata + from tests.musicdata.test_musicdata import CHARTS, KEY, PROV, bgm, master_dir output = tmp_path / "music.json" musicdata.export( @@ -547,7 +550,7 @@ def test_music_data_preserves_asset_hash(tmp_path): def test_directory_hca_key_tracks_replaced_base_apk(tmp_path, monkeypatch): - from nnnotes import cri + from nnnotes.formats import cri apk = tmp_path / "base.apk" apk.write_bytes(b"first") diff --git a/tests/test_master.py b/tests/source/test_master.py similarity index 98% rename from tests/test_master.py rename to tests/source/test_master.py index 95ba8c5..959b187 100644 --- a/tests/test_master.py +++ b/tests/source/test_master.py @@ -3,10 +3,10 @@ import numpy as np import pytest -import synth -from nnnotes import master -from nnnotes.master import MasterKey +from nnnotes.source import master +from nnnotes.source.master import MasterKey +from tests.helpers import synth KEY = MasterKey(synth.MASTER_KEY, synth.MASTER_IV) TABLE = synth.MASTER_TABLE diff --git a/tests/unity/__init__.py b/tests/unity/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_blob_memo.py b/tests/unity/test_blob_memo.py similarity index 98% rename from tests/test_blob_memo.py rename to tests/unity/test_blob_memo.py index 85e0b3c..3e2a212 100644 --- a/tests/test_blob_memo.py +++ b/tests/unity/test_blob_memo.py @@ -6,7 +6,8 @@ from UnityPy.helpers.TypeTreeNode import TypeTreeNode, get_common_strings from UnityPy.streams import EndianBinaryReader, EndianBinaryWriter -from nnnotes import cache, unity +from nnnotes.core import cache +from nnnotes.unity import reader as unity ORIGINAL = TypeTreeNode.parse_blob.__func__._original COMMON = list(get_common_strings().values()) diff --git a/tests/test_fontsettings.py b/tests/unity/test_fontsettings.py similarity index 98% rename from tests/test_fontsettings.py rename to tests/unity/test_fontsettings.py index 61f4d95..2a10e0e 100644 --- a/tests/test_fontsettings.py +++ b/tests/unity/test_fontsettings.py @@ -4,8 +4,8 @@ import pytest -from nnnotes import export, fontsettings -from nnnotes.fontsettings import END_OF_TEXT, SPACE +from nnnotes.unity import export, fontsettings +from nnnotes.unity.fontsettings import END_OF_TEXT, SPACE FACE = {"m_PointSize": 40.0, "m_Scale": 1.0} STYLE = { diff --git a/tests/test_mesh_texture_guards.py b/tests/unity/test_mesh_texture_guards.py similarity index 98% rename from tests/test_mesh_texture_guards.py rename to tests/unity/test_mesh_texture_guards.py index 8e251dd..2a42cc8 100644 --- a/tests/test_mesh_texture_guards.py +++ b/tests/unity/test_mesh_texture_guards.py @@ -6,7 +6,8 @@ import pytest from UnityPy.enums import TextureFormat -from nnnotes import export, unity +from nnnotes.unity import export +from nnnotes.unity import reader as unity def fake_handler(monkeypatch, **fields): diff --git a/tests/test_player.py b/tests/unity/test_player.py similarity index 99% rename from tests/test_player.py rename to tests/unity/test_player.py index f9d9c36..cd81d40 100644 --- a/tests/test_player.py +++ b/tests/unity/test_player.py @@ -5,7 +5,7 @@ import pytest -from nnnotes.player import PlayerData +from nnnotes.unity.player import PlayerData def bare_player() -> PlayerData: diff --git a/tests/test_speed_exact.py b/tests/unity/test_speed_exact.py similarity index 98% rename from tests/test_speed_exact.py rename to tests/unity/test_speed_exact.py index 71c9f7f..24fa7c5 100644 --- a/tests/test_speed_exact.py +++ b/tests/unity/test_speed_exact.py @@ -18,9 +18,12 @@ from PIL import Image from UnityPy.helpers.TypeTreeNode import TypeTreeNode -from nnnotes import cache, cri, encoders, export, jsonio, shader, unity -from nnnotes.export import Exporter, TexelPacker -from nnnotes.unity import SceneGraph +from nnnotes.core import cache, jsonio +from nnnotes.formats import cri, encoders, shader +from nnnotes.unity import export +from nnnotes.unity import reader as unity +from nnnotes.unity.export import Exporter, TexelPacker +from nnnotes.unity.reader import SceneGraph @pytest.fixture(autouse=True) diff --git a/tests/test_textstyle.py b/tests/unity/test_textstyle.py similarity index 98% rename from tests/test_textstyle.py rename to tests/unity/test_textstyle.py index a085312..6e15b11 100644 --- a/tests/test_textstyle.py +++ b/tests/unity/test_textstyle.py @@ -4,8 +4,10 @@ import pytest -from nnnotes import advui, fontsettings, languages, textstyle, tmpfont -from nnnotes.config import ConfigError +from nnnotes.content import advui +from nnnotes.source import languages +from nnnotes.source.config import ConfigError +from nnnotes.unity import fontsettings, textstyle, tmpfont WHITE = {"r": 1.0, "g": 1.0, "b": 1.0, "a": 1.0} DARK = {"r": 0.2, "g": 0.2, "b": 0.2, "a": 0.8} diff --git a/tests/test_typetrees.py b/tests/unity/test_typetrees.py similarity index 98% rename from tests/test_typetrees.py rename to tests/unity/test_typetrees.py index e543dff..7a84d71 100644 --- a/tests/test_typetrees.py +++ b/tests/unity/test_typetrees.py @@ -7,13 +7,12 @@ from pathlib import Path import pytest -import synth from UnityPy.helpers.TypeTreeHelper import read_typetree from UnityPy.streams.EndianBinaryReader import EndianBinaryReader from nnnotes import cli -from nnnotes.config import ConfigError -from nnnotes.player import ( +from nnnotes.source.config import ConfigError +from nnnotes.unity.player import ( HEADER, TYPETREE_FORMAT, PlayerData, @@ -24,8 +23,9 @@ typetree_key, typetree_node, ) +from tests.helpers import synth -TYPETREE_DIR = resources.files("nnnotes").joinpath("typetrees") +TYPETREE_DIR = resources.files("nnnotes.unity").joinpath("typetrees") EMBEDDED = sorted(f.name[: -len(".json")] for f in TYPETREE_DIR.iterdir() if f.name.endswith(".json")) ALIGN = 0x4000 @@ -374,7 +374,7 @@ def gen_typetrees(): """scripts/gen_typetrees.py (a maintainer tool, not part of the package).""" if "gen_typetrees" not in sys.modules: spec = importlib.util.spec_from_file_location( - "gen_typetrees", Path(__file__).resolve().parents[1] / "scripts" / "gen_typetrees.py" + "gen_typetrees", Path(__file__).resolve().parents[2] / "scripts" / "gen_typetrees.py" ) mod = importlib.util.module_from_spec(spec) sys.modules["gen_typetrees"] = mod