diff --git a/CHANGELOG.md b/CHANGELOG.md index 338640f..2ccbec8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ schemas in `nnnotes.schemas` and the documented Python API. Before 1.0 a minor v ### Architecture - Site construction uses shared product requests, named depot catalogs and ordered layout publication. Player page collection and catalog model queries have dedicated interfaces. +- Conversion memoization is bounded within each process, with persistent reuse provided by depot task results. ## [0.2.1](https://github.com/MetaSekaiLab/nnnotes/compare/v0.2.0...v0.2.1) - 2026-10-13 @@ -207,4 +208,3 @@ schemas in `nnnotes.schemas` and the documented Python API. Before 1.0 a minor v - List MetaMiku as an author ([cc43e13](https://github.com/MetaSekaiLab/nnnotes/commit/cc43e132875206d02d8665b19c06b27255ab0037)) - Point the project links at MetaSekaiLab/nnnotes ([a2b8304](https://github.com/MetaSekaiLab/nnnotes/commit/a2b83040bcf06771028bbfd5e90c10c225bde3e9)) - English package readme; check commits on pull requests only ([e602dd5](https://github.com/MetaSekaiLab/nnnotes/commit/e602dd5ebf6dda0f18da5ace54a02dbb893a796f)) - diff --git a/docs/architecture.md b/docs/architecture.md index 3b5056c..385d351 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -44,6 +44,8 @@ decrypted bundles in the depot's `downloads/` directory. The site's `Builder` composes product snapshots, catalog indexes and player page files. Product assembly and conversion belong to the pipeline and content layers. `catalog_models` queries model subjects from a source catalog. +Converters use bounded process-local memoization for repeated reads and computations. Persistent reuse is keyed by depot tasks and their content inputs. Clearing a process memo releases its values; a later task can reuse its committed depot result. + `export episode`, `export shader`, `export audio` and `music-data` expose subject-specific data and conversions through the Session. All commands use the readers and converters below the pipeline layer. ## Outputs diff --git a/src/nnnotes/content/storyfonts.py b/src/nnnotes/content/storyfonts.py index 213af91..f60380f 100644 --- a/src/nnnotes/content/storyfonts.py +++ b/src/nnnotes/content/storyfonts.py @@ -41,7 +41,7 @@ has no sprite asset. Same inputs (font file, episode, game data) and library versions give the same bytes; generated glyph cells are kept -in the process cache (disk layer when the build configures one) by font file, sizes and glyph. +in the process cache by font file, sizes and glyph. """ from __future__ import annotations @@ -85,7 +85,7 @@ UI_RUBY_FIELDS = ("_rubyMarginTop",) KOREAN_ADJUST_CLASS = "LocalizeKoreanAdjust" # the font style a text gets in Korean (LocalizeKoreanAdjust.Apply) UI_TEXT_CLASSES = ("UIText", "UIRubyText") # Fwk.UI.UIText and its subclass -GLYPHS = cache.bucket("storyglyphs", disk=True) +GLYPHS = cache.bucket("storyglyphs") FACE_KEYS = ( "m_FaceIndex", "m_FamilyName", diff --git a/src/nnnotes/core/cache.py b/src/nnnotes/core/cache.py index e1ee210..6d98961 100644 --- a/src/nnnotes/core/cache.py +++ b/src/nnnotes/core/cache.py @@ -2,14 +2,11 @@ A bucket maps a content key to a value computed from exactly the inputs the key covers, so a hit returns what the computation would return: the extractors' outputs stay byte-identical with or without the cache. Buckets live in the -process (LRU, bounded by a byte budget, thread-safe). Buckets that hold bytes can also keep them on disk -(`configure(directory=...)`): processes of one build then share the results, and a later run with the same inputs -reuses them. Keys of the disk layer are salted with the source of the module that produces the value and the -versions of the libraries it calls, so a code or library change never reads an older result. +process (LRU, bounded by a byte budget, thread-safe). Persistent task results belong to the depot. from nnnotes.core import cache cache.configure(enabled=False) # every get misses, nothing is stored - cache.configure(memory_mb=512, directory="work/cache") + cache.configure(memory_mb=512) PNG = cache.bucket("png", salt=cache.source_salt(__file__)) k = PNG.key(data, width, height) png = PNG.get(k) @@ -30,7 +27,6 @@ from pathlib import Path from typing import Any, Protocol, TypedDict, cast -DISK_MAGIC = b"NNC1" DEFAULT_MEMORY_MB = 128 @@ -39,7 +35,6 @@ class Settings(TypedDict): enabled: bool memory: int # byte budget of each bucket - directory: Path | None # disk layer of the byte buckets closures: int # loaded bundle closures kept per thread closure_bytes: int # their bundle bytes @@ -47,7 +42,6 @@ class Settings(TypedDict): _settings: Settings = { "enabled": True, "memory": DEFAULT_MEMORY_MB << 20, - "directory": None, "closures": 4, "closure_bytes": 512 << 20, } @@ -59,20 +53,17 @@ def configure( *, enabled: bool | None = None, memory_mb: int | None = None, - directory: str | os.PathLike[str] | None = None, closures: int | None = None, closure_mb: int | None = None, ) -> Settings: - """Set the caches of this process: `enabled`; `memory_mb` (byte budget of each bucket); `directory` (disk layer - of the byte buckets; "" or None for none); `closures` / `closure_mb` (loaded bundle closures kept by + """Set the caches of this process: `enabled`; `memory_mb` (byte budget of each bucket); + `closures` / `closure_mb` (loaded bundle closures kept by unity.load_closure, per thread: count and bundle bytes). Clears the buckets. Returns the settings.""" with _lock: if enabled is not None: _settings["enabled"] = bool(enabled) if memory_mb is not None: _settings["memory"] = max(0, int(memory_mb)) << 20 - if directory is not None: - _settings["directory"] = Path(directory) if directory else None if closures is not None: _settings["closures"] = max(0, int(closures)) if closure_mb is not None: @@ -91,18 +82,18 @@ def enabled() -> bool: return _settings["enabled"] -def bucket(name: str, *, salt: str = "", disk: bool = False, max_item: int | None = None) -> Bucket: - """The process's bucket `name` (created on first use). `disk`: its values are bytes and may be kept on disk; +def bucket(name: str, *, salt: str = "", max_item: int | None = None) -> Bucket: + """The process's bucket `name` (created on first use). `max_item`: values larger than this many bytes are not kept (default: a quarter of the budget).""" with _lock: b = _buckets.get(name) if b is None: - b = _buckets[name] = Bucket(name, salt, disk, max_item) + b = _buckets[name] = Bucket(name, salt, max_item) return b def stats() -> dict[str, dict[str, int]]: - """Per bucket: items, bytes, hits, misses, disk hits, disk writes, disk write errors.""" + """Per bucket: items, bytes, hits and misses.""" return {n: b.stats() for n, b in sorted(_buckets.items())} @@ -174,12 +165,12 @@ def file_id(path: str | os.PathLike[str]) -> tuple[str, int, int]: class Bucket: """A memo cache of values by content key (see the module documentation).""" - def __init__(self, name: str, salt: str, disk: bool, max_item: int | None): - self.name, self.salt, self.disk, self._max_item = name, salt, disk, max_item + def __init__(self, name: str, salt: str, max_item: int | None): + self.name, self.salt, self._max_item = name, salt, max_item self._items: OrderedDict[str, tuple[Any, int]] = OrderedDict() # key -> (value, byte size) self._bytes = 0 self._lock = threading.Lock() - self.hits = self.misses = self.disk_hits = self.disk_writes = self.disk_errors = 0 + self.hits = self.misses = 0 def key(self, *parts: object) -> str: return key(self.name, self.salt, parts) @@ -198,32 +189,16 @@ def get(self, k: str) -> Any: self._items.move_to_end(k) self.hits += 1 return hit[0] - directory = _settings["directory"] - if self.disk and directory is not None: - data = _disk_read(self._path(directory, k)) - if data is not None: - with self._lock: - self.disk_hits += 1 - self._remember(k, data, len(data)) - return data with self._lock: self.misses += 1 return None def put(self, k: str, value: Any, size: int | None = None) -> None: - """Store `value` (bytes for a disk bucket) under `k`; `size`: its byte size (default len(value)).""" + """Store `value` under `k`; `size`: its byte size (default len(value)).""" if not _settings["enabled"]: return n = len(value) if size is None else int(size) self._remember(k, value, n) - directory = _settings["directory"] - if self.disk and directory is not None and n <= max(self._limit(), 64 << 20): - ok = _disk_write(self._path(directory, k), value) - with self._lock: - if ok: - self.disk_writes += 1 - else: - self.disk_errors += 1 def _remember(self, k: str, value: Any, n: int) -> None: if n > self._limit(): @@ -239,9 +214,6 @@ def _remember(self, k: str, value: Any, n: int) -> None: _, (_, m) = self._items.popitem(last=False) self._bytes -= m - def _path(self, directory: Path, k: str) -> Path: - return directory / self.name / k[:2] / k - def clear(self) -> None: with self._lock: self._items.clear() @@ -254,23 +226,9 @@ def stats(self) -> dict[str, int]: "bytes": self._bytes, "hits": self.hits, "misses": self.misses, - "diskHits": self.disk_hits, - "diskWrites": self.disk_writes, - "diskErrors": self.disk_errors, } -def _disk_read(p: Path) -> bytes | None: - try: - raw = p.read_bytes() - except OSError: - return None - if len(raw) < 12 or raw[:4] != DISK_MAGIC: - return None - (n,) = struct.unpack_from(" Path: """A new temporary file name next to `dst` for writing it atomically: short whatever `dst`'s name (a long name plus a suffix would pass the file system's name limit), random, so unique across processes and threads.""" @@ -293,20 +251,9 @@ def write_atomic(dst: Path, data: bytes) -> None: raise -def _disk_write(p: Path, data: bytes) -> bool: - """Write a disk entry atomically; False when the file system refused (counted as diskErrors: a cache that - cannot write only misses later, the result is the same).""" - try: - p.parent.mkdir(parents=True, exist_ok=True) - write_atomic(p, DISK_MAGIC + struct.pack(" bytes: - """One bytes value from a header and blobs (for disk buckets whose value is several files).""" + """One bytes value from a header and blobs.""" out = [struct.pack(" ACB / AWB bytes, layout _keys: dict[tuple, int] = {} _keys_lock = threading.Lock() diff --git a/src/nnnotes/formats/shader.py b/src/nnnotes/formats/shader.py index 85ad550..5f82343 100644 --- a/src/nnnotes/formats/shader.py +++ b/src/nnnotes/formats/shader.py @@ -32,7 +32,7 @@ from ..core import cache from ..core.jsonio import dumps, write_json -DUMPS = cache.bucket("shaders", salt=cache.source_salt(__file__, "UnityPy"), disk=True) +DUMPS = cache.bucket("shaders", salt=cache.source_salt(__file__, "UnityPy")) # UnityEngine.Rendering ShaderCompilerPlatform value -> tag PLATFORM = { diff --git a/src/nnnotes/unity/export.py b/src/nnnotes/unity/export.py index ef961ca..8ba2707 100644 --- a/src/nnnotes/unity/export.py +++ b/src/nnnotes/unity/export.py @@ -55,8 +55,8 @@ HEADER = ("m_GameObject", "m_Script") _SCALARS = (int, float, str, bool, type(None)) _LIBS = ("UnityPy", "Pillow", "texture2ddecoder", "astc-encoder-py", "etcpak", "numpy") -# decoded textures and their PNG encodings by content (export.py's source and the libraries salt the disk layer) -PNG = cache.bucket("png", salt=cache.source_salt(__file__, *_LIBS), disk=True) +# decoded textures and PNG encodings memoized by content within this process +PNG = cache.bucket("png", salt=cache.source_salt(__file__, *_LIBS)) PIXELS = cache.bucket("pixels", salt=cache.source_salt(__file__, *_LIBS)) diff --git a/tests/core/test_cache.py b/tests/core/test_cache.py index 314ebc2..4ab16db 100644 --- a/tests/core/test_cache.py +++ b/tests/core/test_cache.py @@ -9,12 +9,11 @@ def fresh_cache(): """Every test starts with the default settings and empty buckets, and leaves them so.""" before = cache.settings() - cache.configure(enabled=True, memory_mb=cache.DEFAULT_MEMORY_MB, directory="") + cache.configure(enabled=True, memory_mb=cache.DEFAULT_MEMORY_MB) yield cache.configure( enabled=before["enabled"], memory_mb=before["memory"] >> 20, - directory=before["directory"] or "", closures=before["closures"], closure_mb=before["closure_bytes"] >> 20, ) @@ -66,22 +65,14 @@ def test_non_bytes_values_with_size(): assert b.get(b.key("o")) == {"a": 1} -def test_disk_layer_shared_and_checked(tmp_path): - cache.configure(directory=tmp_path / "c") - b = cache.bucket("t-disk", salt="s1", disk=True) +def test_bucket_clear_releases_values(): + b = cache.bucket("t-clear", salt="s1") k = b.key("v") b.put(k, b"payload") - files = [p for p in (tmp_path / "c").rglob("*") if p.is_file()] - assert len(files) == 1 and not files[0].name.endswith(".part") - cache.clear() # memory gone: the disk answers - assert b.get(k) == b"payload" and b.stats()["diskHits"] == 1 - other = cache.bucket("t-disk-other-salt", salt="s2", disk=True) - assert other.get(other.key("v")) is None # another salt: another key - files[0].write_bytes(files[0].read_bytes()[:-1]) # a truncated file is a miss + assert b.get(k) == b"payload" and b.stats()["items"] == 1 cache.clear() assert b.get(k) is None - files[0].write_bytes(b"garbage") - assert b.get(k) is None + assert b.stats()["items"] == b.stats()["bytes"] == 0 def test_pack_round_trip(): diff --git a/tests/source/test_catalog_lazy.py b/tests/source/test_catalog_lazy.py index c061698..048de6d 100644 --- a/tests/source/test_catalog_lazy.py +++ b/tests/source/test_catalog_lazy.py @@ -171,14 +171,6 @@ def refuse(a, b): assert list(tmp_path.iterdir()) == [] -def test_disk_cache_entries_use_the_short_temporary_name(tmp_path): - from nnnotes.core import cache - - p = tmp_path / "ab" / ("c" * 64) - assert cache._disk_write(p, b"payload") and cache._disk_read(p) == b"payload" - assert sorted(x.name for x in p.parent.iterdir()) == [p.name] - - # ---------------------------------------------------------------- the command line def run(argv, capsys): code = 0 diff --git a/tests/unity/test_speed_exact.py b/tests/unity/test_speed_exact.py index 45c9a4b..6f8c112 100644 --- a/tests/unity/test_speed_exact.py +++ b/tests/unity/test_speed_exact.py @@ -28,10 +28,10 @@ @pytest.fixture(autouse=True) def fresh_cache(): - cache.configure(enabled=True, memory_mb=cache.DEFAULT_MEMORY_MB, directory="") + cache.configure(enabled=True, memory_mb=cache.DEFAULT_MEMORY_MB) unity.clear_closures() yield - cache.configure(enabled=True, memory_mb=cache.DEFAULT_MEMORY_MB, directory="", closures=4, closure_mb=512) + cache.configure(enabled=True, memory_mb=cache.DEFAULT_MEMORY_MB, closures=4, closure_mb=512) unity.clear_closures() @@ -561,13 +561,15 @@ def test_decode_parallel_and_cached_equal_sequential(tmp_path, tools, fmt): assert (len(tools.calls) == n) == (run == "hit") # replayed: no tool ran -def test_decode_disk_cache_and_levels(tmp_path, tools): - cache.configure(directory=tmp_path / "c") +def test_decode_memo_and_levels(tmp_path, tools): cri.decode(None, "S", tmp_path / "a", key=0, flac_level=0) - cache.clear() # another process: the disk layer n = len(tools.calls) cri.decode(None, "S", tmp_path / "b", key=0, flac_level=0) assert len(tools.calls) == n and tree(tmp_path / "a") == tree(tmp_path / "b") + cache.clear() + cri.decode(None, "S", tmp_path / "fresh", key=0, flac_level=0) + assert len(tools.calls) > n and tree(tmp_path / "a") == tree(tmp_path / "fresh") + n = len(tools.calls) cri.decode(None, "S", tmp_path / "c12", key=0) # another level: decoded again assert len(tools.calls) > n assert tree(tmp_path / "a")["a.flac"] != tree(tmp_path / "c12")["a.flac"]