From 29a135009e79c92c32bef1d3e267acd2a2d88523 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Wed, 14 Oct 2026 21:53:17 +0800 Subject: [PATCH 1/2] refactor!: keep downloads, results and runs in one depot `[paths] depot` (`--depot`, `NNNOTES_PATHS_DEPOT`) names one directory for everything a configuration reads and computes: - `downloads/`: the game files as downloaded, under their own names (catalogs, decrypted bundles, raw CDN files, files read from the APK, one directory per catalog source); - `cas/`, `ac/`, `inputs/`, `catalogs/`, `runs/` and `costs/`: the results of the stages by content, their task results, input identities, catalog versions and run manifests. The `depot` command group replaces the `store` group and adds `depot gc`, which removes the runs, results and contents that no kept run reaches (`--keep N` latest runs per selection, `--dry-run`). Task inputs locate their bytes with the locator kinds `depot`, `download` (a path in `downloads/`), `file` and `catalog`. The catalog API speaks of downloads: `Catalog(catalog_bytes, download_dir)`, `Catalog.downloaded`, `Catalog.downloaded_raw`, `Catalog.file_name`, `jp.Source.download_dir`, and `jp.open_catalog` takes the downloads directory. Co-Authored-By: Claude Opus 5.5 --- README.md | 8 +- README.zh-CN.md | 8 +- docs/architecture.md | 10 +- docs/assets.md | 90 +++--- docs/cli.md | 38 ++- docs/commands.md | 31 +- docs/configuration.md | 35 ++- docs/contracts.md | 43 +-- docs/determinism.md | 6 +- docs/export.md | 2 +- docs/jp.md | 14 +- docs/music-data.md | 4 +- docs/quickstart.md | 6 +- docs/schemas.md | 6 +- docs/ui.md | 2 +- docs/views.md | 8 +- src/nnnotes/cli/__init__.py | 6 +- src/nnnotes/cli/assets.py | 151 +++++----- src/nnnotes/cli/browse.py | 21 +- src/nnnotes/cli/bundle.py | 4 +- src/nnnotes/cli/catalog.py | 26 +- src/nnnotes/cli/common.py | 8 +- src/nnnotes/cli/{store.py => depot.py} | 41 ++- src/nnnotes/cli/export.py | 8 +- src/nnnotes/cli/musicdata.py | 2 +- src/nnnotes/content/ui.py | 4 +- src/nnnotes/formats/cri.py | 2 +- src/nnnotes/musicdata/deckdata.py | 12 +- src/nnnotes/pipeline/__init__.py | 2 +- src/nnnotes/pipeline/catalogdb.py | 28 +- src/nnnotes/pipeline/census.py | 6 +- src/nnnotes/pipeline/contract.py | 2 +- src/nnnotes/pipeline/cristages.py | 84 +++--- src/nnnotes/pipeline/{store.py => depot.py} | 171 ++++++++--- src/nnnotes/pipeline/layout.py | 36 +-- src/nnnotes/pipeline/link.py | 28 +- src/nnnotes/pipeline/modelstages.py | 28 +- src/nnnotes/pipeline/objexport.py | 40 +-- src/nnnotes/pipeline/orchestrate.py | 50 ++-- src/nnnotes/pipeline/plan.py | 20 +- src/nnnotes/pipeline/stages.py | 48 +-- src/nnnotes/pipeline/views.py | 26 +- src/nnnotes/pipeline/voices.py | 42 +-- src/nnnotes/schemas/catalogs.schema.json | 2 +- src/nnnotes/schemas/common.schema.json | 6 +- src/nnnotes/schemas/music-data.schema.json | 2 +- src/nnnotes/schemas/run.schema.json | 2 +- src/nnnotes/site/sitefiles.py | 2 +- src/nnnotes/source/catalog.py | 78 ++--- src/nnnotes/source/jp.py | 15 +- src/nnnotes/source/nnnotes.example.toml | 12 +- src/nnnotes/source/session.py | 31 +- tests/cli/test_assets.py | 198 ++++++------ tests/cli/test_cli.py | 49 +-- tests/cli/test_voices.py | 12 +- tests/content/test_ui.py | 2 +- tests/musicdata/test_deckdata.py | 8 +- tests/musicdata/test_musicdata.py | 4 +- tests/pipeline/test_catalogdb.py | 32 +- tests/pipeline/test_census.py | 20 +- tests/pipeline/test_contract.py | 14 +- tests/pipeline/test_cristages.py | 168 +++++------ .../pipeline/{test_store.py => test_depot.py} | 58 ++-- tests/pipeline/test_depot_gc.py | 74 +++++ tests/pipeline/test_goldens.py | 42 +-- tests/pipeline/test_layout.py | 86 +++--- tests/pipeline/test_link.py | 50 ++-- tests/pipeline/test_modelstages.py | 114 +++---- tests/pipeline/test_objexport.py | 282 +++++++++--------- tests/pipeline/test_orchestrate.py | 36 +-- tests/pipeline/test_plan.py | 76 ++--- tests/pipeline/test_stages.py | 90 +++--- tests/pipeline/test_views.py | 70 ++--- tests/pipeline/test_voices.py | 80 ++--- tests/pipeline/test_voices_story.py | 46 +-- tests/site/test_live_options_web.py | 2 +- tests/site/test_pipeline.py | 8 +- tests/site/test_readsets.py | 2 +- tests/source/test_catalog.py | 24 +- tests/source/test_catalog_lazy.py | 46 +-- tests/source/test_catalog_versions.py | 40 +-- tests/source/test_configfile.py | 8 +- tests/source/test_jp.py | 64 ++-- 83 files changed, 1688 insertions(+), 1474 deletions(-) rename src/nnnotes/cli/{store.py => depot.py} (54%) rename src/nnnotes/pipeline/{store.py => depot.py} (75%) rename tests/pipeline/{test_store.py => test_depot.py} (83%) create mode 100644 tests/pipeline/test_depot_gc.py diff --git a/README.md b/README.md index 84ce615..d1836c4 100644 --- a/README.md +++ b/README.md @@ -47,13 +47,13 @@ questions: `nnnotes config init --user --no-input --set SECTION.KEY=VALUE ...`. | `config init\|set\|unset\|check\|path` | write, edit and check the config file | | `servers` | the game API's server list: regions with their CDN and API roots | | `catalog keys\|browse` | the addressable keys of a region's catalog; a local web page to browse catalogs and bundles | -| `catalog versions\|import\|fetch\|diff` | catalog versions kept in the store, and what changed between two | -| `bundle pull` | fetch and decrypt the bundle closure of keys into the cache | +| `catalog versions\|import\|fetch\|diff` | catalog versions kept in the depot, and what changed between two | +| `bundle pull` | fetch and decrypt the bundle closure of keys into the depot | | `master version\|download\|decode` | the master data version a region serves, its download and its decoding to JSON | -| `export assets` | every object of the selected bundles in common formats, through the content-addressed store | +| `export assets` | every object of the selected bundles in common formats, through the content-addressed depot | | `export episode\|story\|live\|live2d\|spot\|room\|shader\|audio\|ui` | one subject: an ADV episode, a story, a live, a Live2D model, a spot, a room, shaders, a cue sheet, the UI library | | `apk settings\|hca-key` | the player's graphics settings and the CRI HCA keycode of `base.apk` | -| `store plan\|run\|verify` | what `export assets` would run and why, single tasks, the store's check | +| `depot plan\|run\|verify\|gc` | what `export assets` would run and why, single tasks, the depot's check and collection | | `voices list\|search\|summary\|get` | the character and story voices of the master data | | `site build` | the static site of ournotes-player: charts, Live2D models and stories, in several regions and languages | | `music-data` | every song and chart in one JSON file, with the deck model's chart statistics | diff --git a/README.zh-CN.md b/README.zh-CN.md index 243f5f2..0d49d32 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -45,13 +45,13 @@ nnnotes site build -o out/site --all --player path/to/ournotes-player | `config init\|set\|unset\|check\|path` | 写入、修改和检查配置文件 | | `servers` | 游戏 API 的服务器列表：各区服及其 CDN 与 API 根地址 | | `catalog keys\|browse` | 区服 catalog 的资源键；在本地网页中浏览 catalog 与资源包 | -| `catalog versions\|import\|fetch\|diff` | store 中保存的 catalog 版本，以及两个版本之间的变化 | -| `bundle pull` | 下载并解密资源键的依赖闭包到缓存 | +| `catalog versions\|import\|fetch\|diff` | depot 中保存的 catalog 版本，以及两个版本之间的变化 | +| `bundle pull` | 下载并解密资源键的依赖闭包到 depot | | `master version\|download\|decode` | 区服当前的 masterdata 版本、下载，以及解码为 JSON | -| `export assets` | 经内容寻址的 store，把所选资源包中的每个对象导出为通用格式 | +| `export assets` | 经内容寻址的 depot，把所选资源包中的每个对象导出为通用格式 | | `export episode\|story\|live\|live2d\|spot\|room\|shader\|audio\|ui` | 单个对象：ADV 剧情集、完整剧情、谱面、Live2D 模型、据点、房间、着色器、cue sheet、UI 库 | | `apk settings\|hca-key` | `base.apk` 中的画面设置与 CRI HCA 密钥 | -| `store plan\|run\|verify` | `export assets` 将运行什么及原因、单独运行任务、检查 store | +| `depot plan\|run\|verify\|gc` | `export assets` 将运行什么及原因、单独运行任务、检查和回收 depot | | `voices list\|search\|summary\|get` | masterdata 中的角色语音与剧情语音 | | `site build` | ournotes-player 静态站点：谱面、Live2D 模型与剧情，支持多个区服与语言 | | `music-data` | 全部歌曲与谱面的单个 JSON 文件，附组卡模型的谱面统计 | diff --git a/docs/architecture.md b/docs/architecture.md index e259fe3..8b5646b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -10,7 +10,7 @@ subpackage of `nnnotes` and imports only the layers below it. |---|---|---| | Command line | `nnnotes.cli` | one module per command group; parses options, reads the configuration, prints summaries and `--json` reports | | Products | `nnnotes.site`, `nnnotes.musicdata` | the ournotes-player site (charts, models, stories, indexes); the music data file and its deck statistics | -| Pipeline | `nnnotes.pipeline` | stages, task descriptions and results, the content-addressed store, plans, layouts, the catalog versions, the views of the master data | +| Pipeline | `nnnotes.pipeline` | stages, task descriptions and results, the content-addressed depot, plans, layouts, the catalog versions, the views of the master data | | Content | `nnnotes.content` | the game's subjects as exports: ADV episodes and stories, lives and scores, spots, rooms, the UI library | | Unity objects | `nnnotes.unity` | bundle reading, the typetree exporter, the player data, Live2D prefabs, text styles and TextMesh Pro fonts | | File formats | `nnnotes.formats` | formats read from bytes: CRI ACB/AWB and HCA, USM movies, CRI Lips data, shaders, motions | @@ -27,12 +27,12 @@ and print, and the work they run is in the library modules below them. Every setting comes from the configuration ([sources and precedence](configuration.md#sources-and-precedence)). `nnnotes.source.session.Session` turns one configuration into the data sources a command needs: the catalog of a region, its decoded master data and the master data version it serves, the APK, the bundle and master data keys and -the store root. It reads a setting when it is first needed and raises a `ConfigError` naming the setting when it is missing or wrong; +the depot root. It reads a setting when it is first needed and raises a `ConfigError` naming the setting when it is missing or wrong; the command line turns that error into exit status 2. The catalog (`nnnotes.source.catalog.Catalog`) resolves an addressable key to its bundle and the bundle's dependency closure, takes a bundle from the APK when the APK has it and from the CDN otherwise, and keeps fetched and -decrypted bundles in the cache directory. +decrypted bundles in the depot's `downloads/` directory. ## Two ways to export @@ -42,8 +42,8 @@ decrypted bundles in the cache directory. - **Every asset.** `export assets` splits the work into stages ([stages](stages.md)). A task is one stage applied to one subject (a bundle, a content id, a view); its key covers exactly what its output depends on: the stage and converter versions, the input contents and the options ([contracts](contracts.md)). Results and their files are - stored once by content hash in the store, and the output directory is laid out from the store. A second run finds - every unchanged result in the store and only lays out what changed; `store plan` says what would run and why + stored once by content hash in the depot, and the output directory is laid out from the depot. A second run finds + every unchanged result in the depot and only lays out what changed; `depot plan` says what would run and why ([asset export](assets.md)). Both ways use the same readers below the pipeline layer, so a texture, shader, mesh or audio stream is converted by diff --git a/docs/assets.md b/docs/assets.md index 4e089bf..f52f1e6 100644 --- a/docs/assets.md +++ b/docs/assets.md @@ -5,29 +5,29 @@ meshes as binary glTF, text assets and fonts as their stored bytes, prefabs, mat class as JSON. It also writes the semantic views of the master data (cards, characters, stamps, …) with the files of the objects they name. The work is split into stages ([stages.md](stages.md)); each stage runs one task per subject (a bundle, a view, …), and a task's result is stored under a key that covers exactly what its output depends on -([contracts.md](contracts.md)). A second run over the same data finds every result in the store and only +([contracts.md](contracts.md)). A second run over the same data finds every result in the depot and only rewrites what changed. ``` nnnotes export assets -o out/assets nnnotes export assets -o out/cards --select group:membercard --views cards -nnnotes store plan --select key:Image/Jacket/ -nnnotes store plan --check && echo "up to date" +nnnotes depot plan --select key:Image/Jacket/ +nnnotes depot plan --check && echo "up to date" ``` ## Commands ``` nnnotes export assets -o OUT [--layout original[,cas]] [--select group:G|key:PREFIX|bundle:GLOB ...] [--views all|none|a,b] - [--store DIR] [--workers N] [--memory GiB] [--fetch-workers N] [--stored-catalog L|SHA] + [--depot DIR] [--workers N] [--memory GiB] [--fetch-workers N] [--stored-catalog L|SHA] [--png-level N] [--flac-level N] [--only-class C,...] [--link auto|clone|hard|copy] [--strict] [--dry-run] [--explain] -nnnotes store plan [the selection and parameters of export] [-o OUT] [--json] [--since RUN] [--check] [--census] +nnnotes depot plan [the selection and parameters of export] [-o OUT] [--json] [--since RUN] [--check] [--census] [--emit-tasks DIR] [--why TASK] -nnnotes store run TASK.json|DIR [...] [--store DIR] [--fetch] [--force] +nnnotes depot run TASK.json|DIR [...] [--depot DIR] [--fetch] [--force] nnnotes catalog versions [--json] | import FILE [--label L] [--apk-catalog FILE] [--resource-version V] | fetch [--label L] | diff OLD NEW [--json] -nnnotes store verify [--quick] [--workers N] +nnnotes depot verify [--quick] [--workers N] ``` ### export assets @@ -38,24 +38,24 @@ nnnotes store verify [--quick] [--workers N] | `--select SEL` | what to export (repeatable, the union; default everything): `group:G` the bundles of an Addressables group (the stable bundle name up to `_assets_`, else up to its first `_`), `key:PREFIX` the bundles and raw files the catalog keys under `PREFIX` load, `bundle:GLOB` the bundles and raw files whose stable name matches | | `--views` | the semantic views to build ([views.md](views.md)): `all`, `none` or names, comma-separated; default `all` when master data is set, else `none` | | `--layout` | `original`, `cas` or both (below) | -| `--store DIR` | the store (`[paths] store`; default `/store`) | +| `--depot DIR` | the depot (`[paths] depot`) | | `--workers N` | worker processes (default: the CPUs the process may run on, its affinity mask as `taskset` or a container's cpuset sets it; `0`: every task in this process) | | `--memory GiB` | memory budget (default: 80 % of the physical memory where it can be read, else none) | -| `--fetch-workers N` | parallel downloads of bundles not in the cache (default 8) | -| `--stored-catalog` | a catalog version of the store (`catalog versions`) instead of the current catalog | +| `--fetch-workers N` | parallel downloads of bundles not downloaded yet (default 8) | +| `--stored-catalog` | a catalog version of the depot (`catalog versions`) instead of the current catalog | | `--png-level N` | zlib level 0-9 of the PNG files (default 6); part of the task keys | | `--flac-level N` | FLAC compression level 0-12 of the CRI audio and the movies' audio (default 8; every level decodes to the same samples; 12 writes files 0.2-1.3 % smaller in about five times the encoding time); part of the task keys | | `--only-class C,...` | export only objects of these Unity classes; part of the task keys | -| `--link` | how the layout files are made from the store (below): `auto` (default; `[export] link`), `clone`, `hard` or `copy` | +| `--link` | how the layout files are made from the depot (below): `auto` (default; `[export] link`), `clone`, `hard` or `copy` | | `--strict` | exit 1 when a view has gaps: a required role whose address names no catalog key or no fitting sub-object (empty values and roles that do not apply are not gaps); in `voices`, a voice whose cue sheet has no catalog key or whose decoded sheet lacks its cue, and a story episode without a catalog key | | `--dry-run` | print the plan and stop | -| `--explain` | print every setting of the run first: store, context, selection, pipeline, stages not installed, parameters, workers, memory budget, recycling, layouts | +| `--explain` | print every setting of the run first: depot, context, selection, pipeline, stages not installed, parameters, workers, memory budget, recycling, layouts | A run resolves the context (region, language, catalog version) and indexes the catalog, identifies the selected -bundles (the sha256 of each decrypted bundle, remembered by path, size and modification time; bundles not in the -cache are fetched), takes the census of each bundle, links the scripts and addresses, exports the bundles, runs the +bundles (the sha256 of each decrypted bundle, remembered by path, size and modification time; bundles not +downloaded yet are fetched into the depot's `downloads/`), takes the census of each bundle, links the scripts and addresses, exports the bundles, runs the stages that need those results (atlas sprites, views), writes the layouts and the reports. The selected raw files -(CRI audio and movies) not in the cache are fetched while the bundle stages run. The census also covers +(CRI audio and movies) not downloaded yet are fetched while the bundle stages run. The census also covers the bundles the selected ones depend on, so script classes and atlas textures in other bundles resolve. Tasks run in spawned worker processes that stay alive between tasks (no more of them start than a stage has tasks to @@ -64,16 +64,16 @@ the running tasks and its own fits the budget, and a task estimated above the bu replaced after 500 tasks or when its resident memory after a task is above 3 GiB, or above 40 % of its part of the memory budget (the budget divided by the workers) when that is less, but not below 256 MiB: a worker keeps much of its largest task's memory, which the budget does not count (`--explain` prints the threshold). The estimates come -from each stage's cost model, scaled per stage by the costs measured in earlier runs of the store -(`/costs/calibration.json`, updated after every export; a task's measured peak counts from its worker's -memory at its start). None of this changes an output: a run with one worker and a run with many write the same store +from each stage's cost model, scaled per stage by the costs measured in earlier runs of the depot +(`/costs/calibration.json`, updated after every export; a task's measured peak counts from its worker's +memory at its start). None of this changes an output: a run with one worker and a run with many write the same depot and the same files. A failed object is reported with its task's result and stays failed until something in its key changes. A task that fails as a whole (an exception, an input that cannot be read, a worker that dies) leaves no result and runs again next time. -### store plan +### depot plan What `export assets` with the same flags would do, without running or fetching anything: every task with its status (`hit`: its result is stored, `run`, `unknown`: it reads the result of a task that would run first, or of a bundle @@ -90,21 +90,21 @@ gives per stage its counts and its first 20 tasks that are not hits; `--json` ha | `--why TASK` | one task: its status, reasons, and its key parts beside those of the previous run | Reasons: `new-input`, `input-changed`, `stage-version`, `atom-changed`, `params-changed`, `context-changed`, -`previous-failed`, `result-missing`, `forced`; hits are `unchanged` or `cached` (the result is in the store from +`previous-failed`, `result-missing`, `forced`; hits are `unchanged` or `cached` (the result is in the depot from another run); unknown tasks are `pending` on a task, or on `fetch:` for a bundle whose content is not known yet. See [contracts.md](contracts.md#plan-nnnotesplan1). -### store run +### depot run -Runs task descriptions (`nnnotes.task/1`, as `plan --emit-tasks` writes them) into a store: each is checked (its key +Runs task descriptions (`nnnotes.task/1`, as `plan --emit-tasks` writes them) into a depot: each is checked (its key is recomputed, its stage must be installed at its version), its inputs are read by content id through their -locators (the store, the cache, a file, or with `--fetch` the catalog), and its result is stored exactly as `export assets` -would store it. Several tasks run one after another in one process; stages that do not read bundles never load +locators (the depot, its downloads, a file, or with `--fetch` the catalog), and its result is stored exactly as +`export assets` would store it. Several tasks run one after another in one process; stages that do not read bundles never load UnityPy. `--force` runs tasks whose result is stored again (the new result must be identical). ### catalog versions, import, fetch, diff -The catalog versions of a store: `import` a catalog file (with the APK's catalog from `[paths] apk` or +The catalog versions of a depot: `import` a catalog file (with the APK's catalog from `[paths] apk` or `--apk-catalog`), `fetch` the current catalog of the configured region and language, `list` them, and `diff` two of them (labels, version ids or sha256 prefixes): keys added, removed and changed, bundles and raw files by stable name, the keys that reach a changed file, and, when a run over each version has stored its address table, the objects by @@ -112,14 +112,22 @@ stable address (an object keeps its stable address while its object id changes w is labelled by `--label`, else by its resource version, else by the first 12 hex digits of its sha256. `fetch` records the resource version it downloads (none for `main`); `import` takes `--resource-version`, else the JP catalog's source version, else the version in a file name `catalog__.bin` as `fetch` and the -cache name them (none for `main`). `export assets` imports the current catalog when it is not imported yet. +downloads name them (none for `main`). `export assets` imports the current catalog when it is not imported yet. -### store verify +### depot verify Checks that every stored content has the sha256 its name says (`--quick`: sizes only) and that every result names its own key, has the key of its parts and names stored contents of the right size. Exit 1 when a problem is found. `--workers` hashes in parallel (default: the CPUs the process may use; `0`: one at a time). +### depot gc + +Removes what no kept run reaches. The kept runs are, for each selection, the `--keep N` latest (default: every run; +the latest run of a selection is always kept). A kept run reaches the results of its tasks, the contents those +results name (their artifacts, and the inputs their task keys name when the depot holds them) and its layout +manifests with the files they list. Every other run, result (with its measured cost) and content is removed. +`--dry-run` counts what would be removed. A layout made of hard links or clones keeps its files. + ## Layouts A layout places the artifacts of a run as files; it never changes their bytes. With one layout the files are @@ -150,16 +158,16 @@ manifest; the run manifest and the printed summary name the method and why it wa | `--link` | the files | extra space | an edit of a file | |---|---|---|---| -| `clone` | clones (reflinks) of the store objects: Linux FICLONE (Btrfs, XFS, bcachefs, …), macOS APFS `clonefile` | none until a file is written (a quota that counts each file's blocks, such as an XFS project quota, counts them in full) | changes that file only | -| `hard` | hard links to the store objects (same file system) | none | fails, except for the superuser: the store objects are read-only (on Windows a delete fails too) | +| `clone` | clones (reflinks) of the depot objects: Linux FICLONE (Btrfs, XFS, bcachefs, …), macOS APFS `clonefile` | none until a file is written (a quota that counts each file's blocks, such as an XFS project quota, counts them in full) | changes that file only | +| `hard` | hard links to the depot objects (same file system) | none | fails, except for the superuser: the depot objects are read-only (on Windows a delete fails too) | | `copy` | copies | the whole layout again | changes that file only | | `auto` (default) | `clone` where the file system makes clones, else `hard` on the same file system, else `copy` | | | `auto` tries each method once, on a temporary file, before the run starts. An explicit `clone` or `hard` that does not work there is a usage error (exit 2) before anything runs; a file that fails with the chosen method is reported (summary, layout report, exit 1), left out of the manifest and placed by the next run: no file falls back to -another method. The store writes its objects read-only; a layout clears the attribute of a file it owns before it -replaces or removes it where the system needs that (Windows), and the store object stays read-only. +another method. The depot writes its objects read-only; a layout clears the attribute of a file it owns before it +replaces or removes it where the system needs that (Windows), and the depot object stays read-only. `cas` is `assets/.` per distinct content, plus the manifest. Each layout follows from the other: `cas` from the `original` manifest alone, `original` from `cas` and the results. The views are the exception: `original` @@ -171,7 +179,7 @@ has them with the paths of their objects, `cas` the view documents they are made | File | Content | |---|---| -| `run.json` | the run manifest (`nnnotes.run/1`): context, selection, parameters, stage versions, every task with its key and status, the layout manifests, a summary; also stored as `/runs/.json` | +| `run.json` | the run manifest (`nnnotes.run/1`): context, selection, parameters, stage versions, every task with its key and status, the layout manifests, a summary; also stored as `/runs/.json` | | `coverage.json` | per class the census total and the count of every item status (`exported`, `contained`, `generic`, `unsupported`, `failed`, and `missing`: objects no task covered; an object two tasks cover, such as a sprite whose image `sprite.crop` makes, counts once); per reason code its count and five examples | | `failures.json` | failed tasks and failed objects, with their codes and messages | | `sources.json` | files of the selection that could not be read: `source.absent` (a file the catalog names that is not there, or an APK file without `[paths] apk`), `source.error` (the download failed) | @@ -185,28 +193,28 @@ and the reports directory. | Command | 0 | 1 | 2 | 3 | 4 | |---|---|---|---|---|---| | `export assets` | complete (unsupported objects allowed) | a task, an object or a download failed (reports written); with `--strict` also view gaps | usage or settings | aborted: a setting or tool a task needs is missing, or interrupted; results written so far stay valid | | -| `store plan` | | with `--check`: something would run or change | usage or settings | | | -| `store run` | every task ran or hit | a task failed or its result is partial | usage or settings | | a task is incompatible (edited, unknown stage or version); nothing of the batch runs | -| `store verify` | no problem | problems found | usage or settings | | | +| `depot plan` | | with `--check`: something would run or change | usage or settings | | | +| `depot run` | every task ran or hit | a task failed or its result is partial | usage or settings | | a task is incompatible (edited, unknown stage or version); nothing of the batch runs | +| `depot verify` | no problem | problems found | usage or settings | | | ## Running tasks elsewhere -Task descriptions carry everything a task needs; any machine with the same nnnotes version and access to the store +Task descriptions carry everything a task needs; any machine with the same nnnotes version and access to the depot can run them. A task can only be described once the content of its inputs is known, so the bundles are identified first (`plan --census` fetches and hashes them and takes their census). Then plan the tasks that can run now, run them in batches on as many workers as you like, and plan again until nothing is left; `export assets` finally writes the layouts and reports (every task is a hit by then). ```sh -store=/data/nnnotes-store -nnnotes store plan --store "$store" --census > /dev/null +depot=/data/nnnotes-depot +nnnotes depot plan --depot "$depot" --census > /dev/null for round in 1 2 3 4 5 6 7 8; do rm -rf tasks - nnnotes store plan --store "$store" --emit-tasks tasks > /dev/null + nnnotes depot plan --depot "$depot" --emit-tasks tasks > /dev/null ls tasks/*.json > /dev/null 2>&1 || break # nothing left to run - ls tasks/*.json | xargs -n 50 -P 16 nnnotes store run --store "$store" --fetch + ls tasks/*.json | xargs -n 50 -P 16 nnnotes depot run --depot "$depot" --fetch done -nnnotes export assets --store "$store" -o out/assets +nnnotes export assets --depot "$depot" -o out/assets ``` Each round runs the tasks whose inputs are known: the catalog index and the censuses, then the script and address diff --git a/docs/cli.md b/docs/cli.md index 1b38fe7..7077399 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -11,10 +11,9 @@ to standard error. Exit status 2 is a usage or settings error. | `--config` `FILE` | TOML config file (else NNNOTES_CONFIG, else ./nnnotes.toml) | | `--region` `REGION` | region: a [servers.] table ([catalog] region) | | `--language` `LANG` | catalog and client language: ja, en, zh-Hant, zh-Hans or ko ([catalog] language) | -| `--catalog` `FILE` | catalog .bin file ([paths] catalog; else downloaded into the cache) | +| `--catalog` `FILE` | catalog .bin file ([paths] catalog; else downloaded into the depot) | | `--catalog-version` `VERSION` | the resource version of an international catalog ([catalog] version; else the version the game API serves when an API root is configured, otherwise main) | -| `--cache` `DIR` | cache directory ([paths] cache) | -| `--store` `DIR` | store directory ([paths] store; default /store) | +| `--depot` `DIR` | depot directory ([paths] depot) | | `--master` `DIR` | decoded master data directory ([paths] master) | | `--apk` `FILE` | base.apk ([paths] apk) | | `--ffmpeg` `EXE` | ffmpeg executable ([paths] ffmpeg; else on PATH) | @@ -97,7 +96,7 @@ browse the configured regions' catalogs and bundles in a local web page ## nnnotes catalog versions -the catalog versions imported into the store (--json: nnnotes.catalogs/1) +the catalog versions imported into the depot (--json: nnnotes.catalogs/1) ## nnnotes catalog import @@ -112,7 +111,7 @@ import a catalog file (with the APK's catalog when [paths] apk is set) ## nnnotes catalog fetch -fetch the region's current catalog and import it into the store +fetch the region's current catalog and import it into the depot | Argument | Description | | --- | --- | @@ -120,7 +119,7 @@ fetch the region's current catalog and import it into the store ## nnnotes catalog diff -keys, bundles, raw files (and objects) that differ between two catalog versions of the store (--json: nnnotes.catalog-diff/1) +keys, bundles, raw files (and objects) that differ between two catalog versions of the depot (--json: nnnotes.catalog-diff/1) | Argument | Description | | --- | --- | @@ -129,7 +128,7 @@ keys, bundles, raw files (and objects) that differ between two catalog versions ## nnnotes bundle pull -fetch the bundle closure of keys into the cache (prints the cached bundle files) +fetch the bundle closure of keys into the depot (prints the downloaded bundle files) | Argument | Description | | --- | --- | @@ -164,14 +163,14 @@ Exactly one of `--version`, `--latest`. ## nnnotes export assets -every object of the selected bundles -> common formats (the store and its layouts) +every object of the selected bundles -> common formats (the depot and its layouts) | Argument | Description | | --- | --- | | `-o`, `--out` `OUT` | output directory (required) | | `--select` `SEL` | group:, key: or bundle: (repeatable; default: everything) | | `--views` `all\|none\|a,b` | semantic views to build (default: all when master data is set, else none) | -| `--stored-catalog` `LABEL\|SHA` | a catalog version of the store (`catalog versions`; default: the current catalog) | +| `--stored-catalog` `LABEL\|SHA` | a catalog version of the depot (`catalog versions`; default: the current catalog) | | `--png-level` `N` | PNG compression level 0-9 (changes the output) | | `--flac-level` `N` | FLAC compression level 0-12 of the CRI audio (default 8; every level decodes to the same samples; changes the files) | | `--only-class` `C,...` | export only objects of these classes | @@ -180,7 +179,7 @@ every object of the selected bundles -> common formats (the store and its layout | `--fetch-workers` `N` | parallel downloads (default 8) | | `--layout` `LAYOUT` | original, cas or original,cas (default original) | | `--explain` | print every setting of the run first | -| `--link` `LINK` | how the layout files are made from the store: auto (default; [export] link): a clone where the file system makes them, else a hard link (read-only), else a copy; or that method only (one of `auto`, `clone`, `hard`, `copy`) | +| `--link` `LINK` | how the layout files are made from the depot: auto (default; [export] link): a clone where the file system makes them, else a hard link (read-only), else a copy; or that method only (one of `auto`, `clone`, `hard`, `copy`) | | `--strict` | exit 1 when a view has gaps | | `--dry-run` | print the plan, run nothing | @@ -305,7 +304,7 @@ find the HCA keycode of the CRI audio (never printed) | --- | --- | | `--write` `DIR` | write it as DIR/.hcakey (the file vgmstream reads) | -## nnnotes store plan +## nnnotes depot plan what `export assets` would run and why, without running it (--json: nnnotes.plan/1) @@ -314,7 +313,7 @@ what `export assets` would run and why, without running it (--json: nnnotes.plan | `-o`, `--out` `OUT` | output directory of the export (counts the layout files to write) | | `--select` `SEL` | group:, key: or bundle: (repeatable; default: everything) | | `--views` `all\|none\|a,b` | semantic views to build (default: all when master data is set, else none) | -| `--stored-catalog` `LABEL\|SHA` | a catalog version of the store (`catalog versions`; default: the current catalog) | +| `--stored-catalog` `LABEL\|SHA` | a catalog version of the depot (`catalog versions`; default: the current catalog) | | `--png-level` `N` | PNG compression level 0-9 (changes the output) | | `--flac-level` `N` | FLAC compression level 0-12 of the CRI audio (default 8; every level decodes to the same samples; changes the files) | | `--only-class` `C,...` | export only objects of these classes | @@ -329,9 +328,9 @@ what `export assets` would run and why, without running it (--json: nnnotes.plan | `--emit-tasks` `DIR` | write the runnable tasks as task descriptions | | `--why` `TASK` | the reasons and key parts of one task | -## nnnotes store run +## nnnotes depot run -run task descriptions (nnnotes.task/1) into the store +run task descriptions (nnnotes.task/1) into the depot | Argument | Description | | --- | --- | @@ -339,7 +338,7 @@ run task descriptions (nnnotes.task/1) into the store | `--fetch` | fetch inputs through their catalog locators | | `--force` | run tasks whose result is stored (it must not change) | -## nnnotes store verify +## nnnotes depot verify check every stored content and result @@ -348,6 +347,15 @@ check every stored content and result | `--quick` | check sizes only, do not hash the contents | | `--workers` `N` | parallel hashing (default the CPUs this process may use; 0: one at a time) | +## nnnotes depot gc + +remove the runs, results and contents no kept run reaches + +| Argument | Description | +| --- | --- | +| `--keep` `N` | keep the N latest runs of each selection (default: every run; the latest run of a selection is kept) | +| `--dry-run` | count what would be removed, remove nothing | + ## nnnotes voices list the voices matching the filters (the text column in the --language language, default [catalog] language) diff --git a/docs/commands.md b/docs/commands.md index 7dc367c..076101c 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -5,11 +5,11 @@ nnnotes [options] ``` Global options (before or after the command name): `--config`, `--region`, `--language`, `--catalog`, -`--catalog-version`, `--cache`, `--store`, `--master`, `--apk`, `--ffmpeg`, `--vgmstream`, `--node` and `--json`, +`--catalog-version`, `--depot`, `--master`, `--apk`, `--ffmpeg`, `--vgmstream`, `--node` and `--json`, plus `--version` and `--help`. They set the settings described in [configuration.md](configuration.md), which also lists the settings each command needs. `nnnotes --help` prints the options of a command; -[cli.md](cli.md) lists every command with its options. The asset export commands (`export assets`, `store plan`, -`store run`, `store verify` and the catalog versions of the store) are described in [assets.md](assets.md). +[cli.md](cli.md) lists every command with its options. The asset export commands (`export assets`, `depot plan`, +`depot run`, `depot verify` and the catalog versions of the depot) are described in [assets.md](assets.md). Commands that write files take the output path with `-o/--out` (required). A command prints a short summary on standard output; with `--json` it prints its report as JSON (UTF-8). Every JSON file is written by one @@ -38,26 +38,27 @@ Write the config file from the template (asking in a terminal, else from `--set` check every setting's origin and format, and list where the config file is looked up. No `config` command prints a setting's value. Details in [configuration.md](configuration.md#writing-the-config-file). -## Cache +## Downloads -`bundle pull` and every extractor read bundles through the cache (`[paths] cache`): +`bundle pull` and every extractor read the game files from the depot's `downloads/` directory (`[paths] depot`), +which keeps the files under their own names: ``` -/catalog_main_.bin the legacy main catalog (explicit main or no API/version pin) -/bundles/ bundles of the dependency closures, decrypted (UnityFS) -/raw/ raw CDN files stored as they are (e.g. CRI cue sheet data) -/catalogs// catalogs downloaded by `catalog browse` -/international/// - versioned catalog, bundles and raw files; isolated from old main caches +/downloads/catalog_main_.bin the main catalog (explicit main or no API/version pin) +/downloads/bundles/ bundles of the dependency closures, decrypted (UnityFS) +/downloads/raw/ raw CDN files stored as they are (e.g. CRI cue sheet data) +/downloads/catalogs// catalogs downloaded by `catalog browse` +/downloads/international/// + versioned catalog, bundles and raw files of one CDN and version ``` -Files already in the cache are not downloaded again. When `[paths] apk` is set, the APK's own catalog is merged with +Files already downloaded are not downloaded again. When `[paths] apk` is set, the APK's own catalog is merged with the region's and bundles that ship inside the APK are read from it. For international regions with a configured API, catalog-based commands discover `resource_version` first and download `catalog__.bin`. `catalog fetch` labels the bytes with the version used to select that file, rather than querying a newer label after downloading main. Use `--catalog-version VERSION` or -the [version settings](configuration.md) to pin builds and use cached data offline. An explicit `--catalog FILE` +the [version settings](configuration.md) to pin builds and use downloaded data offline. An explicit `--catalog FILE` bypasses discovery. JP keeps its existing Version-driven path. ## catalog keys @@ -85,8 +86,8 @@ it, decrypted; `bundles/` lists every bundle directly. nnnotes bundle pull KEY [KEY ...] ``` -Fetches the bundle closure (the key's location and all its dependencies) of each key into the cache and prints the -cached path of every bundle. +Fetches the bundle closure (the key's location and all its dependencies) of each key into the depot's downloads and +prints the path of every bundle. ## servers diff --git a/docs/configuration.md b/docs/configuration.md index 2cfcb5a..f0764aa 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -59,7 +59,7 @@ No `config` command prints a setting's value. The config files these commands wr The global flags go before or after the command name: ```bash -nnnotes --config ~/nnnotes.toml --cache /data/nnnotes-cache bundle pull +nnnotes --config ~/nnnotes.toml --depot /data/nnnotes-depot bundle pull NNNOTES_PATHS_MASTER=/data/master nnnotes export episode 10462 -o episode.json ``` @@ -101,8 +101,7 @@ exist when they are given. | `[bootstrap] api` | `NNNOTES_BOOTSTRAP_API` | — | API root that serves the server list (`nnnotes servers`), same format | | `[client] version` | `NNNOTES_CLIENT_VERSION` | — | client version sent to the game's API, e.g. `1.0.1`; unset: the `versionName` of `[paths] apk` | | `[paths] catalog` | `NNNOTES_PATHS_CATALOG` | `--catalog` | a catalog `.bin` file to read instead of discovering/downloading a catalog; explicit files stay usable offline | -| `[paths] cache` | `NNNOTES_PATHS_CACHE` | `--cache` | cache directory (created when missing) | -| `[paths] store` | `NNNOTES_PATHS_STORE` | `--store` | store directory of the asset export ([assets.md](assets.md)); unset: `<[paths] cache>/store` | +| `[paths] depot` | `NNNOTES_PATHS_DEPOT` | `--depot` | depot directory (created when missing): the downloaded game files, the results of the stages, catalog versions and runs ([commands.md](commands.md#downloads), [contracts.md](contracts.md#depot)) | | `[paths] master` | `NNNOTES_PATHS_MASTER` | `--master` | decoded master data directory, one `.json` per table; the flag overrides `[servers.] master` | | `[paths] apk` | `NNNOTES_PATHS_APK` | `--apk` | the game's `base.apk` (the story export reads the CRI Lips library from it, or from `split_config.arm64_v8a.apk` next to it) | | `[paths] ffmpeg` | `NNNOTES_PATHS_FFMPEG` | `--ffmpeg` | `ffmpeg` executable; unset: `ffmpeg` on `PATH` | @@ -111,7 +110,7 @@ exist when they are given. | `[paths] player` | `NNNOTES_PATHS_PLAYER` | `--player` (of `site build`) | ournotes-player checkout (built) or installed package | | `[paths] fonts.` | `NNNOTES_PATHS_FONTS_` (e.g. `NNNOTES_PATHS_FONTS_ZH_HANT`) | `--font =PATH` (of `site build`) | font file (OpenType or TrueType) the story text of a language is drawn with (`--fonts open`); also read for the other story languages whose game fonts fall back to that language's font | | `[paths] fonts.emoji` | `NNNOTES_PATHS_FONTS_EMOJI` | `--font emoji=PATH` (of `site build`) | colour emoji font (PNG bitmap glyphs: CBDT or sbix) the stories' emoji sprites are drawn from (`--fonts open`); Noto Color Emoji (SIL Open Font License 1.1) is the tested font. Optional: without it the sprites keep their layout with empty glyphs, listed in `ui/fonts.json` `coverage.sprites.missing` | -| `[export] link` | `NNNOTES_EXPORT_LINK` | `--link` (of `export assets`) | how `export assets` makes the layout files from the store: `auto` (unset), `clone`, `hard` or `copy` ([assets.md](assets.md)) | +| `[export] link` | `NNNOTES_EXPORT_LINK` | `--link` (of `export assets`) | how `export assets` makes the layout files from the depot: `auto` (unset), `clone`, `hard` or `copy` ([assets.md](assets.md)) | Hex values are case-insensitive; a `0x` prefix and surrounding whitespace are ignored. All values are TOML strings except `languages` (array of strings). @@ -136,15 +135,15 @@ nnnotes --region tw --catalog-version 1.0.0.201 export live2d MODEL_ID -o out/mo ``` Alternatively set `NNNOTES_SERVERS_TW_CATALOG_VERSION` in the build environment; workers inherit this pin. -`--catalog-version` selects a remote resource release; `--stored-catalog LABEL|SHA` of `export assets` and `store -plan` selects a catalog version of the store. To inspect the historical main catalog, explicitly pin `main`. -Versioned caches are isolated by CDN root and resource version; a cached pin needs the CDN setting to identify -its directory but performs no API/catalog download. A missing versioned file never falls back to main. +`--catalog-version` selects a remote resource release; `--stored-catalog LABEL|SHA` of `export assets` and `depot +plan` selects a catalog version of the depot. To inspect the historical main catalog, explicitly pin `main`. +Versioned downloads are isolated by CDN root and resource version; a pin whose catalog is downloaded needs the CDN +setting to identify its directory but performs no API/catalog download. A missing versioned file never falls back to main. ## What each command needs -"Catalog" below means: `[paths] cache`; either `[paths] catalog` or `[catalog] language` (the catalog is then -downloaded into the cache on first use); and, for what must be downloaded, `[catalog] region` and that region's +"Catalog" below means: `[paths] depot`; either `[paths] catalog` or `[catalog] language` (the catalog is then +downloaded into the depot on first use); and, for what must be downloaded, `[catalog] region` and that region's `cdn`, `[bundle] key` and `nonce_seed`. `[paths] apk` is optional for the catalog: when it is set, the APK's own catalog and bundles are merged in, and bundles that ship inside the APK can only be read with it. @@ -157,8 +156,8 @@ story text); it is not a setting. | Command | Settings | |---|---| -| `catalog keys` | `[paths] cache`; `[paths] catalog`, or `[catalog] language` (plus region and `cdn` while the catalog is not in the cache); no bundle key | -| `catalog browse` | `[paths] cache`, `[bundle] key` + `nonce_seed`, and per region `cdn` + `languages` (`name` optional) | +| `catalog keys` | `[paths] depot`; `[paths] catalog`, or `[catalog] language` (plus region and `cdn` while the catalog is not downloaded); no bundle key | +| `catalog browse` | `[paths] depot`, `[bundle] key` + `nonce_seed`, and per region `cdn` + `languages` (`name` optional) | | `bundle pull` | catalog | | `master decode` | `[master] key` + `iv` | | `master download` | `[catalog] region` and that region's `cdn`; `--latest` also that region's `api` and the client version | @@ -178,13 +177,13 @@ story text); it is not a setting. | `site build --live2d` / `--all-live2d` | catalog (bundles from the CDN of the site's first region), `[paths] apk`, `[paths] player`; not `node`; master data only for the model names (optional: without it `models.json` has no names) | | `site build --player-only` / `--reingest-json` | `[paths] player` | | `music-data` | catalog, `[master] key` + `iv` (not with `--decoded-master`); `--master-files` also `[catalog] region`; `--apk-master` also `[paths] apk`; `--decoded-master` also `[catalog] region` and master data with its `MasterManifest.json`; `[paths] apk` (optional otherwise) for the client version | -| `export assets`, `store plan` | the store (`[paths] store` or `[paths] cache`); catalog (bundles are fetched into the cache); `[paths] apk` for the bundles inside the APK (without it they are reported as `source.absent`); master data for `--views`; with `--stored-catalog` a catalog version of the store instead of the current catalog | -| `store run` | the store; `[paths] cache` for inputs located in the cache; `--fetch` also what fetching needs (region, `cdn`, bundle key, `[paths] apk`) | -| `catalog versions` / `import` / `diff`, `store verify` | the store; `import` reads the APK's catalog from `[paths] apk` when it is set | -| `catalog fetch` | the store, `[catalog] region` and `language`, that region's `cdn`; its `api` (optional) for the resource version label | +| `export assets`, `depot plan` | catalog (bundles are fetched into the depot); `[paths] apk` for the bundles inside the APK (without it they are reported as `source.absent`); master data for `--views`; with `--stored-catalog` a catalog version of the depot instead of the current catalog | +| `depot run` | the depot; `--fetch` also what fetching needs (region, `cdn`, bundle key, `[paths] apk`) | +| `catalog versions` / `import` / `diff`, `depot verify` | the depot; `import` reads the APK's catalog from `[paths] apk` when it is set | +| `catalog fetch` | the depot, `[catalog] region` and `language`, that region's `cdn`; its `api` (optional) for the resource version label | The region, its `cdn`, `[bundle] key` and `nonce_seed` are read only when a file must be downloaded: when the -catalog and every bundle a command needs are in the cache, none of them is needed. A download that needs a missing +catalog and every bundle a command needs are downloaded, none of them is needed. A download that needs a missing one stops the command with one line naming the file and the setting (see Errors). ## Errors @@ -194,7 +193,7 @@ the setting, its environment variable and its flag where there is one; it never ``` nnnotes: setting catalog.region is not set: give it as `region` in the [catalog] table of the config file, the environment variable NNNOTES_CATALOG_REGION or --region -nnnotes: .bundle is not in the cache: setting bundle.key is not set: give it as `key` in the [bundle] table of the config file, the environment variable NNNOTES_BUNDLE_KEY +nnnotes: .bundle is not downloaded: setting bundle.key is not set: give it as `key` in the [bundle] table of the config file, the environment variable NNNOTES_BUNDLE_KEY nnnotes: setting bundle.key: must be 16 bytes (32 hex digits) nnnotes: setting paths.apk: file not found nnnotes: ffmpeg not found on PATH: give its path as `ffmpeg` in the [paths] table of the config file, the environment variable NNNOTES_PATHS_FFMPEG or --ffmpeg diff --git a/docs/contracts.md b/docs/contracts.md index c9e0f2d..930f5ab 100644 --- a/docs/contracts.md +++ b/docs/contracts.md @@ -55,7 +55,7 @@ One stage applied to one subject, with everything needed to run it "params": {"png": {"encoder": "pillow", "level": 6}, "blobMin": 4096}, "atoms": {"png.encode": "pillow-12.3.0/1", "texture.decode": "unitypy-1.25.3/1"}, "inputs": [{"role": "bundle", "sha256": "…", "size": 123456, "name": "membercard_…_3fa…c2.bundle", - "locators": [{"kind": "store"}, {"kind": "cache", "path": "bundles/membercard_…_3fa…c2.bundle"}]}], + "locators": [{"kind": "depot"}, {"kind": "download", "path": "bundles/membercard_…_3fa…c2.bundle"}]}], "context": {"scripts": {"CAB-…:12345": "Assembly-CSharp|Fwk.Sound|SplitAcbData"}}, "key": "…", "cost": {"cpuSeconds": 1.2, "peakBytes": 450000000}} ``` @@ -83,10 +83,10 @@ Locator kinds, tried in order to find an input's bytes (each checked against the | Kind | Where | |---|---| -| `store` | the store's `cas/` | -| `cache` | `path` relative to the cache directory | +| `depot` | the depot's `cas/` | +| `download` | `path` relative to the depot's `downloads/` | | `file` | an absolute `path` | -| `catalog` | fetched through a catalog version of the store (`catalog`: its version id, `location`: the location id in its index) | +| `catalog` | fetched through a catalog version of the depot (`catalog`: its version id, `location`: the location id in its index) | ## Artifact record: nnnotes.artifact/1 @@ -142,7 +142,7 @@ stage, an input that cannot be read, a worker process that dies) leaves no resul What one run covered ([run.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/run.schema.json)): the context (region, language, the catalog version, and the master data its views read: the version named by `MasterManifest.json` and the sha256 of each table), the selection, the normalized parameters per stage, the stage versions, every task with its -key (`null` when it could not be described) and status (`hit`: the result was in the store, `ran`, `failed`), the +key (`null` when it could not be described) and status (`hit`: the result was in the depot, `ran`, `failed`), the sha256 of each layout manifest written, and a summary (task and item counts). The run id is the key hash of the context, selection, params, stages, the tasks' ids and keys, and the layouts, so a run with the same inputs has the same id whether its tasks ran or hit. The run log (`runs/.log.jsonl`) holds what is not deterministic: @@ -181,11 +181,11 @@ task id in the previous run manifest: | `params-changed` | run | a parameter changed (`path`, `old`, `new`) | | `context-changed` | run | a consumed context entry changed (`entry`) | | `previous-failed` | run | the task failed in the previous run | -| `result-missing` | run | same key as before, but its result is not in the store | +| `result-missing` | run | same key as before, but its result is not in the depot | | `key-changed` | run | the key changed for none of the reasons above | | `forced` | run | asked to run again | | `unchanged` | hit | same key as in the previous run | -| `cached` | hit | the result is in the store from another run | +| `cached` | hit | the result is in the depot from another run | | `pending` | unknown | the task reads the result of `task`, which would run first (`fetch:`: an input file whose content is not known yet) | ## Layout manifest: nnnotes.layout/1 @@ -221,8 +221,8 @@ Documents written by particular stages (artifacts of no object: their ids are `< | `nnnotes.artifacts/1` | `link.artifacts` | [artifacts.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/artifacts.schema.json) | | `nnnotes.view/1` | `view.` | [view.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/view.schema.json) | -The catalog versions of a store, `nnnotes.catalogs/1` ([catalogs.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/catalogs.schema.json), -`/catalogs/index.json`), and the difference between two of them, `nnnotes.catalog-diff/1` +The catalog versions of a depot, `nnnotes.catalogs/1` ([catalogs.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/catalogs.schema.json), +`/catalogs/index.json`), and the difference between two of them, `nnnotes.catalog-diff/1` ([catalog-diff.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/catalog-diff.schema.json), `nnnotes catalog diff --json`), are written by the commands. @@ -230,23 +230,24 @@ The music data file of `nnnotes music-data`, `nnnotes.music-data/2` ([music-data.schema.json](https://github.com/MetaSekaiLab/nnnotes/blob/main/src/nnnotes/schemas/music-data.schema.json)), names its format in a `format` field; it is described in [music-data.md](music-data.md). -## Store +## Depot -The store is a directory, and the cache of all stage work: +The depot is a directory: the downloaded game files, and the results of all stage work by content: ``` -/cas/sha256/ab/ bytes by content id, written once (no extension: records carry it) -/ac/ab/.json results by task key (written after the bytes they name) -/inputs/.jsonl input identities: name -> sha256, size, with the path, size and mtime of the +/downloads/ the game files as downloaded, under their own names ([commands.md](commands.md#downloads)) +/cas/sha256/ab/ bytes by content id, written once (no extension: records carry it) +/ac/ab/.json results by task key (written after the bytes they name) +/inputs/.jsonl input identities: name -> sha256, size, with the path, size and mtime of the file that was hashed (bundles.jsonl, raw.jsonl, files.jsonl) -/catalogs/ versioned catalogs -/runs/.json run manifests -/runs/.log.jsonl the log of the run's last execution -/runs/selections/ the id of the latest run of a selection (key hash of the selection list) -/costs/.json the measured cost of a task's last run +/catalogs/ versioned catalogs +/runs/.json run manifests +/runs/.log.jsonl the log of the run's last execution +/runs/selections/ the id of the latest run of a selection (key hash of the selection list) +/costs/.json the measured cost of a task's last run ``` Every file is written to a temporary name and renamed into place, so concurrent writers never see or leave a partial file and an interrupted task leaves no result. `cas/` and `ac/` depend only on the tasks run: two runs of the -same tasks, with any number of workers, leave identical trees. `inputs/`, `runs/*.log.jsonl` and `costs/` are -local and not deterministic. +same tasks, with any number of workers, leave identical trees. `downloads/`, `inputs/`, `runs/*.log.jsonl` and +`costs/` are local and not deterministic. diff --git a/docs/determinism.md b/docs/determinism.md index 550828c..5b7113d 100644 --- a/docs/determinism.md +++ b/docs/determinism.md @@ -19,7 +19,7 @@ fixes the bytes, what can change them, and how the tests hold it. textures, shaders and audio are converted from the stored data with fixed encoder settings. - **No clocks.** No output holds the time of a run or a path of the machine it ran on. A result of the asset export holds no input name, locator, timestamp or traceback; the measured cost of a task (CPU and wall seconds, memory) is - kept in the store's cost report, apart from the results. + kept in the depot's cost report, apart from the results. ## What can change the bytes @@ -32,7 +32,7 @@ Another version of an encoder can write the same content as other bytes: while lossy formats depend on the encoder version. - zlib and brotli encode the compressed assets of a site. -So the guarantee is per installation: pin the versions to reproduce a result exactly. The store keeps this visible: +So the guarantee is per installation: pin the versions to reproduce a result exactly. The depot keeps this visible: a task's key includes the implementation ids of the atoms it uses, with the versions of the libraries they call, so another installation computes other keys instead of mixing results. @@ -44,7 +44,7 @@ another installation computes other keys instead of mixing results. library versions of `.github/goldens-constraints.txt` and sets `GOLDENS_STRICT=1`; elsewhere a record made with other library versions is skipped. - **Two runs, one result.** The tests run exports twice over synthetic inputs and compare the bytes, and compare - one worker with several: a store built with four workers equals one built with one, and parallel audio decoding + one worker with several: a depot built with four workers equals one built with one, and parallel audio decoding equals sequential decoding. - **Writer tests.** The JSON writer's tests cover the non-finite literals, the NaN error and the line endings. diff --git a/docs/export.md b/docs/export.md index 8251fdd..6cce4fc 100644 --- a/docs/export.md +++ b/docs/export.md @@ -1,7 +1,7 @@ # Export stories and lives The `export` commands each write one subject as a directory or file. They read the catalog, fetch and decrypt the -bundles they need into the cache, and need decoded master data where the subject is defined by it +bundles they need into the depot, and need decoded master data where the subject is defined by it ([what each command needs](configuration.md#what-each-command-needs)). To export every object of many bundles at once, use [`export assets`](assets.md) instead. diff --git a/docs/jp.md b/docs/jp.md index df5067e..63d0862 100644 --- a/docs/jp.md +++ b/docs/jp.md @@ -25,7 +25,7 @@ apk = "/path/to/jp/base.apk" master = "/path/to/jp/master" [paths] -cache = "/path/to/cache" +depot = "/path/to/depot" ``` Keep `[bundle] key/nonce_seed` and `[master] key/iv` in your configuration. Master keys are unnecessary when @@ -79,17 +79,17 @@ the JP hash in `provenance.catalog.resourceHash`. JP catalogs are `catalog_main.bin` in a version/hash directory, without a language suffix. The HTTP payload can be gzip; nnnotes keeps its original bytes and SHA-256, and bounds decompression before parsing. Remote bundle and CRI locations use `{Fwk.Resource.RemoteAssetDir}`. These are indexed, resolved and downloaded through the selected -snapshot, including in `catalog browse`, the other `catalog` commands, `export assets`, `store plan` and -`store run`. +snapshot, including in `catalog browse`, the other `catalog` commands, `export assets`, `depot plan` and +`depot run`. -The cache lives under `/jp////`. It cannot reuse the -international catalog cache. Embedded files are further isolated under `apk//`, so an APK -update cannot reuse old local bundles when the CDN snapshot stays unchanged. Each cached `catalog_main.bin` has a `catalog_main.bin.source.json` with its SHA-256 +The downloads live under `/downloads/jp////`, apart from the +international downloads. Embedded files are further isolated under `apk//`, so an APK +update cannot reuse old local bundles when the CDN snapshot stays unchanged. Each downloaded `catalog_main.bin` has a `catalog_main.bin.source.json` with its SHA-256 and public source metadata. For an offline `--catalog` or `catalog import`, copy the pair together. Credentials are reacquired only when a missing file must be downloaded. Catalog database records include this source metadata, and JP catalog identity includes version/hash even when -the catalog bytes remain the same. Old international records retain their previous IDs. Cached historical files +the catalog bytes remain the same. Old international records retain their previous IDs. Downloaded historical files can be read offline; downloading a missing historical file fails if Version now selects a different snapshot. Refresh the catalog to use the new snapshot. Historical local bundles require the APK set matching the imported APK catalog. A JP master download likewise rejects a different current master snapshot. diff --git a/docs/music-data.md b/docs/music-data.md index 88d3a67..77b6eba 100644 --- a/docs/music-data.md +++ b/docs/music-data.md @@ -54,7 +54,7 @@ Each master data file is checked against the SHA-256 the manifest lists and deco With `--decoded-master` the tables are read as decoded, and the master data version and each file's SHA-256 are the manifest's: the decoded tables cannot be checked against the files as served, so the file records what the manifest lists (the same values as `--master-files` on those files). -Charts are read from the catalog of `[catalog] language` (bundles fetched into the cache as for every command); the +Charts are read from the catalog of `[catalog] language` (bundles fetched into the depot as for every command); the BGM length from the cue sheet's ACB (its `CueTable` and `WaveformTable`, no audio is decoded). `FILE` ending in `.gz` is written gzip-compressed. The command prints a summary (`songs`, `charts`, `deck`: the deck model's commit, `unplayable`, `bytes`, `sha256` of the file, ...). @@ -102,7 +102,7 @@ A **text** is an object with one string per language of `languages` (`{"ja": ... |---|---| | `region` | the region whose master data this is (a configured region name), or `embedded` for the APK's master data | | `client.versionName`, `client.versionCode` | the APK's version name and code (null without `[paths] apk`) | -| `catalog.resourceVersion` | the resource version used to select the downloaded catalog, or recorded for an explicit file in the catalog store (`nnnotes catalog fetch` / `import`); null when unknown | +| `catalog.resourceVersion` | the resource version used to select the downloaded catalog, or recorded for an explicit file in the catalog depot (`nnnotes catalog fetch` / `import`); null when unknown | | `catalog.sha256` | SHA-256 of the remote catalog file the charts were read with | | `master.source` | `api` (`--master-files`, `--decoded-master`: the region's files) or `embedded` (`--apk-master`) | | `master.version` | the `version` of the master data manifest | diff --git a/docs/quickstart.md b/docs/quickstart.md index 93d1c43..8c8cc44 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -21,7 +21,7 @@ nnnotes config init --user ``` In a terminal this asks for each setting in turn, among them the bundle and master data keys, the names of your -regions with each region's CDN base and API root, the catalog language, and the paths of the cache, the APK and the +regions with each region's CDN base and API root, the catalog language, and the paths of the depot, the APK and the tools. Enter leaves a setting empty; the keys are read without echo. The file is written to the per-user location, which every command reads from any directory (`nnnotes config path` shows it). @@ -41,7 +41,7 @@ set it. nnnotes catalog keys --prefix Adv/ --limit 5 ``` -The first command downloads the region's catalog into the cache; the keys it lists are the addressable keys of the +The first command downloads the region's catalog into the depot; the keys it lists are the addressable keys of the game's assets. `nnnotes catalog browse` serves a local web page to browse the catalogs and bundles of every configured region. @@ -74,5 +74,5 @@ where the next story reuses them. Add `--json` to any command to get its report - [Export stories and lives](export.md): the other single-subject exports and their options. - [Build an ournotes-player site](site.md): charts, models and stories as one static site. -- [Export every asset](assets.md): every object of the selected bundles through the content-addressed store. +- [Export every asset](assets.md): every object of the selected bundles through the content-addressed depot. - [Command reference](cli.md): every command and option. diff --git a/docs/schemas.md b/docs/schemas.md index 051f6fe..04a7d89 100644 --- a/docs/schemas.md +++ b/docs/schemas.md @@ -19,7 +19,7 @@ schemas.load("plan") # the schema as a dict | `artifacts` | `nnnotes.artifacts/1` | the artifact table: the artifacts of every object, with content id, size and extension | | `catalog-diff` | `nnnotes.catalog-diff/1` | the difference between two catalog versions (`catalog diff`) | | `catalog-index` | `nnnotes.catalog-index/1` | the index of a catalog pair: remote and APK catalogs, every location in catalog order | -| `catalogs` | `nnnotes.catalogs/1` | the catalog versions of a store (`catalog versions`) | +| `catalogs` | `nnnotes.catalogs/1` | the catalog versions of a depot (`catalog versions`) | | `census` | `nnnotes.census/1` | the census of one bundle: serialized files, type trees and objects, read without decoding | | `common` | | definitions shared by the other schemas | | `cost` | `nnnotes.cost/1` | the measured cost of a task's last run | @@ -27,11 +27,11 @@ schemas.load("plan") # the schema as a dict | `failures` | `nnnotes.failures/1` | the failed tasks and items of a run | | `layout` | `nnnotes.layout/1` | the files a layout owns in its output directory | | `music-data` | `nnnotes.music-data/2` | every song and chart of one master data version ([music data](music-data.md)) | -| `plan` | `nnnotes.plan/1` | what a run would do and why (`store plan`) | +| `plan` | `nnnotes.plan/1` | what a run would do and why (`depot plan`) | | `result` | `nnnotes.result/1` | the result of a task, a function of its key alone | | `run` | `nnnotes.run/1` | a run manifest: context, selection, tasks with keys and statuses, layouts | | `scripts` | `nnnotes.scripts/1` | every MonoScript of the censuses, by object id | -| `task` | `nnnotes.task/1` | one stage applied to one subject, with everything needed to run it (`store run`) | +| `task` | `nnnotes.task/1` | one stage applied to one subject, with everything needed to run it (`depot run`) | | `view` | `nnnotes.view/1` | a semantic view of a master table, each row with its names and the objects it names | The documents of the asset export are described in [contracts](contracts.md), the views in [views](views.md). The diff --git a/docs/ui.md b/docs/ui.md index 57c1686..5251ce8 100644 --- a/docs/ui.md +++ b/docs/ui.md @@ -18,7 +18,7 @@ index entry and makes the command exit with status 1. Existing unrelated entries The command uses the same `ApkSet` reader as the existing extractors: one APK, base.apk with adjacent splits, a directory of splits, or an APKS/XAPK archive. Explicit `--apk` wins over the selected region's APK configuration. -The configured cache and bundle decryption settings are used when needed. Readable unencrypted local bundles do +The configured depot and bundle decryption settings are used when needed. Readable unencrypted local bundles do not require a key. Default Unity resources are read from the selected APK set through the existing player reader. ```sh diff --git a/docs/views.md b/docs/views.md index b3cb362..af73b43 100644 --- a/docs/views.md +++ b/docs/views.md @@ -109,7 +109,7 @@ resolver's name). ## Stages `nnnotes.pipeline.views:stages` gives one stage per view, `view.` (docs/stages.md), run by `nnnotes export assets ---views` and planned by `nnnotes store plan`: +--views` and planned by `nnnotes depot plan`: - Subjects: the catalog subjects of `link.addresses` (the export command has one, `main`), when master data is set and the view is selected (the planning facts `master` and `views`). @@ -132,7 +132,7 @@ resolver's name). ## Views ### `backgrounds` -Profile backgrounds: the rows store full addresses. Table `MasterBackground`. Vars: `path` = `_assetPath`, `thumbnail` +Profile backgrounds: the rows depot full addresses. Table `MasterBackground`. Vars: `path` = `_assetPath`, `thumbnail` = `_thumbnailAssetPath`. Names: `name` = `_nameTextId`, `description` = `_descriptionTextId`. Prefixes: `Image/Background/`, `thumbnail/Background/`. @@ -331,7 +331,7 @@ Song jackets. Table `MasterLiveMusic`. Vars: `jacket` = `_jacketAssetName`. Name ### `login_bonuses` -Login bonus sheets: the rows store full addresses. Table `MasterLoginBonus`. Vars: `sheet` = `_sheetImageAsset`, +Login bonus sheets: the rows depot full addresses. Table `MasterLoginBonus`. Vars: `sheet` = `_sheetImageAsset`, `background` = `_backgroundImageAsset`. Names: `name` = `_nameTextID`. Prefixes: `LoginBonus/`. | Role | Address | Expect | Condition | Required | @@ -399,7 +399,7 @@ Skill icons (a skill names its icon by id). Table `MasterSkillIcon`. Vars: `icon ### `spots` -Home spots: the rows store full addresses. Table `MasterHomeSpot`. Vars: `thumbnail` = `_thumbnailAssetPath`, +Home spots: the rows depot full addresses. Table `MasterHomeSpot`. Vars: `thumbnail` = `_thumbnailAssetPath`, `background` = `_backgroundAssetPath`, `situation` = `_situationAssetPath`. Names: `name` = `_nameTextId`. Prefixes: `Image/Spot/`. diff --git a/src/nnnotes/cli/__init__.py b/src/nnnotes/cli/__init__.py index 0020406..0aa5979 100644 --- a/src/nnnotes/cli/__init__.py +++ b/src/nnnotes/cli/__init__.py @@ -13,7 +13,7 @@ nnnotes master version|download|decode nnnotes export assets|episode|story|live|live2d|spot|room|shader|audio|ui ... -o OUT nnnotes apk settings -o FILE | hca-key [--write DIR] - nnnotes store plan|run|verify + nnnotes depot plan|run|verify|gc nnnotes voices list|search|summary|get nnnotes site build -o SITE nnnotes music-data -o FILE @@ -36,10 +36,10 @@ from .. import __version__ from ..source.config import ConfigError from ..source.gameapi import GameApiError -from . import apk, bundle, catalog, config, export, master, musicdata, servers, site, store, voices +from . import apk, bundle, catalog, config, depot, export, master, musicdata, servers, site, voices from .common import GLOBALS, Formatter, fill_globals, load_config -GROUPS = (config, servers, catalog, bundle, master, export, apk, store, voices, site, musicdata) +GROUPS = (config, servers, catalog, bundle, master, export, apk, depot, voices, site, musicdata) def build_parser() -> argparse.ArgumentParser: diff --git a/src/nnnotes/cli/assets.py b/src/nnnotes/cli/assets.py index 3536ff9..55d7a7b 100644 --- a/src/nnnotes/cli/assets.py +++ b/src/nnnotes/cli/assets.py @@ -1,22 +1,22 @@ -"""The asset commands: the default orchestration of the stages (`export assets`), its plan and single tasks (`store -plan`, `store run`). +"""The asset commands: the default orchestration of the stages (`export assets`), its plan and single tasks (`depot +plan`, `depot run`). nnnotes export assets -o OUT [--layout original[,cas]] [--select group:G|key:PREFIX|bundle:GLOB ...] [--views all|none|a,b] [--workers N] [--memory GiB] [--fetch-workers N] [--stored-catalog L|SHA] [--png-level N] [--flac-level N] [--only-class C,...] [--link auto|clone|hard|copy] [--strict] [--dry-run] [--explain] - nnnotes store plan [the selection and parameters of export assets] [-o OUT] [--since RUN] [--check] [--census] + nnnotes depot plan [the selection and parameters of export assets] [-o OUT] [--since RUN] [--check] [--census] [--emit-tasks DIR] [--why TASK] - nnnotes store run TASK.json|DIR [...] [--fetch] [--force] + nnnotes depot run TASK.json|DIR [...] [--fetch] [--force] The pipeline is the registry STAGE_SOURCES in order: each entry names the module and attribute of a stage (a Stage class) or of a family of stages (a function returning them, `view.`). An entry whose module or attribute is not installed is left out of the pipeline and named by --explain. Stage modules import heavy libraries (UnityPy) only -when a task runs, so `store plan`, and `store run` of stages that do not read bundles, never load them. +when a task runs, so `depot plan`, and `depot run` of stages that do not read bundles, never load them. Facts the stages read while their tasks are described (stages.Env): - catalogs {"main": [Input remote, Input apk]}: the catalog version's files in /catalogs/ + catalogs {"main": [Input remote, Input apk]}: the catalog version's files in /catalogs/ index the catalog index document (nnnotes.catalog-index/1) bundles {stable bundle name: Input "bundle"} of the selected bundles and their dependencies (a bundle whose bytes are not known yet: a Pending "fetch:") @@ -34,11 +34,11 @@ dependency is its own bundle), so script and atlas references resolve. Task cost estimates (scheduling only: never part of a key) are each stage's estimate scaled per stage by the -costs measured in earlier runs (/costs/calibration.json, updated after every export). +costs measured in earlier runs (/costs/calibration.json, updated after every export). Exit codes: export assets 0 complete (unsupported objects allowed), 1 item or task failures (reports written), 2 -usage or settings, 3 aborted (results written so far stay valid); store run 0, 1 (a task failed or a result is -partial), 2, 4 an incompatible task (nothing of the batch runs); store plan --check 1 when anything would run or +usage or settings, 3 aborted (results written so far stay valid); depot run 0, 1 (a task failed or a result is +partial), 2, 4 an incompatible task (nothing of the batch runs); depot plan --check 1 when anything would run or change. """ @@ -147,7 +147,7 @@ class Installed: def installed(names=None) -> Installed: - """The stages of STAGE_SOURCES (`names`: only the sources of these stage names, for `store run`).""" + """The stages of STAGE_SOURCES (`names`: only the sources of these stage names, for `depot run`).""" stages, of, missing = {}, {}, [] for src in STAGE_SOURCES: if names is not None and not any(src.matches(n) for n in names): @@ -257,19 +257,20 @@ def _bundle_key(cfg): class CatalogFetcher: """The fetcher of "catalog" locators ({catalog: version id, location: location id}): the file is fetched through - that catalog version of the store (the settings' CDN, bundle key and APK) into the cache. Crosses to worker - processes as its settings.""" + that catalog version of the depot (the settings' CDN, bundle key and APK) into the depot's downloads/. Crosses to + worker processes as its settings.""" - def __init__(self, store_root_, cache, cfg): - self.root, self.cache, self.cfg = Path(store_root_), Path(cache) if cache else None, cfg + def __init__(self, root, cfg): + self.root, self.cfg = Path(root), cfg + self.downloads = self.root / "downloads" self._catalogs: dict = {} self._lock = threading.Lock() def __getstate__(self): - return {"root": self.root, "cache": self.cache, "cfg": self.cfg} + return {"root": self.root, "cfg": self.cfg} def __setstate__(self, state): - self.__init__(state["root"], state["cache"], state["cfg"]) + self.__init__(state["root"], state["cfg"]) def catalog(self, vid: str): """(Catalog, {location id: location}) of a stored catalog version.""" @@ -279,34 +280,32 @@ def catalog(self, vid: str): with self._lock: hit = self._catalogs.get(vid) if hit is None: - if self.cache is None: - raise self.cfg.missing("paths", "cache") db = catalogdb.CatalogDB(self.root) v = next((x for x in db.versions() if x["id"] == vid), None) if v is None: - raise FileNotFoundError(f"catalog version {vid[:12]} is not in the store") + raise FileNotFoundError(f"catalog version {vid[:12]} is not in the depot") remote, apk = db.catalog_bytes(v) cfg, region = self.cfg, v.get("region") region = region or cfg.get("catalog", "region") if region: cfg = cfg.for_region(region) - source, client, cache = None, None, self.cache + source, client, downloads = None, None, self.downloads if v.get("source") is not None: from ..source.jp import Client, Source source = Source.from_dict(v["source"]) client = Client(cfg, cfg.region()) - cache = source.cache_dir(cache) + downloads = source.download_dir(downloads) cat = Catalog( remote, - cache, + downloads, cdn=lambda: cfg.cdn(cfg.region()), bundle_key=lambda: _bundle_key(cfg), apk_catalog=apk, source=source, client=client, ) - # Replay imported offsets; the current APK is needed only for uncached local files. + # Replay imported offsets; the current APK is needed only for local files not downloaded yet. cat.apk = cfg.path("paths", "apk") hit = self._catalogs[vid] = (cat, catalogdb.by_id(catalogdb.index(remote, apk))) return hit @@ -327,12 +326,12 @@ def fetch_location(self, vid: str, lid: str) -> Path: return cat.fetch_raw({"internal_id": iid}) def _apk_file(self, internal_id: str, cat) -> Path: - """A raw APK file through its snapshot's cache; only a cache miss needs the matching APK.""" + """A raw APK file through its snapshot's downloads; only a file not read yet needs the matching APK.""" from ..core.cache import write_atomic from ..source.catalog import APK_AA_DIR rel = apk_rel(internal_id) - dst = cat.local_cache_dir() / "raw" / rel + dst = cat.local_download_dir() / "raw" / rel if not (dst.is_file() and dst.stat().st_size > 0): if cat.apk is None: raise self.cfg.missing("paths", "apk") @@ -354,7 +353,7 @@ def apk_rel(internal_id: str) -> str: return internal_id[len(LOCAL_PREFIX) :].lstrip("/") if internal_id.startswith(LOCAL_PREFIX) else internal_id -def _worker_init(store, cfg) -> None: +def _worker_init(depot, cfg) -> None: """Every worker process: the settings of the command.""" use(cfg) @@ -366,7 +365,7 @@ def _log(message: str) -> None: # ---------------------------------------------------------------- cost calibration -CALIBRATION = "calibration.json" # /costs/calibration.json +CALIBRATION = "calibration.json" # /costs/calibration.json MIN_SAMPLES = 10 @@ -388,7 +387,7 @@ def estimate(self, subject, env, inputs): def load_calibration(root: Path) -> dict: - """{stage: {"cpu": factor, "peak": factor, "samples": n}} of a store ({} when none was measured).""" + """{stage: {"cpu": factor, "peak": factor, "samples": n}} of a depot ({} when none was measured).""" try: doc = contract.loads((Path(root) / "costs" / CALIBRATION).read_bytes()) except (OSError, ValueError): @@ -442,26 +441,25 @@ def calibrate(log: list[dict], factors: dict, worker_bytes: int) -> dict: def write_calibration(root: Path, factors: dict) -> None: - from ..pipeline.store import write_file + from ..pipeline.depot import write_file write_file(Path(root) / "costs" / CALIBRATION, contract.encode({"stages": factors})) # ---------------------------------------------------------------- the workspace of a command class Workspace: - """What export and plan share: the store, the catalog version, its index, the selection, the pipeline and its + """What export and plan share: the depot, the catalog version, its index, the selection, the pipeline and its parameters, the facts.""" def __init__(self, args, cfg, log=_log): from ..pipeline.catalogdb import CatalogDB - from ..pipeline.store import Store + from ..pipeline.depot import Depot self.args, self.cfg, self.log = args, cfg, log - self.root = Session(cfg).store_root() - self.cache = cfg.path("paths", "cache") + self.root = Session(cfg).depot_root() self.apk = cfg.path("paths", "apk") - self.fetcher = CatalogFetcher(self.root, self.cache, cfg) - self.store = Store(self.root, cache=self.cache, fetch=self.fetcher) + self.fetcher = CatalogFetcher(self.root, cfg) + self.depot = Depot(self.root, fetch=self.fetcher) self.db = CatalogDB(self.root) try: self.selection = parse_selection(getattr(args, "select", None)) @@ -521,24 +519,24 @@ def catalogs_fact(self) -> dict: def index_task(self) -> Task: stage = self.installed.stages["catalog.index"] - return describe(stage, CATALOG_SUBJECT, None, Env(self.store, {"catalogs": self.catalogs_fact()})) + return describe(stage, CATALOG_SUBJECT, None, Env(self.depot, {"catalogs": self.catalogs_fact()})) def index(self) -> dict: - """The catalog index: from the store when catalog.index has run, else computed here.""" + """The catalog index: from the depot when catalog.index has run, else computed here.""" if self._index is None: from ..pipeline import catalogdb t = self.index_task() - doc = self.store.result(t.key) + doc = self.depot.result(t.key) if doc is not None: (rec,) = [a for a in doc["artifacts"] if a["id"] == contract.artifact_id(t.id, "index")] - self._index = self.store.document(rec["content"]["sha256"]) + self._index = self.depot.document(rec["content"]["sha256"]) else: self._index = catalogdb.index(self.remote, self.apk_catalog) return self._index def run_index(self) -> None: - execute(self.index_task(), self.store, self.installed.stages) + execute(self.index_task(), self.depot, self.installed.stages) self._index = None def context(self) -> dict: @@ -695,7 +693,7 @@ def facts(self, pipeline: list[str], fetch: bool, background: bool = False) -> d facts["raw"] = Later(self._later) else: facts["raw"] = job[0](*job[1:]) - facts["boot"] = boot_input(self.store, self.apk) + facts["boot"] = boot_input(self.depot, self.apk) self.selected = sel self._facts, self._names = facts, {} return facts @@ -713,14 +711,15 @@ def problems(self) -> list[dict]: """Files of the selection that could not be read: {kind, stable, name, code, message}, sorted.""" return sorted(self._problems, key=lambda p: (p["kind"], p["stable"])) - def _cache_rel(self, kind: str, entry: dict, loc: dict) -> str: + def _download_rel(self, kind: str, entry: dict, loc: dict) -> str: + """Where the file of a catalog entry is in the depot's downloads/ (the path of its "download" locator).""" from ..source.addressables import remote_path prefix = "" if self.version.get("source"): from ..source.jp import Source - prefix = Source.from_dict(self.version["source"]).cache_dir(Path(".")).as_posix() + "/" + prefix = Source.from_dict(self.version["source"]).download_dir(Path(".")).as_posix() + "/" if not entry.get("remote", True) and self.version.get("apk"): prefix += "apk/" + self.version["apk"]["sha256"] + "/" if kind == "bundle": @@ -739,15 +738,15 @@ def _inputs(self, kind: str, entries: list[dict], fetch: bool) -> Inputs: out, todo = {}, [] for e in entries: loc = locs[e["location"]] - rel = self._cache_rel(kind, e, loc) - locators = ([{"kind": "cache", "path": rel}] if rel else []) + [ + rel = self._download_rel(kind, e, loc) + locators = ([{"kind": "download", "path": rel}] if rel else []) + [ {"kind": "catalog", "catalog": self.version["id"], "location": e["location"]} ] - path = self.cache / rel if self.cache is not None and rel else None + path = self.depot.downloads / rel if rel else None if path is not None and path.is_file() and path.stat().st_size > 0: todo.append((e, locators, path)) continue - known = self.store.named(memo_kind, memo_prefix + e["name"]) if e["name"] != e["stable"] else None + known = self.depot.named(memo_kind, memo_prefix + e["name"]) if e["name"] != e["stable"] else None if known is not None: # a hashed file name: its recorded identity holds out[e["stable"]] = Input(kind, known[0], known[1], e["name"], tuple(locators)) elif not e["remote"] and self.apk is None: @@ -766,7 +765,7 @@ def one(job): try: if path is None: path = self.fetcher.fetch_location(self.version["id"], e["location"]) - sha, size = self.store.identify(path, memo_kind, memo_prefix + e["name"]) + sha, size = self.depot.identify(path, memo_kind, memo_prefix + e["name"]) except ConfigError: raise except KeyError as x: # not in the APK @@ -809,8 +808,7 @@ def explain(self, pipeline: list[str], params: dict, orch=None, layouts=(), plac ctx = self.context() c = ctx["catalog"] lines = [ - f"store: {self.root}", - f"cache: {self.cache if self.cache is not None else 'not set'}", + f"depot: {self.root}", f"context: region {ctx['region'] or '-'}, language {ctx['language'] or '-'}, catalog " f"{c['label'] or '-'} ({c['id'][:12]}), APK catalog {(c['apkCatalogSha'] or '-')[:12]}", f"selection: {', '.join(self.selection)}", @@ -837,13 +835,13 @@ def explain(self, pipeline: list[str], params: dict, orch=None, layouts=(), plac # ---------------------------------------------------------------- plan def previous(self, since: str | None) -> dict | None: if not since: - return self.store.latest_run(self.selection) - doc = self.store.run(since) if contract.is_sha256(since) else None + return self.depot.latest_run(self.selection) + doc = self.depot.run(since) if contract.is_sha256(since) else None if doc is None: runs = sorted(p.name[:-5] for p in (self.root / "runs").glob("*.json") if p.name.startswith(since)) if len(runs) != 1: usage(self.args, f"--since {since}: {'no run' if not runs else f'{len(runs)} runs'} with this id") - doc = self.store.run(runs[0]) + doc = self.depot.run(runs[0]) return doc def plan(self, pipeline: list[str], params: dict, *, since=None, out=None, layouts=()): @@ -853,7 +851,7 @@ def plan(self, pipeline: list[str], params: dict, *, since=None, out=None, layou p = plan_mod.plan( self.scheduled, pipeline, - self.store, + self.depot, params=params, facts=facts, previous=self.previous(since), @@ -867,7 +865,7 @@ def plan(self, pipeline: list[str], params: dict, *, since=None, out=None, layou hits = [(n["id"], n["key"]) for n in p.doc["nodes"] if n["status"] == "hit" and self._output(n["stage"])] placed = self.layout_input(dict(hits)) for tid, key in hits: - placed.add(tid, self.store.committed(key, tid)) + placed.add(tid, self.depot.committed(key, tid)) for name, d in layout_dirs(Path(out), layouts).items(): entries, _ = self.layout_entries(name, placed, store_derived=False) p.doc["summary"]["layouts"][name] = layout.diff(d, entries) @@ -885,7 +883,7 @@ def layout_input(self, tasks: dict) -> LayoutInput: stages, asked = self.installed.stages, _Asked() makers = sorted(t for t in tasks if hasattr(stages[contract.parse_task_id(t)[0]], "derived")) for tid in makers: - stages[contract.parse_task_id(tid)[0]].derived(tid, self.store.result(tasks[tid]), self.store, asked) + stages[contract.parse_task_id(tid)[0]].derived(tid, self.depot.result(tasks[tid]), self.depot, asked) return LayoutInput(asked.objects_asked, makers) def layout_entries(self, name: str, placed: LayoutInput, *, store_derived: bool = True) -> tuple[list[dict], dict]: @@ -938,7 +936,7 @@ def names(self, stage_name: str) -> dict: stage = self.installed.stages[stage_name] got = {} if hasattr(stage, "names"): - env = Env(self.store, self._facts) + env = Env(self.depot, self._facts) for tid, key in sorted(self._done.items()): env.done(tid, key) got = stage.names(env) @@ -958,8 +956,8 @@ def _derived(self, sources, entries: list[dict], store_derived: bool, objects: d out = [] for s in makers: stage = stages[contract.parse_task_id(s.task)[0]] - for rel, data in stage.derived(s.task, s.result, self.store, paths): - sha = self.store.put(data) if store_derived else contract.sha256(data) + for rel, data in stage.derived(s.task, s.result, self.depot, paths): + sha = self.depot.put(data) if store_derived else contract.sha256(data) out.append( { "path": layout.escape_path(rel.split("/")), @@ -1129,7 +1127,7 @@ def view_gaps(results) -> int: def _write_report(d: Path, name: str, doc) -> Path: - from ..pipeline.store import write_file + from ..pipeline.depot import write_file p = d / name write_file(p, contract.encode(doc)) @@ -1149,7 +1147,7 @@ def cmd_export(args, cfg): pipeline = ws.pipeline() params = ws.params(pipeline) orch = Orchestrator( - ws.store, + ws.depot, ws.scheduled, workers=ws.workers(), memory=ws.memory(), @@ -1229,7 +1227,7 @@ def visit(tid, doc): written, shas, unplaced = {}, {}, 0 for name, d in layout_dirs(out, names).items(): entries, report = ws.layout_entries(name, placed) - w = layout.materialize(d, name, {}, entries, layout.from_store(ws.store), link=placement[0]) + w = layout.materialize(d, name, {}, entries, layout.from_depot(ws.depot), link=placement[0]) shas[name] = w["sha256"] written[name] = { "dir": str(d), @@ -1316,7 +1314,7 @@ def cmd_plan(args, cfg): ws.run_index() facts = ws.facts(["catalog.index", "unity.census"], fetch=True) Orchestrator( - ws.store, + ws.depot, ws.scheduled, workers=ws.workers(), memory=ws.memory(), @@ -1348,7 +1346,7 @@ def why(ws: Workspace, p, tid: str, since=None) -> dict: new = None if node["key"] is not None: t = next((t for t in p.tasks if t.key == node["key"]), None) - doc = None if t is not None else ws.store.result(node["key"]) + doc = None if t is not None else ws.depot.result(node["key"]) new = t.key_parts() if t is not None else (doc["keyParts"] if doc else None) prev = ws.previous(since) old_task = next((t for t in prev.get("tasks", []) if t["id"] == tid), None) if prev is not None else None @@ -1356,7 +1354,7 @@ def why(ws: Workspace, p, tid: str, since=None) -> dict: if prev is not None and old_task is not None: previous = {"run": prev["id"], "key": old_task.get("key"), "status": old_task["status"]} if old_task.get("key"): - doc = ws.store.result(old_task["key"]) + doc = ws.depot.result(old_task["key"]) old = doc["keyParts"] if doc else None return { "task": tid, @@ -1379,7 +1377,7 @@ def why_lines(r: dict) -> list[str]: ] -# ---------------------------------------------------------------- store run +# ---------------------------------------------------------------- depot run def run_lines(r: dict) -> list[str]: lines = [] for t in r["tasks"]: @@ -1398,8 +1396,8 @@ def _task_files(items) -> list[Path]: def cmd_run_stage(args, cfg): + from ..pipeline.depot import Depot from ..pipeline.orchestrate import run_one - from ..pipeline.store import Store problems, tasks = [], [] for f in _task_files(args.tasks): @@ -1423,13 +1421,12 @@ def cmd_run_stage(args, cfg): for m in problems: print(f"nnnotes: incompatible task: {m}", file=sys.stderr) sys.exit(EXIT_INCOMPATIBLE) - root = Session(cfg).store_root() - cache = cfg.path("paths", "cache") - store = Store(root, cache=cache, fetch=CatalogFetcher(root, cache, cfg) if args.fetch else None) + root = Session(cfg).depot_root() + depot = Depot(root, fetch=CatalogFetcher(root, cfg) if args.fetch else None) use(cfg) out, bad = [], 0 for t in tasks: - r = run_one(t, store, inst.stages, args.force) + r = run_one(t, depot, inst.stages, args.force) if r["ok"]: out.append({"id": t.id, "key": t.key, "status": r["status"], "result": r["resultStatus"]}) bad += r["resultStatus"] != "ok" @@ -1449,9 +1446,9 @@ def cmd_run_stage(args, cfg): # ---------------------------------------------------------------- catalog versions -def addresses_of(store, vid: str) -> dict | None: +def addresses_of(depot, vid: str) -> dict | None: """The address table (link.addresses) of the latest run over a catalog version, None when no run has one.""" - runs = sorted((store.root / "runs").glob("*.json"), key=lambda p: p.stat().st_mtime, reverse=True) + runs = sorted((depot.root / "runs").glob("*.json"), key=lambda p: p.stat().st_mtime, reverse=True) for p in runs: try: doc = contract.loads(p.read_bytes()) @@ -1467,10 +1464,10 @@ def addresses_of(store, vid: str) -> dict | None: ), None, ) - res = store.result(t["key"]) if t else None + res = depot.result(t["key"]) if t else None if res: (a,) = res["artifacts"] - return contract.loads(store.read(a["content"]["sha256"])) + return contract.loads(depot.read(a["content"]["sha256"])) return None @@ -1486,7 +1483,7 @@ def _gib(s: str) -> float: def run_args(c) -> None: - """The selection and parameters shared by `export assets` and `store plan`.""" + """The selection and parameters shared by `export assets` and `depot plan`.""" c.add_argument( "--select", action="append", @@ -1501,7 +1498,7 @@ def run_args(c) -> None: c.add_argument( "--stored-catalog", metavar="LABEL|SHA", - help="a catalog version of the store (`catalog versions`; default: the current catalog)", + help="a catalog version of the depot (`catalog versions`; default: the current catalog)", ) c.add_argument("--png-level", type=int, metavar="N", help="PNG compression level 0-9 (changes the output)") c.add_argument( diff --git a/src/nnnotes/cli/browse.py b/src/nnnotes/cli/browse.py index 9e384f6..93f2c79 100644 --- a/src/nnnotes/cli/browse.py +++ b/src/nnnotes/cli/browse.py @@ -49,13 +49,14 @@ class Region: class Server(ThreadingHTTPServer): - """The browser's HTTP server: its regions, the bundle key, the cache, and the catalogs read so far.""" + """The browser's HTTP server: its regions, the bundle key, the depot's downloads/, and the catalogs read so + far.""" - def __init__(self, address, regions: list[Region], bundle_key: BundleKey, cache: Path): + def __init__(self, address, regions: list[Region], bundle_key: BundleKey, downloads: Path): super().__init__(address, Handler) self.regions = {r.name: r for r in regions} self.bundle_key = bundle_key - self.cache = Path(cache) + self.downloads = Path(downloads) self.catalogs: dict[tuple[str, str], tuple[dict, dict]] = {} self.jp_catalogs: dict[tuple[str, str], Catalog] = {} @@ -87,12 +88,14 @@ def catalog(self, region: Region, language: str): key = (region.name, language) if key not in self.server.catalogs: if region.config is not None and region.config.provider(region.name) == "jp": - cat = open_catalog(region.config, region.name, bundle_key=self.server.bundle_key) + cat = open_catalog(region.config, region.name, self.server.downloads, bundle_key=self.server.bundle_key) self.server.catalogs[key] = browse(cat.entries) self.server.jp_catalogs[key] = cat return self.server.catalogs[key] version = Session(region.config).catalog_version(region.name) if region.config is not None else "main" - cat = Catalog.load(language, self.server.cache / "catalogs" / region.name, cdn=region.cdn, version=version) + cat = Catalog.load( + language, self.server.downloads / "catalogs" / region.name, cdn=region.cdn, version=version + ) self.server.catalogs[key] = browse(cat.entries) return self.server.catalogs[key] @@ -174,10 +177,10 @@ def do_GET(self): self.listing(region.label + "/" + language + "/" + prefix, rows) -def serve(regions: list[Region], bundle_key: BundleKey, cache: Path, port: int, host: str = "127.0.0.1") -> None: - """Browse the catalogs of `regions` and download decrypted bundles on `host:port` (catalogs cached under - /catalogs//).""" - server = Server((host, port), regions, bundle_key, cache) +def serve(regions: list[Region], bundle_key: BundleKey, downloads: Path, port: int, host: str = "127.0.0.1") -> None: + """Browse the catalogs of `regions` and download decrypted bundles on `host:port` (catalogs downloaded into + `downloads`/catalogs//).""" + server = Server((host, port), regions, bundle_key, downloads) print(f"nnnotes catalog browse: serving on {host}:{port}", flush=True) try: server.serve_forever() diff --git a/src/nnnotes/cli/bundle.py b/src/nnnotes/cli/bundle.py index 72a94bb..14b98a6 100644 --- a/src/nnnotes/cli/bundle.py +++ b/src/nnnotes/cli/bundle.py @@ -1,4 +1,4 @@ -"""`nnnotes bundle`: fetch bundles into the cache.""" +"""`nnnotes bundle`: fetch bundles into the depot's downloads.""" from __future__ import annotations @@ -16,6 +16,6 @@ def cmd_pull(args, cfg): def register(sub) -> None: bs = group(sub, "bundle", "the bundles of the catalog") c = command( - bs, "pull", cmd_pull, "fetch the bundle closure of keys into the cache (prints the cached bundle files)" + bs, "pull", cmd_pull, "fetch the bundle closure of keys into the depot (prints the downloaded bundle files)" ) c.add_argument("keys", nargs="+", metavar="KEY", help="addressable keys (`catalog keys` lists them)") diff --git a/src/nnnotes/cli/catalog.py b/src/nnnotes/cli/catalog.py index bc055ca..f1f6f14 100644 --- a/src/nnnotes/cli/catalog.py +++ b/src/nnnotes/cli/catalog.py @@ -1,4 +1,4 @@ -"""`nnnotes catalog`: the region's addressable keys, a local catalog browser, and the catalog versions of the store +"""`nnnotes catalog`: the region's addressable keys, a local catalog browser, and the catalog versions of the depot (import, fetch, list, diff).""" from __future__ import annotations @@ -40,14 +40,14 @@ def cmd_browse(args, cfg): if not langs: raise cfg.missing(f"servers.{r}", "languages") regions.append(Region(r, cfg.get(f"servers.{r}", "name") or r, cfg.cdn(r), langs, cfg)) - serve(regions, Session(cfg).bundle_key(), cfg.require_path("paths", "cache"), args.port, args.host) + serve(regions, Session(cfg).bundle_key(), Session(cfg).downloads(), args.port, args.host) -# ---------------------------------------------------------------- catalog versions of the store +# ---------------------------------------------------------------- catalog versions of the depot def _db(cfg): from ..pipeline.catalogdb import CatalogDB - return CatalogDB(Session(cfg).store_root()) + return CatalogDB(Session(cfg).depot_root()) def version_line(v: dict) -> str: @@ -145,7 +145,7 @@ def cmd_fetch(args, cfg): if cfg.provider(region) == "jp": from ..source.jp import open_catalog - cat = open_catalog(cfg, region, apk=cfg.path("paths", "apk")) + cat = open_catalog(cfg, region, Session(cfg).downloads(), apk=cfg.path("paths", "apk")) src, source = cat.sources(), cat.source assert source is not None # a JP catalog names its version source v = _db(cfg).add( @@ -163,7 +163,7 @@ def cmd_fetch(args, cfg): cdn = cfg.cdn(region) language = cfg.require("catalog", "language") version = Session(cfg).catalog_version(region) - filename = Catalog.cache_file(language, Path("."), version=version).name + filename = Catalog.file_name(language, version=version) try: data, hash_text = catalogdb.fetch(cdn, language, version=version) except OSError as e: # URLError, HTTPError, timeouts (the message has no URL) @@ -190,7 +190,7 @@ def cmd_fetch(args, cfg): def cmd_diff(args, cfg): from ..pipeline import catalogdb - from ..pipeline.store import Store + from ..pipeline.depot import Depot from .assets import addresses_of db = _db(cfg) @@ -200,9 +200,9 @@ def cmd_diff(args, cfg): vs.append(db.get(ref)) except (KeyError, ValueError) as e: usage(args, f"{ref}: {e.args[0] if e.args else e}") - store = Store(Session(cfg).store_root()) + depot = Depot(Session(cfg).depot_root()) a, b = vs - d = catalogdb.diff(db.index(a), db.index(b), addresses_of(store, a["id"]), addresses_of(store, b["id"])) + d = catalogdb.diff(db.index(a), db.index(b), addresses_of(depot, a["id"]), addresses_of(depot, b["id"])) if args.json: report(args, d) else: @@ -210,14 +210,14 @@ def cmd_diff(args, cfg): def register(sub) -> None: - cs = group(sub, "catalog", "the region's catalog: its keys, a local browser, and the catalog versions of the store") + cs = group(sub, "catalog", "the region's catalog: its keys, a local browser, and the catalog versions of the depot") c = command(cs, "keys", cmd_keys, "the addressable keys of the catalog (the total on standard error)") c.add_argument("--prefix", default="", help="only the keys that start with this prefix") c.add_argument("--limit", type=int, default=200, help="at most N keys (default 200; 0: all)") c = command(cs, "browse", cmd_browse, "browse the configured regions' catalogs and bundles in a local web page") c.add_argument("--port", type=int, default=8000, help="port (default 8000)") c.add_argument("--host", default="127.0.0.1", help="address to listen on (default 127.0.0.1)") - command(cs, "versions", cmd_versions, "the catalog versions imported into the store (--json: nnnotes.catalogs/1)") + command(cs, "versions", cmd_versions, "the catalog versions imported into the depot (--json: nnnotes.catalogs/1)") c = command(cs, "import", cmd_import, "import a catalog file (with the APK's catalog when [paths] apk is set)") c.add_argument("file", help="catalog__.bin") c.add_argument("--label", help="label of the version (default: its resource version, else the sha256 prefix)") @@ -227,13 +227,13 @@ def register(sub) -> None: help="the game's resource version of this catalog (default: the JP source's version, else the " "version in the file name catalog__.bin)", ) - c = command(cs, "fetch", cmd_fetch, "fetch the region's current catalog and import it into the store") + c = command(cs, "fetch", cmd_fetch, "fetch the region's current catalog and import it into the depot") c.add_argument("--label", help="label of the version (default: the resource version when an API root is set)") c = command( cs, "diff", cmd_diff, - "keys, bundles, raw files (and objects) that differ between two catalog versions of the store " + "keys, bundles, raw files (and objects) that differ between two catalog versions of the depot " "(--json: nnnotes.catalog-diff/1)", ) c.add_argument("old", help="label, version id or sha256 prefix") diff --git a/src/nnnotes/cli/common.py b/src/nnnotes/cli/common.py index 40a786b..257dd59 100644 --- a/src/nnnotes/cli/common.py +++ b/src/nnnotes/cli/common.py @@ -23,13 +23,12 @@ ("catalog", "language"): ("language", "--language"), ("catalog", "version"): ("catalog_version", "--catalog-version"), ("paths", "catalog"): ("catalog", "--catalog"), - ("paths", "cache"): ("cache", "--cache"), ("paths", "master"): ("master", "--master"), ("paths", "apk"): ("apk", "--apk"), ("paths", "ffmpeg"): ("ffmpeg", "--ffmpeg"), ("paths", "vgmstream"): ("vgmstream", "--vgmstream"), ("paths", "node"): ("node", "--node"), - ("paths", "store"): ("store", "--store"), + ("paths", "depot"): ("depot", "--depot"), ("paths", "player"): ("player", "--player"), } # the global options: dest -> (flag, metavar, help); --player is an option of `site build` only @@ -41,15 +40,14 @@ "LANG", "catalog and client language: ja, en, zh-Hant, zh-Hans or ko ([catalog] language)", ), - "catalog": ("--catalog", "FILE", "catalog .bin file ([paths] catalog; else downloaded into the cache)"), + "catalog": ("--catalog", "FILE", "catalog .bin file ([paths] catalog; else downloaded into the depot)"), "catalog_version": ( "--catalog-version", "VERSION", "the resource version of an international catalog ([catalog] version; else the version the game API " "serves when an API root is configured, otherwise main)", ), - "cache": ("--cache", "DIR", "cache directory ([paths] cache)"), - "store": ("--store", "DIR", "store directory ([paths] store; default /store)"), + "depot": ("--depot", "DIR", "depot directory ([paths] depot)"), "master": ("--master", "DIR", "decoded master data directory ([paths] master)"), "apk": ("--apk", "FILE", "base.apk ([paths] apk)"), "ffmpeg": ("--ffmpeg", "EXE", "ffmpeg executable ([paths] ffmpeg; else on PATH)"), diff --git a/src/nnnotes/cli/store.py b/src/nnnotes/cli/depot.py similarity index 54% rename from src/nnnotes/cli/store.py rename to src/nnnotes/cli/depot.py index 4736a4f..77f5d69 100644 --- a/src/nnnotes/cli/store.py +++ b/src/nnnotes/cli/depot.py @@ -1,5 +1,5 @@ -"""`nnnotes store`: what `export assets` would run and why (plan), task descriptions run into the store (run), and the -check of every stored content and result (verify).""" +"""`nnnotes depot`: what `export assets` would run and why (plan), task descriptions run into the depot (run), the +check of every stored content and result (verify), and the removal of what no kept run reaches (gc).""" from __future__ import annotations @@ -8,21 +8,39 @@ from ..source.config import usable_cpus from ..source.session import Session from . import assets -from .common import command, group, report, workers_arg +from .common import command, group, report, usage, workers_arg def cmd_verify(args, cfg): - from ..pipeline.store import Store + from ..pipeline.depot import Depot workers = usable_cpus() if args.workers is None else args.workers - r = Store(Session(cfg).store_root()).verify(quick=args.quick, workers=workers) + r = Depot(Session(cfg).depot_root()).verify(quick=args.quick, workers=workers) report(args, r) if r["problems"]: sys.exit(assets.EXIT_FAILED) +def cmd_gc(args, cfg): + from ..pipeline.depot import Depot + + if args.keep is not None and args.keep < 1: + usage(args, "--keep must be at least 1") + r = Depot(Session(cfg).depot_root()).gc(keep=args.keep, dry_run=args.dry_run) + report( + args, + r, + lambda r: [ + f"{'would remove' if args.dry_run else 'removed'}: {r['runs']['removed']} runs, " + f"{r['results']['removed']} results, {r['contents']['removed']} contents " + f"({r['bytesRemoved'] / 1e6:.1f} MB)", + f"kept: {r['runs']['kept']} runs, {r['results']['kept']} results, {r['contents']['kept']} contents", + ], + ) + + def register(sub) -> None: - ss = group(sub, "store", "the store of `export assets`: its plan, single tasks, its check") + ss = group(sub, "depot", "the depot: its plan, single tasks, its check, its collection") c = command( ss, @@ -38,7 +56,7 @@ def register(sub) -> None: c.add_argument("--emit-tasks", metavar="DIR", help="write the runnable tasks as task descriptions") c.add_argument("--why", metavar="TASK", help="the reasons and key parts of one task") - c = command(ss, "run", assets.cmd_run_stage, "run task descriptions (nnnotes.task/1) into the store") + c = command(ss, "run", assets.cmd_run_stage, "run task descriptions (nnnotes.task/1) into the depot") c.add_argument("tasks", nargs="+", metavar="TASK", help="task description files, or directories of them") c.add_argument("--fetch", action="store_true", help="fetch inputs through their catalog locators") c.add_argument("--force", action="store_true", help="run tasks whose result is stored (it must not change)") @@ -46,3 +64,12 @@ def register(sub) -> None: c = command(ss, "verify", cmd_verify, "check every stored content and result") c.add_argument("--quick", action="store_true", help="check sizes only, do not hash the contents") workers_arg(c, "parallel hashing", "the CPUs this process may use", zero="one at a time") + + c = command(ss, "gc", cmd_gc, "remove the runs, results and contents no kept run reaches") + c.add_argument( + "--keep", + type=int, + metavar="N", + help="keep the N latest runs of each selection (default: every run; the latest run of a selection is kept)", + ) + c.add_argument("--dry-run", action="store_true", help="count what would be removed, remove nothing") diff --git a/src/nnnotes/cli/export.py b/src/nnnotes/cli/export.py index a9bbfb6..a59b440 100644 --- a/src/nnnotes/cli/export.py +++ b/src/nnnotes/cli/export.py @@ -1,4 +1,4 @@ -"""`nnnotes export`: the selected bundles of the catalog into the store and its layouts (`assets`), and single +"""`nnnotes export`: the selected bundles of the catalog into the depot and its layouts (`assets`), and single subjects into a directory or file: an ADV episode, a story, a live, a Live2D model, a spot, a room, shaders, a cue sheet, the APK's UI library.""" @@ -235,20 +235,20 @@ def cmd_ui(args, cfg): def register(sub) -> None: - es = group(sub, "export", "game data -> common formats: the store's asset export and single subjects") + es = group(sub, "export", "game data -> common formats: the depot's asset export and single subjects") c = command( es, "assets", assets.cmd_export, - "every object of the selected bundles -> common formats (the store and its layouts)", + "every object of the selected bundles -> common formats (the depot and its layouts)", ) out_arg(c, "output directory") assets.run_args(c) c.add_argument( "--link", choices=("auto", "clone", "hard", "copy"), - help="how the layout files are made from the store: auto (default; [export] link): a clone where " + help="how the layout files are made from the depot: auto (default; [export] link): a clone where " "the file system makes them, else a hard link (read-only), else a copy; or that method only", ) c.add_argument("--strict", action="store_true", help="exit 1 when a view has gaps") diff --git a/src/nnnotes/cli/musicdata.py b/src/nnnotes/cli/musicdata.py index dab1236..c39d7e7 100644 --- a/src/nnnotes/cli/musicdata.py +++ b/src/nnnotes/cli/musicdata.py @@ -41,7 +41,7 @@ def cmd_music_data(args, cfg): None if args.no_bgm else musicdata.catalog_bgm(cat), region=region, client=deckdata.apk_client(apk) if apk is not None else {}, - catalog=deckdata.catalog_info(cat, session.store_root()), + catalog=deckdata.catalog_info(cat, session.depot_root()), deck=deck, full=args.full, jacket=musicdata.catalog_jacket(cat) if args.jackets else None, diff --git a/src/nnnotes/content/ui.py b/src/nnnotes/content/ui.py index 1e929b8..559348e 100644 --- a/src/nnnotes/content/ui.py +++ b/src/nnnotes/content/ui.py @@ -397,9 +397,7 @@ def export_apk_library( "region": cfg.get("catalog", "region"), "client": {"versionName": manifest_version_name(manifest), "versionCode": manifest_version_code(manifest)}, } - cat = Catalog( - catalog_data, cfg.require_path("paths", "cache"), bundle_key=lambda: Session(cfg).bundle_key(), apk=apk - ) + cat = Catalog(catalog_data, Session(cfg).downloads(), bundle_key=lambda: Session(cfg).bundle_key(), apk=apk) keys = keys or cat.keys(prefix) if limit is not None: keys = keys[:limit] diff --git a/src/nnnotes/formats/cri.py b/src/nnnotes/formats/cri.py index c617df5..9612584 100644 --- a/src/nnnotes/formats/cri.py +++ b/src/nnnotes/formats/cri.py @@ -22,7 +22,7 @@ `decode` writes one file per vgmstream stream (a name repeated within the sheet, e.g. the layered waveforms of one cue, gets `_`), `cues.json` (first stream per name) and `streams.json` (every stream in vgmstream order). `decode_acb_bytes` does the same for ACB bytes -given directly (no catalog, no cache: the cri.audio stage keeps its results in the store); `split_acb` and +given directly (no catalog, no cache: the cri.audio stage keeps its results in the depot); `split_acb` and `embedded_acb` give the ACB bytes of the two in-bundle layouts from their fields. HCA streams are decrypted with the keycode from the game's own boot data (see diff --git a/src/nnnotes/musicdata/deckdata.py b/src/nnnotes/musicdata/deckdata.py index 9a4a3ed..d3f0ed6 100644 --- a/src/nnnotes/musicdata/deckdata.py +++ b/src/nnnotes/musicdata/deckdata.py @@ -444,28 +444,28 @@ def apk_client(apk) -> dict: return {"versionName": manifest_version_name(data), "versionCode": manifest_version_code(data)} -def catalog_info(cat, store_root=None) -> dict: +def catalog_info(cat, depot_root=None) -> dict: """{resourceVersion, sha256} of a catalog's remote catalog file (resource_version).""" sha = hashlib.sha256(cat.sources()["remote"]).hexdigest() source = getattr(cat, "source", None) return { "resourceVersion": source.version if source - else (getattr(cat, "resource_version", None) or resource_version(store_root, sha)), + else (getattr(cat, "resource_version", None) or resource_version(depot_root, sha)), "sha256": sha, **({"resourceHash": source.hash} if source else {}), } -def resource_version(store_root, remote_sha: str) -> str | None: - """The resource version recorded for the remote catalog `remote_sha` by the catalog versions of a store +def resource_version(depot_root, remote_sha: str) -> str | None: + """The resource version recorded for the remote catalog `remote_sha` by the catalog versions of a depot (the latest import that has one), None when none is recorded. Read only.""" - if store_root is None: + if depot_root is None: return None from ..pipeline.catalogdb import CatalogDB try: - versions = CatalogDB(store_root).versions() + versions = CatalogDB(depot_root).versions() except (OSError, ValueError): return None for v in reversed(versions): diff --git a/src/nnnotes/pipeline/__init__.py b/src/nnnotes/pipeline/__init__.py index 261d5d6..4c7284d 100644 --- a/src/nnnotes/pipeline/__init__.py +++ b/src/nnnotes/pipeline/__init__.py @@ -1 +1 @@ -"""The content-addressed export pipeline: stages, tasks, the store, layouts and the atoms.""" +"""The content-addressed export pipeline: stages, tasks, the depot, layouts and the atoms.""" diff --git a/src/nnnotes/pipeline/catalogdb.py b/src/nnnotes/pipeline/catalogdb.py index b1545c6..fd3655d 100644 --- a/src/nnnotes/pipeline/catalogdb.py +++ b/src/nnnotes/pipeline/catalogdb.py @@ -1,4 +1,4 @@ -"""Catalog versions: the catalog index, a store of catalog versions, and the difference between two versions. +"""Catalog versions: the catalog index, a depot of catalog versions, and the difference between two versions. The catalog index (nnnotes.catalog-index/1) is everything a catalog pair says, decoded (addressables.parse_locations): the remote catalog and, when given, the APK's local catalog. It holds every location (primary key, internal id, @@ -9,10 +9,10 @@ references to APK files may come from another build). The index is a function of the catalog bytes alone. A catalog version is identified by the sha256 of its remote catalog and of its APK catalog (none without an APK). -The versions live in the store: +The versions live in the depot: - /catalogs/.bin each catalog file (remote and APK catalogs alike), by content - /catalogs/index.json the versions (nnnotes.catalogs/1): id, labels, region, language, the two + /catalogs/.bin each catalog file (remote and APK catalogs alike), by content + /catalogs/index.json the versions (nnnotes.catalogs/1): id, labels, region, language, the two catalogs (sha256, size, buildResultHash), the CDN's `.hash` text when it was served (an opaque change signal), resource version, APK version name @@ -35,8 +35,8 @@ from ..source.catalog import APK_CATALOG, Catalog, file_name, location_kind from . import contract from .contract import Cost, Input +from .depot import write_file from .stages import Output, Stage -from .store import write_file CATALOGS = "nnnotes.catalogs/1" DIFF = "nnnotes.catalog-diff/1" @@ -365,7 +365,7 @@ def format_diff(d: dict) -> str: return "\n".join(lines) + "\n" -# ---------------------------------------------------------------- the store of versions +# ---------------------------------------------------------------- the depot of versions def version_id(remote_sha: str, apk_sha: str | None, source: dict | None = None) -> str: """The id of a catalog version: the key hash of its two catalogs' content ids.""" identity = {"remote": remote_sha, "apk": apk_sha} @@ -399,7 +399,7 @@ def apk_version(apk: Path | None) -> str | None: def fetch(cdn: str, language: str, timeout: float = 120, *, version: str = "main") -> tuple[bytes, str | None]: """The remote catalog of `language` from a CDN base, and the text of its `.hash` file (None when not served).""" - base = cdn.rstrip("/") + "/asset/Android/" + Catalog.cache_file(language, Path("."), version=version).name + base = cdn.rstrip("/") + "/asset/Android/" + Catalog.file_name(language, version=version) with urllib.request.urlopen(base, timeout=timeout) as r: data = r.read() try: @@ -411,10 +411,10 @@ def fetch(cdn: str, language: str, timeout: float = 120, *, version: str = "main class CatalogDB: - """The catalog versions of a store (/catalogs/).""" + """The catalog versions of a depot (/catalogs/).""" - def __init__(self, store_root): - self.root = Path(store_root) / "catalogs" + def __init__(self, depot_root): + self.root = Path(depot_root) / "catalogs" def path(self, sha: str) -> Path: if not contract.is_sha256(sha): @@ -540,7 +540,7 @@ def index(self, version: dict) -> dict: return index(*self.catalog_bytes(version)) def inputs(self, version: dict) -> list[Input]: - """The inputs of the catalog.index task of a version (roles "remote" and "apk"), read from this store.""" + """The inputs of the catalog.index task of a version (roles "remote" and "apk"), read from this depot.""" out = [] for role in CATALOG_NAMES: rec = version[role] @@ -579,12 +579,12 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: size = sum(i.size for i in inputs) return Cost(0.2 + size / 10e6, (60 << 20) + 40 * size) - def run(self, task, store) -> Output: + def run(self, task, depot) -> Output: roles = {i.role for i in task.inputs} doc = index( - store.input_bytes(task.input("remote")), store.input_bytes(task.input("apk")) if "apk" in roles else None + depot.input_bytes(task.input("remote")), depot.input_bytes(task.input("apk")) if "apk" in roles else None ) - content = store.add(contract.encode(doc), "json") + content = depot.add(contract.encode(doc), "json") rec = contract.artifact( contract.artifact_id(task.id, "index"), content, diff --git a/src/nnnotes/pipeline/census.py b/src/nnnotes/pipeline/census.py index 4eb71e2..2875e36 100644 --- a/src/nnnotes/pipeline/census.py +++ b/src/nnnotes/pipeline/census.py @@ -330,9 +330,9 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: size = sum(i.size for i in inputs) return Cost(0.002 + size / 40e6, (32 << 20) + 12 * size) - def run(self, task, store) -> Output: - doc = census_file(store.open_input(task.input("bundle"))) - content = store.add(contract.encode(doc), "json") + def run(self, task, depot) -> Output: + doc = census_file(depot.open_input(task.input("bundle"))) + content = depot.add(contract.encode(doc), "json") rec = contract.artifact( contract.artifact_id(task.id, "census"), content, diff --git a/src/nnnotes/pipeline/contract.py b/src/nnnotes/pipeline/contract.py index f2dd92b..8c958c8 100644 --- a/src/nnnotes/pipeline/contract.py +++ b/src/nnnotes/pipeline/contract.py @@ -56,7 +56,7 @@ RESULT_STATUSES = ("ok", "partial") RUN_TASK_STATUSES = ("hit", "ran", "failed") # locator kinds of a task input: where the bytes can be read (never part of the key) -LOCATOR_KINDS = ("store", "cache", "file", "catalog") +LOCATOR_KINDS = ("depot", "download", "file", "catalog") # an object's primary artifact is the one whose role comes first here; its other artifacts are secondary PRIMARY_ROLES = ("image", "mesh", "audio", "video", "font", "data", "prefab", "json") MEDIA_TYPES = { diff --git a/src/nnnotes/pipeline/cristages.py b/src/nnnotes/pipeline/cristages.py index 87f314d..8d4d794 100644 --- a/src/nnnotes/pipeline/cristages.py +++ b/src/nnnotes/pipeline/cristages.py @@ -70,30 +70,30 @@ # ---------------------------------------------------------------- inputs shared by the stages -def boot_input(store, apk) -> Input | None: +def boot_input(depot, apk) -> Input | None: """The input "boot" of the CRI stages: the boot data of an APK (`assets/bin/Data/data.unity3d`, where the HCA - keycode is read), stored in `store` once; the APK is hashed once (store.identify memo) and its boot data is + keycode is read), stored in `depot` once; the APK is hashed once (depot.identify memo) and its boot data is remembered by the APK's content id. A path that is not an .apk / .zip is the boot data itself. None without an APK.""" if apk is None: return None apk = Path(apk) if not apk.is_dir() and apk.suffix.lower() not in (".apk", ".zip", ".apks", ".xapk"): - sha, size = store.identify(apk) + sha, size = depot.identify(apk) return Input("boot", sha, size, apk.name, ({"kind": "file", "path": str(apk.resolve())},)) if apk.is_dir(): with ApkSet(apk) as z: data = z.read(BOOT_IN_APK) - sha = store.put(data) - return Input("boot", sha, len(data), "data.unity3d", ({"kind": "store"},)) - apk_sha, _ = store.identify(apk) + sha = depot.put(data) + return Input("boot", sha, len(data), "data.unity3d", ({"kind": "depot"},)) + apk_sha, _ = depot.identify(apk) name = f"boot:{apk_sha}" - known = store.named("files", name) - if known is None or not store.has(*known): + known = depot.named("files", name) + if known is None or not depot.has(*known): with ApkSet(apk) as z: data = z.read(BOOT_IN_APK) - known = store.identify(store.path(store.put(data)), "files", name) - return Input("boot", known[0], known[1], "data.unity3d", ({"kind": "store"},)) + known = depot.identify(depot.path(depot.put(data)), "files", name) + return Input("boot", known[0], known[1], "data.unity3d", ({"kind": "depot"},)) def _boot(env) -> Input: @@ -103,14 +103,14 @@ def _boot(env) -> Input: return inp -def hca_key(store, inp: Input) -> int: +def hca_key(depot, inp: Input) -> int: """The HCA keycode in the boot data `inp` (crikey.find_key), read once per content and process.""" with _lock: if inp.sha256 in _keys: return _keys[inp.sha256] from ..formats import crikey - key = crikey.find_key(store.open_input(inp)) + key = crikey.find_key(depot.open_input(inp)) with _lock: _keys[inp.sha256] = key return key @@ -153,19 +153,19 @@ def _tool(which: str) -> str: return tool(which, "vgmstream-cli" if which == "vgmstream" else "ffmpeg") -def _magic_of(store, inp: Input, name: str) -> bytes: - """The first 4 bytes of a raw file, from a local copy (the store, the cache, a file); one that is not fetched +def _magic_of(depot, inp: Input, name: str) -> bytes: + """The first 4 bytes of a raw file, from a local copy (the depot, the cache, a file); one that is not fetched yet is Pending("fetch:").""" with _lock: m = _magic.get(inp.sha256) if m is None: - from .store import InputMissing + from .depot import InputMissing local = Input( inp.role, inp.sha256, inp.size, inp.name, tuple(loc for loc in inp.locators if loc.get("kind") != "catalog") ) try: - path = store.open_input(local) + path = depot.open_input(local) except InputMissing: raise Pending(f"fetch:{inp.name or name}") from None with open(path, "rb") as f: @@ -187,7 +187,7 @@ def _raw(env) -> dict[str, tuple[Input, bytes]]: raise Pending(f"fetch:{stable}") if isinstance(inp, Exception): raise inp - out[stable] = (inp, _magic_of(env.store, inp, stable)) + out[stable] = (inp, _magic_of(env.depot, inp, stable)) except Pending as p: waiting.append(p) if waiting: @@ -279,11 +279,11 @@ def _check_atoms(self, task, want: dict) -> None: ) def _artifact( - self, task, store, role: str, data: bytes, ext: str, kind: str, fmt: str, facts: dict, media: str | None = None + self, task, depot, role: str, data: bytes, ext: str, kind: str, fmt: str, facts: dict, media: str | None = None ) -> dict: return contract.artifact( contract.artifact_id(task.id, role), - store.add(data, ext, media), + depot.add(data, ext, media), contract.provenance(task), {"kind": kind, "format": fmt, "facts": facts}, ) @@ -353,11 +353,11 @@ def entry(sha: str) -> dict: a = raw[other][0] e["awb"] = Input("awb", a.sha256, a.size, a.name, a.locators) for tid in tids: - for a in _export_artifacts(env.store, env.key(tid), ACB_KIND): + for a in _export_artifacts(env.depot, env.key(tid), ACB_KIND): c = a["content"] e = entry(c["sha256"]) if e["acb"] is None: - e["acb"] = Input("acb", c["sha256"], c["size"], None, ({"kind": "store"},)) + e["acb"] = Input("acb", c["sha256"], c["size"], None, ({"kind": "depot"},)) e["depends"].add(tid) sheet = (a["semantics"].get("facts") or {}).get("cueSheet") if sheet: @@ -378,7 +378,7 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: size = sum(i.size for i in inputs if i.role == "acb") return Cost(0.3 + size * 2.0e-6, (64 << 20) + 8 * size) - def run(self, task, store) -> Output: + def run(self, task, depot) -> Output: from ..formats import cri vgm, ff = _tool("vgmstream"), _tool("ffmpeg") @@ -390,8 +390,8 @@ def run(self, task, store) -> Output: "unsupported.cri.awb_external", "the cue sheet's waveforms are streamed from an external AWB" ), ) - acb = store.input_bytes(task.input("acb")) - key = hca_key(store, task.input("boot")) + acb = depot.input_bytes(task.input("acb")) + key = hca_key(depot, task.input("boot")) level = int(task.params["flac"]["level"]) with tempfile.TemporaryDirectory(prefix="nnnotes-cri-") as tmp: out = Path(tmp) @@ -410,7 +410,7 @@ def run(self, task, store) -> Output: arts = [ self._artifact( task, - store, + depot, s["file"], (out / s["file"]).read_bytes(), "flac", @@ -423,7 +423,7 @@ def run(self, task, store) -> Output: arts.append( self._artifact( task, - store, + depot, "cues.json", (out / "cues.json").read_bytes(), "json", @@ -435,7 +435,7 @@ def run(self, task, store) -> Output: arts.append( self._artifact( task, - store, + depot, "streams.json", (out / "streams.json").read_bytes(), "json", @@ -447,11 +447,11 @@ def run(self, task, store) -> Output: return self._item(task, arts) -def _export_artifacts(store, key: str, kind: str) -> list[dict]: +def _export_artifacts(depot, key: str, kind: str) -> list[dict]: """The artifact records of semantics kind `kind` (cri.acb, cri.usm) of a unity.export result (a result without the kind is not parsed).""" try: - data = store.result_path(key).read_bytes() + data = depot.result_path(key).read_bytes() except OSError: return [] if f'"{kind}"'.encode() not in data: @@ -521,9 +521,9 @@ def entry(inp: Input) -> dict: e["names"] |= cat.keys.get(stable) or {stable} for tid in tids: bundle = contract.parse_task_id(tid, EXPORT)[1] - for a in _export_artifacts(env.store, env.key(tid), USM_KIND): + for a in _export_artifacts(env.depot, env.key(tid), USM_KIND): c = a["content"] - e = entry(Input("usm", c["sha256"], c["size"], None, ({"kind": "store"},))) + e = entry(Input("usm", c["sha256"], c["size"], None, ({"kind": "depot"},))) e["depends"].add(tid) movie = (a["semantics"].get("facts") or {}).get("movie") keys = {k for k in cat.bundle_keys.get(bundle, ()) if movie and k.rsplit("/", 1)[-1] == movie} @@ -542,13 +542,13 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: size = sum(i.size for i in inputs if i.role == "usm") return Cost(0.3 + size * 1.1e-7, (96 << 20) + 12 * size) - def run(self, task, store) -> Output: + def run(self, task, depot) -> Output: from ..formats import usm ff, vgm = _tool("ffmpeg"), _tool("vgmstream") self._check_atoms(task, movie_atoms(ff, vgm)) - movie = store.input_bytes(task.input("usm")) - key = hca_key(store, task.input("boot")) + movie = depot.input_bytes(task.input("usm")) + key = hca_key(depot, task.input("boot")) try: streams = usm.demux(movie, key) except usm.Unsupported as e: @@ -556,7 +556,7 @@ def run(self, task, store) -> Output: arts, why, tracks = [], None, [] for s in streams: if s.kind == "subtitle": - made, problem = self._subtitle(task, store, s) + made, problem = self._subtitle(task, depot, s) arts += made if problem is None: tracks.append(s) @@ -573,7 +573,7 @@ def run(self, task, store) -> Output: kind = "audio.stream" else: facts, kind = usm.video_facts(s), "video.stream" if s.kind == "video" else "video.alpha" - arts.append(self._artifact(task, store, f"{s.name}.{ext}", s.data, ext, kind, fmt, facts, media)) + arts.append(self._artifact(task, depot, f"{s.name}.{ext}", s.data, ext, kind, fmt, facts, media)) tracks.append(s) out_fmt = task.params["format"] with tempfile.TemporaryDirectory(prefix="nnnotes-usm-") as tmp: @@ -591,11 +591,11 @@ def run(self, task, store) -> Output: facts = {"tracks": [_track(s, out_fmt, reencoded) for s in tracks]} fmt, media = MOVIE_FORMATS[out_fmt] arts.append( - self._artifact(task, store, dst.name, dst.read_bytes(), out_fmt, "video.movie", fmt, facts, media) + self._artifact(task, depot, dst.name, dst.read_bytes(), out_fmt, "video.movie", fmt, facts, media) ) return self._item(task, arts, why) - def _subtitle(self, task, store, s) -> tuple[list[dict], str | None]: + def _subtitle(self, task, depot, s) -> tuple[list[dict], str | None]: """The artifacts of a subtitle channel and why it is not a track (None when it is): the records as JSON, and as SubRip and WebVTT when every text is UTF-8; the bytes as stored (`.sbt`) when they are not records.""" from ..core.jsonio import dumps @@ -606,12 +606,12 @@ def _subtitle(self, task, store, s) -> tuple[list[dict], str | None]: except ValueError as e: return [ self._artifact( - task, store, f"{s.name}.sbt", s.data, "sbt", "subtitle.stream", "sbt", {"channel": s.channel} + task, depot, f"{s.name}.sbt", s.data, "sbt", "subtitle.stream", "sbt", {"channel": s.channel} ) ], f"{s.name}: {e}" facts = {"channel": s.channel, "records": len(records)} doc = dumps(usm.subtitle_json(s.channel, records), indent=1, ensure_ascii=False).encode() - arts = [self._artifact(task, store, f"{s.name}.json", doc, "json", "subtitle.records", "json", facts)] + arts = [self._artifact(task, depot, f"{s.name}.json", doc, "json", "subtitle.records", "json", facts)] try: srt, vtt = usm.subtitle_srt(records), usm.subtitle_webvtt(records) except UnicodeDecodeError: @@ -619,7 +619,7 @@ def _subtitle(self, task, store, s) -> tuple[list[dict], str | None]: arts.append( self._artifact( task, - store, + depot, f"{s.name}.srt", srt.encode(), "srt", @@ -631,7 +631,7 @@ def _subtitle(self, task, store, s) -> tuple[list[dict], str | None]: ) arts.append( self._artifact( - task, store, f"{s.name}.vtt", vtt.encode(), "vtt", "subtitle.track", "webvtt", facts, "text/vtt" + task, depot, f"{s.name}.vtt", vtt.encode(), "vtt", "subtitle.track", "webvtt", facts, "text/vtt" ) ) return arts, None diff --git a/src/nnnotes/pipeline/store.py b/src/nnnotes/pipeline/depot.py similarity index 75% rename from src/nnnotes/pipeline/store.py rename to src/nnnotes/pipeline/depot.py index 8e8acc5..027fae0 100644 --- a/src/nnnotes/pipeline/store.py +++ b/src/nnnotes/pipeline/depot.py @@ -1,21 +1,24 @@ -"""The content-addressed store of the asset stages: a documented format, and the cache of their work. +"""The depot: the files a configuration downloads, the results of the stages by content, and the record of runs. - /cas/sha256/ab/ bytes by content id, write-once (no extension: the records carry it) - /ac/ab/.json task results (nnnotes.result/1) by task key; written after every byte they + /downloads/ the game files as downloaded: catalogs, decrypted bundles (bundles/), + raw files (raw/), files read from the APK, each source in its own + directory (international///, jp////) + /cas/sha256/ab/ bytes by content id, write-once (no extension: the records carry it) + /ac/ab/.json task results (nnnotes.result/1) by task key; written after every byte they name is in cas/, so a result present is a complete result (the commit) - /inputs/.jsonl input identities: name -> sha256, size, with the (path, size, mtime) of the + /inputs/.jsonl input identities: name -> sha256, size, with the (path, size, mtime) of the file that was hashed (a memo: a file seen unchanged is not hashed again); bundles.jsonl, raw.jsonl, files.jsonl (files verified by path only) - /catalogs/ versioned catalogs (catalogdb) - /runs/.json run manifests (nnnotes.run/1), deterministic - /runs/.log.jsonl the log of the run's last execution (timings, workers, errors) - /runs/selections/ the id of the latest run of a selection (digest of its selection) - /costs/.json the measured cost of a task's last run (nnnotes.cost/1) + /catalogs/ versioned catalogs (catalogdb) + /runs/.json run manifests (nnnotes.run/1), deterministic + /runs/.log.jsonl the log of the run's last execution (timings, workers, errors) + /runs/selections/ the id of the latest run of a selection (digest of its selection) + /costs/.json the measured cost of a task's last run (nnnotes.cost/1) Every file is written through a temporary file and a rename, so concurrent writers (threads, processes, machines on a shared file system) never see or leave a partial file, and a crash before a result's entry leaves no false hit. The files of cas/ are read-only (a layout may hard-link them: an edit of such a file fails instead of changing the -store). +depot). Only costs/, runs/ and inputs/ hold non-deterministic data; cas/ and ac/ are a function of the tasks run. """ @@ -39,15 +42,15 @@ class InputMissing(FileNotFoundError): """No locator of an input gives a file with its content id.""" -class StoreConflict(RuntimeError): - """A result already in the store differs from the one being committed for the same key: the stage's output is +class DepotConflict(RuntimeError): + """A result already in the depot differs from the one being committed for the same key: the stage's output is not a function of its key (a nondeterministic stage, or a change without a version bump).""" def _replace(tmp: Path, dst: Path, same) -> bool: """Rename `tmp` onto `dst`; False when another writer put `dst` first (`same(dst)` true) and `tmp` was dropped. On some systems a rename onto (or a read of) a file another process is replacing or reading fails for a - moment: retried, as sitefiles.Store does.""" + moment: retried, as sitefiles.Depot does.""" try: for i in range(20): try: @@ -61,7 +64,7 @@ def _replace(tmp: Path, dst: Path, same) -> bool: except PermissionError: pass time.sleep(0.1 * (i + 1)) - raise RuntimeError(f"could not store {dst}") + raise RuntimeError(f"could not depot {dst}") except BaseException: _drop(tmp) raise @@ -90,7 +93,7 @@ def _read(p: Path) -> bytes | None: def _write(dst: Path, data: bytes, same=None, read_only: bool = False) -> bool: """Write `data` to `dst` atomically (creating the directory); with `same`, keep an existing `dst` for which - same(dst) holds; `read_only`: the file is read-only (store objects). True when this call wrote the file.""" + same(dst) holds; `read_only`: the file is read-only (depot objects). True when this call wrote the file.""" dst.parent.mkdir(parents=True, exist_ok=True) tmp = temp_path(dst) try: @@ -157,29 +160,33 @@ def _lines(path: Path): continue -class Store: - """A store directory. `cache`: the directory "cache" locators are relative to; `fetch(input, locator) -> Path`: - how a "catalog" locator is fetched (none: such locators are skipped).""" +class Depot: + """A depot directory. `fetch(input, locator) -> Path`: how a "catalog" locator is fetched (none: such locators + are skipped).""" - def __init__(self, root, *, cache=None, fetch=None): + def __init__(self, root, *, fetch=None): self.root = Path(root) - self.cache = Path(cache) if cache else None self.fetch = fetch self.written = self.reused = 0 - self._have: set[str] = set() # content ids this store object wrote or found + self._have: set[str] = set() # content ids this depot object wrote or found self._memo: dict | None = None # (path, size, mtime_ns) -> (sha256, size) self._names: dict[str, dict] = {} # kind -> {name: (sha256, size)} self._docs: dict[str, dict] = {} # the documents document() decoded last self._lock = threading.Lock() - def __getstate__(self): # a store crosses to worker processes as its settings - return {"root": self.root, "cache": self.cache, "fetch": self.fetch} + def __getstate__(self): # a depot crosses to worker processes as its settings + return {"root": self.root, "fetch": self.fetch} def __setstate__(self, state): - self.__init__(state["root"], cache=state["cache"], fetch=state["fetch"]) + self.__init__(state["root"], fetch=state["fetch"]) def __repr__(self) -> str: - return f"Store({str(self.root)!r})" + return f"Depot({str(self.root)!r})" + + @property + def downloads(self) -> Path: + """The directory of downloaded game files ("download" locators are relative to it).""" + return self.root / "downloads" @property def catalogs(self) -> Path: @@ -315,7 +322,7 @@ def committed(self, key: str, task: str) -> dict: """The committed result of a task key that has one (a FileNotFoundError naming `task` when it is missing).""" doc = self.result(key) if doc is None: - raise FileNotFoundError(f"{task}: its result {key} is not in the store") + raise FileNotFoundError(f"{task}: its result {key} is not in the depot") return doc def has_result(self, key: str) -> bool: @@ -323,7 +330,7 @@ def has_result(self, key: str) -> bool: def commit(self, doc: dict) -> bool: """Write a result entry, after checking that the bytes of every artifact it names are stored. Write-once: an - entry already there must be identical (else StoreConflict). True when this call wrote it.""" + entry already there must be identical (else DepotConflict). True when this call wrote it.""" if doc.get("schema") != contract.RESULT: raise ValueError("not a result document") for a in doc["artifacts"]: @@ -338,7 +345,7 @@ def same(p: Path) -> bool: if old is None: return False if old != data: - raise StoreConflict(f"result {doc['key']} differs from the one in the store ({dst})") + raise DepotConflict(f"result {doc['key']} differs from the one in the depot ({dst})") return True if same(dst): @@ -401,27 +408,24 @@ def named(self, kind: str, name: str) -> tuple[str, int] | None: return self._names.get(kind, {}).get(name) def open_input(self, inp: Input) -> Path: - """A file holding the bytes of `inp`: its locators in order (none: the store), each checked against the - content id (store: by name and size; files: hashed, memoized). Raises InputMissing.""" + """A file holding the bytes of `inp`: its locators in order (none: the depot), each checked against the + content id (depot: by name and size; files: hashed, memoized). Raises InputMissing.""" tried = [] - for loc in inp.locators or ({"kind": "store"},): + for loc in inp.locators or ({"kind": "depot"},): kind = loc["kind"] try: - if kind == "store": + if kind == "depot": if self.has(inp.sha256, inp.size): return self.path(inp.sha256) - tried.append("store: absent") + tried.append("depot: absent") continue if kind == "catalog": if self.fetch is None: tried.append("catalog: no fetcher") continue p = Path(self.fetch(inp, loc)) - elif kind == "cache": - if self.cache is None: - tried.append("cache: no cache directory") - continue - p = self.cache / loc["path"] + elif kind == "download": + p = self.downloads / loc["path"] else: p = Path(loc["path"]) if not p.is_file() or p.stat().st_size != inp.size: @@ -440,7 +444,7 @@ def input_bytes(self, inp: Input) -> bytes: # ---------------------------------------------------------------- verification def verify(self, *, quick: bool = False, workers: int = 8, limit: int = 100) -> dict: - """Check the store: every cas/ file has the content its name says (`quick`: only that it is a file of a + """Check the depot: every cas/ file has the content its name says (`quick`: only that it is a file of a well-formed name; its size is checked against the results naming it), every ac/ entry is a result whose name is its key, whose key is the key of its parts and whose artifacts' bytes are stored with their size. -> {"contents", "results", "problems": [{path, problem}] (the first `limit`, sorted), "problemCount"}.""" @@ -540,3 +544,92 @@ def cost(self, key: str) -> dict | None: return contract.loads((self.root / "costs" / f"{key}.json").read_bytes()) except OSError: return None + + # ---------------------------------------------------------------- collection + def gc(self, *, keep: int | None = None, dry_run: bool = False) -> dict: + """Remove what no kept run reaches. The kept runs: of each selection the `keep` latest by the time their + manifest was written (None: every run), and always the run the selection names as its latest. A kept run + reaches the results of its tasks, the contents those results name (their artifacts, and the inputs their key + parts name when the depot holds them) and the layout manifests it names with the contents they list. + Removed: the other runs (manifest and log), results and their measured costs, and contents. `dry_run`: count + only. -> {"runs", "results", "contents": {"kept", "removed"}, "bytesRemoved"}.""" + runs_dir = self.root / "runs" + manifests = sorted(p for p in runs_dir.glob("*.json") if p.is_file()) + latest = set() + for p in (runs_dir / "selections").glob("*"): + try: + latest.add(p.read_text(encoding="ascii").strip()) + except OSError: + continue + by_selection: dict[str, list[tuple[int, Path, dict]]] = {} + for p in manifests: + try: + doc = contract.loads(p.read_bytes()) + except (OSError, ValueError): + continue + sel = contract.digest(list(doc.get("selection") or ())) + by_selection.setdefault(sel, []).append((p.stat().st_mtime_ns, p, doc)) + kept, dropped = [], [] + for runs in by_selection.values(): + runs.sort(key=lambda r: r[0], reverse=True) + for i, (_, p, doc) in enumerate(runs): + (kept if keep is None or i < keep or doc.get("id") in latest else dropped).append((p, doc)) + keys: set[str] = set() + shas: set[str] = set() + for _, doc in kept: + for t in doc.get("tasks") or (): + if t.get("key"): + keys.add(t["key"]) + for sha in (doc.get("layouts") or {}).values(): + shas.add(sha) + if self.has(sha): + _collect_shas(contract.loads(self.read(sha)), shas) + for key in keys: + res = self.result(key) + if res is None: + continue + shas.update(a["content"]["sha256"] for a in res.get("artifacts", ())) + shas.update(sha for _, sha in (res.get("keyParts") or {}).get("inputs", ())) + removed_bytes = 0 + out: dict = {"runs": {"kept": len(kept), "removed": len(dropped)}} + for p, _ in dropped: + if not dry_run: + p.unlink(missing_ok=True) + p.with_name(p.name[: -len(".json")] + ".log.jsonl").unlink(missing_ok=True) + n_kept = n_removed = 0 + for p in sorted((self.root / "ac").glob("*/*.json")): + key = p.name[: -len(".json")] + if key in keys: + n_kept += 1 + continue + n_removed += 1 + if not dry_run: + p.unlink(missing_ok=True) + (self.root / "costs" / f"{key}.json").unlink(missing_ok=True) + out["results"] = {"kept": n_kept, "removed": n_removed} + n_kept = n_removed = 0 + for p in sorted((self.root / "cas" / "sha256").glob("*/*")): + if p.name in shas: + n_kept += 1 + continue + n_removed += 1 + removed_bytes += p.stat().st_size + if not dry_run: + _drop(p) + self._have.discard(p.name) + out["contents"] = {"kept": n_kept, "removed": n_removed} + out["bytesRemoved"] = removed_bytes + return out + + +def _collect_shas(doc, out: set[str]) -> None: + """Add every "sha256" value of a document (a layout manifest names its files' contents so).""" + if isinstance(doc, dict): + sha = doc.get("sha256") + if isinstance(sha, str) and contract.is_sha256(sha): + out.add(sha) + for v in doc.values(): + _collect_shas(v, out) + elif isinstance(doc, list): + for v in doc: + _collect_shas(v, out) diff --git a/src/nnnotes/pipeline/layout.py b/src/nnnotes/pipeline/layout.py index 6bc7495..1ef6591 100644 --- a/src/nnnotes/pipeline/layout.py +++ b/src/nnnotes/pipeline/layout.py @@ -39,9 +39,9 @@ place is reported and left out of the manifest (the next run tries it again); no file falls back to another method. Placement (PLACEMENTS; probe() picks one per run for `auto` and checks an explicit one): a clone (reflink: Linux -FICLONE, macOS clonefile) shares the store object's blocks until either file is written, so it is independent and -takes no space; a hard link is the store object itself, read-only like every object of the store (an edit fails -instead of changing the store); a copy is independent and writable, and takes the space again. Every method gives +FICLONE, macOS clonefile) shares the depot object's blocks until either file is written, so it is independent and +takes no space; a hard link is the depot object itself, read-only like every object of the depot (an edit fails +instead of changing the depot); a copy is independent and writable, and takes the space again. Every method gives the same files and the same manifest. """ @@ -57,7 +57,7 @@ from ..core.cache import temp_path from . import contract -from .store import write_doc, write_file +from .depot import write_doc, write_file MANIFEST = "manifest.json" JOURNAL = ".nnnotes-layout-journal.json" @@ -366,9 +366,9 @@ def paths(entries_: list[dict]) -> dict[str, str]: # ---------------------------------------------------------------- writing -def from_store(store): - """A byte source reading the store's cas/.""" - return store.path +def from_depot(depot): + """A byte source reading the depot's cas/.""" + return depot.path def from_layout(directory): @@ -429,7 +429,7 @@ def diff(out_dir, entries_: list[dict]) -> dict: def materialize(out_dir, name: str, params: dict, entries_: list[dict], source, *, link: str = "copy") -> dict: """Write layout `name` into `out_dir` (see the module documentation). `source(sha256) -> Path` gives the bytes - (from_store, from_layout); `link`: the placement (PLACEMENTS). -> {"manifest": the layout document (without + (from_depot, from_layout); `link`: the placement (PLACEMENTS). -> {"manifest": the layout document (without the files not placed), "sha256": of its file, "write", "remove", "keep", "cloned", "linked", "copied", "failed": [{path, message}] of the files not placed}.""" if link not in PLACEMENTS: @@ -449,7 +449,7 @@ def materialize(out_dir, name: str, params: dict, entries_: list[dict], source, stats = {"cloned": 0, "linked": 0, "copied": 0} failed = [] - def restore(p): # a replaced or removed hard link's store object: read-only again + def restore(p): # a replaced or removed hard link's depot object: read-only again if p in owned: try: _read_only(Path(source(owned[p][0]))) @@ -498,7 +498,7 @@ def _place(src: Path, dst: Path, link: str, stats: dict, restore) -> None: if link == "clone": clone(src, tmp) elif link == "hard": - _read_only(src) # an object stored before the store made them read-only + _read_only(src) # an object stored before the depot made them read-only os.link(src, tmp) else: shutil.copyfile(src, tmp) @@ -513,7 +513,7 @@ def _place(src: Path, dst: Path, link: str, stats: dict, restore) -> None: def _read_only(p: Path) -> None: - """Make a store object read-only (its hard links in layouts are the object itself).""" + """Make a depot object read-only (its hard links in layouts are the object itself).""" try: mode = os.stat(p).st_mode if mode & 0o222: @@ -524,7 +524,7 @@ def _read_only(p: Path) -> None: def _writable(p: Path) -> bool: """Clear the read-only attribute of a layout file about to be replaced or removed, where the system refuses - to do either to a read-only file (Windows; the attribute belongs to the file, so a hard link's store object + to do either to a read-only file (Windows; the attribute belongs to the file, so a hard link's depot object loses it too: True, the caller sets it again). Elsewhere nothing: False.""" if os.name != "nt": return False @@ -570,7 +570,7 @@ def clone(src: Path, dst: Path) -> None: def _why(e: OSError, what: str) -> str: if e.errno in _ELSEWHERE: - return "store on another file system" + return "depot on another file system" if "platform" in str(e): return f"{what} unsupported on this platform" code = e.errno if e.errno is None else errno.errorcode.get(e.errno, e.errno) @@ -578,7 +578,7 @@ def _why(e: OSError, what: str) -> str: def probe(out_dir, store_dir, want: str = "auto") -> tuple[str, str]: - """The placement of a layout in `out_dir` whose files come from the store objects in `store_dir`, for `want` + """The placement of a layout in `out_dir` whose files come from the depot objects in `store_dir`, for `want` (LINKS), tried once on a temporary file in each: (placement, reason). auto: a clone where the file system makes them, else a hard link on the same file system, else a copy. An explicit clone or hard link that does not work raises LayoutError.""" @@ -586,10 +586,10 @@ def probe(out_dir, store_dir, want: str = "auto") -> tuple[str, str]: raise ValueError(f"unknown link mode {want!r} (one of {', '.join(LINKS)})") if want == "copy": return "copy", "copy: as asked" - out, store = Path(out_dir), Path(store_dir) + out, depot = Path(out_dir), Path(store_dir) out.mkdir(parents=True, exist_ok=True) - store.mkdir(parents=True, exist_ok=True) - src, made = temp_path(store / "probe"), [] + depot.mkdir(parents=True, exist_ok=True) + src, made = temp_path(depot / "probe"), [] try: with open(src, "xb") as f: f.write(b"nnnotes placement probe\n") @@ -598,7 +598,7 @@ def probe(out_dir, store_dir, want: str = "auto") -> tuple[str, str]: made.append(temp_path(out / "probe")) try: clone(src, made[-1]) - return "clone", "clone: the file system shares the store objects' blocks" + return "clone", "clone: the file system shares the depot objects' blocks" except OSError as e: why = _why(e, "clone") if want == "clone": diff --git a/src/nnnotes/pipeline/link.py b/src/nnnotes/pipeline/link.py index f0222c4..9a6ea97 100644 --- a/src/nnnotes/pipeline/link.py +++ b/src/nnnotes/pipeline/link.py @@ -246,8 +246,8 @@ def _census_input(env, tid: str) -> Input: return env.input_of(tid, "census", as_role="census:" + contract.parse_task_id(tid, census.CensusStage.name)[1]) -def _read(store, task, role: str) -> dict: - return contract.loads(store.input_bytes(task.input(role))) +def _read(depot, task, role: str) -> dict: + return contract.loads(depot.input_bytes(task.input(role))) class ScriptsStage(Stage): @@ -275,9 +275,9 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: size = sum(i.size for i in inputs) return Cost(0.05 + size / 50e6, (40 << 20) + 8 * size) - def run(self, task, store) -> Output: - doc = scripts(_read(store, task, i.role) for i in task.inputs) - content = store.add(contract.encode(doc), "json") + def run(self, task, depot) -> Output: + doc = scripts(_read(depot, task, i.role) for i in task.inputs) + content = depot.add(contract.encode(doc), "json") rec = contract.artifact( contract.artifact_id(task.id, "scripts"), content, @@ -303,7 +303,7 @@ def subjects(self, env) -> list[str]: def inputs(self, subject: str, env) -> list[Input]: tid = contract.task_id(catalogdb.IndexStage.name, subject) idx = env.input_of(tid, "index") - stables = {b["stable"] for b in env.store.document(idx.sha256)["bundles"]} + stables = {b["stable"] for b in env.depot.document(idx.sha256)["bundles"]} wanted = {contract.task_id(census.CensusStage.name, s) for s in stables} pending = [t for t in env.pending(census.CensusStage.name) if t in wanted] if pending: @@ -314,13 +314,13 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: size = sum(i.size for i in inputs) return Cost(0.5 + size / 50e6, (200 << 20) + size // 2) - def run(self, task, store) -> Output: - idx = _read(store, task, "index") + def run(self, task, depot) -> Output: + idx = _read(depot, task, "index") docs = ( - (i.role.split(":", 1)[1], _read(store, task, i.role)) for i in task.inputs if i.role.startswith("census:") + (i.role.split(":", 1)[1], _read(depot, task, i.role)) for i in task.inputs if i.role.startswith("census:") ) doc = addresses(idx, docs) - content = store.add(contract.encode(doc), "json") + content = depot.add(contract.encode(doc), "json") facts = { "bundles": len(doc["bundles"]), "keys": len(doc["keys"]), @@ -426,16 +426,16 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: n = len(self._tasks(env)) return Cost(0.05 + 0.002 * n, (60 << 20) + 200_000 * n) - def run(self, task, store) -> Output: + def run(self, task, depot) -> Output: def results(): for tid, key in sorted(task.context.get("results", {}).items()): - doc = store.result(key) + doc = depot.result(key) if doc is None: - raise FileNotFoundError(f"result of {tid} ({key}) is not in the store") + raise FileNotFoundError(f"result of {tid} ({key}) is not in the depot") yield tid, doc objects, contained, arts = artifact_table(results()) - content = store.add_chunks(encode_artifacts(objects, contained, arts), "json") + content = depot.add_chunks(encode_artifacts(objects, contained, arts), "json") facts = {"objects": len(objects), "contained": len(contained), "artifacts": len(arts)} rec = contract.artifact( contract.artifact_id(task.id, "artifacts"), diff --git a/src/nnnotes/pipeline/modelstages.py b/src/nnnotes/pipeline/modelstages.py index b7059dc..1fadad3 100644 --- a/src/nnnotes/pipeline/modelstages.py +++ b/src/nnnotes/pipeline/modelstages.py @@ -163,18 +163,18 @@ def closure_bundles(self, key: str) -> list[str]: class TaskCatalog: """What the model extractors ask of a Catalog (live2d.extract_model through export.Exporter: fetch_key, entry, apk), answered from a task: its key, the internal id of the key's location and the bundles of the key's closure - in load order, each bundle read through the store (Store.open_input: its locators, checked against its content + in load order, each bundle read through the depot (Depot.open_input: its locators, checked against its content id). Another key is not in it (KeyError). `apk` is not None: the extractors need the APK's bundles, and those of the closure are inputs of the task like the others.""" - def __init__(self, key: str, internal_id: str, bundles: list[Input], store): - self.key, self.internal_id, self.bundles, self.store = key, internal_id, list(bundles), store + def __init__(self, key: str, internal_id: str, bundles: list[Input], depot): + self.key, self.internal_id, self.bundles, self.depot = key, internal_id, list(bundles), depot self.apk = APK_FROM_INPUTS @classmethod - def from_task(cls, task, store) -> TaskCatalog: + def from_task(cls, task, depot) -> TaskCatalog: """The catalog of a live2d.model task: its subject, context internalId and inputs "bundle:".""" - return cls(task.subject, task.context["internalId"], _ordered(task), store) + return cls(task.subject, task.context["internalId"], _ordered(task), depot) def _check(self, key: str) -> None: if key != self.key: @@ -203,7 +203,7 @@ def resolve(self, key: str) -> list[Bundle]: def fetch_key(self, key: str) -> list[Path]: """The files of the closure's bundles, in load order.""" self._check(key) - return [self.store.open_input(inp) for inp in self.bundles] + return [self.depot.open_input(inp) for inp in self.bundles] # ---------------------------------------------------------------- live2d.model @@ -267,11 +267,11 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: size = sum(i.size for i in inputs) return Cost(0.9 + size * 2.2e-7, (420 << 20) + 10 * size) - def run(self, task, store) -> Output: + def run(self, task, depot) -> Output: _check_atoms(task, self.ATOMS) from ..unity import live2d - cat = TaskCatalog.from_task(task, store) + cat = TaskCatalog.from_task(task, depot) arts = [] with tempfile.TemporaryDirectory(prefix="nnnotes-live2d-") as d: # TaskCatalog answers the catalog calls of the model extractor @@ -283,7 +283,7 @@ def run(self, task, store) -> Output: arts.append( contract.artifact( contract.artifact_id(task.id, rel), - store.add(p.read_bytes(), ext), + depot.add(p.read_bytes(), ext), contract.provenance(task), {"kind": kind, "format": ext, "facts": facts}, ) @@ -301,14 +301,14 @@ def skeleton_index(env, selected) -> dict: pending = env.pending(CENSUS) if pending: raise Pending(pending[0]) - table = contract.loads(env.store.read(env.input_of(SCRIPTS_TASK, "scripts").sha256))["scripts"] + table = contract.loads(env.depot.read(env.input_of(SCRIPTS_TASK, "scripts").sha256))["scripts"] scripts = {oid for oid, entry in table.items() if entry.rsplit("|", 1)[-1] == SKELETON_CLASS} files: dict[str, str] = {} externals: dict[str, list[str]] = {} subjects: dict[str, tuple] = {} for tid in env.tasks(CENSUS): stable = contract.parse_task_id(tid, CENSUS)[1] - doc = contract.loads(env.store.read(env.artifact(tid, "census")["content"]["sha256"])) + doc = contract.loads(env.depot.read(env.artifact(tid, "census")["content"]["sha256"])) for f in doc["files"]: files.setdefault(f["name"].casefold(), stable) externals.setdefault(f["name"].casefold(), [x["file"] for x in f["externals"]]) @@ -411,13 +411,13 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: size = sum(i.size for i in inputs) return Cost(0.27 + size * 1.9e-7, (440 << 20) + 20 * size) - def run(self, task, store) -> Output: + def run(self, task, depot) -> Output: _check_atoms(task, self.ATOMS) from ..content import spot from ..unity import reader as unity oid = task.context["object"] - paths = [store.open_input(i) for i in task.inputs if i.role.startswith("bundle:")] + paths = [depot.open_input(i) for i in task.inputs if i.role.startswith("bundle:")] env = unity.load_closure(paths) file, pid = contract.parse_object_id(oid) written: dict[str, bytes] = {} @@ -446,7 +446,7 @@ def run(self, task, store) -> Output: arts.append( contract.artifact( contract.artifact_id(task.id, name), - store.add(data, ext), + depot.add(data, ext), contract.provenance(task), {"kind": kind, "format": ext, "facts": facts}, ) diff --git a/src/nnnotes/pipeline/objexport.py b/src/nnnotes/pipeline/objexport.py index 2e885bd..a1fbb0d 100644 --- a/src/nnnotes/pipeline/objexport.py +++ b/src/nnnotes/pipeline/objexport.py @@ -309,10 +309,10 @@ class RefExporter(_ExporterBase): code runs here in reference mode: `ref` gives $ref values, `deref` resolves inside the bundle only (a MonoScript of another bundle from the script table), and MonoBehaviour classes come from the task's script table.""" - def __init__(self, view, task, store, fns: dict): + def __init__(self, view, task, depot, fns: dict): # no catalog: the objects come from the task's bundle view super().__init__(None, Path("."), class_of=self._script_name) # pyright: ignore[reportArgumentType] - self.view, self.task, self.store, self.fns = view, task, store, fns + self.view, self.task, self.depot, self.fns = view, task, depot, fns self.blob_min = int(task.params["blobMin"]) self.png_level = int(task.params["png"]["level"]) self.scripts = dict(task.context.get("scripts", {})) @@ -631,11 +631,11 @@ def _emit( refs=(), owner: str | None = None, ) -> tuple[str, dict]: - """Store `data` and stage the artifact "#" (owner: the object `key`, else a task-level name).""" + """Depot `data` and stage the artifact "#" (owner: the object `key`, else a task-level name).""" aid = contract.artifact_id(owner or contract.object_id(*key), role) if aid in self.emitted or any(a["id"] == aid for a in self.staged): raise ValueError(f"artifact {aid} made twice") - content = self.store.add(data, ext) + content = self.depot.add(data, ext) prov = contract.provenance(self.task, atoms=self._atoms(self.conv), obj=None if owner else self._facts(key)) sem = {"kind": kind, "format": fmt, "facts": facts} if issues: @@ -1428,15 +1428,15 @@ def _check_atoms(task, reader: Impl | None = None) -> None: class _Docs: - """A few JSON documents of the store by content id (the planning reads each census several times).""" + """A few JSON documents of the depot by content id (the planning reads each census several times).""" def __init__(self, size: int = 64): self.size, self.docs = size, {} - def get(self, store, sha: str) -> dict: + def get(self, depot, sha: str) -> dict: doc = self.docs.get(sha) if doc is None: - doc = contract.loads(store.read(sha)) + doc = contract.loads(depot.read(sha)) if len(self.docs) >= self.size: self.docs.pop(next(iter(self.docs))) self.docs[sha] = doc @@ -1513,7 +1513,7 @@ def inputs(self, subject: str, env) -> list[Input]: def census(self, subject: str, env) -> dict: inp = env.input_of(contract.task_id(CENSUS, subject), "census") - return self._docs.get(env.store, inp.sha256) + return self._docs.get(env.depot, inp.sha256) def uses(self, subject: str, env) -> dict: """The atoms and converters of the classes the bundle's census lists; with the task's `classes` (env.params @@ -1533,7 +1533,7 @@ def context(self, subject: str, env) -> dict: refs = census_mod.script_refs(self.census(subject, env)) if not refs: return {} - table = self._docs.get(env.store, env.input_of(SCRIPTS_TASK, "scripts").sha256)["scripts"] + table = self._docs.get(env.depot, env.input_of(SCRIPTS_TASK, "scripts").sha256)["scripts"] return {"scripts": link.script_context(table, refs)} def depends(self, subject: str, env) -> list[str]: @@ -1551,12 +1551,12 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: n = res = 0 return Cost(0.12 + size * 3.3e-7 + n * 8.8e-5 + res * 6.6e-8, (80 << 20) + 80 * size + 17500 * n) - def run(self, task, store) -> Output: + def run(self, task, depot) -> Output: reader = self.reader if self.reader is not None else resolve("reader") _check_atoms(task, reader) - view = reader.fn(store.input_bytes(task.input("bundle"))) + view = reader.fn(depot.input_bytes(task.input("bundle"))) fns = {n: resolve(n).fn for n in task.atoms if n in ATOM_NAMES and n != "reader"} - arts, items = exporter_type()(view, task, store, fns).run_bundle() + arts, items = exporter_type()(view, task, depot, fns).run_bundle() return Output(arts, items) @@ -1570,7 +1570,7 @@ def _census_classes(env, subject: str) -> dict: def _store_input(role: str, record: dict) -> Input: c = record["content"] - return Input(role, c["sha256"], c["size"], None, ({"kind": "store"},)) + return Input(role, c["sha256"], c["size"], None, ({"kind": "depot"},)) def _pptr_oid(v) -> str | None: @@ -1674,11 +1674,11 @@ def inputs(self, subject: str, env) -> list[Input]: idx, atlas, reqs = self._subject(subject, env) rec = idx["atlases"][atlas][1] ins = [_store_input("atlas", rec)] - entries = _entries(self._docs.get(env.store, rec["content"]["sha256"])) + entries = _entries(self._docs.get(env.depot, rec["content"]["sha256"])) needed = set() for owner, _tid, meta in reqs: ins.append(_store_input("sprite:" + owner, meta)) - doc = self._docs.get(env.store, meta["content"]["sha256"]) + doc = self._docs.get(env.depot, meta["content"]["sha256"]) entry = _entry(entries, doc["sprite"].get("m_RenderDataKey")) if entry is not None: needed.update(t for t in (_pptr_oid(entry.get("texture")), _pptr_oid(entry.get("alphaTexture"))) if t) @@ -1697,7 +1697,7 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: n = sum(1 for i in inputs if i.role.startswith("sprite:")) return Cost(0.05 + n * 0.01 + tex * 5e-8, (72 << 20) + 40 * tex) - def run(self, task, store) -> Output: + def run(self, task, depot) -> Output: from PIL import Image from .atoms import sprite as sprite_atom @@ -1705,7 +1705,7 @@ def run(self, task, store) -> Output: _check_atoms(task) crop, encode = resolve("sprite.crop").fn, resolve("png.encode").fn level = int(task.params["png"]["level"]) - entries = _entries(contract.loads(store.input_bytes(task.input("atlas")))) + entries = _entries(contract.loads(depot.input_bytes(task.input("atlas")))) images = {} def image(oid: str): @@ -1714,7 +1714,7 @@ def image(oid: str): inp = task.input("texture:" + oid) except KeyError: raise Unsupported("no_data.sprite", f"no image of the atlas texture {oid}") from None - im = Image.open(io.BytesIO(store.input_bytes(inp))) + im = Image.open(io.BytesIO(depot.input_bytes(inp))) im.load() images[oid] = im return images[oid] @@ -1724,14 +1724,14 @@ def image(oid: str): if not inp.role.startswith("sprite:"): continue oid = inp.role[len("sprite:") :] - meta = contract.loads(store.input_bytes(inp)) + meta = contract.loads(depot.input_bytes(inp)) try: img = _crop_meta(meta, entries, image, crop, sprite_atom) aid = contract.artifact_id(oid, "image") arts.append( contract.artifact( aid, - store.add(encode(img, level), "png"), + depot.add(encode(img, level), "png"), contract.provenance(task, obj=meta.get("object")), { "kind": "sprite.image", diff --git a/src/nnnotes/pipeline/orchestrate.py b/src/nnnotes/pipeline/orchestrate.py index b3b7f62..6d54736 100644 --- a/src/nnnotes/pipeline/orchestrate.py +++ b/src/nnnotes/pipeline/orchestrate.py @@ -2,7 +2,7 @@ Stages run in the order given; a stage's tasks are described once the stages before it are done (Env: the facts of the caller plus the keys of the tasks already done), so every input is known when a task is described. Per stage: -a task whose result is in the store is a hit; the others run in parallel, then the next stage starts. +a task whose result is in the depot is a hit; the others run in parallel, then the next stage starts. The pool: up to `workers` spawned processes (no more than a stage has tasks to run), long-lived (each imports what its stages need once), one task at a time each. Tasks start in order of decreasing estimated CPU (longest @@ -39,7 +39,7 @@ from .stages import Env, Pending, Stage, describe, execute if TYPE_CHECKING: - from .store import Store + from .depot import Depot WORKER_BYTES = 70 << 20 # resident memory of an idle worker process (interpreter and imports), per worker # failure codes of tasks (failed items carry the codes of their stages) @@ -101,13 +101,13 @@ def failure(task: str, stage: str, code: str, message: str, inp: str | None = No def _error_code(e: BaseException) -> str: from .contract import IncompatibleTask - from .store import InputMissing, StoreConflict + from .depot import DepotConflict, InputMissing if isinstance(e, IncompatibleTask): return "task.incompatible" if isinstance(e, InputMissing): return "task.input" - if isinstance(e, StoreConflict): + if isinstance(e, DepotConflict): return "task.conflict" if isinstance(e, MemoryError): return "task.memory" @@ -116,14 +116,14 @@ def _error_code(e: BaseException) -> str: return "task.error" -def run_one(task: Task, store, stages: dict, force: bool) -> dict: +def run_one(task: Task, depot, stages: dict, force: bool) -> dict: """Execute one task and measure it: the reply a worker sends (status, result status, cost, or error). The cost's peak memory is the process's, with the resident memory at the task's start (what the process held from its imports and earlier tasks) beside it.""" start = rss() reset_peak() try: - ex = execute(task, store, stages, force=force) + ex = execute(task, depot, stages, force=force) except ConfigError as e: return {"ok": False, "code": "config", "message": str(e), "trace": "", "rss": rss()} except Exception as e: @@ -137,14 +137,14 @@ def run_one(task: Task, store, stages: dict, force: bool) -> dict: reply = {"ok": True, "status": ex.status, "resultStatus": ex.result_status, "rss": rss()} if ex.cost is not None: cost = dict(ex.cost, peakRssBytes=peak_rss(), startRssBytes=start) - store.write_cost(task.key, cost) + depot.write_cost(task.key, cost) reply["cost"] = cost return reply -def _worker_main(conn, store, stages: dict, initializer, initargs) -> None: +def _worker_main(conn, depot, stages: dict, initializer, initargs) -> None: if initializer is not None: - initializer(store, *initargs) + initializer(depot, *initargs) while True: try: msg = conn.recv() @@ -158,16 +158,16 @@ def _worker_main(conn, store, stages: dict, initializer, initargs) -> None: except Exception as e: conn.send({"ok": False, "code": "task.incompatible", "message": str(e), "trace": "", "rss": rss()}) continue - conn.send(run_one(task, store, stages, force)) + conn.send(run_one(task, depot, stages, force)) class _Worker: - def __init__(self, ctx, n: int, store, stages, initializer, initargs): + def __init__(self, ctx, n: int, depot, stages, initializer, initargs): self.n = n self.conn, child = ctx.Pipe() self.proc = ctx.Process( target=_worker_main, - args=(child, store, stages, initializer, initargs), + args=(child, depot, stages, initializer, initargs), name=f"nnnotes-worker-{n}", daemon=True, ) @@ -251,7 +251,7 @@ class Run: The reports (summary, failures, coverage) come from one reading of every result (scan), kept while the tasks stay the same.""" - store: Store + depot: Depot context: dict selection: list params: dict @@ -267,7 +267,7 @@ def results(self): for tid in sorted(self.tasks): t = self.tasks[tid] if t["status"] != "failed": - yield tid, self.store.committed(t["key"], tid) + yield tid, self.depot.committed(t["key"], tid) def scan(self, visit=None, census=None) -> None: """Read every result once, by task id, for the reports; `visit(task id, result)` sees each as well (what @@ -328,7 +328,7 @@ def write(self, layouts: dict | None = None, placement: dict | None = None) -> d """Write the run manifest (layouts: {name: sha256 of its layout manifest}; placement: how their files were made) and this execution's log.""" doc = self.manifest(layouts, placement) - self.store.write_run(doc, self.log) + self.depot.write_run(doc, self.log) return doc def coverage(self, census: dict | None = None) -> dict: @@ -447,12 +447,12 @@ def coverage(results, census: dict | None = None) -> dict: class Orchestrator: - """Runs pipelines of stages into a store (see the module documentation). `initializer(store, *initargs)` runs - first in every worker (settings, caches, the store's fetcher); it must be importable (spawned processes).""" + """Runs pipelines of stages into a depot (see the module documentation). `initializer(depot, *initargs)` runs + first in every worker (settings, caches, the depot's fetcher); it must be importable (spawned processes).""" def __init__( self, - store, + depot, stages: dict[str, Stage], *, workers: int = 0, @@ -464,7 +464,7 @@ def __init__( initargs: tuple = (), log=_log, ): - self.store, self.stages = store, stages + self.depot, self.stages = depot, stages self.workers, self.memory, self.worker_bytes = max(0, int(workers)), memory, int(worker_bytes) self.recycle_tasks, self.recycle_rss = recycle_tasks, recycle_rss self.initializer, self.initargs = initializer, tuple(initargs) @@ -504,7 +504,7 @@ def run( ) -> Run: """Run the stages named in `pipeline`, in order. `params`: {stage: parameters}; `facts`: the planning facts; `context` / `selection`: recorded in the run manifest; `force`: task ids or stage names ("*": all) - run again even when their result is in the store (the new result must be identical).""" + run again even when their result is in the depot (the new result must be identical).""" params = params or {} unknown = [n for n in pipeline if n not in self.stages] if unknown: @@ -514,8 +514,8 @@ def run( if late: raise ValueError(f"stage {n} runs before {', '.join(late)}, which it reads") norm = {n: self.stages[n].normalize(params.get(n)) for n in pipeline} - run = Run(self.store, dict(context or {}), list(selection), norm, {n: self.stages[n].version for n in pipeline}) - env = Env(self.store, facts) + run = Run(self.depot, dict(context or {}), list(selection), norm, {n: self.stages[n].version for n in pipeline}) + env = Env(self.depot, facts) force = set(force) for line in self.explain(): run.log.append({"event": "knob", "value": line}) @@ -564,7 +564,7 @@ def _stage(self, stage: Stage, params: dict, env: Env, force: set, run: Run) -> forced = "*" in force or stage.name in force todo = [] for t in sorted(tasks, key=lambda t: t.id): - if not (forced or t.id in force) and self.store.has_result(t.key): + if not (forced or t.id in force) and self.depot.has_result(t.key): run.tasks[t.id] = {"id": t.id, "key": t.key, "status": "hit"} env.done(t.id, t.key) else: @@ -605,7 +605,7 @@ def _execute(self, tasks: list[Task], force, run: Run) -> dict[str, dict]: if t is None: break self._record(run, t, sched, None) - replies[t.id] = run_one(t, self.store, self.stages, force(t)) + replies[t.id] = run_one(t, self.depot, self.stages, force(t)) self._done(run, t, replies[t.id], None) sched.finished(t.id) if replies[t.id].get("code") == "config": @@ -617,7 +617,7 @@ def _execute(self, tasks: list[Task], force, run: Run) -> dict[str, dict]: def _spawn(self) -> _Worker: self._started += 1 - return _Worker(mp.get_context("spawn"), self._started, self.store, self.stages, self.initializer, self.initargs) + return _Worker(mp.get_context("spawn"), self._started, self.depot, self.stages, self.initializer, self.initargs) def _pool_run(self, sched: Scheduler, force, replies: dict, run: Run) -> None: while len(self._pool) < min(self.workers, len(sched.queue)): # no more processes than tasks diff --git a/src/nnnotes/pipeline/plan.py b/src/nnnotes/pipeline/plan.py index 781edba..1cba6ee 100644 --- a/src/nnnotes/pipeline/plan.py +++ b/src/nnnotes/pipeline/plan.py @@ -1,6 +1,6 @@ """Plan: what a run of a pipeline would do, and why, without running anything. -Every subject of every stage is a node. A node is a `hit` (its result is in the store), `run` (it would run), or +Every subject of every stage is a node. A node is a `hit` (its result is in the depot), `run` (it would run), or `unknown` (it reads the result of a task that would run first, or of one that has not run: its inputs are not known yet). Why a node runs is the difference between its key parts and those of the same task id in a previous run manifest (the latest of the same selection, or one named with --since): @@ -12,9 +12,9 @@ params-changed{path,old,new} an output-affecting parameter (path: /name/sub...) context-changed{entry} a consumed context entry previous-failed the task failed in the previous run - result-missing same key as before, but the result is not in the store + result-missing same key as before, but the result is not in the depot forced asked to run again -Hit reasons: `unchanged` (same key as before), `cached` (the key's result is in the store from another run, e.g. +Hit reasons: `unchanged` (same key as before), `cached` (the key's result is in the depot from another run, e.g. content that changed back). Unknown: `pending{task}`. The plan document (nnnotes.plan/1) lists the nodes in pipeline order with key, status, reasons and cost estimate, @@ -27,9 +27,9 @@ from pathlib import Path from . import contract +from .depot import write_file from .orchestrate import WORKER_BYTES from .stages import Env, Pending, Stage, describe -from .store import write_file def _leaves(old, new, path: str = ""): @@ -138,7 +138,7 @@ def _short(v) -> str: def plan( stages: dict[str, Stage], pipeline: list[str], - store, + depot, *, params: dict | None = None, facts: dict | None = None, @@ -154,7 +154,7 @@ def plan( `memory`, `worker_bytes`: as the orchestrator's, for the peak-memory estimate; `layouts`: {name: {write, remove, keep}} (layout.diff).""" params = params or {} - env = Env(store, facts) + env = Env(depot, facts) force = set(force) prev = {t["id"]: t for t in (previous or {}).get("tasks", [])} nodes, edges, runnable = [], set(), [] @@ -198,13 +198,13 @@ def plan( ) continue old = prev.get(tid) - if not (forced or tid in force) and store.has_result(t.key): + if not (forced or tid in force) and depot.has_result(t.key): env.done(tid, t.key) reasons = [{"code": "unchanged" if old is not None and old.get("key") == t.key else "cached"}] status = "hit" else: env.waiting(tid) - reasons = _run_reasons(t, old, store, forced or tid in force) + reasons = _run_reasons(t, old, depot, forced or tid in force) status = "run" runnable.append(t) nodes.append( @@ -220,7 +220,7 @@ def plan( return Plan(_doc(nodes, edges, runnable, workers, memory, worker_bytes, layouts or {}), runnable) -def _run_reasons(t, old: dict | None, store, forced: bool) -> list[dict]: +def _run_reasons(t, old: dict | None, depot, forced: bool) -> list[dict]: if forced: return [{"code": "forced"}] if old is None: @@ -230,7 +230,7 @@ def _run_reasons(t, old: dict | None, store, forced: bool) -> list[dict]: out.append({"code": "previous-failed"}) if old.get("key") == t.key: return out or [{"code": "result-missing"}] - prior = store.result(old["key"]) if old.get("key") else None + prior = depot.result(old["key"]) if old.get("key") else None if prior is not None: reasons = why(prior["keyParts"], t.key_parts()) or [{"code": "key-changed"}] for r in reasons: diff --git a/src/nnnotes/pipeline/stages.py b/src/nnnotes/pipeline/stages.py index 95ab75f..ced2e4b 100644 --- a/src/nnnotes/pipeline/stages.py +++ b/src/nnnotes/pipeline/stages.py @@ -10,13 +10,13 @@ context(subject, env) the context entries its task consumes (only those) depends(subject, env) the task ids whose results it reads (the plan's edges) estimate(subject, env, inputs) the expected cost (CPU seconds, peak bytes) - run(task, store) the work: reads the inputs through the store, writes the bytes of its artifacts - into the store, returns their records and one item per covered object + run(task, depot) the work: reads the inputs through the depot, writes the bytes of its artifacts + into the depot, returns their records and one item per covered object A stage that needs the output of a task that has not run raises Pending from inputs/uses/context (the environment does it for it: Env.key / Env.result). While a task is described, `env.params` holds its normalized parameters, for answers that depend on them (the atoms of the converters a parameter selects). `describe` turns the answers into a -Task; `execute` runs a Task once and commits its result (the store's action cache) after the artifacts' bytes are +Task; `execute` runs a Task once and commits its result (the depot's action cache) after the artifacts' bytes are stored. The output of `run` must be a function of the task's key parts: a change of what a stage writes is a change of its version (golden tests check). Deterministic per-object failures are items (status `failed`, cached with the result); an exception out of `run` @@ -58,7 +58,7 @@ def __init__(self, task: str, failed: bool = False): @dataclass class Output: - """What Stage.run returns: artifact records (their bytes already in the store) and one item per object.""" + """What Stage.run returns: artifact records (their bytes already in the depot) and one item per object.""" artifacts: list = field(default_factory=list) items: list = field(default_factory=list) @@ -99,7 +99,7 @@ def depends(self, subject: str, env: Env) -> list[str]: def estimate(self, subject: str, env: Env, inputs: list[Input]) -> Cost: return Cost(0.0, sum(i.size for i in inputs)) - def run(self, task: Task, store) -> Output: + def run(self, task: Task, depot) -> Output: raise NotImplementedError @@ -118,11 +118,11 @@ class Env: the tasks of the current plan or run with their keys. `key` / `result` raise Pending for a task that has not run (or failed); a stage lets that propagate.""" - def __init__(self, store, facts: dict | None = None): - self.store = store + def __init__(self, depot, facts: dict | None = None): + self.depot = depot self.facts = dict(facts or {}) self.params: dict = {} # the normalized parameters of the task being described - self._keys: dict[str, str] = {} # task id -> key, for tasks whose result is in the store + self._keys: dict[str, str] = {} # task id -> key, for tasks whose result is in the depot self._open: dict[str, bool] = {} # task id -> failed?, for tasks without a result def fact(self, name: str): @@ -148,7 +148,7 @@ def key(self, tid: str) -> str: raise Pending(tid, self._open.get(tid, False)) def result(self, tid: str) -> dict: - doc = self.store.result(self.key(tid)) + doc = self.depot.result(self.key(tid)) if doc is None: raise Pending(tid) return doc @@ -175,9 +175,9 @@ def artifact(self, tid: str, role: str) -> dict: raise KeyError(f"result of {tid} has no artifact {aid}") def input_of(self, tid: str, role: str, as_role: str | None = None) -> Input: - """A task input made of the artifact "#" (read from the store).""" + """A task input made of the artifact "#" (read from the depot).""" c = self.artifact(tid, role)["content"] - return Input(as_role or role, c["sha256"], c["size"], None, ({"kind": "store"},)) + return Input(as_role or role, c["sha256"], c["size"], None, ({"kind": "depot"},)) def describe(stage: Stage, subject: str, params: dict | None, env: Env) -> Task: @@ -202,7 +202,7 @@ def describe(stage: Stage, subject: str, params: dict | None, env: Env) -> Task: @dataclass class Execution: - """The outcome of execute: `status` hit (the result was in the store) or ran; the measured cost of a run.""" + """The outcome of execute: `status` hit (the result was in the depot) or ran; the measured cost of a run.""" task: str key: str @@ -211,10 +211,10 @@ class Execution: cost: dict | None = None -def execute(task: Task, store, stages: dict[str, Stage], *, force: bool = False) -> Execution: +def execute(task: Task, depot, stages: dict[str, Stage], *, force: bool = False) -> Execution: """Run `task` once: verify that this installation has its stage at its version and can read every input with its content id, run the stage, store its artifacts' bytes, then commit the result (write-once; a result already - in the store is a hit unless `force`, which runs again and requires identical output). Exceptions fail the task + in the depot is a hit unless `force`, which runs again and requires identical output). Exceptions fail the task and leave no result. The measured cost (never part of a key) counts the CPU time of this process and of the child processes the stage ran and waited for (childCpuSeconds; null where the platform does not report it).""" stage = stages.get(task.stage) @@ -226,16 +226,16 @@ def execute(task: Task, store, stages: dict[str, Stage], *, force: bool = False) f"the task was described for version {task.version}" ) if not force: - hit = store.result(task.key) + hit = depot.result(task.key) if hit is not None: return Execution(task.id, task.key, "hit", hit["status"]) for i in task.inputs: - store.open_input(i) + depot.open_input(i) children = children_cpu() cpu, wall = time.process_time(), time.perf_counter() - out = stage.run(task, store) + out = stage.run(task, depot) doc = contract.result(task, out.artifacts, out.items) - store.commit(doc) + depot.commit(doc) own = time.process_time() - cpu now = children_cpu() child = None if children is None or now is None else max(0.0, now - children) @@ -273,17 +273,17 @@ def library_versions(dists) -> dict[str, str]: return out -def golden(stage: Stage, fixtures: dict, store, libraries=()) -> dict: +def golden(stage: Stage, fixtures: dict, depot, libraries=()) -> dict: """The golden record of a stage: its version, the atoms its fixtures use, the versions of `libraries`, and per - fixture the key and the sha256 of the canonical result. `fixtures`: {name: callable(store) -> Task} (each - builds its inputs into the store).""" + fixture the key and the sha256 of the canonical result. `fixtures`: {name: callable(depot) -> Task} (each + builds its inputs into the depot).""" atoms, out = {}, {} for name in sorted(fixtures): - task = fixtures[name](store) + task = fixtures[name](depot) if task.stage != stage.name: raise ValueError(f"fixture {name} is a task of {task.stage}, not {stage.name}") - execute(task, store, {stage.name: stage}, force=True) - doc = store.result(task.key) + execute(task, depot, {stage.name: stage}, force=True) + doc = depot.result(task.key) atoms.update(task.atoms) out[name] = {"key": task.key, "result": contract.sha256(contract.encode(doc))} return { diff --git a/src/nnnotes/pipeline/views.py b/src/nnnotes/pipeline/views.py index 926956d..18c7012 100644 --- a/src/nnnotes/pipeline/views.py +++ b/src/nnnotes/pipeline/views.py @@ -862,14 +862,14 @@ def _address_index(env, tid: str) -> AddressIndex: sha = env.artifact(tid, "addresses")["content"]["sha256"] ix = _index_cache.get(sha) if ix is None: - ix = _index_cache[sha] = AddressIndex.from_addresses(contract.loads(env.store.read(sha))) + ix = _index_cache[sha] = AddressIndex.from_addresses(contract.loads(env.depot.read(sha))) while len(_index_cache) > 2: _index_cache.popitem(last=False) return ix -def _stored(store, role: str, data: bytes) -> Input: - return Input(role, store.put(data), len(data), None, ({"kind": "store"},)) +def _stored(depot, role: str, data: bytes) -> Input: + return Input(role, depot.put(data), len(data), None, ({"kind": "depot"},)) def _rank(aid: str) -> tuple: @@ -953,7 +953,7 @@ def master_inputs(self, env) -> list[Input]: p = mdir / f"{t}.json" if not p.is_file(): raise ViewError(f"view {self.view}: master table {t} is not in the master data") - sha, size = env.store.identify(p) + sha, size = env.depot.identify(p) out.append(Input(MASTER_ROLE + t, sha, size, p.name, ({"kind": "file", "path": str(p)},))) return out @@ -966,27 +966,27 @@ def inputs(self, subject: str, env) -> list[Input]: rec = Recorder(index) build(self.rules, self.view, tables, rec) return masters + [ - _stored(env.store, "addresses", contract.encode(rec.subset())), - _stored(env.store, "rules", self.rules_bytes), + _stored(env.depot, "addresses", contract.encode(rec.subset())), + _stored(env.depot, "rules", self.rules_bytes), ] def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: size = sum(i.size for i in inputs) return Cost(0.05 + size / 20e6, (48 << 20) + 16 * size) - def run(self, task, store) -> Output: - rules = contract.loads(store.input_bytes(task.input("rules"))) + def run(self, task, depot) -> Output: + rules = contract.loads(depot.input_bytes(task.input("rules"))) check_rules(rules) if list(rules["views"]) != [self.view] or rules["views"][self.view]["version"] != task.version: raise ViewError(f"task {task.id}: its rules are not those of view {self.view} version {task.version}") tables = { - i.role[len(MASTER_ROLE) :]: _cached_rows(i.sha256, functools.partial(store.input_bytes, i)) + i.role[len(MASTER_ROLE) :]: _cached_rows(i.sha256, functools.partial(depot.input_bytes, i)) for i in task.inputs if i.role.startswith(MASTER_ROLE) } - index = AddressIndex.from_subset(contract.loads(store.input_bytes(task.input("addresses")))) + index = AddressIndex.from_subset(contract.loads(depot.input_bytes(task.input("addresses")))) doc = build(rules, self.view, tables, index) - content = store.add(contract.encode(doc), "json") + content = depot.add(contract.encode(doc), "json") entries: dict[str, int] = {} for c in doc["coverage"]["roles"].values(): for s, n in c["counts"].items(): @@ -1005,13 +1005,13 @@ def run(self, task, store) -> Output: ) return Output([rec], []) - def derived(self, task_id: str, result: dict, store, paths) -> list[tuple[str, bytes]]: + def derived(self, task_id: str, result: dict, depot, paths) -> list[tuple[str, bytes]]: """The view document with the layout paths of its objects (with_paths), at view_path.""" aid = contract.artifact_id(task_id, VIEW_ROLE) rec = next((a for a in result["artifacts"] if a["id"] == aid), None) if rec is None: raise ValueError(f"result of {task_id} has no artifact {aid}") - doc = contract.loads(store.read(rec["content"]["sha256"])) + doc = contract.loads(depot.read(rec["content"]["sha256"])) subject = contract.parse_task_id(task_id, self.name)[1] return [(view_path(self.view, subject), contract.encode(with_paths(doc, paths)))] diff --git a/src/nnnotes/pipeline/voices.py b/src/nnnotes/pipeline/voices.py index d0f80e4..52eefa9 100644 --- a/src/nnnotes/pipeline/voices.py +++ b/src/nnnotes/pipeline/voices.py @@ -828,7 +828,7 @@ def _address_parts(env, tid: str) -> tuple: sha = env.artifact(tid, "addresses")["content"]["sha256"] hit = _address_cache.get(sha) if hit is None: - doc = contract.loads(env.store.read(sha)) + doc = contract.loads(env.depot.read(sha)) hit = (views.AddressIndex.from_addresses(doc), dict(doc.get("files") or {})) _address_cache.clear() _address_cache[sha] = hit @@ -925,7 +925,7 @@ def _plan(self, subject: str, env) -> tuple: tuple((t, env.key(t)) for t in tids), tuple((t, env.key(t)) for t in exports), tuple(i.sha256 for i in masters), - id(env.store), + id(env.depot), id(env.facts.get("index")), ) if self._memo is not None and self._memo[0] == sig: @@ -978,21 +978,21 @@ def _episodes(self, env, index, tables, files: dict, exports: set) -> tuple[dict c = a["content"] out.setdefault(asset, {})[part] = {"object": oid, "artifact": aid, "sha256": c["sha256"]} contents.setdefault( - c["sha256"], Input(f"adv:{c['sha256']}", c["sha256"], c["size"], None, ({"kind": "store"},)) + c["sha256"], Input(f"adv:{c['sha256']}", c["sha256"], c["size"], None, ({"kind": "depot"},)) ) read.add(tid) return dict(sorted(out.items())), [contents[k] for k in sorted(contents)], sorted(read) def inputs(self, subject: str, env) -> list[Input]: masters, subset, sheets, acbs, episodes, contents, _ = self._plan(subject, env) - store = env.store + depot = env.depot return ( masters + [ - views._stored(store, "addresses", contract.encode(subset)), - views._stored(store, "rules", self.rules_bytes), - views._stored(store, "sheets", contract.encode(sheets)), - views._stored(store, "episodes", contract.encode(episodes)), + views._stored(depot, "addresses", contract.encode(subset)), + views._stored(depot, "rules", self.rules_bytes), + views._stored(depot, "sheets", contract.encode(sheets)), + views._stored(depot, "episodes", contract.encode(episodes)), ] + acbs + contents @@ -1005,7 +1005,7 @@ def _masters(self, env) -> list[Input]: p = mdir / f"{t}.json" if not p.is_file(): raise views.ViewError(f"view {VIEW}: master table {t} is not in the master data") - sha, size = env.store.identify(p) + sha, size = env.depot.identify(p) out.append(Input(views.MASTER_ROLE + t, sha, size, p.name, ({"kind": "file", "path": str(p)},))) return out @@ -1022,30 +1022,30 @@ def estimate(self, subject: str, env, inputs: list[Input]) -> Cost: + 12 * max(story, default=0), ) - def run(self, task, store) -> Output: - rules = contract.loads(store.input_bytes(task.input("rules"))) + def run(self, task, depot) -> Output: + rules = contract.loads(depot.input_bytes(task.input("rules"))) check_rules(rules) if rules[VIEW]["version"] != task.version: raise views.ViewError(f"task {task.id}: its rules are not those of view {VIEW} version {task.version}") masters = [i for i in task.inputs if i.role.startswith(views.MASTER_ROLE)] tables = { - i.role[len(views.MASTER_ROLE) :]: views._cached_rows(i.sha256, functools.partial(store.input_bytes, i)) + i.role[len(views.MASTER_ROLE) :]: views._cached_rows(i.sha256, functools.partial(depot.input_bytes, i)) for i in masters } - index = views.AddressIndex.from_subset(contract.loads(store.input_bytes(task.input("addresses")))) - sheets = contract.loads(store.input_bytes(task.input("sheets"))) + index = views.AddressIndex.from_subset(contract.loads(depot.input_bytes(task.input("addresses")))) + sheets = contract.loads(depot.input_bytes(task.input("sheets"))) results = task.context.get("audio", {}) roles = {i.role for i in task.inputs} cues: dict[str, dict] = {} audio = {} for sheet, s in sorted(sheets.items()): key = results.get(s["task"]) - res = store.result(key) if key is not None else None + res = depot.result(key) if key is not None else None if res is None: raise views.ViewError(f"task {task.id}: no result of {s['task']}") role = f"acb:{s['acb']}" if role in roles and s["acb"] not in cues: - cues[s["acb"]] = acb.cue_streams(store.input_bytes(task.input(role))) + cues[s["acb"]] = acb.cue_streams(depot.input_bytes(task.input(role))) audio[sheet] = sheet_audio(res, cues.get(s["acb"])) snapshot = { "tables": {i.role[len(views.MASTER_ROLE) :]: i.sha256 for i in masters}, @@ -1057,13 +1057,13 @@ def run(self, task, store) -> Output: if "episodes" in roles: snapshot["episodes"] = task.input("episodes").sha256 column = rules[VIEW]["sound"]["sheet"] - for asset, parts in sorted(contract.loads(store.input_bytes(task.input("episodes"))).items()): - got = {p: store.input_bytes(task.input(f"adv:{x['sha256']}")) for p, x in parts.items()} + for asset, parts in sorted(contract.loads(depot.input_bytes(task.input("episodes"))).items()): + got = {p: depot.input_bytes(task.input(f"adv:{x['sha256']}")) for p, x in parts.items()} story[asset] = storyvoices.episode( got.get("script"), got.get("text"), got.get("sound"), got.get("sheets"), sheet_column=column ) doc = build(rules, tables, index, audio, snapshot, story) - content = store.add(contract.encode(doc), "json") + content = depot.add(contract.encode(doc), "json") facts = { "rows": len(doc["rows"]), "entries": counts(doc), @@ -1078,13 +1078,13 @@ def run(self, task, store) -> Output: ) return Output([rec], []) - def derived(self, task_id: str, result: dict, store, paths) -> list[tuple[str, bytes]]: + def derived(self, task_id: str, result: dict, depot, paths) -> list[tuple[str, bytes]]: """The voices document with the layout path of every stream, at views/voices.json.""" aid = contract.artifact_id(task_id, views.VIEW_ROLE) rec = next((a for a in result["artifacts"] if a["id"] == aid), None) if rec is None: raise ValueError(f"result of {task_id} has no artifact {aid}") - doc = contract.loads(store.read(rec["content"]["sha256"])) + doc = contract.loads(depot.read(rec["content"]["sha256"])) subject = contract.parse_task_id(task_id, self.name)[1] return [(views.view_path(VIEW, subject), contract.encode(with_paths(doc, paths)))] diff --git a/src/nnnotes/schemas/catalogs.schema.json b/src/nnnotes/schemas/catalogs.schema.json index 7a05fb7..f37287b 100644 --- a/src/nnnotes/schemas/catalogs.schema.json +++ b/src/nnnotes/schemas/catalogs.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "urn:nnnotes:schema:catalogs", "title": "nnnotes.catalogs/1", - "description": "The catalog versions of a store (/catalogs/index.json), in import order: id (key hash of the two catalogs' content ids), labels, region, language, the remote and APK catalogs (content id, size, build result hash), the CDN's .hash text when it was served, resource version and APK version name.", + "description": "The catalog versions of a depot (/catalogs/index.json), in import order: id (key hash of the two catalogs' content ids), labels, region, language, the remote and APK catalogs (content id, size, build result hash), the CDN's .hash text when it was served, resource version and APK version name.", "type": "object", "required": [ "schema", diff --git a/src/nnnotes/schemas/common.schema.json b/src/nnnotes/schemas/common.schema.json index d11e92a..d36e735 100644 --- a/src/nnnotes/schemas/common.schema.json +++ b/src/nnnotes/schemas/common.schema.json @@ -103,8 +103,8 @@ "properties": { "kind": { "enum": [ - "store", - "cache", + "depot", + "download", "file", "catalog" ] @@ -125,7 +125,7 @@ "properties": { "kind": { "enum": [ - "cache", + "download", "file" ] } diff --git a/src/nnnotes/schemas/music-data.schema.json b/src/nnnotes/schemas/music-data.schema.json index 148c345..b5c1934 100644 --- a/src/nnnotes/schemas/music-data.schema.json +++ b/src/nnnotes/schemas/music-data.schema.json @@ -69,7 +69,7 @@ "string", "null" ], - "description": "the resource version recorded for the catalog in the catalog store, null when none is recorded" + "description": "the resource version recorded for the catalog in the catalog depot, null when none is recorded" }, "resourceHash": { "type": "string", diff --git a/src/nnnotes/schemas/run.schema.json b/src/nnnotes/schemas/run.schema.json index c032502..3e814b7 100644 --- a/src/nnnotes/schemas/run.schema.json +++ b/src/nnnotes/schemas/run.schema.json @@ -171,7 +171,7 @@ } }, "placement": { - "description": "How the layouts' files were made from the store (not part of the id, and not of the layout manifests, which every method writes alike): `clone` (sharing the store objects' blocks), `hard` (hard links to the read-only store objects) or `copy`, with the reason `auto` chose it or `as asked`.", + "description": "How the layouts' files were made from the depot (not part of the id, and not of the layout manifests, which every method writes alike): `clone` (sharing the depot objects' blocks), `hard` (hard links to the read-only depot objects) or `copy`, with the reason `auto` chose it or `as asked`.", "type": "object", "required": [ "method", diff --git a/src/nnnotes/site/sitefiles.py b/src/nnnotes/site/sitefiles.py index 158faae..aded184 100644 --- a/src/nnnotes/site/sitefiles.py +++ b/src/nnnotes/site/sitefiles.py @@ -286,7 +286,7 @@ def _lock_fetches(cat, lock) -> None: f = getattr(cat, name) def locked(x, _f=f, _name=name): - hit = cat.cached(x) if _name == "fetch" else cat.cached_raw(x) + hit = cat.downloaded(x) if _name == "fetch" else cat.downloaded_raw(x) if hit is not None: return hit with lock: diff --git a/src/nnnotes/source/catalog.py b/src/nnnotes/source/catalog.py index 94654b9..cd5979f 100644 --- a/src/nnnotes/source/catalog.py +++ b/src/nnnotes/source/catalog.py @@ -1,8 +1,8 @@ -"""Addressables catalog access: dependency closures, a bundle cache, remote and APK-local bundles. +"""Addressables catalog access: dependency closures, downloaded files, remote and APK-local bundles. Remote bundles are fetched from the region's CDN and decrypted (addressables.decrypt); local bundles (shipped inside the APK) are read from a user-supplied base.apk. Nothing here bundles or redistributes game data: it only reads -what the user points it at into a local cache the user controls. +what the user points it at into a directory of downloads the user controls (the depot's downloads/). """ from __future__ import annotations @@ -44,7 +44,7 @@ T = TypeVar("T") -# a setting given as its value, or as a function that reads it when a file not in the cache needs it +# a setting given as its value, or as a function that reads it when a file not downloaded yet needs it Lazy: TypeAlias = "T | Callable[[], T] | None" @@ -94,8 +94,9 @@ class Catalog: shared prefabs) and the remote catalog (downloadable content). Entry offsets are per-catalog, so APK entries are moved into their own offset namespace. - `cdn`: the region's CDN base (needed only to download what the cache lacks); `bundle_key`: the bundle - decryption key (needed only for bundles not yet in the cache). Either may be a function that returns it, called + `download_dir`: the directory of this catalog's downloaded files. `cdn`: the region's CDN base (needed only to + download what is not downloaded yet); `bundle_key`: the bundle decryption key (needed only for bundles not + downloaded yet). Either may be a function that returns it, called the first time it is needed (a ConfigError it raises is raised naming the file that needed the setting). `apk_catalog`: the stored catalog to replay without opening an APK; JP local reads verify it before extraction. """ @@ -103,7 +104,7 @@ class Catalog: def __init__( self, catalog_bytes: bytes, - cache_dir: Path, + download_dir: Path, *, cdn: Lazy[str] = None, bundle_key: Lazy[BundleKey] = None, @@ -114,8 +115,8 @@ def __init__( resource_version: str | None = None, ): self._cdn, self._bundle_key = cdn, bundle_key - self.cache_dir = Path(cache_dir) - self.cache_dir.mkdir(parents=True, exist_ok=True) + self.download_dir = Path(download_dir) + self.download_dir.mkdir(parents=True, exist_ok=True) self.apk = Path(apk) if apk else None self.source, self.client = source, client self.resource_version = resource_version @@ -138,11 +139,11 @@ def check_apk(self, archive: ApkSet) -> None: if not matches: raise ConfigError("JP historical catalog needs the APK set it was imported with") - def local_cache_dir(self) -> Path: + def local_download_dir(self) -> Path: """JP embedded files also depend on the APK catalog, independently of the CDN snapshot.""" if self.source is not None and "apk" in self._sources: - return self.cache_dir / "apk" / hashlib.sha256(self._sources["apk"]).hexdigest() - return self.cache_dir + return self.download_dir / "apk" / hashlib.sha256(self._sources["apk"]).hexdigest() + return self.download_dir def _parse(self) -> tuple[list[Location], dict[int, Location], dict[str, list[Location]]]: """(entries, by offset, by primary key), parsed the first time a lookup needs them (a command that only @@ -177,35 +178,38 @@ def _by_key(self) -> dict[str, list[Location]]: def load( cls, language: str, - cache_dir: Path, + download_dir: Path, *, cdn: Lazy[str] = None, bundle_key: Lazy[BundleKey] = None, apk: Path | None = None, version: str = "main", ) -> Catalog: - """The remote catalog of `language` from the cache, downloaded from the CDN on first use (`cdn` / - `bundle_key`: as for Catalog). A versioned catalog is isolated by CDN root and version; main retains - its legacy offline cache path. Missing versioned files never fall back to main.""" - cache_dir = Path(cache_dir) - filename = cls.cache_file(language, Path("."), version=version).name + """The remote catalog of `language` from `download_dir`, downloaded from the CDN on first use (`cdn` / + `bundle_key`: as for Catalog). A versioned catalog is isolated by CDN root and version + (international///); main is at the top of `download_dir`. Missing versioned + files never fall back to main.""" + download_dir = Path(download_dir) + filename = cls.file_name(language, version=version) if version != "main": cdn = _setting(cdn, filename) if not cdn: - raise ConfigError("a versioned catalog needs its CDN root to identify the cache") - cache_dir = cache_dir / "international" / hashlib.sha256(cdn.rstrip("/").encode()).hexdigest() / version - cat = cache_dir / filename + raise ConfigError("a versioned catalog needs its CDN root to identify its downloads") + download_dir = ( + download_dir / "international" / hashlib.sha256(cdn.rstrip("/").encode()).hexdigest() / version + ) + cat = download_dir / filename if not cat.exists(): root = _setting(cdn, cat.name) if not root: - raise FileNotFoundError(f"{cat} not cached and no CDN base given") + raise FileNotFoundError(f"{cat} not downloaded and no CDN base given") cat.parent.mkdir(parents=True, exist_ok=True) data = download(root.rstrip("/") + f"/asset/Android/{cat.name}") parse(data) # Never install a corrupt/error response as a reusable catalog. _write_atomic(cat, data) return cls( cat.read_bytes(), - cache_dir, + download_dir, cdn=cdn, bundle_key=bundle_key, apk=apk, @@ -229,11 +233,11 @@ def cdn(self) -> str | None: return None if callable(v) or not v else v.rstrip("/") @staticmethod - def cache_file(language: str, cache_dir: Path, *, version: str = "main") -> Path: - """Where the remote catalog of `language` is cached.""" + def file_name(language: str, *, version: str = "main") -> str: + """The file name of the remote catalog of `language` (on the CDN and in a download directory).""" check_catalog_version(version) check_catalog_version(language) - return Path(cache_dir) / f"catalog_{version}_{language}.bin" + return f"catalog_{version}_{language}.bin" # --- lookup ------------------------------------------------------------ def keys(self, prefix: str = "") -> list[str]: @@ -336,20 +340,20 @@ def resolve(self, key: str) -> list[Bundle]: return list(out.values()) # --- fetch ------------------------------------------------------------- - def cached(self, b: Bundle) -> Path | None: - """The file fetch(b) returns when the bundle is in the cache already, else None.""" - dst = (self.cache_dir if b.remote else self.local_cache_dir()) / "bundles" / b.name + def downloaded(self, b: Bundle) -> Path | None: + """The file fetch(b) returns when the bundle is downloaded already, else None.""" + dst = (self.download_dir if b.remote else self.local_download_dir()) / "bundles" / b.name return dst if dst.is_file() and dst.stat().st_size > 0 else None - def cached_raw(self, e: Location) -> Path | None: - """The file fetch_raw(e) returns when it is in the cache already, else None.""" + def downloaded_raw(self, e: Location) -> Path | None: + """The file fetch_raw(e) returns when it is downloaded already, else None.""" rel = remote_path(e["internal_id"]) - dst = self.cache_dir / "raw" / rel.lstrip("/") if rel is not None else None + dst = self.download_dir / "raw" / rel.lstrip("/") if rel is not None else None return dst if dst is not None and dst.is_file() and dst.stat().st_size > 0 else None def fetch(self, b: Bundle) -> Path: """Local path to the decrypted bundle (CDN download or APK read).""" - dst = (self.cache_dir if b.remote else self.local_cache_dir()) / "bundles" / b.name + dst = (self.download_dir if b.remote else self.local_download_dir()) / "bundles" / b.name dst.parent.mkdir(parents=True, exist_ok=True) if dst.exists() and dst.stat().st_size > 0: return dst @@ -374,7 +378,7 @@ def _url(self, internal_id: str) -> str: name = internal_id.rsplit("/", 1)[-1] cdn = self._cdn_for(name) if not cdn: - raise RuntimeError(f"{name} not cached and no CDN base given") + raise RuntimeError(f"{name} not downloaded and no CDN base given") path = remote_path(internal_id) if path is None: raise ValueError(f"{internal_id}: not a remote location") @@ -411,7 +415,7 @@ def fetch_raw(self, e: Location) -> Path: rel = remote_path(iid) if rel is None: raise ValueError(f"not a CDN asset: {iid}") - dst = self.cache_dir / "raw" / rel.lstrip("/") + dst = self.download_dir / "raw" / rel.lstrip("/") dst.parent.mkdir(parents=True, exist_ok=True) if not (dst.exists() and dst.stat().st_size > 0): _write_atomic(dst, self._download(self._url(iid))) @@ -430,7 +434,7 @@ def apk_bundle(self, name_contains: str) -> Path: if len(names) != 1: raise KeyError(f"{name_contains!r}: {len(names)} APK bundles match") name = names[0].rsplit("/", 1)[1] - dst = self.local_cache_dir() / "bundles" / name + dst = self.local_download_dir() / "bundles" / name if not (dst.exists() and dst.stat().st_size > 0): dst.parent.mkdir(parents=True, exist_ok=True) self.check_apk(z) @@ -491,10 +495,10 @@ def _urlopen(url: str, timeout: float) -> bytes: def _setting(v: Lazy[T], needed_by: str) -> T | None: - """`v`, or what it returns when it is a function; its ConfigError names `needed_by` (what is not cached).""" + """`v`, or what it returns when it is a function; its ConfigError names `needed_by` (what is not downloaded).""" if not callable(v): return cast("T | None", v) try: return cast("T", v()) except ConfigError as e: - raise ConfigError(f"{needed_by} is not in the cache: {e}") from None + raise ConfigError(f"{needed_by} is not downloaded: {e}") from None diff --git a/src/nnnotes/source/jp.py b/src/nnnotes/source/jp.py index 7763ec2..9cf141d 100644 --- a/src/nnnotes/source/jp.py +++ b/src/nnnotes/source/jp.py @@ -153,9 +153,10 @@ def from_dict(cls, data: object) -> Source: raise ValueError("invalid JP source metadata") return cls(**fields) - def cache_dir(self, cache: str | os.PathLike[str]) -> Path: + def download_dir(self, downloads: str | os.PathLike[str]) -> Path: + """The directory of this snapshot's files in `downloads`: jp///.""" site = hashlib.sha256(self.cdn.encode()).hexdigest()[:16] - return Path(cache) / "jp" / site / self.version / self.hash + return Path(downloads) / "jp" / site / self.version / self.hash @dataclass(frozen=True) @@ -303,13 +304,15 @@ def write_source(path: str | os.PathLike[str], data: bytes, source: Source) -> N def open_catalog( cfg: Config, region: str, + downloads: Path, *, catalog_file: str | os.PathLike[str] | None = None, bundle_key: BundleKey | Callable[[], BundleKey] | None = None, apk: Path | None = None, ) -> Catalog: """The catalog of the JP region `region`: `catalog_file` with its .source.json sidecar, else the catalog of the - asset version the API serves now (from the cache, downloaded on first use).""" + asset version the API serves now (from `downloads`, the depot's directory of downloaded files, downloaded on first + use).""" client = Client(cfg, region) if catalog_file is not None: data = Path(catalog_file).read_bytes() @@ -318,7 +321,7 @@ def open_catalog( source = client.observe().source if source is None: raise gameapi.GameApiError("JP Version: no Android asset version applies to this client") - path = source.cache_dir(cfg.require_path("paths", "cache")) / "catalog_main.bin" + path = source.download_dir(downloads) / "catalog_main.bin" if path.exists(): data = path.read_bytes() if not sidecar(path).exists(): @@ -328,7 +331,7 @@ def open_catalog( write_atomic(path, data) write_source(path, data, source) elif read_source(path, data) != source: - raise ConfigError("JP cached catalog source differs from the selected version") + raise ConfigError("JP downloaded catalog source differs from the selected version") else: data = client.get(source.catalog_url, source=source, limit=CATALOG_LIMIT) parse_header(data) @@ -337,7 +340,7 @@ def open_catalog( write_source(path, data, source) return Catalog( data, - source.cache_dir(cfg.require_path("paths", "cache")), + source.download_dir(downloads), bundle_key=bundle_key, apk=apk, source=source, diff --git a/src/nnnotes/source/nnnotes.example.toml b/src/nnnotes/source/nnnotes.example.toml index 8f97483..0221b2b 100644 --- a/src/nnnotes/source/nnnotes.example.toml +++ b/src/nnnotes/source/nnnotes.example.toml @@ -61,11 +61,9 @@ version = "" [paths] # catalog .bin file to read instead of downloading; JP source metadata is stored in .source.json catalog = "" -# cache directory: downloaded catalogs, decrypted bundles, raw CDN files (flag --cache) -cache = "" -# store of the asset export: contents, task results, catalog versions, run manifests; empty: /store -# (flag --store) -store = "" +# depot: downloaded game files, the results of the stages by content, catalog versions, run manifests +# (flag --depot) +depot = "" # decoded master data directory, one
.json per table (`nnnotes master decode`) (flag --master) master = "" # the game's base.apk: APK-local bundles, player settings, CRI HCA keycode (flag --apk) @@ -80,6 +78,6 @@ node = "" player = "" [export] -# how `nnnotes export assets` makes the layout files from the store: auto (a clone where the file system makes them, else -# a hard link to the read-only store object, else a copy), clone, hard or copy; empty: auto (flag --link) +# how `nnnotes export assets` makes the layout files from the depot: auto (a clone where the file system makes them, else +# a hard link to the read-only depot object, else a copy), clone, hard or copy; empty: auto (flag --link) link = "" diff --git a/src/nnnotes/source/session.py b/src/nnnotes/source/session.py index 7ad2c88..aba1841 100644 --- a/src/nnnotes/source/session.py +++ b/src/nnnotes/source/session.py @@ -12,7 +12,7 @@ from . import gameapi from .addressables import BundleKey from .catalog import Catalog -from .config import Config, ConfigError, check_catalog_version, describe +from .config import Config, ConfigError, check_catalog_version from .master import MasterKey __all__ = ["Session"] @@ -49,18 +49,13 @@ def required_path(self, section: str, key: str, kind: str = "file") -> Path: assert p is not None # require_path: the setting is set return p - def store_root(self) -> Path: - """The store directory: `[paths] store`, else the store/ directory of `[paths] cache`.""" - p = self.cfg.path("paths", "store") - if p is not None: - return p - cache = self.cfg.path("paths", "cache") - if cache is not None: - return cache / "store" - raise ConfigError( - "setting paths.store is not set (nor paths.cache, whose store/ directory is the default): " - f"give it as {describe('paths', 'store', '--store')}" - ) + def depot_root(self) -> Path: + """The depot directory: `[paths] depot`, required.""" + return self.cfg.require_path("paths", "depot") + + def downloads(self) -> Path: + """The directory of the depot that holds the downloaded game files (its downloads/ directory).""" + return self.depot_root() / "downloads" def apk(self, region: str | None = None) -> Path: """`[paths] apk` of `region`, required.""" @@ -106,20 +101,20 @@ def catalog(self, bundles: bool = True, region: str | None = None) -> Catalog: """The catalog of `[catalog] language` (merged with the APK's when `[paths] apk` is set), fetching from the CDN of `region`. International versioned catalogs are isolated by CDN root and resource version. `bundles`: bundles will be fetched; else only the catalog is read. An explicit `[paths] catalog` file bypasses - discovery; a version pin bypasses the API but needs the CDN root to identify the cache. The bundle key is + discovery; a version pin bypasses the API but needs the CDN root to identify the downloads. The bundle key is only read for a missing encrypted bundle.""" cfg = self.cfg.for_region(region) if region else self.cfg session = Session(cfg) - cache = cfg.require_path("paths", "cache") + downloads = session.downloads() catbin = session.path("paths", "catalog") apk = session.path("paths", "apk") key = (lambda: session.bundle_key()) if bundles else None if cfg.provider() == "jp": from .jp import open_catalog - return open_catalog(cfg, cfg.region(), catalog_file=catbin, bundle_key=key, apk=apk) + return open_catalog(cfg, cfg.region(), downloads, catalog_file=catbin, bundle_key=key, apk=apk) cdn = (lambda: cfg.cdn(region or cfg.region())) if bundles or catbin is None else None if catbin is not None: - return Catalog(catbin.read_bytes(), cache, cdn=cdn, bundle_key=key, apk=apk) + return Catalog(catbin.read_bytes(), downloads, cdn=cdn, bundle_key=key, apk=apk) language = cfg.require("catalog", "language") - return Catalog.load(language, cache, cdn=cdn, bundle_key=key, apk=apk, version=session.catalog_version()) + return Catalog.load(language, downloads, cdn=cdn, bundle_key=key, apk=apk, version=session.catalog_version()) diff --git a/tests/cli/test_assets.py b/tests/cli/test_assets.py index 0c7089b..8778e07 100644 --- a/tests/cli/test_assets.py +++ b/tests/cli/test_assets.py @@ -1,4 +1,4 @@ -"""The asset commands on synthetic catalogs: export assets, store plan, store run, catalog versions, store verify. +"""The asset commands on synthetic catalogs: export assets, depot plan, depot run, catalog versions, depot verify. Bundles here are UnityFS-signed files whose body is the census document a real census would find; the census and export stages of the pipeline are replaced by small stages that read them (the registry is patched), the catalog, @@ -19,8 +19,8 @@ from nnnotes.pipeline import contract from nnnotes.pipeline.census import CensusStage from nnnotes.pipeline.contract import Cost +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.stages import Output, Stage -from nnnotes.pipeline.store import Store from nnnotes.source.config import ConfigError from tests.helpers import synth @@ -134,27 +134,28 @@ def catalog_bytes(extra=()) -> bytes: return synth.CatalogWriter().build(entries) -def setup_data(tmp_path, docs=None, cached=None, extra=()) -> dict: - """A catalog file and a cache with the (decrypted) bundles; -> the global flags.""" +def setup_data(tmp_path, docs=None, downloaded=None, extra=(), depots=("s",)) -> dict: + """A catalog file and, in the downloads of each depot, the (decrypted) bundles; -> the global flags.""" docs = censuses() if docs is None else docs - cache = tmp_path / "cache" - (cache / "bundles").mkdir(parents=True, exist_ok=True) - for n, doc in docs.items(): - if cached is None or n in cached: - (cache / "bundles" / NAMES[n]).write_bytes(bundle_bytes(doc)) + for depot in depots: + downloads = tmp_path / depot / "downloads" + (downloads / "bundles").mkdir(parents=True, exist_ok=True) + for n, doc in docs.items(): + if downloaded is None or n in downloaded: + (downloads / "bundles" / NAMES[n]).write_bytes(bundle_bytes(doc)) cat = tmp_path / "catalog.bin" cat.write_bytes(catalog_bytes(extra)) - return {"cache": cache, "catalog": cat} + return {"downloads": tmp_path / depots[0] / "downloads", "catalog": cat} # ---------------------------------------------------------------- stand-in stages class FakeCensus(CensusStage): """unity.census over the synthetic bundles (their body is the census).""" - def run(self, task, store): - data = store.input_bytes(task.input("bundle")) + def run(self, task, depot): + data = depot.input_bytes(task.input("bundle")) doc = contract.loads(data[8:]) - content = store.add(contract.encode(doc), "json") + content = depot.add(contract.encode(doc), "json") rec = contract.artifact( contract.artifact_id(task.id, "census"), content, @@ -193,9 +194,9 @@ def inputs(self, subject, env): def estimate(self, subject, env, inputs): return Cost(0.01, 1 << 20) - def run(self, task, store): - doc = contract.loads(store.input_bytes(task.input("census"))) - scripts = contract.loads(store.input_bytes(task.input("scripts")))["scripts"] + def run(self, task, depot): + doc = contract.loads(depot.input_bytes(task.input("census"))) + scripts = contract.loads(depot.input_bytes(task.input("scripts")))["scripts"] if any(o["class"] == "Crash" for f in doc["files"] for o in f["objects"]): raise ConfigError("setting paths.tool is not set") container = {(e["file"], e["pathId"]): e["path"] for e in doc["assetBundles"][0]["container"]} @@ -229,7 +230,7 @@ def run(self, task, store): arts.append( contract.artifact( aid, - store.add(contract.encode(body), "json"), + depot.add(contract.encode(body), "json"), contract.provenance(task, obj=facts), {"kind": "fake.json"}, ) @@ -275,7 +276,7 @@ def nn(capsys, *argv): def flags(d: dict) -> list: - return ["--cache", d["cache"], "--catalog", d["catalog"]] + return ["--catalog", d["catalog"]] def tree(root: Path, skip=()) -> dict: @@ -287,7 +288,7 @@ def tree(root: Path, skip=()) -> dict: return out -def export(capsys, tmp_path, d, store, out, *more): +def export(capsys, tmp_path, d, depot, out, *more): return nn( capsys, *flags(d), @@ -295,8 +296,8 @@ def export(capsys, tmp_path, d, store, out, *more): "assets", "-o", tmp_path / out, - "--store", - tmp_path / store, + "--depot", + tmp_path / depot, "--workers", "0", "--json", @@ -331,8 +332,8 @@ def test_select_adds_the_dependencies_to_the_census(): # ---------------------------------------------------------------- export def test_export_writes_layouts_reports_and_hits_the_second_time(tmp_path, capsys, fake): - d = setup_data(tmp_path) - code, out, err = export(capsys, tmp_path, d, "store", "out") + d = setup_data(tmp_path, depots=("depot",)) + code, out, err = export(capsys, tmp_path, d, "depot", "out") assert code == 1, err # the broken object is a failed item summary = json.loads(out) assert summary["items"] == {"exported": 8, "contained": 4, "generic": 0, "unsupported": 0, "failed": 1} @@ -358,23 +359,23 @@ def test_export_writes_layouts_reports_and_hits_the_second_time(tmp_path, capsys "failed": 0, } run = contract.loads((reports / "run.json").read_bytes()) - assert Store(tmp_path / "store").run(run["id"]) == run + assert Depot(tmp_path / "depot").run(run["id"]) == run assert run["context"]["catalog"]["sha"] == contract.sha256(d["catalog"].read_bytes()) first = files - code, out, _ = export(capsys, tmp_path, d, "store", "out") + code, out, _ = export(capsys, tmp_path, d, "depot", "out") again = json.loads(out) assert code == 1 and again["tasks"]["ran"] == 0 and again["run"] == summary["run"] assert again["layouts"]["original"]["write"] == 0 same = lambda t: {k: v for k, v in t.items() if k != f"{assets.REPORTS}/run.json"} # noqa: E731 assert same(tree(tmp_path / "out")) == same(first) # the run's statuses are hits now code, out, _ = nn( - capsys, *flags(d), "store", "plan", "-o", tmp_path / "out", "--store", tmp_path / "store", "--check" + capsys, *flags(d), "depot", "plan", "-o", tmp_path / "out", "--depot", tmp_path / "depot", "--check" ) assert code == 0, out def test_export_is_the_same_with_worker_processes(tmp_path, capsys, fake): - d = setup_data(tmp_path) + d = setup_data(tmp_path, depots=("s0", "s2")) export(capsys, tmp_path, d, "s0", "o0") code, _, err = nn( capsys, @@ -383,7 +384,7 @@ def test_export_is_the_same_with_worker_processes(tmp_path, capsys, fake): "assets", "-o", tmp_path / "o2", - "--store", + "--depot", tmp_path / "s2", "--workers", "2", @@ -391,37 +392,38 @@ def test_export_is_the_same_with_worker_processes(tmp_path, capsys, fake): "original,cas", ) assert code == 1, err - assert tree(tmp_path / "s0", ["inputs", "runs", "costs"]) == tree(tmp_path / "s2", ["inputs", "runs", "costs"]) + skip = ["inputs", "runs", "costs", "downloads"] + assert tree(tmp_path / "s0", skip) == tree(tmp_path / "s2", skip) o0, o2 = tree(tmp_path / "o0"), tree(tmp_path / "o2" / "original") assert {k: v for k, v in o0.items() if not k.startswith(assets.REPORTS)} == o2 cas = tree(tmp_path / "o2" / "cas") assert all(p.startswith("assets/") or p == "manifest.json" for p in cas) -def test_plan_emit_tasks_and_run_stage_give_the_exported_store(tmp_path, capsys, fake): - d = setup_data(tmp_path) +def test_plan_emit_tasks_and_run_stage_give_the_exported_depot(tmp_path, capsys, fake): + d = setup_data(tmp_path, depots=("orch", "q")) export(capsys, tmp_path, d, "orch", "o") rounds = 0 while True: tasks = tmp_path / f"round{rounds}" code, out, err = nn( - capsys, *flags(d), "store", "plan", "--store", tmp_path / "q", "--emit-tasks", tasks, "--json" + capsys, *flags(d), "depot", "plan", "--depot", tmp_path / "q", "--emit-tasks", tasks, "--json" ) assert code == 0, err doc = json.loads(out) if not any(n["status"] == "run" for n in doc["nodes"]): break - code, out, err = nn(capsys, *flags(d), "store", "run", tasks, "--store", tmp_path / "q") + code, out, err = nn(capsys, *flags(d), "depot", "run", tasks, "--depot", tmp_path / "q") assert code in (0, 1), err rounds += 1 assert rounds == 4 # index + census; scripts + addresses; export; artifacts - skip = ["inputs", "runs", "costs", "catalogs"] + skip = ["inputs", "runs", "costs", "catalogs", "downloads"] assert tree(tmp_path / "orch", skip) == tree(tmp_path / "q", skip) def test_plan_explains_and_waits_for_bundles_not_fetched(tmp_path, capsys, fake): - d = setup_data(tmp_path, cached={"card_1", "shared", "bg_x"}) - code, out, _ = nn(capsys, *flags(d), "store", "plan", "--store", tmp_path / "s", "--json", "--check") + d = setup_data(tmp_path, downloaded={"card_1", "shared", "bg_x"}) + code, out, _ = nn(capsys, *flags(d), "depot", "plan", "--depot", tmp_path / "s", "--json", "--check") assert code == 1 nodes = {n["id"]: n for n in json.loads(out)["nodes"]} card2 = contract.stable_bundle_name(NAMES["card_2"]) @@ -429,21 +431,21 @@ def test_plan_explains_and_waits_for_bundles_not_fetched(tmp_path, capsys, fake) assert nodes[f"unity.census:{card2}"]["reasons"] == [{"code": "pending", "task": f"fetch:{NAMES['card_2']}"}] assert nodes["link.scripts:all"]["status"] == "unknown" assert nodes["catalog.index:main"]["reasons"] == [{"code": "new-input"}] - code, out, _ = nn(capsys, *flags(d), "store", "plan", "--store", tmp_path / "s") + code, out, _ = nn(capsys, *flags(d), "depot", "plan", "--depot", tmp_path / "s") assert "unknown" in out and "tasks:" in out def test_parameters_change_keys_and_are_explained(tmp_path, capsys, fake): d = setup_data(tmp_path) export(capsys, tmp_path, d, "s", "o") - base = [*flags(d), "store", "plan", "--store", tmp_path / "s", "--json"] + base = [*flags(d), "depot", "plan", "--depot", tmp_path / "s", "--json"] code, out, _ = nn(capsys, *base, "--png-level", "3") runs = [n for n in json.loads(out)["nodes"] if n["status"] == "run"] assert {n["stage"] for n in runs} == {"unity.export"} assert runs[0]["reasons"] == [{"code": "params-changed", "path": "/png/level", "old": 6, "new": 3}] tid = runs[0]["id"] code, out, _ = nn( - capsys, *flags(d), "store", "plan", "--store", tmp_path / "s", "--png-level", "3", "--why", tid, "--json" + capsys, *flags(d), "depot", "plan", "--depot", tmp_path / "s", "--png-level", "3", "--why", tid, "--json" ) w = json.loads(out) assert w["keyParts"]["old"]["params"]["png"]["level"] == 6 and w["keyParts"]["new"]["params"]["png"]["level"] == 3 @@ -465,7 +467,7 @@ def test_usage_errors(tmp_path, capsys, fake, monkeypatch): code, _, err = export(capsys, tmp_path, d, "s", "o", "--flac-level", "5") assert code == 2 and "no stage of the pipeline encodes FLAC" in err code, _, err = nn(capsys, "export", "assets", "-o", tmp_path / "o") - assert code == 2 and "paths.store" in err # no store and no cache + assert code == 2 and "paths.depot" in err # no depot def test_the_flac_level_is_a_parameter_of_the_stages_that_encode_flac(tmp_path, capsys, fake, monkeypatch): @@ -473,7 +475,7 @@ def test_the_flac_level_is_a_parameter_of_the_stages_that_encode_flac(tmp_path, d = setup_data(tmp_path) assert export(capsys, tmp_path, d, "s", "o", "--flac-level", "13")[0] == 2 export(capsys, tmp_path, d, "s", "o") - base = [*flags(d), "store", "plan", "--store", tmp_path / "s", "--flac-level", "5"] + base = [*flags(d), "depot", "plan", "--depot", tmp_path / "s", "--flac-level", "5"] code, out, _ = nn(capsys, *base, "--json") tid = next(n["id"] for n in json.loads(out)["nodes"] if n["stage"] == "unity.export") code, out, _ = nn(capsys, *base, "--why", tid, "--json") @@ -550,7 +552,7 @@ def test_export_dry_run_and_explain(tmp_path, capsys, fake): # ---------------------------------------------------------------- run-stage def test_run_stage_refuses_incompatible_tasks(tmp_path, capsys, fake): d = setup_data(tmp_path) - nn(capsys, *flags(d), "store", "plan", "--store", tmp_path / "s", "--emit-tasks", tmp_path / "t") + nn(capsys, *flags(d), "depot", "plan", "--depot", tmp_path / "s", "--emit-tasks", tmp_path / "t") (f,) = [ p for p in sorted((tmp_path / "t").glob("*.json")) @@ -559,7 +561,7 @@ def test_run_stage_refuses_incompatible_tasks(tmp_path, capsys, fake): doc = contract.loads(f.read_bytes()) edited = tmp_path / "edited.json" edited.write_bytes(contract.encode(dict(doc, params={"x": 1}))) - code, _, err = nn(capsys, "store", "run", f, edited, "--store", tmp_path / "s") + code, _, err = nn(capsys, "depot", "run", f, edited, "--depot", tmp_path / "s") assert code == 4 and "incompatible task" in err assert not (tmp_path / "s" / "ac").exists() # nothing of the batch ran other = tmp_path / "other.json" @@ -567,8 +569,8 @@ def test_run_stage_refuses_incompatible_tasks(tmp_path, capsys, fake): other.write_bytes( contract.encode(contract.Task("catalog.index", 9, t.subject, t.params, t.atoms, t.inputs, t.context).to_json()) ) - assert nn(capsys, "store", "run", other, "--store", tmp_path / "s")[0] == 4 - code, out, _ = nn(capsys, "store", "run", f, "--store", tmp_path / "s", "--json") + assert nn(capsys, "depot", "run", other, "--depot", tmp_path / "s")[0] == 4 + code, out, _ = nn(capsys, "depot", "run", f, "--depot", tmp_path / "s", "--json") assert code == 0 and json.loads(out)["summary"] == {"tasks": 1, "hit": 0, "ran": 1, "failed": 0} @@ -585,19 +587,19 @@ def main(argv): cli.main(argv) except SystemExit as e: assert not e.code, e.code -main(args + ["store", "plan", "--store", {str(tmp_path / "s")!r}, "--emit-tasks", {str(tmp_path / "t")!r}]) +main(args + ["depot", "plan", "--depot", {str(tmp_path / "s")!r}, "--emit-tasks", {str(tmp_path / "t")!r}]) index = [str(p) for p in Path({str(tmp_path / "t")!r}).glob("*.json") if contract.loads(p.read_bytes())["stage"]["name"] == "catalog.index"] assert len(index) == 1, index -main(["store", "run", *index, "--store", {str(tmp_path / "s")!r}]) -main(args + ["store", "plan", "--store", {str(tmp_path / "s")!r}, "--json"]) +main(["depot", "run", *index, "--depot", {str(tmp_path / "s")!r}]) +main(args + ["depot", "plan", "--depot", {str(tmp_path / "s")!r}, "--json"]) assert "UnityPy" not in sys.modules, "UnityPy imported" """ r = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, cwd=tmp_path) assert r.returncode == 0, r.stderr[-3000:] -# ---------------------------------------------------------------- catalogs and store +# ---------------------------------------------------------------- catalogs and depot def test_catalog_versions_import_list_and_diff(tmp_path, capsys): a = tmp_path / "a.bin" a.write_bytes(catalog_bytes()) @@ -608,7 +610,7 @@ def test_catalog_versions_import_list_and_diff(tmp_path, capsys): ] b = tmp_path / "b.bin" b.write_bytes(synth.CatalogWriter().build(entries)) - s = ["--store", tmp_path / "s"] + s = ["--depot", tmp_path / "s"] assert nn(capsys, "catalog", "import", a, "--label", "v1", *s)[0] == 0 code, out, _ = nn(capsys, "catalog", "import", b, *s) assert code == 0 and contract.sha256(b.read_bytes())[:12] in out @@ -625,7 +627,7 @@ def test_catalog_versions_import_list_and_diff(tmp_path, capsys): def test_catalog_import_takes_the_resource_version_from_the_file_name(tmp_path, capsys): - s = ["--store", tmp_path / "s"] + s = ["--depot", tmp_path / "s"] named = tmp_path / "catalog_1.0.0.201_en.bin" named.write_bytes(catalog_bytes()) main = tmp_path / "catalog_main_en.bin" @@ -650,31 +652,31 @@ def test_catalog_import_resource_version_option(): catalog_cli.import_resource_version("../x", file, None) -def test_store_verify(tmp_path, capsys, fake): +def test_depot_verify(tmp_path, capsys, fake): d = setup_data(tmp_path) export(capsys, tmp_path, d, "s", "o") - code, out, _ = nn(capsys, "store", "verify", "--store", tmp_path / "s", "--json") + code, out, _ = nn(capsys, "depot", "verify", "--depot", tmp_path / "s", "--json") r = json.loads(out) assert code == 0 and r["problemCount"] == 0 and r["contents"] > 0 and r["results"] > 0 victim = next(p for p in sorted((tmp_path / "s" / "cas").rglob("*")) if p.is_file()) - assert os.stat(victim).st_mode & 0o222 == 0 # store objects are read-only + assert os.stat(victim).st_mode & 0o222 == 0 # depot objects are read-only os.chmod(victim, 0o666) victim.write_bytes(victim.read_bytes() + b"x") - code, out, _ = nn(capsys, "store", "verify", "--store", tmp_path / "s", "--json") + code, out, _ = nn(capsys, "depot", "verify", "--depot", tmp_path / "s", "--json") problems = {p["problem"] for p in json.loads(out)["problems"]} assert code == 1 and "content differs from its id" in problems assert any(p.endswith("bytes of another size") for p in problems) - code, out, _ = nn(capsys, "store", "verify", "--quick", "--store", tmp_path / "s", "--json") + code, out, _ = nn(capsys, "depot", "verify", "--quick", "--depot", tmp_path / "s", "--json") assert code == 1 and all("another size" in p["problem"] for p in json.loads(out)["problems"]) # ---------------------------------------------------------------- the command line def test_the_commands_are_part_of_nnnotes(tmp_path, capsys, fake): try: - cli.build_parser().parse_args(["store", "verify"]) + cli.build_parser().parse_args(["depot", "verify"]) except SystemExit: pytest.skip("the command line does not register the asset commands") - assert ("paths", "store") in common.FLAG_SETTINGS + assert ("paths", "depot") in common.FLAG_SETTINGS d = setup_data(tmp_path) with pytest.raises(SystemExit) as e: cli.main( @@ -684,7 +686,7 @@ def test_the_commands_are_part_of_nnnotes(tmp_path, capsys, fake): "assets", "-o", str(tmp_path / "o"), - "--store", + "--depot", str(tmp_path / "s"), "--workers", "0", @@ -694,7 +696,7 @@ def test_the_commands_are_part_of_nnnotes(tmp_path, capsys, fake): capsys.readouterr() with pytest.raises(SystemExit) as e: cli.main( - [*map(str, flags(d)), "store", "plan", "-o", str(tmp_path / "o"), "--store", str(tmp_path / "s"), "--check"] + [*map(str, flags(d)), "depot", "plan", "-o", str(tmp_path / "o"), "--depot", str(tmp_path / "s"), "--check"] ) assert e.value.code == 0 @@ -721,21 +723,21 @@ def inputs(self, subject, env): def estimate(self, subject, env, inputs): return Cost(0.01, 1 << 20) - def run(self, task, store): + def run(self, task, depot): oids = ["CAB-none:9"] for inp in task.inputs: - for f in contract.loads(store.input_bytes(inp))["files"]: + for f in contract.loads(depot.input_bytes(inp))["files"]: oids += [contract.object_id(f["name"], o["pathId"]) for o in f["objects"]] rec = contract.artifact( contract.artifact_id(task.id, "objects"), - store.add(contract.encode(sorted(oids)), "json"), + depot.add(contract.encode(sorted(oids)), "json"), contract.provenance(task), {"kind": "fake.objects"}, ) return Output([rec], []) - def derived(self, task_id, result, store, paths): - oids = contract.loads(store.read(result["artifacts"][0]["content"]["sha256"])) + def derived(self, task_id, result, depot, paths): + oids = contract.loads(depot.read(result["artifacts"][0]["content"]["sha256"])) doc = {o: {a: paths.artifact(a) for a in paths.objects(o)} for o in oids} return [("derived/objects.json", contract.encode(doc))] @@ -755,7 +757,7 @@ def test_derived_documents_read_the_paths_of_the_objects_they_name(tmp_path, cap assert doc[card] == {f"{card}#json": "Assets/Game/Card/1/full.png.json"} assert doc[f"{CABS['card_1']}:1"] == doc[card] # contained in it assert (tmp_path / "o" / "Assets" / "Game" / "Card" / "1" / "full.png.json").is_file() - code, out, _ = nn(capsys, *flags(d), "store", "plan", "-o", tmp_path / "o", "--store", tmp_path / "s", "--check") + code, out, _ = nn(capsys, *flags(d), "depot", "plan", "-o", tmp_path / "o", "--depot", tmp_path / "s", "--check") assert code == 0, out @@ -782,7 +784,7 @@ def test_views_are_placed_with_paths_and_every_document_validates(tmp_path, caps "assets", "-o", tmp_path / "o", - "--store", + "--depot", tmp_path / "s", "--workers", "0", @@ -803,16 +805,16 @@ def test_views_are_placed_with_paths_and_every_document_validates(tmp_path, caps assert not any(p.startswith("_tasks/view.") for p in placed) # placed through the derived documents doc = contract.loads((tmp_path / "o" / "original" / "views" / "cards.json").read_bytes()) assert doc["layout"] == "original" and doc["view"] == "cards" - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") run = contract.loads((tmp_path / "o" / assets.REPORTS / "run.json").read_bytes()) v = validators() checked = set() for t in run["tasks"]: - res = store.result(t["key"]) + res = depot.result(t["key"]) for a in res["artifacts"]: if a["content"]["ext"] != "json": continue - body = contract.loads(store.read(a["content"]["sha256"])) + body = contract.loads(depot.read(a["content"]["sha256"])) name = body.get("schema") if isinstance(body, dict) else None if name in v: errors = [e.message for e in v[name].iter_errors(body)] @@ -831,9 +833,9 @@ def test_views_are_placed_with_paths_and_every_document_validates(tmp_path, caps *flags(d), "--master", master, - "store", + "depot", "plan", - "--store", + "--depot", tmp_path / "s", "--json", "--views", @@ -856,7 +858,7 @@ def test_views_are_placed_with_paths_and_every_document_validates(tmp_path, caps errors = [e.message for e in v[name].iter_errors(body)] assert errors == [], (name, errors[:3]) vid = run["context"]["catalog"]["id"] - code, out, _ = nn(capsys, "catalog", "diff", vid[:12], vid[:12], "--json", "--store", tmp_path / "s") + code, out, _ = nn(capsys, "catalog", "diff", vid[:12], vid[:12], "--json", "--depot", tmp_path / "s") diff = json.loads(out) assert diff["objects"]["summary"]["unchanged"] > 0 # the address tables of the runs assert [e.message for e in v["nnnotes.catalog-diff/1"].iter_errors(diff)] == [] @@ -927,12 +929,12 @@ def test_export_records_the_calibration_and_plan_uses_it(tmp_path, capsys, fake) assert contract.loads(cal.read_bytes()) == {"stages": {}} # four bundles: too few assets.write_calibration(tmp_path / "s", {"unity.export": {"cpu": 2.0, "peak": 3.0, "samples": 50}}) code, out, err = nn( - capsys, *flags(d), "store", "plan", "--store", tmp_path / "s", "--json", "--png-level", "1", "--explain" + capsys, *flags(d), "depot", "plan", "--depot", tmp_path / "s", "--json", "--png-level", "1", "--explain" ) assert "cost calibration unity.export: CPU x2.0, peak memory x3.0 (50 measured tasks)" in err runs = [n for n in json.loads(out)["nodes"] if n["stage"] == "unity.export"] assert runs and all(n["cost"] == {"cpuSeconds": 0.02, "peakBytes": 3 << 20} for n in runs) - code, out, _ = nn(capsys, *flags(d), "store", "plan", "--store", tmp_path / "s", "--check") + code, out, _ = nn(capsys, *flags(d), "depot", "plan", "--depot", tmp_path / "s", "--check") assert code == 0 # keys do not change @@ -958,10 +960,10 @@ def subjects(self, env): def names(self, env): return {"s1": ["Cri/Sound/y", "Cri/Sound/x"]} - def run(self, task, store): + def run(self, task, depot): arts = [ contract.artifact( - contract.artifact_id(task.id, role), store.add(data, ext), contract.provenance(task), {"kind": "toy"} + contract.artifact_id(task.id, role), depot.add(data, ext), contract.provenance(task), {"kind": "toy"} ) for role, data, ext in (("a.flac", b"fLaC" + task.subject.encode(), "flac"), ("cues.json", b"[]", "json")) ] @@ -982,7 +984,7 @@ def uses(self, subject, env): cfg = active() return {"tool": str(cfg.path("paths", "ffmpeg")) if cfg is not None else "unset"} - def run(self, task, store): + def run(self, task, depot): return Output([], []) @@ -1023,7 +1025,7 @@ def test_plan_and_export_describe_with_the_same_settings(tmp_path, capsys, monke d = setup_data(tmp_path, censuses(broken=False)) tool = tmp_path / "ffmpeg-bin" code, _, err = nn( - capsys, *flags(d), "--ffmpeg", tool, "store", "plan", "--store", tmp_path / "s", "--emit-tasks", tmp_path / "t" + capsys, *flags(d), "--ffmpeg", tool, "depot", "plan", "--depot", tmp_path / "s", "--emit-tasks", tmp_path / "t" ) assert code == 0, err (planned,) = [ @@ -1041,7 +1043,7 @@ def test_plan_and_export_describe_with_the_same_settings(tmp_path, capsys, monke "assets", "-o", tmp_path / "o", - "--store", + "--depot", tmp_path / "s", "--workers", "0", @@ -1061,14 +1063,14 @@ def test_plan_knows_the_files_the_apk_does_not_have(tmp_path, capsys, fake, monk ) d = setup_data(tmp_path, censuses(broken=False), extra=[("apkonly.bundle", synth.local("apkonly.bundle"), [])]) monkeypatch.setattr("nnnotes.pipeline.catalogdb.apk_version", lambda apk: None) - code, out, err = nn(capsys, *flags(d), "--apk", apk, "store", "plan", "--store", tmp_path / "s", "--json") + code, out, err = nn(capsys, *flags(d), "--apk", apk, "depot", "plan", "--depot", tmp_path / "s", "--json") nodes = {n["id"]: n for n in json.loads(out)["nodes"]} assert "unity.census:apkonly" not in nodes # not pending on a fetch that cannot happen code, out, err = nn( - capsys, *flags(d), "--apk", apk, "store", "plan", "--store", tmp_path / "s", "--emit-tasks", tmp_path / "t" + capsys, *flags(d), "--apk", apk, "depot", "plan", "--depot", tmp_path / "s", "--emit-tasks", tmp_path / "t" ) - assert nn(capsys, *flags(d), "store", "run", tmp_path / "t", "--store", tmp_path / "s")[0] == 0 - code, out, _ = nn(capsys, *flags(d), "--apk", apk, "store", "plan", "--store", tmp_path / "s", "--json") + assert nn(capsys, *flags(d), "depot", "run", tmp_path / "t", "--depot", tmp_path / "s")[0] == 0 + code, out, _ = nn(capsys, *flags(d), "--apk", apk, "depot", "plan", "--depot", tmp_path / "s", "--json") nodes = {n["id"]: n for n in json.loads(out)["nodes"]} assert nodes["link.scripts:all"]["status"] == "run" @@ -1076,7 +1078,7 @@ def test_plan_knows_the_files_the_apk_does_not_have(tmp_path, capsys, fake, monk def test_plan_census_takes_the_censuses_first(tmp_path, capsys, fake): d = setup_data(tmp_path, censuses(broken=False)) code, out, err = nn( - capsys, *flags(d), "store", "plan", "--store", tmp_path / "s", "--census", "--workers", "0", "--json" + capsys, *flags(d), "depot", "plan", "--depot", tmp_path / "s", "--census", "--workers", "0", "--json" ) assert code == 0, err nodes = json.loads(out)["nodes"] @@ -1105,7 +1107,7 @@ def test_names_see_the_tasks_that_ran(tmp_path, capsys, monkeypatch): assert code == 0, err placed = tree(tmp_path / "o") assert "Named/4/a.flac" in placed and "Named/0/a.flac" not in placed # four bundles were exported - code, out, _ = nn(capsys, *flags(d), "store", "plan", "-o", tmp_path / "o", "--store", tmp_path / "s", "--check") + code, out, _ = nn(capsys, *flags(d), "depot", "plan", "-o", tmp_path / "o", "--depot", tmp_path / "s", "--check") assert code == 0, out # plan places them the same way @@ -1151,8 +1153,8 @@ def subjects(self, env): def inputs(self, subject, env): return [env.fact("raw")[subject]] - def run(self, task, store): - content = store.add(store.input_bytes(task.input("raw")), "acb") + def run(self, task, depot): + content = depot.add(depot.input_bytes(task.input("raw")), "acb") return Output( [ contract.artifact( @@ -1192,7 +1194,7 @@ def raw_export(capsys, tmp_path, d, apk): "assets", "-o", tmp_path / "o", - "--store", + "--depot", tmp_path / "s", "--workers", "0", @@ -1202,7 +1204,7 @@ def raw_export(capsys, tmp_path, d, apk): def test_raw_files_of_the_apk_are_read_from_it(tmp_path, capsys, monkeypatch): apk, name = raw_apk(tmp_path, monkeypatch) d = setup_data(tmp_path, censuses(broken=False)) - code, out, err = nn(capsys, *flags(d), "--apk", apk, "store", "plan", "--store", tmp_path / "s", "--json") + code, out, err = nn(capsys, *flags(d), "--apk", apk, "depot", "plan", "--depot", tmp_path / "s", "--json") nodes = {n["id"]: n for n in json.loads(out)["nodes"]} assert nodes["cri.audio:cri_assets_embcri/sound/initialse"]["reasons"] == [ {"code": "pending", "task": f"fetch:{name}"} @@ -1216,7 +1218,7 @@ def test_raw_files_of_the_apk_are_read_from_it(tmp_path, capsys, monkeypatch): "assets", "-o", tmp_path / "o", - "--store", + "--depot", tmp_path / "s", "--workers", "0", @@ -1224,13 +1226,13 @@ def test_raw_files_of_the_apk_are_read_from_it(tmp_path, capsys, monkeypatch): ) assert code == 0, err assert json.loads(out)["sources"] == {} - assert (d["cache"] / "raw" / "Android" / name).read_bytes() == b"@UTF sheet" # where a CDN file would be + assert (d["downloads"] / "raw" / "Android" / name).read_bytes() == b"@UTF sheet" # where a CDN file would be run = contract.loads((tmp_path / "o" / assets.REPORTS / "run.json").read_bytes()) assert {t["id"]: t["status"] for t in run["tasks"]}["cri.audio:cri_assets_embcri/sound/initialse"] == "ran" code, out, _ = nn( - capsys, *flags(d), "--apk", apk, "store", "plan", "-o", tmp_path / "o", "--store", tmp_path / "s", "--check" + capsys, *flags(d), "--apk", apk, "depot", "plan", "-o", tmp_path / "o", "--depot", tmp_path / "s", "--check" ) - assert code == 0, out # read from the cache now + assert code == 0, out # read from the downloads now def test_the_raw_files_are_fetched_while_the_bundle_stages_run(tmp_path, capsys, monkeypatch): @@ -1246,11 +1248,11 @@ def inputs(self, kind, entries, fetch): order.append(("raw fetch", threading.current_thread() is threading.main_thread())) return real_inputs(self, kind, entries, fetch) - def run(self, task, store): + def run(self, task, depot): if not exported.is_set(): order.append("unity.export") exported.set() - return real_run(self, task, store) + return real_run(self, task, depot) monkeypatch.setattr(assets.Workspace, "_inputs", inputs) monkeypatch.setattr(FakeExport, "run", run) @@ -1261,7 +1263,7 @@ def run(self, task, store): assert "fetched while the bundle stages run" in err and "0 raw files not readable" in err run_doc = contract.loads((tmp_path / "o" / assets.REPORTS / "run.json").read_bytes()) assert {t["id"]: t["status"] for t in run_doc["tasks"]}["cri.audio:cri_assets_embcri/sound/initialse"] == "ran" - assert (d["cache"] / "raw" / "Android" / name).read_bytes() == b"@UTF sheet" + assert (d["downloads"] / "raw" / "Android" / name).read_bytes() == b"@UTF sheet" def test_a_setting_the_raw_fetch_lacks_aborts_the_run(tmp_path, capsys, monkeypatch): @@ -1309,7 +1311,7 @@ def interrupted(*a, **kw): code, _, err = raw_export(capsys, tmp_path, d, apk) assert code == 3 and "interrupted" in err assert ended.wait(10) - assert not (d["cache"] / "raw" / "Android" / name).exists() # not fetched after the interrupt + assert not (d["downloads"] / "raw" / "Android" / name).exists() # not fetched after the interrupt def test_worker_counts_follow_the_cpus_this_process_may_use(monkeypatch): diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index 36cb2cf..0f85f43 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -64,7 +64,7 @@ def test_version_and_help(capsys): "master", "export", "apk", - "store", + "depot", "voices", "site", "music-data", @@ -133,8 +133,8 @@ def test_catalog_listing(catalog_file, tmp_path, capsys): [ "--catalog", str(catalog_file), - "--cache", - str(tmp_path / "cache"), + "--depot", + str(tmp_path / "depot"), "catalog", "keys", "--prefix", @@ -148,11 +148,11 @@ def test_catalog_listing(catalog_file, tmp_path, capsys): def test_missing_settings_exit_2_and_name_the_setting(catalog_file, tmp_path, cdn, capsys, monkeypatch): - base = ["--catalog", str(catalog_file), "--cache", str(tmp_path / "cache")] + base = ["--catalog", str(catalog_file), "--depot", str(tmp_path / "depot")] code, out, err = run(base + ["bundle", "pull", "Char/A"], capsys) assert code == 2 and out == "" assert err.strip() == ( - "nnnotes: a_01.bundle is not in the cache: setting catalog.region is not set: give it " + "nnnotes: a_01.bundle is not downloaded: setting catalog.region is not set: give it " "as `region` in the [catalog] table of the config file, the environment variable " "NNNOTES_CATALOG_REGION or --region (no config file was found: `nnnotes config init` " "writes one to fill in)" @@ -164,7 +164,7 @@ def test_missing_settings_exit_2_and_name_the_setting(catalog_file, tmp_path, cd assert code == 2 and "bundle.key" in err and "NNNOTES_BUNDLE_KEY" in err assert cdn not in err and len(err.strip().splitlines()) == 1 code, _, err = run(["bundle", "pull", "Char/A"], capsys) - assert code == 2 and "paths.cache" in err and "NNNOTES_PATHS_CACHE" in err and "--cache" in err + assert code == 2 and "paths.depot" in err and "NNNOTES_PATHS_DEPOT" in err and "--depot" in err def test_malformed_setting_names_setting_not_value(catalog_file, tmp_path, cdn, capsys, monkeypatch): @@ -176,8 +176,8 @@ def test_malformed_setting_names_setting_not_value(catalog_file, tmp_path, cdn, [ "--catalog", str(catalog_file), - "--cache", - str(tmp_path / "cache"), + "--depot", + str(tmp_path / "depot"), "--region", "zz", "bundle", @@ -187,14 +187,14 @@ def test_malformed_setting_names_setting_not_value(catalog_file, tmp_path, cdn, capsys, ) assert code == 2 and err.strip() == ( - "nnnotes: a_01.bundle is not in the cache: setting bundle.key: must be 16 bytes (32 hex digits)" + "nnnotes: a_01.bundle is not downloaded: setting bundle.key: must be 16 bytes (32 hex digits)" ) assert bad not in err def test_missing_path_on_disk(tmp_path, capsys): code, _, err = run( - ["--cache", str(tmp_path / "c"), "--catalog", str(tmp_path / "absent.bin"), "catalog", "keys"], capsys + ["--depot", str(tmp_path / "c"), "--catalog", str(tmp_path / "absent.bin"), "catalog", "keys"], capsys ) assert code == 2 and "setting paths.catalog: file" in err @@ -206,14 +206,14 @@ def test_pull_from_config_file(catalog_file, tmp_path, cdn, capsys): f'[bundle]\nkey = "{KEY_HEX}"\nnonce_seed = "{SEED_HEX}"\n' f'[catalog]\nregion = "zz"\nlanguage = "xx"\n' f'[servers.zz]\ncdn = "{cdn}"\n' - f'[paths]\ncatalog = "../{catalog_file.name}"\ncache = "cache"\n', + f'[paths]\ncatalog = "../{catalog_file.name}"\ndepot = "depot"\n', encoding="utf-8", ) code, out, err = run(["--config", str(conf), "bundle", "pull", "Char/A", "Char/B"], capsys) assert code == 0, err - cached = tmp_path / "conf" / "cache" / "bundles" / "a_01.bundle" - assert cached.read_bytes() == synth.fake_bundle("a_01.bundle") - assert [Path(line).resolve() for line in out.splitlines()] == [cached.resolve()] * 2 + pulled = tmp_path / "conf" / "depot" / "downloads" / "bundles" / "a_01.bundle" + assert pulled.read_bytes() == synth.fake_bundle("a_01.bundle") + assert [Path(line).resolve() for line in out.splitlines()] == [pulled.resolve()] * 2 assert KEY_HEX not in out + err @@ -263,13 +263,14 @@ def test_web_needs_the_player(tmp_path, capsys): assert code == 2 and "--player" in err and "NNNOTES_PATHS_PLAYER" in err -def test_catalog_from_the_cache_needs_no_region(catalog_file, tmp_path, capsys): - cache = tmp_path / "cache" - code, _, err = run(["--cache", str(cache), "--language", "xx", "catalog", "keys"], capsys) +def test_catalog_from_the_downloads_needs_no_region(catalog_file, tmp_path, capsys): + depot = tmp_path / "depot" + downloads = depot / "downloads" + code, _, err = run(["--depot", str(depot), "--language", "xx", "catalog", "keys"], capsys) assert code == 2 and "catalog.region" in err and "NNNOTES_CATALOG_REGION" in err - cache.mkdir() - (cache / "catalog_main_xx.bin").write_bytes(catalog_file.read_bytes()) - code, out, err = run(["--cache", str(cache), "--language", "xx", "catalog", "keys", "--prefix", "Char/"], capsys) + downloads.mkdir(parents=True) + (downloads / "catalog_main_xx.bin").write_bytes(catalog_file.read_bytes()) + code, out, err = run(["--depot", str(depot), "--language", "xx", "catalog", "keys", "--prefix", "Char/"], capsys) assert code == 0, err assert out.splitlines() == ["Char/A", "Char/B"] @@ -284,8 +285,8 @@ def test_apk_is_a_required_setting(argv, catalog_file, tmp_path, cdn, capsys, mo [ "--catalog", str(catalog_file), - "--cache", - str(tmp_path / "cache"), + "--depot", + str(tmp_path / "depot"), "--region", "zz", "--master", @@ -342,8 +343,8 @@ def data(catalog_file, tmp_path, cdn, monkeypatch): return [ "--catalog", str(catalog_file), - "--cache", - str(tmp_path / "cache"), + "--depot", + str(tmp_path / "depot"), "--region", "zz", "--language", diff --git a/tests/cli/test_voices.py b/tests/cli/test_voices.py index 28d9396..722f070 100644 --- a/tests/cli/test_voices.py +++ b/tests/cli/test_voices.py @@ -7,7 +7,7 @@ from nnnotes import cli from nnnotes.content import storyvoices from nnnotes.formats import cri -from nnnotes.pipeline.store import Store +from nnnotes.pipeline.depot import Depot from nnnotes.source.session import Session from tests.pipeline import test_voices as tv from tests.pipeline import test_voices_story as ts @@ -47,7 +47,7 @@ def decode(cat, sheet, out_dir, key=None, fmt="flac", **kw): def test_list_search_summary(tmp_path, capsysbinary): - out = tv.export_dir(tmp_path, Store(tmp_path / "s")) + out = tv.export_dir(tmp_path, Depot(tmp_path / "s")) command(["voices", "list", "--from", str(out), "--character", "Alpha", "--category", "Talk.CharacterRankUp"]) line = capsysbinary.readouterr().out.decode("utf-8") assert ( @@ -76,8 +76,8 @@ def test_without_an_export_the_content_is_built(tmp_path, capsysbinary, session) def test_get_copies_the_exported_files(tmp_path, capsysbinary): - store = Store(tmp_path / "s") - out = tv.export_dir(tmp_path, store) + depot = Depot(tmp_path / "s") + out = tv.export_dir(tmp_path, depot) dst = tmp_path / "got" command(["voices", "get", "MasterHomeSpot:10001", "-o", str(dst), "--from", str(out)]) assert sorted(p.name for p in dst.iterdir()) == ["IntroRandom.flac", "IntroRandom_2.flac", "voice.json"] @@ -114,7 +114,7 @@ def test_get_decodes_at_the_flac_level_given(tmp_path, capsysbinary, monkeypatch monkeypatch.setattr(cri, "acb_data", lambda cat, sheet: ({"acb": tv.sheet_acb(sheet)}, cri.RAW_ACB)) monkeypatch.setattr(cri, "decode", decode) monkeypatch.setattr(Session, "catalog", lambda self, bundles=True, region=None: tv.Catalog()) - out = tv.export_dir(tmp_path, Store(tmp_path / "s")) + out = tv.export_dir(tmp_path, Depot(tmp_path / "s")) command(["voices", "get", "101", "-o", str(tmp_path / "g"), "--from", str(out), "--flac-level", "5"]) assert decode.calls == [("VoiceSystem_01", "flac", 5)] for argv, message in ( @@ -127,7 +127,7 @@ def test_get_decodes_at_the_flac_level_given(tmp_path, capsysbinary, monkeypatch def test_story_rows_are_filtered(tmp_path, capsysbinary): - out = ts.export_dir(tmp_path, Store(tmp_path / "s")) + out = ts.export_dir(tmp_path, Depot(tmp_path / "s")) command( ["voices", "list", "--from", str(out), "--source", "story", "--episode", "900001", "--status", "ok"], tv.Cfg("en"), diff --git a/tests/content/test_ui.py b/tests/content/test_ui.py index d1380bc..f17ebab 100644 --- a/tests/content/test_ui.py +++ b/tests/content/test_ui.py @@ -176,7 +176,7 @@ def test_cli_selects_the_jp_split_apk_and_keeps_public_provenance(tmp_path, expo { "catalog": {"region": "jp"}, "servers": {"jp": {"apk": str(apk)}}, - "paths": {"apk": "wrong-global.apk", "cache": str(tmp_path / "cache")}, + "paths": {"apk": "wrong-global.apk", "depot": str(tmp_path / "depot")}, }, environ={}, ) diff --git a/tests/musicdata/test_deckdata.py b/tests/musicdata/test_deckdata.py index 558aba3..522168d 100644 --- a/tests/musicdata/test_deckdata.py +++ b/tests/musicdata/test_deckdata.py @@ -590,10 +590,10 @@ def test_decoded_master_checks(tmp_path): def test_resource_version_from_the_catalog_store(tmp_path): remote = synth.CatalogWriter().build([("a_01.bundle", synth.remote("a_01.bundle"), [])]) sha = hashlib.sha256(remote).hexdigest() - assert deckdata.resource_version(tmp_path / "store", sha) is None - CatalogDB(tmp_path / "store").add(remote, None, region="xx", language="en", resource_version="1.2.3") - assert deckdata.resource_version(tmp_path / "store", sha) == "1.2.3" - assert deckdata.resource_version(tmp_path / "store", "0" * 64) is None + assert deckdata.resource_version(tmp_path / "depot", sha) is None + CatalogDB(tmp_path / "depot").add(remote, None, region="xx", language="en", resource_version="1.2.3") + assert deckdata.resource_version(tmp_path / "depot", sha) == "1.2.3" + assert deckdata.resource_version(tmp_path / "depot", "0" * 64) is None assert deckdata.resource_version(None, sha) is None diff --git a/tests/musicdata/test_musicdata.py b/tests/musicdata/test_musicdata.py index f7d4d3f..8cd8c4d 100644 --- a/tests/musicdata/test_musicdata.py +++ b/tests/musicdata/test_musicdata.py @@ -1260,7 +1260,7 @@ def test_command(tmp_path, capsys, monkeypatch): monkeypatch.setenv("NNNOTES_MASTER_IV", synth.MASTER_IV.hex()) monkeypatch.setattr(Session, "catalog", lambda self, **kw: FakeCatalog(CHARTS)) monkeypatch.setattr(score, "fetch_chart", lambda cat, name: cat.charts[name]) - base = ["--region", "xx", "--cache", str(tmp_path / "cache"), "music-data", "--master-files", str(d), "--no-bgm"] + base = ["--region", "xx", "--depot", str(tmp_path / "depot"), "music-data", "--master-files", str(d), "--no-bgm"] code, stdout, err = run(base + ["--no-deck", "-o", str(out)], capsys) assert code == 0, err r = json.loads(stdout) @@ -1297,7 +1297,7 @@ def test_command_decoded_master(tmp_path, capsys, monkeypatch): fake = FakeDeck() real = musicdata.Deck monkeypatch.setattr(musicdata, "Deck", lambda **kw: real(module=fake, **kw)) - common = ["--region", "xx", "--cache", str(tmp_path / "cache")] + common = ["--region", "xx", "--depot", str(tmp_path / "depot")] options = ["--no-bgm", "--full", "-o"] code, stdout, err = run( common + ["--master", str(dec), "music-data", "--decoded-master"] + options + [str(out)], capsys diff --git a/tests/pipeline/test_catalogdb.py b/tests/pipeline/test_catalogdb.py index 1b40ba2..8a2c34b 100644 --- a/tests/pipeline/test_catalogdb.py +++ b/tests/pipeline/test_catalogdb.py @@ -15,8 +15,8 @@ stable_raw_name, version_id, ) +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.stages import Env, describe, execute, registry -from nnnotes.pipeline.store import Store from nnnotes.source.addressables import parse, parse_header, parse_keys, parse_locations from nnnotes.source.catalog import APK_OFFSET_BASE, Catalog, file_name, location_kind from tests.helpers import synth @@ -158,18 +158,18 @@ def test_location_kind_and_file_name(): def test_catalog_bundles_raw_files_locations(tmp_path): remote = build(remote_entries()) - cat = Catalog(remote, tmp_path / "cache") + cat = Catalog(remote, tmp_path / "downloads") assert [b.name for b in cat.bundles()] == [f"chara_a_{H1}.bundle", f"chara_b_{H2}.bundle", "shared.bundle"] assert [b.remote for b in cat.bundles()] == [True, True, False] assert [file_name(e["internal_id"]) for e in cat.raw_files()] == [RAW] - assert cat.cached_raw(cat.raw_files()[0]) is None + assert cat.downloaded_raw(cat.raw_files()[0]) is None assert cat.sources() == {"remote": remote} locs = cat.locations() assert [e["offset"] for e in locs] == [e["offset"] for e in cat.entries] assert {e["catalog"] for e in locs} == {"remote"} assert [e["kind"] for e in locs] == ["asset", "asset", "bundle", "bundle", "asset", "bundle", "raw", "asset"] - with_apk = Catalog(remote, tmp_path / "cache", apk=make_apk(tmp_path / "base.apk")) + with_apk = Catalog(remote, tmp_path / "downloads", apk=make_apk(tmp_path / "base.apk")) shared = [b for b in with_apk.bundles() if b.name == "shared.bundle"] assert len(shared) == 1 and shared[0].offset >= APK_OFFSET_BASE # the APK catalog's location wins assert [file_name(e["internal_id"]) for e in with_apk.raw_files()] == ["cri_emb/voice_" + H4, RAW] @@ -292,9 +292,9 @@ def test_diff_with_addresses(): assert "~ object chara_a/Assets/Game/Char/A.png (object, byteSize)" in format_diff(d) -# ---------------------------------------------------------------- versions in the store +# ---------------------------------------------------------------- versions in the depot def test_catalog_db(tmp_path): - db = CatalogDB(tmp_path / "store") + db = CatalogDB(tmp_path / "depot") assert db.versions() == [] remote, apk = build(remote_entries()), build(APK) v = db.add(remote, apk, region="r1", language="xx", hash_text="00ff" * 8) @@ -320,7 +320,7 @@ def test_catalog_db(tmp_path): assert db.index(v) == index(remote, apk) ins = db.inputs(v) assert [i.role for i in ins] == ["remote", "apk"] and ins[0].locators[0]["path"] == str(db.path(rsha)) - assert (tmp_path / "store" / "catalogs" / "index.json").read_bytes() == contract.encode( + assert (tmp_path / "depot" / "catalogs" / "index.json").read_bytes() == contract.encode( {"schema": "nnnotes.catalogs/1", "versions": db.versions()} ) assert default_label("ab" * 32) == "ab" * 6 and default_label("ab" * 32, "9.9") == "9.9" @@ -338,22 +338,22 @@ def test_fetch_with_and_without_hash(tmp_path): # ---------------------------------------------------------------- stage def test_index_stage(tmp_path): - db = CatalogDB(tmp_path / "store") + db = CatalogDB(tmp_path / "depot") remote, apk = build(remote_entries()), build(APK) v = db.add(remote, apk) - store = Store(tmp_path / "store") + depot = Depot(tmp_path / "depot") stage = IndexStage() - env = Env(store, {"catalogs": {"main": db.inputs(v)}}) + env = Env(depot, {"catalogs": {"main": db.inputs(v)}}) assert stage.subjects(env) == ["main"] task = describe(stage, "main", None, env) assert task.id == "catalog.index:main" - ex = execute(task, store, registry(stage)) + ex = execute(task, depot, registry(stage)) assert ex.status == "ran" and ex.result_status == "ok" - (rec,) = store.result(task.key)["artifacts"] + (rec,) = depot.result(task.key)["artifacts"] assert rec["id"] == "catalog.index:main#index" and rec["semantics"]["facts"]["bundles"] == 3 - assert contract.loads(store.read(rec["content"]["sha256"])) == index(remote, apk) - assert execute(task, store, registry(stage)).status == "hit" - only_remote = Env(store, {"catalogs": {"main": db.inputs(v)[:1]}}) + assert contract.loads(depot.read(rec["content"]["sha256"])) == index(remote, apk) + assert execute(task, depot, registry(stage)).status == "hit" + only_remote = Env(depot, {"catalogs": {"main": db.inputs(v)[:1]}}) assert describe(stage, "main", None, only_remote).key != task.key with pytest.raises(ValueError, match="expected remote"): - describe(stage, "main", None, Env(store, {"catalogs": {"main": db.inputs(v)[1:]}})) + describe(stage, "main", None, Env(depot, {"catalogs": {"main": db.inputs(v)[1:]}})) diff --git a/tests/pipeline/test_census.py b/tests/pipeline/test_census.py index 40647a5..933a34b 100644 --- a/tests/pipeline/test_census.py +++ b/tests/pipeline/test_census.py @@ -5,8 +5,8 @@ from nnnotes.pipeline import census, contract from nnnotes.pipeline.contract import Input +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.stages import Env, Pending, describe, execute, registry -from nnnotes.pipeline.store import Store from nnnotes.unity import reader as unity SCRIPTS = "CAB-5c1e0000000000000000000000000000" @@ -300,26 +300,26 @@ def load(path): return scene_bundle()[0] monkeypatch.setattr(unity, "load", load) - store = Store(tmp_path / "store") - sha, size = store.identify(bundle, "bundles", bundle.name) + depot = Depot(tmp_path / "depot") + sha, size = depot.identify(bundle, "bundles", bundle.name) inp = Input("bundle", sha, size, bundle.name, ({"kind": "file", "path": str(bundle)},)) stage = census.CensusStage() - env = Env(store, {"bundles": {"x": inp}}) + env = Env(depot, {"bundles": {"x": inp}}) assert stage.subjects(env) == ["x"] task = describe(stage, "x", None, env) assert task.id == "unity.census:x" and task.atoms == census.CensusStage.ATOMS assert task.atoms["reader"].startswith("unitypy-") assert task.cost.peak_bytes > size - assert execute(task, store, registry(stage)).status == "ran" - doc = store.result(task.key) + assert execute(task, depot, registry(stage)).status == "ran" + doc = depot.result(task.key) (rec,) = doc["artifacts"] assert rec["id"] == "unity.census:x#census" and rec["semantics"]["facts"]["objects"] == 7 assert rec["provenance"]["inputs"] == [{"role": "bundle", "sha256": sha}] - assert contract.loads(store.read(rec["content"]["sha256"])) == census.census_env(scene_bundle()[0]) + assert contract.loads(depot.read(rec["content"]["sha256"])) == census.census_env(scene_bundle()[0]) assert loaded == [str(bundle)] - assert execute(task, store, registry(stage)).status == "hit" and len(loaded) == 1 + assert execute(task, depot, registry(stage)).status == "hit" and len(loaded) == 1 with pytest.raises(ValueError, match="expected 'bundle'"): - describe(stage, "x", None, Env(store, {"bundles": {"x": Input("other", sha, size)}})) + describe(stage, "x", None, Env(depot, {"bundles": {"x": Input("other", sha, size)}})) for waiting in (None, Pending("fetch:x_1.bundle")): with pytest.raises(Pending, match="fetch:x"): - describe(stage, "x", None, Env(store, {"bundles": {"x": waiting}})) + describe(stage, "x", None, Env(depot, {"bundles": {"x": waiting}})) diff --git a/tests/pipeline/test_contract.py b/tests/pipeline/test_contract.py index aaaa396..a0f1026 100644 --- a/tests/pipeline/test_contract.py +++ b/tests/pipeline/test_contract.py @@ -11,8 +11,8 @@ from nnnotes.pipeline import contract, layout from nnnotes.pipeline import plan as plan_mod from nnnotes.pipeline.contract import Cost, IncompatibleTask, Input, Task +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.orchestrate import FAILURE_CODES, Orchestrator -from nnnotes.pipeline.store import Store from tests.pipeline.test_stages import PIPELINE, toy_files, toy_stages ROOT = Path(__file__).resolve().parents[2] @@ -94,7 +94,7 @@ def a_task(**kw) -> Task: subject="b", params={"png": {"level": 6}}, atoms={"png.encode": "pillow/1"}, - inputs=(Input("bundle", SHA, 10, "b_x.bundle", ({"kind": "cache", "path": "bundles/b_x.bundle"},)),), + inputs=(Input("bundle", SHA, 10, "b_x.bundle", ({"kind": "download", "path": "bundles/b_x.bundle"},)),), context={"scripts": {"CAB-1:1": "A|B|C"}}, cost=Cost(1.5, 100), ) @@ -248,13 +248,13 @@ def test_every_document_has_a_valid_schema(): def test_the_documents_of_a_run_validate(tmp_path): v = validators() - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") facts = {"files": toy_files(tmp_path / "in", {"a": "x\n\nFAIL\n?q", "b": "CRASH", "c": "y"})} - p = plan_mod.plan(toy_stages(), PIPELINE, store, facts=facts, layouts={"cas": {"write": 1, "remove": 0, "keep": 0}}) - run = Orchestrator(store, toy_stages(), log=lambda m: None).run(PIPELINE, facts=facts, selection=["all"]) + p = plan_mod.plan(toy_stages(), PIPELINE, depot, facts=facts, layouts={"cas": {"write": 1, "remove": 0, "keep": 0}}) + run = Orchestrator(depot, toy_stages(), log=lambda m: None).run(PIPELINE, facts=facts, selection=["all"]) sources = [layout.Source(tid, doc) for tid, doc in run.results()] entries, _ = layout.entries("original", sources) - w = layout.materialize(tmp_path / "out", "original", {}, entries, layout.from_store(store)) + w = layout.materialize(tmp_path / "out", "original", {}, entries, layout.from_depot(depot)) manifest = run.write({"original": w["sha256"]}) docs = [ (contract.PLAN, p.doc), @@ -268,7 +268,7 @@ def test_the_documents_of_a_run_validate(tmp_path): docs.append((contract.RESULT, doc)) docs += [(contract.ARTIFACT, a) for a in doc["artifacts"]] keys = [t["key"] for t in manifest["tasks"] if t["status"] == "ran"] - docs += [(contract.COST, store.cost(k)) for k in keys] + docs += [(contract.COST, depot.cost(k)) for k in keys] for name, doc in docs: errors = [e.message for e in v[name].iter_errors(contract.loads(contract.dumps(doc)))] assert errors == [], (name, errors) diff --git a/tests/pipeline/test_cristages.py b/tests/pipeline/test_cristages.py index 89d9002..6881be6 100644 --- a/tests/pipeline/test_cristages.py +++ b/tests/pipeline/test_cristages.py @@ -18,8 +18,8 @@ from nnnotes.pipeline import contract, cristages from nnnotes.pipeline.contract import IncompatibleTask, Input, Task from nnnotes.pipeline.cristages import AUDIO, AudioStage, MovieStage +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.stages import Env, Pending, describe, execute, registry -from nnnotes.pipeline.store import Store from tests.formats.test_usm import ADX, HCA, KEY, MPEG1, VP9, audio_header, build, chunk, rand, sbt, video_header from tests.unity.test_speed_exact import FakeTools @@ -79,11 +79,11 @@ def tools(): yield fake -def file_input(store, tmp: Path, stable: str, data: bytes, role="raw") -> Input: +def file_input(depot, tmp: Path, stable: str, data: bytes, role="raw") -> Input: p = tmp / "files" / stable.replace("/", "_") p.parent.mkdir(parents=True, exist_ok=True) p.write_bytes(data) - sha, size = store.identify(p, "raw", stable) + sha, size = depot.identify(p, "raw", stable) return Input(role, sha, size, f"{stable}_{'0' * 32}", ({"kind": "file", "path": str(p)},)) @@ -106,26 +106,26 @@ def catalog_index(keys: dict, bundles=()) -> dict: } -def planning(store, tmp, files: dict, keys=None, boot=True, bundles=()) -> Env: - raw = {s: file_input(store, tmp, s, d) for s, d in files.items()} +def planning(depot, tmp, files: dict, keys=None, boot=True, bundles=()) -> Env: + raw = {s: file_input(depot, tmp, s, d) for s, d in files.items()} b = tmp / "data.unity3d" b.write_bytes(b"UnityFS boot data") facts = { "raw": raw, "index": catalog_index(keys or {}, bundles), - "boot": cristages.boot_input(store, b) if boot else None, + "boot": cristages.boot_input(depot, b) if boot else None, } - return Env(store, facts) + return Env(depot, facts) -def export_result(store, env, subject: str, acbs: dict, usms=None) -> str: +def export_result(depot, env, subject: str, acbs: dict, usms=None) -> str: """A committed unity.export result holding `cri.acb` artifacts {cue sheet: ACB bytes} and `cri.usm` artifacts {movie asset name: USM bytes}.""" - t = Task(EXPORT, 1, subject, {}, {}, (Input("bundle", store.put(subject.encode()), len(subject)),)) + t = Task(EXPORT, 1, subject, {}, {}, (Input("bundle", depot.put(subject.encode()), len(subject)),)) recs = [ contract.artifact( f"CAB-{subject}:{i + 1}#acb", - store.add(data, "acb"), + depot.add(data, "acb"), contract.provenance(t), {"kind": "cri.acb", "format": "acb", "facts": {"cueSheet": sheet, "layout": "embedded"}}, ) @@ -134,13 +134,13 @@ def export_result(store, env, subject: str, acbs: dict, usms=None) -> str: recs += [ contract.artifact( f"CAB-{subject}:{100 + i}#usm", - store.add(data, "usm"), + depot.add(data, "usm"), contract.provenance(t), {"kind": "cri.usm", "format": "usm", "facts": {"movie": name}}, ) for i, (name, data) in enumerate(sorted((usms or {}).items())) ] - store.commit(contract.result(t, recs, [])) + depot.commit(contract.result(t, recs, [])) env.done(t.id, t.key) return t.id @@ -150,52 +150,52 @@ def export_result(store, env, subject: str, acbs: dict, usms=None) -> str: def run(stage, env, subject, params=None): t = describe(stage, subject, params, env) - execute(t, env.store, registry(stage), force=True) - return t, env.store.result(t.key) + execute(t, env.depot, registry(stage), force=True) + return t, env.depot.result(t.key) def art(doc, aid): return next(a for a in doc["artifacts"] if a["id"] == aid) -def load(store, doc, aid): - return store.read(art(doc, aid)["content"]["sha256"]) +def load(depot, doc, aid): + return depot.read(art(doc, aid)["content"]["sha256"]) # ---------------------------------------------------------------- the key's source def test_boot_input_from_an_apk_is_stored_once(tmp_path): - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") apk = tmp_path / "base.apk" with zipfile.ZipFile(apk, "w") as z: z.writestr(cristages.BOOT_IN_APK, b"UnityFS boot") z.writestr("classes.dex", b"dex") - a = cristages.boot_input(store, apk) + a = cristages.boot_input(depot, apk) assert (a.role, a.sha256, a.size, a.name, a.locators) == ( "boot", contract.sha256(b"UnityFS boot"), 12, "data.unity3d", - ({"kind": "store"},), + ({"kind": "depot"},), ) - assert store.read(a.sha256) == b"UnityFS boot" + assert depot.read(a.sha256) == b"UnityFS boot" with mock.patch.object(zipfile, "ZipFile", side_effect=AssertionError("read again")): - assert cristages.boot_input(Store(tmp_path / "s"), apk) == a # remembered by the APK's content + assert cristages.boot_input(Depot(tmp_path / "s"), apk) == a # remembered by the APK's content d = tmp_path / "data.unity3d" d.write_bytes(b"UnityFS other") - b = cristages.boot_input(store, d) + b = cristages.boot_input(depot, d) assert b.sha256 == contract.sha256(b"UnityFS other") and b.locators[0]["kind"] == "file" - assert cristages.boot_input(store, None) is None + assert cristages.boot_input(depot, None) is None def test_hca_key_is_read_once_per_content(tmp_path): - store = Store(tmp_path / "s") - inp = Input("boot", store.put(b"boot"), 4, None, ({"kind": "store"},)) + depot = Depot(tmp_path / "s") + inp = Input("boot", depot.put(b"boot"), 4, None, ({"kind": "depot"},)) calls = [] with ( mock.patch.object(cristages, "_keys", {}), mock.patch.object(crikey, "find_key", lambda p: calls.append(Path(p).read_bytes()) or 77), ): - assert cristages.hca_key(store, inp) == cristages.hca_key(store, inp) == 77 + assert cristages.hca_key(depot, inp) == cristages.hca_key(depot, inp) == 77 assert calls == [b"boot"] @@ -217,15 +217,15 @@ def test_tool_ids_name_the_version_and_the_executable(tmp_path): # ---------------------------------------------------------------- cri.audio def test_audio_subjects_by_content(tmp_path, tools): - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") other = b"@UTF" + b"another sheet" env = planning( - store, + depot, tmp_path, {"cri_assets_cri/sound/bgm_x": ACB, "cri_assets_cri/video/m": usm_file()[0]}, {"Cri/Sound/bgm_x": ["cri_assets_cri/sound/bgm_x"], "Cri/Video/m": ["cri_assets_cri/video/m"]}, ) - tid = export_result(store, env, "cri_bundle", {"bgm_x": ACB, "se_live": other}) + tid = export_result(depot, env, "cri_bundle", {"bgm_x": ACB, "se_live": other}) stage = AudioStage() sha, other_sha = contract.sha256(ACB), contract.sha256(other) assert stage.subjects(env) == sorted([sha, other_sha]) # the raw ACB and its bundle copy: one @@ -236,20 +236,20 @@ def test_audio_subjects_by_content(tmp_path, tools): assert t.input("acb").locators[0]["kind"] == "file" and t.atoms == FAKE_ATOMS assert t.params == {"flac": {"level": 8}} b = describe(stage, other_sha, None, env) - assert b.input("acb").locators == ({"kind": "store"},) - waiting = Env(store, env.facts) + assert b.input("acb").locators == ({"kind": "depot"},) + waiting = Env(depot, env.facts) waiting.waiting(tid) with pytest.raises(Pending, match=tid): stage.subjects(waiting) with pytest.raises(ValueError, match="set \\[paths\\] apk"): - describe(stage, sha, None, planning(store, tmp_path, {"cri_assets_cri/sound/bgm_x": ACB}, boot=False)) + describe(stage, sha, None, planning(depot, tmp_path, {"cri_assets_cri/sound/bgm_x": ACB}, boot=False)) def test_audio_waits_for_raw_files_not_fetched(tmp_path, tools): from nnnotes.cli.assets import Inputs - store = Store(tmp_path / "s") - env = planning(store, tmp_path, {"cri_assets_cri/sound/a": ACB}) + depot = Depot(tmp_path / "s") + env = planning(depot, tmp_path, {"cri_assets_cri/sound/a": ACB}) env.facts["raw"] = Inputs({**env.facts["raw"], "cri_assets_cri/sound/b": Pending("fetch:b_1")}) with pytest.raises(Pending, match="fetch:b_1"): AudioStage().subjects(env) @@ -260,8 +260,8 @@ def test_audio_waits_for_raw_files_not_fetched(tmp_path, tools): def test_audio_artifacts_are_the_files_of_cri_decode(tmp_path, tools, monkeypatch): - store = Store(tmp_path / "s") - env = planning(store, tmp_path, {"cri_assets_cri/sound/bgm_x": ACB}) + depot = Depot(tmp_path / "s") + env = planning(depot, tmp_path, {"cri_assets_cri/sound/bgm_x": ACB}) sha = contract.sha256(ACB) t, doc = run(AudioStage(), env, sha) monkeypatch.setattr(cri, "acb_data", lambda cat, sheet: ({"acb": ACB}, cri.RAW_ACB)) @@ -276,7 +276,7 @@ def test_audio_artifacts_are_the_files_of_cri_decode(tmp_path, tools, monkeypatc assert files == ["a.flac", "a_3.flac", "b.flac", "c.flac", "c_5.flac", "cues.json", "streams.json"] assert sorted(a["id"] for a in doc["artifacts"]) == [f"{t.id}#{f}" for f in files] for f in files: - assert load(store, doc, f"{t.id}#{f}") == (ref / f).read_bytes(), f + assert load(depot, doc, f"{t.id}#{f}") == (ref / f).read_bytes(), f s = art(doc, f"{t.id}#a_3.flac") assert s["content"]["mediaType"] == "audio/flac" and "object" not in s["provenance"] assert s["semantics"] == { @@ -300,7 +300,7 @@ def test_audio_artifacts_are_the_files_of_cri_decode(tmp_path, tools, monkeypatc def test_audio_two_runs_and_levels(tmp_path, tools): - a, b = Store(tmp_path / "a"), Store(tmp_path / "b") + a, b = Depot(tmp_path / "a"), Depot(tmp_path / "b") ea, eb = planning(a, tmp_path / "x", {"s/a": ACB}), planning(b, tmp_path / "y", {"s/a": ACB}) ta, da = run(AudioStage(), ea, contract.sha256(ACB)) tb, db = run(AudioStage(), eb, contract.sha256(ACB)) @@ -313,9 +313,9 @@ def test_audio_two_runs_and_levels(tmp_path, tools): def test_an_acb_with_an_external_awb_is_unsupported(tmp_path, tools): - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") env = planning( - store, + depot, tmp_path, {"cri_assets_cri/sound/v": ACB, "cri_assets_cri/sound/v_awb": AWB}, {"Cri/Sound/v": ["cri_assets_cri/sound/v", "cri_assets_cri/sound/v_awb"]}, @@ -340,22 +340,22 @@ def test_an_acb_with_an_external_awb_is_unsupported(tmp_path, tools): def test_a_task_of_other_tools_is_incompatible(tmp_path, tools): - store = Store(tmp_path / "s") - env = planning(store, tmp_path, {"s/a": ACB}) + depot = Depot(tmp_path / "s") + env = planning(depot, tmp_path, {"s/a": ACB}) t = describe(AudioStage(), contract.sha256(ACB), None, env) with mock.patch.object(cristages, "tool_id", lambda which, exe: f"{which}-other/1"): with pytest.raises(IncompatibleTask, match="flac.encode, hca.decode differ"): - execute(t, store, registry(AudioStage())) + execute(t, depot, registry(AudioStage())) # ---------------------------------------------------------------- cri.movie def test_movie_streams_as_stored_and_the_mkv(tmp_path, tools): from nnnotes.formats import usm - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") data, frames = usm_file() env = planning( - store, + depot, tmp_path, {"membercard/1/movie/anime_part1": data, "cri_assets_cri/sound/x": ACB}, {"MemberCard/1/movie/anime_part1": ["membercard/1/movie/anime_part1"]}, @@ -367,8 +367,8 @@ def test_movie_streams_as_stored_and_the_mkv(tmp_path, tools): t, doc = run(stage, env, sha) assert t.params == {"format": "mkv", "flac": {"level": 8}} and set(t.atoms) == MOVIE_ATOMS video, audio = usm.demux(data, KEY) - assert load(store, doc, f"{t.id}#video.ivf") == video.data == b"".join(frames) - assert load(store, doc, f"{t.id}#audio.adx") == audio.data + assert load(depot, doc, f"{t.id}#video.ivf") == video.data == b"".join(frames) + assert load(depot, doc, f"{t.id}#audio.adx") == audio.data assert art(doc, f"{t.id}#video.ivf")["semantics"] == { "kind": "video.stream", "format": "ivf", @@ -382,20 +382,20 @@ def test_movie_streams_as_stored_and_the_mkv(tmp_path, tools): {"kind": "audio", "channel": 0, "codec": "flac", "source": "adx"}, ] } - assert b"-c:a|flac|-compression_level|8" in load(store, doc, mkv["id"]) + assert b"-c:a|flac|-compression_level|8" in load(depot, doc, mkv["id"]) assert doc["items"][0]["status"] == "exported" and doc["items"][0]["class"] == "USM" with fake_tools(key=KEY): tw, dw = run(stage, env, sha, {"format": "webm"}) webm = art(dw, f"{tw.id}#movie.webm") assert tw.params == {"format": "webm"} and webm["content"]["mediaType"] == "video/webm" - assert b"libopus" in load(store, dw, webm["id"]) and b"-c:v|copy" in load(store, dw, webm["id"]) + assert b"libopus" in load(depot, dw, webm["id"]) and b"-c:v|copy" in load(depot, dw, webm["id"]) assert f"{tw.id}#movie.mkv" not in {a["id"] for a in dw["artifacts"]} def test_movie_without_audio(tmp_path, tools): - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") data, _ = usm_file(audio=False) - env = planning(store, tmp_path, {"v/m": data}) + env = planning(depot, tmp_path, {"v/m": data}) with fake_tools(key=KEY): t, doc = run(MovieStage(), env, contract.sha256(data)) assert sorted(a["id"] for a in doc["artifacts"]) == [f"{t.id}#movie.mkv", f"{t.id}#video.ivf"] @@ -420,9 +420,9 @@ def test_movie_every_stream_kind(tmp_path, tools): channels: every stream as stored, the subtitles as JSON / SubRip / WebVTT, every track in the Matroska file.""" from nnnotes.formats import usm - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") data = build(EVERY) - env = planning(store, tmp_path, {"v/m": data}) + env = planning(depot, tmp_path, {"v/m": data}) with fake_tools(key=KEY) as fake: t, doc = run(MovieStage(), env, contract.sha256(data)) roles = sorted(contract.parse_artifact_id(a["id"])[1] for a in doc["artifacts"]) @@ -444,21 +444,21 @@ def test_movie_every_stream_kind(tmp_path, tools): ) assert doc["items"][0]["status"] == "exported" streams = {s.name: s for s in usm.demux(data, KEY)} - assert load(store, doc, f"{t.id}#alpha.m1v") == streams["alpha"].data == b"".join(MPEG1) - assert load(store, doc, f"{t.id}#audio_2.hca") == b"".join(HCA) + assert load(depot, doc, f"{t.id}#alpha.m1v") == streams["alpha"].data == b"".join(MPEG1) + assert load(depot, doc, f"{t.id}#audio_2.hca") == b"".join(HCA) alpha = art(doc, f"{t.id}#alpha.m1v") assert alpha["content"]["mediaType"] == "video/mpeg" and alpha["semantics"]["kind"] == "video.alpha" assert alpha["semantics"]["facts"]["alphaType"] == 1 and alpha["semantics"]["facts"]["displayHeight"] == 180 assert art(doc, f"{t.id}#audio_2.hca")["semantics"]["facts"]["cipher"] == 56 - assert json.loads(load(store, doc, f"{t.id}#subtitle.json")) == { + assert json.loads(load(depot, doc, f"{t.id}#subtitle.json")) == { "channel": 0, "records": [ {"language": 0, "timeUnit": 1000, "start": 0, "duration": 1500, "text": "こんにちは", "terminator": 1}, {"language": 0, "timeUnit": 1000, "start": 2000, "duration": 1000, "text": "b", "terminator": 1}, ], } - assert load(store, doc, f"{t.id}#subtitle_1.srt") == b"1\n00:00:00,000 --> 00:00:01,500\nen\n\n" - assert load(store, doc, f"{t.id}#subtitle.vtt").startswith( + assert load(depot, doc, f"{t.id}#subtitle_1.srt") == b"1\n00:00:00,000 --> 00:00:01,500\nen\n\n" + assert load(depot, doc, f"{t.id}#subtitle.vtt").startswith( "WEBVTT\n\n00:00:00.000 --> 00:00:01.500\nこんにちは\n".encode() ) assert ["-i", "-o"] in fake.calls # vgmstream decoded the HCA stream @@ -472,7 +472,7 @@ def test_movie_every_stream_kind(tmp_path, tools): ("subtitle", 0, "srt"), ("subtitle", 1, "srt"), ] - args = load(store, doc, mkv["id"]).split(b"\x00")[0] + args = load(depot, doc, mkv["id"]).split(b"\x00")[0] assert ( b"|+genpts|-f|mpegvideo|-i|video.m1v|" in args and b"|-f|wav|-i|audio_2.wav|" in args @@ -485,9 +485,9 @@ def test_movie_every_stream_kind(tmp_path, tools): def test_movie_webm_forms(tmp_path, tools): """format webm: MPEG-1 video with its alpha re-encoded as VP9 with alpha; the first audio stream only; no subtitles.""" - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") data = build(EVERY) - env = planning(store, tmp_path, {"v/m": data}) + env = planning(depot, tmp_path, {"v/m": data}) with fake_tools(key=KEY): t, doc = run(MovieStage(), env, contract.sha256(data), {"format": "webm"}) webm = art(doc, f"{t.id}#movie.webm") @@ -496,7 +496,7 @@ def test_movie_webm_forms(tmp_path, tools): {"kind": "alpha", "channel": 0, "codec": "vp9", "source": "mpeg1"}, {"kind": "audio", "channel": 0, "codec": "opus", "source": "adx"}, ] - args = load(store, doc, webm["id"]) + args = load(depot, doc, webm["id"]) assert b"alphamerge,format=yuva420p" in args and b"|-c:v|libvpx-vp9|-b:v|0|-crf|20|" in args assert b"audio_1" not in args and b"srt" not in args @@ -504,7 +504,7 @@ def test_movie_webm_forms(tmp_path, tools): def test_movie_subtitles_that_cannot_be_read(tmp_path, tools): """Subtitle bytes that are not records: stored as they are (`.sbt`); a text that is not UTF-8: the JSON only. Either way the movie is `unsupported` with its other artifacts, and the Matroska file has no track for them.""" - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") data = build( [ (b"@SFV", 0, None, VP9, "video"), @@ -512,7 +512,7 @@ def test_movie_subtitles_that_cannot_be_read(tmp_path, tools): (b"@SBT", 1, None, [sbt(0, 1000, 0, 10, b"\x82\xa0")], None), ] ) - env = planning(store, tmp_path, {"v/m": data}) + env = planning(depot, tmp_path, {"v/m": data}) with fake_tools(key=KEY): t, doc = run(MovieStage(), env, contract.sha256(data)) (item,) = doc["items"] @@ -524,7 +524,7 @@ def test_movie_subtitles_that_cannot_be_read(tmp_path, tools): "video.ivf", ] assert sorted(item["artifacts"]) == sorted(a["id"] for a in doc["artifacts"]) - assert json.loads(load(store, doc, f"{t.id}#subtitle_1.json"))["records"][0]["textHex"] == "82a0" + assert json.loads(load(depot, doc, f"{t.id}#subtitle_1.json"))["records"][0]["textHex"] == "82a0" assert art(doc, f"{t.id}#movie.mkv")["semantics"]["facts"]["tracks"] == [ {"kind": "video", "channel": 0, "codec": "vp9"} ] @@ -539,9 +539,9 @@ def test_movie_subtitles_that_cannot_be_read(tmp_path, tools): ], ) def test_unsupported_movies(tmp_path, tools, streams, extra, code): - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") data, _ = usm_file(streams=streams, extra=extra) - env = planning(store, tmp_path, {"v/m": data}) + env = planning(depot, tmp_path, {"v/m": data}) with fake_tools(key=KEY): t, doc = run(MovieStage(), env, contract.sha256(data)) (item,) = doc["items"] @@ -551,11 +551,11 @@ def test_unsupported_movies(tmp_path, tools, streams, extra, code): def test_movie_held_in_a_bundle(tmp_path, tools): """A USM artifact of unity.export is a subject like a raw USM (one task for both copies), named by the keys of its bundle that end in the movie asset's name; it waits for unity.export.""" - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") held, _ = usm_file(audio=False) raw, _ = usm_file() env = planning( - store, + depot, tmp_path, {"cri_assets_cri/video/m": raw}, { @@ -565,7 +565,7 @@ def test_movie_held_in_a_bundle(tmp_path, tools): }, bundles=("embcri_video_common",), ) - tid = export_result(store, env, "embcri_video_common", {}, {"warmup": held, "loose": raw}) + tid = export_result(depot, env, "embcri_video_common", {}, {"warmup": held, "loose": raw}) stage = MovieStage() sha, raw_sha = contract.sha256(held), contract.sha256(raw) assert stage.subjects(env) == sorted([sha, raw_sha]) @@ -573,10 +573,10 @@ def test_movie_held_in_a_bundle(tmp_path, tools): assert stage.depends(sha, env) == [tid] and stage.depends(raw_sha, env) == [tid] with fake_tools(key=KEY): t = describe(stage, sha, None, env) - assert t.input("usm").locators == ({"kind": "store"},) and sorted(i.role for i in t.inputs) == ["boot", "usm"] + assert t.input("usm").locators == ({"kind": "depot"},) and sorted(i.role for i in t.inputs) == ["boot", "usm"] t, doc = run(stage, env, sha) - assert doc["items"][0]["status"] == "exported" and load(store, doc, f"{t.id}#video.ivf") == b"".join(VP9) - waiting = Env(store, env.facts) + assert doc["items"][0]["status"] == "exported" and load(depot, doc, f"{t.id}#video.ivf") == b"".join(VP9) + waiting = Env(depot, env.facts) waiting.waiting(tid) with pytest.raises(Pending, match=tid): stage.subjects(waiting) @@ -623,9 +623,9 @@ class GoldenAudio(AudioStage): def ATOMS(self) -> dict: return dict(FAKE_ATOMS) - def run(self, task, store): + def run(self, task, depot): with fake_tools(): - return super().run(task, store) + return super().run(task, depot) class GoldenMovie(MovieStage): @@ -639,15 +639,15 @@ def ATOMS(self) -> dict: "hca.decode": "vgmstream-fake/1", } - def run(self, task, store): + def run(self, task, depot): with fake_tools(key=KEY): - return super().run(task, store) + return super().run(task, depot) def _fixture(stage, role, data: bytes, params=None): - def build(store): - sha = store.put(data) - boot = store.put(b"UnityFS boot data") + def build(depot): + sha = depot.put(data) + boot = depot.put(b"UnityFS boot data") return Task( stage.name, stage.version, @@ -655,8 +655,8 @@ def build(store): stage.normalize(params), stage.ATOMS, ( - Input(role, sha, len(data), None, ({"kind": "store"},)), - Input("boot", boot, 17, None, ({"kind": "store"},)), + Input(role, sha, len(data), None, ({"kind": "depot"},)), + Input("boot", boot, 17, None, ({"kind": "depot"},)), ), ) @@ -705,6 +705,6 @@ def test_golden_providers_run_twice_alike(tmp_path): for provider in (audio_golden, movie_golden): p = provider() - a = golden(p["stage"], p["fixtures"], Store(tmp_path / f"{p['stage'].name}a"), p["libraries"]) - b = golden(p["stage"], p["fixtures"], Store(tmp_path / f"{p['stage'].name}b"), p["libraries"]) + a = golden(p["stage"], p["fixtures"], Depot(tmp_path / f"{p['stage'].name}a"), p["libraries"]) + b = golden(p["stage"], p["fixtures"], Depot(tmp_path / f"{p['stage'].name}b"), p["libraries"]) assert a == b and golden_problems(a, b, p["stage"]) == [] diff --git a/tests/pipeline/test_store.py b/tests/pipeline/test_depot.py similarity index 83% rename from tests/pipeline/test_store.py rename to tests/pipeline/test_depot.py index 56c596b..c81bb48 100644 --- a/tests/pipeline/test_store.py +++ b/tests/pipeline/test_depot.py @@ -1,4 +1,4 @@ -"""The content-addressed store: write-once bytes, results as the commit point, input identities, runs.""" +"""The content-addressed depot: write-once bytes, results as the commit point, input identities, runs.""" import hashlib import multiprocessing as mp @@ -8,24 +8,24 @@ import pytest from nnnotes.pipeline import contract -from nnnotes.pipeline import store as store_mod +from nnnotes.pipeline import depot as depot_mod from nnnotes.pipeline.contract import Input, Task -from nnnotes.pipeline.store import InputMissing, Store, StoreConflict +from nnnotes.pipeline.depot import Depot, DepotConflict, InputMissing -def a_result(s: Store, task: Task, payload: bytes) -> dict: +def a_result(s: Depot, task: Task, payload: bytes) -> dict: aid = contract.artifact_id(task.id, "data.bin") art = contract.artifact(aid, s.add(payload, "bin"), contract.provenance(task), {"kind": "test"}) return contract.result(task, [art], []) def test_put_is_write_once_and_content_addressed(tmp_path): - s = Store(tmp_path) + s = Depot(tmp_path) sha = s.put(b"hello") assert sha == hashlib.sha256(b"hello").hexdigest() assert s.path(sha) == tmp_path / "cas" / "sha256" / sha[:2] / sha and s.read(sha) == b"hello" assert s.put(b"hello") == sha and (s.written, s.reused) == (1, 1) - assert Store(tmp_path).put(b"hello") == sha # another writer finds it + assert Depot(tmp_path).put(b"hello") == sha # another writer finds it assert s.add(b"{}", "JSON") == { "sha256": contract.sha256(b"{}"), "size": 2, @@ -40,7 +40,7 @@ def test_put_is_write_once_and_content_addressed(tmp_path): def test_commit_needs_the_bytes_and_is_write_once(tmp_path): - s = Store(tmp_path) + s = Depot(tmp_path) task = Task("t.x", 1, "a") doc = a_result(s, task, b"payload") assert s.result(task.key) is None @@ -49,7 +49,7 @@ def test_commit_needs_the_bytes_and_is_write_once(tmp_path): assert s.result_path(task.key).read_bytes() == contract.encode(doc) assert s.commit(doc) is False # identical: kept other = dict(doc, items=[contract.item("F:1", "unsupported", why=contract.reason("x"))]) - with pytest.raises(StoreConflict): + with pytest.raises(DepotConflict): s.commit(other) t2 = Task("t.x", 1, "b") missing = contract.result( @@ -70,7 +70,7 @@ def test_commit_needs_the_bytes_and_is_write_once(tmp_path): def _crash_after_put(root: str, key_file: str) -> None: - s = Store(root) + s = Depot(root) task = Task("t.x", 1, "crash") Path(key_file).write_text(task.key) a_result(s, task, b"written before the crash") # bytes stored, result never committed @@ -82,7 +82,7 @@ def test_a_crash_before_the_result_leaves_no_hit(tmp_path): p.start() p.join(60) assert p.exitcode == 9 - s = Store(tmp_path / "s") + s = Depot(tmp_path / "s") key = (tmp_path / "k").read_text() assert s.result(key) is None and not s.has_result(key) assert s.has(contract.sha256(b"written before the crash")) @@ -90,7 +90,7 @@ def test_a_crash_before_the_result_leaves_no_hit(tmp_path): def _writer(root: str, n: int, barrier) -> None: - s = Store(root) + s = Depot(root) barrier.wait() for i in range(40): task = Task("t.x", 1, f"s{i % 10}") @@ -108,7 +108,7 @@ def test_concurrent_writers_across_processes(tmp_path): for p in procs: p.join(120) assert [p.exitcode for p in procs] == [0, 0, 0, 0] - s = Store(tmp_path) + s = Depot(tmp_path) for i in range(10): task = Task("t.x", 1, f"s{i}") doc = s.result(task.key) @@ -123,13 +123,13 @@ def test_concurrent_writers_across_processes(tmp_path): def test_identify_memoizes_by_path_size_and_mtime(tmp_path, monkeypatch): f = tmp_path / "in.bin" f.write_bytes(b"abc") - s = Store(tmp_path / "s") + s = Depot(tmp_path / "s") assert s.identify(f, "bundles", "in.bin") == (contract.sha256(b"abc"), 3) assert s.named("bundles", "in.bin") == (contract.sha256(b"abc"), 3) calls = [] - real = store_mod.hashlib.file_digest - monkeypatch.setattr(store_mod.hashlib, "file_digest", lambda *a: calls.append(1) or real(*a)) - fresh = Store(tmp_path / "s") # the memo is on disk + real = depot_mod.hashlib.file_digest + monkeypatch.setattr(depot_mod.hashlib, "file_digest", lambda *a: calls.append(1) or real(*a)) + fresh = Depot(tmp_path / "s") # the memo is on disk assert fresh.identify(f) == (contract.sha256(b"abc"), 3) and calls == [] assert fresh.named("bundles", "in.bin")[1] == 3 and fresh.named("bundles", "other") is None f.write_bytes(b"abcd") @@ -137,15 +137,15 @@ def test_identify_memoizes_by_path_size_and_mtime(tmp_path, monkeypatch): assert fresh.identify(f) == (contract.sha256(b"abcd"), 4) and calls == [1] with open(tmp_path / "s" / "inputs" / "files.jsonl", "a", encoding="utf-8") as g: g.write('{"torn": ') # a writer that died mid-line - assert Store(tmp_path / "s").identify(f) == (contract.sha256(b"abcd"), 4) + assert Depot(tmp_path / "s").identify(f) == (contract.sha256(b"abcd"), 4) def test_open_input_tries_locators_in_order(tmp_path): - s = Store(tmp_path / "s", cache=tmp_path / "cache") + s = Depot(tmp_path / "s") data = b"bundle bytes" sha = contract.sha256(data) - (tmp_path / "cache" / "bundles").mkdir(parents=True) - (tmp_path / "cache" / "bundles" / "b.bundle").write_bytes(data) + (tmp_path / "s" / "downloads" / "bundles").mkdir(parents=True) + (tmp_path / "s" / "downloads" / "bundles" / "b.bundle").write_bytes(data) (tmp_path / "wrong.bin").write_bytes(b"other bytes!") inp = Input( "bundle", @@ -153,12 +153,12 @@ def test_open_input_tries_locators_in_order(tmp_path): len(data), "b.bundle", ( - {"kind": "store"}, + {"kind": "depot"}, {"kind": "file", "path": str(tmp_path / "wrong.bin")}, - {"kind": "cache", "path": "bundles/b.bundle"}, + {"kind": "download", "path": "bundles/b.bundle"}, ), ) - assert s.open_input(inp) == tmp_path / "cache" / "bundles" / "b.bundle" + assert s.open_input(inp) == tmp_path / "s" / "downloads" / "bundles" / "b.bundle" s.put(data) assert s.open_input(inp) == s.path(sha) lonely = Input("bundle", contract.sha256(b"x"), 1, "x", ({"kind": "catalog", "catalog": "c", "internalId": "i"},)) @@ -167,12 +167,12 @@ def test_open_input_tries_locators_in_order(tmp_path): (tmp_path / "fetched").write_bytes(b"x") s.fetch = lambda inp, loc: tmp_path / "fetched" assert s.open_input(lonely) == tmp_path / "fetched" - with pytest.raises(InputMissing, match="store: absent"): + with pytest.raises(InputMissing, match="depot: absent"): s.open_input(Input("x", contract.sha256(b"y"), 1)) def test_runs_costs_and_the_latest_run_of_a_selection(tmp_path): - s = Store(tmp_path) + s = Depot(tmp_path) doc = contract.run_doc( context={"region": "r"}, selection=["group:a"], @@ -190,17 +190,17 @@ def test_runs_costs_and_the_latest_run_of_a_selection(tmp_path): assert s.catalogs == tmp_path / "catalogs" -def test_a_store_crosses_processes_as_its_settings(tmp_path): +def test_a_depot_crosses_processes_as_its_settings(tmp_path): import pickle - s = Store(tmp_path / "s", cache=tmp_path / "c") + s = Depot(tmp_path / "s") s.put(b"x") t = pickle.loads(pickle.dumps(s)) - assert (t.root, t.cache, t.written) == (s.root, s.cache, 0) + assert (t.root, t.downloads, t.written) == (s.root, s.downloads, 0) def test_write_doc_writes_the_canonical_bytes_as_it_encodes(tmp_path): - from nnnotes.pipeline.store import write_doc + from nnnotes.pipeline.depot import write_doc docs = [ {"b": [1, 2, {"é": "x\n"}], "a": {}, "c": [], "d": None, "e": 1.5, "f": "立ち絵" * 1000}, diff --git a/tests/pipeline/test_depot_gc.py b/tests/pipeline/test_depot_gc.py new file mode 100644 index 0000000..326cc6d --- /dev/null +++ b/tests/pipeline/test_depot_gc.py @@ -0,0 +1,74 @@ +"""The collection of the depot: what kept runs reach stays, the rest goes.""" + +import os + +from nnnotes.pipeline import contract +from nnnotes.pipeline.contract import Input, Task +from nnnotes.pipeline.depot import Depot + + +def commit(d: Depot, task: Task, payload: bytes) -> dict: + aid = contract.artifact_id(task.id, "data.bin") + art = contract.artifact(aid, d.add(payload, "bin"), contract.provenance(task), {"kind": "test"}) + doc = contract.result(task, [art], []) + d.commit(doc) + return doc + + +def write_run(d: Depot, selection, tasks, layouts=None, mtime=None) -> dict: + doc = contract.run_doc( + context={"region": "r"}, + selection=selection, + params={}, + stages={"t.x": 1}, + tasks=[{"id": t.id, "key": t.key, "status": "ran"} for t in tasks], + layouts=layouts or {}, + summary={}, + ) + p = d.write_run(doc) + if mtime is not None: + os.utime(p, ns=(mtime, mtime)) + return doc + + +def test_gc_keeps_what_the_kept_runs_reach(tmp_path): + d = Depot(tmp_path) + stored = d.put(b"a stored input") + new = Task("t.x", 1, "a", inputs=(Input("doc", stored, 14, None, ({"kind": "depot"},)),)) + old = Task("t.x", 1, "b") + new_art = commit(d, new, b"new")["artifacts"][0]["content"]["sha256"] + old_art = commit(d, old, b"old")["artifacts"][0]["content"]["sha256"] + orphan = d.put(b"no result names it") + first = write_run(d, ["group:a"], [old], mtime=1_000_000_000) + latest = write_run(d, ["group:a"], [new], mtime=2_000_000_000) + + counted = d.gc(keep=1, dry_run=True) + assert counted["runs"] == {"kept": 1, "removed": 1} and counted["contents"] == {"kept": 2, "removed": 2} + assert d.run(first["id"]) is not None and d.has(orphan) # a dry run removes nothing + + r = d.gc(keep=1) + assert r["runs"] == {"kept": 1, "removed": 1} and r["results"] == {"kept": 1, "removed": 1} + assert r["contents"] == {"kept": 2, "removed": 2} + assert d.run(first["id"]) is None and d.run(latest["id"]) is not None + assert d.result(new.key) is not None and d.result(old.key) is None + assert d.has(new_art) and d.has(stored) and not d.has(old_art) and not d.has(orphan) + + +def test_gc_keeps_every_run_by_default_and_the_latest_of_a_selection(tmp_path): + d = Depot(tmp_path) + a, b = Task("t.x", 1, "a"), Task("t.x", 1, "b") + commit(d, a, b"a") + commit(d, b, b"b") + write_run(d, ["group:a"], [a], mtime=2_000_000_000) + latest = write_run(d, ["group:a"], [b], mtime=1_000_000_000) # written last: the selection's latest run + assert d.gc()["results"] == {"kept": 2, "removed": 0} + r = d.gc(keep=1) + assert r["runs"] == {"kept": 2, "removed": 0} and d.latest_run(["group:a"]) == latest + + +def test_gc_keeps_the_files_of_a_layout_a_run_names(tmp_path): + d = Depot(tmp_path) + placed = d.put(b"a derived file") + manifest = d.add(contract.encode({"files": [{"path": "x.json", "sha256": placed, "size": 14}]}), "json") + write_run(d, ["group:a"], [], layouts={"original": manifest["sha256"]}) + assert d.gc()["contents"] == {"kept": 2, "removed": 0} and d.has(placed) diff --git a/tests/pipeline/test_goldens.py b/tests/pipeline/test_goldens.py index c2e0d1d..bc0dc7c 100644 --- a/tests/pipeline/test_goldens.py +++ b/tests/pipeline/test_goldens.py @@ -6,8 +6,8 @@ {"stage": "unity.export", "version": 1, "atoms": {...}, "libraries": {"Pillow": "12.3.0", ...}, "provider": ":", "fixtures": {"": {"key": "...", "result": "..."}}} -The provider (a function of a test module) returns {"stage": Stage, "fixtures": {name: callable(store) -> Task}, -"libraries": [distribution names]}; each fixture builds its synthetic inputs into the store. Every atom the stage +The provider (a function of a test module) returns {"stage": Stage, "fixtures": {name: callable(depot) -> Task}, +"libraries": [distribution names]}; each fixture builds its synthetic inputs into the depot. Every atom the stage declares must be used by a fixture. A record made with other library versions is skipped (another encoder version may write other bytes for the same content) unless GOLDENS_STRICT=1, as in the pinned CI job, where it fails. GOLDENS_UPDATE=1 rewrites the records from the current code (after a version bump). @@ -20,8 +20,8 @@ import pytest from nnnotes.pipeline import contract +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.stages import Output, golden, golden_problems, library_versions -from nnnotes.pipeline.store import Store from tests.pipeline.test_stages import Src GOLDEN_DIR = Path(__file__).resolve().parents[1] / "golden" @@ -32,9 +32,9 @@ def provided(spec: str) -> dict: return getattr(importlib.import_module(module), name)() -def record(spec: str, store: Store) -> dict: +def record(spec: str, depot: Depot) -> dict: p = provided(spec) - doc = golden(p["stage"], p["fixtures"], store, p.get("libraries", ())) + doc = golden(p["stage"], p["fixtures"], depot, p.get("libraries", ())) return dict(doc, provider=spec) @@ -43,31 +43,31 @@ def test_golden_record(path, tmp_path): expected = contract.loads(path.read_bytes()) p = provided(expected["provider"]) if os.environ.get("GOLDENS_UPDATE") == "1": - path.write_bytes(contract.encode(record(expected["provider"], Store(tmp_path / "store")))) + path.write_bytes(contract.encode(record(expected["provider"], Depot(tmp_path / "depot")))) return installed = library_versions(expected["libraries"]) if installed != expected["libraries"]: if os.environ.get("GOLDENS_STRICT") != "1": pytest.skip(f"recorded with {expected['libraries']}, installed {installed}") pytest.fail(f"the pinned job has {installed}, the record needs {expected['libraries']}") - actual = record(expected["provider"], Store(tmp_path / "store")) + actual = record(expected["provider"], Depot(tmp_path / "depot")) assert golden_problems(expected, actual, p["stage"]) == [] # ---------------------------------------------------------------- the machinery itself (toy stage) def text_fixture(text: str): - def build(store): + def build(depot): from nnnotes.pipeline.contract import Input, Task data = text.encode() - sha = store.put(data) + sha = depot.put(data) return Task( "toy.src", 1, text[:4], {"suffix": ""}, dict(Src.ATOMS), - (Input("file", sha, len(data), None, ({"kind": "store"},)),), + (Input("file", sha, len(data), None, ({"kind": "depot"},)),), ) return build @@ -82,16 +82,16 @@ def toy_provider(): def test_a_regenerated_record_matches(tmp_path): - a = record("tests.pipeline.test_goldens:toy_provider", Store(tmp_path / "a")) - b = record("tests.pipeline.test_goldens:toy_provider", Store(tmp_path / "b")) + a = record("tests.pipeline.test_goldens:toy_provider", Depot(tmp_path / "a")) + b = record("tests.pipeline.test_goldens:toy_provider", Depot(tmp_path / "b")) assert a == b and golden_problems(a, b, Src()) == [] assert a["libraries"] == library_versions(["pytest"]) and set(a["fixtures"]) == {"plain", "items"} assert a["atoms"] == Src.ATOMS and a["version"] == 1 class Changed(Src): - def run(self, task, store): - out = super().run(task, store) + def run(self, task, depot): + out = super().run(task, depot) return Output( out.artifacts, [dict(i, reason=contract.reason("new", "x")) if i["status"] == "unsupported" else i for i in out.items], @@ -99,21 +99,21 @@ def run(self, task, store): def test_an_output_change_without_a_version_bump_fails(tmp_path): - expected = record("tests.pipeline.test_goldens:toy_provider", Store(tmp_path / "a")) + expected = record("tests.pipeline.test_goldens:toy_provider", Depot(tmp_path / "a")) fixtures = toy_provider()["fixtures"] - actual = golden(Changed(), fixtures, Store(tmp_path / "b"), ["pytest"]) + actual = golden(Changed(), fixtures, Depot(tmp_path / "b"), ["pytest"]) assert golden_problems(expected, actual) == ["toy.src: fixture items: output changed without a version bump"] bumped = Changed() bumped.version = 2 - actual = golden(bumped, {n: _versioned(f, 2) for n, f in fixtures.items()}, Store(tmp_path / "c"), ["pytest"]) + actual = golden(bumped, {n: _versioned(f, 2) for n, f in fixtures.items()}, Depot(tmp_path / "c"), ["pytest"]) problems = golden_problems(expected, actual) assert problems[0] == "toy.src: version 2, golden record has 1: regenerate the golden record" assert "toy.src: fixture plain: task key changed" in problems def _versioned(build, version): - def b(store): - t = build(store) + def b(depot): + t = build(depot) return contract.Task(t.stage, version, t.subject, t.params, t.atoms, t.inputs, t.context) return b @@ -123,14 +123,14 @@ def test_every_declared_atom_needs_a_fixture(tmp_path): class More(Src): ATOMS = {**Src.ATOMS, "mesh.glb": "toy-glb/1"} - doc = record("tests.pipeline.test_goldens:toy_provider", Store(tmp_path / "a")) + doc = record("tests.pipeline.test_goldens:toy_provider", Depot(tmp_path / "a")) assert golden_problems(doc, doc, More()) == ["toy.src: no fixture uses mesh.glb"] missing = dict(doc, fixtures={"plain": doc["fixtures"]["plain"]}) assert golden_problems(missing, doc) == ["toy.src: fixture items has no golden entry"] def test_records_are_canonical_documents(tmp_path): - doc = record("tests.pipeline.test_goldens:toy_provider", Store(tmp_path / "a")) + doc = record("tests.pipeline.test_goldens:toy_provider", Depot(tmp_path / "a")) path = tmp_path / "toy.src.json" path.write_bytes(contract.encode(doc)) assert contract.loads(path.read_bytes()) == doc diff --git a/tests/pipeline/test_layout.py b/tests/pipeline/test_layout.py index 924d992..b4a8655 100644 --- a/tests/pipeline/test_layout.py +++ b/tests/pipeline/test_layout.py @@ -8,11 +8,11 @@ from nnnotes.pipeline import contract, layout from nnnotes.pipeline.contract import Task +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.layout import LayoutError, Source, escape_segment, unescape_path -from nnnotes.pipeline.store import Store -def bundle(store, subject, objects, loose=(), root=True): +def bundle(depot, subject, objects, loose=(), root=True): """A unity.export-like result: objects [(file, pathId, class, name, container, {role: (bytes, ext)})], loose artifacts of no object [(role, bytes, ext)].""" task = Task("unity.export", 1, subject) @@ -26,14 +26,14 @@ def bundle(store, subject, objects, loose=(), root=True): for role, (data, ext) in roles.items(): aid = contract.artifact_id(oid, role) arts.append( - contract.artifact(aid, store.add(data, ext), contract.provenance(task, obj=obj), {"kind": "test"}) + contract.artifact(aid, depot.add(data, ext), contract.provenance(task, obj=obj), {"kind": "test"}) ) ids.append(aid) items.append(contract.item(oid, "exported", artifacts=ids, cls=cls)) for role, data, ext in loose: arts.append( contract.artifact( - contract.artifact_id(task.id, role), store.add(data, ext), contract.provenance(task), {"kind": "test"} + contract.artifact_id(task.id, role), depot.add(data, ext), contract.provenance(task), {"kind": "test"} ) ) return Source(task.id, contract.result(task, arts, items), f"_bundles/{subject}" if root else None) @@ -60,7 +60,7 @@ def test_escaping_is_reversible_and_safe(): def test_objects_of_one_container(tmp_path): - s = Store(tmp_path / "s") + s = Depot(tmp_path / "s") src = bundle( s, "card", @@ -104,7 +104,7 @@ def test_objects_of_one_container(tmp_path): def test_the_primary_is_the_class_the_extension_names(tmp_path): - s = Store(tmp_path / "s") + s = Depot(tmp_path / "s") src = bundle( s, "b", @@ -118,7 +118,7 @@ def test_the_primary_is_the_class_the_extension_names(tmp_path): def test_objects_without_container_and_artifacts_of_no_object(tmp_path): - s = Store(tmp_path / "s") + s = Depot(tmp_path / "s") src = bundle( s, "b1", @@ -143,7 +143,7 @@ def test_objects_without_container_and_artifacts_of_no_object(tmp_path): def test_a_bundle_of_several_serialized_files_names_them(tmp_path): - s = Store(tmp_path / "s") + s = Depot(tmp_path / "s") src = bundle( s, "scene", @@ -161,7 +161,7 @@ def test_a_bundle_of_several_serialized_files_names_them(tmp_path): def test_a_container_in_two_bundles_is_told_apart_by_bundle(tmp_path): - s = Store(tmp_path / "s") + s = Depot(tmp_path / "s") a = bundle( s, "group_a", @@ -181,7 +181,7 @@ def test_a_container_in_two_bundles_is_told_apart_by_bundle(tmp_path): def test_the_catalog_case_and_case_collisions(tmp_path): - s = Store(tmp_path / "s") + s = Depot(tmp_path / "s") src = bundle( s, "b", @@ -219,7 +219,7 @@ def test_equal_content_at_one_path_shares_the_file(): def two_bundles(tmp_path): - s = Store(tmp_path / "s") + s = Depot(tmp_path / "s") a = bundle( s, "a", @@ -245,8 +245,8 @@ def test_cas_and_original_derive_from_each_other(tmp_path): f"assets/{a['content']['sha256']}.{a['content']['ext']}" for src in sources for a in src.result["artifacts"] } orig, _ = layout.entries("original", sources) - w_cas = layout.materialize(tmp_path / "cas", "cas", {}, cas, layout.from_store(s)) - w_orig = layout.materialize(tmp_path / "orig", "original", {}, orig, layout.from_store(s)) + w_cas = layout.materialize(tmp_path / "cas", "cas", {}, cas, layout.from_depot(s)) + w_orig = layout.materialize(tmp_path / "orig", "original", {}, orig, layout.from_depot(s)) assert w_cas["manifest"]["schema"] == contract.LAYOUT and w_orig["write"] == len(orig) # original from cas + the records; cas from the original manifest alone layout.materialize(tmp_path / "orig2", "original", {}, orig, layout.from_layout(tmp_path / "cas")) @@ -265,15 +265,15 @@ def test_materialize_is_incremental_and_touches_only_its_files(tmp_path): (out / "assets").mkdir(parents=True) (out / "notes.txt").write_text("mine") # a file of the user orig, _ = layout.entries("original", sources) - first = layout.materialize(out, "original", {}, orig, layout.from_store(s)) + first = layout.materialize(out, "original", {}, orig, layout.from_depot(s)) assert layout.diff(out, orig) == {"write": 0, "remove": 0, "keep": len(orig)} - again = layout.materialize(out, "original", {}, orig, layout.from_store(s)) + again = layout.materialize(out, "original", {}, orig, layout.from_depot(s)) assert (again["write"], again["remove"], again["keep"]) == (0, 0, len(orig)) assert again["sha256"] == first["sha256"] (out / "assets" / "t.png").unlink() # damaged output is written again changed, _ = layout.entries("original", sources[:1]) # bundle b dropped assert layout.diff(out, changed) == {"write": 1, "remove": 1, "keep": len(changed) - 1} - w = layout.materialize(out, "original", {}, changed, layout.from_store(s)) + w = layout.materialize(out, "original", {}, changed, layout.from_depot(s)) assert (w["write"], w["remove"]) == (1, 1) assert not (out / "assets" / "text").exists() and (out / "notes.txt").read_text() == "mine" assert (out / "assets" / "t.png").exists() and not (out / layout.JOURNAL).exists() @@ -286,7 +286,7 @@ def test_an_interrupted_run_is_reconciled(tmp_path): (out / "half").mkdir(parents=True) (out / "half" / "written.png").write_bytes(b"partial") # written by a run that was interrupted (out / layout.JOURNAL).write_bytes(contract.encode({"write": ["half/written.png"], "remove": []})) - w = layout.materialize(out, "original", {}, orig, layout.from_store(s)) + w = layout.materialize(out, "original", {}, orig, layout.from_depot(s)) assert w["remove"] == 1 and not (out / "half").exists() and not (out / layout.JOURNAL).exists() @@ -297,27 +297,27 @@ def test_a_file_that_is_not_the_layouts_is_never_overwritten(tmp_path): (out / "assets").mkdir(parents=True) (out / "assets" / "t.png").write_bytes(b"someone else's") with pytest.raises(LayoutError, match="not this layout's: assets/t.png"): - layout.materialize(out, "original", {}, orig, layout.from_store(s)) + layout.materialize(out, "original", {}, orig, layout.from_depot(s)) assert (out / "assets" / "t.png").read_bytes() == b"someone else's" (out / "assets" / "t.png").write_bytes(png("t")[0]) # the right content: adopted - w = layout.materialize(out, "original", {}, orig, layout.from_store(s)) + w = layout.materialize(out, "original", {}, orig, layout.from_depot(s)) assert w["write"] == len(orig) def test_hard_links(tmp_path): s, sources = two_bundles(tmp_path) cas, _ = layout.entries("cas", sources) - w = layout.materialize(tmp_path / "out", "cas", {}, cas, layout.from_store(s), link="hard") + w = layout.materialize(tmp_path / "out", "cas", {}, cas, layout.from_depot(s), link="hard") assert w["linked"] == w["write"] and w["copied"] == w["cloned"] == 0 and w["failed"] == [] e = cas[0] assert os.path.samefile(tmp_path / "out" / e["path"], s.path(e["sha256"])) - assert os.stat(tmp_path / "out" / e["path"]).st_mode & 0o222 == 0 # the store object itself: read-only + assert os.stat(tmp_path / "out" / e["path"]).st_mode & 0o222 == 0 # the depot object itself: read-only if not (hasattr(os, "geteuid") and os.geteuid() == 0): # (the superuser writes anyway) with pytest.raises(PermissionError): (tmp_path / "out" / e["path"]).write_bytes(b"edited") for bad in ("soft", "auto"): with pytest.raises(ValueError): - layout.materialize(tmp_path / "out", "cas", {}, cas, layout.from_store(s), link=bad) + layout.materialize(tmp_path / "out", "cas", {}, cas, layout.from_depot(s), link=bad) def _clone_by_copy(src, dst): @@ -335,20 +335,20 @@ def fail(*a, **k): def test_the_probe_prefers_clone_then_hard_link_then_copy(tmp_path, monkeypatch): import errno - out, store = tmp_path / "out", tmp_path / "s" / "cas" / "sha256" + out, depot = tmp_path / "out", tmp_path / "s" / "cas" / "sha256" monkeypatch.setattr(layout, "clone", _clone_by_copy) - assert layout.probe(out, store)[0] == "clone" and layout.probe(out, store, "clone")[0] == "clone" + assert layout.probe(out, depot)[0] == "clone" and layout.probe(out, depot, "clone")[0] == "clone" monkeypatch.setattr(layout, "clone", _no(errno.EOPNOTSUPP)) - how, why = layout.probe(out, store) + how, why = layout.probe(out, depot) assert how == "hard" and why.startswith("hard links: clone unsupported on this file system (") with pytest.raises(LayoutError, match="--link clone: clone unsupported"): - layout.probe(out, store, "clone") + layout.probe(out, depot, "clone") monkeypatch.setattr(layout.os, "link", _no(errno.EXDEV)) - assert layout.probe(out, store) == ("copy", "copy: store on another file system") - with pytest.raises(LayoutError, match="--link hard: store on another file system"): - layout.probe(out, store, "hard") - assert layout.probe(out, store, "copy") == ("copy", "copy: as asked") - assert list(out.iterdir()) == [] and list(store.iterdir()) == [] # nothing left behind + assert layout.probe(out, depot) == ("copy", "copy: depot on another file system") + with pytest.raises(LayoutError, match="--link hard: depot on another file system"): + layout.probe(out, depot, "hard") + assert layout.probe(out, depot, "copy") == ("copy", "copy: as asked") + assert list(out.iterdir()) == [] and list(depot.iterdir()) == [] # nothing left behind def test_every_placement_gives_the_same_files_and_manifest(tmp_path, monkeypatch): @@ -357,13 +357,13 @@ def test_every_placement_gives_the_same_files_and_manifest(tmp_path, monkeypatch monkeypatch.setattr(layout, "clone", _clone_by_copy) got = {} for how in layout.PLACEMENTS: - w = layout.materialize(tmp_path / how, "original", {}, orig, layout.from_store(s), link=how) + w = layout.materialize(tmp_path / how, "original", {}, orig, layout.from_depot(s), link=how) counts = {k: w[k] for k in ("cloned", "linked", "copied")} assert counts[{"clone": "cloned", "hard": "linked", "copy": "copied"}[how]] == w["write"] == len(orig) got[how] = (files(tmp_path / how), w["sha256"]) assert got["clone"] == got["hard"] == got["copy"] (tmp_path / "copy" / orig[0]["path"]).write_bytes(b"a copy is writable") - assert all(os.stat(s.path(e["sha256"])).st_mode & 0o222 == 0 for e in orig) # store objects: read-only + assert all(os.stat(s.path(e["sha256"])).st_mode & 0o222 == 0 for e in orig) # depot objects: read-only def test_a_file_the_placement_cannot_make_is_reported_and_placed_next_time(tmp_path, monkeypatch): @@ -374,29 +374,29 @@ def test_a_file_the_placement_cannot_make_is_reported_and_placed_next_time(tmp_p real = os.link victim = s.path(cas[0]["sha256"]) monkeypatch.setattr(layout.os, "link", lambda a, b: _no(errno.EMLINK)() if Path(a) == victim else real(a, b)) - w = layout.materialize(tmp_path / "out", "cas", {}, cas, layout.from_store(s), link="hard") + w = layout.materialize(tmp_path / "out", "cas", {}, cas, layout.from_depot(s), link="hard") assert [f["path"] for f in w["failed"]] == [cas[0]["path"]] and w["write"] == len(cas) - 1 assert cas[0]["path"] not in {e["path"] for e in layout.read_manifest(tmp_path / "out")["entries"]} monkeypatch.setattr(layout.os, "link", real) - w = layout.materialize(tmp_path / "out", "cas", {}, cas, layout.from_store(s), link="hard") + w = layout.materialize(tmp_path / "out", "cas", {}, cas, layout.from_depot(s), link="hard") assert w["failed"] == [] and w["write"] == 1 and len(layout.read_manifest(tmp_path / "out")["entries"]) == len(cas) def test_a_layout_replaces_and_removes_its_read_only_files(tmp_path): s, sources = two_bundles(tmp_path) cas, _ = layout.entries("cas", sources) - layout.materialize(tmp_path / "out", "cas", {}, cas, layout.from_store(s), link="hard") - layout.materialize(tmp_path / "out", "cas", {}, cas[1:], layout.from_store(s), link="copy") # remove one + layout.materialize(tmp_path / "out", "cas", {}, cas, layout.from_depot(s), link="hard") + layout.materialize(tmp_path / "out", "cas", {}, cas[1:], layout.from_depot(s), link="copy") # remove one assert not (tmp_path / "out" / cas[0]["path"]).exists() - assert os.stat(s.path(cas[0]["sha256"])).st_mode & 0o222 == 0 # its store object stays read-only + assert os.stat(s.path(cas[0]["sha256"])).st_mode & 0o222 == 0 # its depot object stays read-only other = [dict(cas[1], path=cas[2]["path"])] + [e for e in cas[1:] if e["path"] != cas[2]["path"]] - layout.materialize(tmp_path / "out", "cas", {}, other, layout.from_store(s), link="copy") # replace one + layout.materialize(tmp_path / "out", "cas", {}, other, layout.from_depot(s), link="copy") # replace one assert (tmp_path / "out" / cas[2]["path"]).read_bytes() == s.read(cas[1]["sha256"]) assert os.stat(s.path(cas[2]["sha256"])).st_mode & 0o222 == 0 def test_an_objects_artifacts_from_several_results_are_placed_together(tmp_path): - s = Store(tmp_path / "s") + s = Depot(tmp_path / "s") export = bundle( s, "atlas_b", @@ -421,16 +421,16 @@ def test_an_objects_artifacts_from_several_results_are_placed_together(tmp_path) def test_derivation_keeps_lower_case_extensions(tmp_path): - s = Store(tmp_path / "s") + s = Depot(tmp_path / "s") src = bundle(s, "b", [("CAB-1", 1, "Texture2D", "x", "Assets/X.PNG", {"image": png("x")})]) orig, _ = layout.entries("original", [src]) assert [e["path"] for e in orig] == ["Assets/X.PNG"] - layout.materialize(tmp_path / "o", "original", {}, orig, layout.from_store(s)) + layout.materialize(tmp_path / "o", "original", {}, orig, layout.from_depot(s)) assert layout.cas_from_manifest(layout.read_manifest(tmp_path / "o")) == layout.entries("cas", [src])[0] def test_objects_without_container_go_under_the_root_of_their_file(tmp_path): - s = Store(tmp_path / "s") + s = Depot(tmp_path / "s") export = bundle(s, "sprites_b", [("CAB-s", 3, "Sprite", "loose", None, {"meta": (b"{}", "json")})]) crop = bundle(s, "atlas_a:7", [("CAB-s", 3, "Sprite", "loose", None, {"image": png("c")})]) crop.root = "_tasks/sprite.crop/atlas_a:7" diff --git a/tests/pipeline/test_link.py b/tests/pipeline/test_link.py index f6d5f3f..9f24dc9 100644 --- a/tests/pipeline/test_link.py +++ b/tests/pipeline/test_link.py @@ -3,8 +3,8 @@ from nnnotes.pipeline import catalogdb, census, contract, link from nnnotes.pipeline.catalogdb import CatalogDB, IndexStage, index from nnnotes.pipeline.contract import Input +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.stages import Env, Pending, describe, execute, registry -from nnnotes.pipeline.store import Store from tests.helpers import synth HA, HB, HC = "a" * 32, "b" * 32, "c" * 32 @@ -241,32 +241,32 @@ def test_diff_addresses(): # ---------------------------------------------------------------- stages -def census_task(store, subject, doc): +def census_task(depot, subject, doc): """A census task with a stored result made of `doc` (no bundle read).""" data = contract.encode(doc) inp = Input("bundle", contract.sha256(subject.encode()), 1, subject + ".bundle") - task = describe(census.CensusStage(), subject, None, Env(store, {"bundles": {subject: inp}})) - content = store.add(data, "json") + task = describe(census.CensusStage(), subject, None, Env(depot, {"bundles": {subject: inp}})) + content = depot.add(data, "json") rec = contract.artifact( contract.artifact_id(task.id, "census"), content, contract.provenance(task), {"kind": "unity.census", "format": "json", "facts": census.facts(doc)}, ) - store.commit(contract.result(task, [rec], [])) + depot.commit(contract.result(task, [rec], [])) return task def test_link_stages(tmp_path): - store = Store(tmp_path / "store") - db = CatalogDB(tmp_path / "store") + depot = Depot(tmp_path / "depot") + db = CatalogDB(tmp_path / "depot") v = db.add(synth.CatalogWriter().build(catalog_entries())) - env = Env(store, {"catalogs": {"main": db.inputs(v)}}) + env = Env(depot, {"catalogs": {"main": db.inputs(v)}}) itask = describe(IndexStage(), "main", None, env) - execute(itask, store, registry(IndexStage())) + execute(itask, depot, registry(IndexStage())) env.done(itask.id, itask.key) docs = censuses() - tasks = {s: census_task(store, s, d) for s, d in docs.items()} + tasks = {s: census_task(depot, s, d) for s, d in docs.items()} env.waiting(tasks["card_a"].id) # not run yet for s, t in tasks.items(): if s != "card_a": @@ -282,27 +282,27 @@ def test_link_stages(tmp_path): env.done(tasks["card_a"].id, tasks["card_a"].key) st = describe(stages["link.scripts"], "all", None, env) assert st.id == "link.scripts:all" and [i.role for i in st.inputs] == ["census:scripts"] - execute(st, store, stages) - (rec,) = store.result(st.key)["artifacts"] + execute(st, depot, stages) + (rec,) = depot.result(st.key)["artifacts"] assert rec["id"] == "link.scripts:all#scripts" and rec["semantics"]["facts"] == {"scripts": 2} - assert contract.loads(store.read(rec["content"]["sha256"])) == link.scripts([docs["scripts"]]) + assert contract.loads(depot.read(rec["content"]["sha256"])) == link.scripts([docs["scripts"]]) at = describe(stages["link.addresses"], "main", None, env) assert [i.role for i in at.inputs] == ["census:card_a", "census:scene_x", "census:shared", "index"] - execute(at, store, stages) - (rec,) = store.result(at.key)["artifacts"] + execute(at, depot, stages) + (rec,) = depot.result(at.key)["artifacts"] assert rec["id"] == "link.addresses:main#addresses" and rec["semantics"]["facts"]["objects"] == 5 expected = link.addresses(index(*db.catalog_bytes(v)), {s: docs[s] for s in ("card_a", "scene_x", "shared")}) - assert contract.loads(store.read(rec["content"]["sha256"])) == expected - assert execute(at, store, stages).status == "hit" + assert contract.loads(depot.read(rec["content"]["sha256"])) == expected + assert execute(at, depot, stages).status == "hit" def test_artifacts_table_and_stage(tmp_path): - store = Store(tmp_path / "store") + depot = Depot(tmp_path / "depot") arts, items = [], [] for pid, status in ((1, "exported"), (2, "contained"), (3, "failed")): oid = contract.object_id(CAB_A, pid) if status == "exported": - c = store.add(b"{}\n", "json") + c = depot.add(b"{}\n", "json") aid = contract.artifact_id(oid, "json") arts.append({"id": aid, "content": c}) items.append(contract.item(oid, "exported", artifacts=[aid])) @@ -315,9 +315,9 @@ def test_artifacts_table_and_stage(tmp_path): assert table["contained"] == {f"{CAB_A}:2": f"{CAB_A}:1#json"} assert table["artifacts"][f"{CAB_A}:1#json"]["ext"] == "json" stage = link.ArtifactsStage() - env = Env(store, {}) + env = Env(depot, {}) export = contract.Task("unity.export", 1, "card_a") - store.commit( + depot.commit( contract.result( export, [contract.artifact(a["id"], a["content"], contract.provenance(export), {"kind": "x"}) for a in arts], @@ -331,10 +331,10 @@ def test_artifacts_table_and_stage(tmp_path): env.waiting("sprite.crop:atlas", failed=True) # failed tasks are skipped t = describe(stage, "all", None, env) assert t.context == {"results": {export.id: export.key}} and t.inputs == () - execute(t, store, registry(stage)) - (rec,) = store.result(t.key)["artifacts"] + execute(t, depot, registry(stage)) + (rec,) = depot.result(t.key)["artifacts"] assert rec["id"] == "link.artifacts:all#artifacts" - assert contract.loads(store.read(rec["content"]["sha256"]))["objects"] == table["objects"] + assert contract.loads(depot.read(rec["content"]["sha256"]))["objects"] == table["objects"] def test_the_artifact_table_is_written_entry_by_entry_with_the_same_bytes(): @@ -372,7 +372,7 @@ def res(n, oid, aid, *, within=None): def test_store_add_chunks_is_add_of_the_joined_bytes(tmp_path): - s = Store(tmp_path / "s") + s = Depot(tmp_path / "s") c = s.add_chunks(iter([b"ab", b"", "é".encode()]), ".JSON") assert c == contract.content("abé".encode(), "json") and s.read(c["sha256"]) == "abé".encode() assert s.add_chunks([b"a", "bé".encode()], "json") == c and (s.written, s.reused) == (1, 1) diff --git a/tests/pipeline/test_modelstages.py b/tests/pipeline/test_modelstages.py index 0df3759..49afcd8 100644 --- a/tests/pipeline/test_modelstages.py +++ b/tests/pipeline/test_modelstages.py @@ -12,9 +12,9 @@ from nnnotes.pipeline.catalogdb import index from nnnotes.pipeline.census import CensusStage from nnnotes.pipeline.contract import IncompatibleTask, Input, Task +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.modelstages import Closures, Live2DStage, SpineStage, TaskCatalog from nnnotes.pipeline.stages import Env, Pending, describe, execute, registry -from nnnotes.pipeline.store import Store from nnnotes.source.catalog import Catalog from tests.helpers import synth @@ -54,7 +54,7 @@ def make_apk(path, entries=APK): def test_closures_are_those_of_catalog_resolve(tmp_path): remote = build(remote_entries()) - cat = Catalog(remote, tmp_path / "cache", apk=make_apk(tmp_path / "base.apk")) + cat = Catalog(remote, tmp_path / "downloads", apk=make_apk(tmp_path / "base.apk")) doc = index(remote, build(APK)) c = Closures(doc) names = {b["stable"]: b["name"] for b in doc["bundles"]} @@ -84,23 +84,23 @@ def bundles_fact(doc, data, locators=True): out = {} for b in doc["bundles"]: body = data[b["stable"]] - locs = ({"kind": "cache", "path": f"bundles/{b['name']}"},) if locators else () + locs = ({"kind": "download", "path": f"bundles/{b['name']}"},) if locators else () out[b["stable"]] = Input("bundle", contract.sha256(body), len(body), b["name"], locs) return out -def live2d_env(store, entries=None, data=None, selected=None, locators=True): +def live2d_env(depot, entries=None, data=None, selected=None, locators=True): doc = index(build(entries or remote_entries()), build(APK)) data = data or contents() bundles = bundles_fact(doc, data, locators) - return Env(store, {"index": doc, "bundles": bundles, "selected": sorted(selected or bundles)}) + return Env(depot, {"index": doc, "bundles": bundles, "selected": sorted(selected or bundles)}) def test_live2d_subjects_inputs_and_context(tmp_path): stage = Live2DStage() - env = live2d_env(Store(tmp_path / "s")) + env = live2d_env(Depot(tmp_path / "s")) assert stage.subjects(env) == [M1, M2] # model keys with their bundle - assert stage.subjects(live2d_env(Store(tmp_path / "s"), selected=["model_m2", "shared"])) == [M2] + assert stage.subjects(live2d_env(Depot(tmp_path / "s"), selected=["model_m2", "shared"])) == [M2] t = describe(stage, M1, None, env) assert t.id == "live2d.model:" + M1 and t.params == {} and t.atoms == Live2DStage.ATOMS assert [(i.role, i.name) for i in t.inputs] == [ @@ -111,32 +111,32 @@ def test_live2d_subjects_inputs_and_context(tmp_path): ] assert t.inputs[2].sha256 == contract.sha256(contents()["model_m1"]) assert t.context == {"internalId": PREFAB} - assert t.inputs[0].locators == ({"kind": "cache", "path": "bundles/monoscripts.bundle"},) + assert t.inputs[0].locators == ({"kind": "download", "path": "bundles/monoscripts.bundle"},) assert set(Live2DStage.ATOMS) == {"live2d.extract_model"} assert all(lib.lower() in Live2DStage.ATOMS["live2d.extract_model"] for lib in modelstages.LIBRARIES) def test_live2d_keys_follow_the_closure_contents_and_order_only(tmp_path): - stage, store = Live2DStage(), Store(tmp_path / "s") - key = describe(stage, M1, None, live2d_env(store)).key + stage, depot = Live2DStage(), Depot(tmp_path / "s") + key = describe(stage, M1, None, live2d_env(depot)).key # other file names, no locators, another catalog with more keys: the same task renamed = [(e[0].replace(H[1], H[8]), e[1].replace(H[1], H[8]), *e[2:]) for e in remote_entries()] renamed.append(("Other/key", "Assets/Game/other.asset", [3])) - assert describe(stage, M1, None, live2d_env(store, renamed, locators=False)).key == key + assert describe(stage, M1, None, live2d_env(depot, renamed, locators=False)).key == key # a bundle of the closure changed, the closure's order changed, the key's location changed: another task - assert describe(stage, M1, None, live2d_env(store, data=contents(b"!"))).key != key - assert describe(stage, M1, None, live2d_env(store, remote_entries(order=(3, 2, 5)))).key != key - assert describe(stage, M1, None, live2d_env(store, remote_entries(m1_prefab=PREFAB + "x"))).key != key + assert describe(stage, M1, None, live2d_env(depot, data=contents(b"!"))).key != key + assert describe(stage, M1, None, live2d_env(depot, remote_entries(order=(3, 2, 5)))).key != key + assert describe(stage, M1, None, live2d_env(depot, remote_entries(m1_prefab=PREFAB + "x"))).key != key # a bundle outside the closure changed: the same task other = dict(contents(), model_m2=b"UnityFS m2 changed") - assert describe(stage, M1, None, live2d_env(store, data=other)).key == key + assert describe(stage, M1, None, live2d_env(depot, data=other)).key == key bumped = type("Bumped", (Live2DStage,), {"version": 2})() - assert describe(bumped, M1, None, live2d_env(store)).key != key + assert describe(bumped, M1, None, live2d_env(depot)).key != key def test_live2d_waits_for_unfetched_bundles_and_names_unreadable_ones(tmp_path): - stage, store = Live2DStage(), Store(tmp_path / "s") - env = live2d_env(store) + stage, depot = Live2DStage(), Depot(tmp_path / "s") + env = live2d_env(depot) env.facts["bundles"] = dict(env.facts["bundles"], tex=Pending("fetch:tex")) with pytest.raises(Pending, match="fetch:tex"): describe(stage, M1, None, env) @@ -148,17 +148,17 @@ def test_live2d_waits_for_unfetched_bundles_and_names_unreadable_ones(tmp_path): describe(stage, M1, None, env) -def stored_task(store, stage=None): - """The live2d.model task of M1 with every bundle in the store (store locators only).""" +def stored_task(depot, stage=None): + """The live2d.model task of M1 with every bundle in the depot (depot locators only).""" for body in contents().values(): - store.put(body) - return describe(stage or Live2DStage(), M1, None, live2d_env(store, locators=False)) + depot.put(body) + return describe(stage or Live2DStage(), M1, None, live2d_env(depot, locators=False)) def test_task_catalog_answers_from_the_task(tmp_path): - store = Store(tmp_path / "s") - t = stored_task(store) - cat = TaskCatalog.from_task(t, store) + depot = Depot(tmp_path / "s") + t = stored_task(depot) + cat = TaskCatalog.from_task(t, depot) assert cat.apk is not None and cat.has(M1) and not cat.has(M2) assert cat.keys("Character/") == [M1] and cat.keys("Other/") == [] assert cat.entry(M1)["internal_id"] == PREFAB and cat.entries_for(M2) == [] @@ -174,7 +174,7 @@ def test_task_catalog_answers_from_the_task(tmp_path): assert [ i.role for i in TaskCatalog( - "k", "Assets/k", modelstages._ordered(Task(Live2DStage.name, 1, "k", inputs=tuple(many))), store + "k", "Assets/k", modelstages._ordered(Task(Live2DStage.name, 1, "k", inputs=tuple(many))), depot ).bundles ] == ["bundle:1", "bundle:2", "bundle:10"] @@ -204,12 +204,12 @@ def test_live2d_run_stores_every_file_of_the_extractor(tmp_path, monkeypatch): calls = [] monkeypatch.setattr(live2d, "extract_model", fake_extract(calls)) - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") stage = Live2DStage() - t = stored_task(store, stage) - ex = execute(t, store, registry(stage)) + t = stored_task(depot, stage) + ex = execute(t, depot, registry(stage)) assert ex.status == "ran" and calls == [(M1, PREFAB)] - doc = store.result(t.key) + doc = depot.result(t.key) tid = "live2d.model:" + M1 got = {contract.parse_artifact_id(a["id"])[1]: a for a in doc["artifacts"]} assert sorted(got) == [ @@ -233,18 +233,18 @@ def test_live2d_run_stores_every_file_of_the_extractor(tmp_path, monkeypatch): "textures/texture_00-1a2b3c4d.png": ("live2d.texture", "png"), } order = ["monoscripts", "shared", "model_m1", "tex"] - assert store.read(got["m1.moc3"]["content"]["sha256"]) == b"MOC3" + b"".join(contents()[s] for s in order) + assert depot.read(got["m1.moc3"]["content"]["sha256"]) == b"MOC3" + b"".join(contents()[s] for s in order) assert got["m1.model3.json"]["semantics"]["facts"] == { "expressions": 1, "motions": 1, "name": "m1", "textures": ["textures/texture_00-1a2b3c4d.png"], } - assert execute(t, store, registry(stage)).status == "hit" - assert execute(t, store, registry(stage), force=True).status == "ran" # the same result again + assert execute(t, depot, registry(stage)).status == "hit" + assert execute(t, depot, registry(stage), force=True).status == "ran" # the same result again other = Task(t.stage, t.version, t.subject, t.params, {"live2d.extract_model": "unitypy-0/1"}, t.inputs, t.context) with pytest.raises(IncompatibleTask): - execute(other, store, registry(stage)) + execute(other, depot, registry(stage)) def test_the_model_key_and_libraries_are_those_of_the_extractors(): @@ -356,36 +356,36 @@ def spine_censuses(): } -def census_result(store, env, subject, doc, body): +def census_result(depot, env, subject, doc, body): inp = Input("bundle", contract.sha256(body), len(body), subject + ".bundle", ()) - t = describe(CensusStage(), subject, None, Env(store, {"bundles": {subject: inp}})) + t = describe(CensusStage(), subject, None, Env(depot, {"bundles": {subject: inp}})) rec = contract.artifact( contract.artifact_id(t.id, "census"), - store.add(contract.encode(doc), "json"), + depot.add(contract.encode(doc), "json"), contract.provenance(t), {"kind": "unity.census", "format": "json", "facts": census.facts(doc)}, ) - store.commit(contract.result(t, [rec], [])) + depot.commit(contract.result(t, [rec], [])) env.done(t.id, t.key) return inp -def spine_env(store, docs=None, data=None, selected=("other_d", "scene_e", "spine_a")): +def spine_env(depot, docs=None, data=None, selected=("other_d", "scene_e", "spine_a")): docs = docs or spine_censuses() data = data or {} - env = Env(store, {"selected": sorted(selected)}) + env = Env(depot, {"selected": sorted(selected)}) env.facts["bundles"] = { - s: census_result(store, env, s, d, data.get(s, b"UnityFS " + s.encode())) for s, d in sorted(docs.items()) + s: census_result(depot, env, s, d, data.get(s, b"UnityFS " + s.encode())) for s, d in sorted(docs.items()) } st = describe(link.ScriptsStage(), "all", None, env) - execute(st, store, registry(link.ScriptsStage())) + execute(st, depot, registry(link.ScriptsStage())) env.done(st.id, st.key) return env def test_spine_subjects_closures_and_context(tmp_path): - stage, store = SpineStage(), Store(tmp_path / "s") - env = spine_env(store) + stage, depot = SpineStage(), Depot(tmp_path / "s") + env = spine_env(depot) assert stage.subjects(env) == ["other_d:5", "scene_e.sharedAssets:7", "spine_a:10"] t = describe(stage, "spine_a:10", None, env) assert t.id == "spine.skeleton:spine_a:10" and t.atoms == SpineStage.ATOMS and t.params == {} @@ -409,16 +409,16 @@ def test_spine_subjects_closures_and_context(tmp_path): def test_spine_keys_follow_the_closure_contents_only(tmp_path): stage = SpineStage() - key = describe(stage, "spine_a:10", None, spine_env(Store(tmp_path / "a"))).key - env = spine_env(Store(tmp_path / "b"), data={"other_d": b"UnityFS other, changed"}) + key = describe(stage, "spine_a:10", None, spine_env(Depot(tmp_path / "a"))).key + env = spine_env(Depot(tmp_path / "b"), data={"other_d": b"UnityFS other, changed"}) assert describe(stage, "spine_a:10", None, env).key == key - env = spine_env(Store(tmp_path / "c"), data={"shader_c": b"UnityFS shader, changed"}) + env = spine_env(Depot(tmp_path / "c"), data={"shader_c": b"UnityFS shader, changed"}) assert describe(stage, "spine_a:10", None, env).key != key def test_spine_waits_for_censuses_and_the_script_table(tmp_path): - stage, store = SpineStage(), Store(tmp_path / "s") - env = spine_env(store) + stage, depot = SpineStage(), Depot(tmp_path / "s") + env = spine_env(depot) env.waiting("unity.census:tex_b") with pytest.raises(Pending, match="unity.census:tex_b"): stage.subjects(env) @@ -490,10 +490,10 @@ def fake_skeleton(pages=1): def test_spine_run_stores_the_files_of_the_skeleton(tmp_path, monkeypatch): from nnnotes.unity import reader as unity - stage, store = SpineStage(), Store(tmp_path / "s") - env = spine_env(store) + stage, depot = SpineStage(), Depot(tmp_path / "s") + env = spine_env(depot) for s in ("scripts", "shader_c", "spine_a", "tex_b"): - store.put(b"UnityFS " + s.encode()) + depot.put(b"UnityFS " + s.encode()) loaded = [] def load_closure(paths): @@ -503,16 +503,16 @@ def load_closure(paths): monkeypatch.setattr(unity, "load_closure", load_closure) sda = fake_skeleton() t = describe(stage, "spine_a:10", None, env) - assert execute(t, store, registry(stage)).status == "ran" + assert execute(t, depot, registry(stage)).status == "ran" assert loaded == [[b"UnityFS scripts", b"UnityFS shader_c", b"UnityFS spine_a", b"UnityFS tex_b"]] - doc = store.result(t.key) + doc = depot.result(t.key) got = {contract.parse_artifact_id(a["id"])[1]: a for a in doc["artifacts"]} assert {r: (a["semantics"]["kind"], a["content"]["ext"]) for r, a in got.items()} == { "chr.skel": ("spine.skeleton", "skel"), "chr.atlas": ("spine.atlas", "atlas"), "chr.png": ("spine.page", "png"), } - assert store.read(got["chr.skel"]["content"]["sha256"]) == b"\x01skel" + assert depot.read(got["chr.skel"]["content"]["sha256"]) == b"\x01skel" assert got["chr.skel"]["semantics"]["facts"]["atlases"] == ["chr.atlas"] (item,) = doc["items"] assert item == { @@ -523,7 +523,7 @@ def load_closure(paths): } sda = fake_skeleton(pages=2) # a writer error: a failed item, cached t2 = Task(t.stage, t.version, t.subject, {}, t.atoms, t.inputs[:3], t.context) # another key - assert execute(t2, store, registry(stage)).result_status == "partial" - doc2 = store.result(t2.key) + assert execute(t2, depot, registry(stage)).result_status == "partial" + doc2 = depot.result(t2.key) assert doc2["artifacts"] == [] and doc2["items"][0]["reason"]["code"] == "failed.export" assert "1 pages vs 2 materials" in doc2["items"][0]["reason"]["message"] diff --git a/tests/pipeline/test_objexport.py b/tests/pipeline/test_objexport.py index f4e9ad2..d0019dc 100644 --- a/tests/pipeline/test_objexport.py +++ b/tests/pipeline/test_objexport.py @@ -19,9 +19,9 @@ from nnnotes.pipeline.atoms import Impl, astc, estimate, gltf from nnnotes.pipeline.atoms.reader import UnityBundle from nnnotes.pipeline.contract import IncompatibleTask, Input, Task +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.objexport import CONVERTERS, CROP, EXPORT, ExportStage, SpriteCropStage, export_atoms from nnnotes.pipeline.stages import Env, describe, execute, registry -from nnnotes.pipeline.store import Store from tests.helpers.fakeunity import ( F32, NORMAL, @@ -573,9 +573,9 @@ def classes_of(name): return sorted({r.type.name for f in BUNDLES[name]().assets for r in f.objects.values()}) -def task_for(store, name, classes=None, params=None, scripts=None): +def task_for(depot, name, classes=None, params=None, scripts=None): data = f"fake:{name}".encode() - sha = store.put(data) + sha = depot.put(data) stage = ExportStage(FAKE_READER) p = stage.normalize(dict(params or {}, classes=classes)) return Task( @@ -584,15 +584,15 @@ def task_for(store, name, classes=None, params=None, scripts=None): name, p, export_atoms(classes_of(name), FAKE_READER.id), - (Input("bundle", sha, len(data), f"{name}.bundle", ({"kind": "store"},)),), + (Input("bundle", sha, len(data), f"{name}.bundle", ({"kind": "depot"},)),), {"scripts": dict(SCRIPTS if scripts is None else scripts)}, ) -def run(store, task): +def run(depot, task): stage = ExportStage(FAKE_READER) - execute(task, store, registry(stage), force=True) - return store.result(task.key) + execute(task, depot, registry(stage), force=True) + return depot.result(task.key) def by_object(doc): @@ -603,12 +603,12 @@ def art(doc, aid): return next(a for a in doc["artifacts"] if a["id"] == aid) -def load(store, doc, aid): - return store.read(art(doc, aid)["content"]["sha256"]) +def load(depot, doc, aid): + return depot.read(art(doc, aid)["content"]["sha256"]) -def obj(store, doc, aid): - return contract.loads(load(store, doc, aid)) +def obj(depot, doc, aid): + return contract.loads(load(depot, doc, aid)) def oid(pid, cab=CAB): @@ -617,14 +617,14 @@ def oid(pid, cab=CAB): @pytest.fixture def exported(tmp_path): - store = Store(tmp_path / "store") - task = task_for(store, "all") - return store, task, run(store, task) + depot = Depot(tmp_path / "depot") + task = task_for(depot, "all") + return depot, task, run(depot, task) # ---------------------------------------------------------------- items def test_every_object_has_exactly_one_item(exported): - store, task, doc = exported + depot, task, doc = exported objects = [contract.object_id(f.name, pid) for f in bundle_all().assets for pid in f.objects] items = by_object(doc) assert sorted(items) == sorted(objects) and len(doc["items"]) == len(objects) @@ -675,11 +675,11 @@ def test_every_object_has_exactly_one_item(exported): for a in doc["artifacts"]: owner, _role = contract.parse_artifact_id(a["id"]) assert ("object" in a["provenance"]) == (not owner.startswith("unity.export:")) - assert store.has(a["content"]["sha256"], a["content"]["size"]) + assert depot.has(a["content"]["sha256"], a["content"]["size"]) def test_provenance_facts_and_atoms(exported): - store, task, doc = exported + depot, task, doc = exported a = art(doc, f"{oid(10)}#image") assert a["provenance"]["object"] == { "file": CAB, @@ -706,8 +706,8 @@ def test_provenance_facts_and_atoms(exported): # ---------------------------------------------------------------- converters def test_textures(exported): - store, _task, doc = exported - png = load(store, doc, f"{oid(10)}#image") + depot, _task, doc = exported + png = load(depot, doc, f"{oid(10)}#image") img = Image.open(__import__("io").BytesIO(png)) assert img.size == (W, H) and img.mode == "RGBA" expected = Image.frombytes("RGBA", (W, H), TEX_PIXELS).transpose(Image.Transpose.FLIP_TOP_BOTTOM) @@ -725,15 +725,15 @@ def test_textures(exported): "exportedMips": [0], "mode": "RGBA", } - a = load(store, doc, f"{oid(15)}#image") + a = load(depot, doc, f"{oid(15)}#image") assert astc.parse_header(a) == {"block": (4, 4, 1), "size": (8, 8, 1)} and a[16:] == bytes(range(64)) assert art(doc, f"{oid(15)}#image")["content"]["ext"] == "astc" - empty = obj(store, doc, f"{oid(14)}#json") + empty = obj(depot, doc, f"{oid(14)}#json") assert empty["m_Width"] == 0 and empty["image data"] == {"$hex": ""} def test_sprites_same_bundle(exported): - store, _task, doc = exported + depot, _task, doc = exported from nnnotes.pipeline.atoms import png, sprite from nnnotes.pipeline.atoms.mesh import arrays from nnnotes.pipeline.atoms.reader import mesh_input_from_typetree @@ -754,11 +754,11 @@ def test_sprites_same_bundle(exported): 100.0, canvas={"rect": tt["m_Rect"], "offset": src["textureRectOffset"], "pivot": tt["m_Pivot"]}, ) - assert load(store, doc, f"{oid(pid)}#image") == png.encode(want) - meta = obj(store, doc, f"{oid(pid)}#meta") + assert load(depot, doc, f"{oid(pid)}#image") == png.encode(want) + meta = obj(depot, doc, f"{oid(pid)}#meta") assert meta["image"]["texture"] == {"$ref": {"file": CAB, "pathId": 10, "class": "Texture2D", "name": "tex"}} assert meta["sprite"]["m_Name"] == tt["m_Name"] - meta = obj(store, doc, f"{oid(12)}#meta") + meta = obj(depot, doc, f"{oid(12)}#meta") assert meta["image"]["source"] == "atlas" and meta["image"]["atlas"]["$ref"]["pathId"] == 20 assert meta["sprite"]["m_RD"]["m_VertexData"]["m_DataSize"]["$hex"] # small: inline refs = art(doc, f"{oid(12)}#image")["semantics"]["refs"] @@ -766,18 +766,18 @@ def test_sprites_same_bundle(exported): def test_sprite_with_its_atlas_elsewhere_gets_a_meta_only(exported): - store, _task, doc = exported + depot, _task, doc = exported assert by_object(doc)[oid(13)]["artifacts"] == [f"{oid(13)}#meta"] a = art(doc, f"{oid(13)}#meta") assert a["semantics"]["facts"] == {"crop": CROP, "atlas": f"{ATLAS_CAB}:99"} - meta = obj(store, doc, a["id"]) + meta = obj(depot, doc, a["id"]) assert meta["image"] == {"source": "atlas", "atlas": {"$ref": {"file": ATLAS_CAB, "pathId": 99}}, "stage": CROP} assert meta["object"]["name"] == "remote" and len(meta["mesh"]["positions"]) == 4 def test_mesh(exported): - store, _task, doc = exported - d, body = gltf.read(load(store, doc, f"{oid(30)}#mesh")) + depot, _task, doc = exported + d, body = gltf.read(load(depot, doc, f"{oid(30)}#mesh")) (prim,) = d["meshes"][0]["primitives"] assert set(prim["attributes"]) == {"POSITION", "NORMAL", "TEXCOORD_0"} and prim["mode"] == 4 pos = gltf.accessor_array(d, body, prim["attributes"]["POSITION"]) @@ -791,18 +791,18 @@ def test_mesh(exported): def test_text_and_fonts(exported): - store, _task, doc = exported + depot, _task, doc = exported a = art(doc, f"{oid(40)}#data") - assert load(store, doc, a["id"]) == b"hello\n" and a["content"]["ext"] == "txt" - assert load(store, doc, f"{oid(41)}#data") == b"\xff\xfe\x00bin" + assert load(depot, doc, a["id"]) == b"hello\n" and a["content"]["ext"] == "txt" + assert load(depot, doc, f"{oid(41)}#data") == b"\xff\xfe\x00bin" assert art(doc, f"{oid(41)}#data")["content"]["ext"] == "bin" f = art(doc, f"{oid(42)}#font") - assert load(store, doc, f["id"]) == _ttf() and f["content"]["ext"] == "ttf" + assert load(depot, doc, f["id"]) == _ttf() and f["content"]["ext"] == "ttf" def test_prefab(exported): - store, _task, doc = exported - p = obj(store, doc, f"{oid(60)}#prefab") + depot, _task, doc = exported + p = obj(depot, doc, f"{oid(60)}#prefab") assert p["name"] == "root" and [n["path"] for n in p["nodes"]] == ["root", "root/child"] root, child = p["nodes"] assert root["pathId"] == 60 and root["transformPathId"] == 61 and root["layer"] == 5 @@ -813,9 +813,9 @@ def test_prefab(exported): assert fields["tex"] == {"$ref": {"file": CAB, "pathId": 10, "class": "Texture2D", "name": "tex"}} assert fields["m_Script"] == {"$ref": {"file": "CAB-scripts", "pathId": 7}} assert math.isnan(fields["value"]) and fields["small"] == {"$hex": "616263"} - assert b'{"$float": "nan"}' in load(store, doc, f"{oid(60)}#prefab").replace(b"\n", b"").replace( + assert b'{"$float": "nan"}' in load(depot, doc, f"{oid(60)}#prefab").replace(b"\n", b"").replace( b" ", b"" - ) or b'"$float"' in load(store, doc, f"{oid(60)}#prefab") + ) or b'"$float"' in load(depot, doc, f"{oid(60)}#prefab") (c,) = child["components"] assert c["fields"]["up"] == {"transform": "root"} assert c["fields"]["comp"] == {"component": "MonoBehaviour", "gameObject": "root", "class": "Behaviour"} @@ -824,8 +824,8 @@ def test_prefab(exported): def test_scriptable_objects_blobs_and_strings(exported): - store, _task, doc = exported - m = obj(store, doc, f"{oid(70)}#json") + depot, _task, doc = exported + m = obj(depot, doc, f"{oid(70)}#json") assert m["$script"] == "A|Game|Local" and m["m_Name"] == "moc" blob = m["_bytes"]["$blob"] assert blob == {"sha256": contract.sha256(MOC), "size": len(MOC), "format": "moc3"} @@ -835,19 +835,19 @@ def test_scriptable_objects_blobs_and_strings(exported): "format": "moc3", "facts": {"field": "_bytes"}, } - assert load(store, doc, b["id"]) == MOC + assert load(depot, doc, b["id"]) == MOC assert m["text"] == {"$hex": b"caf".hex() + "c3"} assert by_object(doc)[oid(70)]["artifacts"] == [f"{oid(70)}#blob:_bytes", f"{oid(70)}#json"] - lost = obj(store, doc, f"{oid(71)}#json") + lost = obj(depot, doc, f"{oid(71)}#json") assert "$script" not in lost and lost["m_Script"] == {"$ref": {"file": "CAB-scripts", "pathId": 9}} def test_cue_sheets_held_in_the_bundle(exported): - store, _task, doc = exported + depot, _task, doc = exported from nnnotes.formats import cri a = art(doc, f"{oid(73)}#acb") - assert load(store, doc, a["id"]) == SPLIT_ACB == cri.split_acb(SPLIT_CHUNKS) + assert load(depot, doc, a["id"]) == SPLIT_ACB == cri.split_acb(SPLIT_CHUNKS) assert a["content"]["ext"] == "acb" and a["semantics"] == { "kind": "cri.acb", "format": "acb", @@ -855,25 +855,25 @@ def test_cue_sheets_held_in_the_bundle(exported): "refs": [{"rel": "chunk", "object": oid(44)}, {"rel": "chunk", "object": oid(45)}], } assert by_object(doc)[oid(73)]["artifacts"] == [f"{oid(73)}#acb", f"{oid(73)}#json"] - assert obj(store, doc, f"{oid(73)}#json")["$script"] == "Assembly-CSharp|Fwk.Sound|SplitAcbData" - assert load(store, doc, f"{oid(44)}#data") == SPLIT_CHUNKS[0] # the chunks stay exported + assert obj(depot, doc, f"{oid(73)}#json")["$script"] == "Assembly-CSharp|Fwk.Sound|SplitAcbData" + assert load(depot, doc, f"{oid(44)}#data") == SPLIT_CHUNKS[0] # the chunks stay exported e = art(doc, f"{oid(74)}#acb") - assert load(store, doc, e["id"]) == EMBEDDED_ACB + assert load(depot, doc, e["id"]) == EMBEDDED_ACB assert e["semantics"]["facts"] == {"cueSheet": "se_test", "layout": "embedded"} - m = obj(store, doc, f"{oid(74)}#json") + m = obj(depot, doc, f"{oid(74)}#json") ref = m["references"]["RefIds"][0] assert ref["type"] == {"class": IMPL[2], "ns": IMPL[1], "asm": IMPL[0]} assert ref["data"]["data"] == { "$blob": {"sha256": contract.sha256(EMBEDDED_ACB), "size": len(EMBEDDED_ACB), "format": "acb"} } assert by_object(doc)[oid(74)]["artifacts"] == [f"{oid(74)}#acb", f"{oid(74)}#json"] # no blob: artifact - small = run(store, task_for(store, "all", classes=["MonoBehaviour"], params={"blobMin": 1 << 20})) - assert load(store, small, f"{oid(74)}#acb") == EMBEDDED_ACB # whatever its size + small = run(depot, task_for(depot, "all", classes=["MonoBehaviour"], params={"blobMin": 1 << 20})) + assert load(depot, small, f"{oid(74)}#acb") == EMBEDDED_ACB # whatever its size -def movie_task(store, converter=True): +def movie_task(depot, converter=True): """The task of the movie bundle: converter cri.usm in its atoms, as uses() gives it for that bundle.""" - t = task_for(store, "movie", scripts=USM_SCRIPTS) + t = task_for(depot, "movie", scripts=USM_SCRIPTS) atoms = {**t.atoms, "cri.usm": "cri.usm/1"} if converter else t.atoms return Task(t.stage, t.version, t.subject, t.params, dict(sorted(atoms.items())), t.inputs, t.context) @@ -882,33 +882,33 @@ def test_movies_held_in_the_bundle(tmp_path): """A movie asset's USM bytes (CriSerializedBytesAssetImpl) are its `usm` artifact whatever their size; the cue sheet beside it keeps its `acb`. Without converter cri.usm the movie asset is exported as before (its bytes are not an ACB: the typetree with generic.error).""" - store = Store(tmp_path / "s") - doc = run(store, movie_task(store)) + depot = Depot(tmp_path / "s") + doc = run(depot, movie_task(depot)) m, a = "CAB-movie:2", "CAB-movie:3" usm = art(doc, f"{m}#usm") - assert load(store, doc, usm["id"]) == WARMUP and usm["content"]["ext"] == "usm" + assert load(depot, doc, usm["id"]) == WARMUP and usm["content"]["ext"] == "usm" assert usm["semantics"] == {"kind": "cri.usm", "format": "usm", "facts": {"movie": "warmup"}} - tree = obj(store, doc, f"{m}#json") + tree = obj(depot, doc, f"{m}#json") assert tree["$script"] == objexport.USM_ASSET_SCRIPT and tree["movieInfo"] == MOVIE_INFO assert tree["references"]["RefIds"][0]["data"]["data"] == { "$blob": {"sha256": contract.sha256(WARMUP), "size": len(WARMUP), "format": "usm"} } assert by_object(doc)[m]["status"] == "exported" and by_object(doc)[m]["artifacts"] == [f"{m}#json", f"{m}#usm"] - assert load(store, doc, f"{a}#acb") == EMBEDDED_ACB - old = run(store, movie_task(store, converter=False)) + assert load(depot, doc, f"{a}#acb") == EMBEDDED_ACB + old = run(depot, movie_task(depot, converter=False)) assert by_object(old)[m]["status"] == "generic" and by_object(old)[m]["reason"]["code"] == "generic.error" - assert f"{m}#usm" not in {x["id"] for x in old["artifacts"]} and load(store, old, f"{a}#acb") == EMBEDDED_ACB + assert f"{m}#usm" not in {x["id"] for x in old["artifacts"]} and load(depot, old, f"{a}#acb") == EMBEDDED_ACB def test_the_movie_converter_is_keyed_by_the_script_context(tmp_path): - store = Store(tmp_path / "s") - env = planning_env(store, ("all", "movie"), scripts=USM_SCRIPTS) + depot = Depot(tmp_path / "s") + env = planning_env(depot, ("all", "movie"), scripts=USM_SCRIPTS) stage = ExportStage(FAKE_READER) assert describe(stage, "movie", None, env).atoms["cri.usm"] == "cri.usm/1" assert "cri.usm" not in describe(stage, "all", None, env).atoms assert "cri.usm" not in describe(stage, "movie", {"classes": ["AssetBundle"]}, env).atoms assert "cri.usm" in stage.ATOMS - plain = planning_env(Store(tmp_path / "p"), ("movie",)) + plain = planning_env(Depot(tmp_path / "p"), ("movie",)) assert "cri.usm" not in describe(stage, "movie", None, plain).atoms @@ -949,7 +949,7 @@ def test_acb_layouts(tmp_path): def test_a_split_sheet_with_a_missing_chunk_gives_the_typetree(tmp_path, monkeypatch): - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") real = objexport.acb_asset monkeypatch.setattr( objexport, @@ -958,21 +958,21 @@ def test_a_split_sheet_with_a_missing_chunk_gives_the_typetree(tmp_path, monkeyp dict(real(tt), chunks=[pptr(44), pptr(99, 1)]) if real(tt) and real(tt)["layout"] == "split" else real(tt) ), ) - doc = run(store, task_for(store, "all", classes=["MonoBehaviour"])) + doc = run(depot, task_for(depot, "all", classes=["MonoBehaviour"])) item = by_object(doc)[oid(73)] assert item["status"] == "generic" and item["reason"]["code"] == "generic.error" assert "is not a TextAsset of the bundle" in item["reason"]["message"] def test_shader(exported): - store, _task, doc = exported + depot, _task, doc = exported from nnnotes.formats import shader env = bundle_all() r = env.assets[0].objects[51] text, recs, codes = shader._render(r, r.tt, "Test/Unlit") # what `nnnotes shader` writes - assert load(store, doc, f"{oid(51)}#json") == text.encode("utf-8") - index = obj(store, doc, f"{oid(51)}#index") + assert load(depot, doc, f"{oid(51)}#json") == text.encode("utf-8") + index = obj(depot, doc, f"{oid(51)}#index") assert ( index["name"] == "Test/Unlit" and [{k: v for k, v in x.items() if k != "role"} for x in index["variants"]] == recs @@ -980,10 +980,10 @@ def test_shader(exported): assert [x["role"] for x in index["variants"]] == ["program:gles3/s0p0_vertex_0", "program:gles3/s0p0_fragment_0"] for x, code in zip(index["variants"], codes): a = art(doc, f"{oid(51)}#{x['role']}") - assert load(store, doc, a["id"]) == code and a["content"]["ext"] == "glsl" + assert load(depot, doc, a["id"]) == code and a["content"]["ext"] == "glsl" assert a["semantics"]["facts"]["keywords"] == x["keywords"] assert codes[0] == b"#ifdef VERTEX\nvoid main(){}\n#endif\n" and index["variants"][0]["keywords"] == ["KW_A"] - t = obj(store, doc, f"{oid(51)}#typetree") + t = obj(depot, doc, f"{oid(51)}#typetree") assert t["m_ParsedForm"]["m_Name"] == "Test/Unlit" and "$hex" in t["compressedBlob"] assert art(doc, f"{oid(51)}#json")["semantics"]["facts"] == { "name": "Test/Unlit", @@ -993,33 +993,33 @@ def test_shader(exported): def test_clip_bindings_name_scripts_of_other_bundles(exported): - store, _task, doc = exported - clip = obj(store, doc, f"{oid(96)}#json") + depot, _task, doc = exported + clip = obj(depot, doc, f"{oid(96)}#json") (b,) = clip["bindings"] assert b["typeID"] == 114 and b["class"] == "Fader" and b["path"] == "" assert census_mod.script_refs(census_doc(open_fake(b"fake:all"))) == sorted(SCRIPTS) + ["CAB-scripts:9"] def test_material_clip_controller(exported): - store, _task, doc = exported - mat = obj(store, doc, f"{oid(50)}#json") + depot, _task, doc = exported + mat = obj(depot, doc, f"{oid(50)}#json") assert mat["shader"] == {"$ref": {"file": ATLAS_CAB, "pathId": 5}} assert mat["textures"]["_MainTex"]["texture"]["$ref"]["pathId"] == 10 - assert mat["colors"]["_Color"]["g"] == INF and b"1e999" in load(store, doc, f"{oid(50)}#json") - clip = obj(store, doc, f"{oid(90)}#json") + assert mat["colors"]["_Color"]["g"] == INF and b"1e999" in load(depot, doc, f"{oid(50)}#json") + clip = obj(depot, doc, f"{oid(90)}#json") assert clip["clip"] == "clip" and clip["bindings"] == [ {"path": "", "typeID": 4, "class": "Transform", "attribute": "m_LocalPosition", "curves": 3} ] - ctl = obj(store, doc, f"{oid(92)}#json") + ctl = obj(depot, doc, f"{oid(92)}#json") assert ctl["clips"] == [{"$ref": {"file": CAB, "pathId": 90, "class": "AnimationClip", "name": "clip"}}] - assert obj(store, doc, f"{oid(95)}#json")["m_Speed"] == INF + assert obj(depot, doc, f"{oid(95)}#json")["m_Speed"] == INF def test_loose_documents(exported): - store, task, doc = exported - s = obj(store, doc, "unity.export:all#scripts.json") + depot, task, doc = exported + s = obj(depot, doc, "unity.export:all#scripts.json") assert list(s["scripts"]) == [oid(80)] and s["scripts"][oid(80)]["m_ClassName"] == "Local" - b = obj(store, doc, "unity.export:all#bundle.json") + b = obj(depot, doc, "unity.export:all#bundle.json") assert b["files"] == [ { "name": CAB, @@ -1039,7 +1039,7 @@ def test_loose_documents(exported): def test_two_runs_are_identical(tmp_path): - a, b = Store(tmp_path / "a"), Store(tmp_path / "b") + a, b = Depot(tmp_path / "a"), Depot(tmp_path / "b") ta, tb = task_for(a, "all"), task_for(b, "all") assert ta.key == tb.key da, db = run(a, ta), run(b, tb) @@ -1049,11 +1049,11 @@ def test_two_runs_are_identical(tmp_path): def test_key_follows_the_census_classes_and_params(tmp_path): - store = Store(tmp_path / "s") - t = task_for(store, "atlas") + depot = Depot(tmp_path / "s") + t = task_for(depot, "atlas") assert "mesh.glb" not in t.atoms and "mesh.arrays" in t.atoms # sprites use mesh.arrays assert {"tex.png", "sprite.png", "generic.json", "bundle.json"} <= set(t.atoms) - assert task_for(store, "atlas", params={"png": {"level": 3}}).key != t.key + assert task_for(depot, "atlas", params={"png": {"level": 3}}).key != t.key assert export_atoms(["Texture2D"]) == export_atoms(["Texture2D", "Texture2D"]) assert set(export_atoms(["Mesh"])) == {"reader", "sniff", "mesh.arrays", "gltf.write", "mesh.glb", "generic.json"} with pytest.raises(ValueError, match="png level"): @@ -1064,66 +1064,66 @@ def test_key_follows_the_census_classes_and_params(tmp_path): def test_png_level_changes_bytes_not_pixels(tmp_path): - store = Store(tmp_path / "s") - d6 = run(store, task_for(store, "all", classes=["Texture2D"])) - d0 = run(store, task_for(store, "all", classes=["Texture2D"], params={"png": {"level": 0}})) - p6, p0 = load(store, d6, f"{oid(10)}#image"), load(store, d0, f"{oid(10)}#image") + depot = Depot(tmp_path / "s") + d6 = run(depot, task_for(depot, "all", classes=["Texture2D"])) + d0 = run(depot, task_for(depot, "all", classes=["Texture2D"], params={"png": {"level": 0}})) + p6, p0 = load(depot, d6, f"{oid(10)}#image"), load(depot, d0, f"{oid(10)}#image") assert p6 != p0 io = __import__("io") assert Image.open(io.BytesIO(p6)).tobytes() == Image.open(io.BytesIO(p0)).tobytes() def test_blob_threshold(tmp_path): - store = Store(tmp_path / "s") - doc = run(store, task_for(store, "all", classes=["MonoBehaviour"], params={"blobMin": 1 << 20})) - m = obj(store, doc, f"{oid(70)}#json") + depot = Depot(tmp_path / "s") + doc = run(depot, task_for(depot, "all", classes=["MonoBehaviour"], params={"blobMin": 1 << 20})) + m = obj(depot, doc, f"{oid(70)}#json") assert m["_bytes"] == {"$hex": MOC.hex()} def test_class_selection(tmp_path): - store = Store(tmp_path / "s") - doc = run(store, task_for(store, "all", classes=["Sprite"])) + depot = Depot(tmp_path / "s") + doc = run(depot, task_for(depot, "all", classes=["Sprite"])) items = by_object(doc) assert set(items) == {oid(11), oid(12), oid(13)} # textures decoded, not exported assert all(i["status"] == "exported" for i in items.values()) - doc = run(store, task_for(store, "all", classes=["GameObject"])) + doc = run(depot, task_for(depot, "all", classes=["GameObject"])) assert set(by_object(doc)) == {oid(p) for p in (60, 61, 62, 63, 64, 65)} def test_scripts_come_from_the_context_only(tmp_path): - store = Store(tmp_path / "s") - doc = run(store, task_for(store, "all", classes=["MonoBehaviour"], scripts={})) + depot = Depot(tmp_path / "s") + doc = run(depot, task_for(depot, "all", classes=["MonoBehaviour"], scripts={})) items = by_object(doc) assert items[oid(70)]["reason"]["code"] == "script.unresolved" assert items[oid(62)]["reason"]["code"] == "script.unresolved" # no hierarchy: one by one def test_a_converter_error_gives_the_typetree(tmp_path, monkeypatch): - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") real = objexport.RefExporter.conv_material def broken(self, info): raise KeyError("m_SavedProperties") monkeypatch.setattr(objexport.RefExporter, "conv_material", broken) - doc = run(store, task_for(store, "all", classes=["Material"])) + doc = run(depot, task_for(depot, "all", classes=["Material"])) item = by_object(doc)[oid(50)] assert item["status"] == "generic" and item["reason"] == { "code": "generic.error", "message": "KeyError: 'm_SavedProperties'", } - assert obj(store, doc, f"{oid(50)}#json")["m_Name"] == "mat" + assert obj(depot, doc, f"{oid(50)}#json")["m_Name"] == "mat" monkeypatch.setattr(objexport.RefExporter, "conv_material", real) def test_a_broken_hierarchy_gives_its_objects_one_by_one(tmp_path, monkeypatch): - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") def broken(self, o): raise RuntimeError("component failed") monkeypatch.setattr(objexport.RefExporter, "component", broken) - doc = run(store, task_for(store, "all", classes=["GameObject"])) + doc = run(depot, task_for(depot, "all", classes=["GameObject"])) items = by_object(doc) assert {i["status"] for i in items.values()} == {"generic"} assert all(i["reason"]["code"] == "generic.error" for i in items.values()) @@ -1131,7 +1131,7 @@ def broken(self, o): def test_unreadable_objects_fail(tmp_path, monkeypatch): - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") real = UnityBundle.typetree def typetree(self, o): @@ -1140,17 +1140,17 @@ def typetree(self, o): return real(self, o) monkeypatch.setattr(UnityBundle, "typetree", typetree) - doc = run(store, task_for(store, "all", classes=["Animator"])) + doc = run(depot, task_for(depot, "all", classes=["Animator"])) (item,) = doc["items"] assert item["status"] == "failed" and item["reason"]["code"] == "failed.read" and doc["status"] == "partial" def test_task_with_other_atoms_is_refused(tmp_path): - store = Store(tmp_path / "s") - t = task_for(store, "all") + depot = Depot(tmp_path / "s") + t = task_for(depot, "all") bad = Task(t.stage, t.version, t.subject, t.params, {**t.atoms, "tex.png": "tex.png/0"}, t.inputs, t.context) with pytest.raises(IncompatibleTask, match="tex.png"): - execute(bad, store, registry(ExportStage(FAKE_READER))) + execute(bad, depot, registry(ExportStage(FAKE_READER))) missing = Task( t.stage, t.version, @@ -1161,7 +1161,7 @@ def test_task_with_other_atoms_is_refused(tmp_path): t.context, ) with pytest.raises(IncompatibleTask, match="mesh.glb"): - execute(missing, store, registry(ExportStage(FAKE_READER))) + execute(missing, depot, registry(ExportStage(FAKE_READER))) # ---------------------------------------------------------------- describing tasks @@ -1206,57 +1206,57 @@ def ref(s): } -def committed(store, env, task, role, doc, kind): +def committed(depot, env, task, role, doc, kind): rec = contract.artifact( contract.artifact_id(task.id, role), - store.add(contract.encode(doc), "json"), + depot.add(contract.encode(doc), "json"), contract.provenance(task), {"kind": kind, "format": "json", "facts": {}}, ) - store.commit(contract.result(task, [rec], [])) + depot.commit(contract.result(task, [rec], [])) env.done(task.id, task.key) -def planning_env(store, names=("all", "atlas"), selected=None, scripts=SCRIPTS): +def planning_env(depot, names=("all", "atlas"), selected=None, scripts=SCRIPTS): names = tuple(sorted(set(names) | set(selected or ()))) bundles = {} for n in names: data = f"fake:{n}".encode() - bundles[n] = Input("bundle", store.put(data), len(data), f"{n}.bundle", ({"kind": "store"},)) - env = Env(store, {"bundles": bundles, "selected": sorted(selected if selected is not None else names)}) + bundles[n] = Input("bundle", depot.put(data), len(data), f"{n}.bundle", ({"kind": "depot"},)) + env = Env(depot, {"bundles": bundles, "selected": sorted(selected if selected is not None else names)}) for n in names: t = Task(census_mod.CensusStage.name, 1, n, {}, {}, (bundles[n],)) doc = census_doc(open_fake(f"fake:{n}".encode())) rec = contract.artifact( contract.artifact_id(t.id, "census"), - store.add(contract.encode(doc), "json"), + depot.add(contract.encode(doc), "json"), contract.provenance(t), {"kind": "unity.census", "format": "json", "facts": census_mod.facts(doc)}, ) - store.commit(contract.result(t, [rec], [])) + depot.commit(contract.result(t, [rec], [])) env.done(t.id, t.key) table = Task("link.scripts", 1, "all") - committed(store, env, table, "scripts", {"schema": "nnnotes.scripts/1", "scripts": scripts}, "link.scripts") + committed(depot, env, table, "scripts", {"schema": "nnnotes.scripts/1", "scripts": scripts}, "link.scripts") return env def test_describe_reads_the_census_and_the_script_table(tmp_path): - store = Store(tmp_path / "s") - env = planning_env(store) + depot = Depot(tmp_path / "s") + env = planning_env(depot) stage = ExportStage(FAKE_READER) assert stage.subjects(env) == ["all", "atlas"] # every closure bundle holding a SpriteAtlas joins a selection that holds sprites (a sprite reaches its # texture through its atlas' bundle); a selection without sprites stays as it is - assert stage.subjects(planning_env(Store(tmp_path / "t"), selected=["all"])) == ["all", "atlas"] - assert stage.subjects(planning_env(Store(tmp_path / "u"), selected=["plain"])) == ["plain"] + assert stage.subjects(planning_env(Depot(tmp_path / "t"), selected=["all"])) == ["all", "atlas"] + assert stage.subjects(planning_env(Depot(tmp_path / "u"), selected=["plain"])) == ["plain"] t = describe(stage, "all", None, env) assert t.atoms == export_atoms(classes_of("all"), FAKE_READER.id) - assert t.context == {"scripts": SCRIPTS} and t.key == task_for(store, "all").key + assert t.context == {"scripts": SCRIPTS} and t.key == task_for(depot, "all").key assert stage.depends("all", env) == ["unity.census:all", "link.scripts:all"] a = describe(stage, "atlas", None, env) assert a.context == {} and "mesh.glb" not in a.atoms assert t.cost.peak_bytes >= 72 << 20 - fresh = Env(store, env.facts) + fresh = Env(depot, env.facts) from nnnotes.pipeline.stages import Pending fresh.waiting("unity.census:all") @@ -1265,11 +1265,11 @@ def test_describe_reads_the_census_and_the_script_table(tmp_path): def test_class_selection_keeps_only_its_converters_in_the_key(tmp_path): - store = Store(tmp_path / "s") - env = planning_env(store) + depot = Depot(tmp_path / "s") + env = planning_env(depot) stage = ExportStage(FAKE_READER) full = describe(stage, "all", None, env) - assert full.key == describe(stage, "all", {"classes": None}, env).key == task_for(store, "all").key + assert full.key == describe(stage, "all", {"classes": None}, env).key == task_for(depot, "all").key meshes = describe(stage, "all", {"classes": ["Mesh"]}, env) assert meshes.atoms == export_atoms(["Mesh"], FAKE_READER.id) and meshes.params["classes"] == ["Mesh"] assert "tex.png" not in meshes.atoms and meshes.key != full.key @@ -1278,45 +1278,45 @@ def test_class_selection_keeps_only_its_converters_in_the_key(tmp_path): assert env.params == {} # set while describing only for classes in (["Mesh"], ["Sprite"], ["GameObject"], ["Material", "AnimationClip"]): t = describe(stage, "all", {"classes": classes}, env) - execute(t, store, registry(stage)) - doc = store.result(t.key) + execute(t, depot, registry(stage)) + doc = depot.result(t.key) assert doc["items"] and all(i["class"] in {*classes, "Transform", "MonoBehaviour"} for i in doc["items"]) -def crop_setup(store): - env = planning_env(store) +def crop_setup(depot): + env = planning_env(depot) stage = ExportStage(FAKE_READER) for n in ("all", "atlas"): t = describe(stage, n, None, env) - execute(t, store, registry(stage)) + execute(t, depot, registry(stage)) env.done(t.id, t.key) return env def test_sprite_crop_equals_the_same_sprite_cropped_in_its_atlas_bundle(tmp_path): - store = Store(tmp_path / "s") - env = crop_setup(store) + depot = Depot(tmp_path / "s") + env = crop_setup(depot) stage = SpriteCropStage() assert stage.subjects(env) == ["atlas:99"] t = describe(stage, "atlas:99", None, env) assert [i.role for i in t.inputs] == ["atlas", f"sprite:{oid(13)}", f"texture:{ATLAS_CAB}:98"] assert stage.depends("atlas:99", env) == ["unity.export:all", "unity.export:atlas"] assert set(t.atoms) == {"png.encode", "sprite.crop"} - execute(t, store, registry(stage)) - doc = store.result(t.key) + execute(t, depot, registry(stage)) + doc = depot.result(t.key) (item,) = doc["items"] assert item == {"object": oid(13), "status": "exported", "artifacts": [f"{oid(13)}#image"], "class": "Sprite"} a = doc["artifacts"][0] assert a["provenance"]["object"]["name"] == "remote" and a["provenance"]["object"]["file"] == CAB - local = store.result(env.key("unity.export:atlas")) + local = depot.result(env.key("unity.export:atlas")) same = art(local, f"{ATLAS_CAB}:97#image") assert a["content"] == same["content"] # the same bytes as a local crop - assert Image.open(__import__("io").BytesIO(store.read(a["content"]["sha256"]))).size == (8, 8) + assert Image.open(__import__("io").BytesIO(depot.read(a["content"]["sha256"]))).size == (8, 8) def test_sprite_crop_waits_for_exports(tmp_path): - store = Store(tmp_path / "s") - env = planning_env(store) + depot = Depot(tmp_path / "s") + env = planning_env(depot) env.waiting("unity.export:all") from nnnotes.pipeline.stages import Pending @@ -1377,7 +1377,7 @@ def export_golden(): classes the converter takes), and the whole bundle.""" def fixture(classes): - return lambda store: task_for(store, "all", classes=classes) + return lambda depot: task_for(depot, "all", classes=classes) fixtures = {name: fixture(classes) for name, classes in CONVERTER_CLASSES.items()} fixtures["all"] = fixture(None) @@ -1388,8 +1388,8 @@ def fixture(classes): def crop_golden(): """Provider of tests/golden/sprite.crop.json.""" - def fixture(store): - env = crop_setup(store) + def fixture(depot): + env = crop_setup(depot) return describe(SpriteCropStage(), "atlas:99", None, env) return {"stage": SpriteCropStage(), "fixtures": {"atlas": fixture}, "libraries": LIBRARIES} @@ -1416,7 +1416,7 @@ def changed(self, info): monkeypatch.setattr(objexport.RefExporter, "conv_font", changed) p = export_golden() - actual = golden(p["stage"], p["fixtures"], Store(tmp_path / "s"), p["libraries"]) + actual = golden(p["stage"], p["fixtures"], Depot(tmp_path / "s"), p["libraries"]) problems = golden_problems(expected, actual, p["stage"]) assert "unity.export: fixture font: output changed without a version bump" in problems assert "unity.export: fixture all: output changed without a version bump" in problems diff --git a/tests/pipeline/test_orchestrate.py b/tests/pipeline/test_orchestrate.py index a3e586a..a4cee60 100644 --- a/tests/pipeline/test_orchestrate.py +++ b/tests/pipeline/test_orchestrate.py @@ -7,10 +7,10 @@ from nnnotes.pipeline import contract from nnnotes.pipeline.contract import Cost, Task +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.orchestrate import Orchestrator, Scheduler, coverage from nnnotes.pipeline.plan import plan from nnnotes.pipeline.stages import execute -from nnnotes.pipeline.store import Store from tests.pipeline.test_stages import PIPELINE, toy_files, toy_stages TEXTS = { @@ -34,11 +34,11 @@ def tree(root: Path, *subs) -> dict: def orchestrate(tmp_path, name, texts, workers=0, stages=None, force=(), **kw): - store = Store(tmp_path / name) + depot = Depot(tmp_path / name) files = toy_files(tmp_path / "in", texts) - orch = Orchestrator(store, stages or toy_stages(), workers=workers, log=lambda m: None, **kw) + orch = Orchestrator(depot, stages or toy_stages(), workers=workers, log=lambda m: None, **kw) run = orch.run(PIPELINE, facts={"files": files}, context={"region": "test"}, selection=["all"], force=force) - return store, run, orch + return depot, run, orch def test_one_worker_and_four_workers_give_identical_stores(tmp_path): @@ -56,7 +56,7 @@ def test_one_worker_and_four_workers_give_identical_stores(tmp_path): def test_run_stage_task_by_task_equals_the_orchestrated_store(tmp_path): s0, _, _ = orchestrate(tmp_path, "orch", TEXTS) - s1 = Store(tmp_path / "tasks-store") + s1 = Depot(tmp_path / "tasks-depot") stages = toy_stages() facts = {"files": toy_files(tmp_path / "in", TEXTS)} rounds = 0 @@ -73,7 +73,7 @@ def test_run_stage_task_by_task_equals_the_orchestrated_store(tmp_path): def test_item_failures_are_cached_task_failures_are_not(tmp_path): texts = {"a": "x\nFAIL", "b": "CRASH", "c": "y"} - store, run, _ = orchestrate(tmp_path, "s", texts) + depot, run, _ = orchestrate(tmp_path, "s", texts) st = {t: v["status"] for t, v in run.tasks.items()} assert st == { "toy.src:a": "ran", @@ -92,7 +92,7 @@ def test_item_failures_are_cached_task_failures_are_not(tmp_path): ("toy.src:b", None, "task.error"), ] assert run.exit_code == 1 - total = store.result(run.tasks["toy.total:all"]["key"]) + total = depot.result(run.tasks["toy.total:all"]["key"]) assert [i[0] for i in total["keyParts"]["inputs"]] == ["count:a", "count:c"] _, again, _ = orchestrate(tmp_path, "s", texts) st = {t: v["status"] for t, v in again.tasks.items()} @@ -101,7 +101,7 @@ def test_item_failures_are_cached_task_failures_are_not(tmp_path): def test_a_dying_worker_fails_its_task_only(tmp_path): - store, run, orch = orchestrate(tmp_path, "s", {"a": "x", "d": "DIE", "c": "y"}, workers=2) + depot, run, orch = orchestrate(tmp_path, "s", {"a": "x", "d": "DIE", "c": "y"}, workers=2) assert run.tasks["toy.src:d"]["status"] == "failed" assert [f["code"] for f in run.failures() if f["task"] == "toy.src:d"] == ["task.worker_died"] assert run.tasks["toy.src:a"]["status"] == run.tasks["toy.src:c"]["status"] == "ran" @@ -110,13 +110,13 @@ def test_a_dying_worker_fails_its_task_only(tmp_path): def test_equal_content_under_two_subjects_is_stored_once(tmp_path): - store, run, _ = orchestrate(tmp_path, "s", {"a": "same", "b": "same"}, workers=2) + depot, run, _ = orchestrate(tmp_path, "s", {"a": "same", "b": "same"}, workers=2) assert all(v["status"] == "ran" for v in run.tasks.values()) assert run.tasks["toy.src:a"]["key"] != run.tasks["toy.src:b"]["key"] # the subject names artifacts - ra, rb = (store.result(run.tasks[f"toy.src:{s}"]["key"]) for s in "ab") + ra, rb = (depot.result(run.tasks[f"toy.src:{s}"]["key"]) for s in "ab") assert [a["id"] for a in rb["artifacts"]] == ["F-b:1#data", "toy.src:b#all.txt"] assert {a["content"]["sha256"] for a in ra["artifacts"]} == {a["content"]["sha256"] for a in rb["artifacts"]} - assert len(tree(store.root, "cas")) == 3 # SAME, the two equal counts, the total + assert len(tree(depot.root, "cas")) == 3 # SAME, the two equal counts, the total def test_no_more_workers_start_than_a_stage_has_tasks(tmp_path): @@ -206,7 +206,7 @@ def test_coverage_and_failures_documents(tmp_path): def test_unknown_or_misordered_stages_are_refused(tmp_path): - orch = Orchestrator(Store(tmp_path / "s"), toy_stages(), log=lambda m: None) + orch = Orchestrator(Depot(tmp_path / "s"), toy_stages(), log=lambda m: None) with pytest.raises(ValueError, match="unknown stages"): orch.run(["toy.nope"], facts={"files": {}}) with pytest.raises(ValueError, match="runs before toy.src"): @@ -218,12 +218,12 @@ def test_a_config_error_stops_the_run(tmp_path): from tests.pipeline.test_stages import Src class NeedsTool(Src): - def run(self, task, store): + def run(self, task, depot): raise ConfigError("setting paths.tool is not set") stages = toy_stages() stages["toy.src"] = NeedsTool() - orch = Orchestrator(Store(tmp_path / "s"), stages, log=lambda m: None) + orch = Orchestrator(Depot(tmp_path / "s"), stages, log=lambda m: None) with pytest.raises(ConfigError, match="paths.tool"): orch.run(PIPELINE, facts={"files": toy_files(tmp_path / "in", {"a": "x", "b": "y"})}) @@ -239,7 +239,7 @@ def subjects(self, env): return ["x"] stages = dict(toy_stages(), **{"toy.after": AfterAll()}) - orch = Orchestrator(Store(tmp_path / "s"), stages, log=lambda m: None) + orch = Orchestrator(Depot(tmp_path / "s"), stages, log=lambda m: None) run = orch.run(["toy.src", "toy.after"], facts={"files": toy_files(tmp_path / "in", {"a": "x", "b": "CRASH"})}) assert run.tasks["toy.after:*"]["status"] == "failed" assert [f["code"] for f in run.failures() if f["task"] == "toy.after:*"] == ["upstream.failed"] @@ -333,10 +333,10 @@ def test_compact_coverage_equals_a_reading_of_whole_items(): def test_the_reports_read_the_results_once_until_the_tasks_change(tmp_path): - store, run, _ = orchestrate(tmp_path, "s", {"a": "x\nFAIL\n?y\n", "b": "z"}) + depot, run, _ = orchestrate(tmp_path, "s", {"a": "x\nFAIL\n?y\n", "b": "z"}) reads = [] - real = store.result - store.result = lambda key: reads.append(key) or real(key) + real = depot.result + depot.result = lambda key: reads.append(key) or real(key) first = (run.summary(), run.failures(), run.coverage()) n = len(reads) assert n == sum(t["status"] != "failed" for t in run.tasks.values()) diff --git a/tests/pipeline/test_plan.py b/tests/pipeline/test_plan.py index 177e017..df9bbe5 100644 --- a/tests/pipeline/test_plan.py +++ b/tests/pipeline/test_plan.py @@ -4,22 +4,22 @@ from nnnotes.pipeline import contract from nnnotes.pipeline.contract import Task +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.orchestrate import Orchestrator from nnnotes.pipeline.plan import plan, why -from nnnotes.pipeline.store import Store from tests.pipeline.test_stages import PIPELINE, toy_files, toy_stages TEXTS = {"a": "alpha\nbeta", "b": "gamma", "c": "delta\n?x"} def setup(tmp_path, texts=TEXTS): - store = Store(tmp_path / "store") + depot = Depot(tmp_path / "depot") facts = {"files": toy_files(tmp_path / "in", texts)} - return store, facts + return depot, facts -def run(store, facts, stages=None, params=None, selection=("all",)): - r = Orchestrator(store, stages or toy_stages(), log=lambda m: None).run( +def run(depot, facts, stages=None, params=None, selection=("all",)): + r = Orchestrator(depot, stages or toy_stages(), log=lambda m: None).run( PIPELINE, params=params, facts=facts, selection=list(selection) ) return r.write() @@ -30,8 +30,8 @@ def nodes(p) -> dict: def test_a_cold_plan_runs_the_first_stage_and_knows_nothing_after_it(tmp_path): - store, facts = setup(tmp_path) - p = plan(toy_stages(), PIPELINE, store, facts=facts) + depot, facts = setup(tmp_path) + p = plan(toy_stages(), PIPELINE, depot, facts=facts) n = nodes(p) assert n["toy.src:a"] == ("run", [{"code": "new-input"}]) assert n["toy.count:a"] == ("unknown", [{"code": "pending", "task": "toy.src:a"}]) @@ -53,73 +53,73 @@ def test_a_cold_plan_runs_the_first_stage_and_knows_nothing_after_it(tmp_path): def test_unchanged_inputs_are_all_hits(tmp_path): - store, facts = setup(tmp_path) - prev = run(store, facts) - p = plan(toy_stages(), PIPELINE, store, facts=facts, previous=prev) + depot, facts = setup(tmp_path) + prev = run(depot, facts) + p = plan(toy_stages(), PIPELINE, depot, facts=facts, previous=prev) assert all(v == ("hit", [{"code": "unchanged"}]) for v in nodes(p).values()) assert p.check() == 0 and p.tasks == [] and p.emit(tmp_path / "none") == [] assert "7 tasks: 7 hits, 0 to run, 0 unknown" in p.text() def test_a_stage_bump_reruns_it_and_leaves_downstream_pending(tmp_path): - store, facts = setup(tmp_path) - prev = run(store, facts) + depot, facts = setup(tmp_path) + prev = run(depot, facts) bumped = toy_stages(Count=2) - n = nodes(plan(bumped, PIPELINE, store, facts=facts, previous=prev)) + n = nodes(plan(bumped, PIPELINE, depot, facts=facts, previous=prev)) assert n["toy.src:a"][0] == "hit" assert n["toy.count:a"] == ("run", [{"code": "stage-version", "old": 1, "new": 2}]) assert n["toy.total:all"][0] == "unknown" - after = run(store, facts, bumped) + after = run(depot, facts, bumped) ran = {t["id"] for t in after["tasks"] if t["status"] == "ran"} assert ran == {"toy.count:a", "toy.count:b", "toy.count:c", "toy.total:all"} - assert plan(bumped, PIPELINE, store, facts=facts, previous=after).check() == 0 + assert plan(bumped, PIPELINE, depot, facts=facts, previous=after).check() == 0 def test_parameter_atom_input_and_context_reasons(tmp_path): - store, facts = setup(tmp_path) - prev = run(store, facts) - n = nodes(plan(toy_stages(), PIPELINE, store, facts=facts, previous=prev, params={"toy.src": {"suffix": "!"}})) + depot, facts = setup(tmp_path) + prev = run(depot, facts) + n = nodes(plan(toy_stages(), PIPELINE, depot, facts=facts, previous=prev, params={"toy.src": {"suffix": "!"}})) assert n["toy.src:b"] == ("run", [{"code": "params-changed", "path": "/suffix", "old": "", "new": "!"}]) stages = toy_stages() stages["toy.src"].ATOMS = {"upper": "toy-upper/2"} - n = nodes(plan(stages, PIPELINE, store, facts=facts, previous=prev)) + n = nodes(plan(stages, PIPELINE, depot, facts=facts, previous=prev)) assert n["toy.src:a"] == ( "run", [{"code": "atom-changed", "atom": "upper", "old": "toy-upper/1", "new": "toy-upper/2"}], ) Path(facts["files"]["b"]).write_bytes(b"GAMMA!") - n = nodes(plan(toy_stages(), PIPELINE, store, facts=facts, previous=prev)) + n = nodes(plan(toy_stages(), PIPELINE, depot, facts=facts, previous=prev)) ((status, reasons),) = [n["toy.src:b"]] assert status == "run" and reasons[0]["code"] == "input-changed" and reasons[0]["role"] == "file" assert reasons[0]["new"] == contract.sha256(b"GAMMA!") and reasons[0]["name"] == "b.txt" assert n["toy.src:a"][0] == "hit" and n["toy.count:b"][0] == "unknown" - n = nodes(plan(toy_stages(), PIPELINE, store, facts=dict(facts, context={"c": "x"}), previous=prev)) + n = nodes(plan(toy_stages(), PIPELINE, depot, facts=dict(facts, context={"c": "x"}), previous=prev)) assert n["toy.src:c"] == ("run", [{"code": "context-changed", "entry": "c"}]) def test_new_failed_forced_and_cached_reasons(tmp_path): - store, facts = setup(tmp_path, {"a": "alpha", "b": "CRASH"}) - prev = run(store, facts) - n = nodes(plan(toy_stages(), PIPELINE, store, facts=facts, previous=prev)) + depot, facts = setup(tmp_path, {"a": "alpha", "b": "CRASH"}) + prev = run(depot, facts) + n = nodes(plan(toy_stages(), PIPELINE, depot, facts=facts, previous=prev)) assert n["toy.src:b"] == ("run", [{"code": "previous-failed"}]) assert n["toy.src:a"] == ("hit", [{"code": "unchanged"}]) - n = nodes(plan(toy_stages(), PIPELINE, store, facts=facts, previous=prev, force=["toy.src:a"])) + n = nodes(plan(toy_stages(), PIPELINE, depot, facts=facts, previous=prev, force=["toy.src:a"])) assert n["toy.src:a"] == ("run", [{"code": "forced"}]) facts["files"].update(toy_files(tmp_path / "more", {"z": "new"})) - n = nodes(plan(toy_stages(), PIPELINE, store, facts=facts, previous=prev)) + n = nodes(plan(toy_stages(), PIPELINE, depot, facts=facts, previous=prev)) assert n["toy.src:z"] == ("run", [{"code": "new-input"}]) - n = nodes(plan(toy_stages(), PIPELINE, store, facts=facts)) # no previous run: hits are cached + n = nodes(plan(toy_stages(), PIPELINE, depot, facts=facts)) # no previous run: hits are cached assert n["toy.src:a"] == ("hit", [{"code": "cached"}]) def test_the_latest_run_of_the_selection_is_the_default_previous(tmp_path): - store, facts = setup(tmp_path) - prev = run(store, facts, selection=["group:x"]) - assert store.latest_run(["group:x"])["id"] == prev["id"] - n = nodes(plan(toy_stages(), PIPELINE, store, facts=facts, previous=store.latest_run(["group:x"]))) + depot, facts = setup(tmp_path) + prev = run(depot, facts, selection=["group:x"]) + assert depot.latest_run(["group:x"])["id"] == prev["id"] + n = nodes(plan(toy_stages(), PIPELINE, depot, facts=facts, previous=depot.latest_run(["group:x"]))) assert n["toy.src:a"] == ("hit", [{"code": "unchanged"}]) @@ -139,8 +139,8 @@ def test_why_lists_every_difference(): def test_emitted_tasks_are_runnable_descriptions(tmp_path): - store, facts = setup(tmp_path) - p = plan(toy_stages(), PIPELINE, store, facts=facts) + depot, facts = setup(tmp_path) + p = plan(toy_stages(), PIPELINE, depot, facts=facts) files = p.emit(tmp_path / "tasks") assert [f.name for f in files] == sorted(f"{t.key}.json" for t in p.tasks) for f in files: @@ -149,11 +149,11 @@ def test_emitted_tasks_are_runnable_descriptions(tmp_path): def test_peak_estimate_oversize_and_layout_counts(tmp_path): - store, facts = setup(tmp_path, {"a": "x" * 100, "b": "y" * 10, "c": "z" * 5}) # peaks 1000, 100, 50 + depot, facts = setup(tmp_path, {"a": "x" * 100, "b": "y" * 10, "c": "z" * 5}) # peaks 1000, 100, 50 p = plan( toy_stages(), PIPELINE, - store, + depot, facts=facts, workers=2, memory=500, @@ -163,8 +163,8 @@ def test_peak_estimate_oversize_and_layout_counts(tmp_path): s = p.doc["summary"] assert s["peakBytes"] == 1000 and s["oversize"] == ["toy.src:a"] and s["budget"] == 500 assert "layout original: write 0 / remove 0 / keep 4" in p.text() - p = plan(toy_stages(), PIPELINE, store, facts=facts, workers=2, memory=None, worker_bytes=10) + p = plan(toy_stages(), PIPELINE, depot, facts=facts, workers=2, memory=None, worker_bytes=10) assert p.doc["summary"]["peakBytes"] == 1000 + 100 + 2 * 10 - run(store, facts) - p = plan(toy_stages(), PIPELINE, store, facts=facts, layouts={"cas": {"write": 1, "remove": 0, "keep": 3}}) + run(depot, facts) + p = plan(toy_stages(), PIPELINE, depot, facts=facts, layouts={"cas": {"write": 1, "remove": 0, "keep": 3}}) assert p.check() == 1 # nothing runs, but a file would be written diff --git a/tests/pipeline/test_stages.py b/tests/pipeline/test_stages.py index 3ebd6c4..8bbebea 100644 --- a/tests/pipeline/test_stages.py +++ b/tests/pipeline/test_stages.py @@ -14,8 +14,8 @@ from nnnotes.pipeline import contract from nnnotes.pipeline.contract import Cost, IncompatibleTask, Input +from nnnotes.pipeline.depot import Depot, DepotConflict, InputMissing from nnnotes.pipeline.stages import Env, Output, Pending, Stage, describe, execute, registry -from nnnotes.pipeline.store import InputMissing, Store, StoreConflict class Src(Stage): @@ -28,7 +28,7 @@ def subjects(self, env): def inputs(self, subject, env): p = Path(env.fact("files")[subject]) - sha, size = env.store.identify(p, "raw", p.name) + sha, size = env.depot.identify(p, "raw", p.name) return [Input("file", sha, size, p.name, ({"kind": "file", "path": str(p)},))] def context(self, subject, env): @@ -38,8 +38,8 @@ def context(self, subject, env): def estimate(self, subject, env, inputs): return Cost(inputs[0].size / 100, inputs[0].size * 10) - def run(self, task, store): - data = store.input_bytes(task.input("file")) + def run(self, task, depot): + data = depot.input_bytes(task.input("file")) if b"CRASH" in data: raise RuntimeError("crash requested") if b"DIE" in data: @@ -48,7 +48,7 @@ def run(self, task, store): arts = [ contract.artifact( whole, - store.add(data.upper() + task.params["suffix"].encode(), "txt"), + depot.add(data.upper() + task.params["suffix"].encode(), "txt"), contract.provenance(task), {"kind": "toy.text"}, ) @@ -68,7 +68,7 @@ def run(self, task, store): arts.append( contract.artifact( aid, - store.add(line.upper().encode(), "txt"), + depot.add(line.upper().encode(), "txt"), contract.provenance(task, obj=obj), {"kind": "toy.line"}, ) @@ -90,8 +90,8 @@ def inputs(self, subject, env): def depends(self, subject, env): return [f"toy.src:{subject}"] - def run(self, task, store): - text = store.input_bytes(task.input("text")).decode() + def run(self, task, depot): + text = depot.input_bytes(task.input("text")).decode() doc = {"lines": sum(1 for line in text.split("\n") if line and not line.startswith(("FAIL", "?")))} if self.version > 1: doc["chars"] = len(text) @@ -99,7 +99,7 @@ def run(self, task, store): return Output( [ contract.artifact( - aid, store.add(contract.encode(doc), "json"), contract.provenance(task), {"kind": "toy.count"} + aid, depot.add(contract.encode(doc), "json"), contract.provenance(task), {"kind": "toy.count"} ) ], [], @@ -121,14 +121,14 @@ def inputs(self, subject, env): def depends(self, subject, env): return [f"toy.count:{s}" for s in sorted(env.fact("files"))] - def run(self, task, store): - n = sum(json.loads(store.input_bytes(i))["lines"] for i in task.inputs) + def run(self, task, depot): + n = sum(json.loads(depot.input_bytes(i))["lines"] for i in task.inputs) aid = contract.artifact_id(task.id, "total.json") return Output( [ contract.artifact( aid, - store.add(contract.encode({"lines": n}), "json"), + depot.add(contract.encode({"lines": n}), "json"), contract.provenance(task), {"kind": "toy.total"}, ) @@ -164,8 +164,8 @@ def toy_files(root: Path, texts: dict) -> dict: # ---------------------------------------------------------------- tests def test_describe_fills_defaults_and_refuses_unknown_parameters(tmp_path): - store = Store(tmp_path / "store") - env = Env(store, {"files": toy_files(tmp_path / "in", {"a": "x\ny"})}) + depot = Depot(tmp_path / "depot") + env = Env(depot, {"files": toy_files(tmp_path / "in", {"a": "x\ny"})}) t = describe(Src(), "a", None, env) assert t.id == "toy.src:a" and t.params == {"suffix": ""} and t.atoms == {"upper": "toy-upper/1"} assert t.input("file").name == "a.txt" and t.cost.peak_bytes == 30 @@ -175,73 +175,73 @@ def test_describe_fills_defaults_and_refuses_unknown_parameters(tmp_path): def test_execute_commits_then_hits(tmp_path): - store = Store(tmp_path / "store") - env = Env(store, {"files": toy_files(tmp_path / "in", {"a": "one\n\nFAIL\n?q"})}) + depot = Depot(tmp_path / "depot") + env = Env(depot, {"files": toy_files(tmp_path / "in", {"a": "one\n\nFAIL\n?q"})}) t = describe(Src(), "a", None, env) - ex = execute(t, store, toy_stages()) + ex = execute(t, depot, toy_stages()) assert (ex.status, ex.result_status) == ("ran", "partial") and ex.cost["cpuSeconds"] >= 0 - doc = store.result(t.key) + doc = depot.result(t.key) assert [i["status"] for i in doc["items"]] == ["exported", "contained", "failed", "unsupported"] arts = {a["id"]: a for a in doc["artifacts"]} assert sorted(arts) == ["F-a:1#data", "toy.src:a#all.txt"] - assert store.read(arts["toy.src:a#all.txt"]["content"]["sha256"]) == b"ONE\n\nFAIL\n?Q" + assert depot.read(arts["toy.src:a#all.txt"]["content"]["sha256"]) == b"ONE\n\nFAIL\n?Q" assert "task" not in doc and doc["keyParts"] == t.key_parts() - assert execute(t, store, toy_stages()).status == "hit" - assert execute(t, store, toy_stages(), force=True).status == "ran" # identical output: accepted + assert execute(t, depot, toy_stages()).status == "hit" + assert execute(t, depot, toy_stages(), force=True).status == "ran" # identical output: accepted def test_execute_refuses_other_versions_and_unknown_stages(tmp_path): - store = Store(tmp_path / "store") - env = Env(store, {"files": toy_files(tmp_path / "in", {"a": "x"})}) + depot = Depot(tmp_path / "depot") + env = Env(depot, {"files": toy_files(tmp_path / "in", {"a": "x"})}) t = describe(Src(), "a", None, env) with pytest.raises(IncompatibleTask, match="version 2 here"): - execute(t, store, toy_stages(Src=2)) + execute(t, depot, toy_stages(Src=2)) with pytest.raises(IncompatibleTask, match="no stage"): - execute(t, store, {}) - assert store.result(t.key) is None + execute(t, depot, {}) + assert depot.result(t.key) is None def test_execute_verifies_inputs_and_caches_no_task_failure(tmp_path): - store = Store(tmp_path / "store") + depot = Depot(tmp_path / "depot") files = toy_files(tmp_path / "in", {"a": "x", "b": "CRASH"}) - env = Env(store, {"files": files}) + env = Env(depot, {"files": files}) ta, tb = describe(Src(), "a", None, env), describe(Src(), "b", None, env) with pytest.raises(RuntimeError, match="crash requested"): - execute(tb, store, toy_stages()) - assert store.result(tb.key) is None + execute(tb, depot, toy_stages()) + assert depot.result(tb.key) is None Path(files["a"]).write_bytes(b"y") # same size, other content: the input is refused with pytest.raises(InputMissing, match="other content"): - execute(ta, store, toy_stages()) - assert store.result(ta.key) is None + execute(ta, depot, toy_stages()) + assert depot.result(ta.key) is None def test_a_stage_whose_output_is_not_a_function_of_its_key_is_caught(tmp_path): class Clock(Src): n = 0 - def run(self, task, store): + def run(self, task, depot): Clock.n += 1 aid = contract.artifact_id(task.id, "all.txt") return Output( [ contract.artifact( - aid, store.add(str(Clock.n).encode(), "txt"), contract.provenance(task), {"kind": "toy.text"} + aid, depot.add(str(Clock.n).encode(), "txt"), contract.provenance(task), {"kind": "toy.text"} ) ], [], ) - store = Store(tmp_path / "store") - env = Env(store, {"files": toy_files(tmp_path / "in", {"a": "x"})}) + depot = Depot(tmp_path / "depot") + env = Env(depot, {"files": toy_files(tmp_path / "in", {"a": "x"})}) t = describe(Clock(), "a", None, env) - execute(t, store, {"toy.src": Clock()}) - with pytest.raises(StoreConflict): - execute(t, store, {"toy.src": Clock()}, force=True) + execute(t, depot, {"toy.src": Clock()}) + with pytest.raises(DepotConflict): + execute(t, depot, {"toy.src": Clock()}, force=True) def test_env_pending_and_failed_tasks(tmp_path): - store = Store(tmp_path / "store") - env = Env(store, {"files": {"a": "unused", "b": "unused"}}) + depot = Depot(tmp_path / "depot") + env = Env(depot, {"files": {"a": "unused", "b": "unused"}}) env.waiting("toy.src:a") env.waiting("toy.src:b", failed=True) with pytest.raises(Pending) as e: @@ -275,7 +275,7 @@ def uses(self, subject, env): want = env.params["classes"] return {k: v for k, v in self.ATOMS.items() if want is None or k in want} - env = Env(Store(tmp_path / "s")) + env = Env(Depot(tmp_path / "s")) assert describe(Selective(), "s", None, env).atoms == {"a": "x/1", "b": "y/1"} t = describe(Selective(), "s", {"classes": ["b"]}, env) assert isinstance(t, Task) and t.atoms == {"b": "y/1"} and env.params == {} @@ -289,7 +289,7 @@ class _Spawner(Stage): def subjects(self, env): return ["s"] - def run(self, task, store): + def run(self, task, depot): import subprocess import sys @@ -301,8 +301,8 @@ def test_the_cost_counts_the_child_processes(tmp_path): from nnnotes.pipeline.contract import Task from nnnotes.pipeline.stages import children_cpu - store = Store(tmp_path / "s") - ex = execute(Task("toy.spawn", 1, "s"), store, registry(_Spawner())) + depot = Depot(tmp_path / "s") + ex = execute(Task("toy.spawn", 1, "s"), depot, registry(_Spawner())) cost = ex.cost if children_cpu() is None: # the platform does not report child processes assert cost["childCpuSeconds"] is None diff --git a/tests/pipeline/test_views.py b/tests/pipeline/test_views.py index 3137335..dd9dda9 100644 --- a/tests/pipeline/test_views.py +++ b/tests/pipeline/test_views.py @@ -12,10 +12,10 @@ from nnnotes.pipeline import contract, views from nnnotes.pipeline.contract import Input +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.orchestrate import Orchestrator from nnnotes.pipeline.plan import plan from nnnotes.pipeline.stages import Env, Output, Pending, Stage, describe, execute, registry -from nnnotes.pipeline.store import Store from nnnotes.pipeline.views import AddressIndex, Obj, Recorder, ViewError, ViewStage from nnnotes.source import languages @@ -757,12 +757,12 @@ def subjects(self, env): return sorted(env.fact("catalogs")) def inputs(self, subject, env): - return [Input("doc", env.store.put(self.data), len(self.data), None, ({"kind": "store"},))] + return [Input("doc", env.depot.put(self.data), len(self.data), None, ({"kind": "depot"},))] - def run(self, task, store): + def run(self, task, depot): rec = contract.artifact( contract.artifact_id(task.id, "addresses"), - store.add(store.input_bytes(task.input("doc")), "json"), + depot.add(depot.input_bytes(task.input("doc")), "json"), contract.provenance(task), {"kind": "link.addresses", "format": "json"}, ) @@ -787,12 +787,12 @@ def write_master(d: Path, tables: dict) -> Path: return d -def env_with(store, doc: dict, master, **facts) -> Env: +def env_with(depot, doc: dict, master, **facts) -> Env: """An environment where link.addresses:main has run with the address table `doc`.""" link = FakeAddresses(doc) - env = Env(store, {"catalogs": {"main": []}, "master": master, **facts}) + env = Env(depot, {"catalogs": {"main": []}, "master": master, **facts}) t = describe(link, "main", None, env) - execute(t, store, {link.name: link}) + execute(t, depot, {link.name: link}) env.done(t.id, t.key) return env @@ -810,29 +810,29 @@ def test_one_stage_per_view(): def test_view_stage_subjects_and_dependencies(tmp_path): - store, mdir = Store(tmp_path / "s"), write_master(tmp_path / "m", STAGE_TABLES) + depot, mdir = Depot(tmp_path / "s"), write_master(tmp_path / "m", STAGE_TABLES) stage = ViewStage("cards", views.load_rules()) - assert stage.subjects(Env(store, {"catalogs": {"main": []}, "master": None})) == [] - assert stage.subjects(Env(store, {"catalogs": {"main": []}, "master": mdir, "views": ["stamps"]})) == [] - assert stage.subjects(Env(store, {"catalogs": {"main": [], "b": []}, "master": mdir, "views": ["cards"]})) == [ + assert stage.subjects(Env(depot, {"catalogs": {"main": []}, "master": None})) == [] + assert stage.subjects(Env(depot, {"catalogs": {"main": []}, "master": mdir, "views": ["stamps"]})) == [] + assert stage.subjects(Env(depot, {"catalogs": {"main": [], "b": []}, "master": mdir, "views": ["cards"]})) == [ "b", "main", ] - env = Env(store, {"catalogs": {"main": []}, "master": mdir}) + env = Env(depot, {"catalogs": {"main": []}, "master": mdir}) assert stage.subjects(env) == ["main"] and stage.depends("main", env) == ["link.addresses:main"] with pytest.raises(Pending) as e: describe(stage, "main", None, env) assert e.value.task == "link.addresses:main" (mdir / "MasterBand.json").unlink() with pytest.raises(ViewError, match="MasterBand is not in the master data"): - describe(stage, "main", None, env_with(store, addresses_doc(stage_entries()), mdir)) + describe(stage, "main", None, env_with(depot, addresses_doc(stage_entries()), mdir)) def test_view_stage_runs_on_its_inputs_alone(tmp_path): rules = views.load_rules() stage = ViewStage("cards", rules) mdir, doc = write_master(tmp_path / "m", STAGE_TABLES), addresses_doc(stage_entries()) - a = Store(tmp_path / "a") + a = Depot(tmp_path / "a") task = describe(stage, "main", None, env_with(a, doc, mdir)) assert (task.id, task.version, task.atoms, task.context) == ("view.cards:main", 1, {}, {}) assert [i.role for i in task.inputs] == [ @@ -851,26 +851,26 @@ def test_view_stage_runs_on_its_inputs_alone(tmp_path): assert view == views.build(rules, "cards", views.MasterTables(mdir), AddressIndex.from_addresses(doc)) assert res["artifacts"][0]["semantics"]["facts"]["rows"] == 3 assert res["artifacts"][0]["semantics"]["facts"]["gaps"] == 0 - # the task description runs in another store holding only its stored inputs: the same result - b = Store(tmp_path / "b") + # the task description runs in another depot holding only its stored inputs: the same result + b = Depot(tmp_path / "b") for i in task.inputs: - if i.locators[0]["kind"] == "store": + if i.locators[0]["kind"] == "depot": b.put(a.read(i.sha256)) again = contract.Task.from_json(json.loads(json.dumps(task.to_json()))) execute(again, b, {stage.name: ViewStage("cards", rules)}) assert contract.encode(b.result(task.key)) == contract.encode(res) # described again elsewhere: the same key; run again: the same output - assert describe(stage, "main", None, env_with(Store(tmp_path / "c"), doc, mdir)).key == task.key + assert describe(stage, "main", None, env_with(Depot(tmp_path / "c"), doc, mdir)).key == task.key assert execute(task, a, {stage.name: stage}, force=True).status == "ran" def test_view_keys_follow_only_their_own_inputs(tmp_path): - store, mdir = Store(tmp_path / "s"), write_master(tmp_path / "m", STAGE_TABLES) + depot, mdir = Depot(tmp_path / "s"), write_master(tmp_path / "m", STAGE_TABLES) rules = views.load_rules() entries = stage_entries() def keys(entries=entries, rules=rules): - env = env_with(store, addresses_doc(entries), mdir) + env = env_with(depot, addresses_doc(entries), mdir) return [describe(ViewStage(n, rules), "main", None, env).key for n in ("cards", "stamps")] cards, stamps = keys() @@ -905,20 +905,20 @@ def keys(entries=entries, rules=rules): def test_png_encoding_never_reaches_view_keys(tmp_path): - store, mdir = Store(tmp_path / "s"), write_master(tmp_path / "m", STAGE_TABLES) + depot, mdir = Depot(tmp_path / "s"), write_master(tmp_path / "m", STAGE_TABLES) stage = ViewStage("cards", views.load_rules()) assert "png" not in stage.PARAMS with pytest.raises(ValueError, match="unknown parameters png"): stage.normalize({"png": {"level": 9}}) - env = env_with(store, addresses_doc(stage_entries()), mdir) + env = env_with(depot, addresses_doc(stage_entries()), mdir) key = describe(stage, "main", None, env).key oid = card_entries()["MemberCard/10/member_full"][1].id for level in (6, 9): # exports of the objects with other PNG bytes t = contract.Task("unity.export", 1, "b", {"png": {"encoder": "pillow", "level": level}}) rec = contract.artifact( - f"{oid}#image", store.add(b"png %d" % level, "png"), contract.provenance(t), {"kind": "texture.image"} + f"{oid}#image", depot.add(b"png %d" % level, "png"), contract.provenance(t), {"kind": "texture.image"} ) - store.commit(contract.result(t, [rec], [contract.item(oid, "exported", artifacts=[rec["id"]])])) + depot.commit(contract.result(t, [rec], [contract.item(oid, "exported", artifacts=[rec["id"]])])) env.done(t.id, t.key) assert describe(stage, "main", None, env).key == key @@ -937,11 +937,11 @@ def artifact(self, aid): def test_derived_view_document_has_layout_paths(tmp_path): - store, mdir = Store(tmp_path / "s"), write_master(tmp_path / "m", STAGE_TABLES) + depot, mdir = Depot(tmp_path / "s"), write_master(tmp_path / "m", STAGE_TABLES) stage = ViewStage("cards", views.load_rules()) - task = describe(stage, "main", None, env_with(store, addresses_doc(stage_entries()), mdir)) - execute(task, store, {stage.name: stage}) - res = store.result(task.key) + task = describe(stage, "main", None, env_with(depot, addresses_doc(stage_entries()), mdir)) + execute(task, depot, {stage.name: stage}) + res = depot.result(task.key) full = next( x.id for x in card_entries()["MemberCard/10/member_full"] if x.name == "member_full" and x.cls == "Sprite" ) @@ -949,7 +949,7 @@ def test_derived_view_document_has_layout_paths(tmp_path): f"{full}#meta": "MemberCard/10/member_full[member_full].png.meta.json", f"{full}#image": "MemberCard/10/member_full[member_full].png", } - out = stage.derived(task.id, res, store, Paths(placed)) + out = stage.derived(task.id, res, depot, Paths(placed)) assert [p for p, _ in out] == ["views/cards.json"] doc = contract.loads(out[0][1]) assert out[0][1] == contract.encode(doc) and doc["layout"] == "original" @@ -960,7 +960,7 @@ def test_derived_view_document_has_layout_paths(tmp_path): assert "path" not in r10["roles"]["skill_sprite"] # missing-key: no object assert "path" not in r10["roles"]["movie"][0] # everything else is the view document - view = contract.loads(store.read(res["artifacts"][0]["content"]["sha256"])) + view = contract.loads(depot.read(res["artifacts"][0]["content"]["sha256"])) for row in doc["rows"]: for e in (x for v in row["roles"].values() for x in (v if isinstance(v, list) else [v])): e.pop("path", None), e.pop("files", None) @@ -970,20 +970,20 @@ def test_derived_view_document_has_layout_paths(tmp_path): def test_views_plan_and_run_after_the_address_table(tmp_path): - store, mdir = Store(tmp_path / "s"), write_master(tmp_path / "m", STAGE_TABLES) + depot, mdir = Depot(tmp_path / "s"), write_master(tmp_path / "m", STAGE_TABLES) vs = views.stages() reg = registry(FakeAddresses(addresses_doc(stage_entries())), *vs) pipeline = ["link.addresses"] + [s.name for s in vs] facts = {"catalogs": {"main": []}, "master": mdir, "views": ["cards", "stamps"]} - nodes = {n["id"]: n for n in plan(reg, pipeline, store, facts=facts).doc["nodes"]} + nodes = {n["id"]: n for n in plan(reg, pipeline, depot, facts=facts).doc["nodes"]} assert sorted(nodes) == ["link.addresses:main", "view.cards:main", "view.stamps:main"] assert nodes["view.cards:main"]["status"] == "unknown" assert nodes["view.cards:main"]["reasons"] == [{"code": "pending", "task": "link.addresses:main"}] - first = Orchestrator(store, reg, log=lambda m: None).run(pipeline, facts=facts) + first = Orchestrator(depot, reg, log=lambda m: None).run(pipeline, facts=facts) assert {t: v["status"] for t, v in first.tasks.items()} == dict.fromkeys(nodes, "ran") - second = Orchestrator(store, reg, log=lambda m: None).run(pipeline, facts=facts) + second = Orchestrator(depot, reg, log=lambda m: None).run(pipeline, facts=facts) assert {t: v["status"] for t, v in second.tasks.items()} == dict.fromkeys(nodes, "hit") - assert plan(reg, pipeline, store, facts=facts).check() == 0 + assert plan(reg, pipeline, depot, facts=facts).check() == 0 def test_views_import_neither_unitypy_nor_numpy(): diff --git a/tests/pipeline/test_voices.py b/tests/pipeline/test_voices.py index be7ab29..3c61e00 100644 --- a/tests/pipeline/test_voices.py +++ b/tests/pipeline/test_voices.py @@ -15,8 +15,8 @@ from nnnotes.formats import acb from nnnotes.pipeline import contract, voices from nnnotes.pipeline.contract import Input, Task +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.stages import Env, Pending, describe, execute, registry -from nnnotes.pipeline.store import Store from nnnotes.pipeline.views import AddressIndex, ViewError from nnnotes.source import languages from tests.pipeline.test_views import FakeAddresses, addresses_doc, write_master @@ -420,8 +420,8 @@ def audio_task(sheet: str, sheets=None) -> Task: return Task("cri.audio", 1, sha, {"flac": {"level": 8}}, {}, (Input("acb", sha, len(data)),)) -def audio_result(store, sheet: str, unsupported: bool = False, sheets=None) -> dict: - """A cri.audio result of a sheet, committed to `store` when given.""" +def audio_result(depot, sheet: str, unsupported: bool = False, sheets=None) -> dict: + """A cri.audio result of a sheet, committed to `depot` when given.""" sheets = sheets or SHEETS t = ( audio_task(sheet, sheets) @@ -442,8 +442,8 @@ def audio_result(store, sheet: str, unsupported: bool = False, sheets=None) -> d for n, name, file, samples in stream_files(sheet, sheets): data = f"{sheet}/{file}".encode() content = ( - store.add(data, "flac") - if store is not None + depot.add(data, "flac") + if depot is not None else {"sha256": contract.sha256(data), "size": len(data), "mediaType": "audio/flac", "ext": "flac"} ) arts.append( @@ -459,18 +459,18 @@ def audio_result(store, sheet: str, unsupported: bool = False, sheets=None) -> d ) ) res = contract.result(t, arts, [contract.item(t.id, "exported", artifacts=[a["id"] for a in arts], cls="ACB")]) - if store is not None: - store.commit(res) + if depot is not None: + depot.commit(res) return res -def audio(store=None, sheets=None) -> dict: +def audio(depot=None, sheets=None) -> dict: sheets = sheets or SHEETS out = { - s: voices.sheet_audio(audio_result(store, s, sheets=sheets), acb.cue_streams(sheet_acb(s, sheets))) + s: voices.sheet_audio(audio_result(depot, s, sheets=sheets), acb.cue_streams(sheet_acb(s, sheets))) for s in sheets } - out["VoiceBroken"] = voices.sheet_audio(audio_result(store, "VoiceBroken", unsupported=True), None) + out["VoiceBroken"] = voices.sheet_audio(audio_result(depot, "VoiceBroken", unsupported=True), None) return out @@ -696,7 +696,7 @@ def test_rules_are_checked(): # ---------------------------------------------------------------- the stage -def planning(store, tmp: Path, keys=KEYS, skip=(), sheets=None, tables=None, doc=None, results=None) -> Env: +def planning(depot, tmp: Path, keys=KEYS, skip=(), sheets=None, tables=None, doc=None, results=None) -> Env: """An environment where link.addresses:main has run (the catalog keys `keys`, or the address table `doc`) and the cri.audio tasks of the decoded sheets have results; the raw ACBs are files under their catalog keys. `results(env)`: commits further results (unity.export) before the environment is returned.""" @@ -709,7 +709,7 @@ def planning(store, tmp: Path, keys=KEYS, skip=(), sheets=None, tables=None, doc p = tmp / "raw" / stable.replace("/", "_") p.parent.mkdir(parents=True, exist_ok=True) p.write_bytes(data) - sha, size = store.identify(p, "raw", stable) + sha, size = depot.identify(p, "raw", stable) raw[stable] = Input("raw", sha, size, f"{stable}_{'0' * 32}", ({"kind": "file", "path": str(p)},)) deps[f"Cri/Sound/{sheet}"] = [stable] lid = {s: f"remote:{i}" for i, s in enumerate(sorted(raw))} @@ -720,13 +720,13 @@ def planning(store, tmp: Path, keys=KEYS, skip=(), sheets=None, tables=None, doc ] cat = {"rawFiles": [{"stable": s, "locations": [lid[s]]} for s in sorted(raw)], "locations": locations} link = FakeAddresses(doc or addresses_doc({k: [] for k in keys})) - env = Env(store, {"catalogs": {"main": []}, "master": mdir, "views": ["voices"], "raw": raw, "index": cat}) + env = Env(depot, {"catalogs": {"main": []}, "master": mdir, "views": ["voices"], "raw": raw, "index": cat}) t = describe(link, "main", None, env) - execute(t, store, {link.name: link}) + execute(t, depot, {link.name: link}) env.done(t.id, t.key) for sheet in sheets: if sheet not in skip: - res = audio_result(store, sheet, sheets=sheets) + res = audio_result(depot, sheet, sheets=sheets) env.done(audio_task(sheet, sheets).id, res["key"]) b = Task("cri.audio", 1, raw["cri_assets_cri/sound/voicebroken"].sha256, {}, {}, ()) broken = contract.result( @@ -738,7 +738,7 @@ def planning(store, tmp: Path, keys=KEYS, skip=(), sheets=None, tables=None, doc ) ], ) - store.commit(broken) + depot.commit(broken) env.done(b.id, broken["key"]) if results is not None: results(env) @@ -748,19 +748,19 @@ def planning(store, tmp: Path, keys=KEYS, skip=(), sheets=None, tables=None, doc def run_stage(env): stage = voices.VoiceStage() task = describe(stage, "main", None, env) - ex = execute(task, env.store, registry(stage)) - res = env.store.result(task.key) - return stage, task, ex, res, contract.loads(env.store.read(res["artifacts"][0]["content"]["sha256"])) + ex = execute(task, env.depot, registry(stage)) + res = env.depot.result(task.key) + return stage, task, ex, res, contract.loads(env.depot.read(res["artifacts"][0]["content"]["sha256"])) def test_stage_subjects_inputs_and_waits(tmp_path): - store = Store(tmp_path / "s") + depot = Depot(tmp_path / "s") stage = voices.VoiceStage() assert stage.name == "view.voices" and stage.version == 2 assert stage.after == ("link.addresses", "unity.export", "cri.audio") - assert stage.subjects(Env(store, {"catalogs": {"main": []}, "master": None})) == [] - assert stage.subjects(Env(store, {"catalogs": {"main": []}, "master": tmp_path, "views": ["cards"]})) == [] - env = planning(store, tmp_path) + assert stage.subjects(Env(depot, {"catalogs": {"main": []}, "master": None})) == [] + assert stage.subjects(Env(depot, {"catalogs": {"main": []}, "master": tmp_path, "views": ["cards"]})) == [] + env = planning(depot, tmp_path) assert stage.subjects(env) == ["main"] task = describe(stage, "main", None, env) roles = sorted(i.role for i in task.inputs) @@ -774,7 +774,7 @@ def test_stage_subjects_inputs_and_waits(tmp_path): "rules", "sheets", ] - sheets = contract.loads(store.read(task.input("sheets").sha256)) + sheets = contract.loads(depot.read(task.input("sheets").sha256)) assert sorted(sheets) == sorted([*SHEETS, "VoiceBroken"]) assert sheets["VoiceSystem_01"] == { "task": audio_task("VoiceSystem_01").id, @@ -782,7 +782,7 @@ def test_stage_subjects_inputs_and_waits(tmp_path): } assert sorted(task.context["audio"]) == sorted(stage.depends("main", env)[1:]) assert stage.depends("main", env)[0] == "link.addresses:main" - waiting = Env(store, env.facts) + waiting = Env(depot, env.facts) for tid in env.tasks("link.addresses"): waiting.done(tid, env.key(tid)) waiting.waiting(audio_task("VoiceSystem_01").id) @@ -791,12 +791,12 @@ def test_stage_subjects_inputs_and_waits(tmp_path): def test_stage_document_equals_the_index_of_its_inputs(tmp_path): - store = Store(tmp_path / "s") - env = planning(store, tmp_path) + depot = Depot(tmp_path / "s") + env = planning(depot, tmp_path) stage, task, ex, res, doc = run_stage(env) assert (ex.status, ex.result_status) == ("ran", "ok") and res["items"] == [] assert [a["id"] for a in res["artifacts"]] == ["view.voices:main#view"] - want = voices.build(voices.load_rules(), TABLES, index(), audio(store)) + want = voices.build(voices.load_rules(), TABLES, index(), audio(depot)) assert doc["rows"] == want["rows"] and doc["coverage"] == want["coverage"] assert sorted(doc["snapshot"]["tables"]) == voices.tables_of(stage.rules) assert doc["snapshot"]["sheets"]["VoiceBroken"]["status"] == "unsupported" @@ -809,20 +809,20 @@ def test_stage_document_equals_the_index_of_its_inputs(tmp_path): } # described again: the same key; run again: the same bytes assert describe(voices.VoiceStage(), "main", None, env).key == task.key - again = execute(task, store, registry(stage), force=True) - assert again.status == "ran" and contract.encode(store.result(task.key)) == contract.encode(res) + again = execute(task, depot, registry(stage), force=True) + assert again.status == "ran" and contract.encode(depot.result(task.key)) == contract.encode(res) # the task crosses to another process as JSON same = contract.Task.from_json(json.loads(json.dumps(task.to_json()))) assert same.key == task.key def test_stage_key_follows_the_audio_and_the_catalog(tmp_path): - store = Store(tmp_path / "s") - env = planning(store, tmp_path) + depot = Depot(tmp_path / "s") + env = planning(depot, tmp_path) key = describe(voices.VoiceStage(), "main", None, env).key - other = planning(Store(tmp_path / "t"), tmp_path / "t", keys=KEYS + ["Cri/Sound/VoiceGone"]) + other = planning(Depot(tmp_path / "t"), tmp_path / "t", keys=KEYS + ["Cri/Sound/VoiceGone"]) assert describe(voices.VoiceStage(), "main", None, other).key != key - fewer = planning(Store(tmp_path / "u"), tmp_path / "u", skip=("InitialVoice",)) + fewer = planning(Depot(tmp_path / "u"), tmp_path / "u", skip=("InitialVoice",)) k2 = describe(voices.VoiceStage(), "main", None, fewer).key assert k2 != key _, _, _, _, doc = run_stage(fewer) @@ -841,12 +841,12 @@ def objects(self, oid): def test_derived_document_has_the_stream_paths(tmp_path): - store = Store(tmp_path / "s") - env = planning(store, tmp_path) + depot = Depot(tmp_path / "s") + env = planning(depot, tmp_path) stage, task, _, res, doc = run_stage(env) tid = audio_task("VoiceSystem_01").id placed = {f"{tid}#Growth_Alpha_RankUp_01.flac": "Cri/Sound/VoiceSystem_01/Growth_Alpha_RankUp_01.flac"} - out = stage.derived(task.id, res, store, Paths(placed)) + out = stage.derived(task.id, res, depot, Paths(placed)) assert [p for p, _ in out] == ["views/voices.json"] d = contract.loads(out[0][1]) assert d["layout"] == "original" @@ -875,18 +875,18 @@ def keys(self, prefix=""): return [k for k in KEYS if k.startswith(prefix)] -def export_dir(tmp: Path, store) -> Path: +def export_dir(tmp: Path, depot) -> Path: """An export directory: views/voices.json with paths and the stream files at them.""" out = tmp / "out" placed = {} - doc = voices.build(voices.load_rules(), TABLES, index(), audio(store)) + doc = voices.build(voices.load_rules(), TABLES, index(), audio(depot)) for r in doc["rows"]: for s in (r.get("audio") or {}).get("streams", []): sheet = r["sound"]["sheet"] placed[s["artifact"]] = f"Cri/Sound/{sheet}/{s['file']}" f = out / placed[s["artifact"]] f.parent.mkdir(parents=True, exist_ok=True) - f.write_bytes(store.read(s["sha256"])) + f.write_bytes(depot.read(s["sha256"])) d = voices.with_paths(doc, Paths(placed)) (out / "views").mkdir(parents=True) (out / "views" / "voices.json").write_bytes(contract.encode(d)) diff --git a/tests/pipeline/test_voices_story.py b/tests/pipeline/test_voices_story.py index 5736017..8fd0dd3 100644 --- a/tests/pipeline/test_voices_story.py +++ b/tests/pipeline/test_voices_story.py @@ -11,8 +11,8 @@ from nnnotes.content import advcommand, storyvoices from nnnotes.pipeline import contract, voices from nnnotes.pipeline.contract import Task +from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.stages import Pending, describe, execute, registry -from nnnotes.pipeline.store import Store from nnnotes.pipeline.views import AddressIndex, Obj, ViewError from nnnotes.source import languages from tests.pipeline.test_views import addresses_doc @@ -482,7 +482,7 @@ def test_story_rules_are_checked(): # ---------------------------------------------------------------- the stage -def export_results(store, env) -> dict: +def export_results(depot, env) -> dict: """unity.export results holding the exported episode parts, one bundle per episode: {asset: {part: oid}}.""" oids = {} for n, (asset, parts) in enumerate(sorted(EPISODES.items())): @@ -495,7 +495,7 @@ def export_results(store, env) -> dict: arts.append( contract.artifact( contract.artifact_id(oid, role), - store.add(data, "json" if part == "script" else "txt"), + depot.add(data, "json" if part == "script" else "txt"), contract.provenance(t), {"kind": "unity.object", "format": "json"}, ) @@ -503,7 +503,7 @@ def export_results(store, env) -> dict: items.append(contract.item(oid, "exported", artifacts=[arts[-1]["id"]])) oids.setdefault(asset, {})[part] = oid res = contract.result(t, arts, items) - store.commit(res) + depot.commit(res) env.done(t.id, res["key"]) return oids @@ -527,24 +527,24 @@ def address_doc() -> dict: return doc -def story_env(store, tmp_path): +def story_env(depot, tmp_path): return planning( - store, + depot, tmp_path, keys=catalog_keys(), sheets=SHEETS2, tables=tables(), doc=address_doc(), - results=lambda env: export_results(store, env), + results=lambda env: export_results(depot, env), ) def test_stage_reads_the_exported_episodes(tmp_path): - store = Store(tmp_path / "s") - env = story_env(store, tmp_path) + depot = Depot(tmp_path / "s") + env = story_env(depot, tmp_path) stage = voices.VoiceStage() task = describe(stage, "main", None, env) - eps = contract.loads(store.read(task.input("episodes").sha256)) + eps = contract.loads(depot.read(task.input("episodes").sha256)) assert sorted(eps) == sorted(EPISODES) assert sorted(eps["adv_script_x_02"]) == ["script", "sound"] x01 = eps["adv_script_x_01"]["script"] @@ -557,44 +557,44 @@ def test_stage_reads_the_exported_episodes(tmp_path): assert sorted(i.role for i in task.inputs if i.role.startswith("adv:")) == sorted(f"adv:{s}" for s in shas) deps = stage.depends("main", env) assert sorted(d for d in deps if d.startswith("unity.export:")) == sorted(f"unity.export:adv_{a}" for a in EPISODES) - ex = execute(task, store, registry(stage)) + ex = execute(task, depot, registry(stage)) assert (ex.status, ex.result_status) == ("ran", "ok") - res = store.result(task.key) - doc = contract.loads(store.read(res["artifacts"][0]["content"]["sha256"])) + res = depot.result(task.key) + doc = contract.loads(depot.read(res["artifacts"][0]["content"]["sha256"])) want = voices.build( - RULES, tables(), AddressIndex({k: [] for k in catalog_keys()}), audio(store, SHEETS2), story=story() + RULES, tables(), AddressIndex({k: [] for k in catalog_keys()}), audio(depot, SHEETS2), story=story() ) assert doc["rows"] == want["rows"] and doc["episodes"] == want["episodes"] assert doc["coverage"] == want["coverage"] assert doc["snapshot"]["episodes"] == task.input("episodes").sha256 assert STORY_SHEET in doc["snapshot"]["sheets"] assert describe(voices.VoiceStage(), "main", None, env).key == task.key - again = execute(task, store, registry(stage), force=True) - assert again.status == "ran" and contract.encode(store.result(task.key)) == contract.encode(res) + again = execute(task, depot, registry(stage), force=True) + assert again.status == "ran" and contract.encode(depot.result(task.key)) == contract.encode(res) def test_stage_waits_for_the_exports(tmp_path): - store = Store(tmp_path / "s") - env = story_env(store, tmp_path) + depot = Depot(tmp_path / "s") + env = story_env(depot, tmp_path) env.waiting("unity.export:adv_adv_script_x_01") with pytest.raises(Pending, match="unity.export"): describe(voices.VoiceStage(), "main", None, env) def test_stage_without_exports_has_no_story_rows(tmp_path): - store = Store(tmp_path / "s") - env = planning(store, tmp_path, keys=catalog_keys(), sheets=SHEETS2, tables=tables(), doc=address_doc()) + depot = Depot(tmp_path / "s") + env = planning(depot, tmp_path, keys=catalog_keys(), sheets=SHEETS2, tables=tables(), doc=address_doc()) stage = voices.VoiceStage() task = describe(stage, "main", None, env) assert not [i for i in task.inputs if i.role.startswith("adv:")] - execute(task, store, registry(stage)) - doc = contract.loads(store.read(store.result(task.key)["artifacts"][0]["content"]["sha256"])) + execute(task, depot, registry(stage)) + doc = contract.loads(depot.read(depot.result(task.key)["artifacts"][0]["content"]["sha256"])) assert story_rows(doc) == [] assert {e["status"] for e in doc["episodes"] if e["asset"] in EPISODES} == {"not-exported"} # ---------------------------------------------------------------- the command -def export_dir(tmp_path, store): +def export_dir(tmp_path, depot): out = tmp_path / "out" doc = build() placed = {} diff --git a/tests/site/test_live_options_web.py b/tests/site/test_live_options_web.py index c0c9db1..59381c3 100644 --- a/tests/site/test_live_options_web.py +++ b/tests/site/test_live_options_web.py @@ -161,7 +161,7 @@ def test_live_option_flag(tmp_path, capsys, monkeypatch): master = option_tables(write_master(tmp_path / "m")) monkeypatch.setenv("NNNOTES_SERVERS_ZZ_CDN", (tmp_path / "cdn").as_uri()) base = [ - "--cache", + "--depot", str(tmp_path / "c"), "--region", "zz", diff --git a/tests/site/test_pipeline.py b/tests/site/test_pipeline.py index fd8ea51..91b227d 100644 --- a/tests/site/test_pipeline.py +++ b/tests/site/test_pipeline.py @@ -151,15 +151,15 @@ def test_catalog_cached(tmp_path): cat = Catalog(synth.CatalogWriter().build([("k", synth.remote("b_1.bundle"), [])]), tmp_path) b = Bundle(1, synth.remote("b_1.bundle"), "b_1.bundle", True) - assert cat.cached(b) is None + assert cat.downloaded(b) is None (tmp_path / "bundles").mkdir() (tmp_path / "bundles" / "b_1.bundle").write_bytes(b"UnityFS\0x") - assert cat.cached(b) == tmp_path / "bundles" / "b_1.bundle" == cat.fetch(b) + assert cat.downloaded(b) == tmp_path / "bundles" / "b_1.bundle" == cat.fetch(b) e = {"internal_id": synth.remote("x/raw_1")} - assert cat.cached_raw(e) is None and cat.cached_raw({"internal_id": "Assets/x"}) is None + assert cat.downloaded_raw(e) is None and cat.downloaded_raw({"internal_id": "Assets/x"}) is None (tmp_path / "raw" / "asset" / "Android" / "x").mkdir(parents=True) (tmp_path / "raw" / "asset" / "Android" / "x" / "raw_1").write_bytes(b"@UTF") - assert cat.cached_raw(e) == cat.fetch_raw(e) + assert cat.downloaded_raw(e) == cat.fetch_raw(e) def test_pipeline_window_keeps_the_read_set_slots_busy(): diff --git a/tests/site/test_readsets.py b/tests/site/test_readsets.py index 3ba5b39..d24e4ee 100644 --- a/tests/site/test_readsets.py +++ b/tests/site/test_readsets.py @@ -456,7 +456,7 @@ def test_web_pair_that_no_region_has_is_a_usage_error(tmp_path, capsys, monkeypa monkeypatch.setenv("NNNOTES_SERVERS_ZZ_CDN", (tmp_path / "cdn").as_uri()) monkeypatch.setattr(web, "build", lambda *a, **k: pytest.fail("nothing may be built")) base = [ - "--cache", + "--depot", str(tmp_path / "c"), "--region", "zz", diff --git a/tests/source/test_catalog.py b/tests/source/test_catalog.py index 795ce92..0349c10 100644 --- a/tests/source/test_catalog.py +++ b/tests/source/test_catalog.py @@ -102,34 +102,34 @@ def make_apk(path): return path -def test_closure_fetch_and_cache(tmp_path): +def test_closure_fetch_and_download(tmp_path): cdn = make_cdn(tmp_path / "cdn") - cat = Catalog(remote_catalog(), tmp_path / "cache", cdn=cdn + "/", bundle_key=KEY) + cat = Catalog(remote_catalog(), tmp_path / "downloads", cdn=cdn + "/", bundle_key=KEY) assert cat.keys("Char/") == ["Char/A", "Char/B"] assert cat.has("Story/物語") and not cat.has("Char/C") assert sorted(b.name for b in cat.resolve("Char/A")) == ["a_01.bundle", "b_02.bundle"] paths = cat.fetch_key("Char/A") assert sorted(p.name for p in paths) == ["a_01.bundle", "b_02.bundle"] for p in paths: - assert p.parent == tmp_path / "cache" / "bundles" + assert p.parent == tmp_path / "downloads" / "bundles" assert p.read_bytes() == synth.fake_bundle(p.name) - # cached bundles are not fetched again + # downloaded bundles are not fetched again for p in (tmp_path / "cdn" / "asset" / "Android").iterdir(): p.unlink() - assert cat.fetch_key("Char/B") == [tmp_path / "cache" / "bundles" / "b_02.bundle"] + assert cat.fetch_key("Char/B") == [tmp_path / "downloads" / "bundles" / "b_02.bundle"] with pytest.raises(KeyError): cat.resolve("Char/C") def test_encrypted_bundle_needs_key(tmp_path): - cat = Catalog(remote_catalog(), tmp_path / "cache", cdn=make_cdn(tmp_path / "cdn")) + cat = Catalog(remote_catalog(), tmp_path / "downloads", cdn=make_cdn(tmp_path / "cdn")) with pytest.raises(RuntimeError, match="no bundle key"): cat.fetch_key("Char/B") def test_apk_catalog_is_merged(tmp_path): apk = make_apk(tmp_path / "base.apk") - cat = Catalog(remote_catalog(), tmp_path / "cache", cdn=make_cdn(tmp_path / "cdn"), bundle_key=KEY, apk=apk) + cat = Catalog(remote_catalog(), tmp_path / "downloads", cdn=make_cdn(tmp_path / "cdn"), bundle_key=KEY, apk=apk) assert "Shared/S" in cat.keys() (b,) = cat.resolve("Shared/S") assert not b.remote and b.offset >= APK_OFFSET_BASE @@ -141,7 +141,7 @@ def test_apk_catalog_is_merged(tmp_path): def test_apk_bundles_skipped_without_apk(tmp_path): - cat = Catalog(synth.CatalogWriter().build(APK_ENTRIES), tmp_path / "cache", bundle_key=KEY) + cat = Catalog(synth.CatalogWriter().build(APK_ENTRIES), tmp_path / "downloads", bundle_key=KEY) assert cat.fetch_key("Shared/S") == [] @@ -149,13 +149,13 @@ def test_load_downloads_catalog_once(tmp_path): cdn_root = tmp_path / "cdn" (cdn_root / "asset" / "Android").mkdir(parents=True) (cdn_root / "asset" / "Android" / "catalog_main_xx.bin").write_bytes(remote_catalog()) - cat = Catalog.load("xx", tmp_path / "cache", cdn=cdn_root.as_uri()) - assert (tmp_path / "cache" / "catalog_main_xx.bin").read_bytes() == remote_catalog() + cat = Catalog.load("xx", tmp_path / "downloads", cdn=cdn_root.as_uri()) + assert (tmp_path / "downloads" / "catalog_main_xx.bin").read_bytes() == remote_catalog() assert cat.keys("Char/") == ["Char/A", "Char/B"] (cdn_root / "asset" / "Android" / "catalog_main_xx.bin").unlink() - assert Catalog.load("xx", tmp_path / "cache").keys("Story/") == ["Story/物語"] + assert Catalog.load("xx", tmp_path / "downloads").keys("Story/") == ["Story/物語"] with pytest.raises(FileNotFoundError): - Catalog.load("yy", tmp_path / "cache") + Catalog.load("yy", tmp_path / "downloads") def test_downloads_keep_one_connection_per_thread(monkeypatch): diff --git a/tests/source/test_catalog_lazy.py b/tests/source/test_catalog_lazy.py index 60aec10..295dbbd 100644 --- a/tests/source/test_catalog_lazy.py +++ b/tests/source/test_catalog_lazy.py @@ -43,12 +43,12 @@ def f(): return f -def test_cached_bundles_read_no_settings(tmp_path): +def test_downloaded_bundles_read_no_settings(tmp_path): cdn = cdn_tree(tmp_path / "cdn") - first = Catalog(synth.CatalogWriter().build(ENTRIES), tmp_path / "cache", cdn=cdn, bundle_key=KEY) + first = Catalog(synth.CatalogWriter().build(ENTRIES), tmp_path / "downloads", cdn=cdn, bundle_key=KEY) (p,) = first.fetch_key("Char/A") again = Catalog( - synth.CatalogWriter().build(ENTRIES), tmp_path / "cache", cdn=never("cdn"), bundle_key=never("bundle key") + synth.CatalogWriter().build(ENTRIES), tmp_path / "downloads", cdn=never("cdn"), bundle_key=never("bundle key") ) assert again.fetch_key("Char/A") == [p] and again.fetch_key("Char/B") == [p] assert p.read_bytes() == synth.fake_bundle("a_01.bundle") @@ -59,7 +59,7 @@ def test_settings_are_read_once_when_a_download_needs_them(tmp_path): calls = [] cat = Catalog( synth.CatalogWriter().build(ENTRIES), - tmp_path / "cache", + tmp_path / "downloads", cdn=lambda: calls.append("cdn") or cdn, bundle_key=lambda: calls.append("key") or KEY, ) @@ -74,19 +74,23 @@ def test_settings_are_read_once_when_a_download_needs_them(tmp_path): def test_missing_setting_names_the_file_and_the_setting(tmp_path): cdn = cdn_tree(tmp_path / "cdn") cat = Catalog( - synth.CatalogWriter().build(ENTRIES), tmp_path / "cache", cdn=missing("servers.zz.cdn"), bundle_key=KEY + synth.CatalogWriter().build(ENTRIES), tmp_path / "downloads", cdn=missing("servers.zz.cdn"), bundle_key=KEY ) with pytest.raises(ConfigError) as e: cat.fetch_key("Char/A") - assert str(e.value) == "a_01.bundle is not in the cache: setting servers.zz.cdn is not set" - cat = Catalog(synth.CatalogWriter().build(ENTRIES), tmp_path / "cache", cdn=cdn, bundle_key=missing("bundle.key")) - with pytest.raises(ConfigError, match="^a_01.bundle is not in the cache: setting bundle.key is not set$"): + assert str(e.value) == "a_01.bundle is not downloaded: setting servers.zz.cdn is not set" + cat = Catalog( + synth.CatalogWriter().build(ENTRIES), tmp_path / "downloads", cdn=cdn, bundle_key=missing("bundle.key") + ) + with pytest.raises(ConfigError, match="^a_01.bundle is not downloaded: setting bundle.key is not set$"): cat.fetch_key("Char/A") def test_missing_key_fails_before_the_download(tmp_path): gone = (tmp_path / "no-such-cdn").as_uri() # a download would fail with URLError - cat = Catalog(synth.CatalogWriter().build(ENTRIES), tmp_path / "cache", cdn=gone, bundle_key=missing("bundle.key")) + cat = Catalog( + synth.CatalogWriter().build(ENTRIES), tmp_path / "downloads", cdn=gone, bundle_key=missing("bundle.key") + ) with pytest.raises(ConfigError, match="bundle.key"): cat.fetch_key("Char/A") @@ -107,7 +111,7 @@ def test_apk_bundles_read_the_key_only_when_encrypted(tmp_path): z.writestr("assets/aa/Android/p_04.bundle", synth.fake_bundle("p_04.bundle")) cat = Catalog( synth.CatalogWriter().build(ENTRIES), - tmp_path / "cache", + tmp_path / "downloads", cdn=never("cdn"), bundle_key=missing("bundle.key"), apk=apk, @@ -115,19 +119,19 @@ def test_apk_bundles_read_the_key_only_when_encrypted(tmp_path): (p,) = cat.fetch_key("Shared/P") # plain: no key assert p.read_bytes() == synth.fake_bundle("p_04.bundle") assert cat.apk_bundle("p_04").read_bytes() == synth.fake_bundle("p_04.bundle") - with pytest.raises(ConfigError, match="^s_03.bundle is not in the cache: setting bundle.key"): + with pytest.raises(ConfigError, match="^s_03.bundle is not downloaded: setting bundle.key"): cat.fetch_key("Shared/S") - with pytest.raises(ConfigError, match="^s_03.bundle is not in the cache"): + with pytest.raises(ConfigError, match="^s_03.bundle is not downloaded"): cat.apk_bundle("s_03") -def test_load_reads_the_cdn_only_for_an_uncached_catalog(tmp_path): +def test_load_reads_the_cdn_only_for_a_catalog_not_downloaded(tmp_path): cdn = cdn_tree(tmp_path / "cdn") - with pytest.raises(ConfigError, match="^catalog_main_xx.bin is not in the cache: setting servers.zz.cdn"): - Catalog.load("xx", tmp_path / "cache", cdn=missing("servers.zz.cdn")) + with pytest.raises(ConfigError, match="^catalog_main_xx.bin is not downloaded: setting servers.zz.cdn"): + Catalog.load("xx", tmp_path / "downloads", cdn=missing("servers.zz.cdn")) calls = [] - Catalog.load("xx", tmp_path / "cache", cdn=lambda: calls.append(1) or cdn) - cat = Catalog.load("xx", tmp_path / "cache", cdn=never("cdn"), bundle_key=never("bundle key")) + Catalog.load("xx", tmp_path / "downloads", cdn=lambda: calls.append(1) or cdn) + cat = Catalog.load("xx", tmp_path / "downloads", cdn=never("cdn"), bundle_key=never("bundle key")) assert calls == [1] and cat.keys("Char/") == ["Char/A", "Char/B"] @@ -186,10 +190,10 @@ def run(argv, capsys): return code, out, err -def test_pull_of_cached_bundles_needs_no_region_cdn_or_key(tmp_path, capsys, monkeypatch): +def test_pull_of_downloaded_bundles_needs_no_region_cdn_or_key(tmp_path, capsys, monkeypatch): catbin = tmp_path / "catalog_main_xx.bin" catbin.write_bytes(synth.CatalogWriter().build(ENTRIES)) - base = ["--catalog", str(catbin), "--cache", str(tmp_path / "cache")] + base = ["--catalog", str(catbin), "--depot", str(tmp_path / "depot")] monkeypatch.setenv("NNNOTES_BUNDLE_KEY", synth.BUNDLE_KEY.hex()) monkeypatch.setenv("NNNOTES_BUNDLE_NONCE_SEED", synth.BUNDLE_SEED.hex()) monkeypatch.setenv("NNNOTES_SERVERS_ZZ_CDN", cdn_tree(tmp_path / "cdn")) @@ -207,9 +211,9 @@ def test_pull_that_downloads_names_the_missing_setting_in_one_line(tmp_path, cap catbin.write_bytes(synth.CatalogWriter().build(ENTRIES)) monkeypatch.setenv("NNNOTES_SERVERS_ZZ_CDN", cdn_tree(tmp_path / "cdn")) code, out, err = run( - ["--catalog", str(catbin), "--cache", str(tmp_path / "cache"), "--region", "zz", "bundle", "pull", "Char/A"], + ["--catalog", str(catbin), "--depot", str(tmp_path / "depot"), "--region", "zz", "bundle", "pull", "Char/A"], capsys, ) assert code == 2 and out == "" and len(err.strip().splitlines()) == 1 - assert err.startswith("nnnotes: a_01.bundle is not in the cache: setting bundle.key") + assert err.startswith("nnnotes: a_01.bundle is not downloaded: setting bundle.key") assert "NNNOTES_BUNDLE_KEY" in err diff --git a/tests/source/test_catalog_versions.py b/tests/source/test_catalog_versions.py index aebfcc4..c0d56cf 100644 --- a/tests/source/test_catalog_versions.py +++ b/tests/source/test_catalog_versions.py @@ -33,7 +33,7 @@ def config(tmp_path, *, api=True, version=None, region="tw"): "servers": { region: {"cdn": "https://cdn.example/prod/" + region, **({"api": "https://api.example"} if api else {})} }, - "paths": {"cache": str(tmp_path / "cache")}, + "paths": {"depot": str(tmp_path / "depot")}, }, environ={}, ) @@ -43,9 +43,9 @@ def test_auto_discovery_ignores_stale_main_and_advances_resource_version(tmp_pat from nnnotes.site.webmodel import catalog_models cfg = config(tmp_path) - cache = cfg.require_path("paths", "cache") - cache.mkdir() - catalog.Catalog.cache_file("en", cache).write_bytes(binary("Old")) + downloads = Session(cfg).downloads() + downloads.mkdir(parents=True) + (downloads / catalog.Catalog.file_name("en")).write_bytes(binary("Old")) publication = gameapi.MasterVersion("master-one", "1.0.0.201") seen = [] @@ -71,20 +71,20 @@ def download(url): publication = gameapi.MasterVersion("master-two", "1.0.0.202") second = Session(cfg).catalog(bundles=False) assert second.has("master-two") and not second.has("master-one") - assert first.cache_dir != second.cache_dir + assert first.download_dir != second.download_dir assert all("catalog_main" not in request for request in seen) -def test_explicit_version_cache_is_offline_and_cdn_scoped(tmp_path, monkeypatch): +def test_explicit_version_download_is_offline_and_cdn_scoped(tmp_path, monkeypatch): cfg = config(tmp_path, version="1.0.0.201") monkeypatch.setattr(Session, "master_version", lambda *args: pytest.fail("pin called Version")) monkeypatch.setattr(catalog, "download", lambda url: binary("First")) first = Session(cfg).catalog(bundles=False) - monkeypatch.setattr(catalog, "download", lambda url: pytest.fail("cached pin downloaded")) + monkeypatch.setattr(catalog, "download", lambda url: pytest.fail("downloaded pin downloaded")) assert Session(cfg).catalog(bundles=False).has("First") monkeypatch.setattr(catalog, "download", lambda url: binary("Other region")) other = Session(config(tmp_path, version="1.0.0.201", region="en")).catalog(bundles=False) - assert other.has("Other region") and first.cache_dir != other.cache_dir + assert other.has("Other region") and first.download_dir != other.download_dir def test_explicit_catalog_file_stays_offline(tmp_path, monkeypatch): @@ -94,7 +94,7 @@ def test_explicit_catalog_file_stays_offline(tmp_path, monkeypatch): { "catalog": {"region": "tw", "version": "1.0.0.201"}, "servers": {"tw": {"api": "https://api.example", "catalog": str(path)}}, - "paths": {"cache": str(tmp_path / "cache")}, + "paths": {"depot": str(tmp_path / "depot")}, }, environ={}, ) @@ -104,9 +104,9 @@ def test_explicit_catalog_file_stays_offline(tmp_path, monkeypatch): def test_missing_versioned_catalog_does_not_fall_back_to_main(tmp_path, monkeypatch): cfg = config(tmp_path, version="1.0.0.201") - cache = cfg.require_path("paths", "cache") - cache.mkdir() - catalog.Catalog.cache_file("en", cache).write_bytes(binary("Old")) + downloads = Session(cfg).downloads() + downloads.mkdir(parents=True) + (downloads / catalog.Catalog.file_name("en")).write_bytes(binary("Old")) urls = [] def missing(url): @@ -117,7 +117,7 @@ def missing(url): with pytest.raises(urllib.error.HTTPError): Session(cfg).catalog(bundles=False) assert len(urls) == 1 and "catalog_1.0.0.201_en.bin" in urls[0] - assert not list((cache / "international").rglob("*.bin")) + assert not list((downloads / "international").rglob("*.bin")) def test_failed_discovery_does_not_claim_main_is_current(tmp_path, monkeypatch): @@ -156,7 +156,7 @@ def test_build_workers_inherit_the_version_pin(tmp_path, monkeypatch): monkeypatch.setenv("NNNOTES_SERVERS_TW_CATALOG_VERSION", "1.0.0.201") monkeypatch.setenv("NNNOTES_SERVERS_TW_CDN", "https://cdn.example/prod/tw") monkeypatch.setenv("NNNOTES_SERVERS_TW_API", "https://unavailable.example") - monkeypatch.setenv("NNNOTES_PATHS_CACHE", str(tmp_path / "cache")) + monkeypatch.setenv("NNNOTES_PATHS_DEPOT", str(tmp_path / "depot")) monkeypatch.setattr(catalog, "download", lambda url: binary(MODEL, EPISODE)) assert Session(Config.load()).catalog(bundles=False).has(MODEL) command = [ @@ -175,7 +175,7 @@ def test_invalid_version_cannot_be_used_as_a_path(tmp_path, version): with pytest.raises(ConfigError): Session(config(tmp_path, version=version)).catalog_version() with pytest.raises(ValueError): - catalog.Catalog.cache_file("en", tmp_path, version=version) + catalog.Catalog.file_name("en", version=version) with pytest.raises(ValueError): configfile.validate(configfile.resolve("catalog.version")[0], version) @@ -187,11 +187,11 @@ def test_empty_api_version_fails_before_download(tmp_path, monkeypatch): Session(config(tmp_path)).catalog(bundles=False) -def test_bad_download_is_not_installed_in_cache(tmp_path, monkeypatch): +def test_bad_download_is_not_installed(tmp_path, monkeypatch): monkeypatch.setattr(catalog, "download", lambda url: b"not a catalog") with pytest.raises(ValueError): Session(config(tmp_path, version="1.0.0.201")).catalog(bundles=False) - assert not list((tmp_path / "cache").rglob("*.bin")) + assert not list((tmp_path / "depot").rglob("*.bin")) def test_fetch_queries_version_before_downloading_and_labels_matching_bytes(tmp_path, monkeypatch, capsys): @@ -211,9 +211,9 @@ def fetch(cdn, language, *, version): monkeypatch.setattr(Session, "master_version", observe) monkeypatch.setattr(catalogdb, "fetch", fetch) - cli.main(["--region", "tw", "--language", "en", "--cache", str(tmp_path / "cache"), "catalog", "fetch"]) + cli.main(["--region", "tw", "--language", "en", "--depot", str(tmp_path / "depot"), "catalog", "fetch"]) assert events == ["version", "1.0.0.201"] - version = catalogdb.CatalogDB(tmp_path / "cache" / "store").versions()[0] + version = catalogdb.CatalogDB(tmp_path / "depot").versions()[0] assert version["resourceVersion"] == "1.0.0.201" assert version["remote"]["sha256"] == hashlib.sha256(data).hexdigest() assert "1.0.0.201" in capsys.readouterr().out @@ -235,7 +235,7 @@ def test_browser_uses_the_same_selector(tmp_path, monkeypatch): calls = [] monkeypatch.setattr(catalog, "download", lambda url: calls.append(url) or binary(MODEL)) handler = browse.Handler.__new__(browse.Handler) - handler.server = SimpleNamespace(catalogs={}, cache=tmp_path / "cache", bundle_key=None) + handler.server = SimpleNamespace(catalogs={}, downloads=tmp_path / "depot" / "downloads", bundle_key=None) handler.catalog(browse.Region("tw", "TW", cfg.cdn("tw"), ["en"], cfg), "en") assert len(calls) == 1 and calls[0].endswith("catalog_1.0.0.201_en.bin") diff --git a/tests/source/test_configfile.py b/tests/source/test_configfile.py index b77900e..acb20a2 100644 --- a/tests/source/test_configfile.py +++ b/tests/source/test_configfile.py @@ -79,7 +79,7 @@ def test_init_with_values(tmp_path, capsys): "--set", "catalog.region=en", "--set", - "paths.cache=cache", + "paths.depot=depot", ], capsys, ) @@ -102,7 +102,7 @@ def test_init_with_values(tmp_path, capsys): "catalog_version": "", } } # the example region table renamed - assert data["paths"]["cache"] == str((tmp_path / "cache").absolute()) + assert data["paths"]["depot"] == str((tmp_path / "depot").absolute()) assert "# CDN base URL of the region" in f.read_text(encoding="utf-8") # the comments stay if os.name != "nt": assert f.stat().st_mode & 0o777 == 0o600 @@ -240,7 +240,7 @@ def test_check_does_not_turn_jp_language_defaults_into_flags(tmp_path, capsys): def test_check_reports_states_not_values(tmp_path, capsys, monkeypatch): (tmp_path / "nnnotes.toml").write_text( f'[bundle]\nkey = "{KEY}"\n[master]\niv = "00"\n[catalog]\nregion = "xx"\n[servers.tw]\ncdn = "ftp://h"\n' - f'[paths]\napk = "missing.apk"\ncache = "c"\n[typo]\nkey = "1"\n', + f'[paths]\napk = "missing.apk"\ndepot = "c"\n[typo]\nkey = "1"\n', encoding="utf-8", ) monkeypatch.setenv("NNNOTES_BUNDLE_NONCE_SEED", "ab") @@ -256,7 +256,7 @@ def test_check_reports_states_not_values(tmp_path, capsys, monkeypatch): assert by["master.key"]["status"] == "unset" assert by["catalog.region"]["status"] == "invalid" assert by["servers.tw.cdn"]["status"] == "invalid" - assert by["paths.apk"]["status"] == "not found" and by["paths.cache"]["status"] == "ok" + assert by["paths.apk"]["status"] == "not found" and by["paths.depot"]["status"] == "ok" assert by["typo.key"]["status"] == "unknown" and by["NNNOTES_BUNDLE_KY"]["status"] == "unknown" assert doc["problems"] == 6 code, out, _ = run(["--region", "tw", "config", "check"], capsys) diff --git a/tests/source/test_jp.py b/tests/source/test_jp.py index f318fb2..60dc8ec 100644 --- a/tests/source/test_jp.py +++ b/tests/source/test_jp.py @@ -173,14 +173,14 @@ def test_authenticated_download_rotation_and_redaction(upstream): assert "Authorization" not in json.dumps(obs.source.to_dict()) -def test_truncated_download_is_not_cached(upstream, tmp_path): +def test_truncated_download_is_not_kept(upstream, tmp_path): client = jp.Client(upstream.cfg, "jp") source = client.observe().source upstream.content_length = len(upstream.body) + 5 cat = Catalog(cat_bytes(), tmp_path, source=source, client=client) with pytest.raises(gameapi.GameApiError, match="truncated"): cat.fetch_raw({"internal_id": jp.PLACEHOLDER + "audio/test.acb"}) - assert cat.cached_raw({"internal_id": jp.PLACEHOLDER + "audio/test.acb"}) is None + assert cat.downloaded_raw({"internal_id": jp.PLACEHOLDER + "audio/test.acb"}) is None def test_declared_download_limit_is_checked(upstream): @@ -246,16 +246,16 @@ def test_placeholder_rejects_unsafe_paths(path): location_kind(jp.PLACEHOLDER + path) -def test_catalog_cache_source_and_offline_replay(upstream, tmp_path, monkeypatch): +def test_catalog_download_source_and_offline_replay(upstream, tmp_path, monkeypatch): cfg = upstream.cfg - cfg._data["paths"] = {"cache": str(tmp_path / "cache")} + cfg._data["paths"] = {"depot": str(tmp_path / "depot")} cat = Session(cfg).catalog(bundles=False) - path = cat.cache_dir / "catalog_main.bin" + path = cat.download_dir / "catalog_main.bin" assert path.read_bytes() == upstream.body assert jp.read_source(path, upstream.body) == cat.source - assert cat.cache_dir != tmp_path / "cache" + assert cat.download_dir != tmp_path / "depot" / "downloads" assert cat.keys("Live/") == ["Live/MusicScore/0001/0001_00"] - record = catalogdb.CatalogDB(tmp_path / "store").add(upstream.body, source=cat.source.to_dict(), region="jp") + record = catalogdb.CatalogDB(tmp_path / "depot").add(upstream.body, source=cat.source.to_dict(), region="jp") assert upstream.credential not in json.dumps(record) cfg._data["paths"]["catalog"] = str(path) monkeypatch.setattr(jp.Client, "observe", lambda *a, **k: pytest.fail("offline read made an API call")) @@ -267,27 +267,27 @@ def test_catalog_cache_source_and_offline_replay(upstream, tmp_path, monkeypatch def test_catalogdb_source_identity_and_fetcher(upstream, tmp_path): cfg = upstream.cfg - cfg._data["paths"] = {"cache": str(tmp_path / "cache")} + cfg._data["paths"] = {"depot": str(tmp_path / "depot")} client = jp.Client(cfg, "jp") source = client.observe().source - db = catalogdb.CatalogDB(tmp_path / "store") + db = catalogdb.CatalogDB(tmp_path / "depot") old = db.add(cat_bytes(), source=source.to_dict(), region="jp") new_source = {**source.to_dict(), "hash": "c" * 32} new = db.add(cat_bytes(), source=new_source, region="jp") assert old["id"] != new["id"] and old["labels"] != new["labels"] assert old["remote"] == new["remote"] - fetcher = CatalogFetcher(tmp_path / "store", tmp_path / "cache", cfg) + fetcher = CatalogFetcher(tmp_path / "depot", cfg) cat, _ = fetcher.catalog(old["id"]) - assert cat.source == source and cat.cache_dir == source.cache_dir(tmp_path / "cache") + assert cat.source == source and cat.download_dir == source.download_dir(tmp_path / "depot" / "downloads") -def test_apk_update_with_same_asset_snapshot_does_not_reuse_embedded_cache(tmp_path): +def test_apk_update_with_same_asset_snapshot_does_not_reuse_embedded_downloads(tmp_path): from nnnotes.source.catalog import APK_AA_DIR, APK_CATALOG source = jp.Source("jp", "1.0", HASH, "https://cdn.invalid") apk = tmp_path / "base.apk" - cache = tmp_path / "cache" - db = catalogdb.CatalogDB(tmp_path / "store") + downloads = tmp_path / "depot" / "downloads" + db = catalogdb.CatalogDB(tmp_path / "depot") cfg = Config({"catalog": {"region": "jp"}, "paths": {"apk": str(apk)}}, environ={}) from nnnotes.cli.assets import Workspace @@ -298,14 +298,14 @@ def test_apk_update_with_same_asset_snapshot_does_not_reuse_embedded_cache(tmp_p ) content = b"UnityFS\0" + bytes([revision]) apk.write_bytes(zip_bytes({APK_CATALOG: local, APK_AA_DIR + "Android/local.bundle": content})) - cat = Catalog(cat_bytes(), source.cache_dir(cache), source=source, apk=apk) + cat = Catalog(cat_bytes(), source.download_dir(downloads), source=source, apk=apk) v = db.add(cat_bytes(), local, source=source.to_dict(), region="jp") workspace = Workspace.__new__(Workspace) workspace.version = v - rel = workspace._cache_rel("bundle", {"name": "local.bundle", "remote": False}, {}) + rel = workspace._download_rel("bundle", {"name": "local.bundle", "remote": False}, {}) assert cat.fetch(cat.resolve("Local")[0]).read_bytes() == content - assert (cache / rel).read_bytes() == content - fetcher = CatalogFetcher(tmp_path / "store", cache, cfg) + assert (downloads / rel).read_bytes() == content + fetcher = CatalogFetcher(tmp_path / "depot", cfg) replay, _ = fetcher.catalog(v["id"]) assert replay.fetch(replay.resolve("Local")[0]).read_bytes() == content assert len(db.versions()) == 2 @@ -329,7 +329,7 @@ def test_remote_replay_does_not_open_the_apk(upstream, tmp_path, apk_state, kind cfg = upstream.cfg cfg._data["bundle"] = {"key": "00" * 16, "nonce_seed": "11"} - cfg._data["paths"] = {"cache": str(tmp_path / "cache")} + cfg._data["paths"] = {"depot": str(tmp_path / "depot")} apk = tmp_path / "base.apk" if apk_state != "unset": cfg._data["paths"]["apk"] = str(apk) @@ -338,18 +338,18 @@ def test_remote_replay_does_not_open_the_apk(upstream, tmp_path, apk_state, kind elif apk_state == "broken": apk.write_bytes(b"not a zip") source = jp.Client(cfg, "jp").observe().source - db = catalogdb.CatalogDB(tmp_path / "store") + db = catalogdb.CatalogDB(tmp_path / "depot") record = db.add(cat_bytes(), local_catalog(), source=source.to_dict(), region="jp") entry = next(e for e in db.index(record)[kind] if e["remote"]) upstream.body = b"UnityFS\0synthetic resource" - fetcher = CatalogFetcher(tmp_path / "store", tmp_path / "cache", cfg) + fetcher = CatalogFetcher(tmp_path / "depot", cfg) assert fetcher.fetch_location(record["id"], entry["location"]).read_bytes() == upstream.body assert len(upstream.seen) == 1 @pytest.mark.parametrize("kind", ["bundles", "rawFiles"]) -@pytest.mark.parametrize("state", ["matching", "different", "no-catalog", "cached"]) -def test_local_replay_validates_only_on_cache_miss(tmp_path, kind, state): +@pytest.mark.parametrize("state", ["matching", "different", "no-catalog", "downloaded"]) +def test_local_replay_validates_only_when_not_downloaded(tmp_path, kind, state): from nnnotes.source.catalog import APK_AA_DIR, APK_CATALOG source = jp.Source("jp", "1.0", HASH, "https://cdn.invalid") @@ -359,20 +359,20 @@ def test_local_replay_validates_only_on_cache_miss(tmp_path, kind, state): members[APK_CATALOG] = local_catalog(2 if state == "different" else 1) apk.write_bytes(zip_bytes(members)) cfg = Config({"catalog": {"region": "jp"}, "paths": {"apk": str(apk)}}, environ={}) - db = catalogdb.CatalogDB(tmp_path / "store") + db = catalogdb.CatalogDB(tmp_path / "depot") record = db.add(cat_bytes(), local_catalog(), source=source.to_dict(), region="jp") entry = next(e for e in db.index(record)[kind] if not e["remote"]) - fetcher = CatalogFetcher(tmp_path / "store", tmp_path / "cache", cfg) + fetcher = CatalogFetcher(tmp_path / "depot", cfg) if state in ("different", "no-catalog"): with pytest.raises(ConfigError, match="APK set it was imported with"): fetcher.fetch_location(record["id"], entry["location"]) return expected = members[APK_AA_DIR + ("Android/local.bundle" if kind == "bundles" else "Android/local.acb")] assert fetcher.fetch_location(record["id"], entry["location"]).read_bytes() == expected - if state == "cached": + if state == "downloaded": apk.unlink() cfg._data["paths"].pop("apk") - replay = CatalogFetcher(tmp_path / "store", tmp_path / "cache", cfg) + replay = CatalogFetcher(tmp_path / "depot", cfg) assert replay.fetch_location(record["id"], entry["location"]).read_bytes() == expected @@ -522,16 +522,16 @@ def test_jp_master_rejects_missing_or_malformed_file_lists(tmp_path, files): assert not (tmp_path / "MasterManifest.json").exists() -def test_export_cache_locator_uses_source_namespace(tmp_path): +def test_export_download_locator_uses_source_namespace(tmp_path): from nnnotes.cli.assets import Workspace source = jp.Source("jp", "1.0.0.300", HASH, "https://cdn.invalid") workspace = Workspace.__new__(Workspace) workspace.version = {"source": source.to_dict()} - rel = workspace._cache_rel("bundle", {"name": "same.bundle"}, {}) - assert tmp_path / rel == source.cache_dir(tmp_path) / "bundles/same.bundle" - rel = workspace._cache_rel("raw", {}, {"internalId": jp.PLACEHOLDER + "audio/sample.acb"}) - assert tmp_path / rel == source.cache_dir(tmp_path) / "raw/audio/sample.acb" + rel = workspace._download_rel("bundle", {"name": "same.bundle"}, {}) + assert tmp_path / rel == source.download_dir(tmp_path) / "bundles/same.bundle" + rel = workspace._download_rel("raw", {}, {"internalId": jp.PLACEHOLDER + "audio/sample.acb"}) + assert tmp_path / rel == source.download_dir(tmp_path) / "raw/audio/sample.acb" def test_music_data_preserves_asset_hash(tmp_path): From 98ca57a9b31a601e7969467e0fa2f57945671755 Mon Sep 17 00:00:00 2001 From: nichinichisou Date: Wed, 14 Oct 2026 21:55:46 +0800 Subject: [PATCH 2/2] refactor(scripts): use the depot in the atom evaluation script The evaluation script reads and writes a depot (`--base DEPOT`, `--depot D`) like the commands do. Co-Authored-By: Claude Opus 5.5 --- scripts/atom_eval.py | 78 ++++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/scripts/atom_eval.py b/scripts/atom_eval.py index 327520e..ece713e 100644 --- a/scripts/atom_eval.py +++ b/scripts/atom_eval.py @@ -11,9 +11,9 @@ m1 DIR --atom NAME=SPEC ... conformance: every object of the checked classes through the reference and through each candidate; per check and class (or texture format): equal / differ / refused / ... - prepare DIR --base STORE census and link.scripts of the bundles into a base store (the tasks of + prepare DIR --base DEPOT census and link.scripts of the bundles into a base depot (the tasks of `m3` are described from it) - m3 DIR --base STORE --store S cost of a stage per bundle: wall, user + system CPU and peak resident + m3 DIR --base DEPOT --depot D cost of a stage per bundle: wall, user + system CPU and peak resident memory per task, in a worker pool (per-worker peak) or with --fresh in a new process per bundle (imports measured apart) imports [--module M ...] import cost of modules, each set in new processes @@ -1036,12 +1036,12 @@ def versions() -> dict: return out -# ---------------------------------------------------------------- prepare (base store) +# ---------------------------------------------------------------- prepare (base depot) def _identify(job): - from nnnotes.pipeline.store import Store + from nnnotes.pipeline.depot import Depot - root, store, name = job - s, size = Store(store).identify(Path(root) / name) + root, depot, name = job + s, size = Depot(depot).identify(Path(root) / name) return name, s, size @@ -1049,20 +1049,20 @@ def _census_task(task): from nnnotes.pipeline import census from nnnotes.pipeline.stages import execute - ex = execute(task, G["store"], {"unity.census": census.CensusStage()}) + ex = execute(task, G["depot"], {"unity.census": census.CensusStage()}) return task.subject, task.key, ex.status def cmd_prepare(a) -> dict: from nnnotes.pipeline import census, contract, link from nnnotes.pipeline.contract import Input + from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.stages import Env, describe, execute - from nnnotes.pipeline.store import Store check_clean_env() root, base = Path(a.dir).resolve(), Path(a.base) items = bundle_list(root, a.bundles) - store = Store(base) + depot = Depot(base) t0 = time.perf_counter() ids = pool_map( _identify, [(str(root), str(base), n) for n, _ in sorted(items, key=lambda x: -x[1])], a.workers, maxtasks=None @@ -1074,14 +1074,14 @@ def cmd_prepare(a) -> dict: subject = stable[names[n]] inputs[subject] = Input("bundle", s, size, names[n], ({"kind": "file", "path": str(root / n)},)) subj_name[subject] = n - env = Env(store, {"bundles": inputs}) + env = Env(depot, {"bundles": inputs}) tasks = [describe(census.CensusStage(), s, None, env) for s in sorted(inputs, key=lambda s: -inputs[s].size)] - G["store"] = store + G["depot"] = depot res = pool_map(_census_task, tasks, a.workers) for t in tasks: env.done(t.id, t.key) st = describe(link.ScriptsStage(), "all", None, env) - execute(st, store, {"link.scripts": link.ScriptsStage()}) + execute(st, depot, {"link.scripts": link.ScriptsStage()}) doc = { "dir": str(root), "bundles": { @@ -1101,8 +1101,8 @@ def cmd_prepare(a) -> dict: # ---------------------------------------------------------------- m3 (stage cost) def _prepared_env(base: Path): from nnnotes.pipeline.contract import Input + from nnnotes.pipeline.depot import Depot from nnnotes.pipeline.stages import Env - from nnnotes.pipeline.store import Store prep = json.loads((base / PREPARED).read_text(encoding="utf-8")) root = Path(prep["dir"]) @@ -1116,16 +1116,16 @@ def _prepared_env(base: Path): ) for s, b in prep["bundles"].items() } - env = Env(Store(base), {"bundles": inputs}) + env = Env(Depot(base), {"bundles": inputs}) for s, k in prep["census"].items(): env.done(f"unity.census:{s}", k) env.done("link.scripts:all", prep["scripts"]) return prep, env -def _new_store(base: Path, path: Path): - """A run store that knows the base store's input identities (so inputs are not hashed again).""" - from nnnotes.pipeline.store import Store +def _new_depot(base: Path, path: Path): + """A run depot that knows the base depot's input identities (so inputs are not hashed again).""" + from nnnotes.pipeline.depot import Depot path.mkdir(parents=True, exist_ok=True) src = base / "inputs" @@ -1134,7 +1134,7 @@ def _new_store(base: Path, path: Path): for f in src.iterdir(): if f.is_file() and not (path / "inputs" / f.name).exists(): shutil.copyfile(f, path / "inputs" / f.name) - return Store(path) + return Depot(path) def _stage(name: str): @@ -1144,7 +1144,7 @@ def _stage(name: str): return {"unity.export": ExportStage, "unity.census": census.CensusStage}[name]() -def _run_task(task, store, stage_name: str) -> dict: +def _run_task(task, depot, stage_name: str) -> dict: from nnnotes.pipeline.stages import execute hwm = reset_hwm() @@ -1152,7 +1152,7 @@ def _run_task(task, store, stage_name: str) -> dict: t0 = time.perf_counter() rec = {"subject": task.subject, "key": task.key} try: - ex = execute(task, store, {stage_name: _stage(stage_name)}, force=True) + ex = execute(task, depot, {stage_name: _stage(stage_name)}, force=True) rec.update(status=ex.status, result=ex.result_status) except Exception as e: # noqa: BLE001 rec.update(status="error", error=message(e), trace=traceback.format_exc()[-1200:]) @@ -1169,7 +1169,7 @@ def _run_task(task, store, stage_name: str) -> dict: def _m3_worker(task) -> dict: - return _run_task(task, G["store"], G["stage"]) + return _run_task(task, G["depot"], G["stage"]) def _apply_atoms(atoms) -> None: @@ -1229,19 +1229,19 @@ def summarize_runs(recs: list) -> dict: def cmd_m3(a) -> dict: check_clean_env() - base, out_store = Path(a.base), Path(a.store) + base, out_depot = Path(a.base), Path(a.depot) atoms = parse_atoms(a.atom) params = parse_params(a.param) or None prep, env = _prepared_env(base) subjects = _subjects(prep, a.bundles) if a.fresh: - return m3_fresh(a, base, out_store, atoms, params, prep, subjects) + return m3_fresh(a, base, out_depot, atoms, params, prep, subjects) imp = _import_stage_modules() _apply_atoms(atoms) tasks = _describe_all(a.stage, subjects, params, env) tasks.sort(key=lambda t: -t.inputs[0].size) - store = _new_store(base, out_store) - G.update(store=store, stage=a.stage) + depot = _new_depot(base, out_depot) + G.update(depot=depot, stage=a.stage) log("m3", a.stage, len(tasks), "tasks,", a.workers, "workers, atoms", dict(atoms), "params", params) jsonl = Path(a.out).with_suffix(".jsonl") if a.out else None fo = open(jsonl, "w", encoding="utf-8", newline="\n") if jsonl else None @@ -1266,7 +1266,7 @@ def take(r): "maxtasks": a.maxtasks, "atoms": dict(atoms), "params": params, - "store": str(out_store.resolve()), + "depot": str(out_depot.resolve()), "taskAtoms": tasks[0].atoms if tasks else None, "wall": round(time.perf_counter() - t0, 2), "childrenCpu": round((c1.children_user - c0.children_user) + (c1.children_system - c0.children_system), 2), @@ -1281,18 +1281,18 @@ def take(r): return doc -def m3_fresh(a, base, out_store, atoms, params, prep, subjects) -> dict: +def m3_fresh(a, base, out_depot, atoms, params, prep, subjects) -> dict: """One new process per bundle: its wall, user + system CPU and peak RSS (wait4), and inside it the import time and the task's own cost.""" - _new_store(base, out_store) + _new_depot(base, out_depot) cmd = [ sys.executable, str(Path(__file__).resolve()), "m3-one", "--base", str(base), - "--store", - str(out_store), + "--depot", + str(out_depot), "--stage", a.stage, ] @@ -1306,7 +1306,7 @@ def m3_fresh(a, base, out_store, atoms, params, prep, subjects) -> dict: queue = sorted(subjects, key=lambda s: -sizes[s]) running: dict[int, tuple] = {} recs = [] - tmp = out_store / ".atom_eval" + tmp = out_depot / ".atom_eval" tmp.mkdir(exist_ok=True) jsonl = Path(a.out).with_suffix(".jsonl") if a.out else None fo = open(jsonl, "w", encoding="utf-8", newline="\n") if jsonl else None @@ -1362,7 +1362,7 @@ def m3_fresh(a, base, out_store, atoms, params, prep, subjects) -> dict: "workers": a.workers, "atoms": dict(atoms), "params": params, - "store": str(out_store.resolve()), + "depot": str(out_depot.resolve()), "wall": round(time.perf_counter() - t0, 2), "tasks": len(recs), "ok": len(ok), @@ -1400,9 +1400,9 @@ def cmd_m3_one(a) -> None: prep, env = _prepared_env(base) (task,) = _describe_all(a.stage, [a.subject], parse_params(a.param) or None, env) describe_cost = {"wall": round(time.perf_counter() - t0, 4), "cpu": round(time.process_time() - cd, 4)} - from nnnotes.pipeline.store import Store + from nnnotes.pipeline.depot import Depot - rec = _run_task(task, Store(a.store), a.stage) + rec = _run_task(task, Depot(a.depot), a.stage) rec.update(imports=imp, describe=describe_cost, processCpuInside=round(time.process_time() - c0, 4)) print(json.dumps(rec, ensure_ascii=False)) @@ -1525,10 +1525,10 @@ def _compare_job(job) -> dict: def cmd_compare(a) -> dict: - from nnnotes.pipeline.store import Store + from nnnotes.pipeline.depot import Depot ra, rb = (json.loads(Path(p).read_text(encoding="utf-8")) for p in (a.run_a, a.run_b)) - G.update(sa=Store(ra["store"]), sb=Store(rb["store"])) + G.update(sa=Depot(ra["depot"]), sb=Depot(rb["depot"])) ta, tb = ra["taskKeys"], rb["taskKeys"] jobs = [(s, ta.get(s), tb.get(s)) for s in sorted(ta.keys() | tb.keys())] total = {"counts": {}, "stats": {}, "examples": {}} @@ -1536,8 +1536,8 @@ def cmd_compare(a) -> dict: merge(total, r) doc = { "command": "compare", - "a": {k: ra.get(k) for k in ("atoms", "params", "store", "workers", "mode")}, - "b": {k: rb.get(k) for k in ("atoms", "params", "store", "workers", "mode")}, + "a": {k: ra.get(k) for k in ("atoms", "params", "depot", "workers", "mode")}, + "b": {k: rb.get(k) for k in ("atoms", "params", "depot", "workers", "mode")}, "subjects": len(jobs), **total, } @@ -1583,7 +1583,7 @@ def common(p, bundles=True): else: p.add_argument("--subject", required=True) p.add_argument("--base", required=True) - p.add_argument("--store", required=True) + p.add_argument("--depot", required=True) p.add_argument("--stage", default="unity.export", choices=("unity.export", "unity.census")) p.add_argument("--atom", action="append") p.add_argument("--param", action="append", help="stage parameter path=value, e.g. png.level=1")