From 5226d922f55aba4d7c44ab09d8baf4b7c9da2eeb Mon Sep 17 00:00:00 2001 From: John Crafts <5889731+Artsen@users.noreply.github.com> Date: Sun, 26 Jul 2026 00:08:32 -0400 Subject: [PATCH] Persist image assets and add Craft service discovery --- .gitignore | 1 + README.md | 29 +- app.py | 14 - check.py | 17 + docs/architecture.md | 35 +- docs/image-workflow.md | 59 ++- docs/product-direction.md | 12 +- docs/recovery.md | 33 ++ docs/service-contract.md | 41 ++ docs/storage.md | 58 +++ docs/testing.md | 7 + glitchcraft/application.py | 40 +- glitchcraft/image_assets.py | 167 --------- glitchcraft/service_contract.py | 96 +++++ glitchcraft/storage/__init__.py | 25 ++ glitchcraft/storage/contracts.py | 162 ++++++++ glitchcraft/storage/errors.py | 33 ++ glitchcraft/storage/manifest.py | 100 +++++ glitchcraft/storage/repository.py | 604 ++++++++++++++++++++++++++++++ glitchcraft/version.py | 9 + glitchcraft/web/routes.py | 465 +++++++++++++++++++---- playwright.config.js | 4 + static/app-manifest.json | 26 ++ static/glitchcraft-mark.svg | 6 + tests/conftest.py | 3 + tests/test_image_assets.py | 367 +++++++++++++++--- tests/test_image_workflow.py | 127 ++++++- tests/test_service_contract.py | 225 +++++++++++ tests/test_storage_manifest.py | 280 ++++++++++++++ 29 files changed, 2661 insertions(+), 384 deletions(-) create mode 100644 docs/recovery.md create mode 100644 docs/service-contract.md create mode 100644 docs/storage.md delete mode 100644 glitchcraft/image_assets.py create mode 100644 glitchcraft/service_contract.py create mode 100644 glitchcraft/storage/__init__.py create mode 100644 glitchcraft/storage/contracts.py create mode 100644 glitchcraft/storage/errors.py create mode 100644 glitchcraft/storage/manifest.py create mode 100644 glitchcraft/storage/repository.py create mode 100644 glitchcraft/version.py create mode 100644 static/app-manifest.json create mode 100644 static/glitchcraft-mark.svg create mode 100644 tests/test_service_contract.py create mode 100644 tests/test_storage_manifest.py diff --git a/.gitignore b/.gitignore index a3dac47..4a76315 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ htmlcov/ node_modules/ playwright-report/ test-results/ +data/ build/ dist/ diff --git a/README.md b/README.md index a09a515..d0e8f13 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,8 @@ scan, distortion, and signal treatments for images and video. The current interface is a transitional Flask/jQuery workspace. Images upload once, keep their original visible, update a deterministic processed preview as controls change, and create a downloadable PNG only when explicitly exported. -Video remains on the legacy preview and background-processing path. +Image sources and explicit exports now have persistent local identity. Video +remains on the temporary legacy preview and background-processing path. ## Requirements @@ -34,7 +35,10 @@ python app.py ``` Open . The default port and existing routes are preserved. -Uploaded and generated files are temporary and ignored by Git. +The managed image library defaults to `data/` and is ignored by Git. Configure +`DATA_ROOT` to move it to another local user-data location. The service has no +authentication and is intended only for loopback use; do not expose it directly +to the public internet. ### Image workflow @@ -45,8 +49,12 @@ Uploaded and generated files are temporary and ignored by Git. - Choose **Export PNG**, then explicitly download the managed result. Automatic previews return temporary PNG bytes and do not create output files. -Sources and exported outputs use opaque, in-memory identity and expire from -temporary storage. Their IDs do not survive an application restart. +Sources and explicit exports use opaque persistent identity and survive restarts +when the same data root is reused. They remain until explicitly deleted; they +are not deleted automatically by age. + +Machine-readable local discovery is available at `/app-manifest.json`, +`/metadata`, `/health`, `/ready`, `/api/capabilities`, and `/api/storage`. ## Validate @@ -64,6 +72,9 @@ statement and branch coverage, and repository consistency checks. - [Architecture](docs/architecture.md) - [Effect engine](docs/effect-engine.md) - [Image workflow and API](docs/image-workflow.md) +- [Persistent storage](docs/storage.md) +- [Recovery](docs/recovery.md) +- [Craft service contract](docs/service-contract.md) - [Testing](docs/testing.md) - [Product direction](docs/product-direction.md) @@ -71,11 +82,11 @@ The internal recipe contract supports a schema version, root seed, ordered effec instances, stable IDs, enabled states, and strict effect-specific parameters. User-facing recipe import/export and seed controls are intentionally deferred. -Task, image-source, and output identity remain in memory with no cancellation, -bounded queue, library, or restart recovery. Storage remains temporary, video -and audio behavior are unchanged, and true geometric distortion/datamoshing are -not implemented. jQuery remains the one CDN dependency for the legacy video -path; Bootstrap and Google Fonts are no longer used. +Image source and explicit output identity are persistent. Video task state, +sources, and outputs remain temporary and process-local with no cancellation, +bounded queue, or restart recovery. Video and audio behavior are unchanged, and +true geometric distortion/datamoshing are not implemented. jQuery remains the +one CDN dependency for the legacy video path. ## License diff --git a/app.py b/app.py index 896b4eb..362a1b4 100644 --- a/app.py +++ b/app.py @@ -5,26 +5,12 @@ from glitchcraft import create_app from glitchcraft.cleanup import CleanupScheduler -from glitchcraft.image_assets import ImageOutputStore, ImageSourceStore app = create_app() -source_store: ImageSourceStore = app.extensions["image_source_store"] -output_store: ImageOutputStore = app.extensions["image_output_store"] - - -def cleanup_image_records() -> None: - source_store.cleanup_expired() - output_store.cleanup_expired() - - -def protected_image_paths() -> set[Path]: - return source_store.known_paths() | output_store.known_paths() cleanup = CleanupScheduler( (Path(app.config[key]) for key in ("UPLOAD_FOLDER", "OUTPUT_FOLDER", "PREVIEW_FOLDER")), - protected_paths=protected_image_paths, - before_cleanup=cleanup_image_records, ) if __name__ == "__main__": diff --git a/check.py b/check.py index 409eb0b..cf67bd0 100644 --- a/check.py +++ b/check.py @@ -1,9 +1,13 @@ """Run the complete local quality baseline with one command.""" +import json import subprocess import sys from pathlib import Path +from glitchcraft.service_contract import CAPABILITY_SLUGS +from glitchcraft.version import APP_ID, APP_VERSION + def run(label: str, command: list[str]) -> None: print(f"\n==> {label}", flush=True) @@ -19,11 +23,24 @@ def repository_consistency() -> None: Path("glitchcraft/effects/registry.py"), Path("docs/effect-engine.md"), Path("docs/image-workflow.md"), + Path("docs/storage.md"), + Path("docs/recovery.md"), + Path("docs/service-contract.md"), + Path("glitchcraft/storage/contracts.py"), + Path("glitchcraft/storage/repository.py"), + Path("glitchcraft/version.py"), Path("static/app.js"), + Path("static/app-manifest.json"), + Path("static/glitchcraft-mark.svg"), ] missing = [str(path) for path in required if not path.is_file()] if missing: raise SystemExit(f"Missing required repository files: {', '.join(missing)}") + manifest = json.loads(Path("static/app-manifest.json").read_text(encoding="utf-8")) + if manifest.get("id") != APP_ID or manifest.get("version") != APP_VERSION: + raise SystemExit("Static application identity does not match the Python runtime.") + if tuple(manifest.get("capabilities", ())) != CAPABILITY_SLUGS: + raise SystemExit("Static capability slugs do not match the runtime contract.") def main() -> None: diff --git a/docs/architecture.md b/docs/architecture.md index be74eff..06f583e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,9 +1,9 @@ # Architecture -GlitchCraft is currently a local Flask prototype with a deterministic processing -core. `app.py` is only a development launcher. `glitchcraft.application.create_app` -constructs the application without starting servers, cleanup threads, schedulers, -or background work during import. +GlitchCraft is a local Flask application with a deterministic processing core. +`app.py` is only a development launcher. `glitchcraft.application.create_app` +constructs the application without starting servers, cleanup threads, +schedulers, or background work during import. The package boundaries are: @@ -11,16 +11,21 @@ The package boundaries are: - `effects`: metadata, isolated randomness, operations, and ordered execution. - `media`: Pillow/OpenCV color boundaries, file I/O, and FFmpeg invocation. - `web`: compatibility routes and request-to-recipe translation. -- `tasks`: the temporary thread-safe in-memory task store. -- `image_assets`: opaque, thread-safe identity and leases for temporary image - sources and explicit outputs. -- `cleanup`: launcher-owned expiration of temporary local files. +- `tasks`: the temporary thread-safe in-memory video task store. +- `storage`: strict manifest contracts, atomic persistence, path ownership, + leases, reconciliation, deletion, metrics, and cleanup for image assets. +- `service_contract`: static capability slugs and runtime availability metadata. +- `cleanup`: launcher-owned expiration of temporary legacy media files. -All processing is local. The application currently trusts the local operator and -does not provide authentication, content isolation, persistent source identity, -or a hardened multi-user deployment model. +All processing is local. The application trusts the local operator and does not +provide authentication, content isolation, or a hardened multi-user deployment +model. Metadata is redacted and broad CORS is not enabled. -Task and image identity state are process-local. A restart loses status, source -IDs, and output IDs. There is no bounded queue, cancellation, recovery, -persistent library, or database. These limitations are intentional in the -current transitional application. +Image sources and explicit outputs live in a versioned managed library and retain +opaque identity across restarts. Manifest mutations are serialized by a process +reentrant lock; multiple writer processes sharing a data root are not supported. + +Video task and asset identity remains process-local. A restart loses video job +status, and there is no bounded queue, cancellation, or persistent video library. +The current Flask process serves the interface and API together on port 5000. A +future 5175/4200 frontend/API split is a plan, not current behavior. diff --git a/docs/image-workflow.md b/docs/image-workflow.md index 74b744d..5a5deb3 100644 --- a/docs/image-workflow.md +++ b/docs/image-workflow.md @@ -1,6 +1,6 @@ # Source-aware image workflow -The transitional Flask/jQuery interface now treats image work as a local, +The transitional Flask/JavaScript interface treats image work as a local, interactive workspace: 1. Upload a supported static image once. @@ -10,51 +10,50 @@ interactive workspace: 5. Compare original and processed frames. 6. Export a PNG only after an explicit user action. -Automatic previews return lossless PNG bytes directly. They do not create output -files. Export runs the same full-resolution source, recipe, seed, frame index, and -`apply_effect_stack` engine, then creates one temporary managed PNG. - -Preview currently processes the full-resolution image. A future workspace may -introduce a bounded preview representation without changing export semantics. +Automatic previews return lossless PNG bytes directly and never create output +records or permanent files. Export runs the same full-resolution source, recipe, +seed, frame index, and `apply_effect_stack` engine, then creates one persistent +managed PNG and exact Recipe v1 snapshot. Preview currently processes the +full-resolution image. ## Identity and lifecycle -`ImageSourceStore` and `ImageOutputStore` issue unpredictable opaque identifiers. -Browser requests contain those IDs rather than server paths or managed -filenames. The stores are thread-safe and remain in memory. Records protect their -files from generic cleanup while active and expire after 24 hours by default. -Missing files and expired IDs become controlled 404 responses. - -Sources and outputs are not persistent across application restart. This is -temporary local storage, not a project library. +`ImageAssetRepository` issues unpredictable opaque identifiers. Browser requests +contain those IDs rather than server paths or managed filenames. A +schema-version-1 manifest persists records and output recipe snapshots. Sources +and outputs survive restart with the same data root and remain until explicit +deletion. Age cleanup never removes manifest-referenced image files. -Uploads are extension-checked before writing, decoded with Pillow, restricted to -PNG, JPEG, BMP, or TIFF, limited to 40 million pixels, and guarded against -decompression-bomb conditions. Animated or mismatched content is rejected. +Uploads are extension-checked before installation, decoded with Pillow, +restricted to PNG, JPEG, BMP, or TIFF, limited to 40 million pixels, and guarded +against decompression-bomb conditions. Animated or mismatched content is +rejected. ## API -- `POST /api/image-sources` — multipart source upload; returns metadata, seed, - opaque source ID, and inline original URL. +- `POST /api/image-sources` — multipart source upload. +- `GET /api/image-sources` and `GET /api/image-sources/` — source library + listing and metadata. - `GET /api/image-sources//original` — inline original with `no-store`. -- `POST /api/image-sources//preview` — strict `{ "recipe": ... }` JSON; - returns inline PNG bytes without creating an output. -- `POST /api/image-sources//export` — the same strict recipe; returns an - opaque output ID, display filename, inline URL, and download URL. +- `POST /api/image-sources//preview` — strict recipe request and inline PNG. +- `POST /api/image-sources//export` — explicit persistent PNG export. +- `GET /api/image-outputs` and `GET /api/image-outputs//metadata` — output + listing, links, source availability, and recipe snapshots. - `GET /api/image-outputs/` — inline exported PNG. - `GET /api/image-outputs//download` — the same PNG as an attachment. +- `DELETE /api/image-outputs/` — explicit output deletion. +- `DELETE /api/image-sources/?cascade=true` — source deletion with optional + dependent-output deletion. The legacy routes remain, and video preview/processing continues through the -legacy path. +temporary legacy path. There is no visible library interface in this PR. ## Transitional frontend -The image-side script owns explicit source, seed, recipe, request, preview, +The image-side script retains explicit source, seed, recipe, request, preview, export, and error state. A 175 ms debounce limits slider requests. An `AbortController` cancels prior requests, a monotonically increasing revision rejects late results, and replaced object URLs are revoked. -jQuery remains CDN-hosted for the legacy video path. Bootstrap and Google Fonts -have been removed; the image workspace uses a system font stack and repository -CSS. The planned family-aligned React workspace and final jQuery removal remain -future work. +jQuery remains CDN-hosted for the legacy video path. The planned family-aligned +React workspace, visible library, and final jQuery removal remain future work. diff --git a/docs/product-direction.md b/docs/product-direction.md index 30b2d26..04b89ab 100644 --- a/docs/product-direction.md +++ b/docs/product-direction.md @@ -15,6 +15,12 @@ The family relationship is a shared precision-oriented shell and interaction grammar. GlitchCraft will retain its own signal, interference, and transformation identity; another application's visual theme will not be copied wholesale. -This first PR deliberately makes no visual redesign. It establishes inspectable -recipes, capability metadata, deterministic processing, truthful task state, and -testable boundaries needed by that future workspace. +The current sequence deliberately avoids a final visual redesign. It establishes +inspectable recipes, deterministic processing, persistent image identity, Craft +discovery metadata, truthful readiness, and testable boundaries needed by that +future workspace. + +A future orchestration dashboard may discover and check GlitchCraft, ColorCraft, +and Web Video Optimizer through related contracts. It is not implemented here +and cannot launch or remotely control this service. The family-aligned React +workspace and proposed 5175/4200 frontend/API split are also deferred. diff --git a/docs/recovery.md b/docs/recovery.md new file mode 100644 index 0000000..c0828e2 --- /dev/null +++ b/docs/recovery.md @@ -0,0 +1,33 @@ +# Storage recovery and reconciliation + +Startup creates an empty schema-version-1 manifest when no managed state exists. +A valid primary loads normally. If the primary is missing or invalid and the +backup is valid, GlitchCraft restores the primary without overwriting the +last-known-good backup, records a redacted warning, writes a private recovery +report, and reports degraded readiness for that process. + +If both primary and backup are invalid, neither is overwritten. Lightweight +`/health` remains available, `/ready` returns `not_ready`, and persistent asset +mutations return controlled 503 responses. Local logs may contain diagnostics; +public responses never include paths, raw manifest contents, temporary names, +tracebacks, or operating-system errors. + +Reconciliation uses deterministic rules: + +- a source record whose managed file is missing is removed; +- an output record whose managed file is missing is removed; +- an output file remains usable when its source record/file is missing, and its + metadata reports `sourceAvailable: false`; +- unreferenced source and output files are reported as orphans and are not + deleted automatically. + +Startup changes and backup restoration create timestamped JSON reports inside +`recovery/`. That directory is not served publicly. `/api/storage` exposes only +redacted counts, bytes, writability, free space when available, and the last +reconciliation summary. + +Atomic manifest replacement reduces partial-write risk but cannot make manifest +and filesystem deletion one disk transaction. Deletion first moves files to a +same-root quarantine, persists one complete manifest mutation, and restores the +files if persistence fails. A crash can still leave quarantined or orphaned +files; reconciliation reports these for explicit cleanup. diff --git a/docs/service-contract.md b/docs/service-contract.md new file mode 100644 index 0000000..e3a0967 --- /dev/null +++ b/docs/service-contract.md @@ -0,0 +1,41 @@ +# Craft service contract + +GlitchCraft exposes a local discovery and operations contract compatible with a +future Craft application dashboard: + +- `GET /app-manifest.json` — static identity, version, truthful port-5000 + defaults, endpoint links, and stable capability slugs. +- `GET /metadata` — request-derived runtime addresses, schema versions, format + and effect metadata, links, and library counts. +- `GET /health` — lightweight process/route liveness only. +- `GET /ready` — `ready`, `degraded`, or `not_ready` operational checks. +- `GET /api/capabilities` — implemented versus currently available image/video + capabilities and optional FFmpeg/FFprobe status. +- `GET /api/storage` — redacted persistent-storage counts, bytes, writability, + free space when available, and reconciliation state. + +Health does not write storage, mutate the manifest, or invoke FFmpeg. Readiness +is `not_ready` for core manifest or image-storage failure. Missing FFmpeg, +backup recovery, or reported orphans produce `degraded` while image work remains +available. An empty healthy library is `ready`. + +The canonical application version is `glitchcraft.version.APP_VERSION`; tests +prevent drift with the static manifest. The provisional SVG mark is discovery +support, not final brand identity. + +## Trust model + +The service has no authentication and is intended for loopback use only. It +should not be exposed directly to the public internet. User media, manifest +records, and recipes remain local. Metadata is redacted. No analytics, cloud +processing, external media service, or broad CORS policy is added. + +The current Flask application serves both web and API traffic at +`http://127.0.0.1:5000`. Ports 5175 and 4200 describe a possible future +frontend/API split and are not advertised as running services. There is no +dashboard remote control or application launch contract in this version. + +Persistent storage currently covers images only. Video jobs and assets remain +temporary, task state remains in memory, and audio behavior is unchanged. Saved +recipe management, WVO handoff, persistent video identity, authentication, and a +visible library interface are deferred. diff --git a/docs/storage.md b/docs/storage.md new file mode 100644 index 0000000..0f1cfa9 --- /dev/null +++ b/docs/storage.md @@ -0,0 +1,58 @@ +# Persistent image storage + +GlitchCraft 0.1.0 uses a configurable managed data root for image sources and +explicit image outputs: + +```text +data/ + manifest.json + manifest.json.bak + sources/images/ + outputs/images/ + temporary/ + recovery/ +``` + +`DATA_ROOT`, `MANIFEST_PATH`, and `TEMPORARY_FOLDER` are configurable through the +Flask factory. Launchers may use `GLITCHCRAFT_DATA_ROOT`, +`GLITCHCRAFT_MANIFEST_PATH`, and `GLITCHCRAFT_TEMPORARY_FOLDER`. Managed paths in +the manifest are normalized relative POSIX paths; absolute paths, +traversal, unknown fields, unsupported kinds, inconsistent record IDs, invalid +dimensions or MIME types, and invalid Recipe v1 snapshots are rejected. + +The schema-version-1 manifest contains immutable source metadata and output +metadata. Output records include the exact ordered recipe and seed used for +export. Record order has no meaning. Unknown manifest fields are rejected until +an explicit compatibility policy is introduced for a later schema version. + +## Lifecycle + +Uploads stage in `temporary/`, decode and validate, then move atomically to a +server-ID-derived source filename before the record is persisted. Explicit +exports follow the same staging/install/persist sequence. A failed manifest +mutation removes the newly installed file. Preview requests return PNG bytes and +never create an output record or permanent file. + +Sources and outputs are not deleted automatically by age. Output deletion is +explicit. Source deletion returns a conflict while outputs reference it unless +`cascade=true` applies one logical manifest mutation for the source and dependent +outputs. Leased assets cannot be deleted. + +Manifest writes are serialized with a process-level reentrant lock. The full +next manifest is validated, written beside the primary, flushed and fsynced, +backed up, and atomically replaced. In-memory state changes only after the +replacement succeeds. One application writer process is supported; distributed +or multi-process locking is not. + +`POST /api/storage/cleanup` defaults to dry-run. Temporary cleanup is age-based. +Orphan cleanup requires explicit opt-in and a configured minimum age. Referenced +or leased assets are never treated as cleanup candidates. + +For `/api/storage`, `schemaVersion`, `manifestState`, source/output counts and +byte counts, `temporaryBytes`, `orphanFileCount`, `cleanupAvailable`, and +`writable` are stable schema-version-1 fields. `freeBytes`, +`missingRecordCount`, and the nested reconciliation detail are operational +diagnostics whose values and granularity may vary by platform. + +Legacy video uploads, jobs, previews, and outputs remain temporary. They are not +manifest records and still use the legacy cleanup lifecycle. diff --git a/docs/testing.md b/docs/testing.md index 2a0429a..9506ebf 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -20,6 +20,7 @@ python -m pytest tests/test_contracts.py python -m pytest tests/test_routes.py python -m pytest tests/test_media.py python -m pytest tests/test_image_assets.py tests/test_image_workflow.py +python -m pytest tests/test_storage_manifest.py tests/test_service_contract.py ``` Tests use synthetic NumPy frames and temporary directories. The narrowly marked @@ -43,3 +44,9 @@ debounced newest-result behavior, explicit export/download, reset, keyboard operation, responsive containment at 320/768/1024/desktop widths, and axe results with no serious or critical violations. GitHub Actions keeps this browser job separate from the fast Python quality job. + +Storage tests cover strict path and manifest validation, atomic replacement, +backup recovery, dual-manifest failure, concurrent thread mutations, restart-safe +IDs, reconciliation, deletion rollback, cleanup, and public response redaction. +Every application fixture overrides the data root, manifest, temporary folder, +and legacy media locations with test-owned temporary directories. diff --git a/glitchcraft/application.py b/glitchcraft/application.py index b43ad36..824109e 100644 --- a/glitchcraft/application.py +++ b/glitchcraft/application.py @@ -1,40 +1,58 @@ """Flask application factory.""" +import os from pathlib import Path from typing import Any from flask import Flask -from glitchcraft.image_assets import ImageOutputStore, ImageSourceStore +from glitchcraft.storage.repository import ImageAssetRepository from glitchcraft.tasks import TaskStore +from glitchcraft.version import APP_VERSION, MANIFEST_SCHEMA_VERSION from glitchcraft.web.routes import bp def create_app(config: dict[str, Any] | None = None) -> Flask: project_root = Path(__file__).resolve().parent.parent + data_root = Path(os.environ.get("GLITCHCRAFT_DATA_ROOT", project_root / "data")) app = Flask( __name__, template_folder=str(project_root / "templates"), static_folder=str(project_root / "static"), ) app.config.from_mapping( - UPLOAD_FOLDER=str(project_root / "uploads"), - OUTPUT_FOLDER=str(project_root / "outputs"), - PREVIEW_FOLDER=str(project_root / "static" / "previews"), + APPLICATION_VERSION=APP_VERSION, + DATA_ROOT=str(data_root), + MANIFEST_PATH=os.environ.get("GLITCHCRAFT_MANIFEST_PATH", str(data_root / "manifest.json")), + MANIFEST_SCHEMA_VERSION=MANIFEST_SCHEMA_VERSION, + TEMPORARY_FOLDER=os.environ.get( + "GLITCHCRAFT_TEMPORARY_FOLDER", str(data_root / "temporary") + ), + UPLOAD_FOLDER=os.environ.get("GLITCHCRAFT_UPLOAD_FOLDER", str(project_root / "uploads")), + OUTPUT_FOLDER=os.environ.get("GLITCHCRAFT_OUTPUT_FOLDER", str(project_root / "outputs")), + PREVIEW_FOLDER=os.environ.get( + "GLITCHCRAFT_PREVIEW_FOLDER", str(project_root / "static" / "previews") + ), MAX_CONTENT_LENGTH=1024 * 1024 * 1024, MAX_IMAGE_PIXELS=40_000_000, - IMAGE_ASSET_MAX_AGE=24 * 60 * 60, + TEMPORARY_MAXIMUM_AGE=24 * 60 * 60, + ORPHAN_CLEANUP_MINIMUM_AGE_HOURS=24, ) if config: app.config.update(config) - for key in ("UPLOAD_FOLDER", "OUTPUT_FOLDER", "PREVIEW_FOLDER"): + data_root = Path(app.config["DATA_ROOT"]) + if config and "DATA_ROOT" in config and "MANIFEST_PATH" not in config: + app.config["MANIFEST_PATH"] = str(data_root / "manifest.json") + if config and "DATA_ROOT" in config and "TEMPORARY_FOLDER" not in config: + app.config["TEMPORARY_FOLDER"] = str(data_root / "temporary") + for key in ("UPLOAD_FOLDER", "OUTPUT_FOLDER", "PREVIEW_FOLDER", "TEMPORARY_FOLDER"): Path(app.config[key]).mkdir(parents=True, exist_ok=True) app.extensions["task_store"] = TaskStore() - app.extensions["image_source_store"] = ImageSourceStore( - maximum_age=float(app.config["IMAGE_ASSET_MAX_AGE"]) - ) - app.extensions["image_output_store"] = ImageOutputStore( - maximum_age=float(app.config["IMAGE_ASSET_MAX_AGE"]) + app.extensions["image_repository"] = ImageAssetRepository( + data_root=data_root, + manifest_path=Path(app.config["MANIFEST_PATH"]), + temporary_folder=Path(app.config["TEMPORARY_FOLDER"]), + temporary_maximum_age=float(app.config["TEMPORARY_MAXIMUM_AGE"]), ) app.register_blueprint(bp) return app diff --git a/glitchcraft/image_assets.py b/glitchcraft/image_assets.py deleted file mode 100644 index 4be69f0..0000000 --- a/glitchcraft/image_assets.py +++ /dev/null @@ -1,167 +0,0 @@ -"""Opaque, thread-safe identity for temporary image sources and outputs.""" - -from __future__ import annotations - -import secrets -import time -from collections.abc import Iterator -from contextlib import contextmanager -from dataclasses import dataclass -from pathlib import Path -from threading import RLock -from typing import Generic, TypeVar - -RecordT = TypeVar("RecordT", bound="ImageAssetRecord") - - -@dataclass(frozen=True) -class ImageAssetRecord: - id: str - path: Path - display_name: str - width: int - height: int - created_at: float - - -@dataclass(frozen=True) -class ImageSourceRecord(ImageAssetRecord): - original_name: str - image_format: str - mime_type: str - seed: int - - -@dataclass(frozen=True) -class ImageOutputRecord(ImageAssetRecord): - source_id: str - recipe_json: str - seed: int - mime_type: str = "image/png" - - -class _ImageAssetStore(Generic[RecordT]): - """Store records independently of paths visible to the browser.""" - - def __init__(self, maximum_age: float = 86400) -> None: - self.maximum_age = maximum_age - self._records: dict[str, RecordT] = {} - self._leases: dict[str, int] = {} - self._lock = RLock() - - def _new_id(self) -> str: - return secrets.token_urlsafe(24) - - def _add(self, record: RecordT) -> RecordT: - with self._lock: - self._records[record.id] = record - self._leases[record.id] = 0 - return record - - def get(self, asset_id: str) -> RecordT | None: - with self._lock: - record = self._records.get(asset_id) - if record is None or self._is_expired(record, time.time()): - return None - if not record.path.is_file(): - self._discard_locked(asset_id) - return None - return record - - @contextmanager - def lease(self, asset_id: str) -> Iterator[RecordT]: - with self._lock: - record = self.get(asset_id) - if record is None: - raise KeyError(asset_id) - self._leases[asset_id] += 1 - try: - yield record - finally: - with self._lock: - if asset_id in self._leases: - self._leases[asset_id] = max(0, self._leases[asset_id] - 1) - - def known_paths(self) -> set[Path]: - with self._lock: - return {record.path.resolve() for record in self._records.values()} - - def cleanup_expired(self, now: float | None = None) -> int: - current = time.time() if now is None else now - removed = 0 - with self._lock: - for asset_id, record in list(self._records.items()): - if self._leases[asset_id] == 0 and self._is_expired(record, current): - record.path.unlink(missing_ok=True) - self._discard_locked(asset_id) - removed += 1 - return removed - - def __len__(self) -> int: - with self._lock: - return len(self._records) - - def _is_expired(self, record: RecordT, now: float) -> bool: - return now - record.created_at >= self.maximum_age - - def _discard_locked(self, asset_id: str) -> None: - self._records.pop(asset_id, None) - self._leases.pop(asset_id, None) - - -class ImageSourceStore(_ImageAssetStore[ImageSourceRecord]): - def create( - self, - *, - path: Path, - original_name: str, - display_name: str, - width: int, - height: int, - image_format: str, - mime_type: str, - seed: int, - created_at: float | None = None, - ) -> ImageSourceRecord: - return self._add( - ImageSourceRecord( - id=self._new_id(), - path=path, - original_name=original_name, - display_name=display_name, - width=width, - height=height, - image_format=image_format, - mime_type=mime_type, - seed=seed, - created_at=time.time() if created_at is None else created_at, - ) - ) - - -class ImageOutputStore(_ImageAssetStore[ImageOutputRecord]): - def create( - self, - *, - path: Path, - source_id: str, - display_name: str, - recipe_json: str, - seed: int, - width: int, - height: int, - created_at: float | None = None, - ) -> ImageOutputRecord: - return self._add( - ImageOutputRecord( - id=self._new_id(), - path=path, - source_id=source_id, - display_name=display_name, - recipe_json=recipe_json, - seed=seed, - width=width, - height=height, - created_at=time.time() if created_at is None else created_at, - ) - ) diff --git a/glitchcraft/service_contract.py b/glitchcraft/service_contract.py new file mode 100644 index 0000000..b90d862 --- /dev/null +++ b/glitchcraft/service_contract.py @@ -0,0 +1,96 @@ +"""Machine-readable Craft discovery and runtime capability metadata.""" + +from __future__ import annotations + +import shutil +from typing import Any + +from glitchcraft.effects.registry import EFFECT_REGISTRY +from glitchcraft.media.image_io import SUPPORTED_IMAGE_FORMATS + +CAPABILITY_SLUGS = ( + "image-effects", + "video-effects", + "seeded-effects", + "ordered-effect-recipes", + "inline-image-preview", + "image-export", + "persistent-image-library", +) +SUPPORTED_VIDEO_EXTENSIONS = ("avi", "mkv", "mov", "mp4") + + +def ffmpeg_available() -> bool: + return shutil.which("ffmpeg") is not None + + +def ffprobe_available() -> bool: + return shutil.which("ffprobe") is not None + + +def image_formats() -> list[dict[str, Any]]: + return [ + { + "format": image_format, + "mimeType": values[0], + "extensions": sorted(values[1]), + } + for image_format, values in SUPPORTED_IMAGE_FORMATS.items() + ] + + +def effect_metadata() -> list[dict[str, Any]]: + return [ + { + "type": definition.type.value, + "name": definition.display_name, + "mediaTypes": list(definition.media_types), + "stochastic": definition.stochastic, + } + for definition in EFFECT_REGISTRY.values() + ] + + +def capability_details(*, storage_available: bool) -> list[dict[str, Any]]: + video_ready = ffmpeg_available() + return [ + {"slug": "image-effects", "exists": True, "available": True, "optional": False}, + { + "slug": "video-effects", + "exists": True, + "available": video_ready, + "optional": True, + }, + {"slug": "seeded-effects", "exists": True, "available": True, "optional": False}, + { + "slug": "ordered-effect-recipes", + "exists": True, + "available": True, + "optional": False, + }, + { + "slug": "inline-image-preview", + "exists": True, + "available": storage_available, + "optional": False, + }, + { + "slug": "image-export", + "exists": True, + "available": storage_available, + "optional": False, + }, + { + "slug": "persistent-image-library", + "exists": True, + "available": storage_available, + "optional": False, + }, + {"slug": "video-preview", "exists": True, "available": True, "optional": True}, + { + "slug": "video-full-processing", + "exists": True, + "available": video_ready, + "optional": True, + }, + ] diff --git a/glitchcraft/storage/__init__.py b/glitchcraft/storage/__init__.py new file mode 100644 index 0000000..fda7f60 --- /dev/null +++ b/glitchcraft/storage/__init__.py @@ -0,0 +1,25 @@ +"""Persistent managed storage for image sources and explicit outputs.""" + +from glitchcraft.storage.errors import ( + AssetConflictError, + AssetNotFoundError, + ManagedPathError, + ManifestReadError, + ManifestValidationError, + ManifestWriteError, + StorageError, + StorageUnavailableError, +) +from glitchcraft.storage.repository import ImageAssetRepository + +__all__ = [ + "AssetConflictError", + "AssetNotFoundError", + "ImageAssetRepository", + "ManagedPathError", + "ManifestReadError", + "ManifestValidationError", + "ManifestWriteError", + "StorageError", + "StorageUnavailableError", +] diff --git a/glitchcraft/storage/contracts.py b/glitchcraft/storage/contracts.py new file mode 100644 index 0000000..2911aec --- /dev/null +++ b/glitchcraft/storage/contracts.py @@ -0,0 +1,162 @@ +"""Strict schema-version-1 contracts for persistent image storage.""" + +from __future__ import annotations + +from datetime import datetime, timedelta +from pathlib import PurePosixPath +from typing import Annotated, Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from glitchcraft.contracts.effects import Recipe +from glitchcraft.version import MANIFEST_SCHEMA_VERSION + +PositiveDimension = Annotated[int, Field(ge=1, le=1_000_000)] +OpaqueId = Annotated[str, Field(min_length=16, max_length=256, pattern=r"^[A-Za-z0-9_-]+$")] +SOURCE_FORMATS = { + "PNG": ("image/png", {".png"}), + "JPEG": ("image/jpeg", {".jpg", ".jpeg"}), + "BMP": ("image/bmp", {".bmp"}), + "TIFF": ("image/tiff", {".tif", ".tiff"}), +} + + +class StorageModel(BaseModel): + """Strict, immutable persistence model with stable JSON aliases.""" + + model_config = ConfigDict( + alias_generator=lambda value: "".join( + [value.split("_")[0], *[part.title() for part in value.split("_")[1:]]] + ), + populate_by_name=True, + extra="forbid", + frozen=True, + ) + + +def validate_managed_path(value: str) -> str: + """Accept only normalized relative POSIX paths without traversal.""" + + if not value or "\\" in value: + raise ValueError("managed paths must use relative POSIX syntax") + path = PurePosixPath(value) + if path.is_absolute() or ".." in path.parts or "." in path.parts: + raise ValueError("managed paths must not be absolute or contain traversal") + if str(path) != value: + raise ValueError("managed paths must be normalized") + return value + + +class ImageSourceRecord(StorageModel): + id: OpaqueId + kind: Literal["image"] = "image" + original_name: Annotated[str, Field(min_length=1, max_length=255)] + stored_name: Annotated[str, Field(min_length=1, max_length=512)] + format: Literal["PNG", "JPEG", "BMP", "TIFF"] + mime_type: Literal["image/png", "image/jpeg", "image/bmp", "image/tiff"] + width: PositiveDimension + height: PositiveDimension + seed: Annotated[int, Field(ge=0, le=(1 << 63) - 1)] + created_at: datetime + + @field_validator("stored_name") + @classmethod + def validate_path(cls, value: str) -> str: + value = validate_managed_path(value) + if not value.startswith("sources/images/"): + raise ValueError("source paths must be inside sources/images") + return value + + @field_validator("created_at") + @classmethod + def validate_timestamp(cls, value: datetime) -> datetime: + if value.tzinfo is None or value.utcoffset() != timedelta(0): + raise ValueError("createdAt must use UTC") + return value + + @model_validator(mode="after") + def validate_format_metadata(self) -> ImageSourceRecord: + expected_mime, extensions = SOURCE_FORMATS[self.format] + if self.mime_type != expected_mime: + raise ValueError("source format and MIME type are inconsistent") + if PurePosixPath(self.stored_name).suffix.lower() not in extensions: + raise ValueError("source format and stored extension are inconsistent") + return self + + +class ImageOutputRecord(StorageModel): + id: OpaqueId + kind: Literal["image"] = "image" + source_id: OpaqueId + file_name: Annotated[str, Field(min_length=1, max_length=255)] + stored_name: Annotated[str, Field(min_length=1, max_length=512)] + mime_type: Literal["image/png"] = "image/png" + width: PositiveDimension + height: PositiveDimension + recipe: Recipe + created_at: datetime + + @field_validator("stored_name") + @classmethod + def validate_path(cls, value: str) -> str: + value = validate_managed_path(value) + if not value.startswith("outputs/images/"): + raise ValueError("output paths must be inside outputs/images") + return value + + @field_validator("created_at") + @classmethod + def validate_timestamp(cls, value: datetime) -> datetime: + if value.tzinfo is None or value.utcoffset() != timedelta(0): + raise ValueError("createdAt must use UTC") + return value + + @model_validator(mode="after") + def validate_output_extension(self) -> ImageOutputRecord: + if PurePosixPath(self.stored_name).suffix.lower() != ".png": + raise ValueError("image outputs must use the PNG extension") + return self + + @property + def seed(self) -> int: + return self.recipe.seed + + +class ManifestDocument(StorageModel): + schema_version: Annotated[int, Field(ge=MANIFEST_SCHEMA_VERSION, le=MANIFEST_SCHEMA_VERSION)] + sources: dict[str, ImageSourceRecord] = Field(default_factory=dict) + outputs: dict[str, ImageOutputRecord] = Field(default_factory=dict) + + @model_validator(mode="after") + def validate_record_keys(self) -> ManifestDocument: + if any(key != record.id for key, record in self.sources.items()): + raise ValueError("source record keys must match record IDs") + if any(key != record.id for key, record in self.outputs.items()): + raise ValueError("output record keys must match record IDs") + return self + + +class ReconciliationReport(StorageModel): + state: Literal["clean", "changed", "recovered", "unavailable"] + recovered_from_backup: bool = False + missing_sources: int = 0 + missing_outputs: int = 0 + orphan_sources: int = 0 + orphan_outputs: int = 0 + manifest_changed: bool = False + occurred_at: datetime + + +class CleanupRequest(StorageModel): + dry_run: bool = True + remove_temporary: bool = True + remove_orphans: bool = False + orphan_minimum_age_hours: Annotated[float, Field(ge=1, le=24 * 365)] = 24 + + +class CleanupResult(StorageModel): + dry_run: bool + temporary_files: int + temporary_bytes: int + orphan_files: int + orphan_bytes: int diff --git a/glitchcraft/storage/errors.py b/glitchcraft/storage/errors.py new file mode 100644 index 0000000..04ddc5d --- /dev/null +++ b/glitchcraft/storage/errors.py @@ -0,0 +1,33 @@ +"""Typed storage failures safe for translation at the HTTP boundary.""" + + +class StorageError(Exception): + """Base class for managed-storage failures.""" + + +class ManifestReadError(StorageError): + """The manifest could not be read.""" + + +class ManifestWriteError(StorageError): + """A validated manifest could not be persisted atomically.""" + + +class ManifestValidationError(StorageError): + """Manifest data does not satisfy the supported contract.""" + + +class AssetNotFoundError(StorageError): + """A requested managed asset is unknown or unavailable.""" + + +class AssetConflictError(StorageError): + """An asset mutation conflicts with references or active use.""" + + +class ManagedPathError(StorageError): + """A managed relative path is unsafe or outside its expected area.""" + + +class StorageUnavailableError(StorageError): + """Persistent storage is not available for the requested operation.""" diff --git a/glitchcraft/storage/manifest.py b/glitchcraft/storage/manifest.py new file mode 100644 index 0000000..0a24708 --- /dev/null +++ b/glitchcraft/storage/manifest.py @@ -0,0 +1,100 @@ +"""Validated loading and atomic replacement of the storage manifest.""" + +from __future__ import annotations + +import json +import os +import tempfile +from pathlib import Path + +from pydantic import ValidationError + +from glitchcraft.storage.contracts import ManifestDocument +from glitchcraft.storage.errors import ( + ManifestReadError, + ManifestValidationError, + ManifestWriteError, +) + + +def serialize_manifest(manifest: ManifestDocument) -> bytes: + payload = manifest.model_dump(mode="json", by_alias=True) + return (json.dumps(payload, indent=2, sort_keys=True) + "\n").encode() + + +def load_manifest(path: Path) -> ManifestDocument: + try: + raw = path.read_bytes() + except OSError as exc: + raise ManifestReadError("The storage manifest could not be read.") from exc + try: + value = json.loads(raw) + return ManifestDocument.model_validate(value) + except (json.JSONDecodeError, UnicodeDecodeError, ValidationError) as exc: + raise ManifestValidationError("The storage manifest is invalid.") from exc + + +class AtomicManifestWriter: + """Persist complete manifests with same-filesystem replacement and backup.""" + + def __init__(self, path: Path, backup_path: Path) -> None: + self.path = path + self.backup_path = backup_path + + def write(self, manifest: ManifestDocument, *, preserve_backup: bool = False) -> None: + data = serialize_manifest(manifest) + temporary: Path | None = None + try: + temporary = self._write_temporary(self.path.parent, data) + if self.path.exists() and not preserve_backup: + self._write_backup(self.path.read_bytes()) + self._replace(temporary, self.path) + temporary = None + self._fsync_directory(self.path.parent) + if not self.backup_path.exists(): + self._write_backup(data) + except (OSError, ManifestWriteError) as exc: + raise ManifestWriteError("The storage manifest could not be saved.") from exc + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + + def _write_backup(self, data: bytes) -> None: + temporary = self._write_temporary(self.backup_path.parent, data) + try: + self._replace(temporary, self.backup_path) + self._fsync_directory(self.backup_path.parent) + finally: + temporary.unlink(missing_ok=True) + + @staticmethod + def _write_temporary(folder: Path, data: bytes) -> Path: + folder.mkdir(parents=True, exist_ok=True) + descriptor, name = tempfile.mkstemp(prefix=".manifest-", suffix=".tmp", dir=folder) + path = Path(name) + try: + with os.fdopen(descriptor, "wb") as stream: + descriptor = -1 + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + except Exception: + if descriptor >= 0: + os.close(descriptor) + path.unlink(missing_ok=True) + raise + return path + + @staticmethod + def _replace(source: Path, destination: Path) -> None: + os.replace(source, destination) + + @staticmethod + def _fsync_directory(folder: Path) -> None: + if os.name == "nt": + return + descriptor = os.open(folder, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) diff --git a/glitchcraft/storage/repository.py b/glitchcraft/storage/repository.py new file mode 100644 index 0000000..c9f2fc5 --- /dev/null +++ b/glitchcraft/storage/repository.py @@ -0,0 +1,604 @@ +"""Thread-safe, single-process repository for persistent image assets.""" + +from __future__ import annotations + +import json +import logging +import os +import secrets +import shutil +import tempfile +from collections.abc import Iterable, Iterator +from contextlib import contextmanager +from datetime import UTC, datetime +from pathlib import Path +from threading import RLock + +from pydantic import ValidationError + +from glitchcraft.contracts.effects import Recipe +from glitchcraft.storage.contracts import ( + CleanupRequest, + CleanupResult, + ImageOutputRecord, + ImageSourceRecord, + ManifestDocument, + ReconciliationReport, +) +from glitchcraft.storage.errors import ( + AssetConflictError, + AssetNotFoundError, + ManagedPathError, + ManifestReadError, + ManifestValidationError, + ManifestWriteError, + StorageUnavailableError, +) +from glitchcraft.storage.manifest import AtomicManifestWriter, load_manifest +from glitchcraft.version import MANIFEST_SCHEMA_VERSION + +logger = logging.getLogger(__name__) + + +def _utc_now() -> datetime: + return datetime.now(UTC) + + +class ImageAssetRepository: + """Own managed paths, persistence, leases, reconciliation, and cleanup. + + Writes are safe between threads in one application process. Multiple writer + processes sharing a data root are intentionally unsupported. + """ + + def __init__( + self, + data_root: Path, + manifest_path: Path, + temporary_folder: Path, + *, + temporary_maximum_age: float = 86400, + ) -> None: + self.data_root = data_root.resolve() + self.manifest_path = manifest_path.resolve() + self.backup_path = self.manifest_path.with_name(f"{self.manifest_path.name}.bak") + self.temporary_folder = temporary_folder.resolve() + for managed_path in (self.manifest_path, self.temporary_folder): + try: + managed_path.relative_to(self.data_root) + except ValueError as exc: + raise ManagedPathError( + "Managed storage locations must remain inside the data root." + ) from exc + self.source_folder = self.data_root / "sources" / "images" + self.output_folder = self.data_root / "outputs" / "images" + self.recovery_folder = self.data_root / "recovery" + self.temporary_maximum_age = temporary_maximum_age + self._lock = RLock() + self._leases: dict[tuple[str, str], int] = {} + self._manifest = ManifestDocument( + schema_version=MANIFEST_SCHEMA_VERSION, sources={}, outputs={} + ) + self._available = True + self._warning: str | None = None + self._writer = AtomicManifestWriter(self.manifest_path, self.backup_path) + self._last_report = ReconciliationReport( + state="clean", + occurred_at=_utc_now(), + ) + self._initialize() + + @property + def available(self) -> bool: + with self._lock: + return self._available + + @property + def warning(self) -> str | None: + with self._lock: + return self._warning + + @property + def last_report(self) -> ReconciliationReport: + with self._lock: + return self._last_report.model_copy(deep=True) + + def _initialize(self) -> None: + for folder in ( + self.data_root, + self.source_folder, + self.output_folder, + self.temporary_folder, + self.recovery_folder, + ): + folder.mkdir(parents=True, exist_ok=True) + + recovered = False + if self.manifest_path.exists(): + try: + manifest = load_manifest(self.manifest_path) + except (ManifestReadError, ManifestValidationError) as primary_error: + manifest = self._recover_backup(primary_error) + recovered = True + elif self.backup_path.exists(): + try: + manifest = load_manifest(self.backup_path) + self._writer.write(manifest, preserve_backup=True) + recovered = True + self._warning = "The primary manifest was restored from backup." + except (ManifestReadError, ManifestValidationError, ManifestWriteError) as exc: + self._mark_unavailable(exc) + return + else: + manifest = ManifestDocument( + schema_version=MANIFEST_SCHEMA_VERSION, sources={}, outputs={} + ) + try: + self._writer.write(manifest) + except ManifestWriteError as exc: + self._mark_unavailable(exc) + return + + self._manifest = manifest + self._reconcile(recovered=recovered) + + def _recover_backup(self, primary_error: Exception) -> ManifestDocument: + if not self.backup_path.exists(): + self._mark_unavailable(primary_error) + return self._manifest + try: + manifest = load_manifest(self.backup_path) + self._writer.write(manifest, preserve_backup=True) + self._warning = "The primary manifest was restored from backup." + return manifest + except (ManifestReadError, ManifestValidationError, ManifestWriteError) as exc: + self._mark_unavailable(exc) + return self._manifest + + def _mark_unavailable(self, exc: Exception) -> None: + logger.error("Persistent storage is unavailable: %s", exc) + self._available = False + self._warning = "Persistent storage is unavailable." + self._last_report = ReconciliationReport( + state="unavailable", + occurred_at=_utc_now(), + ) + + def _require_available(self) -> None: + if not self._available: + raise StorageUnavailableError("Persistent image storage is unavailable.") + + def _reconcile(self, *, recovered: bool) -> None: + if not self._available: + return + sources = dict(self._manifest.sources) + outputs = dict(self._manifest.outputs) + missing_sources = [ + asset_id + for asset_id, record in sources.items() + if not self._resolve(record.stored_name, "sources/images").is_file() + ] + missing_outputs = [ + asset_id + for asset_id, record in outputs.items() + if not self._resolve(record.stored_name, "outputs/images").is_file() + ] + for asset_id in missing_sources: + sources.pop(asset_id) + for asset_id in missing_outputs: + outputs.pop(asset_id) + + changed = bool(missing_sources or missing_outputs) + if changed: + next_manifest = ManifestDocument( + schema_version=MANIFEST_SCHEMA_VERSION, + sources=sources, + outputs=outputs, + ) + try: + self._writer.write(next_manifest) + self._manifest = next_manifest + except ManifestWriteError as exc: + self._mark_unavailable(exc) + return + + orphan_sources, orphan_outputs = self._orphan_paths() + state = "recovered" if recovered else ("changed" if changed else "clean") + self._last_report = ReconciliationReport( + state=state, + recovered_from_backup=recovered, + missing_sources=len(missing_sources), + missing_outputs=len(missing_outputs), + orphan_sources=len(orphan_sources), + orphan_outputs=len(orphan_outputs), + manifest_changed=changed, + occurred_at=_utc_now(), + ) + if recovered or changed: + self._write_recovery_report(self._last_report) + + def _write_recovery_report(self, report: ReconciliationReport) -> None: + name = f"reconciliation-{_utc_now().strftime('%Y%m%dT%H%M%S%fZ')}.json" + path = self.recovery_folder / name + try: + path.write_text( + json.dumps(report.model_dump(mode="json", by_alias=True), indent=2) + "\n", + encoding="utf-8", + ) + except OSError: + logger.exception("Could not write a local storage recovery report") + + def new_temporary_path(self, suffix: str) -> Path: + with self._lock: + self._require_available() + safe_suffix = suffix if suffix.startswith(".") and suffix[1:].isalnum() else ".tmp" + descriptor, name = tempfile.mkstemp( + prefix="asset-", suffix=safe_suffix, dir=self.temporary_folder + ) + os.close(descriptor) + return Path(name) + + def create_source( + self, + *, + staged_path: Path, + extension: str, + original_name: str, + image_format: str, + mime_type: str, + width: int, + height: int, + seed: int, + ) -> ImageSourceRecord: + with self._lock: + self._require_available() + asset_id = self._new_id() + stored_name = f"sources/images/{asset_id}.{extension.lower()}" + final_path = self._resolve(stored_name, "sources/images") + record = ImageSourceRecord( + id=asset_id, + original_name=original_name, + stored_name=stored_name, + format=image_format, + mime_type=mime_type, + width=width, + height=height, + seed=seed, + created_at=_utc_now(), + ) + if final_path.exists(): + raise AssetConflictError("A managed source already exists.") + try: + os.replace(staged_path, final_path) + next_manifest = self._manifest.model_copy( + update={"sources": {**self._manifest.sources, asset_id: record}} + ) + next_manifest = ManifestDocument.model_validate(next_manifest.model_dump()) + self._writer.write(next_manifest) + except (OSError, ValidationError, ManifestWriteError) as exc: + final_path.unlink(missing_ok=True) + if isinstance(exc, ManifestWriteError): + raise + raise ManifestWriteError("The image source could not be saved.") from exc + self._manifest = next_manifest + self._leases[("source", asset_id)] = 0 + return record.model_copy(deep=True) + + def create_output( + self, + *, + staged_path: Path, + source_id: str, + file_name: str, + width: int, + height: int, + recipe: Recipe, + ) -> ImageOutputRecord: + with self._lock: + self._require_available() + if source_id not in self._manifest.sources: + raise AssetNotFoundError("Unknown image source.") + asset_id = self._new_id() + stored_name = f"outputs/images/{asset_id}.png" + final_path = self._resolve(stored_name, "outputs/images") + record = ImageOutputRecord( + id=asset_id, + source_id=source_id, + file_name=file_name, + stored_name=stored_name, + width=width, + height=height, + recipe=recipe, + created_at=_utc_now(), + ) + if final_path.exists(): + raise AssetConflictError("A managed output already exists.") + try: + os.replace(staged_path, final_path) + next_manifest = self._manifest.model_copy( + update={"outputs": {**self._manifest.outputs, asset_id: record}} + ) + next_manifest = ManifestDocument.model_validate(next_manifest.model_dump()) + self._writer.write(next_manifest) + except (OSError, ValidationError, ManifestWriteError) as exc: + final_path.unlink(missing_ok=True) + if isinstance(exc, ManifestWriteError): + raise + raise ManifestWriteError("The image output could not be saved.") from exc + self._manifest = next_manifest + self._leases[("output", asset_id)] = 0 + return record.model_copy(deep=True) + + def get_source(self, asset_id: str) -> ImageSourceRecord: + with self._lock: + self._require_available() + record = self._manifest.sources.get(asset_id) + if record is None or not self._resolve(record.stored_name, "sources/images").is_file(): + raise AssetNotFoundError("Unknown image source.") + return record.model_copy(deep=True) + + def get_output(self, asset_id: str) -> ImageOutputRecord: + with self._lock: + self._require_available() + record = self._manifest.outputs.get(asset_id) + if record is None or not self._resolve(record.stored_name, "outputs/images").is_file(): + raise AssetNotFoundError("Unknown image output.") + return record.model_copy(deep=True) + + def list_sources(self) -> list[ImageSourceRecord]: + with self._lock: + self._require_available() + return sorted( + (record.model_copy(deep=True) for record in self._manifest.sources.values()), + key=lambda record: record.created_at, + reverse=True, + ) + + def list_outputs(self) -> list[ImageOutputRecord]: + with self._lock: + self._require_available() + return sorted( + (record.model_copy(deep=True) for record in self._manifest.outputs.values()), + key=lambda record: record.created_at, + reverse=True, + ) + + def output_count(self, source_id: str) -> int: + with self._lock: + return sum(record.source_id == source_id for record in self._manifest.outputs.values()) + + def source_available(self, source_id: str) -> bool: + with self._lock: + record = self._manifest.sources.get(source_id) + return ( + record is not None and self._resolve(record.stored_name, "sources/images").is_file() + ) + + @contextmanager + def lease_source(self, asset_id: str) -> Iterator[tuple[ImageSourceRecord, Path]]: + with self._lease("source", asset_id) as value: + record, path = value + yield record, path # type: ignore[misc] + + @contextmanager + def lease_output(self, asset_id: str) -> Iterator[tuple[ImageOutputRecord, Path]]: + with self._lease("output", asset_id) as value: + record, path = value + yield record, path # type: ignore[misc] + + @contextmanager + def _lease( + self, kind: str, asset_id: str + ) -> Iterator[tuple[ImageSourceRecord | ImageOutputRecord, Path]]: + with self._lock: + record = self.get_source(asset_id) if kind == "source" else self.get_output(asset_id) + key = (kind, asset_id) + self._leases[key] = self._leases.get(key, 0) + 1 + path = self._resolve(record.stored_name, f"{kind}s/images") + try: + yield record, path + finally: + with self._lock: + self._leases[key] = max(0, self._leases.get(key, 1) - 1) + + def delete_output(self, asset_id: str) -> None: + with self._lock: + self._require_available() + record = self._manifest.outputs.get(asset_id) + if record is None: + raise AssetNotFoundError("Unknown image output.") + self._ensure_not_leased("output", asset_id) + self._delete_records([], [record]) + + def delete_source(self, asset_id: str, *, cascade: bool) -> int: + with self._lock: + self._require_available() + source = self._manifest.sources.get(asset_id) + if source is None: + raise AssetNotFoundError("Unknown image source.") + outputs = [ + record for record in self._manifest.outputs.values() if record.source_id == asset_id + ] + if outputs and not cascade: + raise AssetConflictError("The image source still has exported outputs.") + self._ensure_not_leased("source", asset_id) + for output in outputs: + self._ensure_not_leased("output", output.id) + self._delete_records([source], outputs) + return len(outputs) + + def _delete_records( + self, + sources: list[ImageSourceRecord], + outputs: list[ImageOutputRecord], + ) -> None: + moved: list[tuple[Path, Path]] = [] + try: + for kind, records in (("sources/images", sources), ("outputs/images", outputs)): + for record in records: + original = self._resolve(record.stored_name, kind) + if original.exists(): + quarantine = self.temporary_folder / f"delete-{secrets.token_hex(16)}" + os.replace(original, quarantine) + moved.append((original, quarantine)) + next_sources = dict(self._manifest.sources) + next_outputs = dict(self._manifest.outputs) + for record in sources: + next_sources.pop(record.id, None) + for record in outputs: + next_outputs.pop(record.id, None) + next_manifest = ManifestDocument( + schema_version=MANIFEST_SCHEMA_VERSION, + sources=next_sources, + outputs=next_outputs, + ) + self._writer.write(next_manifest) + except (OSError, ManifestWriteError) as exc: + for original, quarantine in reversed(moved): + if quarantine.exists(): + os.replace(quarantine, original) + if isinstance(exc, ManifestWriteError): + raise + raise ManifestWriteError("The asset deletion could not be saved.") from exc + self._manifest = next_manifest + for _, quarantine in moved: + try: + quarantine.unlink(missing_ok=True) + except OSError: + logger.exception("Could not remove a committed deletion quarantine file") + + def _ensure_not_leased(self, kind: str, asset_id: str) -> None: + if self._leases.get((kind, asset_id), 0): + raise AssetConflictError("The image asset is currently in use.") + + def status(self) -> dict[str, object]: + with self._lock: + source_bytes = self._referenced_bytes(self._manifest.sources.values()) + output_bytes = self._referenced_bytes(self._manifest.outputs.values()) + temporary_bytes = self._folder_bytes(self.temporary_folder) + orphan_sources, orphan_outputs = self._orphan_paths() + try: + free_bytes: int | None = shutil.disk_usage(self.data_root).free + except OSError: + free_bytes = None + return { + "schemaVersion": MANIFEST_SCHEMA_VERSION, + "manifestState": "available" if self._available else "unavailable", + "sourceCount": len(self._manifest.sources), + "outputCount": len(self._manifest.outputs), + "sourceBytes": source_bytes, + "outputBytes": output_bytes, + "temporaryBytes": temporary_bytes, + "orphanFileCount": len(orphan_sources) + len(orphan_outputs), + "missingRecordCount": ( + self._last_report.missing_sources + self._last_report.missing_outputs + ), + "freeBytes": free_bytes, + "cleanupAvailable": self._available, + "writable": self.is_writable(), + "reconciliation": self._last_report.model_dump(mode="json", by_alias=True), + } + + def is_writable(self) -> bool: + with self._lock: + if not self._available: + return False + try: + descriptor, name = tempfile.mkstemp( + prefix=".write-check-", dir=self.temporary_folder + ) + os.close(descriptor) + Path(name).unlink() + return True + except OSError: + return False + + def cleanup(self, options: CleanupRequest) -> CleanupResult: + with self._lock: + self._require_available() + now = _utc_now().timestamp() + temporary = [ + path + for path in self.temporary_folder.iterdir() + if path.is_file() and now - path.stat().st_mtime >= self.temporary_maximum_age + ] + orphan_sources, orphan_outputs = self._orphan_paths() + orphan_cutoff = options.orphan_minimum_age_hours * 3600 + orphans = [ + path + for path in (*orphan_sources, *orphan_outputs) + if now - path.stat().st_mtime >= orphan_cutoff + ] + selected_temporary = temporary if options.remove_temporary else [] + selected_orphans = orphans if options.remove_orphans else [] + result = CleanupResult( + dry_run=options.dry_run, + temporary_files=len(selected_temporary), + temporary_bytes=sum(path.stat().st_size for path in selected_temporary), + orphan_files=len(selected_orphans), + orphan_bytes=sum(path.stat().st_size for path in selected_orphans), + ) + if not options.dry_run: + for path in (*selected_temporary, *selected_orphans): + path.unlink(missing_ok=True) + self._refresh_orphan_report() + return result + + def _refresh_orphan_report(self) -> None: + orphan_sources, orphan_outputs = self._orphan_paths() + self._last_report = self._last_report.model_copy( + update={ + "orphan_sources": len(orphan_sources), + "orphan_outputs": len(orphan_outputs), + } + ) + + def _orphan_paths(self) -> tuple[list[Path], list[Path]]: + referenced_sources = { + self._resolve(record.stored_name, "sources/images") + for record in self._manifest.sources.values() + } + referenced_outputs = { + self._resolve(record.stored_name, "outputs/images") + for record in self._manifest.outputs.values() + } + source_orphans = [ + path + for path in self.source_folder.iterdir() + if path.is_file() and path.resolve() not in referenced_sources + ] + output_orphans = [ + path + for path in self.output_folder.iterdir() + if path.is_file() and path.resolve() not in referenced_outputs + ] + return source_orphans, output_orphans + + def _resolve(self, stored_name: str, expected_prefix: str) -> Path: + if not stored_name.startswith(f"{expected_prefix}/"): + raise ManagedPathError("The managed asset path is invalid.") + candidate = (self.data_root / Path(*stored_name.split("/"))).resolve() + try: + candidate.relative_to(self.data_root) + except ValueError as exc: + raise ManagedPathError("The managed asset path is invalid.") from exc + return candidate + + def _referenced_bytes(self, records: Iterable[ImageSourceRecord | ImageOutputRecord]) -> int: + total = 0 + for record in records: + path = self._resolve( + record.stored_name, + "sources/images" if isinstance(record, ImageSourceRecord) else "outputs/images", + ) + if path.is_file(): + total += path.stat().st_size + return total + + @staticmethod + def _folder_bytes(folder: Path) -> int: + return sum(path.stat().st_size for path in folder.iterdir() if path.is_file()) + + @staticmethod + def _new_id() -> str: + return secrets.token_urlsafe(24) diff --git a/glitchcraft/version.py b/glitchcraft/version.py new file mode 100644 index 0000000..932e945 --- /dev/null +++ b/glitchcraft/version.py @@ -0,0 +1,9 @@ +"""Canonical GlitchCraft application and contract versions.""" + +APP_ID = "glitchcraft" +APP_NAME = "GlitchCraft" +APP_DESCRIPTOR = "Local visual-effects workspace" +APP_VERSION = "0.1.0" +MANIFEST_SCHEMA_VERSION = 1 +RECIPE_SCHEMA_VERSION = 1 +STORAGE_SCHEMA_VERSION = 1 diff --git a/glitchcraft/web/routes.py b/glitchcraft/web/routes.py index 224643d..2579d77 100644 --- a/glitchcraft/web/routes.py +++ b/glitchcraft/web/routes.py @@ -28,13 +28,8 @@ from glitchcraft.contracts.image_workflow import ImageRecipeRequest, ImageSourceOptions from glitchcraft.contracts.legacy import FullVideoRequest, LegacyParameters, UploadMode from glitchcraft.effects.engine import apply_effect_stack +from glitchcraft.effects.registry import EFFECT_REGISTRY from glitchcraft.errors import GlitchCraftError -from glitchcraft.image_assets import ( - ImageOutputRecord, - ImageOutputStore, - ImageSourceRecord, - ImageSourceStore, -) from glitchcraft.media.ffmpeg import reencode_for_browser from glitchcraft.media.image_io import ( SUPPORTED_IMAGE_FORMATS, @@ -44,7 +39,36 @@ save_image_rgb, ) from glitchcraft.media.video import create_video_preview, process_video +from glitchcraft.service_contract import ( + CAPABILITY_SLUGS, + SUPPORTED_VIDEO_EXTENSIONS, + capability_details, + effect_metadata, + ffmpeg_available, + ffprobe_available, + image_formats, +) +from glitchcraft.storage.contracts import ( + CleanupRequest, + ImageOutputRecord, + ImageSourceRecord, +) +from glitchcraft.storage.errors import ( + AssetConflictError, + AssetNotFoundError, + ManifestWriteError, + StorageUnavailableError, +) +from glitchcraft.storage.repository import ImageAssetRepository from glitchcraft.tasks import TaskStore +from glitchcraft.version import ( + APP_DESCRIPTOR, + APP_ID, + APP_NAME, + APP_VERSION, + RECIPE_SCHEMA_VERSION, + STORAGE_SCHEMA_VERSION, +) logger = logging.getLogger(__name__) bp = Blueprint("glitchcraft", __name__) @@ -121,16 +145,12 @@ def _task_store() -> TaskStore: return cast(TaskStore, current_app.extensions["task_store"]) -def _source_store() -> ImageSourceStore: - return cast(ImageSourceStore, current_app.extensions["image_source_store"]) - - -def _output_store() -> ImageOutputStore: - return cast(ImageOutputStore, current_app.extensions["image_output_store"]) +def _repository() -> ImageAssetRepository: + return cast(ImageAssetRepository, current_app.extensions["image_repository"]) def _json_not_found(asset: str) -> tuple[Response, int]: - return jsonify(status="error", message=f"Unknown or expired image {asset}."), 404 + return jsonify(status="error", message=f"Unknown image {asset}."), 404 def _no_store(response: Response) -> Response: @@ -146,11 +166,58 @@ def _parse_image_request() -> ImageRecipeRequest: return ImageRecipeRequest.model_validate(data) -def _process_source(record: ImageSourceRecord, recipe: Recipe) -> bytes: - frame = load_image_rgb(record.path) +def _process_source(path: Path, recipe: Recipe) -> bytes: + frame = load_image_rgb(path) return encode_png(apply_effect_stack(frame, recipe, frame_index=0)) +def _storage_error(exc: Exception) -> tuple[Response, int]: + if isinstance(exc, AssetNotFoundError): + return _json_not_found("asset") + if isinstance(exc, AssetConflictError): + return jsonify(status="error", message=str(exc)), 409 + logger.error("Managed storage request failed: %s", type(exc).__name__) + return jsonify(status="error", message="Persistent image storage is unavailable."), 503 + + +def _reject_query_parameters(allowed: set[str] | None = None) -> None: + unknown = set(request.args) - (allowed or set()) + if unknown: + raise ValueError("Unknown query parameters.") + + +def _source_json(source: ImageSourceRecord) -> dict[str, Any]: + return { + "sourceId": source.id, + "originalName": source.original_name, + "width": source.width, + "height": source.height, + "format": source.format, + "mimeType": source.mime_type, + "seed": source.seed, + "createdAt": source.created_at.isoformat().replace("+00:00", "Z"), + "originalUrl": url_for("glitchcraft.serve_image_source", source_id=source.id), + "outputCount": _repository().output_count(source.id), + } + + +def _output_json(output: ImageOutputRecord) -> dict[str, Any]: + return { + "outputId": output.id, + "sourceId": output.source_id, + "sourceAvailable": _repository().source_available(output.source_id), + "fileName": output.file_name, + "width": output.width, + "height": output.height, + "mimeType": output.mime_type, + "seed": output.seed, + "createdAt": output.created_at.isoformat().replace("+00:00", "Z"), + "previewUrl": url_for("glitchcraft.serve_image_output", output_id=output.id), + "downloadUrl": url_for("glitchcraft.download_image_output", output_id=output.id), + "recipe": output.recipe.model_dump(mode="json", by_alias=True), + } + + def _video_worker( store: TaskStore, task_id: str, @@ -188,6 +255,165 @@ def index() -> str: return render_template("index.html") +@bp.get("/app-manifest.json") +def app_manifest() -> Response: + return current_app.send_static_file("app-manifest.json") + + +@bp.get("/metadata") +def metadata() -> Response | tuple[Response, int]: + try: + _reject_query_parameters() + except ValueError as exc: + return _validation_error(exc) + repository = _repository() + address = request.url_root.rstrip("/") + return jsonify( + id=APP_ID, + name=APP_NAME, + descriptor=APP_DESCRIPTOR, + version=APP_VERSION, + manifestSchemaVersion=1, + recipeSchemaVersion=RECIPE_SCHEMA_VERSION, + storageSchemaVersion=STORAGE_SCHEMA_VERSION, + runtime={ + "webAddress": address, + "apiAddress": address, + "localOnly": True, + }, + capabilities=list(CAPABILITY_SLUGS), + supportedImageFormats=image_formats(), + supportedVideoExtensions=list(SUPPORTED_VIDEO_EXTENSIONS), + effectTypes=[effect_type.value for effect_type in EFFECT_REGISTRY], + library={ + "sources": len(repository.list_sources()) if repository.available else 0, + "outputs": len(repository.list_outputs()) if repository.available else 0, + }, + links={ + "manifest": url_for("glitchcraft.app_manifest"), + "health": url_for("glitchcraft.health"), + "readiness": url_for("glitchcraft.readiness"), + "capabilities": url_for("glitchcraft.capabilities"), + "storage": url_for("glitchcraft.storage_status"), + }, + ) + + +@bp.get("/health") +def health() -> Response: + return jsonify(status="ok", service=APP_ID, version=APP_VERSION) + + +@bp.get("/ready") +def readiness() -> tuple[Response, int]: + repository = _repository() + if not repository.available: + return ( + jsonify( + state="not_ready", + service=APP_ID, + version=APP_VERSION, + checks={ + "manifest": {"state": "unavailable"}, + "storage": {"state": "unavailable"}, + "imageProcessing": {"state": "ok"}, + "videoProcessing": { + "state": "available" if ffmpeg_available() else "unavailable" + }, + }, + ), + 503, + ) + + writable = repository.is_writable() + ffmpeg_ready = ffmpeg_available() + report = repository.last_report + degraded_storage = report.state in {"changed", "recovered"} or ( + report.orphan_sources + report.orphan_outputs > 0 + ) + if not writable: + state = "not_ready" + status = 503 + elif degraded_storage or not ffmpeg_ready: + state = "degraded" + status = 200 + else: + state = "ready" + status = 200 + checks: dict[str, Any] = { + "manifest": { + "state": "degraded" if report.recovered_from_backup else "ok", + }, + "storage": { + "state": "ok" if writable else "unavailable", + }, + "temporaryStorage": { + "state": "ok" if writable else "unavailable", + }, + "effectRegistry": { + "state": "ok" if EFFECT_REGISTRY else "unavailable", + }, + "imageProcessing": {"state": "ok"}, + "videoProcessing": { + "state": "available" if ffmpeg_ready else "unavailable", + "message": None if ffmpeg_ready else "FFmpeg is unavailable.", + }, + "ffprobe": { + "state": "available" if ffprobe_available() else "unavailable", + "required": False, + }, + "reconciliation": { + "state": report.state, + "orphanFiles": report.orphan_sources + report.orphan_outputs, + }, + } + return jsonify(state=state, service=APP_ID, version=APP_VERSION, checks=checks), status + + +@bp.get("/api/capabilities") +def capabilities() -> Response: + repository = _repository() + return jsonify( + schemaVersion=1, + service=APP_ID, + capabilities=capability_details(storage_available=repository.available), + ffmpeg={"available": ffmpeg_available()}, + ffprobe={"available": ffprobe_available(), "required": False}, + imageFormats=image_formats(), + outputFormats=[{"format": "PNG", "mimeType": "image/png"}], + videoExtensions=list(SUPPORTED_VIDEO_EXTENSIONS), + effects=effect_metadata(), + ) + + +@bp.get("/api/storage") +def storage_status() -> Response | tuple[Response, int]: + try: + return jsonify(_repository().status()) + except (StorageUnavailableError, OSError) as exc: + return _storage_error(exc) + + +@bp.post("/api/storage/cleanup") +def cleanup_storage() -> Response | tuple[Response, int]: + data = request.get_json(silent=True) + if data is None: + data = {} + if not isinstance(data, dict): + return _validation_error(ValueError("A valid JSON object is required.")) + try: + options = CleanupRequest.model_validate(data) + configured_minimum = float(current_app.config["ORPHAN_CLEANUP_MINIMUM_AGE_HOURS"]) + if options.remove_orphans and options.orphan_minimum_age_hours < configured_minimum: + raise ValueError("The orphan minimum age is below the configured safety floor.") + result = _repository().cleanup(options) + return jsonify(result.model_dump(mode="json", by_alias=True)) + except (ValidationError, ValueError) as exc: + return _validation_error(exc) + except StorageUnavailableError as exc: + return _storage_error(exc) + + @bp.post("/api/image-sources") def create_image_source() -> tuple[Response, int]: file = request.files.get("image") or request.files.get("input_file") @@ -200,22 +426,23 @@ def create_image_source() -> tuple[Response, int]: extension = _extension(original_name) if not original_name or extension not in IMAGE_EXTENSIONS: return jsonify(status="error", message="Unsupported image extension."), 400 - managed_path = _folder("UPLOAD_FOLDER") / f"image-source-{uuid.uuid4().hex}.{extension}" + staged_path: Path | None = None try: options = ImageSourceOptions.model_validate( {"seed": request.form["seed"]} if "seed" in request.form else {} ) - file.save(managed_path) + staged_path = _repository().new_temporary_path(f".{extension}") + file.save(staged_path) metadata = inspect_image( - managed_path, + staged_path, extension, int(current_app.config["MAX_IMAGE_PIXELS"]), ) seed = options.seed if options.seed is not None else secrets.randbelow(MAX_SEED + 1) - source = _source_store().create( - path=managed_path, + source = _repository().create_source( + staged_path=staged_path, + extension=extension, original_name=original_name, - display_name=original_name, width=metadata.width, height=metadata.height, image_format=metadata.image_format, @@ -223,55 +450,77 @@ def create_image_source() -> tuple[Response, int]: seed=seed, ) except (ValidationError, ValueError) as exc: - managed_path.unlink(missing_ok=True) + if staged_path is not None: + staged_path.unlink(missing_ok=True) return _validation_error(exc) except GlitchCraftError as exc: - managed_path.unlink(missing_ok=True) + if staged_path is not None: + staged_path.unlink(missing_ok=True) logger.info("Image source rejected: %s", exc) return jsonify(status="error", message=str(exc)), 400 + except (StorageUnavailableError, ManifestWriteError, AssetConflictError) as exc: + if staged_path is not None: + staged_path.unlink(missing_ok=True) + return _storage_error(exc) except Exception: - managed_path.unlink(missing_ok=True) + if staged_path is not None: + staged_path.unlink(missing_ok=True) logger.exception("Unexpected image source upload failure") return jsonify(status="error", message="Image upload failed."), 500 - return ( - jsonify( - sourceId=source.id, - originalName=source.original_name, - width=source.width, - height=source.height, - format=source.image_format, - originalUrl=url_for("glitchcraft.serve_image_source", source_id=source.id), - seed=source.seed, - ), - 201, - ) + response = _source_json(source) + response.pop("outputCount") + response.pop("createdAt") + response.pop("mimeType") + return jsonify(response), 201 + + +@bp.get("/api/image-sources") +def list_image_sources() -> Response | tuple[Response, int]: + try: + _reject_query_parameters() + return jsonify(sources=[_source_json(source) for source in _repository().list_sources()]) + except ValueError as exc: + return _validation_error(exc) + except StorageUnavailableError as exc: + return _storage_error(exc) + + +@bp.get("/api/image-sources/") +def get_image_source_metadata(source_id: str) -> Response | tuple[Response, int]: + try: + _reject_query_parameters() + return jsonify(_source_json(_repository().get_source(source_id))) + except ValueError as exc: + return _validation_error(exc) + except (AssetNotFoundError, StorageUnavailableError) as exc: + return _storage_error(exc) @bp.get("/api/image-sources//original") def serve_image_source(source_id: str) -> Response | tuple[Response, int]: try: - with _source_store().lease(source_id) as source: + with _repository().lease_source(source_id) as (source, path): response = send_file( - source.path, + path, mimetype=source.mime_type, as_attachment=False, conditional=False, ) response.headers["Content-Disposition"] = "inline" return _no_store(response) - except KeyError: - return _json_not_found("source") + except (AssetNotFoundError, StorageUnavailableError) as exc: + return _storage_error(exc) @bp.post("/api/image-sources//preview") def preview_image_source(source_id: str) -> Response | tuple[Response, int]: try: image_request = _parse_image_request() - with _source_store().lease(source_id) as source: - png = _process_source(source, image_request.recipe) + with _repository().lease_source(source_id) as (_, path): + png = _process_source(path, image_request.recipe) return _no_store(Response(png, mimetype="image/png")) - except KeyError: - return _json_not_found("source") + except (AssetNotFoundError, StorageUnavailableError) as exc: + return _storage_error(exc) except (ValidationError, ValueError) as exc: return _validation_error(exc) except GlitchCraftError: @@ -284,43 +533,50 @@ def preview_image_source(source_id: str) -> Response | tuple[Response, int]: @bp.post("/api/image-sources//export") def export_image_source(source_id: str) -> tuple[Response, int]: - output_path: Path | None = None + staged_path: Path | None = None try: image_request = _parse_image_request() - with _source_store().lease(source_id) as source: - frame = load_image_rgb(source.path) + with _repository().lease_source(source_id) as (source, source_path): + frame = load_image_rgb(source_path) processed = apply_effect_stack(frame, image_request.recipe, frame_index=0) - output_path = _folder("OUTPUT_FOLDER") / f"image-output-{uuid.uuid4().hex}.png" - save_image_rgb(processed, output_path) - base_name = secure_filename(Path(source.display_name).stem) or "image" + staged_path = _repository().new_temporary_path(".png") + save_image_rgb(processed, staged_path) + inspect_image(staged_path, "png", int(current_app.config["MAX_IMAGE_PIXELS"])) + base_name = secure_filename(Path(source.original_name).stem) or "image" display_name = f"{base_name}-glitchcraft.png" - output = _output_store().create( - path=output_path, + output = _repository().create_output( + staged_path=staged_path, source_id=source.id, - display_name=display_name, - recipe_json=image_request.recipe.model_dump_json(by_alias=True), - seed=image_request.recipe.seed, + file_name=display_name, + recipe=image_request.recipe, width=source.width, height=source.height, ) - except KeyError: - return _json_not_found("source") + except ( + AssetNotFoundError, + AssetConflictError, + StorageUnavailableError, + ManifestWriteError, + ) as exc: + if staged_path is not None: + staged_path.unlink(missing_ok=True) + return _storage_error(exc) except (ValidationError, ValueError) as exc: return _validation_error(exc) except GlitchCraftError: - if output_path is not None: - output_path.unlink(missing_ok=True) + if staged_path is not None: + staged_path.unlink(missing_ok=True) logger.exception("Image export failed for source %s", source_id) return jsonify(status="error", message="Image export failed."), 500 except Exception: - if output_path is not None: - output_path.unlink(missing_ok=True) + if staged_path is not None: + staged_path.unlink(missing_ok=True) logger.exception("Unexpected image export failure for source %s", source_id) return jsonify(status="error", message="Image export failed."), 500 return ( jsonify( outputId=output.id, - fileName=output.display_name, + fileName=output.file_name, previewUrl=url_for("glitchcraft.serve_image_output", output_id=output.id), downloadUrl=url_for("glitchcraft.download_image_output", output_id=output.id), ), @@ -328,12 +584,12 @@ def export_image_source(source_id: str) -> tuple[Response, int]: ) -def _serve_output(output: ImageOutputRecord, *, as_attachment: bool) -> Response: +def _serve_output(output: ImageOutputRecord, path: Path, *, as_attachment: bool) -> Response: response = send_file( - output.path, + path, mimetype=output.mime_type, as_attachment=as_attachment, - download_name=output.display_name if as_attachment else None, + download_name=output.file_name if as_attachment else None, conditional=False, ) if not as_attachment: @@ -344,19 +600,82 @@ def _serve_output(output: ImageOutputRecord, *, as_attachment: bool) -> Response @bp.get("/api/image-outputs/") def serve_image_output(output_id: str) -> Response | tuple[Response, int]: try: - with _output_store().lease(output_id) as output: - return _serve_output(output, as_attachment=False) - except KeyError: - return _json_not_found("output") + with _repository().lease_output(output_id) as (output, path): + return _serve_output(output, path, as_attachment=False) + except (AssetNotFoundError, StorageUnavailableError) as exc: + return _storage_error(exc) @bp.get("/api/image-outputs//download") def download_image_output(output_id: str) -> Response | tuple[Response, int]: try: - with _output_store().lease(output_id) as output: - return _serve_output(output, as_attachment=True) - except KeyError: - return _json_not_found("output") + with _repository().lease_output(output_id) as (output, path): + return _serve_output(output, path, as_attachment=True) + except (AssetNotFoundError, StorageUnavailableError) as exc: + return _storage_error(exc) + + +@bp.get("/api/image-outputs") +def list_image_outputs() -> Response | tuple[Response, int]: + try: + _reject_query_parameters() + return jsonify(outputs=[_output_json(output) for output in _repository().list_outputs()]) + except ValueError as exc: + return _validation_error(exc) + except StorageUnavailableError as exc: + return _storage_error(exc) + + +@bp.get("/api/image-outputs//metadata") +def get_image_output_metadata(output_id: str) -> Response | tuple[Response, int]: + try: + _reject_query_parameters() + return jsonify(_output_json(_repository().get_output(output_id))) + except ValueError as exc: + return _validation_error(exc) + except (AssetNotFoundError, StorageUnavailableError) as exc: + return _storage_error(exc) + + +@bp.delete("/api/image-outputs/") +def delete_image_output(output_id: str) -> Response | tuple[Response, int]: + try: + _reject_query_parameters() + _repository().delete_output(output_id) + return jsonify(status="deleted", outputId=output_id) + except ValueError as exc: + return _validation_error(exc) + except ( + AssetNotFoundError, + AssetConflictError, + StorageUnavailableError, + ManifestWriteError, + ) as exc: + return _storage_error(exc) + + +@bp.delete("/api/image-sources/") +def delete_image_source(source_id: str) -> Response | tuple[Response, int]: + try: + _reject_query_parameters({"cascade"}) + raw_cascade = request.args.get("cascade", "false") + if raw_cascade not in {"true", "false"}: + raise ValueError("cascade must be true or false") + deleted_outputs = _repository().delete_source(source_id, cascade=raw_cascade == "true") + return jsonify( + status="deleted", + sourceId=source_id, + deletedOutputs=deleted_outputs, + ) + except ValueError as exc: + return _validation_error(exc) + except ( + AssetNotFoundError, + AssetConflictError, + StorageUnavailableError, + ManifestWriteError, + ) as exc: + return _storage_error(exc) @bp.post("/upload_preview") diff --git a/playwright.config.js b/playwright.config.js index c790192..7394918 100644 --- a/playwright.config.js +++ b/playwright.config.js @@ -19,6 +19,10 @@ module.exports = defineConfig({ webServer: { command: pythonCommand, url: "http://127.0.0.1:5000", + env: { + ...process.env, + GLITCHCRAFT_DATA_ROOT: "test-results/browser-data", + }, reuseExistingServer: false, timeout: 30_000, }, diff --git a/static/app-manifest.json b/static/app-manifest.json new file mode 100644 index 0000000..7922bd0 --- /dev/null +++ b/static/app-manifest.json @@ -0,0 +1,26 @@ +{ + "capabilities": [ + "image-effects", + "video-effects", + "seeded-effects", + "ordered-effect-recipes", + "inline-image-preview", + "image-export", + "persistent-image-library" + ], + "defaults": { + "apiAddress": "http://127.0.0.1:5000", + "webAddress": "http://127.0.0.1:5000" + }, + "descriptor": "Local visual-effects workspace", + "endpoints": { + "health": "/health", + "metadata": "/metadata", + "readiness": "/ready" + }, + "icon": "/static/glitchcraft-mark.svg", + "id": "glitchcraft", + "name": "GlitchCraft", + "schemaVersion": 1, + "version": "0.1.0" +} diff --git a/static/glitchcraft-mark.svg b/static/glitchcraft-mark.svg new file mode 100644 index 0000000..55df431 --- /dev/null +++ b/static/glitchcraft-mark.svg @@ -0,0 +1,6 @@ + + GlitchCraft provisional signal mark + + + + diff --git a/tests/conftest.py b/tests/conftest.py index a8d8a96..454867c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,6 +20,9 @@ def app(tmp_path: Path): "UPLOAD_FOLDER": str(tmp_path / "uploads"), "OUTPUT_FOLDER": str(tmp_path / "outputs"), "PREVIEW_FOLDER": str(tmp_path / "previews"), + "DATA_ROOT": str(tmp_path / "data"), + "MANIFEST_PATH": str(tmp_path / "data" / "manifest.json"), + "TEMPORARY_FOLDER": str(tmp_path / "data" / "temporary"), } ) return application diff --git a/tests/test_image_assets.py b/tests/test_image_assets.py index 099e07c..453fe48 100644 --- a/tests/test_image_assets.py +++ b/tests/test_image_assets.py @@ -1,71 +1,318 @@ -import time +from __future__ import annotations + +import json +import os from concurrent.futures import ThreadPoolExecutor from pathlib import Path -from glitchcraft.image_assets import ImageOutputStore, ImageSourceStore +import pytest + +from glitchcraft.contracts.effects import Recipe +from glitchcraft.storage.contracts import CleanupRequest +from glitchcraft.storage.errors import ( + AssetConflictError, + AssetNotFoundError, + ManagedPathError, + ManifestWriteError, +) +from glitchcraft.storage.repository import ImageAssetRepository -def create_source(store: ImageSourceStore, path: Path, index: int = 0): - path.write_bytes(b"image") - return store.create( - path=path, - original_name=f"source-{index}.png", - display_name=f"source-{index}.png", - width=2, - height=3, +def repository(tmp_path: Path) -> ImageAssetRepository: + return ImageAssetRepository( + tmp_path / "data", + tmp_path / "data" / "manifest.json", + tmp_path / "data" / "temporary", + temporary_maximum_age=10, + ) + + +def create_source(store: ImageAssetRepository, name: str = "example.png"): + staged = store.new_temporary_path(".png") + staged.write_bytes(b"source") + return store.create_source( + staged_path=staged, + extension="png", + original_name=name, image_format="PNG", mime_type="image/png", - seed=index, + width=8, + height=6, + seed=123, ) -def test_source_ids_are_opaque_unique_and_thread_safe(tmp_path: Path) -> None: - store = ImageSourceStore() - - def add(index: int) -> str: - return create_source(store, tmp_path / f"{index}.png", index).id - - with ThreadPoolExecutor(max_workers=8) as executor: - identifiers = list(executor.map(add, range(40))) - assert len(set(identifiers)) == 40 - assert all(len(identifier) >= 32 for identifier in identifiers) - assert len(store) == 40 - - -def test_source_lease_prevents_cleanup_until_released(tmp_path: Path) -> None: - store = ImageSourceStore(maximum_age=10) - source = create_source(store, tmp_path / "source.png") - with store.lease(source.id): - assert store.cleanup_expired(now=source.created_at + 11) == 0 - assert source.path.exists() - assert store.cleanup_expired(now=source.created_at + 11) == 1 - assert store.get(source.id) is None - assert not source.path.exists() - - -def test_missing_file_and_expired_records_are_unavailable(tmp_path: Path) -> None: - store = ImageSourceStore(maximum_age=1) - source = create_source(store, tmp_path / "source.png") - source.path.unlink() - assert store.get(source.id) is None - - expired = create_source(store, tmp_path / "expired.png") - time.sleep(0.01) - assert store.cleanup_expired(now=expired.created_at + 2) == 1 - - -def test_output_store_tracks_recipe_and_known_path(tmp_path: Path) -> None: - path = tmp_path / "output.png" - path.write_bytes(b"output") - store = ImageOutputStore() - output = store.create( - path=path, - source_id="source", - display_name="result.png", - recipe_json='{"seed":1}', - seed=1, - width=10, - height=20, +def create_output(store: ImageAssetRepository, source_id: str): + staged = store.new_temporary_path(".png") + staged.write_bytes(b"output") + return store.create_output( + staged_path=staged, + source_id=source_id, + file_name="example-glitchcraft.png", + width=8, + height=6, + recipe=Recipe(seed=123), ) - assert store.get(output.id) == output - assert path.resolve() in store.known_paths() + + +def test_records_persist_and_lists_are_caller_independent(tmp_path: Path) -> None: + store = repository(tmp_path) + older = create_source(store, "older.png") + newer = create_source(store, "newer.png") + output = create_output(store, older.id) + + restarted = repository(tmp_path) + assert restarted.get_source(older.id) == older + assert restarted.get_output(output.id) == output + assert [record.id for record in restarted.list_sources()] == [newer.id, older.id] + assert restarted.output_count(older.id) == 1 + assert restarted.source_available(older.id) + + +def test_managed_filenames_derive_only_from_opaque_ids(tmp_path: Path) -> None: + store = repository(tmp_path) + source = create_source(store, "private customer name.png") + output = create_output(store, source.id) + manifest = json.loads(store.manifest_path.read_text(encoding="utf-8")) + + assert manifest["sources"][source.id]["originalName"] == "private customer name.png" + assert manifest["sources"][source.id]["storedName"] == f"sources/images/{source.id}.png" + assert manifest["outputs"][output.id]["storedName"] == f"outputs/images/{output.id}.png" + assert str(tmp_path) not in store.manifest_path.read_text(encoding="utf-8") + + +def test_leases_block_deletion_and_release_after_use(tmp_path: Path) -> None: + store = repository(tmp_path) + source = create_source(store) + output = create_output(store, source.id) + + with store.lease_output(output.id), pytest.raises(AssetConflictError): + store.delete_output(output.id) + store.delete_output(output.id) + with pytest.raises(AssetNotFoundError): + store.get_output(output.id) + + +def test_source_delete_conflict_and_atomic_cascade(tmp_path: Path) -> None: + store = repository(tmp_path) + source = create_source(store) + create_output(store, source.id) + + with pytest.raises(AssetConflictError): + store.delete_source(source.id, cascade=False) + assert store.delete_source(source.id, cascade=True) == 1 + assert store.list_sources() == [] + assert store.list_outputs() == [] + + +def test_failed_manifest_mutation_preserves_state_and_installed_files( + tmp_path: Path, monkeypatch +) -> None: + store = repository(tmp_path) + source = create_source(store) + output = create_output(store, source.id) + previous = store.manifest_path.read_bytes() + + def fail_replace(_source: Path, destination: Path) -> None: + if destination == store.manifest_path: + raise OSError("simulated interruption") + os.replace(_source, destination) + + monkeypatch.setattr(store._writer, "_replace", fail_replace) + with pytest.raises(ManifestWriteError): + store.delete_output(output.id) + assert store.manifest_path.read_bytes() == previous + assert store.get_output(output.id).id == output.id + + +def test_concurrent_mutations_are_sequential_and_complete(tmp_path: Path) -> None: + store = repository(tmp_path) + + with ThreadPoolExecutor(max_workers=4) as pool: + records = list(pool.map(lambda index: create_source(store, f"{index}.png"), range(8))) + + assert len({record.id for record in records}) == 8 + assert len(repository(tmp_path).list_sources()) == 8 + + +def test_cleanup_is_safe_by_default_and_requires_orphan_opt_in(tmp_path: Path, monkeypatch) -> None: + store = repository(tmp_path) + source = create_source(store) + orphan = store.source_folder / "orphan.png" + orphan.write_bytes(b"orphan") + temporary = store.new_temporary_path(".tmp") + temporary.write_bytes(b"temporary") + old = 1 + os.utime(orphan, (old, old)) + os.utime(temporary, (old, old)) + + dry = store.cleanup( + CleanupRequest( + dry_run=True, + remove_temporary=True, + remove_orphans=True, + orphan_minimum_age_hours=1, + ) + ) + assert dry.temporary_files == 1 + assert dry.orphan_files == 1 + assert orphan.exists() and temporary.exists() + + removed = store.cleanup( + CleanupRequest( + dry_run=False, + remove_temporary=True, + remove_orphans=True, + orphan_minimum_age_hours=1, + ) + ) + assert removed.temporary_bytes == len(b"temporary") + assert not orphan.exists() and not temporary.exists() + assert store.get_source(source.id).id == source.id + + +def test_unknown_assets_are_typed(tmp_path: Path) -> None: + store = repository(tmp_path) + with pytest.raises(AssetNotFoundError): + store.get_source("missing") + with pytest.raises(AssetNotFoundError): + store.delete_output("missing") + with pytest.raises(AssetNotFoundError): + store.delete_source("missing", cascade=False) + staged = store.new_temporary_path(".png") + staged.write_bytes(b"output") + with pytest.raises(AssetNotFoundError): + store.create_output( + staged_path=staged, + source_id="abcdefghijklmnop", + file_name="missing.png", + width=1, + height=1, + recipe=Recipe(seed=1), + ) + + +def test_repository_rejects_locations_outside_data_root(tmp_path: Path) -> None: + with pytest.raises(ManagedPathError): + ImageAssetRepository( + tmp_path / "data", + tmp_path / "outside.json", + tmp_path / "data" / "temporary", + ) + with pytest.raises(ManagedPathError): + ImageAssetRepository( + tmp_path / "data", + tmp_path / "data" / "manifest.json", + tmp_path / "outside-temporary", + ) + + +def test_creation_failures_remove_installed_files(tmp_path: Path, monkeypatch) -> None: + store = repository(tmp_path) + source = create_source(store) + staged_source = store.new_temporary_path(".png") + staged_source.write_bytes(b"new source") + monkeypatch.setattr( + store._writer, + "write", + lambda *_args, **_kwargs: (_ for _ in ()).throw(ManifestWriteError("failed")), + ) + with pytest.raises(ManifestWriteError): + store.create_source( + staged_path=staged_source, + extension="png", + original_name="failed.png", + image_format="PNG", + mime_type="image/png", + width=1, + height=1, + seed=1, + ) + assert len(list(store.source_folder.iterdir())) == 1 + + staged_output = store.new_temporary_path(".png") + staged_output.write_bytes(b"new output") + with pytest.raises(ManifestWriteError): + store.create_output( + staged_path=staged_output, + source_id=source.id, + file_name="failed.png", + width=1, + height=1, + recipe=Recipe(seed=1), + ) + assert list(store.output_folder.iterdir()) == [] + + +def test_conflicting_generated_paths_are_rejected(tmp_path: Path, monkeypatch) -> None: + store = repository(tmp_path) + source = create_source(store) + monkeypatch.setattr(store, "_new_id", lambda: source.id) + staged = store.new_temporary_path(".png") + staged.write_bytes(b"duplicate") + with pytest.raises(AssetConflictError): + store.create_source( + staged_path=staged, + extension="png", + original_name="duplicate.png", + image_format="PNG", + mime_type="image/png", + width=1, + height=1, + seed=1, + ) + + +def test_status_and_writable_failures_are_redacted(tmp_path: Path, monkeypatch) -> None: + store = repository(tmp_path) + monkeypatch.setattr( + "glitchcraft.storage.repository.shutil.disk_usage", + lambda _path: (_ for _ in ()).throw(OSError("private drive")), + ) + assert store.status()["freeBytes"] is None + monkeypatch.setattr( + "glitchcraft.storage.repository.tempfile.mkstemp", + lambda **_kwargs: (_ for _ in ()).throw(OSError("private directory")), + ) + assert not store.is_writable() + + +def test_cleanup_can_skip_each_category_and_normalizes_suffix(tmp_path: Path) -> None: + store = repository(tmp_path) + assert store.new_temporary_path("unsafe").suffix == ".tmp" + subdirectory = store.temporary_folder / "nested" + subdirectory.mkdir() + result = store.cleanup( + CleanupRequest( + dry_run=True, + remove_temporary=False, + remove_orphans=False, + ) + ) + assert result.temporary_files == 0 + assert result.orphan_files == 0 + + +def test_status_counts_both_asset_types_and_ignores_missing_files(tmp_path: Path) -> None: + store = repository(tmp_path) + source = create_source(store) + output = create_output(store, source.id) + status = store.status() + assert status["sourceBytes"] == len(b"source") + assert status["outputBytes"] == len(b"output") + + source_path = store.data_root / Path(*source.stored_name.split("/")) + output_path = store.data_root / Path(*output.stored_name.split("/")) + source_path.unlink() + output_path.unlink() + missing = store.status() + assert missing["sourceBytes"] == 0 + assert missing["outputBytes"] == 0 + + +def test_resolver_rejects_wrong_prefix_and_escape(tmp_path: Path) -> None: + store = repository(tmp_path) + with pytest.raises(ManagedPathError): + store._resolve("outputs/images/file.png", "sources/images") + with pytest.raises(ManagedPathError): + store._resolve("sources/images/../../../../outside.png", "sources/images") diff --git a/tests/test_image_workflow.py b/tests/test_image_workflow.py index 632d5cf..eb59999 100644 --- a/tests/test_image_workflow.py +++ b/tests/test_image_workflow.py @@ -5,6 +5,8 @@ import pytest from PIL import Image +from glitchcraft.application import create_app + def image_file( *, @@ -135,7 +137,7 @@ def test_preview_is_deterministic_png_and_creates_no_output(client, app) -> None assert first.mimetype == "image/png" assert "no-store" in first.headers["Cache-Control"] assert first.data == second.data - assert not list(Path(app.config["OUTPUT_FOLDER"]).glob("image-output-*.png")) + assert not list((Path(app.config["DATA_ROOT"]) / "outputs" / "images").glob("*.png")) def test_stochastic_preview_changes_by_seed_but_deterministic_does_not(client) -> None: @@ -190,7 +192,8 @@ def test_export_creates_one_opaque_output_identical_to_preview(client, app) -> N assert exported.json["fileName"] == "example-glitchcraft.png" assert "outputId" in exported.json assert str(Path(app.config["OUTPUT_FOLDER"])) not in exported.text - assert len(list(Path(app.config["OUTPUT_FOLDER"]).glob("image-output-*.png"))) == 1 + managed_outputs = Path(app.config["DATA_ROOT"]) / "outputs" / "images" + assert len(list(managed_outputs.glob("*.png"))) == 1 inline = client.get(exported.json["previewUrl"]) download = client.get(exported.json["downloadUrl"]) @@ -235,3 +238,123 @@ def test_unexpected_image_failures_are_controlled(client, monkeypatch) -> None: exported = client.post(f"/api/image-sources/{source_id}/export", json=recipe()) assert exported.status_code == 500 assert "private path" not in exported.text + + +def test_sources_outputs_and_recipe_survive_application_restart(app) -> None: + first_client = app.test_client() + uploaded = upload_source(first_client) + source_id = uploaded.json["sourceId"] + preview = first_client.post(f"/api/image-sources/{source_id}/preview", json=recipe()) + exported = first_client.post(f"/api/image-sources/{source_id}/export", json=recipe()) + output_id = exported.json["outputId"] + + restarted = create_app( + { + "TESTING": True, + "UPLOAD_FOLDER": app.config["UPLOAD_FOLDER"], + "OUTPUT_FOLDER": app.config["OUTPUT_FOLDER"], + "PREVIEW_FOLDER": app.config["PREVIEW_FOLDER"], + "DATA_ROOT": app.config["DATA_ROOT"], + "MANIFEST_PATH": app.config["MANIFEST_PATH"], + "TEMPORARY_FOLDER": app.config["TEMPORARY_FOLDER"], + } + ) + client = restarted.test_client() + assert client.get(f"/api/image-sources/{source_id}/original").status_code == 200 + restored_preview = client.post(f"/api/image-sources/{source_id}/preview", json=recipe()) + assert restored_preview.data == preview.data + assert client.post(f"/api/image-sources/{source_id}/export", json=recipe()).status_code == 201 + assert client.get(f"/api/image-outputs/{output_id}").status_code == 200 + assert client.get(f"/api/image-outputs/{output_id}/download").status_code == 200 + metadata = client.get(f"/api/image-outputs/{output_id}/metadata").json + assert metadata["recipe"] == recipe()["recipe"] + assert metadata["seed"] == 123 + + +def test_library_metadata_sorting_deletion_and_cascade(client) -> None: + first = upload_source(client, filename="first.png").json + second = upload_source(client, filename="second.png").json + exported = client.post(f"/api/image-sources/{first['sourceId']}/export", json=recipe()).json + + sources = client.get("/api/image-sources").json["sources"] + assert [item["sourceId"] for item in sources] == [ + second["sourceId"], + first["sourceId"], + ] + assert sources[1]["outputCount"] == 1 + assert client.get(f"/api/image-sources/{first['sourceId']}").status_code == 200 + outputs = client.get("/api/image-outputs").json["outputs"] + assert outputs[0]["outputId"] == exported["outputId"] + assert "storedName" not in str(outputs) + + conflict = client.delete(f"/api/image-sources/{first['sourceId']}") + assert conflict.status_code == 409 + malformed = client.delete(f"/api/image-sources/{first['sourceId']}?cascade=sometimes") + assert malformed.status_code == 400 + deleted = client.delete(f"/api/image-sources/{first['sourceId']}?cascade=true") + assert deleted.json["deletedOutputs"] == 1 + assert client.get(f"/api/image-outputs/{exported['outputId']}").status_code == 404 + assert client.delete(f"/api/image-sources/{second['sourceId']}").status_code == 200 + + +def test_output_remains_available_when_missing_source_is_reconciled(app) -> None: + client = app.test_client() + source_id = upload_source(client).json["sourceId"] + output_id = client.post(f"/api/image-sources/{source_id}/export", json=recipe()).json[ + "outputId" + ] + manifest = Path(app.config["MANIFEST_PATH"]) + payload = __import__("json").loads(manifest.read_text(encoding="utf-8")) + stored_name = payload["sources"][source_id]["storedName"] + (Path(app.config["DATA_ROOT"]) / Path(*stored_name.split("/"))).unlink() + + restarted = create_app( + { + "TESTING": True, + "DATA_ROOT": app.config["DATA_ROOT"], + "MANIFEST_PATH": app.config["MANIFEST_PATH"], + "TEMPORARY_FOLDER": app.config["TEMPORARY_FOLDER"], + "UPLOAD_FOLDER": app.config["UPLOAD_FOLDER"], + "OUTPUT_FOLDER": app.config["OUTPUT_FOLDER"], + "PREVIEW_FOLDER": app.config["PREVIEW_FOLDER"], + } + ) + restarted_client = restarted.test_client() + assert restarted_client.get(f"/api/image-sources/{source_id}").status_code == 404 + assert restarted_client.get(f"/api/image-outputs/{output_id}").status_code == 200 + metadata = restarted_client.get(f"/api/image-outputs/{output_id}/metadata").json + assert metadata["sourceAvailable"] is False + + +@pytest.mark.parametrize( + "path", + [ + "/api/image-sources?surprise=true", + "/api/image-outputs?surprise=true", + ], +) +def test_library_lists_reject_unknown_query_parameters(client, path: str) -> None: + assert client.get(path).status_code == 400 + + +def test_output_delete_and_metadata_validation(client) -> None: + source_id = upload_source(client).json["sourceId"] + output_id = client.post(f"/api/image-sources/{source_id}/export", json=recipe()).json[ + "outputId" + ] + assert client.get(f"/api/image-outputs/{output_id}/metadata?unknown=true").status_code == 400 + assert client.delete(f"/api/image-outputs/{output_id}?unknown=true").status_code == 400 + deleted = client.delete(f"/api/image-outputs/{output_id}") + assert deleted.json == {"status": "deleted", "outputId": output_id} + assert client.delete(f"/api/image-outputs/{output_id}").status_code == 404 + + +def test_export_rejects_non_object_json(client) -> None: + source_id = upload_source(client).json["sourceId"] + assert ( + client.post( + f"/api/image-sources/{source_id}/export", + json=[], + ).status_code + == 400 + ) diff --git a/tests/test_service_contract.py b/tests/test_service_contract.py new file mode 100644 index 0000000..22799ff --- /dev/null +++ b/tests/test_service_contract.py @@ -0,0 +1,225 @@ +from __future__ import annotations + +import json +import os +from io import BytesIO +from pathlib import Path + +import pytest +from PIL import Image + +import glitchcraft.application as application_module +from glitchcraft.application import create_app +from glitchcraft.service_contract import CAPABILITY_SLUGS +from glitchcraft.version import APP_ID, APP_VERSION + + +def test_static_manifest_runtime_version_and_endpoint_consistency(client) -> None: + manifest_response = client.get("/app-manifest.json") + manifest = manifest_response.json + metadata = client.get("/metadata").json + + assert manifest_response.status_code == 200 + assert manifest["id"] == metadata["id"] == APP_ID + assert manifest["version"] == metadata["version"] == APP_VERSION + assert tuple(manifest["capabilities"]) == CAPABILITY_SLUGS + assert manifest["defaults"] == { + "webAddress": "http://127.0.0.1:5000", + "apiAddress": "http://127.0.0.1:5000", + } + for endpoint in manifest["endpoints"].values(): + assert client.get(endpoint).status_code in {200, 503} + assert client.get(manifest["icon"]).status_code == 200 + + +def test_application_derives_managed_paths_from_configured_or_default_root( + tmp_path: Path, monkeypatch +) -> None: + configured = create_app( + { + "TESTING": True, + "DATA_ROOT": str(tmp_path / "configured"), + "UPLOAD_FOLDER": str(tmp_path / "uploads"), + "OUTPUT_FOLDER": str(tmp_path / "outputs"), + "PREVIEW_FOLDER": str(tmp_path / "previews"), + } + ) + assert configured.config["MANIFEST_PATH"] == str(tmp_path / "configured" / "manifest.json") + assert configured.config["TEMPORARY_FOLDER"] == str(tmp_path / "configured" / "temporary") + + fake_module = tmp_path / "package" / "application.py" + monkeypatch.setattr(application_module, "__file__", str(fake_module)) + default = create_app() + assert Path(default.config["DATA_ROOT"]) == tmp_path / "data" + + +def test_metadata_is_request_derived_redacted_and_truthful(client, app) -> None: + response = client.get("/metadata", base_url="http://127.0.0.1:5000") + payload = response.json + serialized = json.dumps(payload) + assert payload["runtime"] == { + "webAddress": "http://127.0.0.1:5000", + "apiAddress": "http://127.0.0.1:5000", + "localOnly": True, + } + assert payload["storageSchemaVersion"] == 1 + assert payload["recipeSchemaVersion"] == 1 + assert set(payload["effectTypes"]) + assert str(app.config["DATA_ROOT"]) not in serialized + assert "example.png" not in serialized + assert client.get("/metadata?unknown=true").status_code == 400 + + +def test_health_is_lightweight_even_when_storage_is_unavailable(tmp_path: Path) -> None: + data = tmp_path / "data" + data.mkdir() + manifest = data / "manifest.json" + backup = data / "manifest.json.bak" + manifest.write_text("{bad", encoding="utf-8") + backup.write_text("{bad-too", encoding="utf-8") + app = create_app( + { + "TESTING": True, + "DATA_ROOT": str(data), + "MANIFEST_PATH": str(manifest), + "TEMPORARY_FOLDER": str(data / "temporary"), + "UPLOAD_FOLDER": str(tmp_path / "uploads"), + "OUTPUT_FOLDER": str(tmp_path / "outputs"), + "PREVIEW_FOLDER": str(tmp_path / "previews"), + } + ) + client = app.test_client() + assert client.get("/health").json == { + "status": "ok", + "service": APP_ID, + "version": APP_VERSION, + } + assert client.get("/ready").status_code == 503 + assert client.get("/ready").json["state"] == "not_ready" + upload = client.post("/api/image-sources", data={}) + assert upload.status_code == 400 + cleanup = client.post("/api/storage/cleanup", json={}) + assert cleanup.status_code == 503 + assert manifest.read_text(encoding="utf-8") == "{bad" + assert backup.read_text(encoding="utf-8") == "{bad-too" + + +def test_readiness_degrades_without_ffmpeg(client, monkeypatch) -> None: + monkeypatch.setattr( + "glitchcraft.service_contract.shutil.which", + lambda executable: None if executable == "ffmpeg" else "available", + ) + response = client.get("/ready") + assert response.status_code == 200 + assert response.json["state"] == "degraded" + assert response.json["checks"]["videoProcessing"]["state"] == "unavailable" + + +def test_readiness_is_not_ready_when_temporary_storage_is_not_writable( + client, app, monkeypatch +) -> None: + repository = app.extensions["image_repository"] + monkeypatch.setattr(repository, "is_writable", lambda: False) + response = client.get("/ready") + assert response.status_code == 503 + assert response.json["state"] == "not_ready" + + +def test_capabilities_are_structured_and_do_not_overclaim(client) -> None: + response = client.get("/api/capabilities") + payload = response.json + slugs = {item["slug"] for item in payload["capabilities"]} + assert set(CAPABILITY_SLUGS).issubset(slugs) + assert "persistent-video-library" not in json.dumps(payload) + assert "audio-preservation" not in json.dumps(payload) + assert payload["outputFormats"] == [{"format": "PNG", "mimeType": "image/png"}] + + +def test_storage_status_cleanup_and_public_json_are_redacted(client, app) -> None: + temporary = Path(app.config["TEMPORARY_FOLDER"]) / "old.tmp" + temporary.write_bytes(b"temporary") + os.utime(temporary, (1, 1)) + orphan_folder = Path(app.config["DATA_ROOT"]) / "outputs" / "images" + orphan = orphan_folder / "orphan.png" + orphan.write_bytes(b"orphan") + os.utime(orphan, (1, 1)) + + status = client.get("/api/storage") + assert status.status_code == 200 + assert status.json["orphanFileCount"] == 1 + assert status.json["temporaryBytes"] == len(b"temporary") + serialized = status.text + assert str(app.config["DATA_ROOT"]) not in serialized + assert "orphan.png" not in serialized + + safe_default = client.post("/api/storage/cleanup", json={}) + assert safe_default.json["dryRun"] is True + assert temporary.exists() and orphan.exists() + too_young = client.post( + "/api/storage/cleanup", + json={ + "dryRun": False, + "removeOrphans": True, + "orphanMinimumAgeHours": 1, + }, + ) + assert too_young.status_code == 400 + cleanup = client.post( + "/api/storage/cleanup", + json={ + "dryRun": False, + "removeTemporary": True, + "removeOrphans": True, + "orphanMinimumAgeHours": 24, + }, + ) + assert cleanup.status_code == 200 + assert cleanup.json["temporaryFiles"] == 1 + assert cleanup.json["orphanFiles"] == 1 + assert not temporary.exists() and not orphan.exists() + + +@pytest.mark.parametrize( + "payload", + [ + [], + {"unknown": True}, + {"orphanMinimumAgeHours": 0}, + {"orphanMinimumAgeHours": 100000}, + ], +) +def test_cleanup_rejects_invalid_contracts(client, payload) -> None: + assert client.post("/api/storage/cleanup", json=payload).status_code == 400 + + +def test_unavailable_storage_disables_asset_apis(tmp_path: Path) -> None: + data = tmp_path / "data" + data.mkdir() + (data / "manifest.json").write_text("{bad", encoding="utf-8") + (data / "manifest.json.bak").write_text("{bad", encoding="utf-8") + app = create_app( + { + "TESTING": True, + "DATA_ROOT": str(data), + "MANIFEST_PATH": str(data / "manifest.json"), + "TEMPORARY_FOLDER": str(data / "temporary"), + "UPLOAD_FOLDER": str(tmp_path / "uploads"), + "OUTPUT_FOLDER": str(tmp_path / "outputs"), + "PREVIEW_FOLDER": str(tmp_path / "previews"), + } + ) + client = app.test_client() + image = BytesIO() + Image.new("RGB", (2, 2), (20, 30, 40)).save(image, "PNG") + image.seek(0) + + upload = client.post( + "/api/image-sources", + data={"image": (image, "example.png")}, + content_type="multipart/form-data", + ) + assert upload.status_code == 503 + assert client.get("/api/image-sources").status_code == 503 + assert client.get("/api/image-outputs").status_code == 503 + assert client.get("/api/image-sources/unknown").status_code == 503 + assert client.get("/api/image-outputs/unknown/metadata").status_code == 503 diff --git a/tests/test_storage_manifest.py b/tests/test_storage_manifest.py new file mode 100644 index 0000000..b9b3a65 --- /dev/null +++ b/tests/test_storage_manifest.py @@ -0,0 +1,280 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from glitchcraft.storage.contracts import ( + ImageOutputRecord, + ImageSourceRecord, + ManifestDocument, +) +from glitchcraft.storage.errors import ManifestReadError, ManifestWriteError +from glitchcraft.storage.manifest import AtomicManifestWriter, load_manifest +from glitchcraft.storage.repository import ImageAssetRepository + + +def valid_source() -> dict: + return { + "id": "abcdefghijklmnop", + "kind": "image", + "originalName": "example.png", + "storedName": "sources/images/abcdefghijklmnop.png", + "format": "PNG", + "mimeType": "image/png", + "width": 2, + "height": 3, + "seed": 7, + "createdAt": "2026-07-26T03:00:00Z", + } + + +def valid_output() -> dict: + return { + "id": "qrstuvwxyzABCDEF", + "kind": "image", + "sourceId": "abcdefghijklmnop", + "fileName": "example-glitchcraft.png", + "storedName": "outputs/images/qrstuvwxyzABCDEF.png", + "mimeType": "image/png", + "width": 2, + "height": 3, + "recipe": {"schemaVersion": 1, "seed": 7, "effects": []}, + "createdAt": "2026-07-26T03:01:00Z", + } + + +def test_manifest_models_accept_valid_document_and_reject_unknown_fields() -> None: + document = ManifestDocument.model_validate( + { + "schemaVersion": 1, + "sources": {"abcdefghijklmnop": valid_source()}, + "outputs": {"qrstuvwxyzABCDEF": valid_output()}, + } + ) + assert document.outputs["qrstuvwxyzABCDEF"].recipe.seed == 7 + + payload = valid_source() | {"futureField": True} + with pytest.raises(ValidationError): + ImageSourceRecord.model_validate(payload) + + +@pytest.mark.parametrize( + ("model", "field", "value"), + [ + (ImageSourceRecord, "storedName", "C:/private/source.png"), + (ImageSourceRecord, "storedName", "sources/images/../private.png"), + (ImageSourceRecord, "storedName", "outputs/images/source.png"), + (ImageOutputRecord, "storedName", "/outputs/images/result.png"), + (ImageOutputRecord, "storedName", "outputs\\images\\result.png"), + ], +) +def test_managed_paths_reject_absolute_traversal_and_wrong_boundaries( + model, field: str, value: str +) -> None: + payload = valid_source() if model is ImageSourceRecord else valid_output() + payload[field] = value + with pytest.raises(ValidationError): + model.model_validate(payload) + + +def test_manifest_rejects_schema_keys_and_embedded_recipe_errors() -> None: + with pytest.raises(ValidationError): + ManifestDocument.model_validate({"schemaVersion": 2}) + with pytest.raises(ValidationError): + ManifestDocument.model_validate( + {"schemaVersion": 1, "sources": {"different": valid_source()}} + ) + output = valid_output() + output["recipe"] = {"schemaVersion": 1, "seed": -1, "effects": []} + with pytest.raises(ValidationError): + ImageOutputRecord.model_validate(output) + + +def test_manifest_rejects_naive_dates_and_non_normal_paths() -> None: + source = valid_source() + source["createdAt"] = "2026-07-26T03:00:00" + with pytest.raises(ValidationError): + ImageSourceRecord.model_validate(source) + source = valid_source() + source["storedName"] = "sources/images" + with pytest.raises(ValidationError): + ImageSourceRecord.model_validate(source) + output = valid_output() + output["createdAt"] = "2026-07-26T03:00:00" + with pytest.raises(ValidationError): + ImageOutputRecord.model_validate(output) + output = valid_output() + output["storedName"] = "outputs/images" + with pytest.raises(ValidationError): + ImageOutputRecord.model_validate(output) + source = valid_source() + source["storedName"] = "sources//images/example.png" + with pytest.raises(ValidationError): + ImageSourceRecord.model_validate(source) + with pytest.raises(ValidationError): + ManifestDocument.model_validate( + {"schemaVersion": 1, "outputs": {"different": valid_output()}} + ) + + +def test_asset_metadata_requires_utc_and_consistent_formats() -> None: + source = valid_source() + source["createdAt"] = "2026-07-26T08:00:00+05:00" + with pytest.raises(ValidationError): + ImageSourceRecord.model_validate(source) + source = valid_source() + source["mimeType"] = "image/jpeg" + with pytest.raises(ValidationError): + ImageSourceRecord.model_validate(source) + source = valid_source() + source["storedName"] = "sources/images/abcdefghijklmnop.jpg" + with pytest.raises(ValidationError): + ImageSourceRecord.model_validate(source) + output = valid_output() + output["storedName"] = "outputs/images/qrstuvwxyzABCDEF.jpg" + with pytest.raises(ValidationError): + ImageOutputRecord.model_validate(output) + + +def test_atomic_writer_creates_stable_primary_and_backup(tmp_path: Path) -> None: + path = tmp_path / "manifest.json" + writer = AtomicManifestWriter(path, tmp_path / "manifest.json.bak") + empty = ManifestDocument(schema_version=1) + writer.write(empty) + first = path.read_bytes() + assert load_manifest(path) == empty + assert (tmp_path / "manifest.json.bak").read_bytes() == first + + changed = ManifestDocument.model_validate( + { + "schemaVersion": 1, + "sources": {"abcdefghijklmnop": valid_source()}, + } + ) + writer.write(changed) + assert load_manifest(path) == changed + assert (tmp_path / "manifest.json.bak").read_bytes() == first + + +def test_failed_atomic_replace_keeps_previous_manifest(tmp_path: Path, monkeypatch) -> None: + path = tmp_path / "manifest.json" + writer = AtomicManifestWriter(path, tmp_path / "manifest.json.bak") + empty = ManifestDocument(schema_version=1) + writer.write(empty) + previous = path.read_bytes() + original_replace = writer._replace + + def fail_primary(source: Path, destination: Path) -> None: + if destination == path: + raise OSError("interrupted") + original_replace(source, destination) + + monkeypatch.setattr(writer, "_replace", fail_primary) + with pytest.raises(Exception, match="could not be saved"): + writer.write( + ManifestDocument.model_validate( + { + "schemaVersion": 1, + "sources": {"abcdefghijklmnop": valid_source()}, + } + ) + ) + assert path.read_bytes() == previous + assert not list(tmp_path.glob(".manifest-*.tmp")) + + +def test_manifest_read_and_temporary_write_failures_are_typed(tmp_path: Path, monkeypatch) -> None: + with pytest.raises(ManifestReadError): + load_manifest(tmp_path) + + writer = AtomicManifestWriter(tmp_path / "manifest.json", tmp_path / "manifest.json.bak") + monkeypatch.setattr( + "glitchcraft.storage.manifest.os.fdopen", + lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("interrupted")), + ) + with pytest.raises(ManifestWriteError): + writer.write(ManifestDocument(schema_version=1)) + assert not list(tmp_path.glob(".manifest-*.tmp")) + + +def test_backup_recovery_and_both_invalid_are_non_destructive(tmp_path: Path) -> None: + store = ImageAssetRepository( + tmp_path / "data", + tmp_path / "data" / "manifest.json", + tmp_path / "data" / "temporary", + ) + primary = store.manifest_path + backup = store.backup_path + valid_backup = backup.read_bytes() + primary.write_text("{broken", encoding="utf-8") + + recovered = ImageAssetRepository(tmp_path / "data", primary, tmp_path / "data" / "temporary") + assert recovered.available + assert recovered.last_report.recovered_from_backup + assert load_manifest(primary).schema_version == 1 + assert list(recovered.recovery_folder.glob("reconciliation-*.json")) + + primary.write_text("{still-broken", encoding="utf-8") + backup.write_text("{also-broken", encoding="utf-8") + primary_before = primary.read_bytes() + backup_before = backup.read_bytes() + unavailable = ImageAssetRepository(tmp_path / "data", primary, tmp_path / "data" / "temporary") + assert not unavailable.available + assert primary.read_bytes() == primary_before + assert backup.read_bytes() == backup_before + assert valid_backup != backup_before + + +def test_missing_primary_recovers_backup_and_invalid_primary_without_backup_fails( + tmp_path: Path, +) -> None: + store = ImageAssetRepository( + tmp_path / "data", + tmp_path / "data" / "manifest.json", + tmp_path / "data" / "temporary", + ) + store.manifest_path.unlink() + recovered = ImageAssetRepository( + tmp_path / "data", + tmp_path / "data" / "manifest.json", + tmp_path / "data" / "temporary", + ) + assert recovered.available + assert recovered.warning + assert recovered.last_report.state == "recovered" + + other = tmp_path / "other" + other.mkdir() + primary = other / "manifest.json" + primary.write_text("{invalid", encoding="utf-8") + unavailable = ImageAssetRepository(other, primary, other / "temporary") + assert not unavailable.available + assert unavailable.warning == "Persistent storage is unavailable." + + +def test_reconciliation_removes_missing_records_but_reports_orphans(tmp_path: Path) -> None: + data = tmp_path / "data" + data.mkdir() + source_folder = data / "sources" / "images" + output_folder = data / "outputs" / "images" + source_folder.mkdir(parents=True) + output_folder.mkdir(parents=True) + (output_folder / "orphan.png").write_bytes(b"orphan") + manifest = { + "schemaVersion": 1, + "sources": {"abcdefghijklmnop": valid_source()}, + "outputs": {"qrstuvwxyzABCDEF": valid_output()}, + } + (data / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + (data / "manifest.json.bak").write_text(json.dumps(manifest), encoding="utf-8") + + store = ImageAssetRepository(data, data / "manifest.json", data / "temporary") + assert store.list_sources() == [] + assert store.list_outputs() == [] + assert store.last_report.missing_sources == 1 + assert store.last_report.missing_outputs == 1 + assert store.last_report.orphan_outputs == 1 + assert (output_folder / "orphan.png").exists()