From 4cf94ee00d3dd3a88d3fb42976305789deea48bb Mon Sep 17 00:00:00 2001 From: John Crafts <5889731+Artsen@users.noreply.github.com> Date: Mon, 27 Jul 2026 23:58:47 -0400 Subject: [PATCH 1/2] feat: add deterministic temporal effect modulation --- docs/architecture.md | 14 + docs/product-direction.md | 12 +- docs/recipes.md | 86 ++++ docs/service-contract.md | 16 +- docs/storage.md | 11 +- docs/temporal-effects.md | 131 ++++++ docs/testing.md | 25 + docs/video-jobs.md | 13 +- docs/video-workflow.md | 14 + glitchcraft/contracts/effects.py | 262 ++++++++++- glitchcraft/effects/engine.py | 25 +- glitchcraft/effects/execution.py | 194 ++++++++ glitchcraft/effects/registry.py | 203 +++++++- glitchcraft/effects/temporal.py | 361 ++++++++++++++ glitchcraft/jobs/contracts.py | 10 +- glitchcraft/jobs/manager.py | 6 + glitchcraft/media/video.py | 54 ++- glitchcraft/service_contract.py | 39 ++ glitchcraft/storage/contracts.py | 6 +- glitchcraft/storage/repository.py | 4 +- glitchcraft/version.py | 4 +- glitchcraft/web/routes.py | 112 ++++- pyproject.toml | 2 +- static/app-manifest.json | 7 +- static/app.js | 438 ++++++++++++++++- static/style.css | 61 +++ templates/index.html | 32 +- tests/browser/image-workflow.spec.js | 95 +++- tests/test_media.py | 50 +- tests/test_service_contract.py | 3 +- tests/test_temporal_effects.py | 671 +++++++++++++++++++++++++++ tests/test_temporal_routes.py | 206 ++++++++ tests/test_video_workflow.py | 20 +- 33 files changed, 3127 insertions(+), 60 deletions(-) create mode 100644 docs/recipes.md create mode 100644 docs/temporal-effects.md create mode 100644 glitchcraft/effects/execution.py create mode 100644 glitchcraft/effects/temporal.py create mode 100644 tests/test_temporal_effects.py create mode 100644 tests/test_temporal_routes.py diff --git a/docs/architecture.md b/docs/architecture.md index 0c06016..fc44385 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -44,3 +44,17 @@ timing summaries survive restart; rolling speed and ETA restart as unknown. FFmpeg stdout carries structured program progress while a separate reader drains bounded stderr. The polling UI owns one timer and one request, rejects obsolete job revisions, and stops on terminal or reset states. + +Recipe v2 adds a separate temporal path without changing Recipe v1 execution. +The strict contract is canonicalized with registry-owned video defaults, then a +`MediaTimelineContext` and immutable `CompiledRecipe` are built once. Each +effect plan stores bounded sorted intervals and supports binary-search +activation. Preview, schedule inspection, full processing, and restart +recompilation use this same compiler. + +Scheduling owns activation and envelope intensity. A typed execution context +owns named deterministic random channels. Operations own visual transformation +and centralized parameter scaling. This separation keeps routes and the video +loop free of effect-specific timing rules. Smooth variation derives random +anchors on demand and is independent of processing order; no mutable global RNG +or previous-frame buffer exists. diff --git a/docs/product-direction.md b/docs/product-direction.md index 2a9d96a..10cbdf4 100644 --- a/docs/product-direction.md +++ b/docs/product-direction.md @@ -24,9 +24,15 @@ delivery-optimization and packaging tool. The transitional interface explains long video work with concise, progressively disclosed telemetry. It truthfully describes applying the enabled stack to each -frame rather than pretending effects run as separate passes. Advanced temporal -feedback, optical flow, frame reordering, codec datamosh, signal modeling, and -new effect algorithms remain intentionally deferred. +frame rather than pretending effects run as separate passes. Recipe v2 now adds +deterministic ranges and bursts, attack/release envelopes, natural parameter +scaling, and coherent named variation to those existing effects. The compiled +schedule API is a stable base for a future React timeline. + +Stateful temporal feedback, optical flow, frame reordering, codec datamosh, +signal modeling, new effect algorithms, draggable keyframes, and preview clips +remain intentionally deferred. Temporal modulation changes independent decoded +frames and does not use prior-frame state. A future orchestration dashboard may discover and check GlitchCraft, ColorCraft, and Web Video Optimizer through related contracts. It is not implemented here diff --git a/docs/recipes.md b/docs/recipes.md new file mode 100644 index 0000000..cc152aa --- /dev/null +++ b/docs/recipes.md @@ -0,0 +1,86 @@ +# Effect recipes + +GlitchCraft 0.3.0 supports recipe schema versions 1 and 2. Recipe documents are +independently versioned inside manifest-v2 records, so adding Recipe v2 does not +change the manifest, storage, or video-job schemas. + +## Recipe v1: exact legacy behavior + +Recipe v1 remains the default for still images and is supported indefinitely for +persisted sources, outputs, and jobs. Its enabled effects run on every frame. +Each stochastic effect receives the existing +`glitchcraft:v1:::` random namespace. Temporal +fields are rejected. + +The v1 engine was not routed through the temporal compiler. Golden frame hashes +lock representative v1 output at multiple frame indexes, which protects image +exports, interrupted video jobs, and deterministic fixtures from reinterpretation. + +## Recipe v2: temporal modulation + +Recipe v2 retains the root seed, ordered instances, stable unique IDs, enabled +state, effect types, and strict parameter models. Each instance also has: + +- `intensity`, from 0 through 1; +- `timing`, using `continuous`, `range`, `sporadic`, or `events`; +- `envelope`, with attack, release, and a curve; +- `variation`, keyed by registry-declared random channel. + +Unknown fields, effect IDs, channels, modes, parameters, and mode-specific timing +fields are rejected. The canonical JSON uses camel-case aliases and can be +persisted without runtime state. + +```json +{ + "schemaVersion": 2, + "seed": 184729, + "effects": [ + { + "id": "horizontal-glitch", + "type": "horizontal_glitch", + "enabled": true, + "intensity": 1, + "parameters": {"count": 6, "shift": 80}, + "timing": { + "mode": "sporadic", + "startSeconds": 0, + "endSeconds": null, + "frequencyPerMinute": 8, + "minimumDurationFrames": 6, + "maximumDurationFrames": 18, + "minimumCooldownFrames": 24, + "maximumCooldownFrames": 90 + }, + "envelope": { + "attackFrames": 1, + "releaseFrames": 5, + "curve": "easeOut" + }, + "variation": { + "layout": {"mode": "perEvent"}, + "offset": {"mode": "smooth", "periodFrames": 6} + } + } + ] +} +``` + +Natural video defaults belong to the Python effect registry and are returned by +`GET /api/effects`. The transitional frontend consumes that metadata rather than +maintaining another copy. An explicit instance value overrides its default. +Recipe v1 never receives these defaults. + +## Still-image rule + +Image requests continue accepting Recipe v1 only. This strict rule prevents a +sporadic video schedule from making a still effect unexpectedly disappear. +Image preview and export behavior therefore remain byte-compatible. Recipe v2 +is accepted by the video preview, schedule, and job APIs. + +## Persistence and restart + +Manifest-v2 image records contain Recipe v1. Manifest-v2 video jobs and outputs +may contain Recipe v1 or Recipe v2. Existing files load without migration or +rewriting. Recipe v2 persists only its portable contract; compiled intervals and +random-generator state are never stored. Restart recovery recompiles the same +schedule from the recipe, source timeline, effect ID, and isolated namespaces. diff --git a/docs/service-contract.md b/docs/service-contract.md index eb19d5f..1d5d739 100644 --- a/docs/service-contract.md +++ b/docs/service-contract.md @@ -40,10 +40,24 @@ state. Readiness reports queue capacity, worker concurrency, manifest migration or reconciliation state, and both required video tools. Saved recipe management, WVO handoff, authentication, and a visible library interface are deferred. -Application version 0.2.1 advertises video-job contract v2 and the +Application version 0.3.0 advertises video-job contract v2 and the `video-render-telemetry` capability. `/metadata` publishes supported phase codes, the client-polling model, a four-Hz runtime update target, FFmpeg program-progress support, and redacted queue totals. The capability is available only when persistent video processing and its manager are available. Discovery never exposes sample buffers, thread names, executable paths, or raw process output. + +The current recipe schema is 2 and supported recipe schemas are `[1, 2]`. +Manifest and storage schemas remain 2. `GET /api/effects` returns registry-owned +temporal defaults, envelope support, intensity scaling, and named variation +channels. `POST /api/video-sources/{sourceId}/effect-schedule` accepts Recipe v2 +and returns a bounded redacted schedule. Preview responses expose only safe +actual-frame and active-effect headers. + +Temporal capability slugs are `temporal-effect-modulation`, +`deterministic-effect-schedules`, `effect-envelopes`, and +`coherent-effect-variation`. Metadata lists continuous, range, sporadic, and +events timing; all five envelope curves; and per-frame, per-event, and smooth +variation. It does not claim feedback, optical flow, datamoshing, keyframes, or +timeline editing. diff --git a/docs/storage.md b/docs/storage.md index 900f46d..4d05028 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -1,6 +1,6 @@ # Persistent media storage -GlitchCraft 0.2.1 uses a configurable managed data root: +GlitchCraft 0.3.0 uses a configurable managed data root: ```text data/ @@ -58,8 +58,15 @@ diagnostics whose values and granularity may vary by platform. Video completion installs the final MP4 and marks its job completed in one manifest mutation. Intermediate job files remain temporary and are never served. -Manifest schema remains v2 in application 0.2.1. Telemetry contract v2 adds only +Manifest schema remains v2 in application 0.3.0. Telemetry contract v2 adds only backward-compatible job defaults: a phase, at most 32 durable milestones, a terminal timing summary, and coarse last-known frame counters. High-frequency FPS, ETA, encoded-time, queue-position, and stale calculations are runtime-only and never cause per-frame manifest writes. + +Manifest-v2 recipe fields are independently discriminated and may contain +Recipe v1 or Recipe v2. Existing Recipe v1 JSON validates directly and is not +migrated or rewritten. Recipe v2 jobs and outputs retain timing, envelope, +intensity, and variation configuration, but not compiled events or runtime RNG +state. Schedule compilation after restart is deterministic and does not add +per-frame persistence. diff --git a/docs/temporal-effects.md b/docs/temporal-effects.md new file mode 100644 index 0000000..082695e --- /dev/null +++ b/docs/temporal-effects.md @@ -0,0 +1,131 @@ +# Deterministic temporal effects + +Temporal modulation controls when and how strongly existing decoded-frame +effects run. It does not add new effect types, retain previous frames, perform +optical flow, or manipulate codec structures. + +## Timeline and frame conversion + +`MediaTimelineContext` contains the rational source frame rate, source duration, +total frames, and media type. A starting time is converted with +`floor(seconds × frame rate)` and is inclusive. An ending time is converted with +`ceil(seconds × frame rate)` and is exclusive. Both are clipped to the source. +The actual preview timestamp is the selected frame index divided by the same +rational rate. + +Video preview seeks to that frame index for Recipe v2. Schedule inspection and +full processing use the same compiler, so a direct preview of frame 250 has the +same activation, envelope, and variation as frame 250 in sequential rendering. + +## Timing modes + +- `continuous` produces one interval over the source, optionally bounded by + start and end times. +- `range` requires one finite start and later finite end. +- `sporadic` precompiles deterministic, non-overlapping bursts. Frequency, + duration, and cooldown ranges are bounded. Cooldown begins after the preceding + event. A partial final event is omitted. +- `events` accepts a nonempty manually placed list. Events are sorted, start is + inclusive, end is exclusive, and overlaps after frame conversion are rejected. + +Each effect is limited to 10,000 compiled events. The inspection response returns +at most 200 events per effect and reports truncation. Invalid or out-of-source +windows are rejected before a persistent job is created. + +Sporadic random values use a schedule namespace derived from the root seed and +stable effect-instance ID. Parameter edits, unrelated effects, and effect order +do not perturb another instance's schedule. Activation is not rolled +independently on every frame. + +## Compiled plan + +`CompiledRecipe` owns immutable `CompiledEffectPlan` values. Each event records +the effect ID, event index, inclusive start, exclusive end, duration, and stable +identity. Start-frame indexes support bounded binary-search activation lookup. +Compilation occurs once per video job or request; schedules are not regenerated +inside the frame loop. + +An activation snapshot provides event position, duration, configured intensity, +envelope intensity, effective intensity, and event identity. Inactive effects +are skipped before their operation runs. + +## Envelopes and intensity + +Supported curves are `linear`, `easeIn`, `easeOut`, `easeInOut`, and `sharp`. +Attack and release lengths are nonnegative and may not exceed the shortest +possible event when combined. + +An instant attack begins at full strength. For an attack of N frames, the first +frame evaluates the curve at `1/N` and frame N−1 reaches one. A release of N +frames begins N frames before the exclusive end; the final interval frame +evaluates to zero. Frames outside the event are exactly zero. + +`effective intensity = configured intensity × envelope value` + +At zero, the operation is skipped and the frame is byte-identical. At one, +configured parameters are fully applied. + +## Natural parameter scaling + +Scaling is centralized in the effect execution layer: + +- noise amount and strength move toward zero; +- pixel size moves toward one, with fractional blending during ramps; +- horizontal band count and displacement move toward zero; +- frame X/Y displacement moves toward zero; +- color-channel offsets move toward zero; +- scan-line darkness moves toward one; +- static density moves toward zero; +- flicker bounds move toward one. + +All scaled values are rebuilt through typed, validated parameter models. Array +math uses bounded float intermediates and returns RGB `uint8` without overflow. + +## Named variation channels + +The registry declares each supported channel, allowed modes, default mode, and +smooth period: + +- noise and static: `detail`; +- horizontal glitch: `layout` and `offset`; +- frame shift: `offset`; +- color bleed: `channels`; +- flicker: `level`; +- pixelation and scan lines: no random channel. + +`perFrame` derives a value from the absolute frame. `perEvent` derives it from +the event index and holds it. `smooth` derives neighboring deterministic anchors +and interpolates with smoothstep. Smooth values require no mutable random walk, +so random-access preview, sequential rendering, restart, and evaluation order +agree. + +Horizontal glitch uses an event-held layout and separately sampled offsets. +With its natural defaults, recognizable bands persist while offsets drift, +envelope-scaled displacement settles, quiet frames follow, and a later event can +select a different layout. Static retains per-frame detail variation. + +## APIs and current UI + +`GET /api/effects` returns parameter metadata, temporal defaults, intensity +support, and channel contracts. `POST +/api/video-sources//effect-schedule` returns a bounded compiled +schedule. Recipe v2 preview responses include safe headers for the actual frame, +actual timestamp, active effect IDs, and effective intensities. + +Video mode builds Recipe v2. Semantic Advanced timing disclosures expose +continuous, range, and sporadic controls, duration/cooldown, envelope, intensity, +and variation. Schedule requests are debounced, abort obsolete work, and ignore +stale responses. The previous valid still preview stays visible while updating. +Image mode stays Recipe v1. + +This remains a transitional Flask/JavaScript interface. It has no draggable +timeline or complete manual-event editor, and preview remains a still frame. +Future React timeline work can consume the schedule endpoint without changing +the recipe or compiler. + +## Domain boundaries + +Temporal modulation evaluates independent decoded frames. It does not use +previous-frame feedback, frame echo, motion trails, optical flow, or stateful +blending. Codec datamoshing changes compressed-frame or motion-vector structures +and remains a separate future processing domain. diff --git a/docs/testing.md b/docs/testing.md index 46fb5f0..85aae89 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -76,3 +76,28 @@ finalizing, verifying, saving, completed, cancel-pending, and canceled states. It checks concise formatting, keyboard-accessible details, ordered effects, previous-result retention, responsive containment, and Axe results without requiring a long render. + +Recipe v1 golden regressions hash representative frames at indexes 0, 1, and 17 +to prove the legacy RNG namespace and pixels remain exact. Temporal tests cover +strict Recipe v2 parsing, rational conversion, every timing mode, event sorting +and overlap rejection, clipping, event limits, schedule seed/ID isolation, +restart recompilation, every envelope curve, intensity scaling for all eight +effects, and random-access per-frame/per-event/smooth variation. + +Pre-encode arrays prove Recipe v2 preview and sequential processing equivalence. +The real FFmpeg/FFprobe test runs a Recipe v2 job through processing, H.264 +finalization, AAC preservation, telemetry, and persisted timing metadata; its +second Recipe v1 job verifies audio removal and legacy video compatibility. +Playwright verifies video mode sends Recipe v2 while image mode sends Recipe v1, +natural horizontal-glitch defaults, timing disclosure keyboard behavior, +schedule summaries, active-preview feedback, responsive controls, existing job +telemetry/cancellation, and no serious or critical Axe findings. + +On the development machine, 200-iteration temporal benchmarks measured an +eight-effect continuous compile at 0.32 ms and an eight-effect sporadic compile +at 11.81 ms for 978 events. Ten thousand randomized timestamp probes across +those eight sporadic plans averaged 1.91 microseconds per effect lookup. A +one-hour 60 fps synthetic schedule containing 5,810 events compiled in 60.33 ms. +These are diagnostic measurements, not platform guarantees; bounded event +counts, one compile per job, binary-search lookup, and inactive-operation +skipping are the enforced properties. diff --git a/docs/video-jobs.md b/docs/video-jobs.md index 3553603..93ee923 100644 --- a/docs/video-jobs.md +++ b/docs/video-jobs.md @@ -23,7 +23,7 @@ and job are committed in one manifest mutation. ## Render telemetry contract v2 -Application 0.2.1 keeps the persistent states above and adds more precise phase +Application 0.2.1 introduced the more precise phase codes: `waiting`, `preparing_source`, `applying_effects`, `finalizing_output`, `verifying_output`, `saving_output`, `completed`, `failed`, and `canceled`. @@ -46,3 +46,14 @@ defaults. Only bounded phase milestones, coarse frame progress, and a terminal timing summary are durable. FFmpeg finalization uses program-progress output and separately drains bounded diagnostics. Unknown keys and malformed optional values are ignored; neither diagnostics nor reader-thread details enter the API. + +Jobs accept and persist Recipe v1 or Recipe v2 while the video-job contract +remains v2. Recipe v2 is compiled once after source metadata is available; +inactive effects are skipped without changing frame telemetry or cancellation. +Restarted jobs rebuild the same plan from their persisted recipe and source +timeline. Compiled intervals and random state are not manifest data. + +The concise telemetry phrase continues counting enabled instances, not active +effects on one scheduled frame. Event-level changes do not create milestones or +manifest writes. Individual job and output metadata expose `recipeVersion` so +clients can distinguish legacy-continuous work from temporal work. diff --git a/docs/video-workflow.md b/docs/video-workflow.md index 8cd4d16..25d0e55 100644 --- a/docs/video-workflow.md +++ b/docs/video-workflow.md @@ -48,3 +48,17 @@ to constant frame rate. Only the first audio stream is preserved. Subtitles, chapters, attachments, multiple audio tracks, and hardware encoding are not preserved or implemented. Detailed delivery optimization remains the responsibility of Web Video Optimizer. + +Video mode now builds Recipe v2 from `/api/effects` registry metadata. Advanced +timing disclosures appear only for enabled video effects and expose continuous, +range, and sporadic modes, bounds, frequency, duration, cooldown, attack, +release, curve, maximum intensity, and supported variation choices. Image mode +continues to build Recipe v1. + +Schedule and preview requests are debounced independently, abort obsolete +requests, and reject stale revisions. The schedule summary shows bounded event +locations. Preview headers report the actual rationally selected frame and +active effects. Moving the timestamp never changes the root seed or schedule, +and the previous valid still remains visible while its replacement loads. +Manual events are supported by the API contract but do not yet have a full +editor. diff --git a/glitchcraft/contracts/effects.py b/glitchcraft/contracts/effects.py index c6bcc72..360b784 100644 --- a/glitchcraft/contracts/effects.py +++ b/glitchcraft/contracts/effects.py @@ -3,9 +3,9 @@ from __future__ import annotations from enum import StrEnum -from typing import Annotated, Any, ClassVar +from typing import Annotated, Any, ClassVar, Literal, TypeAlias -from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, field_validator, model_validator MAX_SEED = (1 << 63) - 1 @@ -112,7 +112,7 @@ def parameter_model(self) -> ParameterModel: class Recipe(ContractModel): - schema_version: Annotated[int, Field(alias="schemaVersion", ge=1, le=1)] = 1 + schema_version: Literal[1] = Field(default=1, alias="schemaVersion") seed: Annotated[int, Field(ge=0, le=MAX_SEED)] effects: list[EffectInstance] = Field(default_factory=list) @@ -128,3 +128,259 @@ def validate_unique_ids(self) -> Recipe: if len(ids) != len(set(ids)): raise ValueError("effect instance IDs must be unique") return self + + +class EnvelopeCurve(StrEnum): + LINEAR = "linear" + EASE_IN = "easeIn" + EASE_OUT = "easeOut" + EASE_IN_OUT = "easeInOut" + SHARP = "sharp" + + +class VariationMode(StrEnum): + PER_FRAME = "perFrame" + PER_EVENT = "perEvent" + SMOOTH = "smooth" + + +class ContinuousTiming(ContractModel): + mode: Literal["continuous"] = "continuous" + start_seconds: Annotated[float, Field(alias="startSeconds", ge=0)] = 0 + end_seconds: Annotated[float | None, Field(alias="endSeconds")] = None + model_config: ClassVar[ConfigDict] = ConfigDict( + extra="forbid", validate_assignment=True, populate_by_name=True + ) + + @field_validator("start_seconds") + @classmethod + def validate_start(cls, value: float) -> float: + if not value < float("inf"): + raise ValueError("startSeconds must be finite") + return value + + @model_validator(mode="after") + def validate_window(self) -> ContinuousTiming: + if self.end_seconds is not None: + if not 0 < self.end_seconds < float("inf"): + raise ValueError("endSeconds must be finite and positive") + if self.end_seconds <= self.start_seconds: + raise ValueError("endSeconds must be later than startSeconds") + return self + + +class RangeTiming(ContractModel): + mode: Literal["range"] + start_seconds: Annotated[float, Field(alias="startSeconds", ge=0)] + end_seconds: Annotated[float, Field(alias="endSeconds", gt=0)] + model_config: ClassVar[ConfigDict] = ConfigDict( + extra="forbid", validate_assignment=True, populate_by_name=True + ) + + @field_validator("start_seconds", "end_seconds") + @classmethod + def validate_bound(cls, value: float) -> float: + if not value < float("inf"): + raise ValueError("timing bounds must be finite") + return value + + @model_validator(mode="after") + def validate_window(self) -> RangeTiming: + if self.end_seconds <= self.start_seconds: + raise ValueError("endSeconds must be later than startSeconds") + return self + + +class SporadicTiming(ContractModel): + mode: Literal["sporadic"] + start_seconds: Annotated[float, Field(alias="startSeconds", ge=0)] = 0 + end_seconds: Annotated[float | None, Field(alias="endSeconds")] = None + frequency_per_minute: Annotated[float, Field(alias="frequencyPerMinute", gt=0, le=120)] = 8 + minimum_duration_frames: Annotated[int, Field(alias="minimumDurationFrames", ge=1, le=3600)] = 6 + maximum_duration_frames: Annotated[int, Field(alias="maximumDurationFrames", ge=1, le=3600)] = ( + 18 + ) + minimum_cooldown_frames: Annotated[ + int, Field(alias="minimumCooldownFrames", ge=0, le=36000) + ] = 24 + maximum_cooldown_frames: Annotated[ + int, Field(alias="maximumCooldownFrames", ge=0, le=36000) + ] = 90 + model_config: ClassVar[ConfigDict] = ConfigDict( + extra="forbid", validate_assignment=True, populate_by_name=True + ) + + @field_validator("start_seconds", "frequency_per_minute") + @classmethod + def validate_finite(cls, value: float) -> float: + if not value < float("inf"): + raise ValueError("timing values must be finite") + return value + + @model_validator(mode="after") + def validate_ranges(self) -> SporadicTiming: + if self.end_seconds is not None: + if not 0 < self.end_seconds < float("inf"): + raise ValueError("endSeconds must be finite and positive") + if self.end_seconds <= self.start_seconds: + raise ValueError("endSeconds must be later than startSeconds") + if self.minimum_duration_frames > self.maximum_duration_frames: + raise ValueError("minimumDurationFrames cannot exceed maximumDurationFrames") + if self.minimum_cooldown_frames > self.maximum_cooldown_frames: + raise ValueError("minimumCooldownFrames cannot exceed maximumCooldownFrames") + return self + + +class ManualEffectEvent(ContractModel): + start_seconds: Annotated[float, Field(alias="startSeconds", ge=0)] + duration_frames: Annotated[int, Field(alias="durationFrames", ge=1, le=3600)] + model_config: ClassVar[ConfigDict] = ConfigDict( + extra="forbid", validate_assignment=True, populate_by_name=True + ) + + @field_validator("start_seconds") + @classmethod + def validate_start(cls, value: float) -> float: + if not value < float("inf"): + raise ValueError("startSeconds must be finite") + return value + + +class EventsTiming(ContractModel): + mode: Literal["events"] + events: Annotated[list[ManualEffectEvent], Field(min_length=1, max_length=10000)] + + @field_validator("events") + @classmethod + def canonicalize_events(cls, value: list[ManualEffectEvent]) -> list[ManualEffectEvent]: + return sorted(value, key=lambda event: (event.start_seconds, event.duration_frames)) + + +EffectTiming: TypeAlias = Annotated[ + ContinuousTiming | RangeTiming | SporadicTiming | EventsTiming, + Field(discriminator="mode"), +] + + +class EffectEnvelope(ContractModel): + attack_frames: Annotated[int, Field(alias="attackFrames", ge=0, le=3600)] = 0 + release_frames: Annotated[int, Field(alias="releaseFrames", ge=0, le=3600)] = 0 + curve: EnvelopeCurve = EnvelopeCurve.LINEAR + model_config: ClassVar[ConfigDict] = ConfigDict( + extra="forbid", validate_assignment=True, populate_by_name=True + ) + + +class PerFrameVariation(ContractModel): + mode: Literal["perFrame"] + + +class PerEventVariation(ContractModel): + mode: Literal["perEvent"] + + +class SmoothVariation(ContractModel): + mode: Literal["smooth"] + period_frames: Annotated[int, Field(alias="periodFrames", ge=2, le=3600)] = 6 + model_config: ClassVar[ConfigDict] = ConfigDict( + extra="forbid", validate_assignment=True, populate_by_name=True + ) + + +VariationSpec: TypeAlias = Annotated[ + PerFrameVariation | PerEventVariation | SmoothVariation, + Field(discriminator="mode"), +] + +VARIATION_CHANNEL_MODES: dict[EffectType, dict[str, frozenset[VariationMode]]] = { + EffectType.NOISE: {"detail": frozenset({VariationMode.PER_FRAME, VariationMode.PER_EVENT})}, + EffectType.PIXELATION: {}, + EffectType.HORIZONTAL_GLITCH: { + "layout": frozenset({VariationMode.PER_FRAME, VariationMode.PER_EVENT}), + "offset": frozenset( + {VariationMode.PER_FRAME, VariationMode.PER_EVENT, VariationMode.SMOOTH} + ), + }, + EffectType.FRAME_SHIFT: { + "offset": frozenset( + {VariationMode.PER_FRAME, VariationMode.PER_EVENT, VariationMode.SMOOTH} + ) + }, + EffectType.COLOR_BLEED: { + "channels": frozenset( + {VariationMode.PER_FRAME, VariationMode.PER_EVENT, VariationMode.SMOOTH} + ) + }, + EffectType.SCAN_LINES: {}, + EffectType.STATIC: {"detail": frozenset({VariationMode.PER_FRAME, VariationMode.PER_EVENT})}, + EffectType.FLICKER: { + "level": frozenset({VariationMode.PER_FRAME, VariationMode.PER_EVENT, VariationMode.SMOOTH}) + }, +} + + +class EffectInstanceV2(ContractModel): + id: Annotated[str, Field(min_length=1, max_length=128)] + type: EffectType + enabled: bool = True + intensity: Annotated[float, Field(ge=0, le=1)] = 1 + parameters: dict[str, Any] = Field(default_factory=dict) + timing: EffectTiming = Field(default_factory=ContinuousTiming) + envelope: EffectEnvelope = Field(default_factory=EffectEnvelope) + variation: dict[str, VariationSpec] = Field(default_factory=dict) + + @model_validator(mode="after") + def validate_contract(self) -> EffectInstanceV2: + model_type = PARAMETER_MODELS[self.type] + validated = model_type.model_validate(self.parameters) + object.__setattr__(self, "parameters", validated.model_dump()) + supported = VARIATION_CHANNEL_MODES[self.type] + unknown = set(self.variation) - set(supported) + if unknown: + raise ValueError(f"unknown variation channels for {self.type.value}: {sorted(unknown)}") + for channel, specification in self.variation.items(): + if VariationMode(specification.mode) not in supported[channel]: + raise ValueError( + f"variation mode {specification.mode} is not supported for channel {channel}" + ) + minimum_duration: int | None = None + if isinstance(self.timing, SporadicTiming): + minimum_duration = self.timing.minimum_duration_frames + elif isinstance(self.timing, EventsTiming): + minimum_duration = min(event.duration_frames for event in self.timing.events) + if minimum_duration is not None and ( + self.envelope.attack_frames + self.envelope.release_frames > minimum_duration + ): + raise ValueError("attackFrames plus releaseFrames exceeds the shortest event") + return self + + def parameter_model(self) -> ParameterModel: + return PARAMETER_MODELS[self.type].model_validate(self.parameters) # type: ignore[return-value] + + +class RecipeV2(ContractModel): + schema_version: Literal[2] = Field(default=2, alias="schemaVersion") + seed: Annotated[int, Field(ge=0, le=MAX_SEED)] + effects: list[EffectInstanceV2] = Field(default_factory=list) + model_config: ClassVar[ConfigDict] = ConfigDict( + extra="forbid", + validate_assignment=True, + populate_by_name=True, + ) + + @model_validator(mode="after") + def validate_unique_ids(self) -> RecipeV2: + ids = [effect.id for effect in self.effects] + if len(ids) != len(set(ids)): + raise ValueError("effect instance IDs must be unique") + return self + + +RecipeDocument: TypeAlias = Annotated[Recipe | RecipeV2, Field(discriminator="schema_version")] +RECIPE_DOCUMENT_ADAPTER: TypeAdapter[RecipeDocument] = TypeAdapter(RecipeDocument) + + +def parse_recipe(value: Any) -> Recipe | RecipeV2: + """Parse either supported recipe version without rewriting its representation.""" + + return RECIPE_DOCUMENT_ADAPTER.validate_python(value) diff --git a/glitchcraft/effects/engine.py b/glitchcraft/effects/engine.py index 3a17037..4e946a2 100644 --- a/glitchcraft/effects/engine.py +++ b/glitchcraft/effects/engine.py @@ -3,21 +3,42 @@ import numpy as np from numpy.typing import NDArray -from glitchcraft.contracts.effects import Recipe +from glitchcraft.contracts.effects import Recipe, RecipeDocument, RecipeV2 +from glitchcraft.effects.execution import apply_effect_v2 from glitchcraft.effects.randomness import effect_rng from glitchcraft.effects.registry import get_effect_definition +from glitchcraft.effects.temporal import CompiledRecipe, EffectExecutionContext from glitchcraft.errors import EffectProcessingError from glitchcraft.media.color import RGBFrame, validate_rgb def apply_effect_stack( frame_rgb: NDArray[np.generic], - recipe: Recipe, + recipe: RecipeDocument, frame_index: int = 0, + *, + compiled_recipe: CompiledRecipe | None = None, ) -> RGBFrame: """Apply enabled effects in recipe order and return a caller-independent frame.""" source = validate_rgb(frame_rgb) output = source.copy() + if isinstance(recipe, RecipeV2): + if compiled_recipe is None: + raise ValueError("Recipe v2 requires a compiled temporal plan") + for plan in compiled_recipe.effects: + activation = plan.activation_at(frame_index) + if not activation.active: + continue + context = EffectExecutionContext(recipe.seed, frame_index, plan.effect, activation) + try: + output = validate_rgb(apply_effect_v2(output, context)) + except Exception as exc: + raise EffectProcessingError( + plan.effect.id, plan.effect.type.value, str(exc) + ) from exc + return output + + assert isinstance(recipe, Recipe) for effect in recipe.effects: if not effect.enabled: continue diff --git a/glitchcraft/effects/execution.py b/glitchcraft/effects/execution.py new file mode 100644 index 0000000..79288c2 --- /dev/null +++ b/glitchcraft/effects/execution.py @@ -0,0 +1,194 @@ +"""Recipe v2 effect execution with centralized intensity scaling.""" + +from __future__ import annotations + +from typing import cast + +import cv2 +import numpy as np + +from glitchcraft.contracts.effects import ( + ColorBleedParameters, + EffectType, + FlickerParameters, + FrameShiftParameters, + HorizontalGlitchParameters, + NoiseParameters, + ParameterModel, + PixelationParameters, + ScanLinesParameters, + StaticParameters, +) +from glitchcraft.effects.registry import get_effect_definition +from glitchcraft.effects.temporal import EffectExecutionContext +from glitchcraft.media.color import RGBFrame, validate_rgb + + +def scale_parameters(parameters: ParameterModel, intensity: float) -> ParameterModel: + """Scale a valid parameter model toward its visually inactive identity.""" + + amount = min(1.0, max(0.0, intensity)) + if isinstance(parameters, NoiseParameters): + return parameters.model_copy( + update={ + "amount": parameters.amount * amount, + "strength": round(parameters.strength * amount), + } + ) + if isinstance(parameters, PixelationParameters): + return parameters.model_copy( + update={"pixel_size": max(1, round(1 + (parameters.pixel_size - 1) * amount))} + ) + if isinstance(parameters, HorizontalGlitchParameters): + return parameters.model_copy( + update={ + "count": max(0, round(parameters.count * amount)), + "shift": max(0, round(parameters.shift * amount)), + } + ) + if isinstance(parameters, FrameShiftParameters): + return parameters.model_copy( + update={ + "max_x": max(0, round(parameters.max_x * amount)), + "max_y": max(0, round(parameters.max_y * amount)), + } + ) + if isinstance(parameters, ColorBleedParameters): + return parameters.model_copy( + update={"max_shift": max(0, round(parameters.max_shift * amount))} + ) + if isinstance(parameters, ScanLinesParameters): + return parameters.model_copy(update={"darkness": 1 + (parameters.darkness - 1) * amount}) + if isinstance(parameters, StaticParameters): + return parameters.model_copy(update={"intensity": parameters.intensity * amount}) + if isinstance(parameters, FlickerParameters): + return parameters.model_copy( + update={ + "minimum": 1 + (parameters.minimum - 1) * amount, + "maximum": 1 + (parameters.maximum - 1) * amount, + } + ) + raise TypeError(f"unsupported parameter model: {type(parameters).__name__}") + + +def _blend(source: RGBFrame, transformed: RGBFrame, amount: float) -> RGBFrame: + if amount >= 1: + return transformed + return cast( + RGBFrame, + np.rint(source.astype(np.float32) * (1 - amount) + transformed.astype(np.float32) * amount) + .clip(0, 255) + .astype(np.uint8), + ) + + +def _horizontal_glitch( + frame: RGBFrame, + parameters: HorizontalGlitchParameters, + context: EffectExecutionContext, +) -> RGBFrame: + output = frame.copy() + intensity = context.activation.effective_intensity + if parameters.count == 0 or parameters.shift == 0 or intensity <= 0: + return output + layout = context.rng("layout") + low = max(1, parameters.count - 2) + actual_count = int(layout.integers(low, parameters.count + 3)) + applied_count = max(0, round(actual_count * intensity)) + height = output.shape[0] + maximum_band_height = max(1, height // 10) + bands = [ + ( + int(layout.integers(0, height)), + int(layout.integers(1, maximum_band_height + 1)), + ) + for _ in range(actual_count) + ] + for lane, (start, band_height) in enumerate(bands[:applied_count]): + end = min(height, start + band_height) + shift = round((context.sample("offset", lane) * 2 - 1) * parameters.shift * intensity) + if shift: + output[start:end, :] = np.roll(output[start:end, :], shift, axis=1) + return output + + +def _frame_shift( + frame: RGBFrame, + parameters: FrameShiftParameters, + context: EffectExecutionContext, +) -> RGBFrame: + intensity = context.activation.effective_intensity + dx = round((context.sample("offset", 0) * 2 - 1) * parameters.max_x * intensity) + dy = round((context.sample("offset", 1) * 2 - 1) * parameters.max_y * intensity) + if dx == 0 and dy == 0: + return frame.copy() + matrix = np.array([[1, 0, dx], [0, 1, dy]], dtype=np.float32) + height, width, _ = frame.shape + return cast( + RGBFrame, + cv2.warpAffine( + frame, + matrix, + (width, height), + flags=cv2.INTER_LINEAR, + borderMode=cv2.BORDER_CONSTANT, + borderValue=(0.0, 0.0, 0.0), + ), + ) + + +def _color_bleed( + frame: RGBFrame, + parameters: ColorBleedParameters, + context: EffectExecutionContext, +) -> RGBFrame: + intensity = context.activation.effective_intensity + red = round((context.sample("channels", 0) * 2 - 1) * parameters.max_shift * intensity) + blue = round((context.sample("channels", 1) * 2 - 1) * parameters.max_shift * intensity) + output = frame.copy() + output[:, :, 0] = np.roll(frame[:, :, 0], red, axis=1) + output[:, :, 2] = np.roll(frame[:, :, 2], blue, axis=1) + return output + + +def _flicker( + frame: RGBFrame, + parameters: FlickerParameters, + context: EffectExecutionContext, +) -> RGBFrame: + intensity = context.activation.effective_intensity + minimum = 1 + (parameters.minimum - 1) * intensity + maximum = 1 + (parameters.maximum - 1) * intensity + factor = minimum + (maximum - minimum) * context.sample("level") + return np.clip(frame.astype(np.float32) * factor, 0, 255).astype(np.uint8) + + +def apply_effect_v2(frame: RGBFrame, context: EffectExecutionContext) -> RGBFrame: + """Apply one active effect using its temporal execution context.""" + + intensity = context.activation.effective_intensity + if intensity <= 0: + return frame.copy() + parameters = context.effect.parameter_model() + effect_type = context.effect.type + if effect_type == EffectType.HORIZONTAL_GLITCH: + assert isinstance(parameters, HorizontalGlitchParameters) + return _horizontal_glitch(frame, parameters, context) + if effect_type == EffectType.FRAME_SHIFT: + assert isinstance(parameters, FrameShiftParameters) + return _frame_shift(frame, parameters, context) + if effect_type == EffectType.COLOR_BLEED: + assert isinstance(parameters, ColorBleedParameters) + return _color_bleed(frame, parameters, context) + if effect_type == EffectType.FLICKER: + assert isinstance(parameters, FlickerParameters) + return _flicker(frame, parameters, context) + + scaled = scale_parameters(parameters, intensity) + definition = get_effect_definition(effect_type) + channel = next(iter(context.effect.variation), None) + generator = context.rng(channel) if channel is not None else np.random.default_rng(0) + transformed = validate_rgb(definition.operation(frame, scaled, generator)) + if effect_type == EffectType.PIXELATION and 0 < intensity < 1: + return _blend(frame, transformed, intensity) + return transformed diff --git a/glitchcraft/effects/registry.py b/glitchcraft/effects/registry.py index c54c8e3..76bc937 100644 --- a/glitchcraft/effects/registry.py +++ b/glitchcraft/effects/registry.py @@ -1,11 +1,24 @@ -"""Canonical effect metadata and operation registry.""" +"""Canonical effect metadata, temporal defaults, and operation registry.""" from __future__ import annotations from dataclasses import dataclass from typing import Any -from glitchcraft.contracts.effects import PARAMETER_MODELS, EffectType +from glitchcraft.contracts.effects import ( + PARAMETER_MODELS, + ContinuousTiming, + EffectEnvelope, + EffectType, + EnvelopeCurve, + PerEventVariation, + PerFrameVariation, + RecipeV2, + SmoothVariation, + SporadicTiming, + VariationMode, + VariationSpec, +) from glitchcraft.effects import operations from glitchcraft.effects.operations import Operation @@ -19,6 +32,19 @@ class ParameterMetadata: unit: str | None = None +@dataclass(frozen=True) +class VariationChannelMetadata: + supported_modes: tuple[VariationMode, ...] + default: VariationSpec + + +@dataclass(frozen=True) +class TemporalDefaults: + timing: ContinuousTiming | SporadicTiming + envelope: EffectEnvelope + variation: dict[str, VariationSpec] + + @dataclass(frozen=True) class EffectDefinition: type: EffectType @@ -29,6 +55,9 @@ class EffectDefinition: defaults: dict[str, Any] parameters: dict[str, ParameterMetadata] operation: Operation + temporal_defaults: TemporalDefaults + variation_channels: dict[str, VariationChannelMetadata] + intensity_scaling: bool = True def _definition( @@ -38,20 +67,46 @@ def _definition( stochastic: bool, operation: Operation, parameters: dict[str, ParameterMetadata], + temporal_defaults: TemporalDefaults, + variation_channels: dict[str, VariationChannelMetadata], ) -> EffectDefinition: - defaults = PARAMETER_MODELS[effect_type]().model_dump() return EffectDefinition( effect_type, name, description, ("image", "video"), stochastic, - defaults, + PARAMETER_MODELS[effect_type]().model_dump(), parameters, operation, + temporal_defaults, + variation_channels, ) +def _channel( + modes: tuple[VariationMode, ...], + default: VariationSpec, +) -> VariationChannelMetadata: + return VariationChannelMetadata(modes, default) + + +CONTINUOUS_DETAIL = TemporalDefaults( + ContinuousTiming(), + EffectEnvelope(), + {"detail": PerFrameVariation(mode="perFrame")}, +) +NO_VARIATION_CONTINUOUS = TemporalDefaults(ContinuousTiming(), EffectEnvelope(), {}) +SPORADIC_CONTROL_DEFAULTS = SporadicTiming( + mode="sporadic", + frequency_per_minute=8, + minimum_duration_frames=6, + maximum_duration_frames=18, + minimum_cooldown_frames=24, + maximum_cooldown_frames=90, +) + + EFFECT_REGISTRY: dict[EffectType, EffectDefinition] = { EffectType.NOISE: _definition( EffectType.NOISE, @@ -64,6 +119,13 @@ def _definition( "strength": ParameterMetadata("integer", 0, 100, 1), "monochromatic": ParameterMetadata("boolean"), }, + CONTINUOUS_DETAIL, + { + "detail": _channel( + (VariationMode.PER_FRAME, VariationMode.PER_EVENT), + PerFrameVariation(mode="perFrame"), + ) + }, ), EffectType.PIXELATION: _definition( EffectType.PIXELATION, @@ -72,6 +134,19 @@ def _definition( False, operations.pixelation, {"pixel_size": ParameterMetadata("integer", 1, 500, 1, "pixels")}, + TemporalDefaults( + SporadicTiming( + mode="sporadic", + frequency_per_minute=4, + minimum_duration_frames=24, + maximum_duration_frames=72, + minimum_cooldown_frames=30, + maximum_cooldown_frames=150, + ), + EffectEnvelope(attack_frames=8, release_frames=12, curve=EnvelopeCurve.EASE_IN_OUT), + {}, + ), + {}, ), EffectType.HORIZONTAL_GLITCH: _definition( EffectType.HORIZONTAL_GLITCH, @@ -83,6 +158,31 @@ def _definition( "count": ParameterMetadata("integer", 0, 100, 1, "count"), "shift": ParameterMetadata("integer", 0, 1000, 1, "pixels"), }, + TemporalDefaults( + SporadicTiming( + mode="sporadic", + frequency_per_minute=8, + minimum_duration_frames=6, + maximum_duration_frames=18, + minimum_cooldown_frames=24, + maximum_cooldown_frames=90, + ), + EffectEnvelope(attack_frames=1, release_frames=5, curve=EnvelopeCurve.EASE_OUT), + { + "layout": PerEventVariation(mode="perEvent"), + "offset": SmoothVariation(mode="smooth", period_frames=6), + }, + ), + { + "layout": _channel( + (VariationMode.PER_FRAME, VariationMode.PER_EVENT), + PerEventVariation(mode="perEvent"), + ), + "offset": _channel( + (VariationMode.PER_FRAME, VariationMode.PER_EVENT, VariationMode.SMOOTH), + SmoothVariation(mode="smooth", period_frames=6), + ), + }, ), EffectType.FRAME_SHIFT: _definition( EffectType.FRAME_SHIFT, @@ -94,6 +194,24 @@ def _definition( "max_x": ParameterMetadata("integer", 0, 2000, 1, "pixels"), "max_y": ParameterMetadata("integer", 0, 2000, 1, "pixels"), }, + TemporalDefaults( + SporadicTiming( + mode="sporadic", + frequency_per_minute=3, + minimum_duration_frames=2, + maximum_duration_frames=8, + minimum_cooldown_frames=45, + maximum_cooldown_frames=180, + ), + EffectEnvelope(attack_frames=0, release_frames=2, curve=EnvelopeCurve.EASE_OUT), + {"offset": PerEventVariation(mode="perEvent")}, + ), + { + "offset": _channel( + (VariationMode.PER_FRAME, VariationMode.PER_EVENT, VariationMode.SMOOTH), + PerEventVariation(mode="perEvent"), + ) + }, ), EffectType.COLOR_BLEED: _definition( EffectType.COLOR_BLEED, @@ -102,6 +220,24 @@ def _definition( True, operations.color_bleed, {"max_shift": ParameterMetadata("integer", 0, 1000, 1, "pixels")}, + TemporalDefaults( + SporadicTiming( + mode="sporadic", + frequency_per_minute=5, + minimum_duration_frames=18, + maximum_duration_frames=48, + minimum_cooldown_frames=24, + maximum_cooldown_frames=120, + ), + EffectEnvelope(attack_frames=6, release_frames=10, curve=EnvelopeCurve.EASE_IN_OUT), + {"channels": SmoothVariation(mode="smooth", period_frames=8)}, + ), + { + "channels": _channel( + (VariationMode.PER_FRAME, VariationMode.PER_EVENT, VariationMode.SMOOTH), + SmoothVariation(mode="smooth", period_frames=8), + ) + }, ), EffectType.SCAN_LINES: _definition( EffectType.SCAN_LINES, @@ -113,6 +249,8 @@ def _definition( "gap": ParameterMetadata("integer", 1, 1000, 1, "pixels"), "darkness": ParameterMetadata("number", 0, 1, 0.05, "multiplier"), }, + NO_VARIATION_CONTINUOUS, + {}, ), EffectType.STATIC: _definition( EffectType.STATIC, @@ -121,6 +259,13 @@ def _definition( True, operations.static, {"intensity": ParameterMetadata("number", 0, 1, 0.01, "percent")}, + CONTINUOUS_DETAIL, + { + "detail": _channel( + (VariationMode.PER_FRAME, VariationMode.PER_EVENT), + PerFrameVariation(mode="perFrame"), + ) + }, ), EffectType.FLICKER: _definition( EffectType.FLICKER, @@ -132,9 +277,59 @@ def _definition( "minimum": ParameterMetadata("number", 0, 4, 0.01, "multiplier"), "maximum": ParameterMetadata("number", 0, 4, 0.01, "multiplier"), }, + TemporalDefaults( + SporadicTiming( + mode="sporadic", + frequency_per_minute=10, + minimum_duration_frames=3, + maximum_duration_frames=10, + minimum_cooldown_frames=12, + maximum_cooldown_frames=60, + ), + EffectEnvelope(attack_frames=1, release_frames=2, curve=EnvelopeCurve.EASE_OUT), + {"level": PerFrameVariation(mode="perFrame")}, + ), + { + "level": _channel( + (VariationMode.PER_FRAME, VariationMode.PER_EVENT, VariationMode.SMOOTH), + PerFrameVariation(mode="perFrame"), + ) + }, ), } def get_effect_definition(effect_type: EffectType) -> EffectDefinition: return EFFECT_REGISTRY[effect_type] + + +def apply_recipe_v2_defaults(recipe: RecipeV2) -> RecipeV2: + """Return a canonical recipe with registry-owned natural defaults filled in.""" + + effects = [] + for effect in recipe.effects: + definition = get_effect_definition(effect.type) + timing = ( + definition.temporal_defaults.timing + if "timing" not in effect.model_fields_set + else effect.timing + ) + envelope = ( + definition.temporal_defaults.envelope + if "envelope" not in effect.model_fields_set + else effect.envelope + ) + variation = {**definition.temporal_defaults.variation, **effect.variation} + effects.append( + effect.model_copy( + update={ + "timing": timing.model_copy(deep=True), + "envelope": envelope.model_copy(deep=True), + "variation": { + name: specification.model_copy(deep=True) + for name, specification in variation.items() + }, + } + ) + ) + return recipe.model_copy(update={"effects": effects}) diff --git a/glitchcraft/effects/temporal.py b/glitchcraft/effects/temporal.py new file mode 100644 index 0000000..a1f7755 --- /dev/null +++ b/glitchcraft/effects/temporal.py @@ -0,0 +1,361 @@ +"""Deterministic Recipe v2 timeline compilation and random-access activation.""" + +from __future__ import annotations + +import hashlib +import math +from bisect import bisect_right +from dataclasses import dataclass +from fractions import Fraction + +import numpy as np + +from glitchcraft.contracts.effects import ( + ContinuousTiming, + EffectEnvelope, + EffectInstanceV2, + EventsTiming, + PerEventVariation, + PerFrameVariation, + RangeTiming, + RecipeV2, + SmoothVariation, + SporadicTiming, +) + +MAX_COMPILED_EVENTS = 10000 + + +def _fraction(value: float | int | str) -> Fraction: + return Fraction(str(value)) + + +@dataclass(frozen=True) +class MediaTimelineContext: + """Canonical source timeline used by preview, inspection, and rendering.""" + + frame_rate: Fraction + duration: Fraction + total_frames: int + media_type: str = "video" + + @classmethod + def from_source( + cls, + *, + frame_rate: str | float, + duration_seconds: float, + total_frames: int | None, + media_type: str = "video", + ) -> MediaTimelineContext: + rate = Fraction(frame_rate) if isinstance(frame_rate, str) else _fraction(frame_rate) + duration = _fraction(duration_seconds) + if rate <= 0 or duration <= 0: + raise ValueError("timeline frame rate and duration must be positive") + calculated = max(1, math.ceil(duration * rate)) + frames = total_frames if total_frames is not None else calculated + if frames <= 0: + raise ValueError("timeline total frames must be positive") + return cls(rate, duration, frames, media_type) + + @property + def frame_rate_text(self) -> str: + return f"{self.frame_rate.numerator}/{self.frame_rate.denominator}" + + @property + def duration_seconds(self) -> float: + return float(self.duration) + + def start_frame(self, seconds: float) -> int: + """Convert seconds to an inclusive frame by flooring, clipped to the source.""" + + return min(self.total_frames, max(0, math.floor(_fraction(seconds) * self.frame_rate))) + + def end_frame(self, seconds: float | None) -> int: + """Convert seconds to an exclusive frame by ceiling, clipped to the source.""" + + if seconds is None: + return self.total_frames + return min(self.total_frames, max(0, math.ceil(_fraction(seconds) * self.frame_rate))) + + def frame_seconds(self, frame_index: int) -> float: + return float(Fraction(frame_index, 1) / self.frame_rate) + + +@dataclass(frozen=True) +class EffectEvent: + effect_id: str + event_index: int + start_frame: int + end_frame: int + identity: str + + @property + def duration_frames(self) -> int: + return self.end_frame - self.start_frame + + +@dataclass(frozen=True) +class EffectActivation: + active: bool + event_index: int | None = None + event_relative_frame: int | None = None + event_duration: int | None = None + envelope_intensity: float = 0 + configured_intensity: float = 0 + effective_intensity: float = 0 + event_identity: str | None = None + + +@dataclass(frozen=True) +class CompiledEffectPlan: + effect: EffectInstanceV2 + events: tuple[EffectEvent, ...] + starts: tuple[int, ...] + + def activation_at(self, frame_index: int) -> EffectActivation: + position = bisect_right(self.starts, frame_index) - 1 + if position < 0: + return EffectActivation(active=False) + event = self.events[position] + if frame_index >= event.end_frame: + return EffectActivation(active=False) + relative = frame_index - event.start_frame + envelope = envelope_value(relative, event.duration_frames, self.effect.envelope) + effective = self.effect.intensity * envelope + return EffectActivation( + active=effective > 0, + event_index=event.event_index, + event_relative_frame=relative, + event_duration=event.duration_frames, + envelope_intensity=envelope, + configured_intensity=self.effect.intensity, + effective_intensity=effective, + event_identity=event.identity, + ) + + +@dataclass(frozen=True) +class CompiledRecipe: + recipe: RecipeV2 + timeline: MediaTimelineContext + effects: tuple[CompiledEffectPlan, ...] + + def activation_at(self, effect_id: str, frame_index: int) -> EffectActivation: + for plan in self.effects: + if plan.effect.id == effect_id: + return plan.activation_at(frame_index) + raise KeyError(effect_id) + + +def _seed(root_seed: int, effect_id: str, namespace: str, *parts: object) -> int: + material = ":".join( + ["glitchcraft", "v2", namespace, str(root_seed), effect_id, *(str(part) for part in parts)] + ).encode() + return int.from_bytes(hashlib.sha256(material).digest()[:16], "big") + + +def _rng(root_seed: int, effect_id: str, namespace: str, *parts: object) -> np.random.Generator: + return np.random.default_rng(_seed(root_seed, effect_id, namespace, *parts)) + + +def _event( + recipe: RecipeV2, + effect: EffectInstanceV2, + index: int, + start: int, + end: int, +) -> EffectEvent: + identity = hashlib.sha256( + f"glitchcraft:v2:event:{recipe.seed}:{effect.id}:{index}:{start}:{end}".encode() + ).hexdigest()[:24] + return EffectEvent(effect.id, index, start, end, identity) + + +def _window( + timing: ContinuousTiming | RangeTiming | SporadicTiming, + timeline: MediaTimelineContext, +) -> tuple[int, int]: + return timeline.start_frame(timing.start_seconds), timeline.end_frame(timing.end_seconds) + + +def _compile_sporadic( + recipe: RecipeV2, + effect: EffectInstanceV2, + timing: SporadicTiming, + timeline: MediaTimelineContext, +) -> list[EffectEvent]: + start, end = _window(timing, timeline) + if start >= end: + raise ValueError(f"sporadic window for {effect.id} is outside the source") + generator = _rng(recipe.seed, effect.id, "schedule") + nominal_spacing = max(1, round(float(timeline.frame_rate) * 60 / timing.frequency_per_minute)) + initial_quiet = int(generator.integers(0, nominal_spacing + 1)) + cursor = start + initial_quiet + events: list[EffectEvent] = [] + while cursor < end and len(events) < MAX_COMPILED_EVENTS: + duration = int( + generator.integers( + timing.minimum_duration_frames, + timing.maximum_duration_frames + 1, + ) + ) + event_end = cursor + duration + if event_end > end: + break + events.append(_event(recipe, effect, len(events), cursor, event_end)) + cooldown = int( + generator.integers( + timing.minimum_cooldown_frames, + timing.maximum_cooldown_frames + 1, + ) + ) + baseline = duration + cooldown + extra_limit = max(0, nominal_spacing - baseline) * 2 + extra_quiet = int(generator.integers(0, extra_limit + 1)) if extra_limit else 0 + cursor = event_end + cooldown + extra_quiet + return events + + +def _compile_events( + recipe: RecipeV2, + effect: EffectInstanceV2, + timing: EventsTiming, + timeline: MediaTimelineContext, +) -> list[EffectEvent]: + events: list[EffectEvent] = [] + previous_end = -1 + for manual in timing.events: + start = timeline.start_frame(manual.start_seconds) + end = min(timeline.total_frames, start + manual.duration_frames) + if start >= timeline.total_frames: + raise ValueError(f"manual event for {effect.id} starts outside the source") + if start < previous_end: + raise ValueError(f"manual events for {effect.id} overlap after frame conversion") + if effect.envelope.attack_frames + effect.envelope.release_frames > end - start: + raise ValueError(f"envelope exceeds a clipped manual event for {effect.id}") + events.append(_event(recipe, effect, len(events), start, end)) + previous_end = end + return events + + +def _compile_effect( + recipe: RecipeV2, + effect: EffectInstanceV2, + timeline: MediaTimelineContext, +) -> CompiledEffectPlan: + timing = effect.timing + if not effect.enabled or effect.intensity == 0: + events: list[EffectEvent] = [] + elif isinstance(timing, SporadicTiming): + events = _compile_sporadic(recipe, effect, timing, timeline) + elif isinstance(timing, EventsTiming): + events = _compile_events(recipe, effect, timing, timeline) + else: + start, end = _window(timing, timeline) + if start >= end: + raise ValueError(f"timing window for {effect.id} is outside the source") + else: + duration = end - start + if effect.envelope.attack_frames + effect.envelope.release_frames > duration: + raise ValueError(f"envelope exceeds the configured range for {effect.id}") + events = [_event(recipe, effect, 0, start, end)] + if len(events) >= MAX_COMPILED_EVENTS: + raise ValueError(f"compiled schedule for {effect.id} reaches the safe event limit") + return CompiledEffectPlan(effect, tuple(events), tuple(event.start_frame for event in events)) + + +def compile_recipe(recipe: RecipeV2, timeline: MediaTimelineContext) -> CompiledRecipe: + """Compile immutable, bounded effect intervals once for random-access evaluation.""" + + from glitchcraft.effects.registry import apply_recipe_v2_defaults + + canonical = apply_recipe_v2_defaults(recipe) + return CompiledRecipe( + canonical, + timeline, + tuple(_compile_effect(canonical, effect, timeline) for effect in canonical.effects), + ) + + +def _curve(value: float, curve: str) -> float: + value = min(1.0, max(0.0, value)) + if curve == "linear": + return value + if curve == "easeIn": + return value * value + if curve == "easeOut": + return 1 - (1 - value) * (1 - value) + if curve == "easeInOut": + return value * value * (3 - 2 * value) + if curve == "sharp": + return value**4 + raise ValueError(f"unknown envelope curve: {curve}") + + +def envelope_value(relative_frame: int, duration: int, envelope: EffectEnvelope) -> float: + """Evaluate an inclusive-start/exclusive-end envelope for one active interval.""" + + if relative_frame < 0 or relative_frame >= duration: + return 0 + attack = envelope.attack_frames + release = envelope.release_frames + if attack and relative_frame < attack: + return _curve((relative_frame + 1) / attack, envelope.curve.value) + if release and relative_frame >= duration - release: + return _curve((duration - relative_frame - 1) / release, envelope.curve.value) + return 1 + + +@dataclass(frozen=True) +class EffectExecutionContext: + """Typed random and temporal inputs supplied to a Recipe v2 operation.""" + + root_seed: int + frame_index: int + effect: EffectInstanceV2 + activation: EffectActivation + + def _variation(self, channel: str) -> PerFrameVariation | PerEventVariation | SmoothVariation: + specification = self.effect.variation.get(channel) + if specification is None: + raise KeyError(f"missing variation channel: {channel}") + return specification + + def rng(self, channel: str, lane: int = 0) -> np.random.Generator: + specification = self._variation(channel) + if isinstance(specification, PerEventVariation): + position = self.activation.event_index + else: + position = self.frame_index + return _rng(self.root_seed, self.effect.id, f"variation:{channel}", position, lane) + + def sample(self, channel: str, lane: int = 0) -> float: + specification = self._variation(channel) + if not isinstance(specification, SmoothVariation): + return float(self.rng(channel, lane).random()) + relative = self.activation.event_relative_frame or 0 + lower = relative // specification.period_frames + fraction = (relative % specification.period_frames) / specification.period_frames + smooth_fraction = fraction * fraction * (3 - 2 * fraction) + left = float( + _rng( + self.root_seed, + self.effect.id, + f"smooth:{channel}", + self.activation.event_index, + lower, + lane, + ).random() + ) + right = float( + _rng( + self.root_seed, + self.effect.id, + f"smooth:{channel}", + self.activation.event_index, + lower + 1, + lane, + ).random() + ) + return left + (right - left) * smooth_fraction diff --git a/glitchcraft/jobs/contracts.py b/glitchcraft/jobs/contracts.py index 03b33bf..e3f9051 100644 --- a/glitchcraft/jobs/contracts.py +++ b/glitchcraft/jobs/contracts.py @@ -4,7 +4,7 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator -from glitchcraft.contracts.effects import Recipe +from glitchcraft.contracts.effects import RecipeDocument, RecipeV2 from glitchcraft.storage.contracts import AudioMode @@ -13,7 +13,7 @@ class VideoRequestModel(BaseModel): class VideoPreviewRequest(VideoRequestModel): - recipe: Recipe + recipe: RecipeDocument timestamp_seconds: float = Field(alias="timestampSeconds", ge=0) @field_validator("timestamp_seconds") @@ -25,5 +25,9 @@ def finite_timestamp(cls, value: float) -> float: class VideoJobRequest(VideoRequestModel): - recipe: Recipe + recipe: RecipeDocument audio_mode: AudioMode = Field(default=AudioMode.PRESERVE, alias="audioMode") + + +class EffectScheduleRequest(VideoRequestModel): + recipe: RecipeV2 diff --git a/glitchcraft/jobs/manager.py b/glitchcraft/jobs/manager.py index bdc81c9..121c11e 100644 --- a/glitchcraft/jobs/manager.py +++ b/glitchcraft/jobs/manager.py @@ -9,6 +9,7 @@ from pathlib import Path from threading import Condition, Thread +from glitchcraft.effects.temporal import MediaTimelineContext from glitchcraft.errors import ProcessingCanceled, QueueCapacityError from glitchcraft.jobs.telemetry import ( FinalizationProgressSnapshot, @@ -226,6 +227,11 @@ def progress(snapshot: FrameProgressSnapshot) -> None: job.recipe, progress_hook=progress, cancellation_check=canceled, + timeline_context=MediaTimelineContext.from_source( + frame_rate=source.frame_rate, + duration_seconds=source.duration_seconds, + total_frames=source.frame_count, + ), ) self.repository.update_video_job( job_id, diff --git a/glitchcraft/media/video.py b/glitchcraft/media/video.py index 7762bc8..c9afe43 100644 --- a/glitchcraft/media/video.py +++ b/glitchcraft/media/video.py @@ -8,8 +8,9 @@ import numpy as np from numpy.typing import NDArray -from glitchcraft.contracts.effects import Recipe +from glitchcraft.contracts.effects import Recipe, RecipeDocument, RecipeV2 from glitchcraft.effects.engine import apply_effect_stack +from glitchcraft.effects.temporal import CompiledRecipe, MediaTimelineContext, compile_recipe from glitchcraft.errors import MediaReadError, MediaWriteError, ProcessingCanceled from glitchcraft.jobs.telemetry import FrameProgressSnapshot from glitchcraft.media.color import bgr_to_rgb, rgb_to_bgr @@ -17,6 +18,7 @@ ProgressHook = Callable[[FrameProgressSnapshot], None] CancellationCheck = Callable[[], bool] +ActivationHook = Callable[[int, tuple[tuple[str, float], ...]], None] def create_video_preview(input_path: Path, preview_path: Path, recipe: Recipe) -> None: @@ -26,21 +28,41 @@ def create_video_preview(input_path: Path, preview_path: Path, recipe: Recipe) - def create_video_preview_at( input_path: Path, preview_path: Path, - recipe: Recipe, + recipe: RecipeDocument, *, timestamp_seconds: float, + timeline_context: MediaTimelineContext | None = None, + activation_hook: ActivationHook | None = None, ) -> int: capture = cv2.VideoCapture(str(input_path)) try: if not capture.isOpened(): raise MediaReadError("The video could not be opened.") - if timestamp_seconds and hasattr(capture, "set"): + compiled: CompiledRecipe | None = None + target_frame: int | None = None + if isinstance(recipe, RecipeV2): + timeline = timeline_context or MediaTimelineContext.from_source( + frame_rate=capture.get(cv2.CAP_PROP_FPS), + duration_seconds=( + capture.get(cv2.CAP_PROP_FRAME_COUNT) / capture.get(cv2.CAP_PROP_FPS) + ), + total_frames=max(1, int(capture.get(cv2.CAP_PROP_FRAME_COUNT))), + ) + compiled = compile_recipe(recipe, timeline) + target_frame = min(timeline.total_frames - 1, timeline.start_frame(timestamp_seconds)) + capture.set(cv2.CAP_PROP_POS_FRAMES, target_frame) + elif timestamp_seconds and hasattr(capture, "set"): capture.set(cv2.CAP_PROP_POS_MSEC, timestamp_seconds * 1000) readable, frame_bgr = capture.read() if not readable or frame_bgr is None: raise MediaReadError("A preview frame could not be decoded.") - frame_index = 0 - if timestamp_seconds: + frame_index = target_frame or 0 + if target_frame is not None: + try: + frame_index = max(0, int(capture.get(cv2.CAP_PROP_POS_FRAMES)) - 1) + except (KeyError, TypeError, ValueError): + frame_index = target_frame + elif timestamp_seconds: try: frame_index = max(0, int(capture.get(cv2.CAP_PROP_POS_FRAMES)) - 1) except (KeyError, TypeError, ValueError): @@ -49,7 +71,17 @@ def create_video_preview_at( bgr_to_rgb(cast(NDArray[np.uint8], frame_bgr)), recipe, frame_index=frame_index, + compiled_recipe=compiled, ) + if activation_hook is not None and compiled is not None: + activation_hook( + frame_index, + tuple( + (plan.effect.id, activation.effective_intensity) + for plan in compiled.effects + if (activation := plan.activation_at(frame_index)).active + ), + ) save_image_rgb(processed, preview_path) return frame_index finally: @@ -59,9 +91,10 @@ def create_video_preview_at( def process_video( input_path: Path, output_path: Path, - recipe: Recipe, + recipe: RecipeDocument, progress_hook: ProgressHook | None = None, cancellation_check: CancellationCheck | None = None, + timeline_context: MediaTimelineContext | None = None, ) -> None: if cancellation_check is not None and cancellation_check(): raise ProcessingCanceled("Video processing was canceled.") @@ -76,6 +109,14 @@ def process_video( total = max(0, int(capture.get(cv2.CAP_PROP_FRAME_COUNT))) if fps <= 0 or width <= 0 or height <= 0: raise MediaReadError("The video metadata is invalid.") + compiled: CompiledRecipe | None = None + if isinstance(recipe, RecipeV2): + timeline = timeline_context or MediaTimelineContext.from_source( + frame_rate=fps, + duration_seconds=(total / fps) if total else 1 / fps, + total_frames=total or None, + ) + compiled = compile_recipe(recipe, timeline) writer = cv2.VideoWriter( str(output_path), cv2.VideoWriter_fourcc(*"mp4v"), # type: ignore[attr-defined] @@ -95,6 +136,7 @@ def process_video( bgr_to_rgb(cast(NDArray[np.uint8], frame_bgr)), recipe, frame_index=frame_index, + compiled_recipe=compiled, ) writer.write(rgb_to_bgr(processed)) frame_index += 1 diff --git a/glitchcraft/service_contract.py b/glitchcraft/service_contract.py index 8681d58..f99b344 100644 --- a/glitchcraft/service_contract.py +++ b/glitchcraft/service_contract.py @@ -20,6 +20,10 @@ "bounded-video-jobs", "video-job-cancellation", "video-render-telemetry", + "temporal-effect-modulation", + "deterministic-effect-schedules", + "effect-envelopes", + "coherent-effect-variation", "timestamp-video-preview", "audio-preserving-video-export", "http-range-video-streaming", @@ -53,6 +57,27 @@ def effect_metadata() -> list[dict[str, Any]]: "name": definition.display_name, "mediaTypes": list(definition.media_types), "stochastic": definition.stochastic, + "temporalSupport": True, + "intensityScaling": definition.intensity_scaling, + "naturalVideoDefaults": { + "timing": definition.temporal_defaults.timing.model_dump( + by_alias=True, mode="json" + ), + "envelope": definition.temporal_defaults.envelope.model_dump( + by_alias=True, mode="json" + ), + "variation": { + name: specification.model_dump(by_alias=True, mode="json") + for name, specification in definition.temporal_defaults.variation.items() + }, + }, + "variationChannels": { + name: { + "supportedModes": [mode.value for mode in channel.supported_modes], + "default": channel.default.model_dump(by_alias=True, mode="json"), + } + for name, channel in definition.variation_channels.items() + }, } for definition in EFFECT_REGISTRY.values() ] @@ -119,6 +144,20 @@ def capability_details( "available": video_ready and video_manager_running, "optional": True, }, + *[ + { + "slug": slug, + "exists": True, + "available": video_ready, + "optional": True, + } + for slug in ( + "temporal-effect-modulation", + "deterministic-effect-schedules", + "effect-envelopes", + "coherent-effect-variation", + ) + ], { "slug": "timestamp-video-preview", "exists": True, diff --git a/glitchcraft/storage/contracts.py b/glitchcraft/storage/contracts.py index 7f7a867..eb99e4d 100644 --- a/glitchcraft/storage/contracts.py +++ b/glitchcraft/storage/contracts.py @@ -10,7 +10,7 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator -from glitchcraft.contracts.effects import Recipe +from glitchcraft.contracts.effects import Recipe, RecipeDocument from glitchcraft.jobs.telemetry import ( VideoJobMilestone, VideoJobPhase, @@ -233,7 +233,7 @@ class VideoOutputRecord(StorageModel): has_audio: bool audio_codec: Literal["aac"] | None = None file_size: Annotated[int, Field(ge=1)] - recipe: Recipe + recipe: RecipeDocument created_at: datetime @field_validator("stored_name") @@ -316,7 +316,7 @@ class VideoJobRecord(StorageModel): id: OpaqueId source_id: OpaqueId output_id: OpaqueId | None = None - recipe: Recipe + recipe: RecipeDocument audio_mode: AudioMode = AudioMode.PRESERVE state: VideoJobState = VideoJobState.QUEUED phase: VideoJobPhase = VideoJobPhase.WAITING diff --git a/glitchcraft/storage/repository.py b/glitchcraft/storage/repository.py index e47d922..792c5ae 100644 --- a/glitchcraft/storage/repository.py +++ b/glitchcraft/storage/repository.py @@ -16,7 +16,7 @@ from pydantic import ValidationError -from glitchcraft.contracts.effects import Recipe +from glitchcraft.contracts.effects import Recipe, RecipeDocument from glitchcraft.jobs.telemetry import ( VideoJobMilestone, VideoJobPhase, @@ -738,7 +738,7 @@ def lease_video_source(self, asset_id: str) -> Iterator[tuple[VideoSourceRecord, self._leases[key] = max(0, self._leases.get(key, 1) - 1) def create_video_job( - self, *, source_id: str, recipe: Recipe, audio_mode: str + self, *, source_id: str, recipe: RecipeDocument, audio_mode: str ) -> VideoJobRecord: with self._lock: self.get_video_source(source_id) diff --git a/glitchcraft/version.py b/glitchcraft/version.py index 36ee1ad..c504205 100644 --- a/glitchcraft/version.py +++ b/glitchcraft/version.py @@ -3,8 +3,8 @@ APP_ID = "glitchcraft" APP_NAME = "GlitchCraft" APP_DESCRIPTOR = "Local visual-effects workspace" -APP_VERSION = "0.2.1" +APP_VERSION = "0.3.0" MANIFEST_SCHEMA_VERSION = 2 -RECIPE_SCHEMA_VERSION = 1 +RECIPE_SCHEMA_VERSION = 2 STORAGE_SCHEMA_VERSION = 2 VIDEO_JOB_SCHEMA_VERSION = 2 diff --git a/glitchcraft/web/routes.py b/glitchcraft/web/routes.py index 4840198..b260883 100644 --- a/glitchcraft/web/routes.py +++ b/glitchcraft/web/routes.py @@ -24,13 +24,14 @@ from werkzeug.datastructures import FileStorage from werkzeug.utils import secure_filename -from glitchcraft.contracts.effects import MAX_SEED, Recipe +from glitchcraft.contracts.effects import MAX_SEED, Recipe, RecipeDocument, RecipeV2 from glitchcraft.contracts.image_workflow import ImageRecipeRequest, ImageSourceOptions from glitchcraft.contracts.legacy import FullVideoRequest, LegacyParameters, UploadMode from glitchcraft.effects.engine import apply_effect_stack -from glitchcraft.effects.registry import EFFECT_REGISTRY +from glitchcraft.effects.registry import EFFECT_REGISTRY, SPORADIC_CONTROL_DEFAULTS +from glitchcraft.effects.temporal import MediaTimelineContext, compile_recipe from glitchcraft.errors import ExternalToolError, GlitchCraftError, QueueCapacityError -from glitchcraft.jobs.contracts import VideoJobRequest, VideoPreviewRequest +from glitchcraft.jobs.contracts import EffectScheduleRequest, VideoJobRequest, VideoPreviewRequest from glitchcraft.jobs.manager import VideoJobManager from glitchcraft.jobs.telemetry import FrameProgressSnapshot, VideoJobPhase from glitchcraft.media.ffmpeg import reencode_for_browser @@ -284,6 +285,7 @@ def _video_output_json(output: VideoOutputRecord) -> dict[str, Any]: "streamUrl": url_for("glitchcraft.serve_video_output", output_id=output.id), "downloadUrl": url_for("glitchcraft.download_video_output", output_id=output.id), "recipe": output.recipe.model_dump(mode="json", by_alias=True), + "recipeVersion": output.recipe.schema_version, } @@ -300,6 +302,7 @@ def _video_job_json(job: VideoJobRecord, *, full_telemetry: bool = True) -> dict "audioMode": job.audio_mode, "seed": job.recipe.seed, "recipe": job.recipe.model_dump(mode="json", by_alias=True), + "recipeVersion": job.recipe.schema_version, "recoveredAfterRestart": job.recovered_after_restart, "cancellationRequested": job.cancellation_requested, "createdAt": job.created_at.isoformat().replace("+00:00", "Z"), @@ -344,7 +347,7 @@ def _video_worker( task_id: str, input_path: Path, output_path: Path, - recipe: Recipe, + recipe: RecipeDocument, ) -> None: try: @@ -397,6 +400,7 @@ def metadata() -> Response | tuple[Response, int]: version=APP_VERSION, manifestSchemaVersion=1, recipeSchemaVersion=RECIPE_SCHEMA_VERSION, + supportedRecipeSchemaVersions=[1, 2], storageSchemaVersion=STORAGE_SCHEMA_VERSION, videoJobSchemaVersion=VIDEO_JOB_SCHEMA_VERSION, runtime={ @@ -408,6 +412,13 @@ def metadata() -> Response | tuple[Response, int]: supportedImageFormats=image_formats(), supportedVideoExtensions=list(SUPPORTED_VIDEO_EXTENSIONS), effectTypes=[effect_type.value for effect_type in EFFECT_REGISTRY], + temporalEffects={ + "timingModes": ["continuous", "range", "sporadic", "events"], + "envelopeCurves": ["linear", "easeIn", "easeOut", "easeInOut", "sharp"], + "variationModes": ["perFrame", "perEvent", "smooth"], + "scheduleInspection": "/api/video-sources/{sourceId}/effect-schedule", + "effectMetadata": url_for("glitchcraft.effects"), + }, library={ "imageSources": len(repository.list_sources()) if repository.available else 0, "imageOutputs": len(repository.list_outputs()) if repository.available else 0, @@ -437,6 +448,7 @@ def metadata() -> Response | tuple[Response, int]: "health": url_for("glitchcraft.health"), "readiness": url_for("glitchcraft.readiness"), "capabilities": url_for("glitchcraft.capabilities"), + "effects": url_for("glitchcraft.effects"), "storage": url_for("glitchcraft.storage_status"), }, ) @@ -556,6 +568,17 @@ def capabilities() -> Response: ) +@bp.get("/api/effects") +def effects() -> Response: + return jsonify( + schemaVersion=1, + recipeSchemaVersion=RECIPE_SCHEMA_VERSION, + supportedRecipeSchemaVersions=[1, 2], + sporadicControlDefaults=SPORADIC_CONTROL_DEFAULTS.model_dump(by_alias=True, mode="json"), + effects=effect_metadata(), + ) + + @bp.get("/api/storage") def storage_status() -> Response | tuple[Response, int]: try: @@ -967,6 +990,14 @@ def download_video_source(source_id: str) -> Response | tuple[Response, int]: return response +def _source_timeline(source: VideoSourceRecord) -> MediaTimelineContext: + return MediaTimelineContext.from_source( + frame_rate=source.frame_rate, + duration_seconds=source.duration_seconds, + total_frames=source.frame_count, + ) + + @bp.post("/api/video-sources//preview") def preview_video_source(source_id: str) -> Response | tuple[Response, int]: preview_path: Path | None = None @@ -975,6 +1006,15 @@ def preview_video_source(source_id: str) -> Response | tuple[Response, int]: source = _repository().get_video_source(source_id) if payload.timestamp_seconds >= source.duration_seconds: raise ValueError("The preview timestamp must be inside the video.") + timeline = _source_timeline(source) + active_effects: tuple[tuple[str, float], ...] = () + + def capture_activation( + _frame_index: int, activations: tuple[tuple[str, float], ...] + ) -> None: + nonlocal active_effects + active_effects = activations + preview_path = _repository().new_temporary_path(".png") with _repository().lease_video_source(source_id) as (_, path): frame_index = create_video_preview_at( @@ -982,10 +1022,21 @@ def preview_video_source(source_id: str) -> Response | tuple[Response, int]: preview_path, payload.recipe, timestamp_seconds=payload.timestamp_seconds, + timeline_context=timeline, + activation_hook=capture_activation, ) response = Response(preview_path.read_bytes(), mimetype="image/png") response.headers["X-GlitchCraft-Frame-Index"] = str(frame_index) - response.headers["X-GlitchCraft-Timestamp-Seconds"] = str(payload.timestamp_seconds) + response.headers["X-GlitchCraft-Timestamp-Seconds"] = str( + timeline.frame_seconds(frame_index) + ) + response.headers["X-GlitchCraft-Active-Effect-Count"] = str(len(active_effects)) + response.headers["X-GlitchCraft-Active-Effect-Ids"] = ",".join( + effect_id for effect_id, _ in active_effects + ) + response.headers["X-GlitchCraft-Effective-Intensities"] = ",".join( + f"{effect_id}:{intensity:.6f}" for effect_id, intensity in active_effects + ) return _no_store(response) except (ValidationError, ValueError) as exc: return _validation_error(exc) @@ -998,15 +1049,64 @@ def preview_video_source(source_id: str) -> Response | tuple[Response, int]: preview_path.unlink(missing_ok=True) +@bp.post("/api/video-sources//effect-schedule") +def inspect_effect_schedule(source_id: str) -> Response | tuple[Response, int]: + try: + payload = EffectScheduleRequest.model_validate(request.get_json(silent=True)) + source = _repository().get_video_source(source_id) + timeline = _source_timeline(source) + compiled = compile_recipe(payload.recipe, timeline) + effects_payload = [] + for plan in compiled.effects: + visible = plan.events[:200] + effects_payload.append( + { + "effectId": plan.effect.id, + "type": plan.effect.type, + "mode": plan.effect.timing.mode, + "events": [ + { + "eventIndex": event.event_index, + "startFrame": event.start_frame, + "endFrame": event.end_frame, + "startSeconds": timeline.frame_seconds(event.start_frame), + "endSeconds": timeline.frame_seconds(event.end_frame), + "durationFrames": event.duration_frames, + "identity": event.identity, + } + for event in visible + ], + "eventCount": len(plan.events), + "truncated": len(plan.events) > len(visible), + } + ) + return jsonify( + sourceId=source.id, + recipeVersion=2, + frameRate=timeline.frame_rate_text, + totalFrames=timeline.total_frames, + durationSeconds=timeline.duration_seconds, + effects=effects_payload, + ) + except (ValidationError, ValueError) as exc: + return _validation_error(exc) + except (AssetNotFoundError, StorageUnavailableError) as exc: + return _storage_error(exc) + + @bp.post("/api/video-sources//jobs") def create_video_job(source_id: str) -> tuple[Response, int]: try: if not ffmpeg_available() or not ffprobe_available(): raise ExternalToolError("Video processing tools are unavailable.") payload = VideoJobRequest.model_validate(request.get_json(silent=True)) + source = _repository().get_video_source(source_id) + recipe = payload.recipe + if isinstance(payload.recipe, RecipeV2): + recipe = compile_recipe(payload.recipe, _source_timeline(source)).recipe job = _repository().create_video_job( source_id=source_id, - recipe=payload.recipe, + recipe=recipe, audio_mode=payload.audio_mode.value, ) try: diff --git a/pyproject.toml b/pyproject.toml index c4ef64f..13aaa32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "glitchcraft" -version = "0.2.1" +version = "0.3.0" description = "A local workspace for reproducible glitch and signal effects." readme = "README.md" requires-python = ">=3.11" diff --git a/static/app-manifest.json b/static/app-manifest.json index 03542ba..6edebd8 100644 --- a/static/app-manifest.json +++ b/static/app-manifest.json @@ -11,6 +11,10 @@ "bounded-video-jobs", "video-job-cancellation", "video-render-telemetry", + "temporal-effect-modulation", + "deterministic-effect-schedules", + "effect-envelopes", + "coherent-effect-variation", "timestamp-video-preview", "audio-preserving-video-export", "http-range-video-streaming" @@ -21,6 +25,7 @@ }, "descriptor": "Local visual-effects workspace", "endpoints": { + "effects": "/api/effects", "health": "/health", "metadata": "/metadata", "readiness": "/ready" @@ -29,5 +34,5 @@ "id": "glitchcraft", "name": "GlitchCraft", "schemaVersion": 1, - "version": "0.2.1" + "version": "0.3.0" } diff --git a/static/app.js b/static/app.js index eb7373c..3cef8a2 100644 --- a/static/app.js +++ b/static/app.js @@ -40,6 +40,14 @@ lastAnnouncedBucket: -1, lastAnnouncementAt: 0, videoPreviewUrl: null, + videoPreviewController: null, + videoPreviewRevision: 0, + scheduleController: null, + scheduleRevision: 0, + scheduleTimer: null, + effectMetadata: new Map(), + sporadicDefaults: null, + effectMetadataPromise: null, }; // Kept inspectable while this transitional interface remains in one script. @@ -49,7 +57,7 @@ const enabled = (name) => form.elements[name].checked; const currentMode = () => form.elements.mode.value; - function buildRecipe() { + function buildRecipeV1() { return { schemaVersion: 1, seed: imageState.seed, @@ -122,6 +130,298 @@ }; } + function temporalPanel(effectId) { + return document.querySelector( + `.temporal-control[data-effect-id="${CSS.escape(effectId)}"]`, + ); + } + + function temporalNumber(panel, role) { + const input = panel.querySelector(`[data-temporal-role="${role}"]`); + return Number(input.value); + } + + function buildEffectTiming(effectId) { + const panel = temporalPanel(effectId); + const mode = panel.querySelector('[data-temporal-role="mode"]').value; + const startSeconds = temporalNumber(panel, "start"); + const endInput = panel.querySelector('[data-temporal-role="end"]'); + const endSeconds = endInput.value === "" ? null : Number(endInput.value); + if (mode === "range") { + return {mode, startSeconds, endSeconds}; + } + if (mode === "sporadic") { + return { + mode, + startSeconds, + endSeconds, + frequencyPerMinute: temporalNumber(panel, "frequency"), + minimumDurationFrames: temporalNumber(panel, "minimum-duration"), + maximumDurationFrames: temporalNumber(panel, "maximum-duration"), + minimumCooldownFrames: temporalNumber(panel, "minimum-cooldown"), + maximumCooldownFrames: temporalNumber(panel, "maximum-cooldown"), + }; + } + return {mode: "continuous", startSeconds, endSeconds}; + } + + function buildEffectVariation(effectId) { + const panel = temporalPanel(effectId); + return Object.fromEntries( + [...panel.querySelectorAll("[data-variation-channel]")].map((select) => { + const mode = select.value; + const value = {mode}; + if (mode === "smooth") { + value.periodFrames = Number(select.dataset.periodFrames || 6); + } + return [select.dataset.variationChannel, value]; + }), + ); + } + + function buildRecipeV2() { + const legacy = buildRecipeV1(); + return { + schemaVersion: 2, + seed: legacy.seed, + effects: legacy.effects.map((effect) => { + const panel = temporalPanel(effect.id); + return { + ...effect, + intensity: temporalNumber(panel, "intensity"), + timing: buildEffectTiming(effect.id), + envelope: { + attackFrames: temporalNumber(panel, "attack"), + releaseFrames: temporalNumber(panel, "release"), + curve: panel.querySelector('[data-temporal-role="curve"]').value, + }, + variation: buildEffectVariation(effect.id), + }; + }), + }; + } + + function buildRecipe() { + return currentMode() === "video" ? buildRecipeV2() : buildRecipeV1(); + } + + function labeledInput(fieldset, effectId, labelText, role, options = {}) { + const wrapper = document.createElement("div"); + wrapper.className = "temporal-field"; + const id = `temporal-${effectId}-${role}`; + const label = document.createElement("label"); + label.htmlFor = id; + label.textContent = labelText; + const input = document.createElement("input"); + input.id = id; + input.type = options.type || "number"; + input.dataset.temporalRole = role; + for (const [name, value] of Object.entries(options)) { + if (name !== "type" && value !== null && value !== undefined) { + input[name] = value; + } + } + wrapper.append(label, input); + fieldset.append(wrapper); + return input; + } + + function labeledSelect(fieldset, effectId, labelText, role, values, selected) { + const wrapper = document.createElement("div"); + wrapper.className = "temporal-field"; + const id = `temporal-${effectId}-${role}`; + const label = document.createElement("label"); + label.htmlFor = id; + label.textContent = labelText; + const select = document.createElement("select"); + select.id = id; + select.dataset.temporalRole = role; + for (const [value, text] of values) { + const option = document.createElement("option"); + option.value = value; + option.textContent = text; + option.selected = value === selected; + select.append(option); + } + wrapper.append(label, select); + fieldset.append(wrapper); + return select; + } + + function installTimingPanel(card, metadata) { + const effectId = card.dataset.effectId; + const defaults = metadata.naturalVideoDefaults; + const timing = defaults.timing; + const envelope = defaults.envelope; + const sporadicDefaults = imageState.sporadicDefaults; + const details = document.createElement("details"); + details.className = "temporal-control"; + details.dataset.effectId = effectId; + const summary = document.createElement("summary"); + summary.textContent = "Advanced timing"; + const fieldset = document.createElement("fieldset"); + const legend = document.createElement("legend"); + legend.textContent = `${metadata.name} timing`; + fieldset.append(legend); + labeledSelect( + fieldset, + effectId, + "Timing mode", + "mode", + [ + ["continuous", "Continuous"], + ["range", "Selected range"], + ["sporadic", "Sporadic bursts"], + ], + timing.mode, + ); + const bounds = document.createElement("div"); + bounds.className = "temporal-grid"; + bounds.dataset.timingGroup = "bounds"; + fieldset.append(bounds); + labeledInput(bounds, effectId, "Start time (seconds)", "start", { + min: "0", + step: "0.01", + value: timing.startSeconds ?? 0, + }); + labeledInput(bounds, effectId, "End time (blank means source end)", "end", { + min: "0.01", + step: "0.01", + value: timing.endSeconds ?? "", + }); + const sporadic = document.createElement("div"); + sporadic.className = "temporal-grid"; + sporadic.dataset.timingGroup = "sporadic"; + fieldset.append(sporadic); + labeledInput(sporadic, effectId, "Events per minute", "frequency", { + min: "0.1", + max: "120", + step: "0.1", + value: timing.frequencyPerMinute ?? sporadicDefaults.frequencyPerMinute, + }); + labeledInput(sporadic, effectId, "Minimum duration (frames)", "minimum-duration", { + min: "1", + max: "3600", + step: "1", + value: timing.minimumDurationFrames ?? sporadicDefaults.minimumDurationFrames, + }); + labeledInput(sporadic, effectId, "Maximum duration (frames)", "maximum-duration", { + min: "1", + max: "3600", + step: "1", + value: timing.maximumDurationFrames ?? sporadicDefaults.maximumDurationFrames, + }); + labeledInput(sporadic, effectId, "Minimum cooldown (frames)", "minimum-cooldown", { + min: "0", + max: "36000", + step: "1", + value: timing.minimumCooldownFrames ?? sporadicDefaults.minimumCooldownFrames, + }); + labeledInput(sporadic, effectId, "Maximum cooldown (frames)", "maximum-cooldown", { + min: "0", + max: "36000", + step: "1", + value: timing.maximumCooldownFrames ?? sporadicDefaults.maximumCooldownFrames, + }); + const envelopeFields = document.createElement("div"); + envelopeFields.className = "temporal-grid"; + fieldset.append(envelopeFields); + labeledInput(envelopeFields, effectId, "Attack (frames)", "attack", { + min: "0", + max: "3600", + step: "1", + value: envelope.attackFrames, + }); + labeledInput(envelopeFields, effectId, "Release (frames)", "release", { + min: "0", + max: "3600", + step: "1", + value: envelope.releaseFrames, + }); + labeledInput(envelopeFields, effectId, "Maximum intensity", "intensity", { + min: "0", + max: "1", + step: "0.05", + value: "1", + }); + labeledSelect( + envelopeFields, + effectId, + "Envelope curve", + "curve", + [ + ["linear", "Linear"], + ["easeIn", "Ease in"], + ["easeOut", "Ease out"], + ["easeInOut", "Ease in/out"], + ["sharp", "Sharp"], + ], + envelope.curve, + ); + for (const [channelName, channel] of Object.entries(metadata.variationChannels)) { + const select = labeledSelect( + fieldset, + effectId, + `${channelName} variation`, + `variation-${channelName}`, + channel.supportedModes.map((mode) => [ + mode, + {perFrame: "Change every frame", perEvent: "Stable for burst", smooth: "Smooth drift"}[ + mode + ], + ]), + channel.default.mode, + ); + select.dataset.variationChannel = channelName; + select.dataset.periodFrames = String(channel.default.periodFrames || 6); + } + const help = document.createElement("p"); + help.className = "field-help"; + help.textContent = + "Durations and cooldowns use decoded frames. Schedules are seeded and repeat after restart."; + fieldset.append(help); + details.append(summary, fieldset); + card.append(details); + } + + async function loadEffectMetadata() { + const response = await fetch("/api/effects"); + if (!response.ok) { + throw new Error("Effect timing metadata could not be loaded."); + } + const payload = await response.json(); + imageState.sporadicDefaults = payload.sporadicControlDefaults; + for (const metadata of payload.effects) { + imageState.effectMetadata.set(metadata.type, metadata); + const card = document.querySelector( + `.control-card[data-effect-type="${CSS.escape(metadata.type)}"]`, + ); + if (card) { + installTimingPanel(card, metadata); + } + } + updateTemporalVisibility(); + } + + function updateTemporalVisibility() { + const effects = buildRecipeV1().effects; + for (const effect of effects) { + const panel = temporalPanel(effect.id); + if (!panel) { + continue; + } + panel.hidden = currentMode() !== "video" || !effect.enabled; + const mode = panel.querySelector('[data-temporal-role="mode"]').value; + const sporadic = panel.querySelector('[data-timing-group="sporadic"]'); + sporadic.hidden = mode !== "sporadic"; + for (const input of sporadic.querySelectorAll("input, select")) { + input.disabled = mode !== "sporadic"; + } + const end = panel.querySelector('[data-temporal-role="end"]'); + end.required = mode === "range"; + } + } + function setNotice(message, kind = "status") { imageState.error = kind === "error" ? message : null; notice.textContent = message; @@ -327,6 +627,7 @@ if (imageState.sourceId) { imageWorkspace.hidden = !imageMode; } + updateTemporalVisibility(); } function updateRangeOutputs() { @@ -347,6 +648,7 @@ const body = new FormData(); body.append("video", file); try { + await imageState.effectMetadataPromise; const response = await fetch("/api/video-sources", {method: "POST", body}); if (!response.ok) { throw new Error(await errorMessage(response, "The video could not be uploaded.")); @@ -361,6 +663,7 @@ const timeInput = document.querySelector("#video-preview-time"); timeInput.max = String(Math.max(0, source.durationSeconds - 0.001)); document.querySelector("#preview-section").hidden = false; + await requestEffectSchedule(); await requestVideoPreview(); setNotice(""); } catch (error) { @@ -376,6 +679,10 @@ if (!source) { return; } + imageState.videoPreviewController?.abort(); + const controller = new AbortController(); + const revision = ++imageState.videoPreviewRevision; + imageState.videoPreviewController = controller; try { const response = await fetch( `/api/video-sources/${encodeURIComponent(source.sourceId)}/preview`, @@ -386,27 +693,144 @@ recipe: buildRecipe(), timestampSeconds: Number(document.querySelector("#video-preview-time").value), }), + signal: controller.signal, }, ); if (!response.ok) { throw new Error(await errorMessage(response, "Video preview failed.")); } const blob = await response.blob(); + if (revision !== imageState.videoPreviewRevision) { + return; + } releaseVideoPreview(); imageState.videoPreviewUrl = URL.createObjectURL(blob); document.querySelector("#preview-image").src = imageState.videoPreviewUrl; + const activeCount = Number(response.headers.get("X-GlitchCraft-Active-Effect-Count") || 0); + const activeIds = (response.headers.get("X-GlitchCraft-Active-Effect-Ids") || "") + .split(",") + .filter(Boolean); + document.querySelector("#preview-active-state").textContent = activeCount + ? `${activeCount} scheduled ${activeCount === 1 ? "effect" : "effects"} active: ${activeIds + .map((id) => effectNames[buildRecipeV1().effects.find((effect) => effect.id === id)?.type] || id) + .join(", ")}.` + : "No scheduled effects are active at this preview time."; } catch (error) { - setNotice(error.message, "error"); + if (error.name !== "AbortError" && revision === imageState.videoPreviewRevision) { + setNotice(error.message, "error"); + } + } finally { + if (imageState.videoPreviewController === controller) { + imageState.videoPreviewController = null; + } } } function releaseVideoPreview() { + imageState.videoPreviewController?.abort(); + imageState.videoPreviewController = null; if (imageState.videoPreviewUrl) { URL.revokeObjectURL(imageState.videoPreviewUrl); imageState.videoPreviewUrl = null; } } + function formatScheduleTime(seconds) { + const whole = Math.max(0, Math.floor(seconds)); + const minutes = Math.floor(whole / 60); + return `${String(minutes).padStart(2, "0")}:${String(whole % 60).padStart(2, "0")}`; + } + + function renderEffectSchedule(schedule) { + const enabledIds = new Set( + buildRecipeV1() + .effects.filter((effect) => effect.enabled) + .map((effect) => effect.id), + ); + const effects = schedule.effects.filter((effect) => enabledIds.has(effect.effectId)); + const summaries = effects.map((effect) => { + const name = effectNames[effect.type] || effect.type; + if (effect.mode === "continuous" || effect.mode === "range") { + const first = effect.events[0]; + return first + ? `${name}: continuous ${formatScheduleTime(first.startSeconds)}–${formatScheduleTime( + first.endSeconds, + )}` + : `${name}: inactive in the selected range`; + } + const first = effect.events[0]; + return `${name}: ${effect.eventCount} ${effect.eventCount === 1 ? "burst" : "bursts"}${ + first ? `, first at ${formatScheduleTime(first.startSeconds)}` : "" + }`; + }); + document.querySelector("#schedule-status").textContent = + summaries.join(" · ") || "No enabled effects are scheduled."; + const eventList = document.querySelector("#schedule-events"); + eventList.replaceChildren(); + for (const effect of effects) { + const item = document.createElement("li"); + const name = effectNames[effect.type] || effect.type; + const times = effect.events + .slice(0, 12) + .map( + (event) => + `${formatScheduleTime(event.startSeconds)} (${event.durationFrames} frames)`, + ); + item.textContent = `${name}: ${times.join(", ") || "no events"}`; + if (effect.truncated) { + item.textContent += " (additional events omitted)"; + } + eventList.append(item); + } + } + + async function requestEffectSchedule() { + const source = imageState.videoSource; + if (!source || currentMode() !== "video") { + return; + } + imageState.scheduleController?.abort(); + const controller = new AbortController(); + const revision = ++imageState.scheduleRevision; + imageState.scheduleController = controller; + document.querySelector("#schedule-status").textContent = "Compiling deterministic schedule…"; + try { + const response = await fetch( + `/api/video-sources/${encodeURIComponent(source.sourceId)}/effect-schedule`, + { + method: "POST", + headers: {"Content-Type": "application/json"}, + body: JSON.stringify({recipe: buildRecipeV2()}), + signal: controller.signal, + }, + ); + if (!response.ok) { + throw new Error(await errorMessage(response, "The effect schedule is invalid.")); + } + const schedule = await response.json(); + if (revision === imageState.scheduleRevision) { + renderEffectSchedule(schedule); + } + } catch (error) { + if (error.name !== "AbortError" && revision === imageState.scheduleRevision) { + document.querySelector("#schedule-status").textContent = error.message; + } + } finally { + if (imageState.scheduleController === controller) { + imageState.scheduleController = null; + } + } + } + + function scheduleTemporalRefresh() { + updateTemporalVisibility(); + schedulePreview(); + window.clearTimeout(imageState.scheduleTimer); + if (imageState.videoSource && currentMode() === "video") { + imageState.scheduleTimer = window.setTimeout(requestEffectSchedule, 225); + } + } + const effectNames = { noise: "Noise", pixelation: "Pixelation", @@ -796,14 +1220,14 @@ updateRangeOutputs(); } if (event.target.closest("#effect-controls")) { - schedulePreview(); + scheduleTemporalRefresh(); } }); form.addEventListener("change", (event) => { if (event.target.name === "mode") { updateMode(); } else if (event.target.closest("#effect-controls")) { - schedulePreview(); + scheduleTemporalRefresh(); } }); @@ -813,6 +1237,10 @@ toggleSubControls("scan_lines", "#scan_lines-params"); toggleSubControls("static", "#static-params"); toggleSubControls("flicker", "#flicker-params"); + imageState.effectMetadataPromise = loadEffectMetadata().catch((error) => { + setNotice(error.message, "error"); + throw error; + }); updateRangeOutputs(); updateMode(); @@ -864,5 +1292,7 @@ window.addEventListener("pagehide", () => { stopVideoPolling(); releaseVideoPreview(); + imageState.scheduleController?.abort(); + window.clearTimeout(imageState.scheduleTimer); }); })(); diff --git a/static/style.css b/static/style.css index 88994cd..253dd98 100644 --- a/static/style.css +++ b/static/style.css @@ -480,6 +480,63 @@ progress { padding-left: 1.25rem; } +.temporal-control { + margin-top: var(--space-3); + border-top: 1px solid var(--border); + padding-top: var(--space-3); +} + +.temporal-control summary, +.schedule-summary summary { + width: fit-content; + cursor: pointer; + font-weight: 700; +} + +.temporal-control fieldset { + margin-top: var(--space-3); + border: 0; + padding: 0; +} + +.temporal-control legend { + margin-bottom: var(--space-2); + font-weight: 700; +} + +.temporal-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: var(--space-2); + margin-top: var(--space-2); +} + +.temporal-field { + display: grid; + gap: 0.3rem; + margin-top: var(--space-2); +} + +.temporal-field input, +.temporal-field select { + width: 100%; + min-width: 0; +} + +.schedule-summary { + margin: var(--space-3) 0; + border: 1px solid var(--border); + border-radius: var(--radius); + padding: var(--space-3); + background: var(--surface-raised); +} + +.schedule-events { + max-height: 12rem; + overflow: auto; + padding-left: 1.25rem; +} + .visually-hidden { position: absolute; width: 1px; @@ -521,6 +578,10 @@ progress { .action { flex: 1 1 auto; } + + .temporal-grid { + grid-template-columns: 1fr; + } } @media (max-width: 380px) { diff --git a/templates/index.html b/templates/index.html index 16051c7..ab79fbe 100644 --- a/templates/index.html +++ b/templates/index.html @@ -62,7 +62,7 @@

Choose media and treatment

Effect controls
-
+
10% @@ -81,7 +81,7 @@

Choose media and treatment

-
+
1 px @@ -90,7 +90,7 @@

Choose media and treatment

A value of 1 leaves the image unpixelated.

-
+
-
+
-
+
-
+
-
+
-
+
+
+
+
+

Deterministic schedule

+

Effect timing

+
+

+ Timing will appear after the source is ready. +

+
+

+
+ View scheduled events +
    +
    +
    Processed preview frame from the uploaded video
    diff --git a/tests/browser/image-workflow.spec.js b/tests/browser/image-workflow.spec.js index 1d08c9b..e885ba2 100644 --- a/tests/browser/image-workflow.spec.js +++ b/tests/browser/image-workflow.spec.js @@ -31,6 +31,7 @@ test("uploads once and shows original and processed images without navigation", }) => { const startUrl = page.url(); await uploadImage(page); + expect(await page.evaluate(() => window.__glitchcraftState.recipe.schemaVersion)).toBe(1); await expect(page).toHaveURL(startUrl); await expect(page.getByRole("heading", {name: "Original", exact: true})).toBeVisible(); await expect(page.getByRole("heading", {name: "Processed", exact: true})).toBeVisible(); @@ -148,6 +149,8 @@ test("persistent video UI uploads once, previews a timestamp, and polls its job" page, }) => { let previewTimestamp = null; + let previewRecipe = null; + let scheduleRecipe = null; let polls = 0; let submissions = 0; await page.route("**/api/video-sources", async (route) => { @@ -168,14 +171,74 @@ test("persistent video UI uploads once, previews a timestamp, and polls its job" }); }); await page.route("**/api/video-sources/*/preview", async (route) => { - previewTimestamp = route.request().postDataJSON().timestampSeconds; - await route.fulfill({status: 200, contentType: "image/png", body: png}); + const payload = route.request().postDataJSON(); + previewTimestamp = payload.timestampSeconds; + previewRecipe = payload.recipe; + await route.fulfill({ + status: 200, + contentType: "image/png", + headers: { + "X-GlitchCraft-Active-Effect-Count": "1", + "X-GlitchCraft-Active-Effect-Ids": "legacy-noise", + }, + body: png, + }); + }); + await page.route("**/api/video-sources/*/effect-schedule", async (route) => { + scheduleRecipe = route.request().postDataJSON().recipe; + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + frameRate: "30/1", + totalFrames: 120, + durationSeconds: 4, + effects: [ + { + effectId: "legacy-noise", + type: "noise", + mode: "continuous", + eventCount: 1, + truncated: false, + events: [ + { + eventIndex: 0, + startFrame: 0, + endFrame: 120, + startSeconds: 0, + endSeconds: 4, + durationFrames: 120, + }, + ], + }, + { + effectId: "legacy-horizontal-glitch", + type: "horizontal_glitch", + mode: "sporadic", + eventCount: 2, + truncated: false, + events: [ + { + eventIndex: 0, + startFrame: 30, + endFrame: 42, + startSeconds: 1, + endSeconds: 1.4, + durationFrames: 12, + }, + ], + }, + ], + }), + }); }); await page.route("**/api/video-sources/*/jobs", async (route) => { submissions += 1; const payload = route.request().postDataJSON(); expect(payload.audioMode).toBe("preserve"); expect(payload.recipe.seed).toBe(17); + expect(payload.recipe.schemaVersion).toBe(2); + expect(payload.recipe.effects[2].timing.mode).toBe("sporadic"); await route.fulfill({ status: 202, contentType: "application/json", @@ -338,11 +401,39 @@ test("persistent video UI uploads once, previews a timestamp, and polls its job" await page.getByRole("button", {name: "Create video preview"}).click(); await expect(page.locator("#preview-section")).toBeVisible(); await expect(page.locator("#preview-image")).toHaveAttribute("src", /^blob:/); + expect(previewRecipe.schemaVersion).toBe(2); + expect(scheduleRecipe.schemaVersion).toBe(2); + await expect(page.locator("#preview-active-state")).toContainText("1 scheduled effect active"); + await expect(page.locator("#schedule-status")).toContainText("Noise: continuous"); await expect(page.locator("#video-source-metadata")).toContainText("640 × 360"); await page.locator("#video-preview-time").fill("1.5"); await expect.poll(() => previewTimestamp).toBe(1.5); await page.locator("#glitch").check(); + const glitchTiming = page.locator( + '.temporal-control[data-effect-id="legacy-horizontal-glitch"]', + ); + await expect(glitchTiming).toBeVisible(); + await glitchTiming.locator("summary").click(); + await expect(glitchTiming.locator('[data-temporal-role="mode"]')).toHaveValue("sporadic"); + await expect(glitchTiming.locator('[data-temporal-role="minimum-duration"]')).toHaveValue("6"); + await expect(glitchTiming.locator('[data-temporal-role="release"]')).toHaveValue("5"); + await expect(glitchTiming.locator('[data-variation-channel="layout"]')).toHaveValue( + "perEvent", + ); + await expect(glitchTiming.locator('[data-variation-channel="offset"]')).toHaveValue( + "smooth", + ); + await page.setViewportSize({width: 390, height: 844}); + await expect(glitchTiming).toBeInViewport(); + const accessibility = await new AxeBuilder({page}) + .withTags(["wcag2a", "wcag2aa"]) + .analyze(); + expect( + accessibility.violations.filter((violation) => + ["serious", "critical"].includes(violation.impact), + ), + ).toEqual([]); await page.locator("#scan_lines").check(); await page.getByRole("button", {name: "Process full video"}).click(); await expect(page.locator("#progress-indicator")).toBeVisible(); diff --git a/tests/test_media.py b/tests/test_media.py index b070cbc..086633c 100644 --- a/tests/test_media.py +++ b/tests/test_media.py @@ -6,12 +6,12 @@ import numpy as np import pytest -from glitchcraft.contracts.effects import Recipe +from glitchcraft.contracts.effects import Recipe, RecipeV2 from glitchcraft.errors import ExternalToolError, MediaReadError, MediaWriteError from glitchcraft.media.color import bgr_to_rgb, rgb_to_bgr, validate_rgb from glitchcraft.media.ffmpeg import reencode_for_browser from glitchcraft.media.image_io import load_image_rgb, save_image_rgb -from glitchcraft.media.video import create_video_preview, process_video +from glitchcraft.media.video import create_video_preview, create_video_preview_at, process_video def test_rgb_bgr_round_trip_with_distinct_channels() -> None: @@ -71,6 +71,7 @@ def __init__( self.width = width self.height = height self.released = False + self.position = 0.0 def isOpened(self) -> bool: return self.opened @@ -89,6 +90,10 @@ def get(self, property_id: int) -> float: } return values[property_id] + def set(self, _property_id: int, value: float) -> bool: + self.position = value + return True + def release(self) -> None: self.released = True @@ -139,6 +144,47 @@ def test_preview_and_full_video_share_frame_zero(tmp_path: Path, monkeypatch) -> assert writer.released and full_capture.released +def test_recipe_v2_preview_matches_preencode_processing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + bgr = np.full((3, 4, 3), [200, 90, 20], dtype=np.uint8) + recipe = RecipeV2.model_validate( + { + "schemaVersion": 2, + "seed": 5, + "effects": [ + { + "id": "scan", + "type": "scan_lines", + "parameters": {"gap": 2, "darkness": 0.5}, + "timing": {"mode": "continuous"}, + } + ], + } + ) + activations: list[tuple[tuple[str, float], ...]] = [] + preview_capture = FakeCapture([bgr], fps=24) + monkeypatch.setattr("glitchcraft.media.video.cv2.VideoCapture", lambda _path: preview_capture) + preview = tmp_path / "preview-v2.png" + frame_index = create_video_preview_at( + tmp_path / "source.mp4", + preview, + recipe, + timestamp_seconds=0, + activation_hook=lambda _frame, active: activations.append(active), + ) + assert frame_index == 0 + assert activations == [(("scan", 1.0),)] + + full_capture = FakeCapture([bgr], fps=24) + writer = FakeWriter() + monkeypatch.setattr("glitchcraft.media.video.cv2.VideoCapture", lambda _path: full_capture) + monkeypatch.setattr("glitchcraft.media.video.cv2.VideoWriter", lambda *_args: writer) + monkeypatch.setattr("glitchcraft.media.video.cv2.VideoWriter_fourcc", lambda *_args: 1) + process_video(tmp_path / "source.mp4", tmp_path / "out.mp4", recipe) + assert np.array_equal(load_image_rgb(preview), bgr_to_rgb(writer.frames[0])) + + def test_video_read_and_write_failures_are_typed(tmp_path: Path, monkeypatch) -> None: recipe = Recipe(seed=1) monkeypatch.setattr( diff --git a/tests/test_service_contract.py b/tests/test_service_contract.py index 22f9954..1159985 100644 --- a/tests/test_service_contract.py +++ b/tests/test_service_contract.py @@ -63,7 +63,8 @@ def test_metadata_is_request_derived_redacted_and_truthful(client, app) -> None: "localOnly": True, } assert payload["storageSchemaVersion"] == 2 - assert payload["recipeSchemaVersion"] == 1 + assert payload["recipeSchemaVersion"] == 2 + assert payload["supportedRecipeSchemaVersions"] == [1, 2] assert payload["videoJobSchemaVersion"] == 2 assert payload["videoJobs"]["pollingModel"] == "client-polling" assert payload["videoJobs"]["runtimeUpdateFrequencyHz"] == 4 diff --git a/tests/test_temporal_effects.py b/tests/test_temporal_effects.py new file mode 100644 index 0000000..24f2412 --- /dev/null +++ b/tests/test_temporal_effects.py @@ -0,0 +1,671 @@ +from __future__ import annotations + +import hashlib +from fractions import Fraction +from itertools import pairwise + +import numpy as np +import pytest +from pydantic import ValidationError + +from glitchcraft.contracts.effects import ( + ColorBleedParameters, + EffectEnvelope, + EffectInstance, + EffectInstanceV2, + EffectType, + FlickerParameters, + FrameShiftParameters, + HorizontalGlitchParameters, + NoiseParameters, + PixelationParameters, + Recipe, + RecipeV2, + ScanLinesParameters, + StaticParameters, + parse_recipe, +) +from glitchcraft.effects.engine import apply_effect_stack +from glitchcraft.effects.execution import apply_effect_v2, scale_parameters +from glitchcraft.effects.registry import EFFECT_REGISTRY, apply_recipe_v2_defaults +from glitchcraft.effects.temporal import ( + EffectActivation, + EffectExecutionContext, + MediaTimelineContext, + compile_recipe, + envelope_value, +) + + +def timeline( + *, + rate: str = "30/1", + duration: float = 10, + total: int | None = 300, +) -> MediaTimelineContext: + return MediaTimelineContext.from_source( + frame_rate=rate, + duration_seconds=duration, + total_frames=total, + ) + + +def recipe_v2(effect: dict[str, object], *, seed: int = 42) -> RecipeV2: + return RecipeV2.model_validate({"schemaVersion": 2, "seed": seed, "effects": [effect]}) + + +def test_recipe_v1_golden_frames_remain_byte_exact() -> None: + frame = np.arange(8 * 9 * 3, dtype=np.uint8).reshape(8, 9, 3) + recipe = Recipe( + seed=123456, + effects=[ + EffectInstance( + id="n", + type="noise", + parameters={"amount": 25, "strength": 17, "monochromatic": False}, + ), + EffectInstance(id="p", type="pixelation", parameters={"pixel_size": 3}), + EffectInstance(id="h", type="horizontal_glitch", parameters={"count": 4, "shift": 5}), + EffectInstance(id="s", type="static", parameters={"intensity": 0.08}), + EffectInstance(id="f", type="flicker", parameters={"minimum": 0.8, "maximum": 1.2}), + ], + ) + expected = { + 0: "5023881898e3132d1eee1dc8e404c94ae579489d29613f37f78b5507f5756c76", + 1: "4549ef30c98b1ac4b00134bdb39fd49717867d945aea1b1874870d2fe10da987", + 17: "6442e5f49e903fdaa80f4001936b049610e837a932eaf5ca15888b2aa013a040", + } + for frame_index, digest in expected.items(): + actual = apply_effect_stack(frame, recipe, frame_index) + assert hashlib.sha256(actual.tobytes()).hexdigest() == digest + + +def test_recipe_documents_are_strict_versioned_and_canonical() -> None: + legacy = parse_recipe({"schemaVersion": 1, "seed": 1, "effects": []}) + modern = parse_recipe({"schemaVersion": 2, "seed": 1, "effects": []}) + assert isinstance(legacy, Recipe) + assert isinstance(modern, RecipeV2) + with pytest.raises(ValidationError): + parse_recipe( + { + "schemaVersion": 1, + "seed": 1, + "effects": [ + { + "id": "legacy", + "type": "noise", + "parameters": {}, + "timing": {"mode": "continuous"}, + } + ], + } + ) + with pytest.raises(ValidationError): + parse_recipe({"schemaVersion": 3, "seed": 1, "effects": []}) + with pytest.raises(ValidationError): + recipe_v2( + { + "id": "x", + "type": "noise", + "parameters": {}, + "intensity": 1.01, + } + ) + + +@pytest.mark.parametrize( + "timing", + [ + {"mode": "continuous", "startSeconds": 2, "endSeconds": 1}, + {"mode": "range", "startSeconds": 1, "endSeconds": 1}, + { + "mode": "sporadic", + "frequencyPerMinute": 0, + "minimumDurationFrames": 4, + "maximumDurationFrames": 3, + "minimumCooldownFrames": 5, + "maximumCooldownFrames": 4, + }, + {"mode": "events", "events": []}, + {"mode": "continuous", "frequencyPerMinute": 2}, + ], +) +def test_timing_modes_reject_invalid_or_unused_fields(timing: dict[str, object]) -> None: + with pytest.raises(ValidationError): + recipe_v2({"id": "x", "type": "noise", "parameters": {}, "timing": timing}) + + +def test_variation_channels_and_envelopes_are_strict() -> None: + with pytest.raises(ValidationError, match="unknown variation"): + recipe_v2( + { + "id": "x", + "type": "noise", + "parameters": {}, + "variation": {"offset": {"mode": "perFrame"}}, + } + ) + with pytest.raises(ValidationError, match="not supported"): + recipe_v2( + { + "id": "x", + "type": "noise", + "parameters": {}, + "variation": {"detail": {"mode": "smooth", "periodFrames": 4}}, + } + ) + with pytest.raises(ValidationError): + recipe_v2( + { + "id": "x", + "type": "horizontal_glitch", + "parameters": {}, + "timing": { + "mode": "sporadic", + "minimumDurationFrames": 3, + "maximumDurationFrames": 4, + }, + "envelope": {"attackFrames": 2, "releaseFrames": 2, "curve": "linear"}, + } + ) + with pytest.raises(ValidationError): + recipe_v2( + { + "id": "x", + "type": "horizontal_glitch", + "parameters": {}, + "variation": {"offset": {"mode": "smooth", "periodFrames": 1}}, + } + ) + + +@pytest.mark.parametrize( + "timing", + [ + {"mode": "continuous", "startSeconds": float("inf")}, + {"mode": "continuous", "endSeconds": float("inf")}, + {"mode": "continuous", "endSeconds": 0}, + {"mode": "range", "startSeconds": float("inf"), "endSeconds": float("inf")}, + {"mode": "sporadic", "startSeconds": float("inf")}, + {"mode": "sporadic", "endSeconds": float("inf")}, + {"mode": "sporadic", "startSeconds": 2, "endSeconds": 1}, + { + "mode": "sporadic", + "minimumDurationFrames": 10, + "maximumDurationFrames": 9, + }, + { + "mode": "sporadic", + "minimumCooldownFrames": 10, + "maximumCooldownFrames": 9, + }, + { + "mode": "events", + "events": [{"startSeconds": float("inf"), "durationFrames": 2}], + }, + ], +) +def test_temporal_numbers_must_be_finite_and_ordered(timing: dict[str, object]) -> None: + with pytest.raises(ValidationError): + recipe_v2({"id": "x", "type": "noise", "parameters": {}, "timing": timing}) + + +def test_recipe_v2_effect_ids_are_unique() -> None: + with pytest.raises(ValidationError, match="unique"): + RecipeV2( + seed=1, + effects=[ + EffectInstanceV2(id="same", type="noise"), + EffectInstanceV2(id="same", type="static"), + ], + ) + + +def test_registry_defaults_are_natural_and_overrideable() -> None: + raw = recipe_v2( + {"id": "glitch", "type": "horizontal_glitch", "parameters": {"count": 6, "shift": 80}} + ) + canonical = apply_recipe_v2_defaults(raw) + effect = canonical.effects[0] + assert effect.timing.mode == "sporadic" + assert effect.variation["layout"].mode == "perEvent" + assert effect.variation["offset"].mode == "smooth" + assert all(definition.intensity_scaling for definition in EFFECT_REGISTRY.values()) + + explicit = recipe_v2( + { + "id": "glitch", + "type": "horizontal_glitch", + "parameters": {}, + "timing": {"mode": "continuous", "startSeconds": 1}, + "variation": {"offset": {"mode": "perFrame"}}, + } + ) + configured = apply_recipe_v2_defaults(explicit).effects[0] + assert configured.timing.mode == "continuous" + assert configured.variation["offset"].mode == "perFrame" + assert configured.variation["layout"].mode == "perEvent" + + +def test_timeline_uses_rational_floor_start_and_ceil_end() -> None: + value = timeline(rate="30000/1001", duration=10.01, total=None) + assert value.frame_rate == Fraction(30000, 1001) + assert value.frame_rate_text == "30000/1001" + assert value.start_frame(1.0011) == 30 + assert value.end_frame(1.0011) == 31 + assert value.end_frame(None) == value.total_frames + assert value.frame_seconds(30) == pytest.approx(1.001) + with pytest.raises(ValueError): + MediaTimelineContext.from_source(frame_rate=0, duration_seconds=1, total_frames=1) + with pytest.raises(ValueError): + MediaTimelineContext.from_source(frame_rate=30, duration_seconds=1, total_frames=0) + + +def test_continuous_range_and_manual_schedules_are_canonical() -> None: + continuous = recipe_v2( + { + "id": "n", + "type": "noise", + "parameters": {}, + "timing": {"mode": "continuous", "startSeconds": 1, "endSeconds": 2}, + } + ) + event = compile_recipe(continuous, timeline()).effects[0].events[0] + assert (event.start_frame, event.end_frame) == (30, 60) + + ranged = recipe_v2( + { + "id": "n", + "type": "noise", + "parameters": {}, + "timing": {"mode": "range", "startSeconds": 0.05, "endSeconds": 0.11}, + } + ) + event = compile_recipe(ranged, timeline()).effects[0].events[0] + assert (event.start_frame, event.end_frame) == (1, 4) + + manual = recipe_v2( + { + "id": "n", + "type": "noise", + "parameters": {}, + "timing": { + "mode": "events", + "events": [ + {"startSeconds": 2, "durationFrames": 3}, + {"startSeconds": 1, "durationFrames": 2}, + ], + }, + } + ) + events = compile_recipe(manual, timeline()).effects[0].events + assert [event.start_frame for event in events] == [30, 60] + assert all(event.start_frame <= event.end_frame for event in events) + + +def test_manual_event_overlap_and_outside_source_are_rejected() -> None: + overlapping = recipe_v2( + { + "id": "n", + "type": "noise", + "parameters": {}, + "timing": { + "mode": "events", + "events": [ + {"startSeconds": 1, "durationFrames": 10}, + {"startSeconds": 1.1, "durationFrames": 10}, + ], + }, + } + ) + with pytest.raises(ValueError, match="overlap"): + compile_recipe(overlapping, timeline()) + outside = recipe_v2( + { + "id": "n", + "type": "noise", + "parameters": {}, + "timing": { + "mode": "events", + "events": [{"startSeconds": 20, "durationFrames": 2}], + }, + } + ) + with pytest.raises(ValueError, match="outside"): + compile_recipe(outside, timeline()) + clipped_envelope = recipe_v2( + { + "id": "n", + "type": "noise", + "parameters": {}, + "timing": { + "mode": "events", + "events": [{"startSeconds": 9.9, "durationFrames": 10}], + }, + "envelope": {"attackFrames": 4, "releaseFrames": 4, "curve": "linear"}, + } + ) + with pytest.raises(ValueError, match="clipped"): + compile_recipe(clipped_envelope, timeline()) + + +def test_sporadic_schedule_is_bounded_deterministic_and_isolated() -> None: + first = recipe_v2( + { + "id": "glitch", + "type": "horizontal_glitch", + "parameters": {"count": 6, "shift": 80}, + }, + seed=99, + ) + plan_a = compile_recipe(first, timeline(duration=120, total=3600)).effects[0] + plan_b = compile_recipe(first, timeline(duration=120, total=3600)).effects[0] + assert plan_a.events == plan_b.events + restarted = RecipeV2.model_validate( + compile_recipe(first, timeline(duration=120, total=3600)).recipe.model_dump( + by_alias=True, mode="json" + ) + ) + assert ( + compile_recipe(restarted, timeline(duration=120, total=3600)).effects[0].events + == plan_a.events + ) + assert plan_a.events + assert all(left.end_frame <= right.start_frame for left, right in pairwise(plan_a.events)) + + changed = recipe_v2( + { + "id": "glitch", + "type": "horizontal_glitch", + "parameters": {"count": 7, "shift": 80}, + }, + seed=99, + ) + assert ( + compile_recipe(changed, timeline(duration=120, total=3600)).effects[0].events + == plan_a.events + ) + other_seed = recipe_v2( + {"id": "glitch", "type": "horizontal_glitch", "parameters": {}}, seed=100 + ) + assert ( + compile_recipe(other_seed, timeline(duration=120, total=3600)).effects[0].events + != plan_a.events + ) + invalid_window = recipe_v2( + { + "id": "glitch", + "type": "horizontal_glitch", + "parameters": {}, + "timing": { + "mode": "sporadic", + "startSeconds": 20, + "endSeconds": 21, + }, + } + ) + with pytest.raises(ValueError, match="outside"): + compile_recipe(invalid_window, timeline()) + partial = recipe_v2( + { + "id": "short", + "type": "noise", + "parameters": {}, + "timing": { + "mode": "sporadic", + "frequencyPerMinute": 120, + "minimumDurationFrames": 3, + "maximumDurationFrames": 3, + "minimumCooldownFrames": 0, + "maximumCooldownFrames": 0, + }, + "envelope": {"attackFrames": 0, "releaseFrames": 0, "curve": "linear"}, + } + ) + assert not compile_recipe(partial, timeline(rate="1/1", duration=2, total=2)).effects[0].events + + +def test_schedule_event_count_has_a_hard_limit() -> None: + recipe = recipe_v2( + { + "id": "dense", + "type": "noise", + "parameters": {}, + "timing": { + "mode": "sporadic", + "frequencyPerMinute": 120, + "minimumDurationFrames": 1, + "maximumDurationFrames": 1, + "minimumCooldownFrames": 0, + "maximumCooldownFrames": 0, + }, + } + ) + with pytest.raises(ValueError, match="safe event limit"): + compile_recipe( + recipe, + timeline(rate="240/1", duration=24 * 60 * 60, total=20_736_000), + ) + + +@pytest.mark.parametrize("curve", ["linear", "easeIn", "easeOut", "easeInOut", "sharp"]) +def test_envelope_curves_have_explicit_attack_hold_release(curve: str) -> None: + envelope = EffectEnvelope(attack_frames=2, release_frames=2, curve=curve) + values = [envelope_value(index, 6, envelope) for index in range(6)] + assert all(0 <= value <= 1 for value in values) + assert values[0] < values[1] == values[2] == values[3] == 1 + assert values[4] > values[5] == 0 + assert envelope_value(-1, 6, envelope) == 0 + assert envelope_value(6, 6, envelope) == 0 + assert envelope_value(0, 1, EffectEnvelope()) == 1 + + +def test_activation_lookup_is_random_access_and_exclusive_end() -> None: + recipe = recipe_v2( + { + "id": "n", + "type": "noise", + "parameters": {}, + "timing": {"mode": "range", "startSeconds": 1, "endSeconds": 2}, + "envelope": {"attackFrames": 2, "releaseFrames": 2, "curve": "linear"}, + } + ) + compiled = compile_recipe(recipe, timeline()) + plan = compiled.effects[0] + assert not plan.activation_at(29).active + assert plan.activation_at(30).effective_intensity == 0.5 + assert plan.activation_at(31).effective_intensity == 1 + assert plan.activation_at(58).effective_intensity == 0.5 + assert not plan.activation_at(59).active + assert not plan.activation_at(60).active + assert compiled.activation_at("n", 31).active + with pytest.raises(KeyError): + compiled.activation_at("missing", 0) + too_short = recipe_v2( + { + "id": "n", + "type": "noise", + "parameters": {}, + "timing": {"mode": "range", "startSeconds": 0, "endSeconds": 0.1}, + "envelope": {"attackFrames": 2, "releaseFrames": 2, "curve": "linear"}, + } + ) + with pytest.raises(ValueError, match="envelope"): + compile_recipe(too_short, timeline()) + + +def test_variation_modes_are_order_independent_and_smooth() -> None: + recipe = recipe_v2( + { + "id": "glitch", + "type": "horizontal_glitch", + "parameters": {}, + "timing": { + "mode": "events", + "events": [ + {"startSeconds": 0, "durationFrames": 12}, + {"startSeconds": 1, "durationFrames": 12}, + ], + }, + "variation": { + "layout": {"mode": "perEvent"}, + "offset": {"mode": "smooth", "periodFrames": 4}, + }, + } + ) + compiled = compile_recipe(recipe, timeline()) + plan = compiled.effects[0] + first = plan.activation_at(2) + context = EffectExecutionContext(recipe.seed, 2, plan.effect, first) + same_event = EffectExecutionContext(recipe.seed, 5, plan.effect, plan.activation_at(5)) + assert context.rng("layout").random() == same_event.rng("layout").random() + values = [ + EffectExecutionContext(recipe.seed, frame, plan.effect, plan.activation_at(frame)).sample( + "offset" + ) + for frame in (0, 1, 2, 3, 4) + ] + assert values == [ + EffectExecutionContext(recipe.seed, frame, plan.effect, plan.activation_at(frame)).sample( + "offset" + ) + for frame in (0, 1, 2, 3, 4) + ] + assert max(abs(right - left) for left, right in pairwise(values)) < 1 + second_context = EffectExecutionContext(recipe.seed, 31, plan.effect, plan.activation_at(31)) + assert context.rng("layout").random() != second_context.rng("layout").random() + with pytest.raises(KeyError, match="missing"): + context.sample("unknown") + + +@pytest.mark.parametrize( + ("parameters", "inactive_field", "full_field"), + [ + (NoiseParameters(amount=20, strength=30), ("amount", 0), ("strength", 30)), + (PixelationParameters(pixel_size=9), ("pixel_size", 1), ("pixel_size", 9)), + (HorizontalGlitchParameters(count=6, shift=80), ("count", 0), ("shift", 80)), + (FrameShiftParameters(max_x=10, max_y=20), ("max_x", 0), ("max_y", 20)), + (ColorBleedParameters(max_shift=8), ("max_shift", 0), ("max_shift", 8)), + (ScanLinesParameters(gap=4, darkness=0.4), ("darkness", 1), ("darkness", 0.4)), + (StaticParameters(intensity=0.4), ("intensity", 0), ("intensity", 0.4)), + (FlickerParameters(minimum=0.5, maximum=1.5), ("minimum", 1), ("maximum", 1.5)), + ], +) +def test_parameter_scaling_has_identity_and_full_strength( + parameters: object, + inactive_field: tuple[str, float], + full_field: tuple[str, float], +) -> None: + inactive = scale_parameters(parameters, 0) # type: ignore[arg-type] + full = scale_parameters(parameters, 1) # type: ignore[arg-type] + midpoint = scale_parameters(parameters, 0.5) # type: ignore[arg-type] + assert getattr(inactive, inactive_field[0]) == inactive_field[1] + assert getattr(full, full_field[0]) == full_field[1] + assert midpoint.model_dump() != {} + with pytest.raises(ValidationError): + type(midpoint).model_validate({**midpoint.model_dump(), "unexpected": 1}) + + +def test_scaling_rejects_unknown_models() -> None: + with pytest.raises(TypeError, match="unsupported"): + scale_parameters(object(), 0.5) # type: ignore[arg-type] + + +def test_temporal_execution_skips_zero_and_applies_all_effect_types(frame: np.ndarray) -> None: + for effect_type in EffectType: + definition = EFFECT_REGISTRY[effect_type] + recipe = recipe_v2( + { + "id": effect_type.value, + "type": effect_type.value, + "parameters": definition.defaults, + "timing": {"mode": "continuous"}, + "variation": { + name: channel.default.model_dump(by_alias=True) + for name, channel in definition.variation_channels.items() + }, + } + ) + compiled = compile_recipe(recipe, timeline()) + plan = compiled.effects[0] + active = plan.activation_at(1) + output = apply_effect_stack(frame, recipe, 1, compiled_recipe=compiled) + assert output.shape == frame.shape + assert output.dtype == np.uint8 + inactive = EffectExecutionContext( + recipe.seed, + 1, + plan.effect, + EffectActivation(active=False, effective_intensity=0), + ) + assert np.array_equal(apply_effect_v2(frame, inactive), frame) + assert active.active + with pytest.raises(ValueError, match="compiled"): + apply_effect_stack(frame, recipe_v2({"id": "x", "type": "noise", "parameters": {}})) + + +def test_horizontal_glitch_layout_is_held_and_later_event_changes(frame: np.ndarray) -> None: + recipe = recipe_v2( + { + "id": "g", + "type": "horizontal_glitch", + "parameters": {"count": 6, "shift": 8}, + "timing": { + "mode": "events", + "events": [ + {"startSeconds": 0, "durationFrames": 8}, + {"startSeconds": 1, "durationFrames": 8}, + ], + }, + "envelope": {"attackFrames": 0, "releaseFrames": 0, "curve": "linear"}, + "variation": { + "layout": {"mode": "perEvent"}, + "offset": {"mode": "smooth", "periodFrames": 4}, + }, + } + ) + compiled = compile_recipe(recipe, timeline()) + plan = compiled.effects[0] + first_a = EffectExecutionContext(recipe.seed, 1, plan.effect, plan.activation_at(1)) + first_b = EffectExecutionContext(recipe.seed, 2, plan.effect, plan.activation_at(2)) + second = EffectExecutionContext(recipe.seed, 31, plan.effect, plan.activation_at(31)) + assert first_a.rng("layout").integers(0, 100) == first_b.rng("layout").integers(0, 100) + assert first_a.rng("layout").integers(0, 100) != second.rng("layout").integers(0, 100) + assert apply_effect_v2(frame, first_a).shape == frame.shape + + +def test_disabled_and_zero_intensity_effects_compile_without_events() -> None: + recipe = RecipeV2( + seed=1, + effects=[ + EffectInstanceV2(id="disabled", type="noise", enabled=False), + EffectInstanceV2(id="dry", type="static", intensity=0), + ], + ) + compiled = compile_recipe(recipe, timeline()) + assert all(not plan.events for plan in compiled.effects) + + +def test_zero_parameters_are_safe_for_active_spatial_effects(frame: np.ndarray) -> None: + for effect_type, parameters, variation in [ + ( + "horizontal_glitch", + {"count": 0, "shift": 0}, + {"layout": {"mode": "perEvent"}, "offset": {"mode": "perEvent"}}, + ), + ("frame_shift", {"max_x": 0, "max_y": 0}, {"offset": {"mode": "perEvent"}}), + ]: + recipe = recipe_v2( + { + "id": effect_type, + "type": effect_type, + "parameters": parameters, + "timing": {"mode": "continuous"}, + "variation": variation, + } + ) + compiled = compile_recipe(recipe, timeline()) + assert np.array_equal( + apply_effect_stack(frame, recipe, 0, compiled_recipe=compiled), + frame, + ) diff --git a/tests/test_temporal_routes.py b/tests/test_temporal_routes.py new file mode 100644 index 0000000..dff7ed1 --- /dev/null +++ b/tests/test_temporal_routes.py @@ -0,0 +1,206 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from glitchcraft.storage.repository import MediaAssetRepository + + +def add_video(app, *, duration: float = 10, frames: int = 300): + repository: MediaAssetRepository = app.extensions["media_repository"] + staged = repository.new_temporary_path(".mp4") + staged.write_bytes(b"source-video") + return repository.create_video_source( + staged_path=staged, + extension="mp4", + original_name="timeline.mp4", + container="mp4", + mime_type="video/mp4", + width=16, + height=12, + duration_seconds=duration, + frame_rate="30/1", + frame_count=frames, + video_codec="h264", + has_audio=False, + audio_codec=None, + file_size=12, + seed=123, + pixel_format="yuv420p", + ) + + +def temporal_recipe(*, timing: dict[str, Any] | None = None) -> dict[str, Any]: + return { + "schemaVersion": 2, + "seed": 123, + "effects": [ + { + "id": "glitch", + "type": "horizontal_glitch", + "enabled": True, + "intensity": 0.8, + "parameters": {"count": 6, "shift": 80}, + "timing": timing + or { + "mode": "events", + "events": [{"startSeconds": 1, "durationFrames": 12}], + }, + "envelope": {"attackFrames": 1, "releaseFrames": 5, "curve": "easeOut"}, + "variation": { + "layout": {"mode": "perEvent"}, + "offset": {"mode": "smooth", "periodFrames": 6}, + }, + } + ], + } + + +def test_effect_metadata_and_runtime_contract_are_exposed(client) -> None: + response = client.get("/api/effects") + assert response.status_code == 200 + payload = response.json + assert payload["recipeSchemaVersion"] == 2 + assert payload["supportedRecipeSchemaVersions"] == [1, 2] + horizontal = next( + effect for effect in payload["effects"] if effect["type"] == "horizontal_glitch" + ) + assert horizontal["naturalVideoDefaults"]["timing"]["mode"] == "sporadic" + assert horizontal["variationChannels"]["layout"]["default"]["mode"] == "perEvent" + assert "smooth" in horizontal["variationChannels"]["offset"]["supportedModes"] + + metadata = client.get("/metadata").json + assert metadata["supportedRecipeSchemaVersions"] == [1, 2] + assert metadata["temporalEffects"]["timingModes"] == [ + "continuous", + "range", + "sporadic", + "events", + ] + capabilities = { + item["slug"]: item for item in client.get("/api/capabilities").json["capabilities"] + } + assert capabilities["temporal-effect-modulation"]["exists"] + assert capabilities["deterministic-effect-schedules"]["exists"] + + +def test_schedule_endpoint_is_bounded_and_rejects_legacy(client, app) -> None: + source = add_video(app) + response = client.post( + f"/api/video-sources/{source.id}/effect-schedule", + json={"recipe": temporal_recipe()}, + ) + assert response.status_code == 200 + payload = response.json + assert payload["frameRate"] == "30/1" + assert payload["totalFrames"] == 300 + effect = payload["effects"][0] + assert effect["eventCount"] == 1 + assert effect["events"][0] == { + "eventIndex": 0, + "startFrame": 30, + "endFrame": 42, + "startSeconds": 1.0, + "endSeconds": 1.4, + "durationFrames": 12, + "identity": effect["events"][0]["identity"], + } + assert not effect["truncated"] + assert ( + client.post( + f"/api/video-sources/{source.id}/effect-schedule", + json={"recipe": {"schemaVersion": 1, "seed": 1, "effects": []}}, + ).status_code + == 400 + ) + assert ( + client.post( + "/api/video-sources/missing/effect-schedule", json={"recipe": temporal_recipe()} + ).status_code + == 404 + ) + + +def test_preview_reports_actual_frame_and_active_effects( + client, app, monkeypatch: pytest.MonkeyPatch +) -> None: + source = add_video(app) + + def preview( + _source_path: Path, + preview_path: Path, + _recipe: object, + *, + timestamp_seconds: float, + timeline_context: object, + activation_hook: Any, + ) -> int: + assert timestamp_seconds == 1.1 + assert timeline_context is not None + preview_path.write_bytes(b"png") + activation_hook(33, (("glitch", 0.75),)) + return 33 + + monkeypatch.setattr("glitchcraft.web.routes.create_video_preview_at", preview) + response = client.post( + f"/api/video-sources/{source.id}/preview", + json={"recipe": temporal_recipe(), "timestampSeconds": 1.1}, + ) + assert response.status_code == 200 + assert response.headers["X-GlitchCraft-Frame-Index"] == "33" + assert response.headers["X-GlitchCraft-Timestamp-Seconds"] == "1.1" + assert response.headers["X-GlitchCraft-Active-Effect-Count"] == "1" + assert response.headers["X-GlitchCraft-Active-Effect-Ids"] == "glitch" + assert response.headers["X-GlitchCraft-Effective-Intensities"] == "glitch:0.750000" + + +def test_job_validation_compiles_before_persistence( + client, app, monkeypatch: pytest.MonkeyPatch +) -> None: + source = add_video(app) + monkeypatch.setattr("glitchcraft.web.routes.ffmpeg_available", lambda: True) + monkeypatch.setattr("glitchcraft.web.routes.ffprobe_available", lambda: True) + invalid = temporal_recipe(timing={"mode": "range", "startSeconds": 20, "endSeconds": 21}) + response = client.post( + f"/api/video-sources/{source.id}/jobs", + json={"recipe": invalid, "audioMode": "remove"}, + ) + assert response.status_code == 400 + repository: MediaAssetRepository = app.extensions["media_repository"] + assert repository.list_video_jobs() == [] + + manager = app.extensions["video_job_manager"] + monkeypatch.setattr(manager, "enqueue", lambda _job_id: None) + response = client.post( + f"/api/video-sources/{source.id}/jobs", + json={"recipe": temporal_recipe(), "audioMode": "remove"}, + ) + assert response.status_code == 202 + assert response.json["recipeVersion"] == 2 + stored = repository.get_video_job(response.json["jobId"]) + assert stored.recipe.schema_version == 2 + assert stored.recipe.effects[0].timing.mode == "events" + + defaults_response = client.post( + f"/api/video-sources/{source.id}/jobs", + json={ + "recipe": { + "schemaVersion": 2, + "seed": 123, + "effects": [ + { + "id": "natural", + "type": "horizontal_glitch", + "parameters": {"count": 6, "shift": 80}, + } + ], + }, + "audioMode": "remove", + }, + ) + assert defaults_response.status_code == 202 + canonical = repository.get_video_job(defaults_response.json["jobId"]).recipe + assert canonical.effects[0].timing.mode == "sporadic" + assert canonical.effects[0].variation["layout"].mode == "perEvent" diff --git a/tests/test_video_workflow.py b/tests/test_video_workflow.py index 9621fbc..daeb805 100644 --- a/tests/test_video_workflow.py +++ b/tests/test_video_workflow.py @@ -12,7 +12,7 @@ from flask import Flask from pydantic import ValidationError -from glitchcraft.contracts.effects import Recipe +from glitchcraft.contracts.effects import Recipe, RecipeV2 from glitchcraft.errors import ( ExternalToolError, MediaReadError, @@ -1100,7 +1100,7 @@ def test_video_http_streaming_outputs_and_controlled_errors( assert ( client.post( f"/api/video-sources/{source_id}/jobs", - json={"recipe": {"schemaVersion": 2, "seed": 1, "effects": []}}, + json={"recipe": {"schemaVersion": 3, "seed": 1, "effects": []}}, ).status_code == 400 ) @@ -1222,7 +1222,20 @@ def test_real_media_pipeline_preserves_audio_and_creates_browser_mp4( ) job = repo.create_video_job( source_id=source.id, - recipe=Recipe(seed=source.seed), + recipe=RecipeV2.model_validate( + { + "schemaVersion": 2, + "seed": source.seed, + "effects": [ + { + "id": "scan", + "type": "scan_lines", + "parameters": {"gap": 4, "darkness": 0.8}, + "timing": {"mode": "continuous"}, + } + ], + } + ), audio_mode=AudioMode.PRESERVE, ) manager = VideoJobManager(repo) @@ -1230,6 +1243,7 @@ def test_real_media_pipeline_preserves_audio_and_creates_browser_mp4( completed = repo.get_video_job(job.id) assert completed.state == VideoJobState.COMPLETED assert completed.output_id is not None + assert completed.recipe.schema_version == 2 assert completed.timing_summary is not None assert completed.timing_summary.frames_processed == info.frame_count assert [milestone.phase for milestone in completed.milestones] == [ From 7f3eb43b58a505280bf1cedea111974ca7bf4f13 Mon Sep 17 00:00:00 2001 From: John Crafts <5889731+Artsen@users.noreply.github.com> Date: Tue, 28 Jul 2026 00:02:22 -0400 Subject: [PATCH 2/2] test: stabilize responsive timing assertion --- tests/browser/image-workflow.spec.js | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/browser/image-workflow.spec.js b/tests/browser/image-workflow.spec.js index e885ba2..83ca399 100644 --- a/tests/browser/image-workflow.spec.js +++ b/tests/browser/image-workflow.spec.js @@ -425,6 +425,7 @@ test("persistent video UI uploads once, previews a timestamp, and polls its job" "smooth", ); await page.setViewportSize({width: 390, height: 844}); + await glitchTiming.scrollIntoViewIfNeeded(); await expect(glitchTiming).toBeInViewport(); const accessibility = await new AxeBuilder({page}) .withTags(["wcag2a", "wcag2aa"])