diff --git a/.comfyignore b/.comfyignore index 141662c..f20128e 100644 --- a/.comfyignore +++ b/.comfyignore @@ -1,6 +1,7 @@ -# Dev-only files - kept in git, excluded from the published registry package. +# kept in git, left out of the published registry package tests/ -examples/_* +tools/dev/ +.github/ +docs/media/*.webp *.log -*.log.err -_e_models.yaml +__pycache__/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ac26c89 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,34 @@ +name: CI + +on: + push: + branches: [main, v1] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python: ["3.10", "3.12"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + cache: pip + - name: Install + run: | + pip install torch --index-url https://download.pytorch.org/whl/cpu + pip install numpy pillow opencv-python-headless imageio-ffmpeg pytest ruff + - name: Lint + run: ruff check . + - name: Tests + run: pytest + - name: Generated files are current + run: | + python tools/build_workflows.py --check + python tools/build_docs.py --check + - name: JS syntax + run: node --check js/director_timeline.js diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..fd7c7bb --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,19 @@ +name: Publish to Comfy Registry + +# Runs when pyproject.toml changes on main (i.e. a version bump), or by hand. +# Needs the repository secret REGISTRY_ACCESS_TOKEN (registry.comfy.org). +on: + workflow_dispatch: + push: + branches: [main] + paths: ["pyproject.toml"] + +jobs: + publish: + if: github.repository_owner == 'chillithebillis' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: Comfy-Org/publish-node-action@v1 + with: + personal_access_token: ${{ secrets.REGISTRY_ACCESS_TOKEN }} diff --git a/.gitignore b/.gitignore index ce2bb8a..92068b2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,22 +2,13 @@ __pycache__/ *.pyc *.pyo .pytest_cache/ +.ruff_cache/ .DS_Store *.egg-info/ build/ dist/ - -# local smoke-test artifacts (machine-specific paths / rendered output) -examples/_render_out/ -examples/_test_model_paths.yaml -examples/_e_models.yaml -examples/_promo_frames/ -examples/_palette.png -examples/_server.log -examples/_server.log.err - -# comfy-cli publish artifact + private notes node.zip *.zip +tools/dev/out/ NEXT_STEPS.md REVIEW.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..1f3f201 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,41 @@ +# Changelog + +## 1.0.0 + +Difforum becomes a direction layer: a visual timeline drives any renderer. + +### New +- **Director (timeline)**: a multi-track editor with Scenes (prompt + mood), Camera + (visual move picker, speed, amount, lens, easing, per-block audio reaction) and + Energy (a drawable denoise curve), plus an animated camera preview computed by the + render engine. A single `direction` wire feeds the renderers. +- **Setup** with target model grids: LTX 8k+1, MiniMax H3 17k+5 @ 24 fps, Wan 4k+1. +- Video model bridges: **Guide Frames**, **Keyframes**, **Camera → Prompt**, + **LTX Guides** (core `LTXVAddGuide`), **H3 Guides** (core `MiniMaxH3AddGuide`, multi-keyframe) + and **H3 Shot** (first/last frame, segments). +- **Camera Export** to After Effects (.jsx), Blender (.py) and JSON; **Camera Import** + from them, with helper exporters in `tools/`. +- 7 new moves: tilt up/down, crane up, handheld, drift, breathe, vortex. +- Feedback Sampler outputs the tracked depth per frame and a run report; its fine-tuning + moved to a separate **Render Options** node, so the sampler stays small. +- 10 templates in ComfyUI's template browser, generated from the node definitions. + +### Improved +- One shared feedback engine behind the Feedback Sampler, Live Sampler and Storyboard. +- Depth follows the image in 3D, and pseudo-3D replaces the silent freeze without depth. +- Cadence crossfades between keys. Revealed areas are repainted with extra noise. +- Colour anchoring per scene. Lazy prompt travel. Ring buffer in the Live Sampler. +- Deterministic 3D z-buffer on CUDA / MPS / CPU. + +### Fixed +- The pack no longer imports a top-level `core` module (it collided with other packs). +- The 0.x templates `audio_reactive_video`, `deluxe` and `mesmerize` had camera values + in the wrong fields. The v1 templates are generated, so this cannot recur. +- `feedback_classic` and `hybrid_wan_guides` rendered a static camera (3D move, no depth). + +### Removed / deprecated +- 0.x nodes are kept under Difforum/legacy (hidden, deprecated) until 2.0. See + docs/MIGRATION.md. +- Model Profile / Model Catalog → docs/MODELS.md. Load/Save Video → ComfyUI core + nodes. Look, Grade, Glow, Glitch, Datamosh → legacy only. +- Media moved to `docs/media/` as small WebP files. The published package is ~40 MB lighter. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 341729f..ff3e850 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,37 +4,37 @@ Thanks for your interest. Difforum is MIT-licensed and built to be extended. ## Ground rules -- **Keep dependencies minimal.** Core logic uses numpy and a safe AST evaluator; - torch is only used inside nodes that need it. No librosa / numexpr / pandas, so - the package keeps installing cleanly on Python 3.12 and the Comfy Registry. -- **Run the tests** before opening a PR (no GPU needed): +- **Keep dependencies minimal.** The engine uses numpy and torch (which ship with + ComfyUI); OpenCV only where a node needs it, and imported lazily. No librosa, + numexpr or pandas. +- **Engine first.** Put logic in `core/` (pure, testable, no ComfyUI), and keep + nodes thin. Import the engine relatively (`from ..core import ...`). +- **Every node gets a docstring and tooltips.** They become the in-app + description and `docs/NODES.md`. +- **Style:** match the surrounding code, no em dashes, and run `ruff check .` - ```bash - for t in core audio hybrid models warp color effects look video detail shots flow glitch prompt plot integration; do - python tests/test_$t.py - done - ``` - -- **Match the surrounding style** (naming, comment density, no em dashes). -- **Regenerate the example workflows** if you change a node's inputs: - `python examples/_build_examples.py` then `python examples/_validate_workflows.py`. - -## Developer Certificate of Origin (DCO) - -To keep the project's licensing clean and flexible, contributions are accepted -under the [DCO](https://developercertificate.org/): you certify that you wrote -the patch (or have the right to submit it) and agree it can ship under the -project's MIT license. Sign off each commit: +## Before a pull request ```bash -git commit -s -m "your message" +pytest # no GPU or ComfyUI needed +python tools/build_workflows.py # if you changed any node inputs +python tools/build_docs.py # if you changed docstrings / tooltips +ruff check . ``` -which adds a `Signed-off-by: Your Name ` line. +CI runs all of these, plus `--check` on the generated files. + +## Developer Certificate of Origin + +Contributions are accepted under the [DCO](https://developercertificate.org/). +Sign off each commit with `git commit -s`. -## Scope ideas +## Good places to extend -Good places to extend: new schedule functions, camera presets, effects (the -`core/effects.py` and `core/symmetry.py` style), reactive audio modes, model -catalog recipes, and the realtime path (NDI sink, MIDI/OSC control). See -[DESIGN.md](DESIGN.md) and [REALTIME.md](REALTIME.md). +- New camera moves: `core/camera_presets.py`. Add the expressions and a + `MOVE_INFO` entry; the Director picks it up automatically, and a glyph in + `js/director_timeline.js` is optional. +- New audio reactions: `REACTIONS` and `_react()` in `core/direction.py`. +- More bridges: follow `nodes/bridges.py`. Output plain types, or call core + nodes through `call_comfy_node()`. +- Camera interchange for more DCCs (Nuke, Houdini, Unreal): `core/export.py`. diff --git a/DESIGN.md b/DESIGN.md deleted file mode 100644 index 63a76ca..0000000 --- a/DESIGN.md +++ /dev/null @@ -1,222 +0,0 @@ -# Deforum 2 para ComfyUI - Documento de Design - -> Análise do cenário (junho/2026) e arquitetura proposta para um sucessor moderno -> do Deforum dentro do ComfyUI. - ---- - -## 1. Estado atual do Deforum - -| Projeto | Estado | Tecnologia | Problema | -|---|---|---|---| -| Deforum A1111 (clássico) | Legado | SD1.5/SDXL img2img feedback loop, depth MiDaS/AdaBins, schedules matemáticos | Flicker, sem coerência temporal real, ecossistema A1111 morrendo | -| [XmYx/deforum-comfy-nodes](https://github.com/XmYx/deforum-comfy-nodes) | Ativo (push 06/2026), 199★, 29 issues | Port do deforum clássico; loop via auto-queue (hack contra o DAG do Comfy) | Mesmo paradigma 2023; UX confusa; preso a SD1.5-era | -| Deforum Studio (deforum.studio) | Comercial/cloud | Presets áudio-reativos, render em nuvem | Fechado, pago, fora do ComfyUI | -| Parseq | Ativo | Scheduler visual externo (JSON de keyframes) | Só scheduling - continua útil como integração | - -**O que faz o Deforum ser "Deforum"** (a alma que devemos preservar): -1. **Schedules com expressões matemáticas** - `0:(0), 60:(2.5*sin(2*pi*t/30))` para qualquer parâmetro. -2. **Câmera 2D/3D real** - translation/rotation/zoom warpando o frame anterior via depth. -3. **Feedback loop** - cada frame nasce do anterior (o "look" psicodélico de morphing contínuo). -4. **Cadence** - difundir 1 a cada N frames e interpolar o resto. -5. **Áudio-reatividade** - schedules dirigidos por música. - -**O que envelheceu mal:** flicker frame-a-frame, drift de cor, depth ruim (MiDaS), -interpolação de cadence primitiva, ausência de modelo temporal. - ---- - -## 2. Tecnologias modernas a incorporar (2025–2026) - -### Geração de vídeo -- **Wan 2.2** (14B MoE / 5B): melhor open-source para controle. Variantes-chave: - - **FLF2V (First-Last Frame)**: gera vídeo entre dois keyframes → perfeito para "cadence 2.0". - - **VACE**: aceita frames de controle (depth/flow/pose/scribble) + imagens de referência + - inpainting temporal → é o "img2img temporal" que o Deforum nunca teve. - - **Lightning LoRAs (4-step)** + GGUF para low-VRAM. -- **LTX-Video**: alternativa muito rápida para preview/iteração. -- AnimateDiff: ainda útil como modo retrô/low-VRAM, mas não é mais o estado da arte. - -### Percepção / warp -- **Depth Anything V2** (ou DepthCrafter para depth temporalmente consistente) - substitui MiDaS. -- **RIFE 4.x / FILM / GIMM-VFI** - interpolação para cadence e pós. -- Optical flow (RAFT/UniMatch) para blending anti-flicker e máscaras de oclusão. - -### Áudio-reatividade -- [ComfyUI_Yvann-Nodes](https://github.com/yvann-ba/ComfyUI_Yvann-Nodes) e - [RyanOnTheInside](https://github.com/ryanontheinside/ComfyUI_RyanOnTheInside) (everything-reactivity). -- [FizzNodes](https://github.com/FizzleDorf/ComfyUI_FizzNodes) / AudioScheduler - referência de API de scheduling no Comfy. -- Análise via librosa: amplitude, onsets, bandas de frequência, BPM. - -### Consistência e aceleração -- IPAdapter / Flux Redux para travar estilo entre keyframes. -- Color matching LAB + referência ancorada (anti-drift). -- TeaCache, SageAttention, torch.compile. - ---- - -## 3. Conceito central do Deforum 2 - -> **O warp deixa de ser o frame final e vira condição para um modelo de vídeo.** - -O Deforum original warpa o frame anterior e roda img2img - daí o flicker. -O Deforum 2 mantém o motor de câmera matemático, mas usa os frames warpados como -**guias estruturais** (depth/flow) para o Wan VACE preencher com coerência temporal nativa. - -### Dois modos de operação - -**Modo A - Clássico+ (low VRAM, look retrô melhorado)** -Feedback loop tradicional (SD1.5/SDXL/Flux img2img) com melhorias modernas: -depth Anything V2 no warp, blending por optical flow, color coherence LAB, -máscara de oclusão para re-difundir só o que o warp expôs. - -**Modo B - Híbrido Vídeo (o verdadeiro "Deforum 2")** -1. Motor de câmera gera a trajetória e os guide frames warpados (a cada N frames = "super-cadence", ex. N=16–32). -2. Keyframes são difundidos nos pontos âncora (com IPAdapter para estilo). -3. **Wan 2.2 VACE/FLF2V preenche os intervalos**, condicionado por depth/flow dos guide frames. -4. Resultado: movimento de câmera 100% deforum (matemático, áudio-reativo) com qualidade - e suavidade de modelo de vídeo nativo. Duração ilimitada por encadeamento de segmentos - (overlap de frames entre segmentos, técnica dos video-extenders). - ---- - -## 4. Nodes propostos (schema) - -### Tipos customizados -- `D2_SCHEDULE` - curva avaliada por frame (float[]) com metadados. -- `D2_CAMERA` - lista de matrizes 4×4 (pose por frame) + intrínsecos. -- `D2_AUDIO` - features de áudio alinhadas ao fps (amplitude, onsets, bandas, BPM grid). -- `D2_GUIDES` - pacote de guias por frame: {warped_rgb, depth, flow, occlusion_mask}. - -### Núcleo (Fase 1 - sem GPU) -| Node | Inputs | Outputs | Função | -|---|---|---|---| -| `D2 Anim Setup` | width, height, fps, max_frames, seed_schedule, mode | `D2_PARAMS` | Config global da animação | -| `D2 Schedule` | string keyframes, `D2_PARAMS`, [`D2_AUDIO`] | `D2_SCHEDULE`, FLOAT[] | Parser `0:(v), n:(expr)` com `t, fps, audio.*`, interpolação linear/bezier/step | -| `D2 Prompt Schedule` | string multilinha, CLIP, `D2_PARAMS` | CONDITIONING[] | Prompt travel com blend de conditioning | -| `D2 Audio Analyzer` | AUDIO, `D2_PARAMS`, bandas, smoothing | `D2_AUDIO`, curvas FLOAT[] | librosa → curvas por frame | -| `D2 Camera` | schedules tx/ty/tz/rx/ry/rz/zoom/fov | `D2_CAMERA` | Trajetória de câmera | -| `D2 Parseq Import` | JSON | `D2_SCHEDULE`s | Compatibilidade Parseq | - -### Motor de movimento (Fase 2) -| Node | Inputs | Outputs | Função | -|---|---|---|---| -| `D2 Depth` | IMAGE, modelo (DA-V2 S/B/L, DepthCrafter) | DEPTH | Estimativa de profundidade | -| `D2 Warp 3D` | IMAGE, DEPTH, `D2_CAMERA`, frame_idx | IMAGE, MASK (oclusão) | Reprojeção 3D do frame | -| `D2 Feedback Sampler` | MODEL, VAE, COND[], `D2_CAMERA`, schedules (strength, cfg, noise), cadence, color_coherence, init IMAGE | IMAGE[] | **Node monolítico** com o loop interno (Modo A) | -| `D2 Color Coherence` | IMAGE[], ref, modo (LAB/HSV/none), força | IMAGE[] | Anti-drift de cor | - -### Ponte de vídeo (Fase 3) -| Node | Inputs | Outputs | Função | -|---|---|---|---| -| `D2 Guide Builder` | keyframes IMAGE[], `D2_CAMERA`, DEPTH | `D2_GUIDES` | Warps + depth + flow para os intervalos | -| `D2 VACE Fill` | WAN MODEL, `D2_GUIDES`, keyframes, ref IMAGE, overlap | IMAGE[] | Preenche segmentos via VACE/FLF2V, encadeia com overlap | -| `D2 Interpolate` | IMAGE[], fator, engine (RIFE 4.x/FILM) | IMAGE[] | Pós-cadence | - -### Decisão de arquitetura: o loop -ComfyUI é um DAG sem loops nativos. Opções avaliadas: -- ❌ Auto-queue (abordagem XmYx): frágil, estado global, UX ruim. -- ✅ **Node monolítico com loop interno** (como samplers do AnimateDiff): robusto, cacheável, - permite barra de progresso e preview por frame via callbacks. -- 🔶 Futuro: migrar para subgraph/loop nativo quando a execution model do Comfy estabilizar isso. - ---- - -## 5. Roadmap de implementação - -1. **Fase 1 - Schedule Engine** (puro Python, testável sem GPU) - Parser de expressões (numexpr/AST seguro), curvas, áudio, Parseq import. É a fundação de tudo. -2. **Fase 2 - Modo Clássico+** - Depth Anything V2 + Warp 3D + Feedback Sampler + color coherence. Já entrega um "deforum melhor que o original". -3. **Fase 3 - Modo Híbrido Vídeo** - Guide Builder + VACE Fill com Wan 2.2 (+ Lightning LoRA p/ velocidade, GGUF p/ low-VRAM). -4. **Fase 4 - UX** - Widget JS de preview de curvas no node, presets áudio-reativos, exemplos de workflow. - -## 6. Requisitos de hardware (referência) - -| Modo | VRAM mínima | Confortável | -|---|---|---| -| Clássico+ (SD1.5) | 6 GB | 8 GB | -| Clássico+ (SDXL/Flux) | 10 GB | 16 GB | -| Híbrido (Wan 2.2 5B / GGUF) | 8–10 GB | 12 GB | -| Híbrido (Wan 2.2 14B fp8) | 16 GB | 24 GB | - ---- - -## 7. O esquema HÍBRIDO em detalhe (o "Deforum 2" de verdade) - -### Grafo de nodes - -``` - ┌─────────────────┐ - Anim Setup ──────►│ Difforum Camera │ schedules tx/ty/tz/rot/zoom (+audio) - │ └────────┬────────┘ - │ │ DIFFORUM_CAMERA (poses 4x4 por frame) - │ ▼ - Model Profile ───►┌─────────────────┐◄──── keyframe IMAGE(s) (anchors) - (VRAM 12-32GB) │ Difforum Hybrid │◄──── DIFFORUM_SCHEDULE (strength/cfg/motion) - │ │ Render │◄──── COND (prompt travel) - └───────────►│ │◄──── WanVideoWrapper MODEL/VAE - └────────┬────────┘ - │ IMAGE[] (vídeo) - ┌──────────┴──────────┐ - ▼ ▼ - RIFE/GIMM-VFI Video Combine - (cadence 2.0) (mp4 + áudio) -``` - -### Loop interno do Hybrid Render (por segmento) - -A animação é cortada em **segmentos** de `segment_frames` (do perfil), com -`overlap` frames compartilhados para encadear sem corte visível: - -1. **Âncora**: difunde (ou recebe) o keyframe inicial do segmento. IPAdapter/ - Redux trava o estilo para não driftar entre segmentos. -2. **Guias de câmera**: para cada frame do segmento, warpa a âncora pela `pose` - acumulada da `DIFFORUM_CAMERA` usando depth (Depth Anything V2) → gera - `warped_rgb + depth + máscara de oclusão`. É aqui que mora o controle de - câmera estilo Deforum. -3. **Preenchimento VACE**: o Wan 2.2 recebe os guias como controle estrutural + - a âncora como referência e **gera o segmento com coerência temporal nativa** - (sem o flicker do feedback img2img clássico). -4. **Encadeia**: os últimos `overlap` frames viram a âncora do próximo segmento - (técnica dos video-extenders) → duração ilimitada. -5. **Schedules por frame**: strength, cfg, motion_scale e prompt são lidos das - curvas Difforum a cada frame - tudo áudio-reativo. - -### Modos de motor (família selecionável no Model Profile) - -- **wan22** (recomendado): híbrido completo VACE/FLF2V. O melhor resultado. -- **sd15_animatediff**: o "look AnimateDiff" leve; câmera vira context+motion. -- **ltxv**: previews ultrarrápidos para iterar schedules antes do render final. -- **sdxl**: modo Clássico+ (feedback loop por frame, look deforum puro). - -### Tiers de VRAM (resolvidos automaticamente - `Difforum Model Profile`) - -| VRAM | Modelo (família wan22) | Quant | Resolução | Segmento | Offload | -|---|---|---|---|---|---| -| 12 GB | Wan 2.2 **5B** | GGUF Q5_K_M | 640×640 | 49f | sequential | -| 16 GB | Wan 2.2 **14B** | GGUF Q4_K_M | 832×480 | 65f | model | -| 24 GB | Wan 2.2 **14B** | GGUF Q5_K_M | 1280×720 | 81f | model | -| 32 GB | Wan 2.2 **14B** | fp8 | 1280×720 | 81f | none | - -`quality=fast` usa **Lightning LoRA (4 steps)**; `balanced` 6 steps; `quality` -sobe para 20 steps sem distill. O node auto-detecta a VRAM via torch e cai no -tier certo (abaixo de 12 GB faz clamp pro 5B em vez de quebrar). - -### Status desta fase - -✅ `Difforum Camera` (motor de câmera 2D/3D, poses acumuladas, áudio-reativo) -✅ `Difforum Model Profile` (tiering 12-32GB, GGUF-aware, auto-detect VRAM) -✅ `Difforum Model Catalog` (modelos clássicos treináveis + receitas) -✅ `Difforum Warp (2D/3D)` (warp afim + reprojeção por depth + máscara de oclusão) -✅ `Difforum Feedback Sampler` (Modo Clássico+: warp→re-difunde→cor, vídeo end-to-end) -✅ `Difforum Guide Builder` (ponte híbrida: guias warpados → Wan VACE existente) - -### Por que Guide Builder em vez de um Hybrid Render monolítico - -Acoplar à API interna do WanVideoWrapper quebra a cada versão dele. O Guide -Builder gera batches de IMAGE/MASK padrão que plugam em qualquer grafo VACE do -Wan que o usuário já tenha - robusto e à prova de versão. A câmera continua 100% -Difforum; o Wan faz só o preenchimento temporal. diff --git a/README.md b/README.md index f74d69e..29a7a70 100644 --- a/README.md +++ b/README.md @@ -1,423 +1,127 @@ -# Difforum: Deforum-style animation for ComfyUI +
-Keyframe animation driven by math expressions, with camera moves, audio -reactivity and prompt travel. It brings the Deforum workflow to current models -(SD1.5, SDXL, **Flux**, SD3.5, Wan 2.2) and Python 3.12+. + -![Difforum kaleidoscope promo](examples/difforum_promo.gif) +# Difforum -*Sacred-geometry prompt travel with in-loop kaleidoscope symmetry and Echo -Trails. SDXL (Juggernaut XL) feedback render at 1024x576. Full 720p clip: -[`examples/difforum_promo.mp4`](examples/difforum_promo.mp4). Made entirely with the -Difforum nodes in ComfyUI.* +**Direct the camera. Let any model render it.** -![Difforum 16:9 showcase](examples/difforum_showcase.gif) +A timeline for ComfyUI: draw scenes, camera moves and energy with the mouse, +then render the shot with MiniMax H3, LTX-2, or the classic Deforum +feedback look. The camera also goes out to After Effects and Blender. -*16:9 Classic+ render. Prompt travel `nebula, spiral galaxy, black hole, -newborn star` blended per frame, with the camera flying forward and the feedback -loop reshaping the scene. SDXL at 1024x576, full clip: -[`examples/difforum_showcase.mp4`](examples/difforum_showcase.mp4).* +[![CI](https://github.com/chillithebillis/Difforum/actions/workflows/ci.yml/badge.svg)](https://github.com/chillithebillis/Difforum/actions/workflows/ci.yml) +![ComfyUI](https://img.shields.io/badge/ComfyUI-custom%20nodes-4f8ef7) +![License: MIT](https://img.shields.io/badge/license-MIT-green) -![Difforum audio-reactive](examples/difforum_audio.gif) +
-*Audio-reactive render: bass pumps the zoom, beats pulse the strength and spin. -One node turns any band into a curve. SDXL, full clip: -[`examples/difforum_audio.mp4`](examples/difforum_audio.mp4).* +![The Director timeline](docs/media/director.jpg) -## Why it exists +## What it does -The original [Deforum](https://github.com/deforum-art/sd-webui-deforum) is hard -to run today: its library refuses Python 3.12, it's tied to SD1.5-era img2img, -and the frame-by-frame loop flickers. Difforum rebuilds the same ideas from -scratch and fixes those problems. +Video models render well, but you steer them with words. Difforum gives you +back control over the **camera and the timing**: -What you get: +1. **Direct.** In the **Director** node you lay out *scenes* (prompt + mood), *camera + moves* (picked from 25 visual presets) and an *energy curve*. Press ▶ to + preview the move; the preview uses the same engine that renders. +2. **Render.** One `direction` wire goes to the renderer of your choice: -- The same `0:(expr)` keyframe syntax (`sin`, `cos`, `t`, audio variables), - camera schedules, audio reactivity and prompt travel. -- A model-agnostic sampler that takes plain ComfyUI `MODEL/VAE/CONDITIONING`, so - you can drop in SDXL, **Flux**, SD3.5 or an SD-Turbo/LCM model instead of - 2023-era SD1.5. No lock-in. -- Three render paths over one control layer: **Classic+** (depth-aware warp, - re-diffuse, LAB colour match), **Hybrid** (drives Wan 2.2 VACE), and - **AnimateDiff** (feeds prompt travel and schedules into AnimateDiff-Evolved). -- No `eval()`, no exotic dependencies (safe AST evaluator, numpy FFT audio, torch - warps), 16 test suites, and a clean install on Python 3.12 and the Comfy Registry. + | Renderer | Difforum node | You get | + |---|---|---| + | **MiniMax H3** | H3 Guides / H3 Shot | Your camera turned into keyframes, first/last frames and a camera prompt, with no hand-placed frames | + | **LTX-2 / 2.5** | LTX Guides | Keyframes inside the LTX latent | + | **Any image model** (SDXL, Flux, SD1.5, turbo) | Feedback Sampler | The Deforum look: every frame re-imagines the last | + | **Realtime** (turbo models, webcam) | Live Sampler | Plays inside the node, with Spout / OBS output | -See [DESIGN.md](DESIGN.md) for the architecture, [PERFORMANCE.md](PERFORMANCE.md) -for quality at few steps and Apple Silicon, and [REALTIME.md](REALTIME.md) for the -live-performance direction. +3. **Finish.** **Camera Export** writes the same camera for After Effects (`.jsx`) + and Blender (`.py`) so titles, 3D and comp lock to the AI shot. -## The nodes - -34 nodes in six groups. Everything below is schedulable, and any schedule -can be driven by audio. - -**Schedules.** The GPU-free animation brain: - -| Node | What it does | -|---|---| -| **Difforum · Anim Setup** | Global params (size, fps, frames, seed) shared downstream | -| **Difforum · Schedule** | Parses `0:(expr), 60:(expr)` into a per-frame curve | -| **Difforum · Sample Schedule** | Reads a curve value at one frame | -| **Difforum · Schedule Info** | Debug summary + ASCII sparkline of a curve | -| **Difforum · Schedule Plot** | Renders the computed curve as an IMAGE (Preview-ready) | - -**Direction.** Plan the clip before spending render time on it: - -| Node | What it does | -|---|---| -| **Difforum · Anim Setup+** | Duration in frames *or* seconds, framing by aspect ratio, resolution clamped to what the machine can finish | -| **Difforum · Film Director** | Visual timeline: drag scenes, pick a mood + camera each, and it emits the camera, prompt and strength schedules together | -| **Difforum · Storyboard** | The whole clip warped but never diffused, as a contact sheet + drift readout. Under a second - fix the movement here | -| **Difforum · Camera Keys** | Keyframed camera: moves *blend* into each other instead of cutting, and the lens (fov) is its own interpolated channel | -| **Difforum · Seamless Camera** | Camera path made periodic, so the clip loops with no crossfade | -| **Difforum · Loop Take** | Keeps the settled lap of a multi-lap render and scores the seam | - -**Audio reactivity.** Pure numpy (no librosa): - -| Node | What it does | -|---|---| -| **Difforum · Audio Analyzer** | AUDIO → per-frame curves `amp/low/mid/high/onset/beat` | -| **Difforum · Audio Schedule (reactive)** | Turn a band into a ready curve (bass-pump, beat-pulse…) for any schedule input | +No camera expressions to write. They are still there if you want them. -Connect Audio Analyzer's `audio_curves` to Schedule's `audio` input, then use -the curve names in expressions: `0:(0.2 + 0.8*amp)`, `0:(beat*0.6)`, -`0:(2*low - high)`. - -**Camera, models and rendering.** - -| Node | What it does | -|---|---| -| **Difforum · Camera Move (presets)** | Intuitive camera: pick a move (zoom/orbit/spiral/shake…) + speed + intensity | -| **Difforum · Camera Shots (director)** | Direct the camera like an edit: a shot list (`frame: preset speed intensity`) chains moves across the clip | -| **Difforum · Camera Path Preview** | See the camera direction before rendering: top-down trajectory + zoom strip as an IMAGE | -| **Difforum · Camera (advanced)** | Deforum 2D/3D camera schedules → per-frame poses (audio-reactive) | -| **Difforum · Model Profile** | Auto-resolve model/quant/res/steps for the GPU (12→32GB+, GGUF) | -| **Difforum · Model Catalog** | Classic, trainable model recipes + download/training guide | -| **Difforum · Warp (2D/3D)** | Classic Deforum warp + occlusion mask (force_2d / force_3d) | -| **Difforum · Feedback Sampler** | Classic+ video: warp→re-diffuse→colour-match (LAB/RGB) loop (SD1.5/SDXL) | -| **Difforum · Guide Builder** | Warp anchor along camera path → guide batch for Wan 2.2 VACE | -| **Difforum · Prompt Scenes (travel)** | Prompt travel by scenes - one text box per scene, auto-spaced | -| **Difforum · Prompt Schedule (travel)** | Prompt travel from a `frame: prompt` schedule | -| **Difforum · Prompt Batch (→ AnimateDiff)** | Stacks prompt travel into one batched CONDITIONING for AnimateDiff | - -**Effects.** Mirror, kaleidoscope, temporal trails and detail recovery: - -| Node | What it does | -|---|---| -| **Difforum · Symmetry / Kaleidoscope** | Mirror H/V, 4-fold quad, or N-segment kaleidoscope on a frame or batch | -| **Difforum · Echo Trails** | Long-exposure motion trails across a frame batch (smooth, hypnotic) | -| **Difforum · Detail Guard (anti-mush)** | Unsharp mask + contrast + noise injection to fight feedback-loop blur | -| **Difforum · Flow Stabilize (anti-flicker)** | Blends history along optical flow, gated by photometric confidence: texture stops boiling, motion never ghosts | -| **Difforum · Ping-Pong Loop** | Forward + reverse for a guaranteed loop (reverses motion) | -| **Difforum · Loop Blend (perfect loop)** | True forward loop: the tail morph-fades into the head along optical flow, no reversed motion (wired into the render templates) | -| **Difforum · Glitch (DSP databending)** | Convolution kernels, 1D delay-line databending, bitcrush, VHS jitter/bands/chroma, RGB split | -| **Difforum · Datamosh (optical flow)** | Motion vectors keep flowing while the refresh melts: grid (macroblock), melt, edge modes | +## Quick start -Symmetry is also built into the **Feedback Sampler** (`symmetry` + `symmetry_segments`): -applied *inside* the loop it compounds each frame and the diffusion heals the -seams, giving a living kaleidoscope (`difforum_mesmerize_kaleidoscope.json`). +1. Install from **ComfyUI Manager** (search "Difforum"), or: + ```bash + cd ComfyUI/custom_nodes && git clone https://github.com/chillithebillis/Difforum.git difforum + ``` +2. Restart ComfyUI and open **Templates → Difforum**. +3. Start with **`01_storyboard_no_model`**. It needs no model: load a picture, shape the + timeline, press ▶, then Queue. -![Difforum living kaleidoscope](examples/difforum_kaleidoscope.gif) +## Templates -*In-loop kaleidoscope symmetry on an SDXL feedback render, smoothed with Echo -Trails. The pattern folds every frame and the diffusion reseals the seams, so it -keeps growing on itself. Made entirely with the Difforum nodes.* +| # | Template | What it shows | Needs | +|---|---|---|---| +| 01 | `storyboard_no_model` | Direct a shot and preview the whole move in about a second | an image | +| 02 | `feedback_sdxl` | The Deforum look on a modern model | SDXL / SD1.5 / Flux | +| 03 | `parallax_3d_depth` | Real 3D parallax from a depth map | + [DepthAnythingV2](https://github.com/kijai/ComfyUI-DepthAnythingV2) | +| 04 | `audio_reactive` | Camera moves that react to music, with no expressions | + an audio file | +| 05 | `live_turbo` | Realtime feedback, webcam mirror, VJ output | a turbo model | +| 06 | `seamless_loop` | A loop without a crossfade, for installations | a checkpoint | +| 07 | `ltx_guides` | Director keyframes guiding LTX-2 / 2.5 | your LTX graph | +| 08 | `h3_first_last_frame` | **MiniMax H3 FL2VA**: first frame + the frame where your camera ends | MiniMax H3 fl2va | +| 09 | `camera_to_ae_blender` | Camera out to After Effects / Blender, and back in | nothing | +| 10 | `h3_multikeyframe_guides` | **MiniMax H3**: keyframes along your camera, anchored inside the generation | MiniMax H3 ref2va | -**VJ look and video.** Grade video footage for live visuals, no model required: +### MiniMax H3 in one picture -| Node | What it does | -|---|---| -| **Difforum · VJ Look (presets)** | One-shot grade: neon / cinematic / vaporwave / film / noir / psychedelic, with an intensity that an audio schedule can pulse to the beat | -| **Difforum · Colour Grade** | Exposure, contrast, saturation, white balance, lift/gamma/gain, hue cycling | -| **Difforum · Glow** | Neon bloom (blur the bright areas, screen-blend them back) | -| **Difforum · Load Video** | Read a clip into an IMAGE batch (frame skip, resize cap), via OpenCV | -| **Difforum · Save Video (MP4)** | Write an IMAGE batch back to an h264 MP4, via ffmpeg | - -These are plain IMAGE to IMAGE, so they work on a still, a feedback render or a -whole footage batch. The `difforum_vj_footage.json` template is a complete, -self-contained pipeline (Load Video to VJ Look to Echo Trails to Save Video) that -needs **no checkpoint and no external nodes**: footage in, graded MP4 out. The -video IO uses opencv-python and imageio-ffmpeg (usually already in ComfyUI). - -Three render paths: **Classic+** (Feedback Sampler - self-contained Deforum -video), **Hybrid** (Guide Builder → your Wan 2.2 VACE graph), and **AnimateDiff** -(Prompt Batch → AnimateDiff-Evolved for native motion + Difforum control). See -DESIGN.md §7. Modern bases (SDXL / Flux / SD3.5) drop into the Feedback Sampler - -it is model-agnostic. AnimateDiff motion modules need AnimateDiff's own batch -sampler (the `difforum_animatediff_sd15.json` template wires it). - -The 3D warp exposes `translation_scale` to tune motion vs. the depth estimator's -(non-metric) range; colour coherence does perceptual **LAB** matching by default. - -![Difforum hybrid guide preview](examples/difforum_guide_demo.gif) - -*Hybrid bridge: the Guide Builder warps one anchor image along the camera path -into the guide batch a Wan 2.2 VACE graph consumes (camera by Difforum, fill by Wan).* - -> **Black output?** Some ComfyUI builds make SD1.5 fp16 produce NaN (all-black -> frames). Launch ComfyUI with `--force-fp32` - it's a base-model precision -> issue, not Difforum. - -## Install - -**Manual / git (local or self-hosted):** - -```bash -cd ComfyUI/custom_nodes -git clone https://github.com/chillithebillis/Difforum.git difforum -# deps are minimal (numpy, already present); only needed for a bare env: -pip install -r difforum/requirements.txt ``` - -Restart ComfyUI; the console prints `[Difforum] loaded N nodes`. - -**ComfyUI Manager:** *Custom Nodes Manager* → install via Git URL with the repo -above (or by name once published to the registry). - -**Comfy Registry:** Difforum is registry-ready via `pyproject.toml` (no heavy or -version-locked deps, so it builds on the cloud's Python 3.12). To publish: - -```bash -pip install comfy-cli -comfy node publish # uploads to the Comfy Registry +Load Image ─► Guide Frames ─► Keyframes ─► H3 Guides ─► Guider ─► Sampler ─► Video + audio + ▲ ▲ +Setup ─► Director (camera, scenes) ─► Camera → Prompt ─► MiniMax H3 Reference to Video ``` -**Cloud GPU (self-host):** the quickest way to run Difforum in the cloud today is -a managed GPU host like [RunPod](https://www.runpod.io) or [Vast.ai](https://vast.ai): -spin up a ComfyUI template, then in the Manager install via the Git URL above (or -`git clone` into `custom_nodes`). Everything runs with no gatekeeping. - -**Comfy Cloud** runs a curated set of vetted node packs, so a node has to be -reviewed and added by the Comfy team before it works there. Publishing to the -Registry is the first step; getting onto Comfy Cloud is their manual curation. Use -a self-host GPU (above) in the meantime. - -## Example workflows - -Workflows live in `examples/`, split in two. Drag any `.json` onto the ComfyUI -canvas. - -### `examples/basic/` - one idea at a time - -Little or nothing to download, few nodes, no external packs. - -| File | Needs assets? | Shows | -|---|---|---| -| `difforum_schedule_basic.json` | no | Anim Setup → Schedule → Schedule Info. Runs anywhere, zero downloads. | -| `difforum_models_info.json` | no | Model Profile (auto-VRAM) + Model Catalog (recipes). Reference card. | -| `difforum_vj_footage.json` | a video clip | Load Video → VJ Look → Echo Trails → Save MP4. No checkpoint, no external nodes. | -| `difforum_camera_warp.json` | an image | Load Image → Camera → Warp (2D/3D) → Preview. The warp + occlusion on one frame. | -| `difforum_audio_reactive.json` | an audio file | Audio Analyzer → two Schedules (bass-zoom `1.0+0.6*low`, beat-spin `beat*30`). | -| `difforum_feedback_classic.json` | an SD1.5 checkpoint | Full Classic+ video: txt2img frame 0 → Camera + strength → Feedback Sampler → Save. | -| `difforum_intuitive_controls.json` | SD1.5 checkpoint | Camera **Move presets** + **Prompt Scenes** → feedback video. | -| `difforum_turbo_live.json` | SD-Turbo / LCM ckpt | Low-step (4) fast feedback, the path toward realtime. | - -### `examples/advanced/` - full productions - -Several models, helper node packs, longer graphs. - -| File | Needs assets? | Shows | -|---|---|---| -| `difforum_film_director.json` | SDXL ckpt + DMD2 LoRA | **Start here.** Film Director timeline drives camera, prompts and strength at once; Storyboard previews the whole clip before any diffusion runs. | -| `difforum_parallax_fluid.json` | SDXL ckpt + DMD2 LoRA + DepthAnythingV2 + Frame Interpolation | **Fluid motion**: a depth map turns on real parallax (near pixels travel faster than far ones), RIFE interpolates to 48fps, and the loop settings are tuned against convergence. | -| `difforum_seamless_installation.json` | SDXL ckpt + DMD2 LoRA | **True loop, no crossfade**: periodic camera + 3 laps + Loop Take. For projections that run for hours. | -| `difforum_fast_sdxl.json` | SDXL ckpt + DMD2 LoRA | **Fast recipe**: frame 0 full quality on the base model, loop on a 4-step distill + cadence 2. ~8-10x less diffusion cost. | -| `difforum_ipadapter_coherent.json` (16:9) | ckpt + IPAdapter + style image | **Style-locked** feedback via IPAdapter → strong coherence, low drift. | -| `difforum_audio_reactive_video.json` (16:9) | ckpt + audio file | **Audio Schedule**: bass pumps zoom, beats pulse strength → MP4. | -| `difforum_deluxe_travel_controlnet_video.json` | SD1.5 ckpt + tile ControlNet | Everything: travel + ControlNet + RIFE interpolation → MP4. | -| `difforum_animatediff_sd15.json` (16:9) | SD1.5 ckpt + AnimateDiff-Evolved | Prompt Batch → AnimateDiff native motion + Difforum control. | -| `difforum_qrcode_illusion.json` (16:9) | SD1.5 ckpt + QR-Monster ControlNet + pattern | Locks a spiral/logo/mask in the scene while the loop morphs. | -| `difforum_mesmerize_kaleidoscope.json` | SD1.5 checkpoint | **Living kaleidoscope**: in-loop symmetry, the diffusion heals the seams. | -| `difforum_hybrid_wan_guides.json` | an anchor image | Camera → Guide Builder → guide batch for a Wan 2.2 VACE graph. | -| `difforum_realtime_live.json` | SD-Turbo / LCM ckpt | **Native realtime**: Live Sampler internal loop with a live preview in the node. | - -Templates exercise every node. Regenerate with -`python examples/_build_examples.py`. - -**Prompt pack:** [`examples/PROMPTS.md`](examples/PROMPTS.md) has 10 ready-to-paste -prompt-travel presets (Cosmic Voyage, Sacred Geometry kaleidoscope, Neon City, -Elemental Shift, Audio-reactive Pulse and more) with suggested camera, strength -and symmetry settings for each. - -**Every workflow has a yellow Note on the canvas** listing exactly which models -to download and the folder they go in, so you can set up without guessing. - -## Models & downloads - -Quick reference for the files the templates ask for (drop each in the folder -shown under your ComfyUI `models/` directory): - -| Model | Folder | Where | -|---|---|---| -| SD1.5 checkpoint (DreamShaper 8) | `checkpoints` | civitai.com/models/4384 | -| SDXL checkpoint (Juggernaut XL) | `checkpoints` | civitai.com (search "Juggernaut XL") | -| Flux.1-dev fp8 / schnell | `checkpoints` or `unet` | hf: Comfy-Org/flux1-dev, black-forest-labs/FLUX.1-schnell | -| DMD2 4-step distill LoRA (SDXL) | `loras` | hf: tianweiy/DMD2 (dmd2_sdxl_4step_lora) | -| SD-Turbo / LCM-LoRA (fast) | `checkpoints` / `loras` | hf: stabilityai/sd-turbo, latent-consistency/lcm-lora-sdv1-5 | -| Wan 2.2 5B GGUF + umt5 + wan vae | `unet` / `text_encoders` / `vae` | hf: QuantStack/Wan2.2-TI2V-5B-GGUF, Comfy-Org/Wan_2.1_ComfyUI_repackaged | -| IP-Adapter + CLIP-Vision (ViT-H) | `ipadapter` / `clip_vision` | hf: h94/IP-Adapter | -| QR Code Monster ControlNet | `controlnet` | hf: monster-labs/control_v1p_sd15_qrcode_monster | -| Tile / depth ControlNet (SD1.5) | `controlnet` | hf: comfyanonymous/ControlNet-v1-1_fp16_safetensors | -| AnimateDiff motion module (mm_sd15_v3) | `animatediff_models` | hf: guoyww/animatediff | - -Helper nodes some templates use: ComfyUI-VideoHelperSuite (MP4), -ComfyUI-Frame-Interpolation (RIFE), ComfyUI_IPAdapter_plus, ComfyUI-GGUF, -ComfyUI-WanVideoWrapper, ComfyUI-AnimateDiff-Evolved. The **intuitive nodes** -(`Camera Move` presets, `Prompt Scenes`) are wired into `intuitive_controls`, -`turbo_live` and `camera_warp` for an easier start. - -## Quality & coherence tips - -- **Lock the style with IPAdapter.** Patch the checkpoint's MODEL through - IPAdapter (a reference image) *before* the Feedback Sampler - it keeps colour - and style stable across the whole clip (see `difforum_ipadapter_coherent.json`). -- **ControlNet must match the base family** (SD1.5↔SD1.5, SDXL↔SDXL). Feed the - Feedback Sampler's `control_net` for structure-guided feedback; lower - `control_strength` (~0.3-0.5) if it feels too rigid. -- **Use a modern base.** SDXL / Flux / SD3.5 in the Feedback Sampler look far - cleaner than SD1.5 - it's model-agnostic, just swap the checkpoint. -- **Colour drift?** Keep `color_mode = lab` (default) and `color_coherence` - around 0.7-0.9. **Flicker?** Lower per-frame `strength` (denoise) to ~0.4-0.5, - and run the frames through **Flow Stabilize** (strength ~0.5): it aligns the - previous frame along optical flow before blending, so it removes texture boil - without the ghosting that naive frame-blending causes. -- **Texture re-rolling between frames?** The sampler's `seed_mode = fixed` - (default) reuses the same sampling noise every frame, which calms boiling a - lot - especially visible in kaleidoscope renders. Set `increment` if you want - livelier texture churn. -- **Long videos, faster:** set the Feedback Sampler's `cadence` to 2-3 and only - every Nth frame is diffused - the in-between frames ride the camera warp - (classic Deforum turbo). Roughly N times faster with smoother motion; also on - the Live Sampler for higher realtime FPS. -- **Mushy after many frames?** The Feedback Sampler's `sharpen` (~0.2-0.5) and - `noise` (~0.02-0.05) re-inject detail each frame so the sampler resolves it - into structure instead of amplifying blur; `border = reflection` (default) - stops black edges creeping in on zoom-out and rotation. Same seed, same - camera, frame 48 of a spiral move: - -![Detail guard off vs on](examples/difforum_detail_ab_final.png) - -![Detail guard A/B](examples/difforum_detail_ab.gif) -- **Audio-reactive** is one node: `Difforum · Audio Schedule` turns a band - (bass/beat/onset) into a curve for zoom, strength, cfg - any schedule input. -- **Illusions / hidden patterns:** feed a grayscale pattern (spiral, logo, mask) - to `control_image` with **QR Code Monster** ControlNet - it stays locked in the - scene while the loop morphs around it (`difforum_qrcode_illusion.json`). -- **Mesmerizing / symmetric video:** set the Feedback Sampler's `symmetry` to - `kaleidoscope` (or `mirror_h/v/quad`) so it folds *inside* the loop - the - diffusion heals the seams and the pattern grows on itself. Add **Echo Trails** - after the sampler for smooth motion blur (`difforum_mesmerize_kaleidoscope.json`). - A slow rotation/zoom on the camera keeps feeding the kaleidoscope new material. - -## Performance - -Full detail in [PERFORMANCE.md](PERFORMANCE.md) - cost model, few-step recipes, -Apple Silicon memory budgets and measured throughput. - -Difforum's pixel-effect chain (warp, symmetry, sharpen, noise) runs on the best -available device automatically: CUDA, then Apple **MPS**, then CPU, with a safe -per-session fallback. The Model Profile node detects the device and adapts its -recommendations. Platform notes: - -**Apple Silicon (M-series).** -- **fp8 checkpoints do not work on MPS** (no fp8 kernels). Use **fp16/bf16 or - GGUF** versions instead; the Model Profile swaps its recommendation - automatically on Macs. -- Launch with `--use-pytorch-cross-attention` and set - `PYTORCH_MPS_HIGH_WATERMARK_RATIO=0.0` to unlock the full unified-memory pool. -- fp16 halves memory but is **not faster** than fp32 on M-series ALUs, so the - real speed lever is **few-step models**: SD-Turbo / LCM / Lightning at 1-4 - steps. The Live Sampler and Turbo templates are the best fit for Macs. -- Memory is unified: a 32GB Mac behaves like ~22GB of usable "VRAM" once the OS - and CPU take their share. - -**AnimateDiff (via the Prompt Batch bridge).** -- **AnimateLCM**: `lcm` beta schedule + LCM LoRA + `lcm` sampler, cfg 1.0-2.0, - ~4 steps - the fastest good-looking AnimateDiff path. -- For long clips use AnimateDiff-Evolved's **Context Options** (sliding windows) - and set Sample Settings `noise_type = FreeNoise` so window transitions stop - popping. - -**Wan 2.2.** -- **Lightning LoRA (lightx2v)**: 4 steps on the High model + 4 on the Low model, - cfg 1 - the standard fast setup our Model Profile recommends. -- **MagCache** (better than TeaCache in our reading) and **SageAttention** - (CUDA-only) stack more speed on top. -- Keep resolutions divisible by 16 or the VAE will fail at decode time. - -## Realtime / live - -The **Difforum Live Sampler** is a native realtime engine. Stock ComfyUI re-runs -the graph per "Queue Prompt", so the Live Sampler keeps the model resident and -runs its own **internal loop**: each tick it warps (camera), folds (symmetry), -re-diffuses 1-2 steps (Turbo/LCM) and color-matches, **streaming a live preview -into the node** as it goes. Queue once and watch it generate, no external runtime. - -- Pair with a **few-step model** (SD-Turbo / SDXL-Turbo / LCM / DMD2), 1-2 steps, - cfg ~1, at 512px for live FPS. The `difforum_realtime_live.json` template wires - a Turbo checkpoint to the Live Sampler with a kaleidoscope on. -- **Live webcam ("magic mirror"):** set `live_source` to `0` (webcam device) or a - video path and the loop stylizes the live feed, kaleidoscoping the camera in - realtime. `source_blend` mixes camera vs feedback (needs `opencv-python`). -- Live output to a VJ app: set `stream_dir` (writes frames to a folder that - OBS/Resolume/ffmpeg can read live) or `spout_name` (Spout, needs `SpoutGL`). -- For the highest FPS, add **[TensorRT](https://github.com/comfyanonymous/ComfyUI_TensorRT)** - + **TAESD** (tiny VAE). Map physical knobs/faders with MIDI/OSC. - -If you specifically need browser/WebRTC streaming, the same nodes can run under -ComfyStream. Full plan: [REALTIME.md](REALTIME.md). - -## Schedule syntax - -Deforum-compatible. Each keyframe holds a math expression: - -``` -0:(0), 60:(0.5*sin(2*pi*t/30)), 120:(1.0) -``` +The official *Multiframe Reference* template anchors images you place by hand. +Template 10 anchors frames that come **from your camera move**, so H3 performs +the dolly, orbit or crane you drew. Template 08 does the same with H3's +first/last-frame model, and splits clips longer than 20 s into segments. +Both use ComfyUI's core MiniMax H3 nodes. -Variables: `t`/`f` (frame), `s` (seconds), `fps`, `max_f`, `pi`, `e`, `tau`, -plus any audio curve (e.g. `amp`, `low`, `mid`, `high`) when an audio input is -connected. Functions: `sin cos tan asin acos atan atan2 sinh cosh tanh abs -sqrt exp log log10 pow floor ceil round min max sign clamp clip lerp -smoothstep`. Ternaries work: `t if t > 30 else 0`. +## The nodes -Expressions are evaluated with a whitelisted AST walker - **no `eval()` of -arbitrary code**, no external dependencies. +| Group | Nodes | +|---|---| +| **Setup** | Setup: duration in seconds, aspect, and snapping to each model's grid (H3 17k+5 @ 24 fps, LTX 8k+1, Wan 4k+1) | +| **Direction** | Director (timeline), Camera (keys), Camera (expressions), Storyboard | +| **Curves & prompts** | Schedule, Schedule Plot, Audio Analyzer, Audio Curve, Prompt Travel | +| **Render** | Feedback Sampler, Live Sampler, Render Options | +| **Video model bridges** | Guide Frames, Keyframes, Camera → Prompt, H3 Guides, H3 Shot, LTX Guides | +| **Export** | Camera Export (AE / Blender / JSON), Camera Import | +| **Post** | Loop, Symmetry, Echo Trails, Flow Stabilize, Detail Guard | -Easing between keyframes: `linear`, `ease_in`, `ease_out`, `ease_in_out`, `step`. +The full reference is in **[docs/NODES.md](docs/NODES.md)**. Every node also shows its description inside ComfyUI. -A copy-paste cookbook of ready expressions (bobs, bounces, audio pumps, -ping-pong loops, holds) lives in [MATHS.md](MATHS.md). +## The Deforum look, rebuilt -## Develop / test +![Feedback render](docs/media/kaleidoscope.webp) -No GPU or ComfyUI needed - 16 suites cover the engine, warp, colour, effects, look, video, camera shots, detail, -model catalog, and a full end-to-end orchestration (stub diffusion): +The Feedback Sampler runs on any image model and fixes what made the original +Deforum hard to use: -```bash -for t in core audio hybrid models warp color effects look video detail shots flow glitch prompt plot integration; do - python tests/test_$t.py -done -``` +- **Depth follows the image**, so parallax stays true for the whole clip. +- **3D moves never freeze.** Without a depth map they become pseudo-3D, and the node says so. +- **Cadence without pops.** Only every Nth frame is diffused; the frames in between are crossfaded. +- **Revealed edges are repainted** instead of smeared. +- **Colour locks per scene**, so prompt travel can change the palette. -`test_integration.py` wires every node together (AnimSetup → Audio → Camera → -Schedule → Feedback Sampler → frames, plus the Guide Builder hybrid path) and -runs the real production code path with a stub model/VAE - so the only thing -swapped for a real render is the checkpoint inside ComfyUI. +## Documentation -After changes, restart ComfyUI fully to reload the nodes (look for -`[Difforum] loaded N nodes` in the console). +| | | +|---|---| +| [Bridges](docs/BRIDGES.md) | MiniMax H3, LTX-2, After Effects, Blender | +| [Node reference](docs/NODES.md) | Every input and output | +| [Performance](docs/PERFORMANCE.md) | Speed vs. quality, Apple Silicon | +| [Models](docs/MODELS.md) | What works well, and the settings | +| [Prompt pack](docs/PROMPTS.md) | Ready-made scene sets | +| [Expressions](docs/EXPRESSIONS.md) | The math syntax, for power users | +| [Migrating from 0.x](docs/MIGRATION.md) | Old workflows still open; the new equivalents | +| [Architecture](docs/ARCHITECTURE.md) | How it is built, for contributors | ## Contributing -Issues and pull requests are welcome. Keep the dependency surface minimal (numpy -in core; torch only where a node needs it) and run the test suites before opening -a PR. Sign your commits off (`git commit -s`, a Developer Certificate of Origin -acknowledgement). See [CONTRIBUTING.md](CONTRIBUTING.md). - -## License +`pytest` runs without a GPU or ComfyUI. Templates and the node reference are +generated from the code (`tools/`). See [CONTRIBUTING.md](CONTRIBUTING.md). -MIT, see [LICENSE](LICENSE). You can use, modify and redistribute Difforum, -including commercially, as long as the copyright and license notice are kept. +MIT licensed. diff --git a/REALTIME.md b/REALTIME.md deleted file mode 100644 index 8245b41..0000000 --- a/REALTIME.md +++ /dev/null @@ -1,90 +0,0 @@ -# Difforum Live - realtime / live-performance - -Uma versão **tempo-real** do Difforum, nativa: tipo o StreamDiffusion do -TouchDesigner, mas dentro do ComfyUI e com a alma do Deforum (câmera matemática, -áudio-reatividade, prompt travel, simetria) dirigindo um loop de difusão rápido, -com preview ao vivo e saída para software de VJ. - -## O que já existe: Difforum Live Sampler (nativo) - -O **Difforum Live Sampler** é o motor realtime próprio do Difforum. O ComfyUI -normal reexecuta o grafo a cada "Queue Prompt" (não é streaming), então o Live -Sampler resolve isso com um **loop interno** num único node: - -- modelo **residente** na VRAM (carrega uma vez, não recompila o grafo por frame); -- a cada tick: **warp** (câmera) → **simetria/caleidoscópio** → **re-difusão de - 1-2 steps** (Turbo/LCM) → **color match** → realimenta; -- **preview ao vivo dentro do node** enquanto roda (manda cada frame pra UI por - WebSocket via `ProgressBar`) - você dá Queue uma vez e assiste gerar; -- **sinks opcionais**: `stream_dir` grava os frames numa pasta em tempo real (pra - OBS/Resolume/ffmpeg consumirem) e `spout_name` envia por **Spout** se você tiver - `SpoutGL` (degrada pra no-op se não tiver, nunca quebra); -- no fim retorna o batch de frames (dá pra salvar em MP4 também). - -Isso é "câmera matemática → difusão → tela, ao vivo" sem depender de runtime -externo. Pareie com um modelo **SD-Turbo / SDXL-Turbo / LCM** (1-2 steps, cfg ~1) -e resolução baixa (512) pra ganhar FPS. Template: `difforum_realtime_live.json`. - -### Espelho mágico: webcam ao vivo - -Setando `live_source` o Live Sampler vira um estilizador de **câmera ao vivo** -(o efeito TouchDesigner): cada tick puxa um frame novo, blenda com o feedback -(`source_blend`), aplica o caleidoscópio e difunde. Valores de `live_source`: - -- `""` (vazio) = feedback generativo puro (padrão); -- `"0"` / `"1"` = índice de webcam (via OpenCV); -- um caminho de arquivo = estiliza um vídeo (em loop). - -`source_blend` 0.9 = segue a câmera com um leve rastro; 1.0 = estiliza limpo; -baixo = mais feedback/trails. Precisa de `opencv-python` (já vem na maioria das -instalações ComfyUI). Validado de ponta a ponta com vídeo e webcam. - -### Parâmetros que importam - -| Campo | O que faz | -|---|---| -| `duration_frames` | quantos frames a sessão ao vivo roda | -| `steps` / `cfg` | 1-2 e ~1.0 com Turbo/LCM = realtime | -| `strength` | denoise por frame: baixo = coerente/suave, alto = mais mudança | -| `symmetry` | espelho/caleidoscópio dentro do loop (custo ~zero, hipnótico) | -| `target_fps` | trava o FPS (0 = o mais rápido possível) | -| `live_source` | webcam (`"0"`) ou vídeo (caminho) para o modo espelho ao vivo | -| `source_blend` | quanto da câmera vs feedback (0.9 = câmera com leve rastro) | -| `stream_dir` / `spout_name` | saída ao vivo pra apps de VJ | -| `live_preview` | liga o preview ao vivo no node | - -## Controle ao vivo - -A câmera (presets + intensidade), o prompt travel e a reatividade de áudio já -são Difforum. Para knobs/faders físicos, mapeie **MIDI/OSC** -([RyanOnTheInside](https://github.com/ryanontheinside/ComfyUI_RyanOnTheInside)) -para os parâmetros do Live Sampler. - -## Desempenho esperado (referência) - -| GPU | Modelo | ~FPS @512 | -|---|---|---| -| RTX 3060 12GB | SD1.5 LCM/Turbo, 1 step | ~6-15 | -| RTX 4090 | SD1.5-Turbo + TensorRT | 30-60+ | -| RTX 4090 | SDXL-Turbo | ~10-20 | - -Aceleração extra (opcional): **TensorRT** -([ComfyUI_TensorRT](https://github.com/comfyanonymous/ComfyUI_TensorRT)) compila -o modelo em engine (3-10 min, uma vez) e **TAESD** (VAE minúsculo) deixa o decode -quase instantâneo. É o que leva a FPS alto de verdade. - -## Roadmap - -1. ~~Entrada de webcam~~ - **feito** (`live_source` no Live Sampler). -2. **Sink NDI** nativo (além de Spout/folder) pra Resolume/OBS sem plugin extra. -3. **Live param bus** - MIDI/OSC mapeado direto nos campos do Live Sampler. -4. **A/B prompt crossfade** num fader, estilo VJ. - -## Interop opcional - -Se você precisar especificamente de **streaming web/WebRTC** (mandar pra um -navegador ou pra nuvem), dá pra hospedar os mesmos nodes Difforum sob o -[ComfyStream](https://github.com/yondonfu/comfystream); e pra **FPS máximo** num -palco, o [StreamDiffusion](https://github.com/pschroedl/ComfyUI-StreamDiffusion) -+ TensorRT é o pipeline mais rápido. São caminhos complementares - o Live Sampler -nativo cobre o caso "rodar ao vivo no meu ComfyUI" sem nada disso. diff --git a/__init__.py b/__init__.py index b01a964..128b44a 100644 --- a/__init__.py +++ b/__init__.py @@ -1,122 +1,37 @@ """ -Difforum - a modern, Deforum-style animation toolkit for ComfyUI. +Difforum - camera direction and timeline orchestration for ComfyUI. -Phase 1 ships the GPU-free schedule engine (keyframe parser + safe math -evaluator + audio-reactive hooks). Later phases add depth warping, the -feedback sampler (Classic+ mode) and the Wan 2.2 VACE video bridge -(Hybrid mode). See DESIGN.md. +Direct a shot on a visual timeline (scenes, camera moves, energy), render it +with the Deforum-style feedback sampler on any image model, or hand the same +direction to LTX-2 / MiniMax H3 and export the camera to After Effects and +Blender. See README.md. """ -from .nodes.schedule_nodes import ( - NODE_CLASS_MAPPINGS as _SCHED_CLASSES, - NODE_DISPLAY_NAME_MAPPINGS as _SCHED_NAMES, -) -from .nodes.audio_nodes import ( - NODE_CLASS_MAPPINGS as _AUDIO_CLASSES, - NODE_DISPLAY_NAME_MAPPINGS as _AUDIO_NAMES, -) -from .nodes.hybrid_nodes import ( - NODE_CLASS_MAPPINGS as _HYBRID_CLASSES, - NODE_DISPLAY_NAME_MAPPINGS as _HYBRID_NAMES, -) -from .nodes.catalog_nodes import ( - NODE_CLASS_MAPPINGS as _CATALOG_CLASSES, - NODE_DISPLAY_NAME_MAPPINGS as _CATALOG_NAMES, -) -from .nodes.warp_nodes import ( - NODE_CLASS_MAPPINGS as _WARP_CLASSES, - NODE_DISPLAY_NAME_MAPPINGS as _WARP_NAMES, -) -from .nodes.sampler_nodes import ( - NODE_CLASS_MAPPINGS as _SAMPLER_CLASSES, - NODE_DISPLAY_NAME_MAPPINGS as _SAMPLER_NAMES, -) -from .nodes.guide_nodes import ( - NODE_CLASS_MAPPINGS as _GUIDE_CLASSES, - NODE_DISPLAY_NAME_MAPPINGS as _GUIDE_NAMES, -) -from .nodes.prompt_nodes import ( - NODE_CLASS_MAPPINGS as _PROMPT_CLASSES, - NODE_DISPLAY_NAME_MAPPINGS as _PROMPT_NAMES, -) -from .nodes.live_nodes import ( - NODE_CLASS_MAPPINGS as _LIVE_CLASSES, - NODE_DISPLAY_NAME_MAPPINGS as _LIVE_NAMES, -) -from .nodes.effects_nodes import ( - NODE_CLASS_MAPPINGS as _EFFECTS_CLASSES, - NODE_DISPLAY_NAME_MAPPINGS as _EFFECTS_NAMES, -) -from .nodes.look_nodes import ( - NODE_CLASS_MAPPINGS as _LOOK_CLASSES, - NODE_DISPLAY_NAME_MAPPINGS as _LOOK_NAMES, -) -from .nodes.glitch_nodes import ( - NODE_CLASS_MAPPINGS as _GLITCH_CLASSES, - NODE_DISPLAY_NAME_MAPPINGS as _GLITCH_NAMES, -) -from .nodes.video_nodes import ( - NODE_CLASS_MAPPINGS as _VIDEO_CLASSES, - NODE_DISPLAY_NAME_MAPPINGS as _VIDEO_NAMES, -) -from .nodes.director_nodes import ( - NODE_CLASS_MAPPINGS as _DIRECTOR_CLASSES, - NODE_DISPLAY_NAME_MAPPINGS as _DIRECTOR_NAMES, -) -from .nodes.storyboard_nodes import ( - NODE_CLASS_MAPPINGS as _STORYBOARD_CLASSES, - NODE_DISPLAY_NAME_MAPPINGS as _STORYBOARD_NAMES, -) -from .nodes.loop_nodes import ( - NODE_CLASS_MAPPINGS as _LOOP_CLASSES, - NODE_DISPLAY_NAME_MAPPINGS as _LOOP_NAMES, -) -from .nodes.setup_nodes import ( - NODE_CLASS_MAPPINGS as _SETUP_CLASSES, - NODE_DISPLAY_NAME_MAPPINGS as _SETUP_NAMES, -) +import logging -NODE_CLASS_MAPPINGS = {} -NODE_DISPLAY_NAME_MAPPINGS = {} +from .nodes import NODE_CLASS_MAPPINGS as _V1, NODE_DISPLAY_NAME_MAPPINGS as _V1_NAMES -NODE_CLASS_MAPPINGS.update(_SCHED_CLASSES) -NODE_DISPLAY_NAME_MAPPINGS.update(_SCHED_NAMES) -NODE_CLASS_MAPPINGS.update(_AUDIO_CLASSES) -NODE_DISPLAY_NAME_MAPPINGS.update(_AUDIO_NAMES) -NODE_CLASS_MAPPINGS.update(_HYBRID_CLASSES) -NODE_DISPLAY_NAME_MAPPINGS.update(_HYBRID_NAMES) -NODE_CLASS_MAPPINGS.update(_CATALOG_CLASSES) -NODE_DISPLAY_NAME_MAPPINGS.update(_CATALOG_NAMES) -NODE_CLASS_MAPPINGS.update(_WARP_CLASSES) -NODE_DISPLAY_NAME_MAPPINGS.update(_WARP_NAMES) -NODE_CLASS_MAPPINGS.update(_SAMPLER_CLASSES) -NODE_DISPLAY_NAME_MAPPINGS.update(_SAMPLER_NAMES) -NODE_CLASS_MAPPINGS.update(_GUIDE_CLASSES) -NODE_DISPLAY_NAME_MAPPINGS.update(_GUIDE_NAMES) -NODE_CLASS_MAPPINGS.update(_PROMPT_CLASSES) -NODE_DISPLAY_NAME_MAPPINGS.update(_PROMPT_NAMES) -NODE_CLASS_MAPPINGS.update(_LIVE_CLASSES) -NODE_DISPLAY_NAME_MAPPINGS.update(_LIVE_NAMES) -NODE_CLASS_MAPPINGS.update(_EFFECTS_CLASSES) -NODE_DISPLAY_NAME_MAPPINGS.update(_EFFECTS_NAMES) -NODE_CLASS_MAPPINGS.update(_LOOK_CLASSES) -NODE_DISPLAY_NAME_MAPPINGS.update(_LOOK_NAMES) -NODE_CLASS_MAPPINGS.update(_VIDEO_CLASSES) -NODE_DISPLAY_NAME_MAPPINGS.update(_VIDEO_NAMES) -NODE_CLASS_MAPPINGS.update(_GLITCH_CLASSES) -NODE_DISPLAY_NAME_MAPPINGS.update(_GLITCH_NAMES) -NODE_CLASS_MAPPINGS.update(_DIRECTOR_CLASSES) -NODE_DISPLAY_NAME_MAPPINGS.update(_DIRECTOR_NAMES) -NODE_CLASS_MAPPINGS.update(_STORYBOARD_CLASSES) -NODE_DISPLAY_NAME_MAPPINGS.update(_STORYBOARD_NAMES) -NODE_CLASS_MAPPINGS.update(_LOOP_CLASSES) -NODE_DISPLAY_NAME_MAPPINGS.update(_LOOP_NAMES) -NODE_CLASS_MAPPINGS.update(_SETUP_CLASSES) -NODE_DISPLAY_NAME_MAPPINGS.update(_SETUP_NAMES) +log = logging.getLogger("difforum") + +NODE_CLASS_MAPPINGS = dict(_V1) +NODE_DISPLAY_NAME_MAPPINGS = dict(_V1_NAMES) + +try: # 0.x nodes, hidden and deprecated, so old workflows still open + from .legacy import NODE_CLASS_MAPPINGS as _OLD, NODE_DISPLAY_NAME_MAPPINGS as _OLD_NAMES + for _k, _v in _OLD.items(): + NODE_CLASS_MAPPINGS.setdefault(_k, _v) + NODE_DISPLAY_NAME_MAPPINGS.setdefault(_k, _OLD_NAMES[_k]) +except Exception as exc: # never let legacy code block the v1 pack + log.warning("[Difforum] legacy nodes unavailable: %s", exc) + +try: + from .nodes.routes import register_routes + register_routes() +except Exception as exc: + log.warning("[Difforum] timeline preview endpoints unavailable: %s", exc) -# Serve ./js so the Film Director timeline widget loads in the browser. WEB_DIRECTORY = "./js" __all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"] -print(f"[Difforum] loaded {len(NODE_CLASS_MAPPINGS)} nodes") +print(f"[Difforum] {len(_V1)} nodes ready (+{len(NODE_CLASS_MAPPINGS) - len(_V1)} legacy)") diff --git a/conftest.py b/conftest.py deleted file mode 100644 index 79aec0c..0000000 --- a/conftest.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Test bootstrap: make the pack importable as `difforum` whatever the folder is called. - -`tests/test_integration.py` and `tests/test_prompt.py` import node modules as -`difforum.nodes.*` on purpose: those tests inject a stub top-level `nodes` -module (standing in for ComfyUI's own), so importing the pack's internal -`nodes` subpackage by bare name would collide with the stub. - -The catch is the folder name. A `git clone` produces `Difforum`; a Comfy -Registry install produces `difforum`. Python matches module names -case-sensitively even on macOS, where the filesystem does not - so hardcoding -either spelling breaks the suite on the other layout, and on Linux CI. - -This binds the name to whatever the directory actually is, via a module whose -`__path__` points at the pack. Importing the package's own `__init__` is -deliberately avoided: it pulls in every node module, which needs `comfy` at -import time and is not available under bare pytest. -""" - -import sys -import types -from pathlib import Path - -_PACK = Path(__file__).resolve().parent - -# `from core.x import ...` / `from nodes.x import ...` - how most tests import. -if str(_PACK) not in sys.path: - sys.path.insert(0, str(_PACK)) - -# `from difforum.nodes.x import ...` - the package-qualified form. -if "difforum" not in sys.modules: - _alias = types.ModuleType("difforum") - _alias.__path__ = [str(_PACK)] - sys.modules["difforum"] = _alias diff --git a/core/camera.py b/core/camera.py index d7325c7..f922eb2 100644 --- a/core/camera.py +++ b/core/camera.py @@ -53,6 +53,9 @@ class CameraTrack: zoom: list[float] # per-frame 2D scale factor fov: list[float] # per-frame field of view (degrees) mode: str = "3d" + # a 3D track the user asked to render flat (2d): renderers use the + # pseudo-3D affine even when a depth map is connected + flat: bool = False def __len__(self) -> int: return len(self.deltas) @@ -102,7 +105,9 @@ def col(name: str, default: float) -> list[float]: d = _transform(tx[f], ty[f], 0.0, 0.0, 0.0, rz[f]) else: d = _transform(tx[f], ty[f], tz[f], rx[f], ry[f], rz[f]) - acc = acc @ d + # a delta acts on the previous frame's view (P_f = d_f @ P_{f-1}), so + # the accumulated pose composes on the left + acc = d @ acc deltas.append(d) poses.append(acc.copy()) diff --git a/core/camera_keys.py b/core/camera_keys.py index fbbbb64..0c8501b 100644 --- a/core/camera_keys.py +++ b/core/camera_keys.py @@ -111,7 +111,27 @@ def keys_to_axis_values( Returns (axis values, fov per frame, human summary). """ - keys = parse_camera_keys(text) + return keys_list_to_axis_values(parse_camera_keys(text), max_frames, fps, + extra_vars=extra_vars, blend=blend) + + +def keys_list_to_axis_values( + keys: list[dict], + max_frames: int, + fps: float = 24.0, + extra_vars=None, + blend: float = 1.0, +) -> tuple[dict[str, list[float]], list[float], str]: + """Same as `keys_to_axis_values`, from already-parsed key dicts + (frame, move, speed, intensity, lens, ease) - the timeline UI path.""" + keys = sorted(keys, key=lambda k: k["frame"]) + if not keys: + keys = [{"frame": 0, "move": "still", "speed": 1.0, "intensity": 1.0, + "lens": DEFAULT_LENS, "ease": "ease_in_out"}] + if keys[0]["frame"] != 0: + first = dict(keys[0]) + first.update(frame=0, move="still", speed=1.0, intensity=1.0) + keys.insert(0, first) n = max(1, int(max_frames)) values: dict[str, list[float]] = {ax: [0.0] * n for ax in _AXES} diff --git a/core/camera_presets.py b/core/camera_presets.py index d6c82d2..ce8caea 100644 --- a/core/camera_presets.py +++ b/core/camera_presets.py @@ -14,8 +14,38 @@ "pan_left", "pan_right", "pan_up", "pan_down", "orbit_left", "orbit_right", "roll_cw", "roll_ccw", "spiral", "sway", "dolly_zoom", "rise", "shake", + "tilt_up", "tilt_down", "crane_up", "handheld", "drift", "breathe", "vortex", ) +# UI metadata for the Director timeline: label, group, one-line description. +MOVE_INFO = { + "still": ("Still", "basic", "Locked-off frame; the diffusion still evolves."), + "zoom_in": ("Zoom in", "basic", "Lens magnifies toward the centre."), + "zoom_out": ("Zoom out", "basic", "Lens pulls back from the centre."), + "pan_left": ("Pan left", "basic", "Frame slides left."), + "pan_right": ("Pan right", "basic", "Frame slides right."), + "pan_up": ("Pan up", "basic", "Frame slides up."), + "pan_down": ("Pan down", "basic", "Frame slides down."), + "roll_cw": ("Roll CW", "basic", "Rotates clockwise around the view axis."), + "roll_ccw": ("Roll CCW", "basic", "Rotates counter-clockwise."), + "dolly_in": ("Dolly in", "space", "Camera travels forward into the scene (parallax with depth)."), + "dolly_out": ("Dolly out", "space", "Camera travels backward."), + "orbit_left": ("Orbit left", "space", "Camera circles the subject to the left."), + "orbit_right": ("Orbit right", "space", "Camera circles the subject to the right."), + "tilt_up": ("Tilt up", "space", "Camera pivots upward."), + "tilt_down": ("Tilt down", "space", "Camera pivots downward."), + "rise": ("Rise", "space", "Camera lifts while easing forward."), + "crane_up": ("Crane up", "space", "Rises and tilts down to keep the subject."), + "dolly_zoom": ("Dolly zoom", "space", "Vertigo: push in while the lens widens."), + "spiral": ("Spiral", "fx", "Roll plus zoom - the classic Deforum tunnel."), + "vortex": ("Vortex", "fx", "Roll while pulling back - an outward spiral."), + "sway": ("Sway", "fx", "Slow side-to-side yaw."), + "breathe": ("Breathe", "fx", "Zoom gently pulses in and out."), + "drift": ("Drift", "fx", "Lazy wandering float, no fixed direction."), + "handheld": ("Handheld", "fx", "Small organic operator shake."), + "shake": ("Shake", "fx", "Hard jitter, for impacts."), +} + _AXES = ( "translation_x", "translation_y", "translation_z", "rotation_3d_x", "rotation_3d_y", "rotation_3d_z", "zoom", @@ -108,5 +138,29 @@ def osc(amp, period): # oscillator around 0 elif preset == "shake": out["translation_x"] = osc(2.5 * i, 6.0 / s) out["translation_y"] = osc(2.0 * i, 5.0 / s) + elif preset == "tilt_up": + out["rotation_3d_x"] = c(0.4 * i * s) + elif preset == "tilt_down": + out["rotation_3d_x"] = c(-0.4 * i * s) + elif preset == "crane_up": + out["translation_y"] = c(-1.6 * i * s) + out["rotation_3d_x"] = c(-0.15 * i * s) + elif preset == "handheld": + p1, p2, p3 = 37.0 / s, 53.0 / s, 71.0 / s + out["translation_x"] = (f"0:({0.6 * i:.4g}*sin(2*pi*t/{p1:.4g})" + f"+{0.35 * i:.4g}*sin(2*pi*t/{p3:.4g}+1.3))") + out["translation_y"] = f"0:({0.45 * i:.4g}*sin(2*pi*t/{p2:.4g}+0.7))" + out["rotation_3d_z"] = f"0:({0.08 * i:.4g}*sin(2*pi*t/{p3:.4g}+2.1))" + elif preset == "drift": + p1, p2 = 180.0 / s, 240.0 / s + out["translation_x"] = f"0:({0.9 * i:.4g}*sin(2*pi*t/{p1:.4g}))" + out["translation_y"] = f"0:({0.6 * i:.4g}*sin(2*pi*t/{p2:.4g}+1.1))" + out["rotation_3d_z"] = f"0:({0.1 * i:.4g}*sin(2*pi*t/{p2:.4g}+0.4))" + elif preset == "breathe": + p = max(2.0, 96.0 / s) + out["zoom"] = f"0:(1.0 + {0.006 * i:.4g}*sin(2*pi*t/{p:.4g}))" + elif preset == "vortex": + out["rotation_3d_z"] = c(-0.6 * i * s) + out["zoom"] = c(1.0 - 0.01 * i * s) return out diff --git a/core/direction.py b/core/direction.py new file mode 100644 index 0000000..fda5228 --- /dev/null +++ b/core/direction.py @@ -0,0 +1,534 @@ +""" +Timeline direction: the data model behind the Director's visual timeline. + +A timeline has independent tracks, so camera cuts do not have to line up with +prompt changes: + + scenes [{start, prompt, mood}] what is on screen + camera [{start, move, speed, intensity, lens, ease, react}] the camera + energy [[frame, strength], ...] denoise envelope + guidance [[frame, cfg], ...] (optional) cfg envelope + +`build_direction` turns it into everything a renderer needs - camera axis +curves, lens, strength and cfg per frame, prompt keyframes and a plain-language +camera description - with no ComfyUI or torch dependency, so it is fully +testable and can back a live preview endpoint. + +The 0.x Film Director format (a JSON list of scenes that each carried a camera) +is still accepted and converted. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field + +from .camera_keys import DEFAULT_LENS, EASINGS, keys_list_to_axis_values +from .camera_presets import CAMERA_PRESETS, MOVE_INFO, needs_depth + +MOODS = { + # denoise speed amplitude lens ease + "calm": {"strength": 0.44, "speed": 0.6, "intensity": 0.5, "lens": 46.0, "ease": "ease_in_out"}, + "build": {"strength": 0.52, "speed": 1.0, "intensity": 0.8, "lens": 40.0, "ease": "ease_in_out"}, + "tense": {"strength": 0.57, "speed": 1.2, "intensity": 1.0, "lens": 32.0, "ease": "ease_in"}, + "climax": {"strength": 0.63, "speed": 1.6, "intensity": 1.3, "lens": 62.0, "ease": "ease_in"}, + "resolve": {"strength": 0.44, "speed": 0.5, "intensity": 0.45, "lens": 44.0, "ease": "ease_out"}, + "dream": {"strength": 0.48, "speed": 0.4, "intensity": 0.6, "lens": 52.0, "ease": "ease_in_out"}, +} + +MOOD_COLORS = { + "calm": "#4a90d9", "build": "#e0a020", "tense": "#e2703a", + "climax": "#a45ec4", "resolve": "#37a86b", "dream": "#1f9c8c", +} + +# audio reaction per camera block (applied when an audio analysis is connected) +REACTIONS = { + "none": "No audio reaction", + "beat_pulse": "Zoom punches on every beat", + "onset_shake": "Shakes on transients", + "bass_speed": "Bass drives the move's speed", + "mid_sway": "Mids rock the roll", +} + +STYLES = { + "cinematic": {"color_coherence": 0.85, "color_mode": "lab", "sharpen": 0.25, "noise": 0.025}, + "documentary": {"color_coherence": 0.90, "color_mode": "lab", "sharpen": 0.15, "noise": 0.015}, + "music_video": {"color_coherence": 0.75, "color_mode": "lab", "sharpen": 0.35, "noise": 0.035}, + "psychedelic": {"color_coherence": 0.60, "color_mode": "rgb", "sharpen": 0.20, "noise": 0.045}, +} + + +def default_timeline(frames: int = 120) -> dict: + """A safe starting timeline: only moves that work without a depth map.""" + q = max(1, frames // 4) + return { + "version": 2, + "scenes": [ + {"start": 0, "mood": "calm", "prompt": "misty ancient forest at dawn, volumetric light, wide shot"}, + {"start": q, "mood": "build", "prompt": "glowing roots and moss, bioluminescent details, macro"}, + {"start": 2 * q, "mood": "climax", "prompt": "explosion of light, cosmic transformation, high contrast"}, + {"start": 3 * q, "mood": "resolve", "prompt": "embers becoming stars, calm, vast cosmic scale"}, + ], + "camera": [ + {"start": 0, "move": "zoom_in", "speed": 0.8, "intensity": 0.8, "ease": "ease_in_out"}, + {"start": q, "move": "pan_right", "speed": 1.0, "intensity": 0.8, "ease": "ease_in_out"}, + {"start": 2 * q, "move": "spiral", "speed": 1.4, "intensity": 1.1, "ease": "ease_in"}, + {"start": 3 * q, "move": "zoom_out", "speed": 0.6, "intensity": 0.5, "ease": "ease_out"}, + ], + "energy": [], + "guidance": [], + } + + +@dataclass +class Direction: + frames: int + fps: float + mode: str + axes: dict[str, list[float]] + lens: list[float] + strength: list[float] + cfg: list[float] | None + prompts: list[tuple[int, str]] + scenes: list[dict] + camera_blocks: list[dict] + camera_text: str + warnings: list[str] = field(default_factory=list) + summary: list[str] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# parsing +# --------------------------------------------------------------------------- + +def _num(v, default): + try: + return float(v) + except (TypeError, ValueError): + return default + + +def parse_timeline(raw) -> dict: + """Timeline JSON (v2 dict or 0.x scene list, or a `frame | mood | move | + prompt` table) -> normalised v2 dict.""" + if isinstance(raw, str): + text = raw.strip() + if not text: + return {"version": 2, "scenes": [], "camera": [], "energy": [], "guidance": []} + try: + raw = json.loads(text) + except json.JSONDecodeError: + raw = _parse_table(text) + + if isinstance(raw, list): # 0.x format + scenes, camera = [], [] + for item in raw: + if not isinstance(item, dict): + continue + start = int(_num(item.get("frame_start", item.get("start", 0)), 0)) + mood = str(item.get("mood", "calm")).lower() + scenes.append({"start": start, "mood": mood, "prompt": str(item.get("prompt", ""))}) + m = MOODS.get(mood, MOODS["calm"]) + camera.append({ + "start": start, "move": str(item.get("camera", "still")).lower(), + "speed": m["speed"], "intensity": m["intensity"], + "lens": _num(item.get("lens"), 0.0) or 0.0, + "ease": item.get("ease") or m["ease"], + }) + raw = {"version": 2, "scenes": scenes, "camera": camera, "energy": [], "guidance": []} + + if not isinstance(raw, dict): + raw = {} + + scenes = [] + for s in raw.get("scenes", []) or []: + if isinstance(s, dict): + scenes.append({ + "start": max(0, int(_num(s.get("start", 0), 0))), + "mood": str(s.get("mood", "calm")).lower(), + "prompt": " ".join(str(s.get("prompt", "")).split()), + }) + camera = [] + for c in raw.get("camera", []) or []: + if isinstance(c, dict): + move = str(c.get("move", "still")).lower() + camera.append({ + "start": max(0, int(_num(c.get("start", 0), 0))), + "move": move if move in CAMERA_PRESETS else "still", + "speed": _num(c.get("speed"), 1.0), + "intensity": _num(c.get("intensity"), 1.0), + "lens": _num(c.get("lens"), 0.0), + "ease": c.get("ease") if c.get("ease") in EASINGS else "ease_in_out", + "react": c.get("react") if c.get("react") in REACTIONS else "none", + }) + + def env(key): + pts = [] + for p in raw.get(key, []) or []: + if isinstance(p, (list, tuple)) and len(p) >= 2: + pts.append((max(0, int(_num(p[0], 0))), _num(p[1], 0.5))) + elif isinstance(p, dict): + pts.append((max(0, int(_num(p.get("frame", 0), 0))), _num(p.get("value"), 0.5))) + return sorted(pts) + + return { + "version": 2, + "scenes": sorted(scenes, key=lambda s: s["start"]), + "camera": sorted(camera, key=lambda c: c["start"]), + "energy": env("energy"), + "guidance": env("guidance"), + } + + +def _parse_table(text: str) -> list[dict]: + out = [] + for line in text.splitlines(): + line = line.split("#", 1)[0].strip() + parts = [p.strip() for p in line.split("|")] + if len(parts) < 2: + continue + try: + frame = int(parts[0]) + except ValueError: + continue + out.append({"frame_start": frame, "mood": parts[1], + "camera": parts[2] if len(parts) > 2 else "still", + "prompt": parts[3] if len(parts) > 3 else ""}) + return out + + +# --------------------------------------------------------------------------- +# building +# --------------------------------------------------------------------------- + +def _jitter(seed: int, index: int, channel: str) -> float: + h = hashlib.sha1(f"{seed}:{index}:{channel}".encode()).digest() + return (int.from_bytes(h[:4], "big") / 0xFFFFFFFF) * 2.0 - 1.0 + + +def _envelope(points: list[tuple[int, float]], n: int, smooth: bool = True) -> list[float]: + """Piecewise curve through (frame, value) points, held flat at the ends.""" + if not points: + return [] + pts = sorted(points) + out = [] + j = 0 + for f in range(n): + while j + 1 < len(pts) and pts[j + 1][0] <= f: + j += 1 + f0, v0 = pts[j] + if f <= pts[0][0]: + out.append(pts[0][1]) + continue + if j + 1 >= len(pts): + out.append(v0) + continue + f1, v1 = pts[j + 1] + u = (f - f0) / max(1, f1 - f0) + if smooth: + u = u * u * (3.0 - 2.0 * u) + out.append(v0 + (v1 - v0) * u) + return out + + +def mood_energy(scenes: list[dict], strength_bias: float = 0.0) -> list[tuple[int, float]]: + """Energy points implied by the scene moods (what the 'auto' button draws).""" + pts = [] + for s in scenes: + m = MOODS.get(s["mood"], MOODS["calm"]) + pts.append((s["start"], round(min(0.95, max(0.05, m["strength"] + strength_bias)), 3))) + return pts + + +def _react(axes: dict, blocks: list[dict], n: int, curves: dict | None, warnings: list[str]): + if not any(b.get("react", "none") != "none" for b in blocks): + return + if not curves: + warnings.append("audio reactions are set on camera blocks but no audio is connected") + return + + def curve(name): + c = list(curves.get(name, [])) + return (c + [c[-1] if c else 0.0] * n)[:n] if c else [0.0] * n + + beat, onset, low, mid = curve("beat"), curve("onset"), curve("low"), curve("mid") + for i, b in enumerate(blocks): + kind = b.get("react", "none") + if kind == "none": + continue + start = min(b["start"], n) + end = min(blocks[i + 1]["start"], n) if i + 1 < len(blocks) else n + amt = float(b.get("intensity", 1.0)) + for f in range(start, end): + if kind == "beat_pulse": + axes["zoom"][f] *= 1.0 + 0.035 * amt * beat[f] + elif kind == "onset_shake": + sign = 1.0 if (f % 2) else -1.0 + axes["translation_x"][f] += sign * 4.0 * amt * onset[f] + axes["translation_y"][f] -= sign * 3.0 * amt * onset[f] + elif kind == "bass_speed": + k = 0.3 + 1.4 * low[f] + for ax in ("translation_x", "translation_y", "translation_z", + "rotation_3d_x", "rotation_3d_y", "rotation_3d_z"): + axes[ax][f] *= k + axes["zoom"][f] = 1.0 + (axes["zoom"][f] - 1.0) * k + elif kind == "mid_sway": + axes["rotation_3d_z"][f] += 0.6 * amt * (mid[f] - 0.5) + + +def build_direction( + timeline, + frames: int, + fps: float = 24.0, + mode: str = "2d", + camera_scale: float = 1.0, + strength_bias: float = 0.0, + blend: float = 1.0, + variation: float = 0.0, + variation_seed: int = 0, + audio_curves: dict | None = None, + has_depth: bool | None = None, +) -> Direction: + tl = parse_timeline(timeline) + n = max(1, int(frames)) + fps = float(fps) + warnings: list[str] = [] + var = max(0.0, min(1.0, float(variation))) + + scenes = [s for s in tl["scenes"] if s["start"] < n] + blocks = [b for b in tl["camera"] if b["start"] < n] + dropped = len(tl["scenes"]) - len(scenes) + len(tl["camera"]) - len(blocks) + if dropped: + warnings.append(f"{dropped} block(s) start after the clip ends ({n} frames) and were ignored") + if not blocks: + blocks = [{"start": 0, "move": "still", "speed": 1.0, "intensity": 1.0, + "lens": 0.0, "ease": "ease_in_out", "react": "none"}] + + def mood_at(frame): + cur = "calm" + for s in scenes: + if s["start"] <= frame: + cur = s["mood"] + return MOODS.get(cur, MOODS["calm"]) + + keys = [] + for i, b in enumerate(blocks): + m = mood_at(b["start"]) + js = 1.0 + var * 0.35 * _jitter(variation_seed, i, "speed") + ja = 1.0 + var * 0.35 * _jitter(variation_seed, i, "intensity") + jl = var * 12.0 * _jitter(variation_seed, i, "lens") + lens = b["lens"] if b["lens"] and b["lens"] > 0 else m["lens"] + keys.append({ + "frame": b["start"], "move": b["move"], + "speed": round(b["speed"] * camera_scale * js, 4), + "intensity": round(b["intensity"] * camera_scale * ja, 4), + "lens": round(min(140.0, max(8.0, lens + jl)), 2), + "ease": b["ease"], + }) + axes, lens, _summary = keys_list_to_axis_values(keys, n, fps, extra_vars=audio_curves, blend=blend) + _react(axes, blocks, n, audio_curves, warnings) + + depth_moves = sorted({MOVE_INFO[b["move"]][0] for b in blocks if needs_depth(b["move"])}) + if depth_moves and (mode == "2d"): + warnings.append( + f"{', '.join(depth_moves)}: 3D moves are approximated in 2d mode " + "(dolly -> zoom, orbit -> pan). Set camera_mode 3d + a depth map for real parallax.") + elif depth_moves and has_depth is False: + warnings.append( + f"{', '.join(depth_moves)}: pseudo-3D until a depth map reaches the sampler.") + + energy_pts = tl["energy"] or mood_energy(scenes, strength_bias) + if tl["energy"] and strength_bias: + energy_pts = [(f, min(0.95, max(0.05, v + strength_bias))) for f, v in energy_pts] + strength = _envelope(energy_pts, n) or [0.5] * n + if var: + strength = [min(0.95, max(0.05, v + var * 0.06 * _jitter(variation_seed, 0, "strength"))) + for v in strength] + cfg = _envelope(tl["guidance"], n) if tl["guidance"] else None + + prompts = [(s["start"], s["prompt"]) for s in scenes if s["prompt"]] + if not prompts: + prompts = [(0, "")] + elif prompts[0][0] != 0: + prompts.insert(0, (0, prompts[0][1])) + + summary = [] + for s in scenes: + summary.append(f" scene {s['start'] / fps:6.2f}s {s['mood']:<8} {s['prompt'][:60]}") + for b, k in zip(blocks, keys): + summary.append( + f" camera {b['start'] / fps:5.2f}s {MOVE_INFO[b['move']][0]:<11} " + f"x{k['speed']:<5g} amp {k['intensity']:<5g} lens {k['lens']:g}deg {k['ease']}" + + (f" react:{b['react']}" if b.get("react", "none") != "none" else "")) + + return Direction( + frames=n, fps=fps, mode=mode, axes=axes, lens=lens, strength=strength, cfg=cfg, + prompts=prompts, scenes=scenes, camera_blocks=blocks, + camera_text=describe_blocks(blocks, n, fps), warnings=warnings, summary=summary, + ) + + +# --------------------------------------------------------------------------- +# camera -> words (for prompt-driven video models: H3, LTX, Seedance...) +# --------------------------------------------------------------------------- + +_PHRASE = { + "still": "holds a locked-off frame", + "zoom_in": "zooms in", "zoom_out": "zooms out", + "dolly_in": "pushes in", "dolly_out": "pulls back", + "pan_left": "pans left", "pan_right": "pans right", + "pan_up": "tilts up", "pan_down": "tilts down", + "tilt_up": "tilts up", "tilt_down": "tilts down", + "orbit_left": "orbits left around the subject", "orbit_right": "orbits right around the subject", + "roll_cw": "rolls clockwise", "roll_ccw": "rolls counter-clockwise", + "spiral": "spirals inward, rolling as it pushes in", + "vortex": "spirals outward, rolling as it pulls back", + "sway": "sways gently side to side", + "dolly_zoom": "performs a dolly zoom (vertigo effect)", + "rise": "rises smoothly", "crane_up": "cranes up, tilting down to keep the subject", + "shake": "shakes hard", "handheld": "floats with a subtle handheld feel", + "drift": "drifts slowly and weightlessly", "breathe": "breathes with a gentle push and pull", +} + + +def _speed_word(speed: float, intensity: float) -> str: + e = float(speed) * float(intensity) + if e < 0.45: + return "very slowly" + if e < 0.8: + return "slowly" + if e < 1.3: + return "" + if e < 2.0: + return "quickly" + return "rapidly" + + +def describe_blocks(blocks: list[dict], frames: int, fps: float) -> str: + """One flowing sentence of camera direction, in shot order.""" + parts = [] + for i, b in enumerate(blocks): + phrase = _PHRASE.get(b["move"], b["move"].replace("_", " ")) + sw = _speed_word(b.get("speed", 1.0), b.get("intensity", 1.0)) + if sw and b["move"] not in ("still", "shake", "handheld"): + phrase = f"{phrase} {sw}" + if i == 0: + parts.append(f"The camera {phrase}") + elif b.get("ease") == "step": + parts.append(f"then cuts and {phrase}") + elif i == len(blocks) - 1 and i > 1: + parts.append(f"and finally {phrase}") + else: + parts.append(f"then {phrase}") + return (", ".join(parts) + ".") if parts else "The camera holds a locked-off frame." + + +def describe_timed(blocks: list[dict], frames: int, fps: float) -> str: + lines = [] + for i, b in enumerate(blocks): + end = blocks[i + 1]["start"] if i + 1 < len(blocks) else frames + phrase = _PHRASE.get(b["move"], b["move"]) + sw = _speed_word(b.get("speed", 1.0), b.get("intensity", 1.0)) + lines.append(f"[{b['start'] / fps:.1f}s-{end / fps:.1f}s] camera {phrase}" + + (f" {sw}" if sw else "")) + return "\n".join(lines) + + +def ui_catalog() -> dict: + """Everything the timeline widget needs to draw its pickers - served to the + frontend so the move list lives in exactly one place (Python).""" + return { + "moves": [ + {"id": k, "label": MOVE_INFO[k][0], "group": MOVE_INFO[k][1], + "hint": MOVE_INFO[k][2], "depth": needs_depth(k)} + for k in CAMERA_PRESETS + ], + "moods": [{"id": k, "color": MOOD_COLORS.get(k, "#888"), **v} for k, v in MOODS.items()], + "easings": list(EASINGS), + "reactions": [{"id": k, "hint": v} for k, v in REACTIONS.items()], + "styles": list(STYLES), + "default_lens": DEFAULT_LENS, + } + + +@dataclass +class DirectionBundle: + """What travels on a DIFFORUM_DIRECTION wire: one object instead of five.""" + + params: dict + camera: object # core.camera.CameraTrack + strength: object # core.schedule.Schedule + cfg: object | None # core.schedule.Schedule | None + prompts: object | None # core.prompt.PromptTrack | None + direction: Direction + style: str = "cinematic" + + @property + def look(self) -> dict: + return STYLES.get(self.style, STYLES["cinematic"]) + + +def blocks_in_range(blocks: list[dict], start: int, end: int) -> list[dict]: + """Camera blocks overlapping [start, end), re-timed so the range starts at 0.""" + out = [] + for i, b in enumerate(blocks): + b_end = blocks[i + 1]["start"] if i + 1 < len(blocks) else 10**9 + if b_end <= start or b["start"] >= end: + continue + nb = dict(b) + nb["start"] = max(0, b["start"] - start) + out.append(nb) + return out + + +# normalisers: per-frame magnitude that reads as a "normal speed" move +_TRACK_AXES = ( + ("zoom", 0.012, "zoom_in", "zoom_out"), + ("tz", 1.0, "dolly_out", "dolly_in"), + ("ry", 0.5, "orbit_right", "orbit_left"), + ("rx", 0.35, "tilt_up", "tilt_down"), + ("rz", 0.45, "roll_cw", "roll_ccw"), + ("tx", 1.8, "pan_right", "pan_left"), + ("ty", 1.8, "pan_down", "pan_up"), +) + + +def describe_track(camera, fps: float, window_seconds: float = 1.0) -> tuple[str, str]: + """Plain-language description of any camera track (not only Director + blocks): windows of ~1 s are classified by their dominant motion and runs + of the same move are merged. Returns (sentence, timed lines).""" + import math + + n = len(camera.deltas) + win = max(1, int(round(window_seconds * fps))) + blocks: list[dict] = [] + for s in range(0, n, win): + e = min(n, s + win) + acc = {k: 0.0 for k, *_ in _TRACK_AXES} + for f in range(s, e): + d = camera.deltas[f] + acc["zoom"] += math.log(max(1e-6, float(camera.zoom[f]))) + acc["tz"] += float(d[2][3]) + acc["tx"] += float(d[0][3]) + acc["ty"] += float(d[1][3]) + acc["rz"] += math.degrees(math.atan2(float(d[1][0]), float(d[0][0]))) + acc["ry"] += math.degrees(math.asin(max(-1.0, min(1.0, float(d[0][2]))))) + acc["rx"] += math.degrees(math.atan2(-float(d[1][2]), float(d[2][2]))) + L = e - s + best, best_v = "still", 0.25 + speed = 1.0 + for key, norm, pos, neg in _TRACK_AXES: + if camera.mode == "2d" and key in ("tz", "ry", "rx"): + continue + v = acc[key] / L / norm + if abs(v) > best_v: + best, best_v, speed = (pos if v > 0 else neg), abs(v), abs(v) + zv = acc["zoom"] / L / 0.012 + if best in ("roll_cw", "roll_ccw") and abs(zv) > 0.5: + best = "spiral" if zv > 0 else "vortex" + if blocks and blocks[-1]["move"] == best: + continue + blocks.append({"start": s, "move": best, "speed": speed, "intensity": 1.0, + "ease": "ease_in_out"}) + return describe_blocks(blocks, n, fps), describe_timed(blocks, n, fps) diff --git a/core/engine.py b/core/engine.py new file mode 100644 index 0000000..0b5a755 --- /dev/null +++ b/core/engine.py @@ -0,0 +1,387 @@ +""" +The Difforum feedback engine - one implementation shared by the Feedback +Sampler, the Live Sampler and the Storyboard. + +Per frame: warp (camera) -> symmetry -> detail guard -> [diffuse] -> colour. + +What it fixes over the 0.x loop: + +* **Depth follows the image.** The depth map is re-projected with every 3D + warp, so parallax stays attached to what is on screen instead of to frame 0. +* **No silent freeze without depth.** A 3D camera with no depth map runs a + pseudo-3D affine (dolly -> zoom, orbit/pan -> shift, roll -> roll) and the + run report says so. +* **Pans agree between 2D and 3D.** Lateral 3D translation is measured in + pixels at the reference depth, the same unit the 2D warp uses. +* **Cadence without pops.** In-between frames are a crossfade of the previous + key warped forward and the next key warped *back*, the way Deforum does it. +* **Holes get repainted.** The occlusion mask from the warp gets extra noise, so + the sampler invents new content where the camera revealed the unknown. +* **Colour anchor modes.** `first` (0.x behaviour), `scene` (re-anchors at every + prompt keyframe and lets go during the transition), `rolling` (previous key) + or `none`. + +The engine is torch-only and knows nothing about ComfyUI: diffusion is a +callback, so the whole loop is testable with a stub. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass, field +from typing import Callable, Iterator + +import torch + +from .color import match_color +from .detail import add_noise, sharpen +from .fx import FxRunner +from .symmetry import apply_symmetry +from .warp import ( + _focal, affine_2d, pseudo_3d_params, warp_2d, warp_3d, warp_affine, +) + +ANCHOR_MODES = ("scene", "first", "rolling", "none") +DEPTH_TRACKING = ("follow", "static") +BORDER_MODES = ("reflection", "border", "zeros") + + +def resize_bhwc(image: torch.Tensor, width: int, height: int) -> torch.Tensor: + """[H,W,C] or [B,H,W,C] -> [B,height,width,C] (bilinear).""" + if image.dim() == 3: + image = image.unsqueeze(0) + _b, h, w, _c = image.shape + if h == height and w == width: + return image + chw = image.permute(0, 3, 1, 2) + chw = torch.nn.functional.interpolate( + chw, size=(height, width), mode="bilinear", align_corners=False) + return chw.permute(0, 2, 3, 1) + + +def depth_to_single(depth: torch.Tensor, width: int, height: int) -> torch.Tensor: + """Any IMAGE/MASK-ish depth -> [B,H,W] in 0..1 at the render size.""" + d = depth + if d.dim() == 2: + d = d.unsqueeze(0) + if d.dim() == 3: # MASK [B,H,W] + d = d.unsqueeze(-1) + d = resize_bhwc(d, width, height) + if d.shape[-1] >= 3: + d = d[..., :3].mean(dim=-1) + else: + d = d[..., 0] + return d.clamp(0.0, 1.0) + + +def z_angle_deg(delta) -> float: + return math.degrees(math.atan2(float(delta[1, 0]), float(delta[0, 0]))) + + +@dataclass +class EngineConfig: + width: int + height: int + border: str = "reflection" + symmetry: str = "none" + symmetry_segments: int = 6 + sharpen: float = 0.2 + noise: float = 0.02 + hole_noise: float = 0.25 # extra noise where the warp revealed holes + cadence: int = 1 + color_coherence: float = 0.8 + color_mode: str = "lab" + anchor_mode: str = "scene" + near: float = 1.0 + far: float = 100.0 + invert_depth: bool = False + translation_scale: float = 1.0 + depth_tracking: str = "follow" + reference_depth: float = 0.5 # where pseudo-3D / lateral units are measured + seed: int = 0 + + +@dataclass +class RunReport: + mode: str = "2d" # what actually ran: 2d / 3d / pseudo3d + keys: int = 0 + tweens: int = 0 + notes: list[str] = field(default_factory=list) + + def text(self) -> str: + head = f"engine: {self.mode} keys={self.keys} tweens={self.tweens}" + return "\n".join([head, *[f" {n}" for n in self.notes]]) + + +class FeedbackEngine: + """Stateful camera warp chain. `step()` advances one frame.""" + + def __init__(self, camera, cfg: EngineConfig, depth: torch.Tensor | None = None, + fx: FxRunner | None = None): + self.camera = camera + self.cfg = cfg + self.fx = fx or FxRunner() + self.report = RunReport() + self.depth_log: dict[int, torch.Tensor] = {} + self.depth_seq = None # per-frame depth batch (video depth) + self.depth = None # current tracked depth [1,H,W] + if depth is not None: + d = depth_to_single(depth, cfg.width, cfg.height) + if d.shape[0] > 1: + self.depth_seq = d + self.depth = d[:1] + else: + self.depth = d + self.use_depth = camera.mode == "3d" and self.depth is not None and not camera.flat + if camera.mode == "3d": + self.report.mode = "3d" if self.use_depth else "pseudo3d" + if self.depth is None and not camera.flat: + self.report.notes.append( + "3D camera without a depth map: ran the pseudo-3D fallback " + "(dolly -> zoom, orbit -> pan). Connect a depth map for real parallax." + ) + else: + self.report.mode = "2d" + + # -- geometry ------------------------------------------------------------ + + def _lateral_scale(self, fov: float) -> float: + """3D x/y translation units -> pixels at the reference depth.""" + c = self.cfg + z_ref = c.near + (1.0 - c.reference_depth) * (c.far - c.near) + return z_ref / _focal(c.width, fov) + + def _delta3d(self, f: int, fov: float) -> torch.Tensor: + d = torch.as_tensor(self.camera.deltas[f], dtype=torch.float32).clone() + lat = self._lateral_scale(fov) + d[0, 3] *= lat + d[1, 3] *= lat + return d + + def _affine_for(self, f: int) -> torch.Tensor: + """3x3 pixel-space map for frame f in 2D / pseudo-3D.""" + c = self.cfg + zoom = float(self.camera.zoom[f]) + fov = float(self.camera.fov[f]) + if self.camera.mode == "3d": + tx, ty, ang, z = pseudo_3d_params( + self._delta3d(f, fov), fov, c.width, c.near, c.far, + c.translation_scale, c.reference_depth) + zoom = zoom * z + else: + delta = self.camera.deltas[f] + tx, ty, ang = float(delta[0][3]), float(delta[1][3]), z_angle_deg(delta) + return affine_2d(tx, ty, ang, zoom, c.width, c.height) + + def _depth_for(self, f: int): + if self.depth_seq is not None: + return self.depth_seq[min(f, self.depth_seq.shape[0] - 1)].unsqueeze(0) + return self.depth + + def warp(self, img: torch.Tensor, f: int, track_depth: bool = True): + """Warp one frame forward by the camera step at frame f. + + Returns (image, hole_mask) where hole_mask is 1 where the warp revealed + pixels it could not fill.""" + c = self.cfg + fov = float(self.camera.fov[f]) + depth = self._depth_for(f) + if self.use_depth: + out, m, new_d = warp_3d( + img, depth.to(img.device), self._delta3d(f, fov), fov_deg=fov, + near=c.near, far=c.far, invert_depth=c.invert_depth, + translation_scale=c.translation_scale, return_depth=True) + zoom = float(self.camera.zoom[f]) + if abs(zoom - 1.0) > 1e-6: # 3D mode still honours the zoom channel + out, m2 = warp_2d(out, 0.0, 0.0, 0.0, zoom, padding_mode=c.border) + m = m * m2 + new_d, _ = warp_2d(new_d.unsqueeze(-1), 0.0, 0.0, 0.0, zoom, + padding_mode="border") + new_d = new_d[..., 0] + if track_depth and c.depth_tracking == "follow" and self.depth_seq is None: + self.depth = new_d.detach().to("cpu") + if track_depth: + self.depth_log[f] = (new_d if c.depth_tracking == "follow" else depth).detach().to("cpu") + else: + out, m = warp_affine(img, self._affine_for(f), padding_mode=c.border) + return out, 1.0 - m + + def warp_back(self, img: torch.Tensor, frm: int, to: int, depth=None) -> torch.Tensor: + """Carry an image rendered at frame `frm` back to earlier frame `to` + (inverse of the camera steps to+1..frm). Used for cadence tweens.""" + c = self.cfg + if frm <= to: + return img + if self.use_depth and depth is not None: + acc = torch.eye(4, dtype=torch.float64) + for f in range(to + 1, frm + 1): + fov = float(self.camera.fov[f]) + acc = self._delta3d(f, fov).to(torch.float64) @ acc + inv = torch.linalg.inv(acc).to(torch.float32) + out, _m = warp_3d( + img, depth.to(img.device), inv, fov_deg=float(self.camera.fov[frm]), + near=c.near, far=c.far, invert_depth=c.invert_depth, + translation_scale=c.translation_scale) + return out + acc = torch.eye(3, dtype=torch.float64) + for f in range(to + 1, frm + 1): + acc = self._affine_for(f) @ acc + out, _m = warp_affine(img, torch.linalg.inv(acc), padding_mode=c.border) + return out + + def iter_from_anchor(self, anchor: torch.Tensor, n: int): + """Warp ONE anchor image along the accumulated camera path (no + feedback, no blur build-up) - the guide-frame generator for video + models. Yields (image [1,H,W,3], valid_mask [1,H,W,1]) per frame.""" + c = self.cfg + img = resize_bhwc(anchor, c.width, c.height)[:1, ..., :3].float() + depth0 = self.depth + use3d = self.use_depth + acc3 = torch.eye(4, dtype=torch.float64) + acc2 = torch.eye(3, dtype=torch.float64) + zoom_cum = 1.0 + for f in range(n): + if f > 0: + if use3d: + fov_f = float(self.camera.fov[f]) + acc3 = self._delta3d(f, fov_f).to(torch.float64) @ acc3 + zoom_cum *= float(self.camera.zoom[f]) + else: + acc2 = self._affine_for(f) @ acc2 + if f == 0: + yield img, torch.ones_like(img[..., :1]) + continue + if use3d: + out, m = warp_3d(img, depth0, acc3.to(torch.float32), + fov_deg=float(self.camera.fov[f]), near=c.near, far=c.far, + invert_depth=c.invert_depth, + translation_scale=c.translation_scale) + if abs(zoom_cum - 1.0) > 1e-6: + out, m2 = warp_2d(out, 0.0, 0.0, 0.0, zoom_cum, padding_mode=c.border) + m = m * m2 + else: + out, m = warp_affine(img, acc2, padding_mode=c.border) + yield out.clamp(0.0, 1.0), m + + # -- per-frame pixel chain ------------------------------------------------- + + def prepare(self, img: torch.Tensor, f: int, is_key: bool): + """Warp + symmetry + detail guard (+ noise on key frames).""" + c = self.cfg + + def chain(x): + out, holes = self.warp(x, f) + if c.symmetry != "none": + out = apply_symmetry(out, mode=c.symmetry, segments=int(c.symmetry_segments)) + if c.sharpen > 0.0: + out = sharpen(out, c.sharpen) + if is_key and c.noise > 0.0: + out = add_noise(out, c.noise, seed=c.seed + f) + if is_key and c.hole_noise > 0.0 and float(holes.max()) > 0.0: + extra = add_noise(out, c.hole_noise, seed=c.seed + 7919 * (f + 1)) + out = out * (1.0 - holes) + extra * holes + return out + + return self.fx(chain, img).clamp(0.0, 1.0) + + +def _anchor_strength(cfg: EngineConfig, prompt_track, f: int) -> float: + """Colour-lock strength at frame f. In `scene` mode the lock releases while + the prompt is travelling, so the palette can follow the new scene.""" + s = float(cfg.color_coherence) + if cfg.anchor_mode != "scene" or prompt_track is None: + return s + # lock fades out as the prompt travels towards the next scene; the key + # rendered once that scene is reached becomes the new anchor + return s * (1.0 - prompt_track.transition_weight(f)) + + +def iter_feedback( + engine: FeedbackEngine, + init_image: torch.Tensor, + n_frames: int, + diffuse: Callable[[torch.Tensor, int], torch.Tensor] | None = None, + prompt_track=None, + start_frame: int = 0, + end_frame: int = 0, + on_progress: Callable[[int], None] | None = None, + pre_warp: Callable[[torch.Tensor, int], torch.Tensor] | None = None, +) -> Iterator[tuple[int, torch.Tensor]]: + """Run the loop, yielding (frame_index, [1,H,W,3]) in order. + + `diffuse(image, f)` re-diffuses a key frame (img2img); None = storyboard + mode (camera only). `pre_warp(prev, f)` may alter the previous frame + before it is warped (live camera input). With cadence > 1, in-between frames are buffered until + the next key is known, then crossfaded, so output lags by `cadence` frames. + """ + cfg = engine.cfg + cad = max(1, int(cfg.cadence)) + prev = resize_bhwc(init_image, cfg.width, cfg.height)[:1, ..., :3].float() + anchor = prev.clone() + f0 = max(0, int(start_frame)) + f1 = min(n_frames, int(end_frame)) if int(end_frame) > 0 else n_frames + + if engine.depth is not None: + engine.depth_log[f0] = engine.depth + yield f0, prev + if on_progress: + on_progress(f0) + + last_key_f = f0 + pending: list[tuple[int, torch.Tensor]] = [] + scene_idx = prompt_track.scene_index(f0) if prompt_track is not None else 0 + + for f in range(f0 + 1, f1): + is_key = diffuse is not None and (cad == 1 or (f - f0) % cad == 0 or f == f1 - 1) + if pre_warp is not None: + prev = pre_warp(prev, f) + warped = engine.prepare(prev, f, is_key=is_key) + + if not is_key: + prev = warped + if diffuse is None or cad == 1: + yield f, prev + if on_progress: + on_progress(f) + else: + pending.append((f, warped)) + continue + + image = diffuse(warped, f)[:1, ..., :3].float().cpu() + engine.report.keys += 1 + + # colour anchoring + if cfg.color_mode != "none" and cfg.anchor_mode != "none": + if cfg.anchor_mode == "scene" and prompt_track is not None: + si = prompt_track.scene_index(f) + if si != scene_idx: + scene_idx = si + anchor = image.clone() + strength = _anchor_strength(cfg, prompt_track, f) + if strength > 0.0: + image = match_color(image, anchor, strength=strength, mode=cfg.color_mode) + if cfg.anchor_mode == "rolling": + anchor = image.clone() + + # cadence: crossfade the buffered tweens between the two keys + if pending: + span = f - last_key_f + depth = engine.depth + for (tf, fwd) in pending: + back = engine.warp_back(image, f, tf, depth=depth) + u = (tf - last_key_f) / span + w = u * u * (3.0 - 2.0 * u) + engine.report.tweens += 1 + yield tf, (fwd * (1.0 - w) + back * w).clamp(0.0, 1.0) + if on_progress: + on_progress(tf) + pending = [] + + prev = image + last_key_f = f + yield f, prev + if on_progress: + on_progress(f) + + for (tf, fwd) in pending: # defensive: the last frame is always a key + yield tf, fwd diff --git a/core/export.py b/core/export.py new file mode 100644 index 0000000..5afcf49 --- /dev/null +++ b/core/export.py @@ -0,0 +1,307 @@ +""" +Camera interchange: Difforum <-> After Effects / Blender / JSON. + +Difforum's camera is a per-frame *scene* transform (P_f = d_f @ P_{f-1}) in an +OpenCV-style frame: x right, y down, z forward, world = the first camera. Here +it becomes a real camera-to-world track with a focal length per frame: + +* 3D mode: the deltas as-is (lateral units measured in pixels at the + reference depth, exactly as the renderer uses them), plus the zoom channel + as a focal-length change. +* 2D mode: a nodal camera. Pans become yaw/pitch (tan(angle) = shift / f), + roll stays roll and zoom becomes focal length - the move you would do on a + tripod to get the same 2D image motion. + +Units: one scene unit = the depth-map unit of the renderer (near..far, default +1..100). `unit_scale` converts to Blender metres; After Effects uses pixels, so +the reference depth is placed on the z = 0 layer plane. + +All conversions are plain numpy so they are testable and round-trip. +""" + +from __future__ import annotations + +import json +import math + +import numpy as np + +SCHEMA = "difforum.camera/1" + +# OpenCV world (x right, y down, z fwd) -> Blender world (x right, y fwd, z up) +_CV_TO_BL_WORLD = np.array([[1, 0, 0, 0], [0, 0, 1, 0], [0, -1, 0, 0], [0, 0, 0, 1]], dtype=np.float64) +# OpenCV camera axes -> Blender camera axes (camera looks down -Z, Y up) +_CV_TO_BL_CAM = np.diag([1.0, -1.0, -1.0, 1.0]) + + +def focal_px(width: int, fov_deg: float) -> float: + return 0.5 * width / math.tan(math.radians(fov_deg) * 0.5) + + +def fov_from_focal(width: int, f: float) -> float: + return math.degrees(2.0 * math.atan(0.5 * width / max(f, 1e-6))) + + +def _rot(rx, ry, rz): + from .camera import euler_to_matrix + return euler_to_matrix(rx, ry, rz) + + +def reference_z(near: float = 1.0, far: float = 100.0, reference_depth: float = 0.5) -> float: + return near + (1.0 - reference_depth) * (far - near) + + +def camera_track_to_world(camera, width: int, height: int, near: float = 1.0, + far: float = 100.0, reference_depth: float = 0.5): + """-> (list of 4x4 camera-to-world, focal_px per frame). World = frame 0 camera.""" + n = len(camera.deltas) + z_ref = reference_z(near, far, reference_depth) + pose = np.eye(4) + zoom_cum = 1.0 + c2w, focals = [], [] + for f in range(n): + fov = float(camera.fov[f]) + f0 = focal_px(width, fov) + if f > 0: + d = np.array(camera.deltas[f], dtype=np.float64) + if camera.mode == "3d": + d = d.copy() + lat = z_ref / f0 + d[0, 3] *= lat + d[1, 3] *= lat + else: + tx, ty = float(d[0, 3]), float(d[1, 3]) + roll = math.degrees(math.atan2(d[1, 0], d[0, 0])) + yaw = math.degrees(math.atan(tx / f0)) + pitch = -math.degrees(math.atan(ty / f0)) + d = np.eye(4) + d[:3, :3] = _rot(pitch, yaw, roll) + pose = d @ pose + zoom_cum *= float(camera.zoom[f]) + c2w.append(np.linalg.inv(pose)) + focals.append(f0 * zoom_cum) + return c2w, focals + + +def world_to_camera_track(c2w: list, focals: list, width: int, mode: str = "3d", + near: float = 1.0, far: float = 100.0, reference_depth: float = 0.5): + """Inverse of `camera_track_to_world` (3D): camera-to-world matrices in the + OpenCV frame -> a Difforum CameraTrack whose deltas re-create the move.""" + from .camera import CameraTrack + + z_ref = reference_z(near, far, reference_depth) + base = np.array(c2w[0], dtype=np.float64) + poses, deltas, fovs = [], [], [] + prev = np.eye(4) + for f, (m, fl) in enumerate(zip(c2w, focals)): + pose = np.linalg.inv(np.array(m, dtype=np.float64)) @ base # world0 -> cam f + d = pose @ np.linalg.inv(prev) if f > 0 else np.eye(4) + fov = fov_from_focal(width, float(fl)) + lat = z_ref / focal_px(width, fov) + d = d.copy() + d[0, 3] /= lat + d[1, 3] /= lat + deltas.append(d) + poses.append(pose) + fovs.append(fov) + prev = pose + return CameraTrack(deltas=deltas, poses=poses, zoom=[1.0] * len(deltas), fov=fovs, mode=mode) + + +# --------------------------------------------------------------------------- +# JSON +# --------------------------------------------------------------------------- + +def to_json(camera, width, height, fps, near=1.0, far=100.0, reference_depth=0.5) -> dict: + c2w, focals = camera_track_to_world(camera, width, height, near, far, reference_depth) + return { + "schema": SCHEMA, + "convention": "opencv", + "note": "camera-to-world, x right / y down / z forward, world = first camera, " + "units = Difforum scene units (depth-map units)", + "fps": float(fps), "width": int(width), "height": int(height), + "reference_z": reference_z(near, far, reference_depth), + "frames": [ + {"f": i, "matrix": np.round(m, 8).tolist(), "focal_px": round(float(fl), 5), + "fov_deg": round(fov_from_focal(width, fl), 5)} + for i, (m, fl) in enumerate(zip(c2w, focals)) + ], + } + + +def _ae_matrix(rx, ry, rz): + return _rot(rx, ry, rz) + + +def from_json(data) -> tuple: + """Parse a camera JSON (Difforum, or the Blender / After Effects export + tools in tools/) -> (c2w list in OpenCV frame, focals, width, height, fps).""" + if isinstance(data, str): + data = json.loads(data) + conv = data.get("convention", "opencv") + w, h = int(data["width"]), int(data["height"]) + fps = float(data.get("fps", 24.0)) + c2w, focals = [], [] + for fr in data["frames"]: + if conv == "ae": + # position in px (y down, z forward), rotations in degrees, zoom = focal px + fl = float(fr["zoom"]) + pos = np.array(fr["position"], dtype=np.float64) + m = np.eye(4) + m[:3, :3] = _ae_matrix(*fr.get("rotation", [0.0, 0.0, 0.0])) + origin = np.array([w / 2.0, h / 2.0, -float(data.get("zoom0", fl))]) + scale = float(data.get("reference_z", 50.5)) / float(data.get("zoom0", fl)) + m[:3, 3] = (pos - origin) * scale + else: + m = np.array(fr["matrix"], dtype=np.float64) + if conv == "blender": + m = np.linalg.inv(_CV_TO_BL_WORLD) @ m @ np.linalg.inv(_CV_TO_BL_CAM) + m[:3, 3] /= float(data.get("unit_scale", 0.1)) + fl = float(fr.get("focal_px") or focal_px(w, float(fr.get("fov_deg", 40.0)))) + c2w.append(m) + focals.append(fl) + if c2w: # re-base so the first camera is the origin + base_inv = np.linalg.inv(c2w[0]) + c2w = [base_inv @ m for m in c2w] + return c2w, focals, w, h, fps + + +# --------------------------------------------------------------------------- +# Blender +# --------------------------------------------------------------------------- + +def to_blender_script(camera, width, height, fps, unit_scale=0.1, name="DifforumCam", + near=1.0, far=100.0, reference_depth=0.5) -> str: + c2w, focals = camera_track_to_world(camera, width, height, near, far, reference_depth) + mats = [] + for m in c2w: + mb = _CV_TO_BL_WORLD @ m @ _CV_TO_BL_CAM + mb[:3, 3] *= unit_scale + mats.append(np.round(mb, 6).tolist()) + lens = [round(fl / width * 36.0, 5) for fl in focals] + return f'''# Difforum camera for Blender - run from the Text Editor (Alt+P). +# Creates/updates the camera "{name}", matches fps, resolution and frame range. +import bpy +from mathutils import Matrix + +MATS = {json.dumps(mats)} +LENS = {json.dumps(lens)} +FPS, W, H = {float(fps)!r}, {int(width)}, {int(height)} + +scn = bpy.context.scene +scn.render.fps = int(round(FPS)) +scn.render.fps_base = scn.render.fps / FPS +scn.render.resolution_x, scn.render.resolution_y = W, H +scn.frame_start, scn.frame_end = 1, len(MATS) + +cam_data = bpy.data.cameras.get("{name}") or bpy.data.cameras.new("{name}") +cam_data.sensor_fit = "HORIZONTAL" +cam_data.sensor_width = 36.0 +cam = bpy.data.objects.get("{name}") or bpy.data.objects.new("{name}", cam_data) +if cam.name not in scn.collection.all_objects: + scn.collection.objects.link(cam) +cam.animation_data_clear() +cam_data.animation_data_clear() +for i, (m, lens) in enumerate(zip(MATS, LENS)): + f = i + 1 + cam.matrix_world = Matrix(m) + cam.keyframe_insert("location", frame=f) + cam.keyframe_insert("rotation_euler", frame=f) + cam_data.lens = lens + cam_data.keyframe_insert("lens", frame=f) +scn.camera = cam +print("Difforum: {{}} camera keys on {name}".format(len(MATS))) +''' + + +# --------------------------------------------------------------------------- +# After Effects +# --------------------------------------------------------------------------- + +def _decompose_xyz(r: np.ndarray) -> tuple[float, float, float]: + """Inverse of euler_to_matrix (R = Rz @ Ry @ Rx) -> (rx, ry, rz) degrees.""" + sy = -r[2, 0] + sy = max(-1.0, min(1.0, sy)) + ry = math.asin(sy) + if abs(math.cos(ry)) > 1e-6: + rx = math.atan2(r[2, 1], r[2, 2]) + rz = math.atan2(r[1, 0], r[0, 0]) + else: + rx = math.atan2(-r[1, 2], r[1, 1]) + rz = 0.0 + return math.degrees(rx), math.degrees(ry), math.degrees(rz) + + +def to_ae_jsx(camera, width, height, fps, name="Difforum Camera", + near=1.0, far=100.0, reference_depth=0.5) -> str: + n = len(camera.deltas) + duration = n / float(fps) + if camera.mode == "2d": + # exact 2D: a null whose transform reproduces the image motion + from .warp import affine_2d + acc = np.eye(3) + pos, scl, rot = [], [], [] + for f in range(n): + if f > 0: + d = camera.deltas[f] + ang = math.degrees(math.atan2(float(d[1][0]), float(d[0][0]))) + a = affine_2d(float(d[0][3]), float(d[1][3]), ang, float(camera.zoom[f]), + width, height).numpy() + acc = a @ acc + c = np.array([(width - 1) / 2.0, (height - 1) / 2.0, 1.0]) + p = acc @ c + s = math.hypot(acc[0, 0], acc[1, 0]) + pos.append([round(float(p[0]), 4), round(float(p[1]), 4)]) + scl.append(round(s * 100.0, 5)) + rot.append(round(math.degrees(math.atan2(acc[1, 0], acc[0, 0])), 5)) + body = f'''var POS = {json.dumps(pos)}; +var SCL = {json.dumps(scl)}; +var ROT = {json.dumps(rot)}; +var nul = comp.layers.addNull(comp.duration); +nul.name = "{name} (2D)"; +nul.anchorPoint.setValue([(comp.width - 1) / 2, (comp.height - 1) / 2]); +for (var i = 0; i < POS.length; i++) {{ + var t = i / comp.frameRate; + nul.position.setValueAtTime(t, POS[i]); + nul.scale.setValueAtTime(t, [SCL[i], SCL[i]]); + nul.rotation.setValueAtTime(t, ROT[i]); +}} +alert("Difforum: 2D move on a null. Parent your footage to it.");''' + else: + c2w, focals = camera_track_to_world(camera, width, height, near, far, reference_depth) + z_ref = reference_z(near, far, reference_depth) + f0 = focals[0] + s = f0 / z_ref + pos, rot, zoom = [], [], [] + for m, fl in zip(c2w, focals): + p = m[:3, 3] * s + np.array([width / 2.0, height / 2.0, -f0]) + pos.append([round(float(v), 4) for v in p]) + rot.append([round(v, 5) for v in _decompose_xyz(m[:3, :3])]) + zoom.append(round(float(fl), 4)) + body = f'''var POS = {json.dumps(pos)}; +var ROT = {json.dumps(rot)}; +var ZOOM = {json.dumps(zoom)}; +var cam = comp.layers.addCamera("{name}", [comp.width / 2, comp.height / 2]); +cam.autoOrient = AutoOrientType.NO_AUTO_ORIENT; +for (var i = 0; i < POS.length; i++) {{ + var t = i / comp.frameRate; + cam.position.setValueAtTime(t, POS[i]); + cam.xRotation.setValueAtTime(t, ROT[i][0]); + cam.yRotation.setValueAtTime(t, ROT[i][1]); + cam.zRotation.setValueAtTime(t, ROT[i][2]); + cam.zoom.setValueAtTime(t, ZOOM[i]); +}} +alert("Difforum: " + POS.length + " camera keys. Layers at z=0 sit at the reference depth.");''' + return f'''// Difforum camera for After Effects - File > Scripts > Run Script File... +// Uses the active comp, or creates one that matches the render. +(function () {{ +app.beginUndoGroup("Difforum camera"); +var comp = app.project.activeItem; +if (!(comp && comp instanceof CompItem)) {{ + comp = app.project.items.addComp("Difforum", {int(width)}, {int(height)}, 1, {duration:.6f}, {float(fps)!r}); + comp.openInViewer(); +}} +{body} +app.endUndoGroup(); +}})(); +''' diff --git a/core/loop.py b/core/loop.py index 0ce7b59..4c3726f 100644 --- a/core/loop.py +++ b/core/loop.py @@ -27,7 +27,6 @@ from __future__ import annotations -import math import numpy as np @@ -152,7 +151,6 @@ def seam_report(frames, clip_len: int) -> str: has settled: if lap-to-lap drift is far below the seam error, more laps will not help and the residual is coming from the camera instead. """ - import torch n = len(frames) if n < 2: diff --git a/core/prompt.py b/core/prompt.py index 7145458..1f71737 100644 --- a/core/prompt.py +++ b/core/prompt.py @@ -17,6 +17,7 @@ from __future__ import annotations import re +from collections.abc import Sequence from .schedule import EASINGS, _ease @@ -134,3 +135,88 @@ def blend_conditioning(cond_to, cond_from, to_strength: float): meta["pooled_output"] = pooled_to * to_strength + pooled_from * (1.0 - to_strength) out.append([tw, meta]) return out + + +class PromptTrack(Sequence): + """Per-frame prompt travel, blended lazily. + + Holds one encoded CONDITIONING per keyframe plus the blend plan, and + builds a frame's blended conditioning only when it is asked for (with a + small cache). A 0.x `DIFFORUM_PROMPT` was a fully materialised list, which + for long clips on T5/Gemma-sized encoders meant gigabytes of RAM. Indexing + and len() behave exactly like that list, so older consumers keep working. + """ + + def __init__(self, encoded: list, keyframes: list[tuple[int, str]], + max_frames: int, easing: str = "linear"): + if not encoded: + raise ValueError("PromptTrack needs at least one keyframe") + self.encoded = list(encoded) + self.keyframes = list(keyframes) + self.n = max(1, int(max_frames)) + self.easing = easing + self.plan = plan_blend([f for f, _ in keyframes], self.n, easing) + self._cache: dict[int, object] = {} + + def __len__(self) -> int: + return self.n + + def __getitem__(self, i): + if isinstance(i, slice): + return [self[j] for j in range(*i.indices(self.n))] + i = int(i) + if i < 0: + i += self.n + i = max(0, min(i, self.n - 1)) + li, ri, w = self.plan[i] + key = (li, ri, round(w, 4)) + hit = self._cache.get(key) + if hit is not None: + return hit + if li == ri or w <= 0.0: + out = self.encoded[li] + elif w >= 1.0: + out = self.encoded[ri] + else: + out = blend_conditioning(self.encoded[ri], self.encoded[li], w) + if len(self._cache) > 8: + self._cache.clear() + self._cache[key] = out + return out + + def scene_index(self, f: int) -> int: + """Index of the keyframe the frame is travelling away from.""" + return self.plan[max(0, min(int(f), self.n - 1))][0] + + def transition_weight(self, f: int) -> float: + """0 = holding a keyframe prompt, ->1 = arriving at the next one.""" + li, ri, w = self.plan[max(0, min(int(f), self.n - 1))] + return 0.0 if li == ri else float(w) + + def text_at(self, f: int) -> str: + return self.keyframes[self.scene_index(f)][1] + + +def batch_conditioning(track) -> list: + """Stack a per-frame prompt track into ONE CONDITIONING whose batch dim is + the frame count (frame i = prompt i) - what AnimateDiff-style batch + samplers expect.""" + import torch + + conds = [track[i] for i in range(len(track))] + if not conds: + raise ValueError("empty prompt track") + tensors = [c[0][0] for c in conds] + max_t = max(t.shape[1] for t in tensors) + padded = [] + for t in tensors: + if t.shape[1] < max_t: + pad = torch.zeros((t.shape[0], max_t - t.shape[1], t.shape[2]), + dtype=t.dtype, device=t.device) + t = torch.cat([t, pad], dim=1) + padded.append(t) + meta = dict(conds[0][0][1]) + pooled = [c[0][1].get("pooled_output") for c in conds] + if all(p is not None for p in pooled): + meta["pooled_output"] = torch.cat(pooled, dim=0) + return [[torch.cat(padded, dim=0), meta]] diff --git a/core/warp.py b/core/warp.py index c4a1ab1..fa24c90 100644 --- a/core/warp.py +++ b/core/warp.py @@ -127,6 +127,13 @@ def _depth_to_z(depth: torch.Tensor, near: float, far: float, invert: bool) -> t return near + (1.0 - d) * (far - near) +def _z_to_depth(z: torch.Tensor, near: float, far: float, invert: bool) -> torch.Tensor: + """Inverse of `_depth_to_z`: metric z back to the 0..1 depth-map convention.""" + d = 1.0 - (z - near) / max(far - near, 1e-6) + d = d.clamp(0.0, 1.0) + return 1.0 - d if invert else d + + def warp_3d( image: torch.Tensor, depth: torch.Tensor, @@ -137,20 +144,28 @@ def warp_3d( invert_depth: bool = False, translation_scale: float = 1.0, fill_holes: bool = True, -) -> tuple[torch.Tensor, torch.Tensor]: + return_depth: bool = False, +): """ Perspective warp via depth + a 4x4 scene-point transform. `transform` is applied to unprojected scene points (P' = R@P + T). A positive translation_z therefore pushes the scene away (camera dollies - back); flip the sign upstream for "into the scene". Forward splat with a - z-buffer: nearest point wins, unfilled pixels are holes (mask=0). + back); flip the sign upstream for "into the scene". + + Forward splat with a deterministic z-buffer: for every destination pixel + the nearest source point wins, resolved with `scatter_reduce` so the result + is identical on CPU, CUDA and MPS (plain duplicate-index assignment is + undefined on GPUs and shows up as speckled occlusion). `translation_scale` multiplies the translation part of the transform so the motion magnitude can be tuned to the depth range (depth estimators output - relative/disparity values, not metric units - this is the calibration knob). + relative values, not metric units - this is the calibration knob). - Returns (warped [.,H,W,3], mask [.,H,W,1]). + Returns (warped [.,H,W,3], mask [.,H,W,1]) and, with return_depth=True, the + depth map re-projected into the new view ([.,H,W], same 0..1 convention as + the input). Feeding that depth to the next frame keeps the parallax locked + to the image instead of to frame 0. """ img, squeezed = _as_bhwc(image) b, h, w, c = img.shape @@ -160,7 +175,7 @@ def warp_3d( depth = depth.unsqueeze(0) if depth.dim() == 4: depth = depth[..., 0] - depth = depth.expand(b, h, w) + depth = depth.to(dev, dt).expand(b, h, w) f = _focal(w, fov_deg) cx, cy = (w - 1) / 2.0, (h - 1) / 2.0 @@ -173,8 +188,10 @@ def warp_3d( indexing="ij", ) + n_pix = h * w out = torch.zeros_like(img) mask = torch.zeros((b, h, w), device=dev, dtype=dt) + zout = torch.full((b, h, w), float(far), device=dev, dtype=dt) for i in range(b): z = _depth_to_z(depth[i], near, far, invert_depth) @@ -188,26 +205,119 @@ def warp_3d( ui = up.round().long() vi = vp.round().long() - valid = (ui >= 0) & (ui < w) & (vi >= 0) & (vi < h) + valid = (ui >= 0) & (ui < w) & (vi >= 0) & (vi < h) & (Pp[:, 2] > 1e-4) ui, vi, Zp = ui[valid], vi[valid], Zp[valid] cols = img[i].reshape(-1, c)[valid] dest = vi * w + ui - # paint far -> near so the nearest point ends up on top (z-buffer) - order = torch.argsort(Zp, descending=True) - dest_o, cols_o = dest[order], cols[order] + # z-buffer: nearest depth per destination, then one deterministic + # winner (highest source index) among exact ties + zbuf = torch.full((n_pix,), float("inf"), device=dev, dtype=dt) + zbuf = zbuf.scatter_reduce(0, dest, Zp, reduce="amin", include_self=True) + win = Zp <= zbuf[dest] + src_idx = torch.arange(dest.numel(), device=dev) + best = torch.full((n_pix,), -1, device=dev, dtype=torch.long) + best = best.scatter_reduce(0, dest[win], src_idx[win], reduce="amax", + include_self=True) + filled = best >= 0 flat = out[i].reshape(-1, c) - flat[dest_o] = cols_o - mflat = mask[i].reshape(-1) - mflat[dest_o] = 1.0 + flat[filled] = cols[best[filled]] + mask[i].reshape(-1)[filled] = 1.0 + zout[i].reshape(-1)[filled] = zbuf[filled] - # clean speckle holes in the colour (keep `mask` as the true occlusion) + # clean speckle holes in colour and depth (keep `mask` as the true occlusion) if fill_holes: out = _fill_holes(out, mask) + if return_depth: + zfill = _fill_holes(zout.unsqueeze(-1), mask, iters=16)[..., 0] + zout = torch.where(mask > 0.5, zout, zfill) mask = mask.unsqueeze(-1) + new_depth = _z_to_depth(zout, near, far, invert_depth) if return_depth else None if squeezed: out, mask = out[0], mask[0] + if new_depth is not None: + new_depth = new_depth[0] + if return_depth: + return out, mask, new_depth return out, mask + + +# --------------------------------------------------------------------------- +# 2D affine helpers (composition / inversion, used by cadence tweening and the +# pseudo-3D fallback) +# --------------------------------------------------------------------------- + +def affine_2d(translation_x: float, translation_y: float, angle: float, zoom: float, + width: int, height: int) -> torch.Tensor: + """3x3 matrix of the exact forward map `warp_2d` applies (pixel space). + + p_out = S*R*(p_in - c) + c + t + """ + cx, cy = (width - 1) / 2.0, (height - 1) / 2.0 + a = math.radians(angle) + s = max(float(zoom), 1e-6) + ca, sa = math.cos(a) * s, math.sin(a) * s + m = torch.tensor([ + [ca, -sa, 0.0], + [sa, ca, 0.0], + [0.0, 0.0, 1.0], + ], dtype=torch.float64) + to_c = torch.tensor([[1, 0, cx + translation_x], [0, 1, cy + translation_y], [0, 0, 1]], + dtype=torch.float64) + from_c = torch.tensor([[1, 0, -cx], [0, 1, -cy], [0, 0, 1]], dtype=torch.float64) + return to_c @ m @ from_c + + +def warp_affine(image: torch.Tensor, matrix: torch.Tensor, + padding_mode: str = "reflection") -> tuple[torch.Tensor, torch.Tensor]: + """Warp by a 3x3 forward pixel-space matrix (inverse-sampled).""" + img, squeezed = _as_bhwc(image) + b, h, w, c = img.shape + dev, dt = img.device, img.dtype + inv = torch.linalg.inv(matrix.to(torch.float64)).to(dev, dt) + ys, xs = torch.meshgrid( + torch.arange(h, device=dev, dtype=dt), + torch.arange(w, device=dev, dtype=dt), + indexing="ij", + ) + src_x = inv[0, 0] * xs + inv[0, 1] * ys + inv[0, 2] + src_y = inv[1, 0] * xs + inv[1, 1] * ys + inv[1, 2] + gx = (src_x / max(w - 1, 1)) * 2.0 - 1.0 + gy = (src_y / max(h - 1, 1)) * 2.0 - 1.0 + grid = torch.stack([gx, gy], dim=-1).unsqueeze(0).expand(b, h, w, 2) + warped = torch.nn.functional.grid_sample( + img.permute(0, 3, 1, 2), grid, mode="bilinear", + padding_mode=padding_mode, align_corners=True, + ).permute(0, 2, 3, 1) + inside = ((src_x >= 0) & (src_x <= w - 1) & (src_y >= 0) & (src_y <= h - 1)).to(dt) + mask = inside.unsqueeze(0).expand(b, h, w).unsqueeze(-1) + if squeezed: + warped, mask = warped[0], mask[0] + return warped, mask + + +def pseudo_3d_params(transform, fov_deg: float, width: int, near: float = 1.0, + far: float = 100.0, translation_scale: float = 1.0, + reference_depth: float = 0.5) -> tuple[float, float, float, float]: + """Approximate a 3D camera step with a 2D affine when there is no depth map. + + Projects the scene point at the reference depth on the optical axis through + the transform: its image shift becomes the pan, the change in its distance + becomes the zoom (so a dolly still pushes in), and the in-plane rotation is + the roll. Orbit / dolly / rise presets therefore still move instead of + freezing the frame. Returns (translation_x, translation_y, angle_deg, zoom). + """ + t = torch.as_tensor(transform, dtype=torch.float64) + R, T = t[:3, :3], t[:3, 3] * float(translation_scale) + f = _focal(width, fov_deg) + z_ref = near + (1.0 - float(reference_depth)) * (far - near) + p = R @ torch.tensor([0.0, 0.0, z_ref], dtype=torch.float64) + T + pz = max(float(p[2]), 1e-3) + tx = f * float(p[0]) / pz + ty = f * float(p[1]) / pz + zoom = z_ref / pz + angle = math.degrees(math.atan2(float(R[1, 0]), float(R[0, 0]))) + return tx, ty, angle, zoom diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..792ee19 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,58 @@ +# Architecture + +``` +Setup ──params──► Director ──direction──► Feedback / Live Sampler ──frames──► Post / Save + │ Storyboard + │ Guide Frames ─► Keyframes ─► LTX Guides / H3 Shot + └──────────────────► Camera → Prompt, Camera Export +``` + +## Layers + +| Layer | Where | Depends on | +|---|---|---| +| Engine | `core/` | numpy, torch. No ComfyUI. | +| Nodes | `nodes/` | the engine, plus ComfyUI imported lazily inside functions | +| UI | `js/director_timeline.js` | the `/difforum/catalog` and `/difforum/preview` routes (`nodes/routes.py`) | +| Legacy | `legacy/` | the engine; 0.x nodes flagged `DEPRECATED` | + +Every module imports the engine *relatively* (`from ..core import ...`). The +0.x pack put itself on `sys.path` and imported a top-level `core`, which +collided with any other custom node that did the same. + +## Core modules + +- `schedule.py`, `expr.py`: the Deforum keyframe syntax, evaluated by a + whitelisted AST walker (no `eval`). +- `camera.py`: per-frame 4x4 deltas and accumulated poses. A delta acts on the + previous view: `P_f = d_f · P_{f-1}`. +- `camera_presets.py`: the 25 moves as per-axis expressions, plus UI metadata. +- `camera_keys.py`: blends moves between keyframes (easing, lens channel). +- `direction.py`: the timeline model (scenes / camera / energy / guidance), + audio reactions, and camera → words. `build_direction()` has no ComfyUI or torch + dependency, which is why it can back the live preview endpoint. +- `engine.py`: `FeedbackEngine` + `iter_feedback()`, the one loop behind the + Feedback Sampler, the Live Sampler and the Storyboard. Diffusion is a callback. +- `warp.py`: the 2D affine and 3D depth warps, with a deterministic z-buffer + (`scatter_reduce`), depth re-projection, affine composition and pseudo-3D. +- `export.py`: camera ↔ JSON / After Effects / Blender. +- `prompt.py`: prompt travel. `PromptTrack` blends lazily and indexes like a list. +- `color.py`, `detail.py`, `symmetry.py`, `flow.py`, `effects.py`, `loop.py`, + `audio.py`, `plot.py`, `storyboard.py`: the pixel and signal tools. + +## Units and conventions + +- Images are `[B,H,W,C]` in 0..1 (ComfyUI `IMAGE`). Depth maps: white = near. +- 2D camera: pixels per frame, degrees per frame, and zoom as a per-frame scale. +- 3D camera: rotations in degrees per frame. Lateral translation is in pixels at the + reference depth (so 2D and 3D pans agree), and z translation is in depth units + (`near`..`far`, default 1..100). +- Exported cameras use OpenCV axes (x right, y down, z forward), with world = the + first camera. + +## Keeping things in sync + +- `tools/build_workflows.py` builds `example_workflows/` from the node + definitions, so widget order can never drift. CI runs it with `--check`. +- `tools/build_docs.py` builds `docs/NODES.md` from docstrings and tooltips. + CI runs `--check` on it as well. diff --git a/docs/BRIDGES.md b/docs/BRIDGES.md new file mode 100644 index 0000000..a1462a7 --- /dev/null +++ b/docs/BRIDGES.md @@ -0,0 +1,104 @@ +# Bridges: video models and camera interchange + +Difforum stays the director (camera, timing, prompts) and the video model does +the temporally coherent rendering. The bridges output plain `IMAGE`, `MASK`, +`INT` and `STRING` values, and the LTX bridge calls ComfyUI's own core +`LTXVAddGuide`, so none of them depend on another pack's internals. + +## The building blocks + +| Node | What it gives you | +|---|---| +| **Guide Frames** | One anchor image carried along the Director's camera path. Revealed areas are filled with neutral gray and marked in a mask (`1 = generate` is the convention VACE / LTX / H3 use). Works in 3D with a depth map. | +| **Keyframes** | Picks keyframes from *any* frame batch (Guide Frames, a Storyboard, a Feedback render) on a model's grid. Outputs the keyframes, their indices, a full-length sparse batch, the first and last frame, and the snapped length. | +| **Camera → Prompt** | The camera in words, for prompt-driven models. It uses the Director's blocks when present; otherwise it reads any camera track and names the moves. | + +What to feed Keyframes: + +- **Guide Frames**: exact geometry, gray where the camera reveals new area. Best + for moves that stay inside the frame (push-in, orbit, spiral). +- **Feedback render**: every frame is a finished image, revealed areas included. + Best for pans and long travels. +- **Storyboard**: the cheapest option. Warps only, no diffusion. + +## LTX-2 / 2.5 + +Template: `07_ltx_guides`. + +1. **Setup** target `LTX-2 / 2.5` keeps the length on 8k+1 and the size on 32 px. +2. **Director** → **Guide Frames** (or a Feedback render) → **Keyframes** (grid LTX, + every 1 s; spacing snaps to the 8-frame latent stride). +3. **LTX Guides** takes your LTX positive / negative / VAE and an empty LTX latent + whose length comes from Keyframes. It adds one guide per keyframe: frame 0 at + `first_strength` (1.0 locks the look), the rest at `strength` (0.6-0.8 lets LTX + move naturally between them). +4. Append **Camera → Prompt** (`prompt suffix`) to your LTX prompt, so the text + agrees with the guides. + +To use KJNodes' `LTXVAddGuidesFromBatch` instead, feed it Keyframes' +`sparse_batch`: every frame is black except the keys. + +## MiniMax H3 (multi-keyframe guides) + +Template: `10_h3_multikeyframe_guides`. This is the official *Multiframe Reference* +setup (ref2va model), with the guide images coming from your camera. + +1. **Setup** target `MiniMax H3` locks 24 fps, the 17k+5 length grid and a 32 px size. +2. **Director** → **Guide Frames** (the anchor carried along the camera) → + **Keyframes** (grid H3, every 2 s). +3. **MiniMax H3 Reference to Video** gets the anchor as `` (identity + and style), the prompt from **Camera → Prompt**, and the size and length + from Setup / Keyframes. +4. **H3 Guides** anchors each keyframe at its frame, using the core + `MiniMaxH3AddGuide`. `max_guides` (default 4) keeps the first, the last and + evenly spaced ones in between. Connect an `audio` (and the audio VAE) to + anchor a soundtrack at frame 0. +5. The rest is the official sampling tail: guider → SamplerCustomAdvanced → + video + audio decode → Create Video. The Turbo LoRA node is bypassed; enable it + with Ctrl+B and drop the steps to 4. + +## MiniMax H3 (first / last frame) + +Template: `08_h3_first_last_frame` (fl2va model), wired end to end. + +1. **Setup** target `MiniMax H3` locks 24 fps and the 17k+5 length grid. +2. **H3 Shot** takes the frames (Guide Frames, a Feedback render or a Storyboard) + and returns, for the chosen `segment`: + `first_frame`, `last_frame`, `length`, `width`, `height` (32 px grid) and a + `prompt` built from your shot description plus the camera move of that segment. +3. These feed core **MiniMax H3 Image to Video** directly. The last frame is + the anchor carried to where the camera ends, so H3 performs the move you drew + instead of improvising one. + +**Longer than one generation (20 s):** `segments` tells you how many there are. +Render segment 0, then set `segment` 1 and use the previous render's last frame +as the next first frame. Segments share their boundary frame, so the joins are +seamless. + +## After Effects and Blender + +**Camera Export** writes to `ComfyUI/output/difforum/`: + +- `.jsx`: *File → Scripts → Run Script File* in After Effects. In 3D mode it + creates a keyed 3D camera, and layers at z = 0 sit at the render's reference depth. + In 2D mode it creates a null whose transform reproduces the image motion + exactly; parent your footage or titles to it. +- `_blender.py`: run it from Blender's Text Editor. It creates an animated camera + and matches fps, resolution and frame range. `blender_unit_scale` sets how many + metres one Difforum unit is. +- `.json`: camera-to-world matrices plus the focal length per frame (OpenCV + convention, world = first camera), for Nuke, Houdini, Unreal or your own tools. + +Set `translation_scale` to the same value as the sampler so the exported move +matches the render. + +**Camera Import** goes the other way: block a move in Blender or After Effects, +export it with `tools/blender/difforum_export_camera.py` or +`tools/aftereffects/Difforum Export Camera.jsx`, drop the `.json` into +`ComfyUI/input/`, and it drives the Storyboard, Feedback Sampler or Guide Frames +like any Difforum camera. `retime = match frames` resamples it to your clip length. + +Round trips (Difforum → JSON / Blender / AE → Difforum) are covered by the test +suite. After Effects rotations use X/Y/Z Rotation composed as Rz·Ry·Rx. If a +move looks mirrored in your AE version, check that the camera has no Orientation +values. diff --git a/MATHS.md b/docs/EXPRESSIONS.md similarity index 100% rename from MATHS.md rename to docs/EXPRESSIONS.md diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md new file mode 100644 index 0000000..e1f515f --- /dev/null +++ b/docs/MIGRATION.md @@ -0,0 +1,52 @@ +# Migrating from Difforum 0.x + +Difforum 1.0 goes from 40 nodes to 26 and makes the Director timeline the main +way to drive a shot. Nothing breaks on load: every 0.x node is still +registered, hidden from the node search and moved to **Difforum/legacy**, so +old workflows open and run. Legacy nodes will be removed in 2.0. + +## Node map + +| 0.x | 1.0 | +|---|---| +| Anim Setup, Anim Setup+ | **Setup** (seconds or frames, aspect, target model grid) | +| Film Director | **Director (timeline)**: separate Scenes / Camera / Energy tracks. It outputs the camera, energy and prompts directly (one `direction` wire) instead of text for other nodes to parse | +| Camera Keys | **Camera (keys)**, same syntax, and `loop_mode` included | +| Camera Shots | **Camera (keys)** with `transition 0` (hard cuts) | +| Camera Move (presets) | **Camera (keys)** with a single key, or one Director block | +| Seamless Camera | **Camera (keys)** with `loop_mode` | +| Camera (advanced) | **Camera (expressions)**, where each axis also accepts a schedule socket (e.g. an Audio Curve) | +| Camera Path Preview, Warp (2D/3D), Storyboard | **Storyboard** (sheet + frames + camera path, same engine as the sampler) | +| Schedule / Sample Schedule | **Schedule** | +| Schedule Info / Schedule Plot | **Schedule Plot** (image + text) | +| Audio Analyzer | **Audio Analyzer** (+ `offset_seconds`, a bands plot) | +| Audio Schedule (reactive) | **Audio Curve**, or a per-block audio reaction in the Director | +| Prompt Schedule / Prompt Scenes | **Prompt Travel** (`2.5s:` timestamps allowed) or the Director's Scenes track | +| Prompt Batch (→ AnimateDiff) | **Prompt Travel** with `build_batched` | +| Feedback Sampler | **Feedback Sampler** (see "behaviour changes" below) | +| Live Sampler, Live Step | **Live Sampler** | +| Guide Builder (Wan VACE) | **Guide Frames** (mask convention and hole fill selectable) | +| Loop Take, Loop Blend, Ping-Pong | **Loop** (`method`) | +| Symmetry, Echo Trails, Detail Guard, Flow Stabilize | same names, under Post | +| Model Profile, Model Catalog | [MODELS.md](MODELS.md): recommendations that go stale belong in docs, not in nodes | +| Load Video, Save Video | ComfyUI core **Load Video** / **Create Video** + **Save Video** (these keep audio) | +| VJ Look, Colour Grade, Glow, Glitch, Datamosh | a dedicated grading / glitch pack; out of scope for a direction tool | + +## Behaviour changes in the Feedback Sampler + +- **3D without depth** used to freeze dolly / orbit moves silently. It now runs a + pseudo-3D fallback and reports it. +- **3D lateral moves** are measured in pixels at the reference depth, the same unit + as 2D pans. A 0.x 3D graph with `translation_x` will pan less; raise + `translation_scale` if you relied on the old magnitude. +- **The depth map follows the image** (`depth_tracking = follow`). Set `static` for + the 0.x behaviour. +- **Cadence** in-betweens are crossfaded between keys instead of being plain warps. +- **Colour anchor** defaults to `scene`. Set `anchor_mode = first` for 0.x. +- **The camera pose** accumulates in view order (`P_f = d_f · P_{f-1}`). Poses from + combined rotation + translation moves differ slightly from 0.x; per-frame + deltas are unchanged. +- **Inputs:** `strength_schedule` → `strength`, `cfg_schedule` → `cfg_curve`, + `positive_schedule` → `prompts`. Colour, detail, symmetry, depth calibration + and chunking moved to **Render Options**; without it the Director's style sets + the look. diff --git a/docs/MODELS.md b/docs/MODELS.md new file mode 100644 index 0000000..abd9061 --- /dev/null +++ b/docs/MODELS.md @@ -0,0 +1,35 @@ +# Models that work well + +Difforum takes a plain `MODEL` / `VAE` / `CONDITIONING`, so any image model +ComfyUI loads can drive the Feedback Sampler. These are good starting points. +Check each model's page for current versions and licences. + +## Feedback Sampler (the Deforum look) + +| Goal | Model | Settings | +|---|---|---| +| Quality | SDXL finetunes (e.g. Juggernaut XL, RealVis XL) | 20-28 steps, dpmpp_2m karras, cfg 5-7, energy 0.4-0.6 | +| Speed | SDXL + DMD2 4-step LoRA, or Lightning | 4-8 steps, lcm / euler, cfg 1-1.5, cadence 2 | +| Flux | Flux.1 dev / schnell (fp16 / GGUF on Mac) | 8-20 steps, cfg 1, use FluxGuidance on the positive prompt | +| Stylised / light | SD1.5 finetunes (DreamShaper, etc.) | 20 steps, cfg 7. On some builds, launch with `--force-fp32` if frames come out black | +| Live | SDXL-Turbo, SD-Turbo, LCM LoRA | 1-2 steps, cfg ~1, 512 px | + +A depth model for 3D parallax: **Depth Anything V2** via +[ComfyUI-DepthAnythingV2](https://github.com/kijai/ComfyUI-DepthAnythingV2) +(vitl for quality, vits for speed). + +## Video models (via the bridges) + +| Model | Bridge | Notes | +|---|---|---| +| LTX-2 / 2.5 | **LTX Guides** | Setup target LTX. Runs on Apple Silicon with bf16 / GGUF (no fp8 on MPS). | +| MiniMax H3 | **H3 Shot** | 24 fps, 17k+5 frames, up to 20 s per generation. | +| Wan 2.x VACE | **Guide Frames** → VACE control video + masks | Setup target Wan (4k+1, 16 px). | + +## Apple Silicon + +- fp8 checkpoints do not run on MPS. Use fp16 / bf16 or GGUF. +- Launch with `--use-pytorch-cross-attention`, and + `PYTORCH_MPS_HIGH_WATERMARK_RATIO=0.0` to unlock unified memory. +- Few-step models are the real speed lever. Setup's `max_megapixels` keeps long + renders inside memory (about 0.35 MP on 16 GB, 0.65 on 24 GB, 1.0 on 32 GB). diff --git a/docs/NODES.md b/docs/NODES.md new file mode 100644 index 0000000..e8477e5 --- /dev/null +++ b/docs/NODES.md @@ -0,0 +1,645 @@ +# Node reference + +Generated from the code by `tools/build_docs.py`. Every node also shows this text as its description inside ComfyUI. + +## 1 · Setup + +### Difforum · Setup + +`Difforum_Setup` + +Duration, framing and the grid of the model you will render with. + +Say how long the clip is (seconds or frames) and how it is framed (aspect + +long edge); the resolution is derived, snapped to what the target model's +VAE accepts, and the frame count is snapped to the model's length grid +(LTX 8k+1, MiniMax H3 17k+5 at 24 fps, Wan 4k+1). Every other Difforum node +reads these params, so the whole graph agrees on one timeline. + +| input | type | default | notes | +|---|---|---|---| +| `target` | choice (feedback (SD/SDXL/Flux), LTX-2 / 2.5, MiniMax H3, Wan 2.x) | feedback (SD/SDXL/Flux) | Snaps size and length to what this model family accepts. | +| `duration_mode` | choice (seconds, frames) | seconds | | +| `duration` | FLOAT | 5.0 | | +| `fps` | FLOAT | 24.0 | | +| `aspect` | choice (16:9 landscape, 9:16 vertical, 1:1 square, 4:5 social portrait, 4:3 classic, 3:4 portrait, ...) | 16:9 landscape | | +| `long_edge` | INT | 768 | Pixels on the longer side before snapping. | +| `seed` | INT | 0 | | +| `custom_aspect_w` *(optional)* | FLOAT | 16.0 | | +| `custom_aspect_h` *(optional)* | FLOAT | 9.0 | | +| `max_megapixels` *(optional)* | FLOAT | 0.0 | 0 = no cap. Otherwise shrink to fit (e.g. 0.35 for a 16 GB Mac). | + +**Outputs:** `params` (DIFFORUM_PARAMS), `width` (INT), `height` (INT), `frames` (INT), `fps` (FLOAT), `info` (STRING) + +## 2 · Direction + +### Difforum · Director (timeline) + +`Difforum_Director` + +Direct the whole clip on a visual timeline. + +Three tracks, edited with the mouse: **Scenes** (prompt + mood), **Camera** +(pick a move, set speed / amplitude / lens / easing / audio reaction) and +**Energy** (the denoise curve - how much each moment is re-imagined). The +preview panel plays the camera back with the same engine that renders it. + +One `direction` wire carries camera, strength, cfg and prompt travel into +the Feedback Sampler or a video-model bridge. The separate outputs are +there for custom graphs. + +| input | type | default | notes | +|---|---|---|---| +| `params` | DIFFORUM_PARAMS | | | +| `timeline` | STRING | {"version": 2, "scenes": [{"start": 0... | | +| `camera_mode` | choice (2d, 3d) | 2d | 3d = real parallax when a depth map reaches the sampler (pseudo-3D otherwise). | +| `style` | choice (cinematic, documentary, music_video, psychedelic) | cinematic | Colour lock / detail defaults the sampler uses in 'from director' mode. | +| `transition` | FLOAT | 1.0 | How much of each camera block is spent easing into the next. 0 = hard cuts. | +| `camera_scale` | FLOAT | 1.0 | Master multiplier on every move. | +| `energy_bias` | FLOAT | 0.0 | Shifts the whole denoise curve up or down. | +| `variation` | FLOAT | 0.0 | 0 = exactly as drawn. Above 0, seeded nudges per block. | +| `variation_seed` | INT | 0 | | +| `clip` *(optional)* | CLIP | | Connect to encode the scene prompts (prompt travel). | +| `audio` *(optional)* | DIFFORUM_AUDIO | | From Audio Analyzer - enables per-block audio reactions. | + +**Outputs:** `direction` (DIFFORUM_DIRECTION), `camera` (DIFFORUM_CAMERA), `strength` (DIFFORUM_SCHEDULE), `prompts` (DIFFORUM_PROMPT), `camera_text` (STRING), `info` (STRING) + +### Difforum · Camera (keys) + +`Difforum_Camera` + +Keyframed camera as text, one key per line: + + frame: move [speed] [intensity] [lens] [ease] + +Moves blend into each other (`transition` 1.0 = continuous crane move, +0.0 = hard cuts). `loop_mode` makes the path periodic over `cycle_frames` +so the clip loops without a crossfade. The Director timeline writes the +same thing visually - use this node when you prefer typing. + +| input | type | default | notes | +|---|---|---|---| +| `params` | DIFFORUM_PARAMS | | | +| `keys` | STRING | 0: zoom_in 0.6 0.5 40 ease_in_out 30:... | | +| `mode` | choice (2d, 3d) | 2d | | +| `transition` | FLOAT | 1.0 | | +| `speed` | FLOAT | 1.0 | | +| `loop_mode` | choice (harmonic, zero_mean, off) | off | | +| `cycle_frames` | INT | 0 | 0 = whole clip is one cycle. | +| `harmonics` | INT | 3 | | +| `audio` *(optional)* | DIFFORUM_AUDIO | | | + +**Outputs:** `camera` (DIFFORUM_CAMERA), `cycle_frames` (INT), `info` (STRING) + +### Difforum · Camera (expressions) + +`Difforum_CameraExpr` + +Deforum-style camera from math expressions, per axis. + +`0:(0), 60:(2*sin(2*pi*t/30))`, with `t` (frame), `s` (seconds) and any audio +band (`low`, `beat`...) when audio is connected. Each axis also has an +optional schedule socket (e.g. an Audio Curve) that overrides its text. +Values are per-frame increments; zoom is a per-frame scale factor. + +| input | type | default | notes | +|---|---|---|---| +| `params` | DIFFORUM_PARAMS | | | +| `mode` | choice (2d, 3d) | 2d | | +| `fov` | FLOAT | 40.0 | | +| `translation_x` | STRING | 0:(0) | | +| `translation_y` | STRING | 0:(0) | | +| `translation_z` | STRING | 0:(0) | | +| `rotation_3d_x` | STRING | 0:(0) | | +| `rotation_3d_y` | STRING | 0:(0) | | +| `rotation_3d_z` | STRING | 0:(0.3) | | +| `zoom` | STRING | 0:(1.0 + 0.004*sin(2*pi*t/48)) | | +| `audio` *(optional)* | DIFFORUM_AUDIO | | | +| `translation_x_curve` *(optional)* | DIFFORUM_SCHEDULE | | | +| `translation_y_curve` *(optional)* | DIFFORUM_SCHEDULE | | | +| `translation_z_curve` *(optional)* | DIFFORUM_SCHEDULE | | | +| `rotation_3d_x_curve` *(optional)* | DIFFORUM_SCHEDULE | | | +| `rotation_3d_y_curve` *(optional)* | DIFFORUM_SCHEDULE | | | +| `rotation_3d_z_curve` *(optional)* | DIFFORUM_SCHEDULE | | | +| `zoom_curve` *(optional)* | DIFFORUM_SCHEDULE | | | + +**Outputs:** `camera` (DIFFORUM_CAMERA), `info` (STRING) + +### Difforum · Storyboard + +`Difforum_Storyboard` + +The whole clip warped by the camera, no diffusion - in about a second. + +Same engine as the Feedback Sampler with the sampler switched off, so what +you see is exactly the motion the render will get: contact sheet, the +frames themselves, the camera path from above, and a pacing verdict. + +| input | type | default | notes | +|---|---|---|---| +| `init_image` | IMAGE | | | +| `every_nth` | INT | 8 | | +| `columns` | INT | 6 | | +| `preview_scale` | FLOAT | 0.5 | | +| `direction` *(optional)* | DIFFORUM_DIRECTION | | | +| `params` *(optional)* | DIFFORUM_PARAMS | | | +| `camera` *(optional)* | DIFFORUM_CAMERA | | | +| `depth` *(optional)* | IMAGE | | | +| `symmetry` *(optional)* | choice (none, mirror_h, mirror_v, mirror_quad, kaleidoscope) | none | | + +**Outputs:** `sheet` (IMAGE), `frames` (IMAGE), `camera_path` (IMAGE), `info` (STRING) + +## 3 · Curves & Prompts + +### Difforum · Schedule + +`Difforum_Schedule` + +A per-frame curve from Deforum keyframe syntax: `0:(0.4), 48:(0.6+0.1*sin(t/8))`. + +Variables: t/f (frame), s (seconds), fps, max_f, and every audio band when +an Audio Analyzer is connected. Easing applies between keyframes. + +| input | type | default | notes | +|---|---|---|---| +| `params` | DIFFORUM_PARAMS | | | +| `schedule` | STRING | 0:(0.45), 60:(0.6), 119:(0.45) | | +| `easing` | choice (linear, ease_in, ease_out, ease_in_out, step) | ease_in_out | | +| `audio` *(optional)* | DIFFORUM_AUDIO | | | + +**Outputs:** `schedule` (DIFFORUM_SCHEDULE), `info` (STRING) + +### Difforum · Schedule Plot + +`Difforum_SchedulePlot` + +Draw any schedule as an image (and a one-line summary). + +| input | type | default | notes | +|---|---|---|---| +| `schedule` | DIFFORUM_SCHEDULE | | | +| `width` | INT | 512 | | +| `height` | INT | 200 | | + +**Outputs:** `plot` (IMAGE), `info` (STRING) + +### Difforum · Audio Analyzer + +`Difforum_AudioAnalyzer` + +Audio -> per-frame bands: amp, low, mid, high, onset, beat (0..1). + +Use the bands in any expression (`1 + 0.05*low`), in a Director camera +block's audio reaction, or turn one into a curve with Audio Curve. + +| input | type | default | notes | +|---|---|---|---| +| `params` | DIFFORUM_PARAMS | | | +| `audio` | AUDIO | | | +| `smoothing` | FLOAT | 0.25 | | +| `beat_sensitivity` | FLOAT | 1.5 | | +| `offset_seconds` | FLOAT | 0.0 | Start reading the track here (sync to an edit). | + +**Outputs:** `audio_curves` (DIFFORUM_AUDIO), `bands_plot` (IMAGE), `info` (STRING) + +### Difforum · Audio Curve + +`Difforum_AudioCurve` + +One audio band -> a ready curve: e.g. bass-pumped energy +(`low`, add, base 0.45, amount 0.2) or a beat-pulsed cfg. + +| input | type | default | notes | +|---|---|---|---| +| `params` | DIFFORUM_PARAMS | | | +| `audio_curves` | DIFFORUM_AUDIO | | | +| `source` | choice (amp, low, mid, high, onset, beat) | low | | +| `mode` | choice (add, subtract, multiply) | add | | +| `base` | FLOAT | 0.45 | | +| `amount` | FLOAT | 0.2 | | +| `smoothing` | FLOAT | 0.2 | | + +**Outputs:** `schedule` (DIFFORUM_SCHEDULE) + +### Difforum · Prompt Travel + +`Difforum_PromptTravel` + +Prompt travel from `frame: prompt` lines (or `2.5s: prompt`), blended +per frame. Encodes each prompt once and blends lazily, so long clips on big +text encoders stay light. `build_batched` adds one batched conditioning +(frame i = prompt i) for batch samplers such as AnimateDiff. + +| input | type | default | notes | +|---|---|---|---| +| `params` | DIFFORUM_PARAMS | | | +| `clip` | CLIP | | | +| `prompts` | STRING | 0: a serene misty forest, soft light ... | | +| `easing` | choice (linear, ease_in, ease_out, ease_in_out, step) | ease_in_out | | +| `build_batched` *(optional)* | BOOLEAN | False | Also build the batched output (costs RAM on long clips). | + +**Outputs:** `prompts` (DIFFORUM_PROMPT), `first_frame_cond` (CONDITIONING), `batched` (CONDITIONING), `info` (STRING) + +## 4 · Render + +### Difforum · Feedback Sampler + +`Difforum_FeedbackSampler` + +The Deforum look on any image model: every frame is the last one, +moved by the camera and re-imagined by the sampler. + +Plug a Director `direction` wire and a first frame; that is the whole +setup. Works with SD1.5, SDXL, Flux, SD3.5 and turbo/LCM/DMD2 distills +(plain MODEL / VAE / CONDITIONING). Connect a depth map (e.g. Depth +Anything V2 on the first frame) and set the Director to 3d for real +parallax - the depth then follows the image frame by frame. + +Outputs the frames, the tracked depth per frame (for comp / relight / +video-model guides) and a run report. + +| input | type | default | notes | +|---|---|---|---| +| `model` | MODEL | | | +| `positive` | CONDITIONING | | Used when no prompt travel is connected. | +| `negative` | CONDITIONING | | | +| `vae` | VAE | | | +| `init_image` | IMAGE | | The first frame. | +| `steps` | INT | 20 | | +| `cfg` | FLOAT | 6.0 | | +| `sampler_name` | choice (euler, lcm) | euler | | +| `scheduler` | choice (normal, sgm_uniform) | normal | | +| `cadence` | INT | 1 | Diffuse every Nth frame; the rest are crossfaded warps. ~N x faster. | +| `direction` *(optional)* | DIFFORUM_DIRECTION | | | +| `options` *(optional)* | DIFFORUM_OPTIONS | | Difforum · Render Options, for manual control. | +| `params` *(optional)* | DIFFORUM_PARAMS | | | +| `camera` *(optional)* | DIFFORUM_CAMERA | | | +| `strength` *(optional)* | DIFFORUM_SCHEDULE | | Overrides the Director's energy curve. | +| `cfg_curve` *(optional)* | DIFFORUM_SCHEDULE | | | +| `prompts` *(optional)* | DIFFORUM_PROMPT | | | +| `depth` *(optional)* | IMAGE | | Depth of the first frame (white = near), or a per-frame batch. | +| `control_net` *(optional)* | CONTROL_NET | | | +| `control_image` *(optional)* | IMAGE | | | +| `control_strength` *(optional)* | FLOAT | 0.6 | | +| `energy` *(optional)* | FLOAT | 0.5 | Denoise when neither a Director nor a strength curve is connected. | + +**Outputs:** `frames` (IMAGE), `depth` (IMAGE), `report` (STRING) + +### Difforum · Live Sampler + +`Difforum_LiveSampler` + +Realtime Difforum: queue once and watch it play inside the node. + +Pair with a 1-4 step model (SD-Turbo, SDXL-Turbo, LCM, DMD2) at ~512 px. +`live_source` turns it into a magic mirror (webcam "0" or a video path); +`stream_dir` / `spout_name` feed OBS, Resolume or TouchDesigner. Only the +last `keep_frames` frames are returned, so long sessions do not fill RAM. + +| input | type | default | notes | +|---|---|---|---| +| `model` | MODEL | | | +| `positive` | CONDITIONING | | | +| `negative` | CONDITIONING | | | +| `vae` | VAE | | | +| `init_image` | IMAGE | | | +| `run_frames` | INT | 480 | | +| `steps` | INT | 2 | | +| `cfg` | FLOAT | 1.2 | | +| `sampler_name` | choice (euler, lcm) | lcm | | +| `scheduler` | choice (normal, sgm_uniform) | sgm_uniform | | +| `cadence` | INT | 1 | | +| `target_fps` | FLOAT | 0.0 | 0 = as fast as possible. | +| `direction` *(optional)* | DIFFORUM_DIRECTION | | | +| `options` *(optional)* | DIFFORUM_OPTIONS | | | +| `params` *(optional)* | DIFFORUM_PARAMS | | | +| `camera` *(optional)* | DIFFORUM_CAMERA | | | +| `strength` *(optional)* | DIFFORUM_SCHEDULE | | | +| `prompts` *(optional)* | DIFFORUM_PROMPT | | | +| `depth` *(optional)* | IMAGE | | | +| `energy` *(optional)* | FLOAT | 0.5 | | +| `loop_camera` *(optional)* | BOOLEAN | True | | +| `live_source` *(optional)* | STRING | | '' off, '0' webcam, or a video path. | +| `source_blend` *(optional)* | FLOAT | 0.9 | | +| `stream_dir` *(optional)* | STRING | | Folder to write live PNG frames to. | +| `spout_name` *(optional)* | STRING | | | +| `keep_frames` *(optional)* | INT | 240 | | +| `live_preview` *(optional)* | BOOLEAN | True | | + +**Outputs:** `frames` (IMAGE), `report` (STRING) + +### Difforum · Render Options + +`Difforum_RenderOptions` + +Everything the samplers can fine-tune, kept off the sampler itself. + +Without this node the look (colour lock, sharpen, grain) comes from the +Director's style and the rest uses sensible defaults. Connect it to take +manual control: colour anchoring, detail guard, in-loop symmetry, 3D depth +calibration, noise seeding and chunked rendering. + +| input | type | default | notes | +|---|---|---|---| +| `color_coherence` | FLOAT | 0.8 | How hard colours are held to the anchor. | +| `color_mode` | choice (lab, rgb, none) | lab | | +| `anchor_mode` | choice (scene, first, rolling, none) | scene | scene = re-anchor colour at each prompt scene; first = hold frame 0; rolling = previous key; none = free. | +| `sharpen` | FLOAT | 0.2 | | +| `noise` | FLOAT | 0.02 | | +| `hole_noise` | FLOAT | 0.25 | Extra noise where the camera reveals new area, so it is repainted. | +| `symmetry` | choice (none, mirror_h, mirror_v, mirror_quad, kaleidoscope) | none | | +| `symmetry_segments` | INT | 6 | | +| `border` | choice (reflection, border, zeros) | reflection | | +| `depth_tracking` | choice (follow, static) | follow | follow = depth is re-projected with the image every frame. | +| `translation_scale` | FLOAT | 1.0 | 3D motion strength vs. the depth map's range. | +| `near` | FLOAT | 1.0 | | +| `far` | FLOAT | 100.0 | | +| `invert_depth` | BOOLEAN | False | Enable if near things are dark in your depth map. | +| `seed_mode` | choice (fixed, increment) | fixed | fixed = same noise every frame (calmer texture). | +| `start_frame` | INT | 0 | Feedback Sampler: render a chunk; init_image is the frame at start_frame. | +| `end_frame` | INT | 0 | 0 = to the end. | + +**Outputs:** `options` (DIFFORUM_OPTIONS) + +## 5 · Video model bridges + +### Difforum · Guide Frames + +`Difforum_GuideFrames` + +Warp one anchor image along the Director's camera path. + +Produces a guide video whose motion is exactly the camera you drew, for +video models that take control/guide frames (LTX guides, Wan VACE, H3 +first/last frame). Revealed areas are filled with neutral gray and marked +in the mask, using the convention the target expects. + +| input | type | default | notes | +|---|---|---|---| +| `anchor_image` | IMAGE | | | +| `mask_convention` | choice (1 = generate (VACE / LTX / H3), 1 = keep) | 1 = generate (VACE / LTX / H3) | | +| `hole_fill` | choice (gray, stretch edge, black) | gray | | +| `direction` *(optional)* | DIFFORUM_DIRECTION | | | +| `params` *(optional)* | DIFFORUM_PARAMS | | | +| `camera` *(optional)* | DIFFORUM_CAMERA | | | +| `depth` *(optional)* | IMAGE | | Depth of the anchor, for real parallax in 3d mode. | +| `translation_scale` *(optional)* | FLOAT | 1.0 | | + +**Outputs:** `guide_frames` (IMAGE), `masks` (MASK), `info` (STRING) + +### Difforum · Keyframes + +`Difforum_Keyframes` + +Pick keyframes out of any frame batch (a Storyboard, a Feedback render, +guide frames) for a video model, on that model's frame grid. + +Outputs the keyframes, their indices, a full-length *sparse* batch (black +except at keyframes - what `LTXVAddGuidesFromBatch` expects), the first and +last frame (for first-last-frame models) and the snapped clip length. + +| input | type | default | notes | +|---|---|---|---| +| `frames` | IMAGE | | | +| `grid` | choice (LTX-2 / 2.5 (8k+1), MiniMax H3 (17k+5), Wan 2.x (4k+1), any) | LTX-2 / 2.5 (8k+1) | | +| `every_seconds` | FLOAT | 1.0 | Spacing between keyframes. 0 = first and last only. | +| `fps` | FLOAT | 24.0 | | +| `indices` *(optional)* | STRING | | Explicit frame list, e.g. 0, 48, 96, -1 (overrides spacing). | + +**Outputs:** `keyframes` (IMAGE), `indices` (STRING), `sparse_batch` (IMAGE), `first_frame` (IMAGE), `last_frame` (IMAGE), `length` (INT), `info` (STRING) + +### Difforum · Camera → Prompt + +`Difforum_CameraPrompt` + +The camera, in words - for prompt-driven video models (MiniMax H3, +LTX, Seedance, Veo...). Uses the Director's blocks when available, or +reads any camera track and names its moves. + +| input | type | default | notes | +|---|---|---|---| +| `format` | choice (sentence, timed lines, prompt suffix) | sentence | | +| `direction` *(optional)* | DIFFORUM_DIRECTION | | | +| `params` *(optional)* | DIFFORUM_PARAMS | | | +| `camera` *(optional)* | DIFFORUM_CAMERA | | | +| `prefix` *(optional)* | STRING | | Your shot description; the camera sentence is appended. | + +**Outputs:** `text` (STRING) + +### Difforum · LTX Guides + +`Difforum_LTXGuides` + +Put Difforum keyframes into an LTX-2 / 2.5 latent as guides. + +Wraps ComfyUI's core `LTXVAddGuide` once per keyframe, so the camera you +drew becomes the LTX shot. Feed keyframes + indices from the Keyframes +node (from a Storyboard, Guide Frames or a Feedback render), and an empty +LTX latent whose length matches (Setup target = LTX). + +| input | type | default | notes | +|---|---|---|---| +| `positive` | CONDITIONING | | | +| `negative` | CONDITIONING | | | +| `vae` | VAE | | | +| `latent` | LATENT | | | +| `keyframes` | IMAGE | | | +| `indices` | STRING | 0 | | +| `strength` | FLOAT | 0.7 | Guide strength for in-between keyframes. | +| `first_strength` | FLOAT | 1.0 | Frame 0 usually locks the look: keep it high. | +| `last_strength` | FLOAT | 0.7 | | + +**Outputs:** `positive` (CONDITIONING), `negative` (CONDITIONING), `latent` (LATENT), `info` (STRING) + +### Difforum · H3 Shot + +`Difforum_H3Shot` + +Everything a MiniMax H3 first-last-frame shot needs, from Difforum. + +Give it the frames of a Storyboard / Feedback render / Guide Frames and it +returns the first and last frame of the chosen segment, the length on the +H3 17k+5 grid, the size on the 32 px grid, and the camera move of that +segment in words for the prompt. Clips longer than one H3 generation are +split into segments: render segment 0, then feed its last frame as the +next segment's first frame. + +| input | type | default | notes | +|---|---|---|---| +| `frames` | IMAGE | | | +| `segment_length` | choice (auto (from frames), 124 (~5s), 243 (~10s), 362 (~15s), 481 (~20s)) | auto (from frames) | | +| `segment` | INT | 0 | | +| `direction` *(optional)* | DIFFORUM_DIRECTION | | | +| `params` *(optional)* | DIFFORUM_PARAMS | | | +| `camera` *(optional)* | DIFFORUM_CAMERA | | | +| `shot_description` *(optional)* | STRING | | | + +**Outputs:** `first_frame` (IMAGE), `last_frame` (IMAGE), `length` (INT), `width` (INT), `height` (INT), `prompt` (STRING), `segments` (INT), `info` (STRING) + +### Difforum · H3 Guides + +`Difforum_H3Guides` + +Anchor Difforum keyframes inside a MiniMax H3 generation. + +Wraps ComfyUI's core `MiniMaxH3AddGuide` once per keyframe, so the camera +you drew becomes the H3 shot: feed keyframes + indices from the Keyframes +node (grid H3) and the positive + AV latent from `MiniMax H3 Reference to +Video` (or `Image to Video`). Optionally anchor a soundtrack at frame 0, so +an audio-reactive direction and H3's own audio stay in sync. + +H3 is trained with a few guides per clip: `max_guides` keeps the first, +the last and evenly spaced ones in between. + +| input | type | default | notes | +|---|---|---|---| +| `positive` | CONDITIONING | | | +| `latent` | LATENT | | The MiniMax H3 AV latent. | +| `vae` | VAE | | MiniMax H3 video VAE. | +| `keyframes` | IMAGE | | | +| `indices` | STRING | 0 | | +| `max_guides` | INT | 4 | | +| `skip_first` | BOOLEAN | False | Enable when frame 0 is already set (e.g. Image to Video first_frame). | +| `audio_vae` *(optional)* | VAE | | MiniMax H3 audio VAE, needed with audio. | +| `audio` *(optional)* | AUDIO | | Soundtrack anchored at frame 0. | + +**Outputs:** `positive` (CONDITIONING), `info` (STRING) + +## 6 · Export + +### Difforum · Camera Export (AE / Blender / JSON) + +`Difforum_CameraExport` + +Send the Difforum camera to After Effects, Blender or any tool (JSON). + +Writes to `output/difforum/`: + * `.jsx` - run in After Effects: a 3D camera (3d mode) or a transform + null that reproduces the 2D move exactly (2d mode). + * `.py` - run in Blender's Text Editor: an animated camera, fps, + resolution and frame range matched. + * `.json` - camera-to-world matrices + focal length per frame. +Composite titles, 3D elements or a relight pass on top of the AI render +with the exact same camera. + +| input | type | default | notes | +|---|---|---|---| +| `formats` | choice (all, after effects, blender, json) | all | | +| `filename_prefix` | STRING | difforum_camera | | +| `blender_unit_scale` | FLOAT | 0.1 | Metres per Difforum scene unit. | +| `direction` *(optional)* | DIFFORUM_DIRECTION | | | +| `params` *(optional)* | DIFFORUM_PARAMS | | | +| `camera` *(optional)* | DIFFORUM_CAMERA | | | +| `translation_scale` *(optional)* | FLOAT | 1.0 | Match the sampler's value so the exported move matches the render. | + +**Outputs:** `paths` (STRING), `json` (STRING) + +### Difforum · Camera Import + +`Difforum_CameraImport` + +Bring a camera in from Blender, After Effects or JSON. + +Accepts the files written by Camera Export and by the helper scripts in +`tools/` (export the active Blender camera, or the selected After Effects +camera). The result drives the Feedback Sampler, Storyboard or Guide +Frames like any Difforum camera, so a move blocked in 3D becomes the AI +shot. + +| input | type | default | notes | +|---|---|---|---| +| `file` | choice ((put a camera .json in ComfyUI/input)) | | | +| `params` | DIFFORUM_PARAMS | | | +| `retime` | choice (match frames, keep source length) | match frames | | +| `json_text` *(optional)* | STRING | | Paste JSON here instead of choosing a file. | + +**Outputs:** `camera` (DIFFORUM_CAMERA), `info` (STRING) + +## 7 · Post + +### Difforum · Loop + +`Difforum_Loop` + +Make a clip loop. + +* keep settled lap - for renders made with a looping camera (Director / + Camera loop_mode) over several laps: keeps the last lap, no blending. +* flow crossfade - morphs the tail into the head along optical flow. +* ping-pong - forward then backward (motion reverses). +Reports how visible the seam is. + +| input | type | default | notes | +|---|---|---|---| +| `frames` | IMAGE | | | +| `method` | choice (keep settled lap, flow crossfade, ping-pong) | flow crossfade | | +| `cycle_frames` | INT | 0 | keep settled lap: frames per lap (the Camera node outputs it). | +| `blend_frames` | INT | 12 | flow crossfade: overlap length. | + +**Outputs:** `frames` (IMAGE), `seam_info` (STRING) + +### Difforum · Symmetry + +`Difforum_Symmetry` + +Mirror or kaleidoscope a frame or batch (the Feedback Sampler can also +do it inside the loop, where it compounds into a living pattern). + +| input | type | default | notes | +|---|---|---|---| +| `image` | IMAGE | | | +| `mode` | choice (none, mirror_h, mirror_v, mirror_quad, kaleidoscope) | kaleidoscope | | +| `segments` | INT | 6 | | +| `mix` | FLOAT | 1.0 | | +| `flip` *(optional)* | BOOLEAN | False | | +| `center_x` *(optional)* | FLOAT | 0.5 | | +| `center_y` *(optional)* | FLOAT | 0.5 | | +| `angle` *(optional)* | FLOAT | 0.0 | | + +**Outputs:** `image` (IMAGE) + +### Difforum · Echo Trails + +`Difforum_EchoTrails` + +Long-exposure trails across a frame batch. + +| input | type | default | notes | +|---|---|---|---| +| `frames` | IMAGE | | | +| `decay` | FLOAT | 0.6 | | +| `mix` | FLOAT | 0.5 | | + +**Outputs:** `frames` (IMAGE) + +### Difforum · Flow Stabilize + +`Difforum_FlowStabilize` + +Anti-flicker: blends history along optical flow, only where it matches, +so texture stops boiling but motion never ghosts. + +| input | type | default | notes | +|---|---|---|---| +| `frames` | IMAGE | | | +| `strength` | FLOAT | 0.5 | | +| `flow_scale` | FLOAT | 0.5 | | +| `error_gate` | FLOAT | 0.15 | | + +**Outputs:** `frames` (IMAGE) + +### Difforum · Detail Guard + +`Difforum_DetailGuard` + +Unsharp + contrast + grain, for frames that went soft. + +| input | type | default | notes | +|---|---|---|---| +| `image` | IMAGE | | | +| `sharpen` | FLOAT | 0.3 | | +| `contrast` | FLOAT | 1.0 | | +| `grain` | FLOAT | 0.0 | | +| `grain_mode` | choice (gaussian, plasma) | gaussian | | + +**Outputs:** `image` (IMAGE) diff --git a/PERFORMANCE.md b/docs/PERFORMANCE.md similarity index 94% rename from PERFORMANCE.md rename to docs/PERFORMANCE.md index f5f7450..5e40a91 100644 --- a/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -5,7 +5,7 @@ sampler calls in a row, so anything that costs a little per frame costs a lot per clip - and on unified memory the ceiling arrives long before the clock does. The same handful of levers fixes both. -Measured on the `advanced/difforum_film_director.json` graph, 768x432, 120 +Measured on the 0.x Film Director graph (the v1 engine adds cadence crossfades, so the same settings are no slower), 768x432, 120 frames, SDXL + DMD2, M-series with 24 GB unified: | stage | time | share | @@ -85,7 +85,7 @@ more coherent; too low and the loop stops evolving and mushes. - `0.55-0.65` active transformation, for climax beats - `> 0.7` the frame stops being a continuation of the last one -The Film Director's mood presets already sit in this band. `strength_bias` +The Director's mood presets already sit in this band. `energy_bias` shifts the whole clip at once - the fastest single knob for a speed/quality trade. @@ -171,12 +171,12 @@ every setting on this page combined. For a piece that projects for hours, render once at the highest quality the machine allows and loop it properly: -- **Camera Keys** with `loop_mode = harmonic` makes the path periodic, so the +- **Difforum · Camera (keys)** with `loop_mode = harmonic` makes the path periodic, so the last frame flows into the first with the velocity matched. - Render **3 laps** of the cycle and keep the last with **Loop Take**. The early laps let the feedback image settle onto its cycle. - Nothing is blended, so there is no dissolve for the eye to find on repeat. -`advanced/difforum_seamless_installation.json` is wired this way. Three laps +`example_workflows/06_seamless_loop.json` is wired this way. Three laps means three times the render for a clip that plays forever - the right trade for an installation, the wrong one while you are still exploring. diff --git a/examples/PROMPTS.md b/docs/PROMPTS.md similarity index 91% rename from examples/PROMPTS.md rename to docs/PROMPTS.md index 3d223fc..a2cc0a1 100644 --- a/examples/PROMPTS.md +++ b/docs/PROMPTS.md @@ -1,16 +1,17 @@ # Difforum prompt pack A curated set of prompt-travel presets to explore quality and motion. Each one -is ready to paste into **Difforum - Prompt Schedule (travel)** (the `prompts` -box). The frame numbers assume `max_frames` around 48; scale them to your clip -length. Settings under each preset are starting points, not rules. +works as scenes on the **Director** timeline (one scene per line), or pasted +into **Difforum · Prompt Travel** (`frame: prompt` or `2.5s: prompt` per line). +The frame numbers assume a clip of about 48 frames; scale them to yours. +Settings under each preset are starting points, not rules. How to read a preset: -- **Travel** goes into the Prompt Schedule node (`frame: prompt` per line). -- **Camera** values go into the Camera node (advanced) fields, or pick the - closest Camera Move preset. -- **Strength** is the per-frame denoise (Schedule node). Lower = more coherent +- **Travel** = the scenes, in order. +- **Camera** = the closest Director moves; the expression values also work in + **Difforum · Camera (expressions)**. +- **Strength** = the Director's Energy curve (denoise). Lower = more coherent and smoother, higher = more change per frame. - **Symmetry** refers to the Feedback Sampler's `symmetry` option. @@ -93,7 +94,7 @@ Built for the in-loop symmetry. Mesmerizing, hypnotic. - Camera: mode `2d`, rotation_z `0:(0.7)`, zoom `0:(1.012)` - Strength `0:(0.42)`, **Symmetry `kaleidoscope`, segments 6**, Echo Trails on -- See `difforum_mesmerize_kaleidoscope.json` +- Set the Feedback Sampler `symmetry` to kaleidoscope ## 5. Neon City Flythrough @@ -165,7 +166,7 @@ Drive zoom and strength from the audio. Connect Audio Analyzer to the schedules. - Zoom schedule: `0:(1.0 + 0.6*low)` (bass pumps the zoom) - Strength schedule: `0:(0.35 + 0.4*beat)` (beats add change) -- cfg 7.0, see `difforum_audio_reactive_video.json` +- cfg 7.0, see `example_workflows/04_audio_reactive.json` ## 10. Dreamscape Morph diff --git a/docs/media/audio.webp b/docs/media/audio.webp new file mode 100644 index 0000000..c8ca4c9 Binary files /dev/null and b/docs/media/audio.webp differ diff --git a/docs/media/detail_ab.jpg b/docs/media/detail_ab.jpg new file mode 100644 index 0000000..be1ba0c Binary files /dev/null and b/docs/media/detail_ab.jpg differ diff --git a/docs/media/director.jpg b/docs/media/director.jpg new file mode 100644 index 0000000..8b27720 Binary files /dev/null and b/docs/media/director.jpg differ diff --git a/docs/media/icon.png b/docs/media/icon.png new file mode 100644 index 0000000..15da691 Binary files /dev/null and b/docs/media/icon.png differ diff --git a/docs/media/kaleidoscope.webp b/docs/media/kaleidoscope.webp new file mode 100644 index 0000000..d8fb024 Binary files /dev/null and b/docs/media/kaleidoscope.webp differ diff --git a/docs/media/promo.webp b/docs/media/promo.webp new file mode 100644 index 0000000..f8556f4 Binary files /dev/null and b/docs/media/promo.webp differ diff --git a/docs/media/showcase.webp b/docs/media/showcase.webp new file mode 100644 index 0000000..9d88b22 Binary files /dev/null and b/docs/media/showcase.webp differ diff --git a/example_workflows/01_storyboard_no_model.json b/example_workflows/01_storyboard_no_model.json new file mode 100644 index 0000000..a270a57 --- /dev/null +++ b/example_workflows/01_storyboard_no_model.json @@ -0,0 +1,520 @@ +{ + "last_node_id": 9, + "last_link_id": 7, + "nodes": [ + { + "id": 1, + "type": "MarkdownNote", + "pos": [ + 0, + -300 + ], + "size": [ + 520, + 260 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "## Start here - no model needed\n\n1. Load any still in **Load Image**.\n2. Edit the **Director** timeline: drag blocks, pick camera moves, press ▶ to preview.\n3. Queue: the **Storyboard** renders the whole camera move in about a second, and **Camera → Prompt** writes it in words for prompt-driven video models.\n\nWhen the motion feels right, move to 02 (render) or 07/08 (LTX / H3)." + ] + }, + { + "id": 2, + "type": "Difforum_Setup", + "pos": [ + 0, + 0 + ], + "size": [ + 320, + 300 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "links": [ + 1 + ], + "slot_index": 0 + }, + { + "name": "width", + "type": "INT", + "links": [], + "slot_index": 1 + }, + { + "name": "height", + "type": "INT", + "links": [], + "slot_index": 2 + }, + { + "name": "frames", + "type": "INT", + "links": [], + "slot_index": 3 + }, + { + "name": "fps", + "type": "FLOAT", + "links": [], + "slot_index": 4 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 5 + } + ], + "properties": { + "Node name for S&R": "Difforum_Setup" + }, + "widgets_values": [ + "feedback (SD/SDXL/Flux)", + "seconds", + 5.0, + 24.0, + "16:9 landscape", + 768, + 0, + "fixed", + 16.0, + 9.0, + 0.0 + ] + }, + { + "id": 3, + "type": "Difforum_Director", + "pos": [ + 360, + 0 + ], + "size": [ + 800, + 880 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": 1 + }, + { + "name": "clip", + "type": "CLIP", + "link": null + }, + { + "name": "audio", + "type": "DIFFORUM_AUDIO", + "link": null + } + ], + "outputs": [ + { + "name": "direction", + "type": "DIFFORUM_DIRECTION", + "links": [ + 3, + 6 + ], + "slot_index": 0 + }, + { + "name": "camera", + "type": "DIFFORUM_CAMERA", + "links": [], + "slot_index": 1 + }, + { + "name": "strength", + "type": "DIFFORUM_SCHEDULE", + "links": [], + "slot_index": 2 + }, + { + "name": "prompts", + "type": "DIFFORUM_PROMPT", + "links": [], + "slot_index": 3 + }, + { + "name": "camera_text", + "type": "STRING", + "links": [], + "slot_index": 4 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 5 + } + ], + "properties": { + "Node name for S&R": "Difforum_Director" + }, + "widgets_values": [ + "{\"version\": 2, \"scenes\": [{\"start\": 0, \"mood\": \"calm\", \"prompt\": \"misty ancient forest at dawn, volumetric light, wide shot\"}, {\"start\": 30, \"mood\": \"build\", \"prompt\": \"glowing roots and moss, bioluminescent details, macro\"}, {\"start\": 60, \"mood\": \"climax\", \"prompt\": \"explosion of light, cosmic transformation, high contrast\"}, {\"start\": 90, \"mood\": \"resolve\", \"prompt\": \"embers becoming stars, calm, vast cosmic scale\"}], \"camera\": [{\"start\": 0, \"move\": \"zoom_in\", \"speed\": 0.8, \"intensity\": 0.8, \"lens\": 0.0, \"ease\": \"ease_in_out\", \"react\": \"none\"}, {\"start\": 30, \"move\": \"pan_right\", \"speed\": 1.0, \"intensity\": 0.8, \"lens\": 0.0, \"ease\": \"ease_in_out\", \"react\": \"none\"}, {\"start\": 60, \"move\": \"spiral\", \"speed\": 1.4, \"intensity\": 1.1, \"lens\": 0.0, \"ease\": \"ease_in\", \"react\": \"none\"}, {\"start\": 90, \"move\": \"zoom_out\", \"speed\": 0.6, \"intensity\": 0.5, \"lens\": 0.0, \"ease\": \"ease_out\", \"react\": \"none\"}], \"energy\": [], \"guidance\": []}", + "2d", + "cinematic", + 1.0, + 1.0, + 0.0, + 0.0, + 0, + "fixed" + ] + }, + { + "id": 4, + "type": "LoadImage", + "pos": [ + 0, + 360 + ], + "size": [ + 320, + 360 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 2 + ], + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": [], + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "example.png", + null + ] + }, + { + "id": 5, + "type": "Difforum_Storyboard", + "pos": [ + 1200, + 0 + ], + "size": [ + 320, + 220 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "init_image", + "type": "IMAGE", + "link": 2 + }, + { + "name": "direction", + "type": "DIFFORUM_DIRECTION", + "link": 3 + }, + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": null + }, + { + "name": "camera", + "type": "DIFFORUM_CAMERA", + "link": null + }, + { + "name": "depth", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "sheet", + "type": "IMAGE", + "links": [ + 4 + ], + "slot_index": 0 + }, + { + "name": "frames", + "type": "IMAGE", + "links": [], + "slot_index": 1 + }, + { + "name": "camera_path", + "type": "IMAGE", + "links": [ + 5 + ], + "slot_index": 2 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 3 + } + ], + "properties": { + "Node name for S&R": "Difforum_Storyboard" + }, + "widgets_values": [ + 8, + 6, + 0.5, + "none" + ] + }, + { + "id": 6, + "type": "PreviewImage", + "pos": [ + 1560, + 0 + ], + "size": [ + 620, + 380 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 4 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [], + "title": "Contact sheet" + }, + { + "id": 7, + "type": "PreviewImage", + "pos": [ + 1560, + 420 + ], + "size": [ + 300, + 300 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 5 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [], + "title": "Camera path (top view)" + }, + { + "id": 8, + "type": "Difforum_CameraPrompt", + "pos": [ + 1200, + 280 + ], + "size": [ + 320, + 160 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "direction", + "type": "DIFFORUM_DIRECTION", + "link": 6 + }, + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": null + }, + { + "name": "camera", + "type": "DIFFORUM_CAMERA", + "link": null + } + ], + "outputs": [ + { + "name": "text", + "type": "STRING", + "links": [ + 7 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "Difforum_CameraPrompt" + }, + "widgets_values": [ + "sentence", + "" + ] + }, + { + "id": 9, + "type": "PreviewAny", + "pos": [ + 1200, + 480 + ], + "size": [ + 320, + 200 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "source", + "type": "*", + "link": 7 + } + ], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "PreviewAny" + }, + "widgets_values": [], + "title": "Camera in words" + } + ], + "links": [ + [ + 1, + 2, + 0, + 3, + 0, + "DIFFORUM_PARAMS" + ], + [ + 2, + 4, + 0, + 5, + 0, + "IMAGE" + ], + [ + 3, + 3, + 0, + 5, + 1, + "DIFFORUM_DIRECTION" + ], + [ + 4, + 5, + 0, + 6, + 0, + "IMAGE" + ], + [ + 5, + 5, + 2, + 7, + 0, + "IMAGE" + ], + [ + 6, + 3, + 0, + 8, + 0, + "DIFFORUM_DIRECTION" + ], + [ + 7, + 8, + 0, + 9, + 0, + "STRING" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.65, + "offset": [ + 40, + 40 + ] + }, + "difforum": "1.0", + "title": "01 · Storyboard - direct a shot, no model needed" + }, + "version": 0.4 +} diff --git a/example_workflows/02_feedback_sdxl.json b/example_workflows/02_feedback_sdxl.json new file mode 100644 index 0000000..2889dc9 --- /dev/null +++ b/example_workflows/02_feedback_sdxl.json @@ -0,0 +1,962 @@ +{ + "last_node_id": 14, + "last_link_id": 23, + "nodes": [ + { + "id": 1, + "type": "CheckpointLoaderSimple", + "pos": [ + 0, + 0 + ], + "size": [ + 320, + 100 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 7, + 13 + ], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 1, + 2, + 4 + ], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 12, + 14 + ], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "sd_xl_base_1.0.safetensors" + ] + }, + { + "id": 2, + "type": "CLIPTextEncode", + "pos": [ + 360, + 0 + ], + "size": [ + 380, + 140 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 8, + 15 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "cinematic film still, rich detail, volumetric light, 35mm" + ], + "title": "Look / style (positive)" + }, + { + "id": 3, + "type": "CLIPTextEncode", + "pos": [ + 360, + 180 + ], + "size": [ + 380, + 120 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 2 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 9, + 16 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "blurry, low quality, watermark, text, frame, border" + ], + "title": "Negative" + }, + { + "id": 4, + "type": "Difforum_Setup", + "pos": [ + 0, + 360 + ], + "size": [ + 320, + 300 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "links": [ + 3 + ], + "slot_index": 0 + }, + { + "name": "width", + "type": "INT", + "links": [ + 5 + ], + "slot_index": 1 + }, + { + "name": "height", + "type": "INT", + "links": [ + 6 + ], + "slot_index": 2 + }, + { + "name": "frames", + "type": "INT", + "links": [], + "slot_index": 3 + }, + { + "name": "fps", + "type": "FLOAT", + "links": [ + 21 + ], + "slot_index": 4 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 5 + } + ], + "properties": { + "Node name for S&R": "Difforum_Setup" + }, + "widgets_values": [ + "feedback (SD/SDXL/Flux)", + "seconds", + 5.0, + 24.0, + "16:9 landscape", + 1024, + 0, + "fixed", + 16.0, + 9.0, + 0.0 + ] + }, + { + "id": 5, + "type": "Difforum_Director", + "pos": [ + 420, + 360 + ], + "size": [ + 800, + 880 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": 3 + }, + { + "name": "clip", + "type": "CLIP", + "link": 4 + }, + { + "name": "audio", + "type": "DIFFORUM_AUDIO", + "link": null + } + ], + "outputs": [ + { + "name": "direction", + "type": "DIFFORUM_DIRECTION", + "links": [ + 18 + ], + "slot_index": 0 + }, + { + "name": "camera", + "type": "DIFFORUM_CAMERA", + "links": [], + "slot_index": 1 + }, + { + "name": "strength", + "type": "DIFFORUM_SCHEDULE", + "links": [], + "slot_index": 2 + }, + { + "name": "prompts", + "type": "DIFFORUM_PROMPT", + "links": [], + "slot_index": 3 + }, + { + "name": "camera_text", + "type": "STRING", + "links": [], + "slot_index": 4 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 5 + } + ], + "properties": { + "Node name for S&R": "Difforum_Director" + }, + "widgets_values": [ + "{\"version\": 2, \"scenes\": [{\"start\": 0, \"mood\": \"calm\", \"prompt\": \"misty ancient forest at dawn, volumetric light, wide shot\"}, {\"start\": 30, \"mood\": \"build\", \"prompt\": \"glowing roots and moss, bioluminescent details, macro\"}, {\"start\": 60, \"mood\": \"climax\", \"prompt\": \"explosion of light, cosmic transformation, high contrast\"}, {\"start\": 90, \"mood\": \"resolve\", \"prompt\": \"embers becoming stars, calm, vast cosmic scale\"}], \"camera\": [{\"start\": 0, \"move\": \"zoom_in\", \"speed\": 0.8, \"intensity\": 0.8, \"lens\": 0.0, \"ease\": \"ease_in_out\", \"react\": \"none\"}, {\"start\": 30, \"move\": \"pan_right\", \"speed\": 1.0, \"intensity\": 0.8, \"lens\": 0.0, \"ease\": \"ease_in_out\", \"react\": \"none\"}, {\"start\": 60, \"move\": \"spiral\", \"speed\": 1.4, \"intensity\": 1.1, \"lens\": 0.0, \"ease\": \"ease_in\", \"react\": \"none\"}, {\"start\": 90, \"move\": \"zoom_out\", \"speed\": 0.6, \"intensity\": 0.5, \"lens\": 0.0, \"ease\": \"ease_out\", \"react\": \"none\"}], \"energy\": [], \"guidance\": []}", + "2d", + "cinematic", + 1.0, + 1.0, + 0.0, + 0.0, + 0, + "fixed" + ] + }, + { + "id": 6, + "type": "EmptyLatentImage", + "pos": [ + 820, + 0 + ], + "size": [ + 260, + 110 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "width", + "type": "INT", + "link": 5, + "widget": { + "name": "width" + } + }, + { + "name": "height", + "type": "INT", + "link": 6, + "widget": { + "name": "height" + } + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 10 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 1024, + 576, + 1 + ] + }, + { + "id": 7, + "type": "KSampler", + "pos": [ + 1110, + 0 + ], + "size": [ + 300, + 260 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 7 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 8 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 9 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 10 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 11 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 7, + "fixed", + 24, + 6.0, + "dpmpp_2m", + "karras", + 1.0 + ], + "title": "First frame" + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [ + 1440, + 0 + ], + "size": [ + 200, + 60 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 11 + }, + { + "name": "vae", + "type": "VAE", + "link": 12 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 17 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + }, + "widgets_values": [] + }, + { + "id": 9, + "type": "Difforum_FeedbackSampler", + "pos": [ + 1260, + 360 + ], + "size": [ + 360, + 460 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 13 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 15 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 16 + }, + { + "name": "vae", + "type": "VAE", + "link": 14 + }, + { + "name": "init_image", + "type": "IMAGE", + "link": 17 + }, + { + "name": "direction", + "type": "DIFFORUM_DIRECTION", + "link": 18 + }, + { + "name": "options", + "type": "DIFFORUM_OPTIONS", + "link": null + }, + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": null + }, + { + "name": "camera", + "type": "DIFFORUM_CAMERA", + "link": null + }, + { + "name": "strength", + "type": "DIFFORUM_SCHEDULE", + "link": null + }, + { + "name": "cfg_curve", + "type": "DIFFORUM_SCHEDULE", + "link": null + }, + { + "name": "prompts", + "type": "DIFFORUM_PROMPT", + "link": null + }, + { + "name": "depth", + "type": "IMAGE", + "link": null + }, + { + "name": "control_net", + "type": "CONTROL_NET", + "link": null + }, + { + "name": "control_image", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "frames", + "type": "IMAGE", + "links": [ + 19 + ], + "slot_index": 0 + }, + { + "name": "depth", + "type": "IMAGE", + "links": [], + "slot_index": 1 + }, + { + "name": "report", + "type": "STRING", + "links": [ + 23 + ], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "Difforum_FeedbackSampler" + }, + "widgets_values": [ + 20, + 6.0, + "dpmpp_2m", + "karras", + 2, + 0.6, + 0.5 + ] + }, + { + "id": 10, + "type": "Difforum_FlowStabilize", + "pos": [ + 1660, + 360 + ], + "size": [ + 300, + 130 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "frames", + "type": "IMAGE", + "link": 19 + } + ], + "outputs": [ + { + "name": "frames", + "type": "IMAGE", + "links": [ + 20 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "Difforum_FlowStabilize" + }, + "widgets_values": [ + 0.45, + 0.5, + 0.15 + ] + }, + { + "id": 11, + "type": "CreateVideo", + "pos": [ + 2000, + 360 + ], + "size": [ + 260, + 100 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 20 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null + }, + { + "name": "fps", + "type": "FLOAT", + "link": 21, + "widget": { + "name": "fps" + } + } + ], + "outputs": [ + { + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 22 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CreateVideo" + }, + "widgets_values": [ + 24.0 + ] + }, + { + "id": 12, + "type": "SaveVideo", + "pos": [ + 2000, + 500 + ], + "size": [ + 300, + 320 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "video", + "type": "VIDEO", + "link": 22 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "video/difforum", + "auto", + "auto" + ] + }, + { + "id": 13, + "type": "PreviewAny", + "pos": [ + 1660, + 540 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "source", + "type": "*", + "link": 23 + } + ], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "PreviewAny" + }, + "widgets_values": [], + "title": "Run report" + }, + { + "id": 14, + "type": "MarkdownNote", + "pos": [ + 0, + -320 + ], + "size": [ + 560, + 260 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "## Feedback render (SDXL)\n\nThe classic Deforum look on a modern model. **Director** drives camera, energy and prompt travel through one `direction` wire.\n\n- Any SD1.5 / SDXL / Flux checkpoint works (plain MODEL/VAE/CONDITIONING).\n- `cadence 2` diffuses every other frame and crossfades the rest (~2x faster).\n- Faster still: a DMD2 / Lightning / Turbo LoRA at 4-8 steps, cfg 1-2.\n- Director prompts replace the positive prompt once CLIP is connected; the positive box still sets the first frame." + ] + } + ], + "links": [ + [ + 1, + 1, + 1, + 2, + 0, + "CLIP" + ], + [ + 2, + 1, + 1, + 3, + 0, + "CLIP" + ], + [ + 3, + 4, + 0, + 5, + 0, + "DIFFORUM_PARAMS" + ], + [ + 4, + 1, + 1, + 5, + 1, + "CLIP" + ], + [ + 5, + 4, + 1, + 6, + 0, + "INT" + ], + [ + 6, + 4, + 2, + 6, + 1, + "INT" + ], + [ + 7, + 1, + 0, + 7, + 0, + "MODEL" + ], + [ + 8, + 2, + 0, + 7, + 1, + "CONDITIONING" + ], + [ + 9, + 3, + 0, + 7, + 2, + "CONDITIONING" + ], + [ + 10, + 6, + 0, + 7, + 3, + "LATENT" + ], + [ + 11, + 7, + 0, + 8, + 0, + "LATENT" + ], + [ + 12, + 1, + 2, + 8, + 1, + "VAE" + ], + [ + 13, + 1, + 0, + 9, + 0, + "MODEL" + ], + [ + 14, + 1, + 2, + 9, + 3, + "VAE" + ], + [ + 15, + 2, + 0, + 9, + 1, + "CONDITIONING" + ], + [ + 16, + 3, + 0, + 9, + 2, + "CONDITIONING" + ], + [ + 17, + 8, + 0, + 9, + 4, + "IMAGE" + ], + [ + 18, + 5, + 0, + 9, + 5, + "DIFFORUM_DIRECTION" + ], + [ + 19, + 9, + 0, + 10, + 0, + "IMAGE" + ], + [ + 20, + 10, + 0, + 11, + 0, + "IMAGE" + ], + [ + 21, + 4, + 4, + 11, + 2, + "FLOAT" + ], + [ + 22, + 11, + 0, + 12, + 0, + "VIDEO" + ], + [ + 23, + 9, + 2, + 13, + 0, + "STRING" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.65, + "offset": [ + 40, + 40 + ] + }, + "difforum": "1.0", + "title": "02 · Feedback render (SDXL)" + }, + "version": 0.4 +} diff --git a/example_workflows/03_parallax_3d_depth.json b/example_workflows/03_parallax_3d_depth.json new file mode 100644 index 0000000..af95cac --- /dev/null +++ b/example_workflows/03_parallax_3d_depth.json @@ -0,0 +1,1060 @@ +{ + "last_node_id": 16, + "last_link_id": 26, + "nodes": [ + { + "id": 1, + "type": "CheckpointLoaderSimple", + "pos": [ + 0, + 0 + ], + "size": [ + 320, + 100 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 7, + 13 + ], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 1, + 2, + 4 + ], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 12, + 14 + ], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "sd_xl_base_1.0.safetensors" + ] + }, + { + "id": 2, + "type": "CLIPTextEncode", + "pos": [ + 360, + 0 + ], + "size": [ + 380, + 140 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 8, + 15 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "cinematic film still, rich detail, volumetric light, 35mm" + ], + "title": "Look / style (positive)" + }, + { + "id": 3, + "type": "CLIPTextEncode", + "pos": [ + 360, + 180 + ], + "size": [ + 380, + 120 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 2 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 9, + 16 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "blurry, low quality, watermark, text, frame, border" + ], + "title": "Negative" + }, + { + "id": 4, + "type": "Difforum_Setup", + "pos": [ + 0, + 360 + ], + "size": [ + 320, + 300 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "links": [ + 3 + ], + "slot_index": 0 + }, + { + "name": "width", + "type": "INT", + "links": [ + 5 + ], + "slot_index": 1 + }, + { + "name": "height", + "type": "INT", + "links": [ + 6 + ], + "slot_index": 2 + }, + { + "name": "frames", + "type": "INT", + "links": [], + "slot_index": 3 + }, + { + "name": "fps", + "type": "FLOAT", + "links": [ + 24 + ], + "slot_index": 4 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 5 + } + ], + "properties": { + "Node name for S&R": "Difforum_Setup" + }, + "widgets_values": [ + "feedback (SD/SDXL/Flux)", + "seconds", + 5.0, + 24.0, + "16:9 landscape", + 1024, + 0, + "fixed", + 16.0, + 9.0, + 0.0 + ] + }, + { + "id": 5, + "type": "Difforum_Director", + "pos": [ + 420, + 360 + ], + "size": [ + 800, + 880 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": 3 + }, + { + "name": "clip", + "type": "CLIP", + "link": 4 + }, + { + "name": "audio", + "type": "DIFFORUM_AUDIO", + "link": null + } + ], + "outputs": [ + { + "name": "direction", + "type": "DIFFORUM_DIRECTION", + "links": [ + 18 + ], + "slot_index": 0 + }, + { + "name": "camera", + "type": "DIFFORUM_CAMERA", + "links": [], + "slot_index": 1 + }, + { + "name": "strength", + "type": "DIFFORUM_SCHEDULE", + "links": [], + "slot_index": 2 + }, + { + "name": "prompts", + "type": "DIFFORUM_PROMPT", + "links": [], + "slot_index": 3 + }, + { + "name": "camera_text", + "type": "STRING", + "links": [], + "slot_index": 4 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 5 + } + ], + "properties": { + "Node name for S&R": "Difforum_Director" + }, + "widgets_values": [ + "{\"version\": 2, \"scenes\": [{\"start\": 0, \"mood\": \"calm\", \"prompt\": \"misty ancient forest at dawn, volumetric light, wide shot\"}, {\"start\": 30, \"mood\": \"build\", \"prompt\": \"glowing roots and moss, bioluminescent details, macro\"}, {\"start\": 60, \"mood\": \"climax\", \"prompt\": \"explosion of light, cosmic transformation, high contrast\"}, {\"start\": 90, \"mood\": \"resolve\", \"prompt\": \"embers becoming stars, calm, vast cosmic scale\"}], \"camera\": [{\"start\": 0, \"move\": \"dolly_in\", \"speed\": 0.9, \"intensity\": 0.8, \"lens\": 0.0, \"ease\": \"ease_in_out\", \"react\": \"none\"}, {\"start\": 40, \"move\": \"orbit_right\", \"speed\": 1.0, \"intensity\": 0.9, \"lens\": 0.0, \"ease\": \"ease_in_out\", \"react\": \"none\"}, {\"start\": 80, \"move\": \"crane_up\", \"speed\": 0.8, \"intensity\": 0.8, \"lens\": 0.0, \"ease\": \"ease_out\", \"react\": \"none\"}], \"energy\": [], \"guidance\": []}", + "3d", + "cinematic", + 1.0, + 1.0, + 0.0, + 0.0, + 0, + "fixed" + ] + }, + { + "id": 6, + "type": "EmptyLatentImage", + "pos": [ + 820, + 0 + ], + "size": [ + 260, + 110 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "width", + "type": "INT", + "link": 5, + "widget": { + "name": "width" + } + }, + { + "name": "height", + "type": "INT", + "link": 6, + "widget": { + "name": "height" + } + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 10 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 1024, + 576, + 1 + ] + }, + { + "id": 7, + "type": "KSampler", + "pos": [ + 1110, + 0 + ], + "size": [ + 300, + 260 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 7 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 8 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 9 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 10 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 11 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 7, + "fixed", + 24, + 6.0, + "dpmpp_2m", + "karras", + 1.0 + ], + "title": "First frame" + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [ + 1440, + 0 + ], + "size": [ + 200, + 60 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 11 + }, + { + "name": "vae", + "type": "VAE", + "link": 12 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 17, + 20 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + }, + "widgets_values": [] + }, + { + "id": 9, + "type": "Difforum_FeedbackSampler", + "pos": [ + 1260, + 360 + ], + "size": [ + 360, + 460 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 13 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 15 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 16 + }, + { + "name": "vae", + "type": "VAE", + "link": 14 + }, + { + "name": "init_image", + "type": "IMAGE", + "link": 17 + }, + { + "name": "direction", + "type": "DIFFORUM_DIRECTION", + "link": 18 + }, + { + "name": "options", + "type": "DIFFORUM_OPTIONS", + "link": null + }, + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": null + }, + { + "name": "camera", + "type": "DIFFORUM_CAMERA", + "link": null + }, + { + "name": "strength", + "type": "DIFFORUM_SCHEDULE", + "link": null + }, + { + "name": "cfg_curve", + "type": "DIFFORUM_SCHEDULE", + "link": null + }, + { + "name": "prompts", + "type": "DIFFORUM_PROMPT", + "link": null + }, + { + "name": "depth", + "type": "IMAGE", + "link": 21 + }, + { + "name": "control_net", + "type": "CONTROL_NET", + "link": null + }, + { + "name": "control_image", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "frames", + "type": "IMAGE", + "links": [ + 22 + ], + "slot_index": 0 + }, + { + "name": "depth", + "type": "IMAGE", + "links": [], + "slot_index": 1 + }, + { + "name": "report", + "type": "STRING", + "links": [ + 26 + ], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "Difforum_FeedbackSampler" + }, + "widgets_values": [ + 20, + 6.0, + "dpmpp_2m", + "karras", + 2, + 0.6, + 0.5 + ] + }, + { + "id": 10, + "type": "DownloadAndLoadDepthAnythingV2Model", + "pos": [ + 1260, + 0 + ], + "size": [ + 360, + 80 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "da_v2_model", + "type": "DAMODEL", + "links": [ + 19 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "DownloadAndLoadDepthAnythingV2Model" + }, + "widgets_values": [ + "depth_anything_v2_vitl_fp32.safetensors" + ] + }, + { + "id": 11, + "type": "DepthAnything_V2", + "pos": [ + 1260, + 120 + ], + "size": [ + 360, + 80 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "da_model", + "type": "DAMODEL", + "link": 19 + }, + { + "name": "images", + "type": "IMAGE", + "link": 20 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 21 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "DepthAnything_V2" + }, + "widgets_values": [] + }, + { + "id": 12, + "type": "Difforum_FlowStabilize", + "pos": [ + 1660, + 360 + ], + "size": [ + 300, + 130 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "frames", + "type": "IMAGE", + "link": 22 + } + ], + "outputs": [ + { + "name": "frames", + "type": "IMAGE", + "links": [ + 23 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "Difforum_FlowStabilize" + }, + "widgets_values": [ + 0.45, + 0.5, + 0.15 + ] + }, + { + "id": 13, + "type": "CreateVideo", + "pos": [ + 2000, + 360 + ], + "size": [ + 260, + 100 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 23 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null + }, + { + "name": "fps", + "type": "FLOAT", + "link": 24, + "widget": { + "name": "fps" + } + } + ], + "outputs": [ + { + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 25 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CreateVideo" + }, + "widgets_values": [ + 24.0 + ] + }, + { + "id": 14, + "type": "SaveVideo", + "pos": [ + 2000, + 500 + ], + "size": [ + 300, + 320 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "video", + "type": "VIDEO", + "link": 25 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "video/difforum", + "auto", + "auto" + ] + }, + { + "id": 15, + "type": "PreviewAny", + "pos": [ + 1660, + 540 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "source", + "type": "*", + "link": 26 + } + ], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "PreviewAny" + }, + "widgets_values": [], + "title": "Run report" + }, + { + "id": 16, + "type": "MarkdownNote", + "pos": [ + 0, + -320 + ], + "size": [ + 560, + 240 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "## Real parallax\n\nDepth Anything V2 reads the first frame, the Director is in **3d**, and the sampler re-projects the depth with the image every frame - near things move faster than far ones for the whole clip, not just the start.\n\nNeeds **ComfyUI-DepthAnythingV2** (Kijai). Too strong? Lower the sampler's `translation_scale`. The `depth` output is the tracked depth per frame, ready for compositing." + ] + } + ], + "links": [ + [ + 1, + 1, + 1, + 2, + 0, + "CLIP" + ], + [ + 2, + 1, + 1, + 3, + 0, + "CLIP" + ], + [ + 3, + 4, + 0, + 5, + 0, + "DIFFORUM_PARAMS" + ], + [ + 4, + 1, + 1, + 5, + 1, + "CLIP" + ], + [ + 5, + 4, + 1, + 6, + 0, + "INT" + ], + [ + 6, + 4, + 2, + 6, + 1, + "INT" + ], + [ + 7, + 1, + 0, + 7, + 0, + "MODEL" + ], + [ + 8, + 2, + 0, + 7, + 1, + "CONDITIONING" + ], + [ + 9, + 3, + 0, + 7, + 2, + "CONDITIONING" + ], + [ + 10, + 6, + 0, + 7, + 3, + "LATENT" + ], + [ + 11, + 7, + 0, + 8, + 0, + "LATENT" + ], + [ + 12, + 1, + 2, + 8, + 1, + "VAE" + ], + [ + 13, + 1, + 0, + 9, + 0, + "MODEL" + ], + [ + 14, + 1, + 2, + 9, + 3, + "VAE" + ], + [ + 15, + 2, + 0, + 9, + 1, + "CONDITIONING" + ], + [ + 16, + 3, + 0, + 9, + 2, + "CONDITIONING" + ], + [ + 17, + 8, + 0, + 9, + 4, + "IMAGE" + ], + [ + 18, + 5, + 0, + 9, + 5, + "DIFFORUM_DIRECTION" + ], + [ + 19, + 10, + 0, + 11, + 0, + "DAMODEL" + ], + [ + 20, + 8, + 0, + 11, + 1, + "IMAGE" + ], + [ + 21, + 11, + 0, + 9, + 12, + "IMAGE" + ], + [ + 22, + 9, + 0, + 12, + 0, + "IMAGE" + ], + [ + 23, + 12, + 0, + 13, + 0, + "IMAGE" + ], + [ + 24, + 4, + 4, + 13, + 2, + "FLOAT" + ], + [ + 25, + 13, + 0, + 14, + 0, + "VIDEO" + ], + [ + 26, + 9, + 2, + 15, + 0, + "STRING" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.65, + "offset": [ + 40, + 40 + ] + }, + "difforum": "1.0", + "title": "03 · Real parallax (3D + depth)" + }, + "version": 0.4 +} diff --git a/example_workflows/04_audio_reactive.json b/example_workflows/04_audio_reactive.json new file mode 100644 index 0000000..fdd2c16 --- /dev/null +++ b/example_workflows/04_audio_reactive.json @@ -0,0 +1,1125 @@ +{ + "last_node_id": 17, + "last_link_id": 28, + "nodes": [ + { + "id": 1, + "type": "CheckpointLoaderSimple", + "pos": [ + 0, + 0 + ], + "size": [ + 320, + 100 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 7, + 13 + ], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 1, + 2, + 4 + ], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 12, + 14 + ], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "sd_xl_base_1.0.safetensors" + ] + }, + { + "id": 2, + "type": "CLIPTextEncode", + "pos": [ + 360, + 0 + ], + "size": [ + 380, + 140 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 8, + 15 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "cinematic film still, rich detail, volumetric light, 35mm" + ], + "title": "Look / style (positive)" + }, + { + "id": 3, + "type": "CLIPTextEncode", + "pos": [ + 360, + 180 + ], + "size": [ + 380, + 120 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 2 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 9, + 16 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "blurry, low quality, watermark, text, frame, border" + ], + "title": "Negative" + }, + { + "id": 4, + "type": "Difforum_Setup", + "pos": [ + 0, + 360 + ], + "size": [ + 320, + 300 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "links": [ + 3, + 24 + ], + "slot_index": 0 + }, + { + "name": "width", + "type": "INT", + "links": [ + 5 + ], + "slot_index": 1 + }, + { + "name": "height", + "type": "INT", + "links": [ + 6 + ], + "slot_index": 2 + }, + { + "name": "frames", + "type": "INT", + "links": [], + "slot_index": 3 + }, + { + "name": "fps", + "type": "FLOAT", + "links": [ + 21 + ], + "slot_index": 4 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 5 + } + ], + "properties": { + "Node name for S&R": "Difforum_Setup" + }, + "widgets_values": [ + "feedback (SD/SDXL/Flux)", + "seconds", + 10.0, + 24.0, + "16:9 landscape", + 1024, + 0, + "fixed", + 16.0, + 9.0, + 0.0 + ] + }, + { + "id": 5, + "type": "Difforum_Director", + "pos": [ + 420, + 360 + ], + "size": [ + 800, + 880 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": 3 + }, + { + "name": "clip", + "type": "CLIP", + "link": 4 + }, + { + "name": "audio", + "type": "DIFFORUM_AUDIO", + "link": 26 + } + ], + "outputs": [ + { + "name": "direction", + "type": "DIFFORUM_DIRECTION", + "links": [ + 18 + ], + "slot_index": 0 + }, + { + "name": "camera", + "type": "DIFFORUM_CAMERA", + "links": [], + "slot_index": 1 + }, + { + "name": "strength", + "type": "DIFFORUM_SCHEDULE", + "links": [], + "slot_index": 2 + }, + { + "name": "prompts", + "type": "DIFFORUM_PROMPT", + "links": [], + "slot_index": 3 + }, + { + "name": "camera_text", + "type": "STRING", + "links": [], + "slot_index": 4 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 5 + } + ], + "properties": { + "Node name for S&R": "Difforum_Director" + }, + "widgets_values": [ + "{\"version\": 2, \"scenes\": [{\"start\": 0, \"mood\": \"calm\", \"prompt\": \"misty ancient forest at dawn, volumetric light, wide shot\"}, {\"start\": 60, \"mood\": \"build\", \"prompt\": \"glowing roots and moss, bioluminescent details, macro\"}, {\"start\": 120, \"mood\": \"climax\", \"prompt\": \"explosion of light, cosmic transformation, high contrast\"}, {\"start\": 180, \"mood\": \"resolve\", \"prompt\": \"embers becoming stars, calm, vast cosmic scale\"}], \"camera\": [{\"start\": 0, \"move\": \"zoom_in\", \"speed\": 0.7, \"intensity\": 0.8, \"lens\": 0.0, \"ease\": \"ease_in_out\", \"react\": \"beat_pulse\"}, {\"start\": 96, \"move\": \"spiral\", \"speed\": 1.0, \"intensity\": 1.0, \"lens\": 0.0, \"ease\": \"ease_in_out\", \"react\": \"bass_speed\"}, {\"start\": 180, \"move\": \"handheld\", \"speed\": 1.0, \"intensity\": 1.2, \"lens\": 0.0, \"ease\": \"ease_in_out\", \"react\": \"onset_shake\"}], \"energy\": [], \"guidance\": []}", + "2d", + "cinematic", + 1.0, + 1.0, + 0.0, + 0.0, + 0, + "fixed" + ] + }, + { + "id": 6, + "type": "EmptyLatentImage", + "pos": [ + 820, + 0 + ], + "size": [ + 260, + 110 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "width", + "type": "INT", + "link": 5, + "widget": { + "name": "width" + } + }, + { + "name": "height", + "type": "INT", + "link": 6, + "widget": { + "name": "height" + } + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 10 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 1024, + 576, + 1 + ] + }, + { + "id": 7, + "type": "KSampler", + "pos": [ + 1110, + 0 + ], + "size": [ + 300, + 260 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 7 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 8 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 9 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 10 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 11 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 7, + "fixed", + 24, + 6.0, + "dpmpp_2m", + "karras", + 1.0 + ], + "title": "First frame" + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [ + 1440, + 0 + ], + "size": [ + 200, + 60 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 11 + }, + { + "name": "vae", + "type": "VAE", + "link": 12 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 17 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + }, + "widgets_values": [] + }, + { + "id": 9, + "type": "Difforum_FeedbackSampler", + "pos": [ + 1260, + 360 + ], + "size": [ + 360, + 460 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 13 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 15 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 16 + }, + { + "name": "vae", + "type": "VAE", + "link": 14 + }, + { + "name": "init_image", + "type": "IMAGE", + "link": 17 + }, + { + "name": "direction", + "type": "DIFFORUM_DIRECTION", + "link": 18 + }, + { + "name": "options", + "type": "DIFFORUM_OPTIONS", + "link": null + }, + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": null + }, + { + "name": "camera", + "type": "DIFFORUM_CAMERA", + "link": null + }, + { + "name": "strength", + "type": "DIFFORUM_SCHEDULE", + "link": null + }, + { + "name": "cfg_curve", + "type": "DIFFORUM_SCHEDULE", + "link": null + }, + { + "name": "prompts", + "type": "DIFFORUM_PROMPT", + "link": null + }, + { + "name": "depth", + "type": "IMAGE", + "link": null + }, + { + "name": "control_net", + "type": "CONTROL_NET", + "link": null + }, + { + "name": "control_image", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "frames", + "type": "IMAGE", + "links": [ + 19 + ], + "slot_index": 0 + }, + { + "name": "depth", + "type": "IMAGE", + "links": [], + "slot_index": 1 + }, + { + "name": "report", + "type": "STRING", + "links": [ + 23 + ], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "Difforum_FeedbackSampler" + }, + "widgets_values": [ + 20, + 6.0, + "dpmpp_2m", + "karras", + 2, + 0.6, + 0.5 + ] + }, + { + "id": 10, + "type": "Difforum_FlowStabilize", + "pos": [ + 1660, + 360 + ], + "size": [ + 300, + 130 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "frames", + "type": "IMAGE", + "link": 19 + } + ], + "outputs": [ + { + "name": "frames", + "type": "IMAGE", + "links": [ + 20 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "Difforum_FlowStabilize" + }, + "widgets_values": [ + 0.45, + 0.5, + 0.15 + ] + }, + { + "id": 11, + "type": "CreateVideo", + "pos": [ + 2000, + 360 + ], + "size": [ + 260, + 100 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 20 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 27 + }, + { + "name": "fps", + "type": "FLOAT", + "link": 21, + "widget": { + "name": "fps" + } + } + ], + "outputs": [ + { + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 22 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CreateVideo" + }, + "widgets_values": [ + 24.0 + ] + }, + { + "id": 12, + "type": "SaveVideo", + "pos": [ + 2000, + 500 + ], + "size": [ + 300, + 320 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "video", + "type": "VIDEO", + "link": 22 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "video/difforum", + "auto", + "auto" + ] + }, + { + "id": 13, + "type": "PreviewAny", + "pos": [ + 1660, + 540 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "source", + "type": "*", + "link": 23 + } + ], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "PreviewAny" + }, + "widgets_values": [], + "title": "Run report" + }, + { + "id": 14, + "type": "LoadAudio", + "pos": [ + 0, + 720 + ], + "size": [ + 320, + 140 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 25, + 27 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadAudio" + }, + "widgets_values": [ + "track.mp3", + null, + null + ] + }, + { + "id": 15, + "type": "Difforum_AudioAnalyzer", + "pos": [ + 0, + 900 + ], + "size": [ + 320, + 200 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": 24 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 25 + } + ], + "outputs": [ + { + "name": "audio_curves", + "type": "DIFFORUM_AUDIO", + "links": [ + 26 + ], + "slot_index": 0 + }, + { + "name": "bands_plot", + "type": "IMAGE", + "links": [ + 28 + ], + "slot_index": 1 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "Difforum_AudioAnalyzer" + }, + "widgets_values": [ + 0.25, + 1.5, + 0.0 + ] + }, + { + "id": 16, + "type": "PreviewImage", + "pos": [ + 0, + 1140 + ], + "size": [ + 320, + 260 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 28 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [], + "title": "Bands: low / mid / high / beat" + }, + { + "id": 17, + "type": "MarkdownNote", + "pos": [ + 0, + -320 + ], + "size": [ + 560, + 220 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "## Audio reactive, no expressions\n\nEach camera block in the Director has an **Audio** reaction: pulse on beat, shake on hits, bass drives speed, mids rock the roll. The analyzer feeds them; **Create Video** muxes the track back in.\n\nFor custom curves use **Audio Curve** or type bands (`low`, `beat`...) in a Schedule." + ] + } + ], + "links": [ + [ + 1, + 1, + 1, + 2, + 0, + "CLIP" + ], + [ + 2, + 1, + 1, + 3, + 0, + "CLIP" + ], + [ + 3, + 4, + 0, + 5, + 0, + "DIFFORUM_PARAMS" + ], + [ + 4, + 1, + 1, + 5, + 1, + "CLIP" + ], + [ + 5, + 4, + 1, + 6, + 0, + "INT" + ], + [ + 6, + 4, + 2, + 6, + 1, + "INT" + ], + [ + 7, + 1, + 0, + 7, + 0, + "MODEL" + ], + [ + 8, + 2, + 0, + 7, + 1, + "CONDITIONING" + ], + [ + 9, + 3, + 0, + 7, + 2, + "CONDITIONING" + ], + [ + 10, + 6, + 0, + 7, + 3, + "LATENT" + ], + [ + 11, + 7, + 0, + 8, + 0, + "LATENT" + ], + [ + 12, + 1, + 2, + 8, + 1, + "VAE" + ], + [ + 13, + 1, + 0, + 9, + 0, + "MODEL" + ], + [ + 14, + 1, + 2, + 9, + 3, + "VAE" + ], + [ + 15, + 2, + 0, + 9, + 1, + "CONDITIONING" + ], + [ + 16, + 3, + 0, + 9, + 2, + "CONDITIONING" + ], + [ + 17, + 8, + 0, + 9, + 4, + "IMAGE" + ], + [ + 18, + 5, + 0, + 9, + 5, + "DIFFORUM_DIRECTION" + ], + [ + 19, + 9, + 0, + 10, + 0, + "IMAGE" + ], + [ + 20, + 10, + 0, + 11, + 0, + "IMAGE" + ], + [ + 21, + 4, + 4, + 11, + 2, + "FLOAT" + ], + [ + 22, + 11, + 0, + 12, + 0, + "VIDEO" + ], + [ + 23, + 9, + 2, + 13, + 0, + "STRING" + ], + [ + 24, + 4, + 0, + 15, + 0, + "DIFFORUM_PARAMS" + ], + [ + 25, + 14, + 0, + 15, + 1, + "AUDIO" + ], + [ + 26, + 15, + 0, + 5, + 2, + "DIFFORUM_AUDIO" + ], + [ + 27, + 14, + 0, + 11, + 1, + "AUDIO" + ], + [ + 28, + 15, + 1, + 16, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.65, + "offset": [ + 40, + 40 + ] + }, + "difforum": "1.0", + "title": "04 · Audio reactive" + }, + "version": 0.4 +} diff --git a/example_workflows/05_live_turbo.json b/example_workflows/05_live_turbo.json new file mode 100644 index 0000000..ee1b4d6 --- /dev/null +++ b/example_workflows/05_live_turbo.json @@ -0,0 +1,831 @@ +{ + "last_node_id": 12, + "last_link_id": 19, + "nodes": [ + { + "id": 1, + "type": "CheckpointLoaderSimple", + "pos": [ + 0, + 0 + ], + "size": [ + 320, + 100 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 6, + 13 + ], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 1, + 2 + ], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 11, + 14 + ], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "sd_xl_turbo_1.0_fp16.safetensors" + ] + }, + { + "id": 2, + "type": "CLIPTextEncode", + "pos": [ + 360, + 0 + ], + "size": [ + 380, + 140 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 7, + 15 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "liquid chrome sculpture, iridescent, studio light" + ], + "title": "Look / style (positive)" + }, + { + "id": 3, + "type": "CLIPTextEncode", + "pos": [ + 360, + 180 + ], + "size": [ + 380, + 120 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 2 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 8, + 16 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "blurry, low quality, watermark, text, frame, border" + ], + "title": "Negative" + }, + { + "id": 4, + "type": "Difforum_Setup", + "pos": [ + 0, + 360 + ], + "size": [ + 320, + 300 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "links": [ + 3 + ], + "slot_index": 0 + }, + { + "name": "width", + "type": "INT", + "links": [ + 4 + ], + "slot_index": 1 + }, + { + "name": "height", + "type": "INT", + "links": [ + 5 + ], + "slot_index": 2 + }, + { + "name": "frames", + "type": "INT", + "links": [], + "slot_index": 3 + }, + { + "name": "fps", + "type": "FLOAT", + "links": [], + "slot_index": 4 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 5 + } + ], + "properties": { + "Node name for S&R": "Difforum_Setup" + }, + "widgets_values": [ + "feedback (SD/SDXL/Flux)", + "seconds", + 10.0, + 24.0, + "16:9 landscape", + 512, + 0, + "fixed", + 16.0, + 9.0, + 0.0 + ] + }, + { + "id": 5, + "type": "Difforum_Director", + "pos": [ + 420, + 360 + ], + "size": [ + 800, + 880 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": 3 + }, + { + "name": "clip", + "type": "CLIP", + "link": null + }, + { + "name": "audio", + "type": "DIFFORUM_AUDIO", + "link": null + } + ], + "outputs": [ + { + "name": "direction", + "type": "DIFFORUM_DIRECTION", + "links": [ + 18 + ], + "slot_index": 0 + }, + { + "name": "camera", + "type": "DIFFORUM_CAMERA", + "links": [], + "slot_index": 1 + }, + { + "name": "strength", + "type": "DIFFORUM_SCHEDULE", + "links": [], + "slot_index": 2 + }, + { + "name": "prompts", + "type": "DIFFORUM_PROMPT", + "links": [], + "slot_index": 3 + }, + { + "name": "camera_text", + "type": "STRING", + "links": [], + "slot_index": 4 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 5 + } + ], + "properties": { + "Node name for S&R": "Difforum_Director" + }, + "widgets_values": [ + "{\"version\": 2, \"scenes\": [{\"start\": 0, \"mood\": \"calm\", \"prompt\": \"misty ancient forest at dawn, volumetric light, wide shot\"}, {\"start\": 60, \"mood\": \"build\", \"prompt\": \"glowing roots and moss, bioluminescent details, macro\"}, {\"start\": 120, \"mood\": \"climax\", \"prompt\": \"explosion of light, cosmic transformation, high contrast\"}, {\"start\": 180, \"mood\": \"resolve\", \"prompt\": \"embers becoming stars, calm, vast cosmic scale\"}], \"camera\": [{\"start\": 0, \"move\": \"zoom_in\", \"speed\": 0.8, \"intensity\": 0.8, \"lens\": 0.0, \"ease\": \"ease_in_out\", \"react\": \"none\"}, {\"start\": 60, \"move\": \"pan_right\", \"speed\": 1.0, \"intensity\": 0.8, \"lens\": 0.0, \"ease\": \"ease_in_out\", \"react\": \"none\"}, {\"start\": 120, \"move\": \"spiral\", \"speed\": 1.4, \"intensity\": 1.1, \"lens\": 0.0, \"ease\": \"ease_in\", \"react\": \"none\"}, {\"start\": 180, \"move\": \"zoom_out\", \"speed\": 0.6, \"intensity\": 0.5, \"lens\": 0.0, \"ease\": \"ease_out\", \"react\": \"none\"}], \"energy\": [], \"guidance\": []}", + "2d", + "cinematic", + 1.0, + 1.0, + 0.0, + 0.0, + 0, + "fixed" + ] + }, + { + "id": 6, + "type": "EmptyLatentImage", + "pos": [ + 820, + 0 + ], + "size": [ + 260, + 110 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "width", + "type": "INT", + "link": 4, + "widget": { + "name": "width" + } + }, + { + "name": "height", + "type": "INT", + "link": 5, + "widget": { + "name": "height" + } + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 9 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 1024, + 576, + 1 + ] + }, + { + "id": 7, + "type": "KSampler", + "pos": [ + 1110, + 0 + ], + "size": [ + 300, + 260 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 6 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 7 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 8 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 9 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 10 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 7, + "fixed", + 2, + 1.0, + "euler_ancestral", + "sgm_uniform", + 1.0 + ], + "title": "First frame" + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [ + 1440, + 0 + ], + "size": [ + 200, + 60 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 10 + }, + { + "name": "vae", + "type": "VAE", + "link": 11 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 17 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + }, + "widgets_values": [] + }, + { + "id": 9, + "type": "Difforum_LiveSampler", + "pos": [ + 1260, + 360 + ], + "size": [ + 360, + 560 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 13 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 15 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 16 + }, + { + "name": "vae", + "type": "VAE", + "link": 14 + }, + { + "name": "init_image", + "type": "IMAGE", + "link": 17 + }, + { + "name": "direction", + "type": "DIFFORUM_DIRECTION", + "link": 18 + }, + { + "name": "options", + "type": "DIFFORUM_OPTIONS", + "link": 12 + }, + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": null + }, + { + "name": "camera", + "type": "DIFFORUM_CAMERA", + "link": null + }, + { + "name": "strength", + "type": "DIFFORUM_SCHEDULE", + "link": null + }, + { + "name": "prompts", + "type": "DIFFORUM_PROMPT", + "link": null + }, + { + "name": "depth", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "frames", + "type": "IMAGE", + "links": [ + 19 + ], + "slot_index": 0 + }, + { + "name": "report", + "type": "STRING", + "links": [], + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "Difforum_LiveSampler" + }, + "widgets_values": [ + 1200, + 2, + 1.0, + "euler_ancestral", + "sgm_uniform", + 1, + 0.0, + 0.5, + true, + "", + 0.9, + "", + "", + 240, + true + ] + }, + { + "id": 10, + "type": "Difforum_RenderOptions", + "pos": [ + 1260, + 960 + ], + "size": [ + 340, + 560 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "options", + "type": "DIFFORUM_OPTIONS", + "links": [ + 12 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "Difforum_RenderOptions" + }, + "widgets_values": [ + 0.5, + "lab", + "scene", + 0.2, + 0.02, + 0.25, + "kaleidoscope", + 6, + "reflection", + "follow", + 1.0, + 1.0, + 100.0, + false, + "fixed", + 0, + 0 + ], + "title": "Render Options (kaleidoscope)" + }, + { + "id": 11, + "type": "PreviewImage", + "pos": [ + 1660, + 360 + ], + "size": [ + 400, + 300 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 19 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 12, + "type": "MarkdownNote", + "pos": [ + 0, + -320 + ], + "size": [ + 560, + 160 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "## Live\n\nQueue once: the node plays in place. `live_source` = `0` for a webcam magic mirror; `stream_dir` / `spout_name` feed OBS, Resolume or TouchDesigner. Use a 1-4 step model at ~512 px." + ] + } + ], + "links": [ + [ + 1, + 1, + 1, + 2, + 0, + "CLIP" + ], + [ + 2, + 1, + 1, + 3, + 0, + "CLIP" + ], + [ + 3, + 4, + 0, + 5, + 0, + "DIFFORUM_PARAMS" + ], + [ + 4, + 4, + 1, + 6, + 0, + "INT" + ], + [ + 5, + 4, + 2, + 6, + 1, + "INT" + ], + [ + 6, + 1, + 0, + 7, + 0, + "MODEL" + ], + [ + 7, + 2, + 0, + 7, + 1, + "CONDITIONING" + ], + [ + 8, + 3, + 0, + 7, + 2, + "CONDITIONING" + ], + [ + 9, + 6, + 0, + 7, + 3, + "LATENT" + ], + [ + 10, + 7, + 0, + 8, + 0, + "LATENT" + ], + [ + 11, + 1, + 2, + 8, + 1, + "VAE" + ], + [ + 12, + 10, + 0, + 9, + 6, + "DIFFORUM_OPTIONS" + ], + [ + 13, + 1, + 0, + 9, + 0, + "MODEL" + ], + [ + 14, + 1, + 2, + 9, + 3, + "VAE" + ], + [ + 15, + 2, + 0, + 9, + 1, + "CONDITIONING" + ], + [ + 16, + 3, + 0, + 9, + 2, + "CONDITIONING" + ], + [ + 17, + 8, + 0, + 9, + 4, + "IMAGE" + ], + [ + 18, + 5, + 0, + 9, + 5, + "DIFFORUM_DIRECTION" + ], + [ + 19, + 9, + 0, + 11, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.65, + "offset": [ + 40, + 40 + ] + }, + "difforum": "1.0", + "title": "05 · Live (turbo models)" + }, + "version": 0.4 +} diff --git a/example_workflows/06_seamless_loop.json b/example_workflows/06_seamless_loop.json new file mode 100644 index 0000000..000fa9a --- /dev/null +++ b/example_workflows/06_seamless_loop.json @@ -0,0 +1,965 @@ +{ + "last_node_id": 14, + "last_link_id": 24, + "nodes": [ + { + "id": 1, + "type": "CheckpointLoaderSimple", + "pos": [ + 0, + 0 + ], + "size": [ + 320, + 100 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 7, + 13 + ], + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 1, + 2 + ], + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 12, + 14 + ], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "sd_xl_base_1.0.safetensors" + ] + }, + { + "id": 2, + "type": "CLIPTextEncode", + "pos": [ + 360, + 0 + ], + "size": [ + 380, + 140 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 1 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 8, + 15 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "cinematic film still, rich detail, volumetric light, 35mm" + ], + "title": "Look / style (positive)" + }, + { + "id": 3, + "type": "CLIPTextEncode", + "pos": [ + 360, + 180 + ], + "size": [ + 380, + 120 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 2 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 9, + 16 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "blurry, low quality, watermark, text, frame, border" + ], + "title": "Negative" + }, + { + "id": 4, + "type": "Difforum_Setup", + "pos": [ + 0, + 360 + ], + "size": [ + 320, + 300 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "links": [ + 3, + 4, + 18 + ], + "slot_index": 0 + }, + { + "name": "width", + "type": "INT", + "links": [ + 5 + ], + "slot_index": 1 + }, + { + "name": "height", + "type": "INT", + "links": [ + 6 + ], + "slot_index": 2 + }, + { + "name": "frames", + "type": "INT", + "links": [], + "slot_index": 3 + }, + { + "name": "fps", + "type": "FLOAT", + "links": [], + "slot_index": 4 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 5 + } + ], + "properties": { + "Node name for S&R": "Difforum_Setup" + }, + "widgets_values": [ + "feedback (SD/SDXL/Flux)", + "frames", + 360.0, + 24.0, + "16:9 landscape", + 768, + 0, + "fixed", + 16.0, + 9.0, + 0.0 + ] + }, + { + "id": 5, + "type": "Difforum_Camera", + "pos": [ + 420, + 360 + ], + "size": [ + 380, + 360 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": 3 + }, + { + "name": "audio", + "type": "DIFFORUM_AUDIO", + "link": null + } + ], + "outputs": [ + { + "name": "camera", + "type": "DIFFORUM_CAMERA", + "links": [ + 19 + ], + "slot_index": 0 + }, + { + "name": "cycle_frames", + "type": "INT", + "links": [ + 22 + ], + "slot_index": 1 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "Difforum_Camera" + }, + "widgets_values": [ + "0: orbit_right 1.0 0.8\n60: spiral 1.2 1.0", + "2d", + 1.0, + 1.0, + "harmonic", + 120, + 3 + ] + }, + { + "id": 6, + "type": "Difforum_Schedule", + "pos": [ + 420, + 760 + ], + "size": [ + 380, + 160 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": 4 + }, + { + "name": "audio", + "type": "DIFFORUM_AUDIO", + "link": null + } + ], + "outputs": [ + { + "name": "schedule", + "type": "DIFFORUM_SCHEDULE", + "links": [ + 20 + ], + "slot_index": 0 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "Difforum_Schedule" + }, + "widgets_values": [ + "0:(0.5)", + "ease_in_out" + ] + }, + { + "id": 7, + "type": "EmptyLatentImage", + "pos": [ + 820, + 0 + ], + "size": [ + 260, + 110 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "width", + "type": "INT", + "link": 5, + "widget": { + "name": "width" + } + }, + { + "name": "height", + "type": "INT", + "link": 6, + "widget": { + "name": "height" + } + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 10 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 1024, + 576, + 1 + ] + }, + { + "id": 8, + "type": "KSampler", + "pos": [ + 1110, + 0 + ], + "size": [ + 300, + 260 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 7 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 8 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 9 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 10 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 11 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 7, + "fixed", + 24, + 6.0, + "dpmpp_2m", + "karras", + 1.0 + ], + "title": "First frame" + }, + { + "id": 9, + "type": "VAEDecode", + "pos": [ + 1440, + 0 + ], + "size": [ + 200, + 60 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 11 + }, + { + "name": "vae", + "type": "VAE", + "link": 12 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 17 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + }, + "widgets_values": [] + }, + { + "id": 10, + "type": "Difforum_FeedbackSampler", + "pos": [ + 860, + 360 + ], + "size": [ + 360, + 460 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 13 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 15 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 16 + }, + { + "name": "vae", + "type": "VAE", + "link": 14 + }, + { + "name": "init_image", + "type": "IMAGE", + "link": 17 + }, + { + "name": "direction", + "type": "DIFFORUM_DIRECTION", + "link": null + }, + { + "name": "options", + "type": "DIFFORUM_OPTIONS", + "link": null + }, + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": 18 + }, + { + "name": "camera", + "type": "DIFFORUM_CAMERA", + "link": 19 + }, + { + "name": "strength", + "type": "DIFFORUM_SCHEDULE", + "link": 20 + }, + { + "name": "cfg_curve", + "type": "DIFFORUM_SCHEDULE", + "link": null + }, + { + "name": "prompts", + "type": "DIFFORUM_PROMPT", + "link": null + }, + { + "name": "depth", + "type": "IMAGE", + "link": null + }, + { + "name": "control_net", + "type": "CONTROL_NET", + "link": null + }, + { + "name": "control_image", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "frames", + "type": "IMAGE", + "links": [ + 21 + ], + "slot_index": 0 + }, + { + "name": "depth", + "type": "IMAGE", + "links": [], + "slot_index": 1 + }, + { + "name": "report", + "type": "STRING", + "links": [], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "Difforum_FeedbackSampler" + }, + "widgets_values": [ + 20, + 6.0, + "euler", + "normal", + 1, + 0.6, + 0.5 + ] + }, + { + "id": 11, + "type": "Difforum_Loop", + "pos": [ + 1260, + 360 + ], + "size": [ + 300, + 160 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "frames", + "type": "IMAGE", + "link": 21 + }, + { + "name": "cycle_frames", + "type": "INT", + "link": 22, + "widget": { + "name": "cycle_frames" + } + } + ], + "outputs": [ + { + "name": "frames", + "type": "IMAGE", + "links": [ + 23 + ], + "slot_index": 0 + }, + { + "name": "seam_info", + "type": "STRING", + "links": [], + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "Difforum_Loop" + }, + "widgets_values": [ + "keep settled lap", + 0, + 12 + ] + }, + { + "id": 12, + "type": "CreateVideo", + "pos": [ + 1600, + 360 + ], + "size": [ + 260, + 100 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 23 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null + } + ], + "outputs": [ + { + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 24 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CreateVideo" + }, + "widgets_values": [ + 24.0 + ] + }, + { + "id": 13, + "type": "SaveVideo", + "pos": [ + 1600, + 500 + ], + "size": [ + 300, + 320 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "video", + "type": "VIDEO", + "link": 24 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "video/difforum_loop", + "auto", + "auto" + ] + }, + { + "id": 14, + "type": "MarkdownNote", + "pos": [ + 0, + -320 + ], + "size": [ + 560, + 180 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "## Loop without a crossfade\n\nThe Camera path is made periodic over 120 frames and rendered for 3 laps; the feedback settles onto its cycle and **Loop** keeps the last lap. For projections that run for hours." + ] + } + ], + "links": [ + [ + 1, + 1, + 1, + 2, + 0, + "CLIP" + ], + [ + 2, + 1, + 1, + 3, + 0, + "CLIP" + ], + [ + 3, + 4, + 0, + 5, + 0, + "DIFFORUM_PARAMS" + ], + [ + 4, + 4, + 0, + 6, + 0, + "DIFFORUM_PARAMS" + ], + [ + 5, + 4, + 1, + 7, + 0, + "INT" + ], + [ + 6, + 4, + 2, + 7, + 1, + "INT" + ], + [ + 7, + 1, + 0, + 8, + 0, + "MODEL" + ], + [ + 8, + 2, + 0, + 8, + 1, + "CONDITIONING" + ], + [ + 9, + 3, + 0, + 8, + 2, + "CONDITIONING" + ], + [ + 10, + 7, + 0, + 8, + 3, + "LATENT" + ], + [ + 11, + 8, + 0, + 9, + 0, + "LATENT" + ], + [ + 12, + 1, + 2, + 9, + 1, + "VAE" + ], + [ + 13, + 1, + 0, + 10, + 0, + "MODEL" + ], + [ + 14, + 1, + 2, + 10, + 3, + "VAE" + ], + [ + 15, + 2, + 0, + 10, + 1, + "CONDITIONING" + ], + [ + 16, + 3, + 0, + 10, + 2, + "CONDITIONING" + ], + [ + 17, + 9, + 0, + 10, + 4, + "IMAGE" + ], + [ + 18, + 4, + 0, + 10, + 7, + "DIFFORUM_PARAMS" + ], + [ + 19, + 5, + 0, + 10, + 8, + "DIFFORUM_CAMERA" + ], + [ + 20, + 6, + 0, + 10, + 9, + "DIFFORUM_SCHEDULE" + ], + [ + 21, + 10, + 0, + 11, + 0, + "IMAGE" + ], + [ + 22, + 5, + 1, + 11, + 1, + "INT" + ], + [ + 23, + 11, + 0, + 12, + 0, + "IMAGE" + ], + [ + 24, + 12, + 0, + 13, + 0, + "VIDEO" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.65, + "offset": [ + 40, + 40 + ] + }, + "difforum": "1.0", + "title": "06 · Seamless loop (installations)" + }, + "version": 0.4 +} diff --git a/example_workflows/07_ltx_guides.json b/example_workflows/07_ltx_guides.json new file mode 100644 index 0000000..3aeefe1 --- /dev/null +++ b/example_workflows/07_ltx_guides.json @@ -0,0 +1,732 @@ +{ + "last_node_id": 10, + "last_link_id": 13, + "nodes": [ + { + "id": 1, + "type": "Difforum_Setup", + "pos": [ + 0, + 0 + ], + "size": [ + 320, + 300 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "links": [ + 1 + ], + "slot_index": 0 + }, + { + "name": "width", + "type": "INT", + "links": [ + 6 + ], + "slot_index": 1 + }, + { + "name": "height", + "type": "INT", + "links": [ + 7 + ], + "slot_index": 2 + }, + { + "name": "frames", + "type": "INT", + "links": [], + "slot_index": 3 + }, + { + "name": "fps", + "type": "FLOAT", + "links": [ + 5 + ], + "slot_index": 4 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 5 + } + ], + "properties": { + "Node name for S&R": "Difforum_Setup" + }, + "widgets_values": [ + "LTX-2 / 2.5", + "seconds", + 5.0, + 24.0, + "16:9 landscape", + 1216, + 0, + "fixed", + 16.0, + 9.0, + 0.0 + ] + }, + { + "id": 2, + "type": "Difforum_Director", + "pos": [ + 360, + 0 + ], + "size": [ + 800, + 880 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": 1 + }, + { + "name": "clip", + "type": "CLIP", + "link": null + }, + { + "name": "audio", + "type": "DIFFORUM_AUDIO", + "link": null + } + ], + "outputs": [ + { + "name": "direction", + "type": "DIFFORUM_DIRECTION", + "links": [ + 3, + 12 + ], + "slot_index": 0 + }, + { + "name": "camera", + "type": "DIFFORUM_CAMERA", + "links": [], + "slot_index": 1 + }, + { + "name": "strength", + "type": "DIFFORUM_SCHEDULE", + "links": [], + "slot_index": 2 + }, + { + "name": "prompts", + "type": "DIFFORUM_PROMPT", + "links": [], + "slot_index": 3 + }, + { + "name": "camera_text", + "type": "STRING", + "links": [], + "slot_index": 4 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 5 + } + ], + "properties": { + "Node name for S&R": "Difforum_Director" + }, + "widgets_values": [ + "{\"version\": 2, \"scenes\": [{\"start\": 0, \"mood\": \"calm\", \"prompt\": \"misty ancient forest at dawn, volumetric light, wide shot\"}, {\"start\": 30, \"mood\": \"build\", \"prompt\": \"glowing roots and moss, bioluminescent details, macro\"}, {\"start\": 60, \"mood\": \"climax\", \"prompt\": \"explosion of light, cosmic transformation, high contrast\"}, {\"start\": 90, \"mood\": \"resolve\", \"prompt\": \"embers becoming stars, calm, vast cosmic scale\"}], \"camera\": [{\"start\": 0, \"move\": \"zoom_in\", \"speed\": 0.8, \"intensity\": 0.8, \"lens\": 0.0, \"ease\": \"ease_in_out\", \"react\": \"none\"}, {\"start\": 30, \"move\": \"pan_right\", \"speed\": 1.0, \"intensity\": 0.8, \"lens\": 0.0, \"ease\": \"ease_in_out\", \"react\": \"none\"}, {\"start\": 60, \"move\": \"spiral\", \"speed\": 1.4, \"intensity\": 1.1, \"lens\": 0.0, \"ease\": \"ease_in\", \"react\": \"none\"}, {\"start\": 90, \"move\": \"zoom_out\", \"speed\": 0.6, \"intensity\": 0.5, \"lens\": 0.0, \"ease\": \"ease_out\", \"react\": \"none\"}], \"energy\": [], \"guidance\": []}", + "2d", + "cinematic", + 1.0, + 1.0, + 0.0, + 0.0, + 0, + "fixed" + ] + }, + { + "id": 3, + "type": "LoadImage", + "pos": [ + 0, + 360 + ], + "size": [ + 320, + 360 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 2 + ], + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": [], + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "example.png", + null + ], + "title": "Anchor / first frame" + }, + { + "id": 4, + "type": "Difforum_GuideFrames", + "pos": [ + 1200, + 0 + ], + "size": [ + 340, + 200 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "anchor_image", + "type": "IMAGE", + "link": 2 + }, + { + "name": "direction", + "type": "DIFFORUM_DIRECTION", + "link": 3 + }, + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": null + }, + { + "name": "camera", + "type": "DIFFORUM_CAMERA", + "link": null + }, + { + "name": "depth", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "guide_frames", + "type": "IMAGE", + "links": [ + 4 + ], + "slot_index": 0 + }, + { + "name": "masks", + "type": "MASK", + "links": [], + "slot_index": 1 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "Difforum_GuideFrames" + }, + "widgets_values": [ + "1 = generate (VACE / LTX / H3)", + "gray", + 1.0 + ] + }, + { + "id": 5, + "type": "Difforum_Keyframes", + "pos": [ + 1200, + 240 + ], + "size": [ + 340, + 220 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "frames", + "type": "IMAGE", + "link": 4 + }, + { + "name": "fps", + "type": "FLOAT", + "link": 5, + "widget": { + "name": "fps" + } + } + ], + "outputs": [ + { + "name": "keyframes", + "type": "IMAGE", + "links": [ + 9, + 13 + ], + "slot_index": 0 + }, + { + "name": "indices", + "type": "STRING", + "links": [ + 10 + ], + "slot_index": 1 + }, + { + "name": "sparse_batch", + "type": "IMAGE", + "links": [], + "slot_index": 2 + }, + { + "name": "first_frame", + "type": "IMAGE", + "links": [], + "slot_index": 3 + }, + { + "name": "last_frame", + "type": "IMAGE", + "links": [], + "slot_index": 4 + }, + { + "name": "length", + "type": "INT", + "links": [ + 8 + ], + "slot_index": 5 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 6 + } + ], + "properties": { + "Node name for S&R": "Difforum_Keyframes" + }, + "widgets_values": [ + "LTX-2 / 2.5 (8k+1)", + 1.0, + 24.0, + "" + ] + }, + { + "id": 6, + "type": "EmptyLTXVLatentVideo", + "pos": [ + 1200, + 500 + ], + "size": [ + 300, + 160 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "width", + "type": "INT", + "link": 6, + "widget": { + "name": "width" + } + }, + { + "name": "height", + "type": "INT", + "link": 7, + "widget": { + "name": "height" + } + }, + { + "name": "length", + "type": "INT", + "link": 8, + "widget": { + "name": "length" + } + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 11 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyLTXVLatentVideo" + }, + "widgets_values": [ + 1216, + 704, + 121, + 1 + ] + }, + { + "id": 7, + "type": "Difforum_LTXGuides", + "pos": [ + 1580, + 240 + ], + "size": [ + 340, + 260 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "link": null + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + }, + { + "name": "latent", + "type": "LATENT", + "link": 11 + }, + { + "name": "keyframes", + "type": "IMAGE", + "link": 9 + }, + { + "name": "indices", + "type": "STRING", + "link": 10 + } + ], + "outputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "links": [], + "slot_index": 0 + }, + { + "name": "negative", + "type": "CONDITIONING", + "links": [], + "slot_index": 1 + }, + { + "name": "latent", + "type": "LATENT", + "links": [], + "slot_index": 2 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 3 + } + ], + "properties": { + "Node name for S&R": "Difforum_LTXGuides" + }, + "widgets_values": [ + 0.7, + 1.0, + 0.7 + ] + }, + { + "id": 8, + "type": "Difforum_CameraPrompt", + "pos": [ + 1580, + 0 + ], + "size": [ + 340, + 160 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "direction", + "type": "DIFFORUM_DIRECTION", + "link": 12 + }, + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": null + }, + { + "name": "camera", + "type": "DIFFORUM_CAMERA", + "link": null + } + ], + "outputs": [ + { + "name": "text", + "type": "STRING", + "links": [], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "Difforum_CameraPrompt" + }, + "widgets_values": [ + "prompt suffix", + "" + ] + }, + { + "id": 9, + "type": "PreviewImage", + "pos": [ + 1580, + 540 + ], + "size": [ + 340, + 260 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 13 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [], + "title": "Keyframes" + }, + { + "id": 10, + "type": "MarkdownNote", + "pos": [ + 1960, + 0 + ], + "size": [ + 520, + 300 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "## Wire into your LTX graph\n\nConnect your LTX **positive / negative** (text encoded with the Camera → Prompt text appended), and the LTX **VAE** into **LTX Guides**. Its outputs replace the conditioning and latent going into your LTX sampler.\n\n- Setup target = LTX keeps length on 8k+1 and size on 32 px.\n- Guides every 1 s at strength 0.7; frame 0 at 1.0 locks the look.\n- Swap Guide Frames for a Feedback render or a Storyboard to guide with other images." + ] + } + ], + "links": [ + [ + 1, + 1, + 0, + 2, + 0, + "DIFFORUM_PARAMS" + ], + [ + 2, + 3, + 0, + 4, + 0, + "IMAGE" + ], + [ + 3, + 2, + 0, + 4, + 1, + "DIFFORUM_DIRECTION" + ], + [ + 4, + 4, + 0, + 5, + 0, + "IMAGE" + ], + [ + 5, + 1, + 4, + 5, + 1, + "FLOAT" + ], + [ + 6, + 1, + 1, + 6, + 0, + "INT" + ], + [ + 7, + 1, + 2, + 6, + 1, + "INT" + ], + [ + 8, + 5, + 5, + 6, + 2, + "INT" + ], + [ + 9, + 5, + 0, + 7, + 4, + "IMAGE" + ], + [ + 10, + 5, + 1, + 7, + 5, + "STRING" + ], + [ + 11, + 6, + 0, + 7, + 3, + "LATENT" + ], + [ + 12, + 2, + 0, + 8, + 0, + "DIFFORUM_DIRECTION" + ], + [ + 13, + 5, + 0, + 9, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.65, + "offset": [ + 40, + 40 + ] + }, + "difforum": "1.0", + "title": "07 · LTX-2 / 2.5 guides from the Director" + }, + "version": 0.4 +} diff --git a/example_workflows/08_h3_first_last_frame.json b/example_workflows/08_h3_first_last_frame.json new file mode 100644 index 0000000..4be901b --- /dev/null +++ b/example_workflows/08_h3_first_last_frame.json @@ -0,0 +1,1326 @@ +{ + "last_node_id": 22, + "last_link_id": 30, + "nodes": [ + { + "id": 1, + "type": "Difforum_Setup", + "pos": [ + 0, + 0 + ], + "size": [ + 320, + 300 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "links": [ + 1 + ], + "slot_index": 0 + }, + { + "name": "width", + "type": "INT", + "links": [], + "slot_index": 1 + }, + { + "name": "height", + "type": "INT", + "links": [], + "slot_index": 2 + }, + { + "name": "frames", + "type": "INT", + "links": [], + "slot_index": 3 + }, + { + "name": "fps", + "type": "FLOAT", + "links": [], + "slot_index": 4 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 5 + } + ], + "properties": { + "Node name for S&R": "Difforum_Setup" + }, + "widgets_values": [ + "MiniMax H3", + "seconds", + 5.0, + 24.0, + "16:9 landscape", + 832, + 0, + "fixed", + 16.0, + 9.0, + 0.0 + ] + }, + { + "id": 2, + "type": "Difforum_Director", + "pos": [ + 360, + 0 + ], + "size": [ + 800, + 880 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": 1 + }, + { + "name": "clip", + "type": "CLIP", + "link": null + }, + { + "name": "audio", + "type": "DIFFORUM_AUDIO", + "link": null + } + ], + "outputs": [ + { + "name": "direction", + "type": "DIFFORUM_DIRECTION", + "links": [ + 3, + 5 + ], + "slot_index": 0 + }, + { + "name": "camera", + "type": "DIFFORUM_CAMERA", + "links": [], + "slot_index": 1 + }, + { + "name": "strength", + "type": "DIFFORUM_SCHEDULE", + "links": [], + "slot_index": 2 + }, + { + "name": "prompts", + "type": "DIFFORUM_PROMPT", + "links": [], + "slot_index": 3 + }, + { + "name": "camera_text", + "type": "STRING", + "links": [], + "slot_index": 4 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 5 + } + ], + "properties": { + "Node name for S&R": "Difforum_Director" + }, + "widgets_values": [ + "{\"version\": 2, \"scenes\": [{\"start\": 0, \"mood\": \"calm\", \"prompt\": \"misty ancient forest at dawn, light shafts through the canopy\"}, {\"start\": 62, \"mood\": \"build\", \"prompt\": \"the mist lifts, glowing moss and roots in the foreground\"}], \"camera\": [{\"start\": 0, \"move\": \"dolly_in\", \"speed\": 0.8, \"intensity\": 0.8, \"lens\": 0.0, \"ease\": \"ease_in_out\", \"react\": \"none\"}, {\"start\": 62, \"move\": \"orbit_right\", \"speed\": 0.9, \"intensity\": 0.8, \"lens\": 0.0, \"ease\": \"ease_in_out\", \"react\": \"none\"}], \"energy\": [], \"guidance\": []}", + "3d", + "cinematic", + 1.0, + 1.0, + 0.0, + 0.0, + 0, + "fixed" + ] + }, + { + "id": 3, + "type": "LoadImage", + "pos": [ + 0, + 360 + ], + "size": [ + 320, + 360 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 2 + ], + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": [], + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "example.png", + null + ], + "title": "First frame" + }, + { + "id": 4, + "type": "Difforum_GuideFrames", + "pos": [ + 1200, + 0 + ], + "size": [ + 340, + 200 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "anchor_image", + "type": "IMAGE", + "link": 2 + }, + { + "name": "direction", + "type": "DIFFORUM_DIRECTION", + "link": 3 + }, + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": null + }, + { + "name": "camera", + "type": "DIFFORUM_CAMERA", + "link": null + }, + { + "name": "depth", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "guide_frames", + "type": "IMAGE", + "links": [ + 4 + ], + "slot_index": 0 + }, + { + "name": "masks", + "type": "MASK", + "links": [], + "slot_index": 1 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "Difforum_GuideFrames" + }, + "widgets_values": [ + "1 = generate (VACE / LTX / H3)", + "stretch edge", + 1.0 + ] + }, + { + "id": 5, + "type": "Difforum_H3Shot", + "pos": [ + 1200, + 240 + ], + "size": [ + 340, + 300 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "frames", + "type": "IMAGE", + "link": 4 + }, + { + "name": "direction", + "type": "DIFFORUM_DIRECTION", + "link": 5 + }, + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": null + }, + { + "name": "camera", + "type": "DIFFORUM_CAMERA", + "link": null + } + ], + "outputs": [ + { + "name": "first_frame", + "type": "IMAGE", + "links": [ + 8 + ], + "slot_index": 0 + }, + { + "name": "last_frame", + "type": "IMAGE", + "links": [ + 9, + 30 + ], + "slot_index": 1 + }, + { + "name": "length", + "type": "INT", + "links": [ + 13 + ], + "slot_index": 2 + }, + { + "name": "width", + "type": "INT", + "links": [ + 11 + ], + "slot_index": 3 + }, + { + "name": "height", + "type": "INT", + "links": [ + 12 + ], + "slot_index": 4 + }, + { + "name": "prompt", + "type": "STRING", + "links": [ + 10 + ], + "slot_index": 5 + }, + { + "name": "segments", + "type": "INT", + "links": [], + "slot_index": 6 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 7 + } + ], + "properties": { + "Node name for S&R": "Difforum_H3Shot" + }, + "widgets_values": [ + "124 (~5s)", + 0, + "A misty ancient forest at dawn; light shafts cut through the canopy. Soft ambient birdsong, a low wind." + ] + }, + { + "id": 6, + "type": "UNETLoader", + "pos": [ + 1580, + -200 + ], + "size": [ + 420, + 90 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 14 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "UNETLoader" + }, + "widgets_values": [ + "minimax_h3_fl2va_pruned_int8_convrot.safetensors", + "default" + ] + }, + { + "id": 7, + "type": "CLIPLoader", + "pos": [ + 1580, + -80 + ], + "size": [ + 420, + 110 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 6 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPLoader" + }, + "widgets_values": [ + "qwen3vl_32b_minimax_h3_nvfp4_awq.safetensors", + "minimax", + "default" + ] + }, + { + "id": 8, + "type": "VAELoader", + "pos": [ + 1580, + 60 + ], + "size": [ + 420, + 60 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 7, + 24 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAELoader" + }, + "widgets_values": [ + "minimax_h3_video_vae_int8_convrot.safetensors" + ] + }, + { + "id": 9, + "type": "VAELoader", + "pos": [ + 1580, + 150 + ], + "size": [ + 420, + 60 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 26 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAELoader" + }, + "widgets_values": [ + "minimax_h3_audio_vae_fp32.safetensors" + ] + }, + { + "id": 10, + "type": "MiniMaxH3ImageToVideo", + "pos": [ + 1580, + 300 + ], + "size": [ + 420, + 260 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 6 + }, + { + "name": "vae", + "type": "VAE", + "link": 7 + }, + { + "name": "first_frame", + "type": "IMAGE", + "link": 8 + }, + { + "name": "last_frame", + "type": "IMAGE", + "link": 9 + }, + { + "name": "prompt", + "type": "STRING", + "link": 10, + "widget": { + "name": "prompt" + } + }, + { + "name": "width", + "type": "INT", + "link": 11, + "widget": { + "name": "width" + } + }, + { + "name": "height", + "type": "INT", + "link": 12, + "widget": { + "name": "height" + } + }, + { + "name": "length", + "type": "INT", + "link": 13, + "widget": { + "name": "length" + } + } + ], + "outputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "links": [ + 16 + ], + "slot_index": 0 + }, + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 22 + ], + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "MiniMaxH3ImageToVideo" + }, + "widgets_values": [ + "", + 832, + 480, + 124 + ] + }, + { + "id": 11, + "type": "LoraLoaderModelOnly", + "pos": [ + 2060, + 0 + ], + "size": [ + 360, + 90 + ], + "flags": {}, + "order": 10, + "mode": 4, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 14 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 15, + 17 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoraLoaderModelOnly" + }, + "widgets_values": [ + "minimax_h3_fl2v_turbo_8step_v1.0_comfyui_bf16.safetensors", + 1.0 + ], + "title": "Turbo LoRA (Ctrl+B to enable, then steps 4-8)" + }, + { + "id": 12, + "type": "BasicGuider", + "pos": [ + 2060, + 120 + ], + "size": [ + 220, + 50 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 15 + }, + { + "name": "conditioning", + "type": "CONDITIONING", + "link": 16 + } + ], + "outputs": [ + { + "name": "GUIDER", + "type": "GUIDER", + "links": [ + 19 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "BasicGuider" + }, + "widgets_values": [] + }, + { + "id": 13, + "type": "RandomNoise", + "pos": [ + 2060, + 210 + ], + "size": [ + 260, + 90 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "NOISE", + "type": "NOISE", + "links": [ + 18 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "RandomNoise" + }, + "widgets_values": [ + 7, + "fixed" + ] + }, + { + "id": 14, + "type": "KSamplerSelect", + "pos": [ + 2060, + 330 + ], + "size": [ + 260, + 60 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "SAMPLER", + "type": "SAMPLER", + "links": [ + 20 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSamplerSelect" + }, + "widgets_values": [ + "res_multistep" + ] + }, + { + "id": 15, + "type": "BasicScheduler", + "pos": [ + 2060, + 420 + ], + "size": [ + 260, + 110 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 17 + } + ], + "outputs": [ + { + "name": "SIGMAS", + "type": "SIGMAS", + "links": [ + 21 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "BasicScheduler" + }, + "widgets_values": [ + "simple", + 20, + 1.0 + ] + }, + { + "id": 16, + "type": "SamplerCustomAdvanced", + "pos": [ + 2360, + 120 + ], + "size": [ + 260, + 120 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "noise", + "type": "NOISE", + "link": 18 + }, + { + "name": "guider", + "type": "GUIDER", + "link": 19 + }, + { + "name": "sampler", + "type": "SAMPLER", + "link": 20 + }, + { + "name": "sigmas", + "type": "SIGMAS", + "link": 21 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 22 + } + ], + "outputs": [ + { + "name": "output", + "type": "LATENT", + "links": [ + 23, + 25 + ], + "slot_index": 0 + }, + { + "name": "denoised_output", + "type": "LATENT", + "links": [], + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "SamplerCustomAdvanced" + }, + "widgets_values": [] + }, + { + "id": 17, + "type": "VAEDecode", + "pos": [ + 2660, + 120 + ], + "size": [ + 200, + 60 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 23 + }, + { + "name": "vae", + "type": "VAE", + "link": 24 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 27 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + }, + "widgets_values": [] + }, + { + "id": 18, + "type": "VAEDecodeAudio", + "pos": [ + 2660, + 220 + ], + "size": [ + 200, + 60 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 25 + }, + { + "name": "vae", + "type": "VAE", + "link": 26 + } + ], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 28 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecodeAudio" + }, + "widgets_values": [] + }, + { + "id": 19, + "type": "CreateVideo", + "pos": [ + 2900, + 120 + ], + "size": [ + 240, + 100 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 27 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 28 + } + ], + "outputs": [ + { + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 29 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CreateVideo" + }, + "widgets_values": [ + 24.0 + ] + }, + { + "id": 20, + "type": "SaveVideo", + "pos": [ + 2900, + 260 + ], + "size": [ + 420, + 420 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "video", + "type": "VIDEO", + "link": 29 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "video/difforum_h3", + "auto", + "auto" + ] + }, + { + "id": 21, + "type": "PreviewImage", + "pos": [ + 1200, + 580 + ], + "size": [ + 340, + 240 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 30 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [], + "title": "first / last frame" + }, + { + "id": 22, + "type": "MarkdownNote", + "pos": [ + 0, + -330 + ], + "size": [ + 620, + 280 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "## MiniMax H3 · first / last frame\n\nThe **Director** draws the move, **Guide Frames** carries your first frame along it, and **H3 Shot** hands H3 the first frame, the frame where the camera ends, the length (17k+5), the size and a prompt with the camera move in words. H3 then performs *your* move.\n\n- Model files: the names of the official MiniMax H3 templates - swap for yours.\n- Faster: enable the Turbo LoRA (Ctrl+B) and set steps 6-8.\n- Longer than 20 s: raise `segment` on H3 Shot and feed each render's last frame forward." + ] + } + ], + "links": [ + [ + 1, + 1, + 0, + 2, + 0, + "DIFFORUM_PARAMS" + ], + [ + 2, + 3, + 0, + 4, + 0, + "IMAGE" + ], + [ + 3, + 2, + 0, + 4, + 1, + "DIFFORUM_DIRECTION" + ], + [ + 4, + 4, + 0, + 5, + 0, + "IMAGE" + ], + [ + 5, + 2, + 0, + 5, + 1, + "DIFFORUM_DIRECTION" + ], + [ + 6, + 7, + 0, + 10, + 0, + "CLIP" + ], + [ + 7, + 8, + 0, + 10, + 1, + "VAE" + ], + [ + 8, + 5, + 0, + 10, + 2, + "IMAGE" + ], + [ + 9, + 5, + 1, + 10, + 3, + "IMAGE" + ], + [ + 10, + 5, + 5, + 10, + 4, + "STRING" + ], + [ + 11, + 5, + 3, + 10, + 5, + "INT" + ], + [ + 12, + 5, + 4, + 10, + 6, + "INT" + ], + [ + 13, + 5, + 2, + 10, + 7, + "INT" + ], + [ + 14, + 6, + 0, + 11, + 0, + "MODEL" + ], + [ + 15, + 11, + 0, + 12, + 0, + "MODEL" + ], + [ + 16, + 10, + 0, + 12, + 1, + "CONDITIONING" + ], + [ + 17, + 11, + 0, + 15, + 0, + "MODEL" + ], + [ + 18, + 13, + 0, + 16, + 0, + "NOISE" + ], + [ + 19, + 12, + 0, + 16, + 1, + "GUIDER" + ], + [ + 20, + 14, + 0, + 16, + 2, + "SAMPLER" + ], + [ + 21, + 15, + 0, + 16, + 3, + "SIGMAS" + ], + [ + 22, + 10, + 1, + 16, + 4, + "LATENT" + ], + [ + 23, + 16, + 0, + 17, + 0, + "LATENT" + ], + [ + 24, + 8, + 0, + 17, + 1, + "VAE" + ], + [ + 25, + 16, + 0, + 18, + 0, + "LATENT" + ], + [ + 26, + 9, + 0, + 18, + 1, + "VAE" + ], + [ + 27, + 17, + 0, + 19, + 0, + "IMAGE" + ], + [ + 28, + 18, + 0, + 19, + 1, + "AUDIO" + ], + [ + 29, + 19, + 0, + 20, + 0, + "VIDEO" + ], + [ + 30, + 5, + 1, + 21, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.65, + "offset": [ + 40, + 40 + ] + }, + "difforum": "1.0", + "title": "08 · MiniMax H3 first/last frame (FL2VA) from the Director" + }, + "version": 0.4 +} diff --git a/example_workflows/09_camera_to_ae_blender.json b/example_workflows/09_camera_to_ae_blender.json new file mode 100644 index 0000000..c83bd71 --- /dev/null +++ b/example_workflows/09_camera_to_ae_blender.json @@ -0,0 +1,552 @@ +{ + "last_node_id": 9, + "last_link_id": 8, + "nodes": [ + { + "id": 1, + "type": "Difforum_Setup", + "pos": [ + 0, + 0 + ], + "size": [ + 320, + 300 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "links": [ + 1, + 4, + 6 + ], + "slot_index": 0 + }, + { + "name": "width", + "type": "INT", + "links": [], + "slot_index": 1 + }, + { + "name": "height", + "type": "INT", + "links": [], + "slot_index": 2 + }, + { + "name": "frames", + "type": "INT", + "links": [], + "slot_index": 3 + }, + { + "name": "fps", + "type": "FLOAT", + "links": [], + "slot_index": 4 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 5 + } + ], + "properties": { + "Node name for S&R": "Difforum_Setup" + }, + "widgets_values": [ + "feedback (SD/SDXL/Flux)", + "seconds", + 5.0, + 24.0, + "16:9 landscape", + 768, + 0, + "fixed", + 16.0, + 9.0, + 0.0 + ] + }, + { + "id": 2, + "type": "Difforum_Director", + "pos": [ + 360, + 0 + ], + "size": [ + 800, + 880 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": 1 + }, + { + "name": "clip", + "type": "CLIP", + "link": null + }, + { + "name": "audio", + "type": "DIFFORUM_AUDIO", + "link": null + } + ], + "outputs": [ + { + "name": "direction", + "type": "DIFFORUM_DIRECTION", + "links": [ + 2 + ], + "slot_index": 0 + }, + { + "name": "camera", + "type": "DIFFORUM_CAMERA", + "links": [], + "slot_index": 1 + }, + { + "name": "strength", + "type": "DIFFORUM_SCHEDULE", + "links": [], + "slot_index": 2 + }, + { + "name": "prompts", + "type": "DIFFORUM_PROMPT", + "links": [], + "slot_index": 3 + }, + { + "name": "camera_text", + "type": "STRING", + "links": [], + "slot_index": 4 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 5 + } + ], + "properties": { + "Node name for S&R": "Difforum_Director" + }, + "widgets_values": [ + "{\"version\": 2, \"scenes\": [{\"start\": 0, \"mood\": \"calm\", \"prompt\": \"misty ancient forest at dawn, volumetric light, wide shot\"}, {\"start\": 30, \"mood\": \"build\", \"prompt\": \"glowing roots and moss, bioluminescent details, macro\"}, {\"start\": 60, \"mood\": \"climax\", \"prompt\": \"explosion of light, cosmic transformation, high contrast\"}, {\"start\": 90, \"mood\": \"resolve\", \"prompt\": \"embers becoming stars, calm, vast cosmic scale\"}], \"camera\": [{\"start\": 0, \"move\": \"zoom_in\", \"speed\": 0.8, \"intensity\": 0.8, \"lens\": 0.0, \"ease\": \"ease_in_out\", \"react\": \"none\"}, {\"start\": 30, \"move\": \"pan_right\", \"speed\": 1.0, \"intensity\": 0.8, \"lens\": 0.0, \"ease\": \"ease_in_out\", \"react\": \"none\"}, {\"start\": 60, \"move\": \"spiral\", \"speed\": 1.4, \"intensity\": 1.1, \"lens\": 0.0, \"ease\": \"ease_in\", \"react\": \"none\"}, {\"start\": 90, \"move\": \"zoom_out\", \"speed\": 0.6, \"intensity\": 0.5, \"lens\": 0.0, \"ease\": \"ease_out\", \"react\": \"none\"}], \"energy\": [], \"guidance\": []}", + "3d", + "cinematic", + 1.0, + 1.0, + 0.0, + 0.0, + 0, + "fixed" + ] + }, + { + "id": 3, + "type": "Difforum_CameraExport", + "pos": [ + 1200, + 0 + ], + "size": [ + 360, + 200 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "direction", + "type": "DIFFORUM_DIRECTION", + "link": 2 + }, + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": null + }, + { + "name": "camera", + "type": "DIFFORUM_CAMERA", + "link": null + } + ], + "outputs": [ + { + "name": "paths", + "type": "STRING", + "links": [ + 3 + ], + "slot_index": 0 + }, + { + "name": "json", + "type": "STRING", + "links": [], + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "Difforum_CameraExport" + }, + "widgets_values": [ + "all", + "difforum_camera", + 0.1, + 1.0 + ] + }, + { + "id": 4, + "type": "PreviewAny", + "pos": [ + 1600, + 0 + ], + "size": [ + 360, + 160 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "source", + "type": "*", + "link": 3 + } + ], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "PreviewAny" + }, + "widgets_values": [], + "title": "Written files" + }, + { + "id": 5, + "type": "Difforum_CameraImport", + "pos": [ + 1200, + 260 + ], + "size": [ + 360, + 200 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": 4 + } + ], + "outputs": [ + { + "name": "camera", + "type": "DIFFORUM_CAMERA", + "links": [ + 7 + ], + "slot_index": 0 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "Difforum_CameraImport" + }, + "widgets_values": [ + "(put a camera .json in ComfyUI/input)", + "match frames", + "" + ] + }, + { + "id": 6, + "type": "LoadImage", + "pos": [ + 0, + 360 + ], + "size": [ + 320, + 360 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 5 + ], + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": [], + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "example.png", + null + ] + }, + { + "id": 7, + "type": "Difforum_Storyboard", + "pos": [ + 1600, + 260 + ], + "size": [ + 320, + 220 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "init_image", + "type": "IMAGE", + "link": 5 + }, + { + "name": "direction", + "type": "DIFFORUM_DIRECTION", + "link": null + }, + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": 6 + }, + { + "name": "camera", + "type": "DIFFORUM_CAMERA", + "link": 7 + }, + { + "name": "depth", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "sheet", + "type": "IMAGE", + "links": [ + 8 + ], + "slot_index": 0 + }, + { + "name": "frames", + "type": "IMAGE", + "links": [], + "slot_index": 1 + }, + { + "name": "camera_path", + "type": "IMAGE", + "links": [], + "slot_index": 2 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 3 + } + ], + "properties": { + "Node name for S&R": "Difforum_Storyboard" + }, + "widgets_values": [ + 8, + 6, + 0.5, + "none" + ] + }, + { + "id": 8, + "type": "PreviewImage", + "pos": [ + 1960, + 260 + ], + "size": [ + 520, + 320 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 8 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 9, + "type": "MarkdownNote", + "pos": [ + 0, + -320 + ], + "size": [ + 620, + 260 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "## Camera interchange\n\n**Camera Export** writes to `output/difforum/`: a `.jsx` for After Effects (File > Scripts > Run Script File), a `.py` for Blender (Text Editor > Run) and a `.json`. Composite titles or 3D on top of the render with the same camera.\n\n**Camera Import** reads a `.json` from `input/` - from Camera Export, or from `tools/blender/difforum_export_camera.py` / `tools/aftereffects/Difforum Export Camera.jsx` - so a move blocked in Blender or AE drives the AI shot. (The import branch errors until a file is in `input/`: bypass it with Ctrl+B.)" + ] + } + ], + "links": [ + [ + 1, + 1, + 0, + 2, + 0, + "DIFFORUM_PARAMS" + ], + [ + 2, + 2, + 0, + 3, + 0, + "DIFFORUM_DIRECTION" + ], + [ + 3, + 3, + 0, + 4, + 0, + "STRING" + ], + [ + 4, + 1, + 0, + 5, + 0, + "DIFFORUM_PARAMS" + ], + [ + 5, + 6, + 0, + 7, + 0, + "IMAGE" + ], + [ + 6, + 1, + 0, + 7, + 2, + "DIFFORUM_PARAMS" + ], + [ + 7, + 5, + 0, + 7, + 3, + "DIFFORUM_CAMERA" + ], + [ + 8, + 7, + 0, + 8, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.65, + "offset": [ + 40, + 40 + ] + }, + "difforum": "1.0", + "title": "09 · Camera to After Effects / Blender (and back)" + }, + "version": 0.4 +} diff --git a/example_workflows/10_h3_multikeyframe_guides.json b/example_workflows/10_h3_multikeyframe_guides.json new file mode 100644 index 0000000..06715fb --- /dev/null +++ b/example_workflows/10_h3_multikeyframe_guides.json @@ -0,0 +1,1491 @@ +{ + "last_node_id": 24, + "last_link_id": 36, + "nodes": [ + { + "id": 1, + "type": "Difforum_Setup", + "pos": [ + 0, + 0 + ], + "size": [ + 320, + 300 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "links": [ + 1 + ], + "slot_index": 0 + }, + { + "name": "width", + "type": "INT", + "links": [ + 12 + ], + "slot_index": 1 + }, + { + "name": "height", + "type": "INT", + "links": [ + 13 + ], + "slot_index": 2 + }, + { + "name": "frames", + "type": "INT", + "links": [], + "slot_index": 3 + }, + { + "name": "fps", + "type": "FLOAT", + "links": [ + 5 + ], + "slot_index": 4 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 5 + } + ], + "properties": { + "Node name for S&R": "Difforum_Setup" + }, + "widgets_values": [ + "MiniMax H3", + "seconds", + 5.0, + 24.0, + "16:9 landscape", + 832, + 0, + "fixed", + 16.0, + 9.0, + 0.0 + ] + }, + { + "id": 2, + "type": "Difforum_Director", + "pos": [ + 360, + 0 + ], + "size": [ + 800, + 880 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": 1 + }, + { + "name": "clip", + "type": "CLIP", + "link": null + }, + { + "name": "audio", + "type": "DIFFORUM_AUDIO", + "link": null + } + ], + "outputs": [ + { + "name": "direction", + "type": "DIFFORUM_DIRECTION", + "links": [ + 3, + 6 + ], + "slot_index": 0 + }, + { + "name": "camera", + "type": "DIFFORUM_CAMERA", + "links": [], + "slot_index": 1 + }, + { + "name": "strength", + "type": "DIFFORUM_SCHEDULE", + "links": [], + "slot_index": 2 + }, + { + "name": "prompts", + "type": "DIFFORUM_PROMPT", + "links": [], + "slot_index": 3 + }, + { + "name": "camera_text", + "type": "STRING", + "links": [], + "slot_index": 4 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 5 + } + ], + "properties": { + "Node name for S&R": "Difforum_Director" + }, + "widgets_values": [ + "{\"version\": 2, \"scenes\": [{\"start\": 0, \"mood\": \"calm\", \"prompt\": \"misty ancient forest at dawn, light shafts through the canopy\"}, {\"start\": 62, \"mood\": \"build\", \"prompt\": \"the mist lifts, glowing moss and roots in the foreground\"}], \"camera\": [{\"start\": 0, \"move\": \"dolly_in\", \"speed\": 0.8, \"intensity\": 0.8, \"lens\": 0.0, \"ease\": \"ease_in_out\", \"react\": \"none\"}, {\"start\": 62, \"move\": \"orbit_right\", \"speed\": 0.9, \"intensity\": 0.8, \"lens\": 0.0, \"ease\": \"ease_in_out\", \"react\": \"none\"}], \"energy\": [], \"guidance\": []}", + "3d", + "cinematic", + 1.0, + 1.0, + 0.0, + 0.0, + 0, + "fixed" + ] + }, + { + "id": 3, + "type": "LoadImage", + "pos": [ + 0, + 360 + ], + "size": [ + 320, + 360 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 2, + 10 + ], + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": [], + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "example.png", + null + ], + "title": "Anchor image (also the H3 reference)" + }, + { + "id": 4, + "type": "Difforum_GuideFrames", + "pos": [ + 1200, + 0 + ], + "size": [ + 340, + 200 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "anchor_image", + "type": "IMAGE", + "link": 2 + }, + { + "name": "direction", + "type": "DIFFORUM_DIRECTION", + "link": 3 + }, + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": null + }, + { + "name": "camera", + "type": "DIFFORUM_CAMERA", + "link": null + }, + { + "name": "depth", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "guide_frames", + "type": "IMAGE", + "links": [ + 4 + ], + "slot_index": 0 + }, + { + "name": "masks", + "type": "MASK", + "links": [], + "slot_index": 1 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "Difforum_GuideFrames" + }, + "widgets_values": [ + "1 = generate (VACE / LTX / H3)", + "stretch edge", + 1.0 + ] + }, + { + "id": 5, + "type": "Difforum_Keyframes", + "pos": [ + 1200, + 240 + ], + "size": [ + 340, + 220 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "frames", + "type": "IMAGE", + "link": 4 + }, + { + "name": "fps", + "type": "FLOAT", + "link": 5, + "widget": { + "name": "fps" + } + } + ], + "outputs": [ + { + "name": "keyframes", + "type": "IMAGE", + "links": [ + 18, + 36 + ], + "slot_index": 0 + }, + { + "name": "indices", + "type": "STRING", + "links": [ + 19 + ], + "slot_index": 1 + }, + { + "name": "sparse_batch", + "type": "IMAGE", + "links": [], + "slot_index": 2 + }, + { + "name": "first_frame", + "type": "IMAGE", + "links": [], + "slot_index": 3 + }, + { + "name": "last_frame", + "type": "IMAGE", + "links": [], + "slot_index": 4 + }, + { + "name": "length", + "type": "INT", + "links": [ + 14 + ], + "slot_index": 5 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 6 + } + ], + "properties": { + "Node name for S&R": "Difforum_Keyframes" + }, + "widgets_values": [ + "MiniMax H3 (17k+5)", + 2.0, + 24.0, + "" + ] + }, + { + "id": 6, + "type": "Difforum_CameraPrompt", + "pos": [ + 1200, + 500 + ], + "size": [ + 340, + 220 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "direction", + "type": "DIFFORUM_DIRECTION", + "link": 6 + }, + { + "name": "params", + "type": "DIFFORUM_PARAMS", + "link": null + }, + { + "name": "camera", + "type": "DIFFORUM_CAMERA", + "link": null + } + ], + "outputs": [ + { + "name": "text", + "type": "STRING", + "links": [ + 11 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "Difforum_CameraPrompt" + }, + "widgets_values": [ + "sentence", + " shows a misty ancient forest at dawn. Keep its look, light and composition. Soft ambient birdsong, a low wind." + ] + }, + { + "id": 7, + "type": "UNETLoader", + "pos": [ + 1580, + -200 + ], + "size": [ + 420, + 90 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 20 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "UNETLoader" + }, + "widgets_values": [ + "minimax_h3_ref2va_pruned_int8_convrot.safetensors", + "default" + ] + }, + { + "id": 8, + "type": "CLIPLoader", + "pos": [ + 1580, + -80 + ], + "size": [ + 420, + 110 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 7 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPLoader" + }, + "widgets_values": [ + "qwen3vl_32b_minimax_h3_nvfp4_awq.safetensors", + "minimax", + "default" + ] + }, + { + "id": 9, + "type": "VAELoader", + "pos": [ + 1580, + 60 + ], + "size": [ + 420, + 60 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 8, + 17, + 30 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAELoader" + }, + "widgets_values": [ + "minimax_h3_video_vae_int8_convrot.safetensors" + ] + }, + { + "id": 10, + "type": "VAELoader", + "pos": [ + 1580, + 150 + ], + "size": [ + 420, + 60 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 9, + 32 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAELoader" + }, + "widgets_values": [ + "minimax_h3_audio_vae_fp32.safetensors" + ] + }, + { + "id": 11, + "type": "MiniMaxH3ReferenceToVideo", + "pos": [ + 1580, + 300 + ], + "size": [ + 420, + 300 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": 8 + }, + { + "name": "audio_vae", + "type": "VAE", + "link": 9 + }, + { + "name": "ref_images.ref_image_0", + "type": "IMAGE", + "link": 10 + }, + { + "name": "prompt", + "type": "STRING", + "link": 11, + "widget": { + "name": "prompt" + } + }, + { + "name": "width", + "type": "INT", + "link": 12, + "widget": { + "name": "width" + } + }, + { + "name": "height", + "type": "INT", + "link": 13, + "widget": { + "name": "height" + } + }, + { + "name": "length", + "type": "INT", + "link": 14, + "widget": { + "name": "length" + } + } + ], + "outputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "links": [ + 15 + ], + "slot_index": 0 + }, + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 16, + 28 + ], + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "MiniMaxH3ReferenceToVideo" + }, + "widgets_values": [ + "", + 832, + 480, + 124, + "match" + ] + }, + { + "id": 12, + "type": "Difforum_H3Guides", + "pos": [ + 1580, + 640 + ], + "size": [ + 420, + 200 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "link": 15 + }, + { + "name": "latent", + "type": "LATENT", + "link": 16 + }, + { + "name": "vae", + "type": "VAE", + "link": 17 + }, + { + "name": "keyframes", + "type": "IMAGE", + "link": 18 + }, + { + "name": "indices", + "type": "STRING", + "link": 19 + }, + { + "name": "audio_vae", + "type": "VAE", + "link": null + }, + { + "name": "audio", + "type": "AUDIO", + "link": null + } + ], + "outputs": [ + { + "name": "positive", + "type": "CONDITIONING", + "links": [ + 22 + ], + "slot_index": 0 + }, + { + "name": "info", + "type": "STRING", + "links": [], + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "Difforum_H3Guides" + }, + "widgets_values": [ + 4, + false + ] + }, + { + "id": 13, + "type": "LoraLoaderModelOnly", + "pos": [ + 2060, + 0 + ], + "size": [ + 360, + 90 + ], + "flags": {}, + "order": 12, + "mode": 4, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 20 + } + ], + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 21, + 23 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoraLoaderModelOnly" + }, + "widgets_values": [ + "minimax_h3_ref2v_turbo_4step_v0.1_comfyui_bf16.safetensors", + 1.0 + ], + "title": "Turbo LoRA (Ctrl+B to enable, then steps 4-8)" + }, + { + "id": 14, + "type": "BasicGuider", + "pos": [ + 2060, + 120 + ], + "size": [ + 220, + 50 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 21 + }, + { + "name": "conditioning", + "type": "CONDITIONING", + "link": 22 + } + ], + "outputs": [ + { + "name": "GUIDER", + "type": "GUIDER", + "links": [ + 25 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "BasicGuider" + }, + "widgets_values": [] + }, + { + "id": 15, + "type": "RandomNoise", + "pos": [ + 2060, + 210 + ], + "size": [ + 260, + 90 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "NOISE", + "type": "NOISE", + "links": [ + 24 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "RandomNoise" + }, + "widgets_values": [ + 7, + "fixed" + ] + }, + { + "id": 16, + "type": "KSamplerSelect", + "pos": [ + 2060, + 330 + ], + "size": [ + 260, + 60 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "SAMPLER", + "type": "SAMPLER", + "links": [ + 26 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSamplerSelect" + }, + "widgets_values": [ + "res_multistep" + ] + }, + { + "id": 17, + "type": "BasicScheduler", + "pos": [ + 2060, + 420 + ], + "size": [ + 260, + 110 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 23 + } + ], + "outputs": [ + { + "name": "SIGMAS", + "type": "SIGMAS", + "links": [ + 27 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "BasicScheduler" + }, + "widgets_values": [ + "simple", + 20, + 1.0 + ] + }, + { + "id": 18, + "type": "SamplerCustomAdvanced", + "pos": [ + 2360, + 120 + ], + "size": [ + 260, + 120 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [ + { + "name": "noise", + "type": "NOISE", + "link": 24 + }, + { + "name": "guider", + "type": "GUIDER", + "link": 25 + }, + { + "name": "sampler", + "type": "SAMPLER", + "link": 26 + }, + { + "name": "sigmas", + "type": "SIGMAS", + "link": 27 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 28 + } + ], + "outputs": [ + { + "name": "output", + "type": "LATENT", + "links": [ + 29, + 31 + ], + "slot_index": 0 + }, + { + "name": "denoised_output", + "type": "LATENT", + "links": [], + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "SamplerCustomAdvanced" + }, + "widgets_values": [] + }, + { + "id": 19, + "type": "VAEDecode", + "pos": [ + 2660, + 120 + ], + "size": [ + 200, + 60 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 29 + }, + { + "name": "vae", + "type": "VAE", + "link": 30 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 33 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + }, + "widgets_values": [] + }, + { + "id": 20, + "type": "VAEDecodeAudio", + "pos": [ + 2660, + 220 + ], + "size": [ + 200, + 60 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 31 + }, + { + "name": "vae", + "type": "VAE", + "link": 32 + } + ], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 34 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecodeAudio" + }, + "widgets_values": [] + }, + { + "id": 21, + "type": "CreateVideo", + "pos": [ + 2900, + 120 + ], + "size": [ + 240, + 100 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 33 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 34 + } + ], + "outputs": [ + { + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 35 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CreateVideo" + }, + "widgets_values": [ + 24.0 + ] + }, + { + "id": 22, + "type": "SaveVideo", + "pos": [ + 2900, + 260 + ], + "size": [ + 420, + 420 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "name": "video", + "type": "VIDEO", + "link": 35 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "video/difforum_h3", + "auto", + "auto" + ] + }, + { + "id": 23, + "type": "PreviewImage", + "pos": [ + 1200, + 760 + ], + "size": [ + 340, + 240 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 36 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [], + "title": "Keyframes sent to H3" + }, + { + "id": 24, + "type": "MarkdownNote", + "pos": [ + 0, + -330 + ], + "size": [ + 620, + 330 + ], + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "## MiniMax H3 · keyframes along your camera\n\nThe **Director** draws the move. **Guide Frames** carries the anchor image along it, **Keyframes** picks one every 2 s on the H3 grid, and **H3 Guides** anchors them inside the generation with the core `MiniMaxH3AddGuide` - the same mechanism as the official *Multiframe Reference* template, driven by a camera instead of hand-placed images.\n\n- The anchor is also ``, which holds identity and style.\n- **Camera → Prompt** adds the move in words.\n- Swap Guide Frames for a Feedback render or a Storyboard to guide with other images; connect an **audio** to H3 Guides to anchor a soundtrack at frame 0.\n- Model files follow the official template - swap for yours. Turbo LoRA: Ctrl+B, steps 4." + ] + } + ], + "links": [ + [ + 1, + 1, + 0, + 2, + 0, + "DIFFORUM_PARAMS" + ], + [ + 2, + 3, + 0, + 4, + 0, + "IMAGE" + ], + [ + 3, + 2, + 0, + 4, + 1, + "DIFFORUM_DIRECTION" + ], + [ + 4, + 4, + 0, + 5, + 0, + "IMAGE" + ], + [ + 5, + 1, + 4, + 5, + 1, + "FLOAT" + ], + [ + 6, + 2, + 0, + 6, + 0, + "DIFFORUM_DIRECTION" + ], + [ + 7, + 8, + 0, + 11, + 0, + "CLIP" + ], + [ + 8, + 9, + 0, + 11, + 1, + "VAE" + ], + [ + 9, + 10, + 0, + 11, + 2, + "VAE" + ], + [ + 10, + 3, + 0, + 11, + 3, + "IMAGE" + ], + [ + 11, + 6, + 0, + 11, + 4, + "STRING" + ], + [ + 12, + 1, + 1, + 11, + 5, + "INT" + ], + [ + 13, + 1, + 2, + 11, + 6, + "INT" + ], + [ + 14, + 5, + 5, + 11, + 7, + "INT" + ], + [ + 15, + 11, + 0, + 12, + 0, + "CONDITIONING" + ], + [ + 16, + 11, + 1, + 12, + 1, + "LATENT" + ], + [ + 17, + 9, + 0, + 12, + 2, + "VAE" + ], + [ + 18, + 5, + 0, + 12, + 3, + "IMAGE" + ], + [ + 19, + 5, + 1, + 12, + 4, + "STRING" + ], + [ + 20, + 7, + 0, + 13, + 0, + "MODEL" + ], + [ + 21, + 13, + 0, + 14, + 0, + "MODEL" + ], + [ + 22, + 12, + 0, + 14, + 1, + "CONDITIONING" + ], + [ + 23, + 13, + 0, + 17, + 0, + "MODEL" + ], + [ + 24, + 15, + 0, + 18, + 0, + "NOISE" + ], + [ + 25, + 14, + 0, + 18, + 1, + "GUIDER" + ], + [ + 26, + 16, + 0, + 18, + 2, + "SAMPLER" + ], + [ + 27, + 17, + 0, + 18, + 3, + "SIGMAS" + ], + [ + 28, + 11, + 1, + 18, + 4, + "LATENT" + ], + [ + 29, + 18, + 0, + 19, + 0, + "LATENT" + ], + [ + 30, + 9, + 0, + 19, + 1, + "VAE" + ], + [ + 31, + 18, + 0, + 20, + 0, + "LATENT" + ], + [ + 32, + 10, + 0, + 20, + 1, + "VAE" + ], + [ + 33, + 19, + 0, + 21, + 0, + "IMAGE" + ], + [ + 34, + 20, + 0, + 21, + 1, + "AUDIO" + ], + [ + 35, + 21, + 0, + 22, + 0, + "VIDEO" + ], + [ + 36, + 5, + 0, + 23, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.65, + "offset": [ + 40, + 40 + ] + }, + "difforum": "1.0", + "title": "10 · MiniMax H3 multi-keyframe guides from the Director" + }, + "version": 0.4 +} diff --git a/examples/_build_examples.py b/examples/_build_examples.py deleted file mode 100644 index 1960c0c..0000000 --- a/examples/_build_examples.py +++ /dev/null @@ -1,1457 +0,0 @@ -""" -Generate Difforum example workflows as ComfyUI litegraph JSON. - -Run with any python: python examples/_build_examples.py -Produces (next to this script): - difforum_schedule_basic.json - self-contained, no assets, runs anywhere - difforum_audio_reactive.json - needs an audio file via Load Audio - -Also validates that every link references an existing node/slot before writing. -""" - -import json -from pathlib import Path - -HERE = Path(__file__).resolve().parent - -# Where each workflow lands. `basic` is for learning one idea at a time - little -# or nothing to download, few nodes, no external packs. `advanced` is for full -# productions: several models, helper node packs, longer graphs. -TIERS = { - "difforum_schedule_basic.json": "basic", - "difforum_models_info.json": "basic", - "difforum_camera_warp.json": "basic", - "difforum_audio_reactive.json": "basic", - "difforum_intuitive_controls.json": "basic", - "difforum_feedback_classic.json": "basic", - "difforum_vj_footage.json": "basic", - "difforum_turbo_live.json": "basic", - - "difforum_fast_sdxl.json": "advanced", - "difforum_film_director.json": "advanced", - "difforum_parallax_fluid.json": "advanced", - "difforum_seamless_installation.json": "advanced", - "difforum_ipadapter_coherent.json": "advanced", - "difforum_audio_reactive_video.json": "advanced", - "difforum_deluxe_travel_controlnet_video.json": "advanced", - "difforum_animatediff_sd15.json": "advanced", - "difforum_qrcode_illusion.json": "advanced", - "difforum_mesmerize_kaleidoscope.json": "advanced", - "difforum_hybrid_wan_guides.json": "advanced", - "difforum_realtime_live.json": "advanced", -} - - -class WF: - """Tiny builder for litegraph workflow JSON.""" - - def __init__(self): - self.nodes = [] - self.links = [] - self._nid = 0 - self._lid = 0 - - def node(self, type_, pos, inputs, outputs, widgets=None, is_output=False): - self._nid += 1 - node = { - "id": self._nid, - "type": type_, - "pos": pos, - "size": [320, 200], - "flags": {}, - "order": self._nid - 1, - "mode": 0, - "inputs": [ - {"name": n, "type": t, "link": None} for (n, t) in inputs - ], - "outputs": [ - {"name": n, "type": t, "links": [], "slot_index": i} - for i, (n, t) in enumerate(outputs) - ], - "properties": {"Node name for S&R": type_}, - "widgets_values": widgets or [], - } - self.nodes.append(node) - return self._nid - - def link(self, src_id, src_slot, dst_id, dst_slot, type_): - self._lid += 1 - self.links.append([self._lid, src_id, src_slot, dst_id, dst_slot, type_]) - src = next(n for n in self.nodes if n["id"] == src_id) - dst = next(n for n in self.nodes if n["id"] == dst_id) - src["outputs"][src_slot]["links"].append(self._lid) - dst["inputs"][dst_slot]["link"] = self._lid - return self._lid - - def validate(self): - ids = {n["id"] for n in self.nodes} - for lid, s, ss, d, ds, _ in self.links: - assert s in ids and d in ids, f"link {lid} dangling node" - src = next(n for n in self.nodes if n["id"] == s) - dst = next(n for n in self.nodes if n["id"] == d) - assert ss < len(src["outputs"]), f"link {lid} bad src slot" - assert ds < len(dst["inputs"]), f"link {lid} bad dst slot" - return True - - def group(self, title, bounding, color="#3f5159"): - if not hasattr(self, "groups"): - self.groups = [] - self.groups.append({"title": title, "bounding": list(bounding), - "color": color, "font_size": 22, "locked": False}) - - def dump(self): - self.validate() - return { - "last_node_id": self._nid, - "last_link_id": self._lid, - "nodes": self.nodes, - "links": self.links, - "groups": getattr(self, "groups", []), - "config": {}, - "extra": {}, - "version": 0.4, - } - - -def build_basic(): - w = WF() - setup = w.node( - "DifforumAnimSetup", [80, 200], - inputs=[], - outputs=[("params", "DIFFORUM_PARAMS")], - widgets=[768, 768, 24, 120, 0, "fixed"], - ) - sched = w.node( - "DifforumSchedule", [460, 200], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "DIFFORUM_AUDIO")], - outputs=[("schedule", "DIFFORUM_SCHEDULE"), ("values", "FLOAT")], - widgets=["0:(0), 60:(0.5*sin(2*pi*t/30)), 120:(1.0)", "linear"], - ) - info = w.node( - "DifforumScheduleInfo", [860, 120], - inputs=[("schedule", "DIFFORUM_SCHEDULE")], - outputs=[("info", "STRING")], - widgets=[], - is_output=True, - ) - plot = w.node( - "DifforumSchedulePlot", [860, 320], - inputs=[("schedule", "DIFFORUM_SCHEDULE")], - outputs=[("plot", "IMAGE")], - widgets=[512, 256], - ) - preview = w.node( - "PreviewImage", [1200, 320], inputs=[("images", "IMAGE")], - outputs=[], widgets=[], is_output=True, - ) - w.link(setup, 0, sched, 0, "DIFFORUM_PARAMS") - w.link(sched, 0, info, 0, "DIFFORUM_SCHEDULE") - w.link(sched, 0, plot, 0, "DIFFORUM_SCHEDULE") - w.link(plot, 0, preview, 0, "IMAGE") - return w.dump() - - -def build_audio(): - w = WF() - setup = w.node( - "DifforumAnimSetup", [80, 120], - inputs=[], - outputs=[("params", "DIFFORUM_PARAMS")], - widgets=[768, 768, 24, 240, 0, "fixed"], - ) - load = w.node( - "LoadAudio", [80, 420], - inputs=[], - outputs=[("AUDIO", "AUDIO")], - widgets=["your_track.mp3"], - ) - analyzer = w.node( - "DifforumAudioAnalyzer", [460, 360], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "AUDIO")], - outputs=[ - ("audio_curves", "DIFFORUM_AUDIO"), ("amp", "FLOAT"), ("low", "FLOAT"), - ("mid", "FLOAT"), ("high", "FLOAT"), ("onset", "FLOAT"), ("beat", "FLOAT"), - ], - widgets=[0.25, True, 1.5], - ) - zoom = w.node( - "DifforumSchedule", [880, 120], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "DIFFORUM_AUDIO")], - outputs=[("schedule", "DIFFORUM_SCHEDULE"), ("values", "FLOAT")], - widgets=["0:(1.0 + 0.6*low)", "linear"], # zoom pumps on the bass - ) - spin = w.node( - "DifforumSchedule", [880, 380], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "DIFFORUM_AUDIO")], - outputs=[("schedule", "DIFFORUM_SCHEDULE"), ("values", "FLOAT")], - widgets=["0:(beat*30)", "linear"], # kick a spin on every beat - ) - info_zoom = w.node( - "DifforumScheduleInfo", [1280, 120], - inputs=[("schedule", "DIFFORUM_SCHEDULE")], - outputs=[("info", "STRING")], widgets=[], is_output=True, - ) - info_spin = w.node( - "DifforumScheduleInfo", [1280, 380], - inputs=[("schedule", "DIFFORUM_SCHEDULE")], - outputs=[("info", "STRING")], widgets=[], is_output=True, - ) - w.link(setup, 0, analyzer, 0, "DIFFORUM_PARAMS") - w.link(load, 0, analyzer, 1, "AUDIO") - w.link(setup, 0, zoom, 0, "DIFFORUM_PARAMS") - w.link(analyzer, 0, zoom, 1, "DIFFORUM_AUDIO") - w.link(setup, 0, spin, 0, "DIFFORUM_PARAMS") - w.link(analyzer, 0, spin, 1, "DIFFORUM_AUDIO") - w.link(zoom, 0, info_zoom, 0, "DIFFORUM_SCHEDULE") - w.link(spin, 0, info_spin, 0, "DIFFORUM_SCHEDULE") - return w.dump() - - -def build_feedback(): - """Full Classic+ pipeline: txt2img frame 0 -> Deforum feedback -> save.""" - w = WF() - ckpt = w.node( - "CheckpointLoaderSimple", [40, 40], inputs=[], - outputs=[("MODEL", "MODEL"), ("CLIP", "CLIP"), ("VAE", "VAE")], - widgets=["dreamshaper_8.safetensors"], - ) - pos = w.node( - "CLIPTextEncode", [340, 20], inputs=[("clip", "CLIP")], - outputs=[("CONDITIONING", "CONDITIONING")], - widgets=["psychedelic cosmic tunnel, vivid colors, highly detailed"], - ) - neg = w.node( - "CLIPTextEncode", [340, 200], inputs=[("clip", "CLIP")], - outputs=[("CONDITIONING", "CONDITIONING")], - widgets=["blurry, low quality, watermark, text"], - ) - lat = w.node( - "EmptyLatentImage", [340, 380], inputs=[], - outputs=[("LATENT", "LATENT")], widgets=[768, 768, 1], - ) - ks = w.node( - "KSampler", [660, 120], - inputs=[("model", "MODEL"), ("positive", "CONDITIONING"), - ("negative", "CONDITIONING"), ("latent_image", "LATENT")], - outputs=[("LATENT", "LATENT")], - widgets=[0, "fixed", 26, 6.5, "dpmpp_2m", "karras", 1.0], - ) - dec = w.node( - "VAEDecode", [980, 120], - inputs=[("samples", "LATENT"), ("vae", "VAE")], - outputs=[("IMAGE", "IMAGE")], widgets=[], - ) - setup = w.node( - "DifforumAnimSetup", [40, 520], inputs=[], - outputs=[("params", "DIFFORUM_PARAMS")], - widgets=[768, 768, 24, 48, 0, "fixed"], - ) - cam = w.node( - "DifforumCamera", [340, 560], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "DIFFORUM_AUDIO")], - outputs=[("camera", "DIFFORUM_CAMERA"), ("info", "STRING")], - widgets=["3d", 40.0, "0:(0)", "0:(0)", "0:(1.5)", "0:(0)", - "0:(0.5*sin(2*pi*t/120))", "0:(0)", "0:(1.0)"], - ) - strength = w.node( - "DifforumSchedule", [340, 820], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "DIFFORUM_AUDIO")], - outputs=[("schedule", "DIFFORUM_SCHEDULE"), ("values", "FLOAT")], - widgets=["0:(0.45)", "linear"], - ) - prompt = w.node( - "DifforumPromptSchedule", [340, 1060], - inputs=[("params", "DIFFORUM_PARAMS"), ("clip", "CLIP")], - outputs=[("positive_schedule", "DIFFORUM_PROMPT"), - ("first_frame_cond", "CONDITIONING"), ("info", "STRING")], - widgets=["0: psychedelic cosmic tunnel, vivid colors\n" - "24: deep space nebula, glowing stars\n" - "47: intricate fractal mandala, vivid", "ease_in_out"], - ) - fb = w.node( - "DifforumFeedbackSampler", [1000, 480], - inputs=[("model", "MODEL"), ("positive", "CONDITIONING"), - ("negative", "CONDITIONING"), ("vae", "VAE"), - ("params", "DIFFORUM_PARAMS"), ("camera", "DIFFORUM_CAMERA"), - ("init_image", "IMAGE"), ("strength_schedule", "DIFFORUM_SCHEDULE"), - ("depth", "IMAGE"), ("cfg_schedule", "DIFFORUM_SCHEDULE"), - ("positive_schedule", "DIFFORUM_PROMPT")], - outputs=[("frames", "IMAGE")], - widgets=[24, 6.5, "dpmpp_2m", "karras", 0.8, "lab", 1.0, 100.0, False, 1.0, - 0.6, "none", 6, "reflection", 0.25, 0.02, 1, 0, 0], - ) - save = w.node( - "SaveImage", [1700, 480], inputs=[("images", "IMAGE")], - outputs=[], widgets=["Difforum"], is_output=True, - ) - # frame 0 txt2img - w.link(ckpt, 0, ks, 0, "MODEL") - w.link(ckpt, 1, pos, 0, "CLIP") - w.link(ckpt, 1, neg, 0, "CLIP") - w.link(pos, 0, ks, 1, "CONDITIONING") - w.link(neg, 0, ks, 2, "CONDITIONING") - w.link(lat, 0, ks, 3, "LATENT") - w.link(ks, 0, dec, 0, "LATENT") - w.link(ckpt, 2, dec, 1, "VAE") - # difforum control plane - w.link(setup, 0, cam, 0, "DIFFORUM_PARAMS") - w.link(setup, 0, strength, 0, "DIFFORUM_PARAMS") - w.link(setup, 0, prompt, 0, "DIFFORUM_PARAMS") - w.link(ckpt, 1, prompt, 1, "CLIP") - # feedback sampler - w.link(ckpt, 0, fb, 0, "MODEL") - w.link(pos, 0, fb, 1, "CONDITIONING") - w.link(neg, 0, fb, 2, "CONDITIONING") - w.link(ckpt, 2, fb, 3, "VAE") - w.link(setup, 0, fb, 4, "DIFFORUM_PARAMS") - w.link(cam, 0, fb, 5, "DIFFORUM_CAMERA") - w.link(dec, 0, fb, 6, "IMAGE") - w.link(strength, 0, fb, 7, "DIFFORUM_SCHEDULE") - w.link(prompt, 0, fb, 10, "DIFFORUM_PROMPT") - stab = w.node( - "DifforumFlowStabilize", [1340, 480], - inputs=[("frames", "IMAGE")], outputs=[("frames", "IMAGE")], - widgets=[0.5, 0.5, 0.15], - ) - w.link(fb, 0, stab, 0, "IMAGE") - pp = w.node("DifforumLoopBlend", [1520, 660], inputs=[("frames", "IMAGE")], - outputs=[("frames", "IMAGE")], widgets=[12, True]) - w.link(stab, 0, pp, 0, "IMAGE") - w.link(pp, 0, save, 0, "IMAGE") - w.group("1 - FIRST FRAME (txt2img)", [10, -60, 1270, 500], "#33415a") - w.group("2 - ANIMATION BRAIN (camera + schedules + prompt travel)", [10, 470, 900, 920], "#33503f") - w.group("3 - RENDER LOOP (detail guard inside)", [960, 390, 350, 480], "#5a3a33") - w.group("4 - QUALITY: anti-flicker (select + Ctrl+B to bypass)", [1310, 390, 350, 300], "#4a3d5a") - return w.dump() - - -def build_camera_warp(): - """Show the warp: Load an image, warp one frame by the camera, preview it.""" - w = WF() - load = w.node( - "LoadImage", [40, 60], inputs=[], - outputs=[("IMAGE", "IMAGE"), ("MASK", "MASK")], widgets=["example.png"], - ) - setup = w.node( - "DifforumAnimSetup", [40, 360], inputs=[], - outputs=[("params", "DIFFORUM_PARAMS")], widgets=[768, 768, 24, 48, 0, "fixed"], - ) - cam = w.node( - "DifforumCameraMove", [360, 360], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "DIFFORUM_AUDIO")], - outputs=[("camera", "DIFFORUM_CAMERA"), ("info", "STRING")], - widgets=["spiral", 1.0, 1.0, "2d", 40.0], - ) - warp = w.node( - "DifforumWarp", [760, 120], - inputs=[("image", "IMAGE"), ("camera", "DIFFORUM_CAMERA"), ("depth", "IMAGE")], - outputs=[("warped", "IMAGE"), ("occlusion_mask", "MASK")], - widgets=[12, "follow_camera", 1.0, 100.0, False], - ) - prev = w.node( - "PreviewImage", [1120, 120], inputs=[("images", "IMAGE")], - outputs=[], widgets=[], is_output=True, - ) - w.link(setup, 0, cam, 0, "DIFFORUM_PARAMS") - w.link(load, 0, warp, 0, "IMAGE") - w.link(cam, 0, warp, 1, "DIFFORUM_CAMERA") - w.link(warp, 0, prev, 0, "IMAGE") - return w.dump() - - -def build_hybrid_guides(): - """Hybrid bridge: warp an anchor along the camera path into a Wan guide batch.""" - w = WF() - load = w.node( - "LoadImage", [40, 60], inputs=[], - outputs=[("IMAGE", "IMAGE"), ("MASK", "MASK")], widgets=["anchor.png"], - ) - setup = w.node( - "DifforumAnimSetup", [40, 360], inputs=[], - outputs=[("params", "DIFFORUM_PARAMS")], widgets=[832, 480, 16, 81, 0, "fixed"], - ) - cam = w.node( - "DifforumCamera", [360, 360], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "DIFFORUM_AUDIO")], - outputs=[("camera", "DIFFORUM_CAMERA"), ("info", "STRING")], - widgets=["3d", 40.0, "0:(0)", "0:(0)", "0:(1.2)", "0:(0)", - "0:(0.3*sin(2*pi*t/160))", "0:(0)", "0:(1.0)"], - ) - guides = w.node( - "DifforumGuideBuilder", [760, 120], - inputs=[("anchor_image", "IMAGE"), ("camera", "DIFFORUM_CAMERA"), - ("params", "DIFFORUM_PARAMS"), ("depth", "IMAGE")], - outputs=[("guide_frames", "IMAGE"), ("occlusion_masks", "MASK"), ("info", "STRING")], - widgets=["force_2d", 1.0, 100.0, False], - ) - prev = w.node( - "PreviewImage", [1140, 120], inputs=[("images", "IMAGE")], - outputs=[], widgets=[], is_output=True, - ) - w.link(setup, 0, cam, 0, "DIFFORUM_PARAMS") - w.link(load, 0, guides, 0, "IMAGE") - w.link(cam, 0, guides, 1, "DIFFORUM_CAMERA") - w.link(setup, 0, guides, 2, "DIFFORUM_PARAMS") - w.link(guides, 0, prev, 0, "IMAGE") - return w.dump() - - -def build_models_info(): - """Reference card: GPU profile + model recipe + a sampled schedule value.""" - w = WF() - profile = w.node( - "DifforumModelProfile", [40, 60], inputs=[], - outputs=[("profile", "DIFFORUM_PROFILE"), ("summary", "STRING"), - ("width", "INT"), ("height", "INT"), ("segment_frames", "INT")], - widgets=["wan22", "balanced", True, 12, "sdpa"], - is_output=True, - ) - catalog = w.node( - "DifforumModelCatalog", [40, 360], inputs=[], - outputs=[("recipe", "DIFFORUM_RECIPE"), ("guide", "STRING")], - widgets=["wan22_12gb"], is_output=True, - ) - setup = w.node( - "DifforumAnimSetup", [440, 360], inputs=[], - outputs=[("params", "DIFFORUM_PARAMS")], widgets=[768, 768, 24, 60, 0, "fixed"], - ) - sched = w.node( - "DifforumSchedule", [440, 560], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "DIFFORUM_AUDIO")], - outputs=[("schedule", "DIFFORUM_SCHEDULE"), ("values", "FLOAT")], - widgets=["0:(0.2), 30:(0.8), 59:(0.2)", "ease_in_out"], - ) - sample = w.node( - "DifforumSampleSchedule", [840, 560], - inputs=[("schedule", "DIFFORUM_SCHEDULE")], - outputs=[("value", "FLOAT")], widgets=[30], - ) - info = w.node( - "DifforumScheduleInfo", [840, 360], - inputs=[("schedule", "DIFFORUM_SCHEDULE")], - outputs=[("info", "STRING")], widgets=[], is_output=True, - ) - w.link(setup, 0, sched, 0, "DIFFORUM_PARAMS") - w.link(sched, 0, sample, 0, "DIFFORUM_SCHEDULE") - w.link(sched, 0, info, 0, "DIFFORUM_SCHEDULE") - return w.dump() - - -def build_deluxe(): - """Everything: feedback + prompt travel + ControlNet + RIFE interp -> MP4.""" - w = WF() - ckpt = w.node("CheckpointLoaderSimple", [40, 40], inputs=[], - outputs=[("MODEL", "MODEL"), ("CLIP", "CLIP"), ("VAE", "VAE")], - widgets=["dreamshaper_8.safetensors"]) - pos = w.node("CLIPTextEncode", [340, 20], inputs=[("clip", "CLIP")], - outputs=[("CONDITIONING", "CONDITIONING")], - widgets=["psychedelic mandala, vivid colors, intricate, highly detailed"]) - neg = w.node("CLIPTextEncode", [340, 200], inputs=[("clip", "CLIP")], - outputs=[("CONDITIONING", "CONDITIONING")], - widgets=["blurry, low quality, watermark, text"]) - lat = w.node("EmptyLatentImage", [340, 380], inputs=[], - outputs=[("LATENT", "LATENT")], widgets=[768, 768, 1]) - ks = w.node("KSampler", [660, 120], - inputs=[("model", "MODEL"), ("positive", "CONDITIONING"), - ("negative", "CONDITIONING"), ("latent_image", "LATENT")], - outputs=[("LATENT", "LATENT")], - widgets=[0, "fixed", 26, 6.5, "dpmpp_2m", "karras", 1.0]) - dec = w.node("VAEDecode", [980, 120], - inputs=[("samples", "LATENT"), ("vae", "VAE")], - outputs=[("IMAGE", "IMAGE")], widgets=[]) - cn = w.node("ControlNetLoader", [40, 280], inputs=[], - outputs=[("CONTROL_NET", "CONTROL_NET")], - widgets=["control_v11f1e_sd15_tile.pth"]) - setup = w.node("DifforumAnimSetup", [40, 520], inputs=[], - outputs=[("params", "DIFFORUM_PARAMS")], - widgets=[768, 768, 24, 48, 0, "fixed"]) - cam = w.node("DifforumCamera", [340, 560], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "DIFFORUM_AUDIO")], - outputs=[("camera", "DIFFORUM_CAMERA"), ("info", "STRING")], - widgets=["2d", 40.0, "0:(2)", "0:(0)", "0:(0)", "0:(0)", - "0:(0.6)", "0:(0)", "0:(1.02)"]) - strength = w.node("DifforumSchedule", [340, 820], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "DIFFORUM_AUDIO")], - outputs=[("schedule", "DIFFORUM_SCHEDULE"), ("values", "FLOAT")], - widgets=["0:(0.5)", "linear"]) - prompt = w.node("DifforumPromptSchedule", [340, 1060], - inputs=[("params", "DIFFORUM_PARAMS"), ("clip", "CLIP")], - outputs=[("positive_schedule", "DIFFORUM_PROMPT"), - ("first_frame_cond", "CONDITIONING"), ("info", "STRING")], - widgets=["0: psychedelic mandala, vivid colors\n" - "24: cosmic fractal temple, golden light\n" - "47: deep space nebula, glowing stars", "ease_in_out"]) - fb = w.node("DifforumFeedbackSampler", [1000, 480], - inputs=[("model", "MODEL"), ("positive", "CONDITIONING"), - ("negative", "CONDITIONING"), ("vae", "VAE"), - ("params", "DIFFORUM_PARAMS"), ("camera", "DIFFORUM_CAMERA"), - ("init_image", "IMAGE"), ("strength_schedule", "DIFFORUM_SCHEDULE"), - ("depth", "IMAGE"), ("cfg_schedule", "DIFFORUM_SCHEDULE"), - ("positive_schedule", "DIFFORUM_PROMPT"), - ("control_net", "CONTROL_NET"), ("control_image", "IMAGE")], - outputs=[("frames", "IMAGE")], - widgets=[24, 6.5, "dpmpp_2m", "karras", 0.85, "lab", - 1.0, 100.0, False, 1.0, 0.6]) - rife = w.node("RIFE VFI", [1360, 360], inputs=[("frames", "IMAGE")], - outputs=[("IMAGE", "IMAGE")], - widgets=["rife47.pth", 10, 2, True, True, 1.0]) - vhs = w.node("VHS_VideoCombine", [1700, 360], inputs=[("images", "IMAGE")], - outputs=[], is_output=True, - widgets=[16, 0, "Difforum", "video/h264-mp4", False, True]) - save = w.node("SaveImage", [1360, 620], inputs=[("images", "IMAGE")], - outputs=[], widgets=["Difforum"], is_output=True) - # frame 0 txt2img - w.link(ckpt, 0, ks, 0, "MODEL") - w.link(ckpt, 1, pos, 0, "CLIP") - w.link(ckpt, 1, neg, 0, "CLIP") - w.link(pos, 0, ks, 1, "CONDITIONING") - w.link(neg, 0, ks, 2, "CONDITIONING") - w.link(lat, 0, ks, 3, "LATENT") - w.link(ks, 0, dec, 0, "LATENT") - w.link(ckpt, 2, dec, 1, "VAE") - # control plane - w.link(setup, 0, cam, 0, "DIFFORUM_PARAMS") - w.link(setup, 0, strength, 0, "DIFFORUM_PARAMS") - w.link(setup, 0, prompt, 0, "DIFFORUM_PARAMS") - w.link(ckpt, 1, prompt, 1, "CLIP") - # feedback sampler (+ controlnet + prompt travel) - w.link(ckpt, 0, fb, 0, "MODEL") - w.link(pos, 0, fb, 1, "CONDITIONING") - w.link(neg, 0, fb, 2, "CONDITIONING") - w.link(ckpt, 2, fb, 3, "VAE") - w.link(setup, 0, fb, 4, "DIFFORUM_PARAMS") - w.link(cam, 0, fb, 5, "DIFFORUM_CAMERA") - w.link(dec, 0, fb, 6, "IMAGE") - w.link(strength, 0, fb, 7, "DIFFORUM_SCHEDULE") - w.link(prompt, 0, fb, 10, "DIFFORUM_PROMPT") - w.link(cn, 0, fb, 11, "CONTROL_NET") - # output: frames -> interpolate -> video, and also save the PNG sequence - w.link(fb, 0, rife, 0, "IMAGE") - w.link(rife, 0, vhs, 0, "IMAGE") - w.link(fb, 0, save, 0, "IMAGE") - return w.dump() - - -def build_intuitive(): - """Intuitive controls: Camera Move (preset) + Prompt Scenes -> feedback video.""" - w = WF() - ckpt = w.node("CheckpointLoaderSimple", [40, 40], inputs=[], - outputs=[("MODEL", "MODEL"), ("CLIP", "CLIP"), ("VAE", "VAE")], - widgets=["dreamshaper_8.safetensors"]) - pos = w.node("CLIPTextEncode", [340, 20], inputs=[("clip", "CLIP")], - outputs=[("CONDITIONING", "CONDITIONING")], - widgets=["a serene misty forest, soft volumetric light"]) - neg = w.node("CLIPTextEncode", [340, 200], inputs=[("clip", "CLIP")], - outputs=[("CONDITIONING", "CONDITIONING")], - widgets=["blurry, low quality, watermark, text"]) - lat = w.node("EmptyLatentImage", [340, 380], inputs=[], - outputs=[("LATENT", "LATENT")], widgets=[768, 768, 1]) - ks = w.node("KSampler", [660, 120], - inputs=[("model", "MODEL"), ("positive", "CONDITIONING"), - ("negative", "CONDITIONING"), ("latent_image", "LATENT")], - outputs=[("LATENT", "LATENT")], - widgets=[0, "fixed", 26, 6.5, "dpmpp_2m", "karras", 1.0]) - dec = w.node("VAEDecode", [980, 120], inputs=[("samples", "LATENT"), ("vae", "VAE")], - outputs=[("IMAGE", "IMAGE")], widgets=[]) - setup = w.node("DifforumAnimSetup", [40, 460], inputs=[], - outputs=[("params", "DIFFORUM_PARAMS")], widgets=[768, 768, 24, 48, 0, "fixed"]) - move = w.node("DifforumCameraShots", [340, 500], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "DIFFORUM_AUDIO")], - outputs=[("camera", "DIFFORUM_CAMERA"), ("info", "STRING")], - widgets=["0: dolly_in 1.0 1.0\n24: orbit_right 1.0 0.8\n36: spiral 1.0 1.2", "2d", 40.0]) - prev_cam = w.node("DifforumCameraPreview", [660, 540], - inputs=[("camera", "DIFFORUM_CAMERA")], - outputs=[("path_image", "IMAGE")], widgets=[512]) - prev_img = w.node("PreviewImage", [660, 700], inputs=[("images", "IMAGE")], - outputs=[], widgets=[], is_output=True) - strength = w.node("DifforumSchedule", [340, 720], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "DIFFORUM_AUDIO")], - outputs=[("schedule", "DIFFORUM_SCHEDULE"), ("values", "FLOAT")], - widgets=["0:(0.5)", "linear"]) - scenes = w.node("DifforumPromptScenes", [340, 920], - inputs=[("params", "DIFFORUM_PARAMS"), ("clip", "CLIP")], - outputs=[("positive_schedule", "DIFFORUM_PROMPT"), - ("first_frame_cond", "CONDITIONING"), ("info", "STRING")], - widgets=["a serene misty forest, soft light", - "a glowing crystal cave, bioluminescent", - "a vast starry galaxy, swirling nebula", "", "ease_in_out"]) - fb = w.node("DifforumFeedbackSampler", [1000, 460], - inputs=[("model", "MODEL"), ("positive", "CONDITIONING"), - ("negative", "CONDITIONING"), ("vae", "VAE"), - ("params", "DIFFORUM_PARAMS"), ("camera", "DIFFORUM_CAMERA"), - ("init_image", "IMAGE"), ("strength_schedule", "DIFFORUM_SCHEDULE"), - ("depth", "IMAGE"), ("cfg_schedule", "DIFFORUM_SCHEDULE"), - ("positive_schedule", "DIFFORUM_PROMPT")], - outputs=[("frames", "IMAGE")], - widgets=[24, 6.5, "dpmpp_2m", "karras", 0.85, "lab", 1.0, 100.0, False, 1.0, - 0.6, "none", 6, "reflection", 0.25, 0.02, 1, 0, 0]) - save = w.node("SaveImage", [1700, 460], inputs=[("images", "IMAGE")], - outputs=[], widgets=["Difforum"], is_output=True) - w.link(ckpt, 0, ks, 0, "MODEL") - w.link(ckpt, 1, pos, 0, "CLIP") - w.link(ckpt, 1, neg, 0, "CLIP") - w.link(pos, 0, ks, 1, "CONDITIONING") - w.link(neg, 0, ks, 2, "CONDITIONING") - w.link(lat, 0, ks, 3, "LATENT") - w.link(ks, 0, dec, 0, "LATENT") - w.link(ckpt, 2, dec, 1, "VAE") - w.link(setup, 0, move, 0, "DIFFORUM_PARAMS") - w.link(move, 0, prev_cam, 0, "DIFFORUM_CAMERA") - w.link(prev_cam, 0, prev_img, 0, "IMAGE") - w.link(setup, 0, strength, 0, "DIFFORUM_PARAMS") - w.link(setup, 0, scenes, 0, "DIFFORUM_PARAMS") - w.link(ckpt, 1, scenes, 1, "CLIP") - w.link(ckpt, 0, fb, 0, "MODEL") - w.link(pos, 0, fb, 1, "CONDITIONING") - w.link(neg, 0, fb, 2, "CONDITIONING") - w.link(ckpt, 2, fb, 3, "VAE") - w.link(setup, 0, fb, 4, "DIFFORUM_PARAMS") - w.link(move, 0, fb, 5, "DIFFORUM_CAMERA") - w.link(dec, 0, fb, 6, "IMAGE") - w.link(strength, 0, fb, 7, "DIFFORUM_SCHEDULE") - w.link(scenes, 0, fb, 10, "DIFFORUM_PROMPT") - stab = w.node("DifforumFlowStabilize", [1340, 460], - inputs=[("frames", "IMAGE")], outputs=[("frames", "IMAGE")], - widgets=[0.5, 0.5, 0.15]) - w.link(fb, 0, stab, 0, "IMAGE") - pp = w.node("DifforumLoopBlend", [1520, 660], inputs=[("frames", "IMAGE")], - outputs=[("frames", "IMAGE")], widgets=[12, True]) - w.link(stab, 0, pp, 0, "IMAGE") - w.link(pp, 0, save, 0, "IMAGE") - w.group("1 - FIRST FRAME (txt2img)", [10, -60, 1270, 480], "#33415a") - w.group("2 - DIRECTOR (shot list + path preview + scenes)", [10, 440, 940, 900], "#33503f") - w.group("3 - RENDER (detail guard inside)", [960, 380, 350, 460], "#5a3a33") - w.group("4 - QUALITY: anti-flicker (Ctrl+B to bypass)", [1310, 380, 350, 300], "#4a3d5a") - return w.dump() - - -def build_turbo_live(): - """SD-Turbo / LCM tuned for speed: low steps, cfg 1.0, no ControlNet, 512px. - - Use an SD-Turbo checkpoint (sd_turbo) OR SD1.5 + LCM-LoRA. NOTE: don't mix an - SD1.5 ControlNet with an SD2.1-based sd_turbo (dim mismatch) - left out here. - """ - w = WF() - ckpt = w.node("CheckpointLoaderSimple", [40, 40], inputs=[], - outputs=[("MODEL", "MODEL"), ("CLIP", "CLIP"), ("VAE", "VAE")], - widgets=["sd_turbo.safetensors"]) - pos = w.node("CLIPTextEncode", [340, 20], inputs=[("clip", "CLIP")], - outputs=[("CONDITIONING", "CONDITIONING")], - widgets=["psychedelic mandala, vivid colors, intricate"]) - neg = w.node("CLIPTextEncode", [340, 200], inputs=[("clip", "CLIP")], - outputs=[("CONDITIONING", "CONDITIONING")], widgets=["blurry, low quality"]) - lat = w.node("EmptyLatentImage", [340, 380], inputs=[], - outputs=[("LATENT", "LATENT")], widgets=[512, 512, 1]) - ks = w.node("KSampler", [660, 120], - inputs=[("model", "MODEL"), ("positive", "CONDITIONING"), - ("negative", "CONDITIONING"), ("latent_image", "LATENT")], - outputs=[("LATENT", "LATENT")], - widgets=[0, "fixed", 4, 1.0, "euler", "sgm_uniform", 1.0]) - dec = w.node("VAEDecode", [980, 120], inputs=[("samples", "LATENT"), ("vae", "VAE")], - outputs=[("IMAGE", "IMAGE")], widgets=[]) - setup = w.node("DifforumAnimSetup", [40, 460], inputs=[], - outputs=[("params", "DIFFORUM_PARAMS")], widgets=[512, 512, 24, 64, 0, "fixed"]) - move = w.node("DifforumCameraMove", [340, 500], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "DIFFORUM_AUDIO")], - outputs=[("camera", "DIFFORUM_CAMERA"), ("info", "STRING")], - widgets=["spiral", 1.0, 1.0, "2d", 40.0]) - strength = w.node("DifforumSchedule", [340, 720], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "DIFFORUM_AUDIO")], - outputs=[("schedule", "DIFFORUM_SCHEDULE"), ("values", "FLOAT")], - widgets=["0:(0.5)", "linear"]) - scenes = w.node("DifforumPromptScenes", [340, 920], - inputs=[("params", "DIFFORUM_PARAMS"), ("clip", "CLIP")], - outputs=[("positive_schedule", "DIFFORUM_PROMPT"), - ("first_frame_cond", "CONDITIONING"), ("info", "STRING")], - widgets=["psychedelic mandala, vivid colors", - "cosmic fractal temple, golden light", - "deep space nebula, glowing stars", "", "ease_in_out"]) - fb = w.node("DifforumFeedbackSampler", [1000, 460], - inputs=[("model", "MODEL"), ("positive", "CONDITIONING"), - ("negative", "CONDITIONING"), ("vae", "VAE"), - ("params", "DIFFORUM_PARAMS"), ("camera", "DIFFORUM_CAMERA"), - ("init_image", "IMAGE"), ("strength_schedule", "DIFFORUM_SCHEDULE"), - ("depth", "IMAGE"), ("cfg_schedule", "DIFFORUM_SCHEDULE"), - ("positive_schedule", "DIFFORUM_PROMPT")], - outputs=[("frames", "IMAGE")], - # steps=4, cfg=1.0 -> Turbo/LCM settings - widgets=[4, 1.0, "euler", "sgm_uniform", 0.7, "lab", 1.0, 100.0, False, 1.0, - 0.6, "none", 6, "reflection", 0.15, 0.02, 2, 0, 0]) - save = w.node("SaveImage", [1360, 460], inputs=[("images", "IMAGE")], - outputs=[], widgets=["Difforum_turbo"], is_output=True) - w.link(ckpt, 0, ks, 0, "MODEL") - w.link(ckpt, 1, pos, 0, "CLIP") - w.link(ckpt, 1, neg, 0, "CLIP") - w.link(pos, 0, ks, 1, "CONDITIONING") - w.link(neg, 0, ks, 2, "CONDITIONING") - w.link(lat, 0, ks, 3, "LATENT") - w.link(ks, 0, dec, 0, "LATENT") - w.link(ckpt, 2, dec, 1, "VAE") - w.link(setup, 0, move, 0, "DIFFORUM_PARAMS") - w.link(setup, 0, strength, 0, "DIFFORUM_PARAMS") - w.link(setup, 0, scenes, 0, "DIFFORUM_PARAMS") - w.link(ckpt, 1, scenes, 1, "CLIP") - w.link(ckpt, 0, fb, 0, "MODEL") - w.link(pos, 0, fb, 1, "CONDITIONING") - w.link(neg, 0, fb, 2, "CONDITIONING") - w.link(ckpt, 2, fb, 3, "VAE") - w.link(setup, 0, fb, 4, "DIFFORUM_PARAMS") - w.link(move, 0, fb, 5, "DIFFORUM_CAMERA") - w.link(dec, 0, fb, 6, "IMAGE") - w.link(strength, 0, fb, 7, "DIFFORUM_SCHEDULE") - w.link(scenes, 0, fb, 10, "DIFFORUM_PROMPT") - w.link(fb, 0, save, 0, "IMAGE") - return w.dump() - - -def build_animatediff(): - """AnimateDiff look + Difforum control: native motion via AnimateDiff-Evolved, - prompt travel batched by Difforum. Needs comfyui-animatediff-evolved installed. - batch_size MUST equal max_frames so prompt-batch lines up with the latents. - """ - N = 16 - w = WF() - ckpt = w.node("CheckpointLoaderSimple", [40, 40], inputs=[], - outputs=[("MODEL", "MODEL"), ("CLIP", "CLIP"), ("VAE", "VAE")], - widgets=["dreamshaper_8.safetensors"]) - neg = w.node("CLIPTextEncode", [360, 220], inputs=[("clip", "CLIP")], - outputs=[("CONDITIONING", "CONDITIONING")], - widgets=["blurry, low quality, watermark, text"]) - setup = w.node("DifforumAnimSetup", [40, 360], inputs=[], - outputs=[("params", "DIFFORUM_PARAMS")], widgets=[512, 512, 12, N, 0, "fixed"]) - scenes = w.node("DifforumPromptScenes", [360, 380], - inputs=[("params", "DIFFORUM_PARAMS"), ("clip", "CLIP")], - outputs=[("positive_schedule", "DIFFORUM_PROMPT"), - ("first_frame_cond", "CONDITIONING"), ("info", "STRING")], - widgets=["a serene koi pond, lily pads", - "an autumn forest path, falling leaves", - "a snowy mountain peak at dawn", "", "ease_in_out"]) - pbatch = w.node("DifforumPromptBatch", [720, 380], - inputs=[("positive_schedule", "DIFFORUM_PROMPT")], - outputs=[("conditioning", "CONDITIONING")], widgets=[]) - ctx = w.node("ADE_AnimateDiffUniformContextOptions", [40, 560], inputs=[], - outputs=[("CONTEXT_OPTIONS", "CONTEXT_OPTIONS")], - widgets=[16, 1, 4, False, "flat"]) - ade = w.node("ADE_AnimateDiffLoaderGen1", [360, 560], - inputs=[("model", "MODEL"), ("context_options", "CONTEXT_OPTIONS"), - ("motion_lora", "MOTION_LORA")], - outputs=[("MODEL", "MODEL")], - widgets=["v3_sd15_mm.ckpt", "autoselect"]) - lat = w.node("EmptyLatentImage", [360, 760], inputs=[], - outputs=[("LATENT", "LATENT")], widgets=[512, 512, N]) - ks = w.node("KSampler", [760, 560], - inputs=[("model", "MODEL"), ("positive", "CONDITIONING"), - ("negative", "CONDITIONING"), ("latent_image", "LATENT")], - outputs=[("LATENT", "LATENT")], - widgets=[0, "fixed", 26, 6.5, "dpmpp_2m", "karras", 1.0]) - dec = w.node("VAEDecode", [1120, 560], - inputs=[("samples", "LATENT"), ("vae", "VAE")], - outputs=[("IMAGE", "IMAGE")], widgets=[]) - vhs = w.node("VHS_VideoCombine", [1400, 560], inputs=[("images", "IMAGE")], - outputs=[], is_output=True, - widgets=[12, 0, "Difforum_ADE", "video/h264-mp4", False, True]) - w.link(setup, 0, scenes, 0, "DIFFORUM_PARAMS") - w.link(ckpt, 1, scenes, 1, "CLIP") - w.link(ckpt, 1, neg, 0, "CLIP") - w.link(scenes, 0, pbatch, 0, "DIFFORUM_PROMPT") - w.link(ckpt, 0, ade, 0, "MODEL") - w.link(ctx, 0, ade, 1, "CONTEXT_OPTIONS") - w.link(ade, 0, ks, 0, "MODEL") - w.link(pbatch, 0, ks, 1, "CONDITIONING") - w.link(neg, 0, ks, 2, "CONDITIONING") - w.link(lat, 0, ks, 3, "LATENT") - w.link(ks, 0, dec, 0, "LATENT") - w.link(ckpt, 2, dec, 1, "VAE") - w.link(dec, 0, vhs, 0, "IMAGE") - return w.dump() - - -def build_ipadapter_coherent(): - """16:9 Classic+ with IPAdapter style-lock for strong frame-to-frame coherence. - Needs comfyui_ipadapter_plus + an SD1.5 IPAdapter model. Style image stays - consistent across the whole animation -> much less drift.""" - W, H = 768, 432 # 16:9 - w = WF() - ckpt = w.node("CheckpointLoaderSimple", [40, 40], inputs=[], - outputs=[("MODEL", "MODEL"), ("CLIP", "CLIP"), ("VAE", "VAE")], - widgets=["dreamshaper_8.safetensors"]) - style = w.node("LoadImage", [40, 260], inputs=[], - outputs=[("IMAGE", "IMAGE"), ("MASK", "MASK")], widgets=["style.png"]) - ipload = w.node("IPAdapterUnifiedLoader", [360, 40], - inputs=[("model", "MODEL"), ("ipadapter", "IPADAPTER")], - outputs=[("model", "MODEL"), ("ipadapter", "IPADAPTER")], - widgets=["PLUS (high strength)"]) - ipapply = w.node("IPAdapter", [660, 40], - inputs=[("model", "MODEL"), ("ipadapter", "IPADAPTER"), ("image", "IMAGE")], - outputs=[("MODEL", "MODEL")], - widgets=[0.7, "standard", 0.0, 1.0]) - pos = w.node("CLIPTextEncode", [360, 260], inputs=[("clip", "CLIP")], - outputs=[("CONDITIONING", "CONDITIONING")], - widgets=["a dreamlike landscape, painterly, vivid"]) - neg = w.node("CLIPTextEncode", [360, 420], inputs=[("clip", "CLIP")], - outputs=[("CONDITIONING", "CONDITIONING")], widgets=["blurry, low quality"]) - lat = w.node("EmptyLatentImage", [360, 580], inputs=[], - outputs=[("LATENT", "LATENT")], widgets=[W, H, 1]) - ks = w.node("KSampler", [960, 40], - inputs=[("model", "MODEL"), ("positive", "CONDITIONING"), - ("negative", "CONDITIONING"), ("latent_image", "LATENT")], - outputs=[("LATENT", "LATENT")], widgets=[0, "fixed", 26, 6.5, "dpmpp_2m", "karras", 1.0]) - dec = w.node("VAEDecode", [1260, 40], inputs=[("samples", "LATENT"), ("vae", "VAE")], - outputs=[("IMAGE", "IMAGE")], widgets=[]) - setup = w.node("DifforumAnimSetup", [40, 600], inputs=[], - outputs=[("params", "DIFFORUM_PARAMS")], widgets=[W, H, 24, 48, 0, "fixed"]) - cam = w.node("DifforumCameraMove", [360, 760], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "DIFFORUM_AUDIO")], - outputs=[("camera", "DIFFORUM_CAMERA"), ("info", "STRING")], - widgets=["dolly_in", 1.0, 1.0, "2d", 40.0]) - strength = w.node("DifforumSchedule", [360, 980], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "DIFFORUM_AUDIO")], - outputs=[("schedule", "DIFFORUM_SCHEDULE"), ("values", "FLOAT")], - widgets=["0:(0.45)", "linear"]) - fb = w.node("DifforumFeedbackSampler", [1260, 300], - inputs=[("model", "MODEL"), ("positive", "CONDITIONING"), - ("negative", "CONDITIONING"), ("vae", "VAE"), - ("params", "DIFFORUM_PARAMS"), ("camera", "DIFFORUM_CAMERA"), - ("init_image", "IMAGE"), ("strength_schedule", "DIFFORUM_SCHEDULE"), - ("depth", "IMAGE"), ("cfg_schedule", "DIFFORUM_SCHEDULE"), - ("positive_schedule", "DIFFORUM_PROMPT"), - ("control_net", "CONTROL_NET"), ("control_image", "IMAGE")], - outputs=[("frames", "IMAGE")], - widgets=[24, 6.5, "dpmpp_2m", "karras", 0.85, "lab", 1.0, 100.0, False, 1.0, 0.6]) - save = w.node("SaveImage", [1620, 300], inputs=[("images", "IMAGE")], - outputs=[], widgets=["Difforum_ipa"], is_output=True) - w.link(ckpt, 0, ipload, 0, "MODEL") - w.link(ipload, 0, ipapply, 0, "MODEL") - w.link(ipload, 1, ipapply, 1, "IPADAPTER") - w.link(style, 0, ipapply, 2, "IMAGE") - w.link(ckpt, 1, pos, 0, "CLIP") - w.link(ckpt, 1, neg, 0, "CLIP") - w.link(ipapply, 0, ks, 0, "MODEL") - w.link(pos, 0, ks, 1, "CONDITIONING") - w.link(neg, 0, ks, 2, "CONDITIONING") - w.link(lat, 0, ks, 3, "LATENT") - w.link(ks, 0, dec, 0, "LATENT") - w.link(ckpt, 2, dec, 1, "VAE") - w.link(setup, 0, cam, 0, "DIFFORUM_PARAMS") - w.link(setup, 0, strength, 0, "DIFFORUM_PARAMS") - w.link(ipapply, 0, fb, 0, "MODEL") - w.link(pos, 0, fb, 1, "CONDITIONING") - w.link(neg, 0, fb, 2, "CONDITIONING") - w.link(ckpt, 2, fb, 3, "VAE") - w.link(setup, 0, fb, 4, "DIFFORUM_PARAMS") - w.link(cam, 0, fb, 5, "DIFFORUM_CAMERA") - w.link(dec, 0, fb, 6, "IMAGE") - w.link(strength, 0, fb, 7, "DIFFORUM_SCHEDULE") - w.link(fb, 0, save, 0, "IMAGE") - return w.dump() - - -def build_audio_reactive_video(): - """16:9 audio-reactive: bass pumps the zoom, beats pulse the denoise strength.""" - W, H = 768, 432 - w = WF() - ckpt = w.node("CheckpointLoaderSimple", [40, 40], inputs=[], - outputs=[("MODEL", "MODEL"), ("CLIP", "CLIP"), ("VAE", "VAE")], - widgets=["dreamshaper_8.safetensors"]) - pos = w.node("CLIPTextEncode", [340, 20], inputs=[("clip", "CLIP")], - outputs=[("CONDITIONING", "CONDITIONING")], - widgets=["neon synthwave city, glowing grid, vivid"]) - neg = w.node("CLIPTextEncode", [340, 200], inputs=[("clip", "CLIP")], - outputs=[("CONDITIONING", "CONDITIONING")], widgets=["blurry, low quality"]) - lat = w.node("EmptyLatentImage", [340, 360], inputs=[], - outputs=[("LATENT", "LATENT")], widgets=[W, H, 1]) - ks = w.node("KSampler", [660, 120], - inputs=[("model", "MODEL"), ("positive", "CONDITIONING"), - ("negative", "CONDITIONING"), ("latent_image", "LATENT")], - outputs=[("LATENT", "LATENT")], widgets=[0, "fixed", 26, 6.5, "dpmpp_2m", "karras", 1.0]) - dec = w.node("VAEDecode", [980, 120], inputs=[("samples", "LATENT"), ("vae", "VAE")], - outputs=[("IMAGE", "IMAGE")], widgets=[]) - setup = w.node("DifforumAnimSetup", [40, 420], inputs=[], - outputs=[("params", "DIFFORUM_PARAMS")], widgets=[W, H, 24, 96, 0, "fixed"]) - audio = w.node("LoadAudio", [40, 640], inputs=[], - outputs=[("AUDIO", "AUDIO")], widgets=["your_track.mp3"]) - analyzer = w.node("DifforumAudioAnalyzer", [340, 640], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "AUDIO")], - outputs=[("audio_curves", "DIFFORUM_AUDIO"), ("amp", "FLOAT"), ("low", "FLOAT"), - ("mid", "FLOAT"), ("high", "FLOAT"), ("onset", "FLOAT"), ("beat", "FLOAT")], - widgets=[0.25, True, 1.5]) - # beat -> denoise strength pulse - asched = w.node("DifforumAudioSchedule", [700, 640], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "DIFFORUM_AUDIO")], - outputs=[("schedule", "DIFFORUM_SCHEDULE"), ("values", "FLOAT")], - widgets=["beat", "add", 0.4, 0.35, 0.2]) - # bass -> zoom (camera reads audio var `low`) - cam = w.node("DifforumCamera", [340, 860], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "DIFFORUM_AUDIO")], - outputs=[("camera", "DIFFORUM_CAMERA"), ("info", "STRING")], - widgets=["2d", 40.0, "0:(0)", "0:(0)", "0:(0)", "0:(0)", "0:(0.3)", "0:(1.0 + 0.06*low)"]) - fb = w.node("DifforumFeedbackSampler", [1280, 360], - inputs=[("model", "MODEL"), ("positive", "CONDITIONING"), - ("negative", "CONDITIONING"), ("vae", "VAE"), - ("params", "DIFFORUM_PARAMS"), ("camera", "DIFFORUM_CAMERA"), - ("init_image", "IMAGE"), ("strength_schedule", "DIFFORUM_SCHEDULE")], - outputs=[("frames", "IMAGE")], - widgets=[24, 6.5, "dpmpp_2m", "karras", 0.8, "lab", 1.0, 100.0, False, 1.0, 0.6]) - vhs = w.node("VHS_VideoCombine", [1640, 360], inputs=[("images", "IMAGE")], - outputs=[], is_output=True, - widgets=[24, 0, "Difforum_audio", "video/h264-mp4", False, True]) - w.link(ckpt, 0, ks, 0, "MODEL") - w.link(ckpt, 1, pos, 0, "CLIP") - w.link(ckpt, 1, neg, 0, "CLIP") - w.link(pos, 0, ks, 1, "CONDITIONING") - w.link(neg, 0, ks, 2, "CONDITIONING") - w.link(lat, 0, ks, 3, "LATENT") - w.link(ks, 0, dec, 0, "LATENT") - w.link(ckpt, 2, dec, 1, "VAE") - w.link(setup, 0, analyzer, 0, "DIFFORUM_PARAMS") - w.link(audio, 0, analyzer, 1, "AUDIO") - w.link(setup, 0, asched, 0, "DIFFORUM_PARAMS") - w.link(analyzer, 0, asched, 1, "DIFFORUM_AUDIO") - w.link(setup, 0, cam, 0, "DIFFORUM_PARAMS") - w.link(analyzer, 0, cam, 1, "DIFFORUM_AUDIO") - w.link(ckpt, 0, fb, 0, "MODEL") - w.link(pos, 0, fb, 1, "CONDITIONING") - w.link(neg, 0, fb, 2, "CONDITIONING") - w.link(ckpt, 2, fb, 3, "VAE") - w.link(setup, 0, fb, 4, "DIFFORUM_PARAMS") - w.link(cam, 0, fb, 5, "DIFFORUM_CAMERA") - w.link(dec, 0, fb, 6, "IMAGE") - w.link(asched, 0, fb, 7, "DIFFORUM_SCHEDULE") - stab = w.node("DifforumFlowStabilize", [1600, 360], - inputs=[("frames", "IMAGE")], outputs=[("frames", "IMAGE")], - widgets=[0.5, 0.5, 0.15]) - w.link(fb, 0, stab, 0, "IMAGE") - w.link(stab, 0, vhs, 0, "IMAGE") - return w.dump() - - -def build_qrcode_illusion(): - """16:9 hidden-pattern illusion: QR Code Monster ControlNet locks a fixed - grayscale pattern (spiral/logo/mask) while the feedback loop morphs the scene. - Needs an SD1.5 checkpoint + control_v1p_sd15_qrcode_monster + a pattern image.""" - W, H = 768, 432 - w = WF() - ckpt = w.node("CheckpointLoaderSimple", [40, 40], inputs=[], - outputs=[("MODEL", "MODEL"), ("CLIP", "CLIP"), ("VAE", "VAE")], - widgets=["dreamshaper_8.safetensors"]) - pattern = w.node("LoadImage", [40, 260], inputs=[], - outputs=[("IMAGE", "IMAGE"), ("MASK", "MASK")], widgets=["pattern.png"]) - cn = w.node("ControlNetLoader", [40, 480], inputs=[], - outputs=[("CONTROL_NET", "CONTROL_NET")], - widgets=["control_v1p_sd15_qrcode_monster.safetensors"]) - pos = w.node("CLIPTextEncode", [360, 20], inputs=[("clip", "CLIP")], - outputs=[("CONDITIONING", "CONDITIONING")], - widgets=["ornate fractal mandala, molten gold and emerald, intricate"]) - neg = w.node("CLIPTextEncode", [360, 200], inputs=[("clip", "CLIP")], - outputs=[("CONDITIONING", "CONDITIONING")], widgets=["blurry, low quality, flat"]) - lat = w.node("EmptyLatentImage", [360, 360], inputs=[], - outputs=[("LATENT", "LATENT")], widgets=[W, H, 1]) - ks = w.node("KSampler", [660, 120], - inputs=[("model", "MODEL"), ("positive", "CONDITIONING"), - ("negative", "CONDITIONING"), ("latent_image", "LATENT")], - outputs=[("LATENT", "LATENT")], widgets=[0, "fixed", 26, 6.5, "dpmpp_2m", "karras", 1.0]) - dec = w.node("VAEDecode", [980, 120], inputs=[("samples", "LATENT"), ("vae", "VAE")], - outputs=[("IMAGE", "IMAGE")], widgets=[]) - setup = w.node("DifforumAnimSetup", [40, 660], inputs=[], - outputs=[("params", "DIFFORUM_PARAMS")], widgets=[W, H, 24, 48, 0, "fixed"]) - cam = w.node("DifforumCameraMove", [360, 560], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "DIFFORUM_AUDIO")], - outputs=[("camera", "DIFFORUM_CAMERA"), ("info", "STRING")], - widgets=["zoom_in", 0.5, 0.6, "2d", 40.0]) - strength = w.node("DifforumSchedule", [360, 780], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "DIFFORUM_AUDIO")], - outputs=[("schedule", "DIFFORUM_SCHEDULE"), ("values", "FLOAT")], - widgets=["0:(0.4)", "linear"]) # lower denoise -> pattern persists - fb = w.node("DifforumFeedbackSampler", [1280, 300], - inputs=[("model", "MODEL"), ("positive", "CONDITIONING"), - ("negative", "CONDITIONING"), ("vae", "VAE"), - ("params", "DIFFORUM_PARAMS"), ("camera", "DIFFORUM_CAMERA"), - ("init_image", "IMAGE"), ("strength_schedule", "DIFFORUM_SCHEDULE"), - ("depth", "IMAGE"), ("cfg_schedule", "DIFFORUM_SCHEDULE"), - ("positive_schedule", "DIFFORUM_PROMPT"), - ("control_net", "CONTROL_NET"), ("control_image", "IMAGE")], - outputs=[("frames", "IMAGE")], - # control_strength 1.2 -> strong illusion - widgets=[24, 6.5, "dpmpp_2m", "karras", 0.85, "lab", 1.0, 100.0, False, 1.0, 1.2]) - save = w.node("SaveImage", [1640, 300], inputs=[("images", "IMAGE")], - outputs=[], widgets=["Difforum_illusion"], is_output=True) - w.link(ckpt, 1, pos, 0, "CLIP") - w.link(ckpt, 1, neg, 0, "CLIP") - w.link(ckpt, 0, ks, 0, "MODEL") - w.link(pos, 0, ks, 1, "CONDITIONING") - w.link(neg, 0, ks, 2, "CONDITIONING") - w.link(lat, 0, ks, 3, "LATENT") - w.link(ks, 0, dec, 0, "LATENT") - w.link(ckpt, 2, dec, 1, "VAE") - w.link(setup, 0, cam, 0, "DIFFORUM_PARAMS") - w.link(setup, 0, strength, 0, "DIFFORUM_PARAMS") - w.link(ckpt, 0, fb, 0, "MODEL") - w.link(pos, 0, fb, 1, "CONDITIONING") - w.link(neg, 0, fb, 2, "CONDITIONING") - w.link(ckpt, 2, fb, 3, "VAE") - w.link(setup, 0, fb, 4, "DIFFORUM_PARAMS") - w.link(cam, 0, fb, 5, "DIFFORUM_CAMERA") - w.link(dec, 0, fb, 6, "IMAGE") - w.link(strength, 0, fb, 7, "DIFFORUM_SCHEDULE") - w.link(cn, 0, fb, 11, "CONTROL_NET") - w.link(pattern, 0, fb, 12, "IMAGE") - w.link(fb, 0, save, 0, "IMAGE") - return w.dump() - - -def build_realtime(): - """Native realtime: Difforum Live Sampler runs an internal resident-model loop - (warp + kaleidoscope + 1-step Turbo/LCM re-diffuse) and streams a LIVE preview - into the node. Queue once and watch it generate. No external runtime.""" - w = WF() - ckpt = w.node("CheckpointLoaderSimple", [40, 40], inputs=[], - outputs=[("MODEL", "MODEL"), ("CLIP", "CLIP"), ("VAE", "VAE")], - widgets=["sd_turbo.safetensors"]) - pos = w.node("CLIPTextEncode", [360, 20], inputs=[("clip", "CLIP")], - outputs=[("CONDITIONING", "CONDITIONING")], - widgets=["psychedelic stained glass mandala, vivid colors, intricate, glowing"]) - neg = w.node("CLIPTextEncode", [360, 160], inputs=[("clip", "CLIP")], - outputs=[("CONDITIONING", "CONDITIONING")], widgets=["blurry, low quality"]) - lat = w.node("EmptyLatentImage", [360, 300], inputs=[], - outputs=[("LATENT", "LATENT")], widgets=[512, 512, 1]) - ks = w.node("KSampler", [680, 60], - inputs=[("model", "MODEL"), ("positive", "CONDITIONING"), - ("negative", "CONDITIONING"), ("latent_image", "LATENT")], - outputs=[("LATENT", "LATENT")], - widgets=[0, "fixed", 4, 1.0, "euler", "sgm_uniform", 1.0]) - dec = w.node("VAEDecode", [1000, 60], inputs=[("samples", "LATENT"), ("vae", "VAE")], - outputs=[("IMAGE", "IMAGE")], widgets=[]) - setup = w.node("DifforumAnimSetup", [40, 360], inputs=[], - outputs=[("params", "DIFFORUM_PARAMS")], widgets=[512, 512, 24, 120, 0, "fixed"]) - move = w.node("DifforumCameraMove", [360, 460], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "DIFFORUM_AUDIO")], - outputs=[("camera", "DIFFORUM_CAMERA"), ("info", "STRING")], - widgets=["spiral", 1.0, 1.0, "2d", 40.0]) - live = w.node("DifforumLiveSampler", [1000, 320], - inputs=[("model", "MODEL"), ("positive", "CONDITIONING"), - ("negative", "CONDITIONING"), ("vae", "VAE"), - ("params", "DIFFORUM_PARAMS"), ("camera", "DIFFORUM_CAMERA"), - ("init_image", "IMAGE")], - outputs=[("frames", "IMAGE")], - # duration, strength, steps, cfg, sampler, scheduler, color_coh, - # color_mode, symmetry, segments, target_fps - widgets=[240, 0.5, 1, 1.0, "lcm", "sgm_uniform", 0.5, "lab", - "kaleidoscope", 6, 0.0]) - save = w.node("SaveImage", [1360, 320], inputs=[("images", "IMAGE")], - outputs=[], widgets=["Difforum_live"], is_output=True) - w.link(ckpt, 0, ks, 0, "MODEL") - w.link(ckpt, 1, pos, 0, "CLIP") - w.link(ckpt, 1, neg, 0, "CLIP") - w.link(pos, 0, ks, 1, "CONDITIONING") - w.link(neg, 0, ks, 2, "CONDITIONING") - w.link(lat, 0, ks, 3, "LATENT") - w.link(ks, 0, dec, 0, "LATENT") - w.link(ckpt, 2, dec, 1, "VAE") - w.link(setup, 0, move, 0, "DIFFORUM_PARAMS") - w.link(ckpt, 0, live, 0, "MODEL") - w.link(pos, 0, live, 1, "CONDITIONING") - w.link(neg, 0, live, 2, "CONDITIONING") - w.link(ckpt, 2, live, 3, "VAE") - w.link(setup, 0, live, 4, "DIFFORUM_PARAMS") - w.link(move, 0, live, 5, "DIFFORUM_CAMERA") - w.link(dec, 0, live, 6, "IMAGE") - w.link(live, 0, save, 0, "IMAGE") - return w.dump() - - -def build_mesmerize(): - """Living kaleidoscope: feedback loop with in-loop symmetry (the diffusion - heals the seams each frame) + Echo Trails for smooth hypnotic motion.""" - w = WF() - ckpt = w.node( - "CheckpointLoaderSimple", [40, 40], inputs=[], - outputs=[("MODEL", "MODEL"), ("CLIP", "CLIP"), ("VAE", "VAE")], - widgets=["dreamshaper_8.safetensors"], - ) - pos = w.node( - "CLIPTextEncode", [340, 20], inputs=[("clip", "CLIP")], - outputs=[("CONDITIONING", "CONDITIONING")], - widgets=["ornate symmetric mandala, stained glass, iridescent, " - "intricate fractal detail, glowing"], - ) - neg = w.node( - "CLIPTextEncode", [340, 200], inputs=[("clip", "CLIP")], - outputs=[("CONDITIONING", "CONDITIONING")], - widgets=["blurry, low quality, watermark, text, asymmetric"], - ) - lat = w.node( - "EmptyLatentImage", [340, 380], inputs=[], - outputs=[("LATENT", "LATENT")], widgets=[768, 768, 1], - ) - ks = w.node( - "KSampler", [660, 120], - inputs=[("model", "MODEL"), ("positive", "CONDITIONING"), - ("negative", "CONDITIONING"), ("latent_image", "LATENT")], - outputs=[("LATENT", "LATENT")], - widgets=[0, "fixed", 26, 6.5, "dpmpp_2m", "karras", 1.0], - ) - dec = w.node( - "VAEDecode", [980, 120], - inputs=[("samples", "LATENT"), ("vae", "VAE")], - outputs=[("IMAGE", "IMAGE")], widgets=[], - ) - setup = w.node( - "DifforumAnimSetup", [40, 520], inputs=[], - outputs=[("params", "DIFFORUM_PARAMS")], - widgets=[768, 768, 24, 72, 0, "fixed"], - ) - # slow rotation + gentle zoom feeds the kaleidoscope new material - cam = w.node( - "DifforumCamera", [340, 560], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "DIFFORUM_AUDIO")], - outputs=[("camera", "DIFFORUM_CAMERA"), ("info", "STRING")], - widgets=["2d", 40.0, "0:(0)", "0:(0)", "0:(1.0)", "0:(0.6)", - "0:(0)", "0:(0)", "0:(1.01)"], - ) - strength = w.node( - "DifforumSchedule", [340, 820], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "DIFFORUM_AUDIO")], - outputs=[("schedule", "DIFFORUM_SCHEDULE"), ("values", "FLOAT")], - widgets=["0:(0.4)", "linear"], - ) - fb = w.node( - "DifforumFeedbackSampler", [1000, 480], - inputs=[("model", "MODEL"), ("positive", "CONDITIONING"), - ("negative", "CONDITIONING"), ("vae", "VAE"), - ("params", "DIFFORUM_PARAMS"), ("camera", "DIFFORUM_CAMERA"), - ("init_image", "IMAGE"), ("strength_schedule", "DIFFORUM_SCHEDULE")], - outputs=[("frames", "IMAGE")], - # ...,control_strength, symmetry, symmetry_segments - widgets=[24, 6.5, "dpmpp_2m", "karras", 0.85, "lab", 1.0, 100.0, False, 1.0, - 0.6, "kaleidoscope", 6], - ) - echo = w.node( - "DifforumEchoTrails", [1360, 360], - inputs=[("frames", "IMAGE")], - outputs=[("frames", "IMAGE")], widgets=[0.6, 0.4], - ) - save = w.node( - "SaveImage", [1680, 360], inputs=[("images", "IMAGE")], - outputs=[], widgets=["Difforum_mesmerize"], is_output=True, - ) - # frame 0 txt2img - w.link(ckpt, 0, ks, 0, "MODEL") - w.link(ckpt, 1, pos, 0, "CLIP") - w.link(ckpt, 1, neg, 0, "CLIP") - w.link(pos, 0, ks, 1, "CONDITIONING") - w.link(neg, 0, ks, 2, "CONDITIONING") - w.link(lat, 0, ks, 3, "LATENT") - w.link(ks, 0, dec, 0, "LATENT") - w.link(ckpt, 2, dec, 1, "VAE") - # difforum control plane - w.link(setup, 0, cam, 0, "DIFFORUM_PARAMS") - w.link(setup, 0, strength, 0, "DIFFORUM_PARAMS") - # feedback sampler with in-loop kaleidoscope - w.link(ckpt, 0, fb, 0, "MODEL") - w.link(pos, 0, fb, 1, "CONDITIONING") - w.link(neg, 0, fb, 2, "CONDITIONING") - w.link(ckpt, 2, fb, 3, "VAE") - w.link(setup, 0, fb, 4, "DIFFORUM_PARAMS") - w.link(cam, 0, fb, 5, "DIFFORUM_CAMERA") - w.link(dec, 0, fb, 6, "IMAGE") - w.link(strength, 0, fb, 7, "DIFFORUM_SCHEDULE") - stab = w.node("DifforumFlowStabilize", [1330, 480], - inputs=[("frames", "IMAGE")], outputs=[("frames", "IMAGE")], - widgets=[0.5, 0.5, 0.15]) - w.link(fb, 0, stab, 0, "IMAGE") - w.link(stab, 0, echo, 0, "IMAGE") - pp = w.node("DifforumLoopBlend", [1860, 480], inputs=[("frames", "IMAGE")], - outputs=[("frames", "IMAGE")], widgets=[12, True]) - w.link(echo, 0, pp, 0, "IMAGE") - w.link(pp, 0, save, 0, "IMAGE") - w.group("1 - FIRST FRAME (txt2img)", [10, -60, 1270, 500], "#33415a") - w.group("2 - ANIMATION BRAIN", [10, 470, 900, 700], "#33503f") - w.group("3 - RENDER LOOP (kaleidoscope inside)", [960, 390, 350, 480], "#5a3a33") - w.group("4 - QUALITY: anti-flicker + trails (Ctrl+B to bypass)", [1310, 300, 700, 320], "#4a3d5a") - return w.dump() - - -def build_vj_footage(): - """Complete, self-contained VJ pipeline, no model and no external nodes: - Load Video -> VJ Look (grade + glow + chroma + grain) -> Echo Trails -> - Save Video (MP4). For beat reactivity, wire an Audio Schedule into the VJ - Look's intensity_schedule (see the Note).""" - w = WF() - vid = w.node("DifforumLoadVideo", [40, 40], inputs=[], - outputs=[("frames", "IMAGE"), ("frame_count", "INT"), ("fps", "FLOAT")], - widgets=["footage.mp4", 0, 1, 0]) - look = w.node("DifforumLook", [400, 40], inputs=[("image", "IMAGE")], - outputs=[("image", "IMAGE")], widgets=["neon", 1.2, 0]) - echo = w.node("DifforumEchoTrails", [720, 40], inputs=[("frames", "IMAGE")], - outputs=[("frames", "IMAGE")], widgets=[0.4, 0.3]) - save = w.node("DifforumSaveVideo", [1040, 40], - inputs=[("frames", "IMAGE"), ("fps_in", "FLOAT")], - outputs=[("path", "STRING")], widgets=["Difforum_vj", 24.0, 8], is_output=True) - w.link(vid, 0, look, 0, "IMAGE") - w.link(look, 0, echo, 0, "IMAGE") - w.link(echo, 0, save, 0, "IMAGE") - w.link(vid, 2, save, 1, "FLOAT") # source fps -> output fps - return w.dump() - - -def build_fast_sdxl(): - """Fast SDXL recipe: frame 0 at full quality on the base model, then the - loop runs a DMD2 4-step distill LoRA at cfg 1.2 with cadence 2 - about - 8-10x less diffusion cost with the same checkpoint look. The best-fit - template for Apple Silicon too (few steps is the Mac speed lever).""" - w = WF() - ckpt = w.node("CheckpointLoaderSimple", [40, 40], inputs=[], - outputs=[("MODEL", "MODEL"), ("CLIP", "CLIP"), ("VAE", "VAE")], - widgets=["juggernautXL_v9.safetensors"]) - lora = w.node("LoraLoader", [40, 240], - inputs=[("model", "MODEL"), ("clip", "CLIP")], - outputs=[("MODEL", "MODEL"), ("CLIP", "CLIP")], - widgets=["dmd2_sdxl_4step_lora.safetensors", 1.0, 1.0]) - neg = w.node("CLIPTextEncode", [340, 200], inputs=[("clip", "CLIP")], - outputs=[("CONDITIONING", "CONDITIONING")], - widgets=["blurry, low quality, watermark, text, washed out"]) - lat = w.node("EmptyLatentImage", [340, 360], inputs=[], - outputs=[("LATENT", "LATENT")], widgets=[768, 432, 1]) - scenes = w.node("DifforumPromptScenes", [340, 520], - inputs=[("params", "DIFFORUM_PARAMS"), ("clip", "CLIP")], - outputs=[("positive_schedule", "DIFFORUM_PROMPT"), - ("first_frame_cond", "CONDITIONING"), ("info", "STRING")], - widgets=["a luminous coral canyon, volumetric light, highly detailed", - "a crystalline ice cathedral, refracted glow", - "a golden desert storm at dusk, cinematic", "", "ease_in_out"]) - ks = w.node("KSampler", [660, 60], - inputs=[("model", "MODEL"), ("positive", "CONDITIONING"), - ("negative", "CONDITIONING"), ("latent_image", "LATENT")], - outputs=[("LATENT", "LATENT")], - widgets=[0, "fixed", 26, 6.5, "dpmpp_2m", "karras", 1.0]) - dec = w.node("VAEDecode", [980, 60], inputs=[("samples", "LATENT"), ("vae", "VAE")], - outputs=[("IMAGE", "IMAGE")], widgets=[]) - setup = w.node("DifforumAnimSetup", [40, 480], inputs=[], - outputs=[("params", "DIFFORUM_PARAMS")], widgets=[768, 432, 24, 96, 0, "fixed"]) - cam = w.node("DifforumCameraShots", [340, 860], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "DIFFORUM_AUDIO")], - outputs=[("camera", "DIFFORUM_CAMERA"), ("info", "STRING")], - widgets=["0: dolly_in 1.0 1.0\n48: spiral 1.0 1.2", "2d", 40.0]) - strength = w.node("DifforumSchedule", [340, 1060], - inputs=[("params", "DIFFORUM_PARAMS"), ("audio", "DIFFORUM_AUDIO")], - outputs=[("schedule", "DIFFORUM_SCHEDULE"), ("values", "FLOAT")], - widgets=["0:(0.45)", "linear"]) - fb = w.node("DifforumFeedbackSampler", [1000, 460], - inputs=[("model", "MODEL"), ("positive", "CONDITIONING"), - ("negative", "CONDITIONING"), ("vae", "VAE"), - ("params", "DIFFORUM_PARAMS"), ("camera", "DIFFORUM_CAMERA"), - ("init_image", "IMAGE"), ("strength_schedule", "DIFFORUM_SCHEDULE"), - ("depth", "IMAGE"), ("cfg_schedule", "DIFFORUM_SCHEDULE"), - ("positive_schedule", "DIFFORUM_PROMPT")], - outputs=[("frames", "IMAGE")], - widgets=[5, 1.2, "lcm", "sgm_uniform", 0.8, "lab", 1.0, 100.0, False, 1.0, - 0.6, "none", 6, "reflection", 0.25, 0.02, 2, 0, 0]) - stab = w.node("DifforumFlowStabilize", [1340, 460], - inputs=[("frames", "IMAGE")], outputs=[("frames", "IMAGE")], - widgets=[0.5, 0.5, 0.15]) - save = w.node("SaveImage", [1680, 460], inputs=[("images", "IMAGE")], - outputs=[], widgets=["Difforum_fast"], is_output=True) - w.link(ckpt, 0, lora, 0, "MODEL") - w.link(ckpt, 1, lora, 1, "CLIP") - w.link(ckpt, 1, neg, 0, "CLIP") - w.link(ckpt, 1, scenes, 1, "CLIP") - w.link(setup, 0, scenes, 0, "DIFFORUM_PARAMS") - w.link(ckpt, 0, ks, 0, "MODEL") - w.link(scenes, 1, ks, 1, "CONDITIONING") - w.link(neg, 0, ks, 2, "CONDITIONING") - w.link(lat, 0, ks, 3, "LATENT") - w.link(ks, 0, dec, 0, "LATENT") - w.link(ckpt, 2, dec, 1, "VAE") - w.link(setup, 0, cam, 0, "DIFFORUM_PARAMS") - w.link(setup, 0, strength, 0, "DIFFORUM_PARAMS") - w.link(lora, 0, fb, 0, "MODEL") - w.link(scenes, 1, fb, 1, "CONDITIONING") - w.link(neg, 0, fb, 2, "CONDITIONING") - w.link(ckpt, 2, fb, 3, "VAE") - w.link(setup, 0, fb, 4, "DIFFORUM_PARAMS") - w.link(cam, 0, fb, 5, "DIFFORUM_CAMERA") - w.link(dec, 0, fb, 6, "IMAGE") - w.link(strength, 0, fb, 7, "DIFFORUM_SCHEDULE") - w.link(scenes, 0, fb, 10, "DIFFORUM_PROMPT") - w.link(fb, 0, stab, 0, "IMAGE") - pp = w.node("DifforumLoopBlend", [1520, 660], inputs=[("frames", "IMAGE")], - outputs=[("frames", "IMAGE")], widgets=[12, True]) - w.link(stab, 0, pp, 0, "IMAGE") - w.link(pp, 0, save, 0, "IMAGE") - w.group("1 - FIRST FRAME: full quality, base model (26 steps)", [10, -60, 1270, 440], "#33415a") - w.group("2 - DIRECTOR + SCHEDULES", [10, 450, 900, 900], "#33503f") - w.group("3 - FAST LOOP: DMD2 4-step + cadence 2", [960, 380, 350, 480], "#5a3a33") - w.group("4 - QUALITY: anti-flicker (Ctrl+B to bypass)", [1310, 380, 350, 300], "#4a3d5a") - return w.dump() - - -def add_note(data, text, pos=(40, -210)): - """Append a frontend Note node (yellow sticky) with a model checklist.""" - nid = data["last_node_id"] + 1 - data["last_node_id"] = nid - data["nodes"].append({ - "id": nid, "type": "Note", "pos": list(pos), "size": [460, 300], - "flags": {}, "order": len(data["nodes"]), "mode": 0, - "inputs": [], "outputs": [], "title": "READ ME - models needed", - "properties": {"text": ""}, "widgets_values": [text], - "color": "#432", "bgcolor": "#653", - }) - - -# concise, real download pointers per template (folder shown after ->) -NOTES = { - "difforum_schedule_basic.json": - "Difforum: Schedule basics.\n\nNo models needed - this runs anywhere " - "(schedules + curve plot only).", - "difforum_models_info.json": - "Difforum: Reference card.\n\nNo models needed. Model Profile auto-detects " - "your VRAM; Model Catalog lists recipes + download links.", - "difforum_audio_reactive.json": - "Difforum: Audio analysis (no render).\n\nNeeds:\n" - "- an audio file (.mp3/.wav) -> dropped in the Load Audio node\n" - "No checkpoint required.", - "difforum_camera_warp.json": - "Difforum: Warp preview.\n\nNeeds:\n- an input image -> Load Image\n" - "No checkpoint (this only previews the 2D/3D warp + occlusion mask).\n" - "Tip: uses the intuitive Camera Move presets.", - "difforum_hybrid_wan_guides.json": - "Difforum: Hybrid bridge -> Wan 2.2 VACE.\n\nThis builds camera-warped GUIDE " - "frames. To actually render, wire `guide_frames` into a Wan 2.2 VACE graph.\n\n" - "Models:\n- anchor image -> Load Image\n" - "- Wan 2.2 5B GGUF -> models/unet (hf: QuantStack/Wan2.2-TI2V-5B-GGUF)\n" - "- umt5 text encoder -> models/text_encoders (hf: Comfy-Org/Wan_2.1_ComfyUI_repackaged)\n" - "- wan_2.1_vae.safetensors -> models/vae\n" - "- nodes: ComfyUI-WanVideoWrapper + ComfyUI-GGUF", - "difforum_feedback_classic.json": - "Difforum: Classic+ feedback video.\n\nModels:\n" - "- SD1.5 checkpoint -> models/checkpoints (DreamShaper: civitai.com/models/4384)\n" - "- (optional) better VAE vae-ft-mse-840000 -> models/vae (hf: stabilityai/sd-vae-ft-mse)\n\n" - "Quality upgrade: swap the checkpoint for SDXL or Flux (model-agnostic).\n" - "Groups: 4 = anti-flicker (Flow Stabilize) - select it and Ctrl+B to bypass.\n" - "In the sampler: sharpen/noise = detail guard, cadence 2-3 = faster long videos.\n" - "Tip: try Camera Shots + Prompt Scenes for an easier setup.", - "difforum_intuitive_controls.json": - "Difforum: Intuitive controls (Camera Move presets + Prompt Scenes).\n\nModels:\n" - "- SD1.5 checkpoint -> models/checkpoints (DreamShaper: civitai.com/models/4384)\n" - "Swap to SDXL/Flux for higher quality.\n\n" - "Camera Shots = direct like an edit (frame: preset speed intensity), and the\n" - "Path Preview shows the trajectory BEFORE you render. Group 4 = anti-flicker\n" - "(select + Ctrl+B to bypass). cadence 2-3 in the sampler = faster long videos.", - "difforum_turbo_live.json": - "Difforum: Turbo/LCM fast feedback (4 steps, cfg 1).\n\nModels:\n" - "- SD-Turbo -> models/checkpoints (hf: stabilityai/sd-turbo)\n" - " or any SD1.5 + LCM-LoRA (hf: latent-consistency/lcm-lora-sdv1-5 -> models/loras)\n" - "This is the path toward realtime.", - "difforum_deluxe_travel_controlnet_video.json": - "Difforum: Deluxe (travel + ControlNet + interpolation + MP4).\n\nModels:\n" - "- SD1.5 checkpoint -> models/checkpoints (DreamShaper: civitai.com/models/4384)\n" - "- tile ControlNet control_v11f1e_sd15_tile -> models/controlnet\n" - " (hf: comfyanonymous/ControlNet-v1-1_fp16_safetensors)\n" - "- nodes: ComfyUI-Frame-Interpolation (RIFE auto-downloads) + ComfyUI-VideoHelperSuite\n" - "ControlNet must match the base family (SD1.5 here).", - "difforum_ipadapter_coherent.json": - "Difforum: IPAdapter style-lock (coherence).\n\nModels:\n" - "- SD1.5 checkpoint -> models/checkpoints\n" - "- IP-Adapter ip-adapter-plus_sd15.safetensors -> models/ipadapter (hf: h94/IP-Adapter)\n" - "- CLIP-Vision (ViT-H) -> models/clip_vision (hf: h94/IP-Adapter, image_encoder)\n" - "- a style/reference image -> Load Image\n" - "- node: ComfyUI_IPAdapter_plus", - "difforum_audio_reactive_video.json": - "Difforum: Audio-reactive video (bass->zoom, beat->strength).\n\nModels:\n" - "- SD1.5 checkpoint -> models/checkpoints\n" - "- an audio file (.mp3/.wav) -> Load Audio\n" - "- node: ComfyUI-VideoHelperSuite (for the MP4)", - "difforum_animatediff_sd15.json": - "Difforum: AnimateDiff bridge (native motion).\n\nModels:\n" - "- SD1.5 checkpoint -> models/checkpoints\n" - "- motion module mm_sd15_v3.ckpt -> models/animatediff_models (hf: guoyww/animatediff)\n" - "- node: ComfyUI-AnimateDiff-Evolved", - "difforum_qrcode_illusion.json": - "Difforum: Hidden-pattern illusion (QR Monster).\n\nModels:\n" - "- SD1.5 checkpoint -> models/checkpoints\n" - "- control_v1p_sd15_qrcode_monster -> models/controlnet (hf: monster-labs/control_v1p_sd15_qrcode_monster)\n" - "- a grayscale pattern image (spiral/logo/mask) -> Load Image\n" - "control_strength ~1.2 = strong illusion; lower for subtle.", - "difforum_realtime_live.json": - "Difforum: Native realtime (Live Sampler internal loop).\n\n" - "Queue ONCE and watch the node stream a live preview as it generates - the\n" - "Live Sampler keeps the model resident and loops warp + kaleidoscope +\n" - "1-step re-diffuse internally. No external runtime.\n\nModels:\n" - "- SD-Turbo -> models/checkpoints (hf: stabilityai/sd-turbo)\n" - " or any SD1.5 + LCM-LoRA (hf: latent-consistency/lcm-lora-sdv1-5)\n" - "Use 512x512, 1-2 steps, cfg 1. Add TensorRT + TAESD for max FPS.\n\n" - "Magic mirror: set the Live Sampler's live_source to 0 (webcam) or a video\n" - "path to stylize a live feed (kaleidoscope your camera). Needs opencv.\n" - "Live output: set stream_dir (folder for OBS/Resolume) or spout_name\n" - "(needs SpoutGL). See REALTIME.md.", - "difforum_fast_sdxl.json": - "Difforum: FAST SDXL (DMD2 distill + cadence). ~8-10x less diffusion cost\n" - "than the classic recipe, same checkpoint look. Best template for Macs.\n\n" - "Models:\n" - "- SDXL checkpoint -> models/checkpoints (Juggernaut XL: civitai)\n" - "- DMD2 4-step LoRA -> models/loras (hf: tianweiy/DMD2,\n" - " file dmd2_sdxl_4step_lora.safetensors)\n\n" - "How it works: frame 0 renders on the BASE model at 26 steps (full\n" - "quality), the loop runs the DMD2 LoRA at 5 steps cfg 1.2 (lcm sampler)\n" - "with cadence 2. Group 4 = anti-flicker (Ctrl+B to bypass).", - "difforum_mesmerize_kaleidoscope.json": - "Difforum: Living kaleidoscope (in-loop symmetry + Echo Trails).\n\nModels:\n" - "- SD1.5 checkpoint -> models/checkpoints (DreamShaper: civitai.com/models/4384)\n" - "Swap to SDXL/Flux for higher quality.\n\n" - "How it works: the Feedback Sampler's `symmetry` = kaleidoscope folds each\n" - "warped frame, then the diffusion heals the seams, so the pattern grows and\n" - "stays symmetric. Echo Trails adds smooth motion-blur trails.\n" - "Try symmetry mirror_h/mirror_v/mirror_quad, or raise symmetry_segments.\n" - "LOOP tip: rotation-only camera (no net zoom/dolly) + ping-pong export =\n" - "seamless loop. Group 4 = anti-flicker + trails (Ctrl+B to bypass).", - "difforum_vj_footage.json": - "Difforum: VJ look for video footage. Complete + self-contained, NO model\n" - "and NO external nodes.\n\n" - "Load Video -> VJ Look (grade + glow + chroma + grain) -> Echo Trails ->\n" - "Save Video (MP4). Pure post, runs fast.\n\nSetup:\n" - "- put your clip in ComfyUI/input, pick it in Load Video\n" - "- Load Video / Save Video need opencv-python + imageio-ffmpeg (usually\n" - " already in ComfyUI). Set resize_to (e.g. 720) to cap resolution.\n\n" - "VJ Look presets: neon / cinematic / vaporwave / film / noir / psychedelic.\n" - "Beat-reactive: add Anim Setup + Load Audio + Audio Analyzer + Audio Schedule\n" - "(source=beat, base=1.0, amount=0.5) and wire its schedule into the VJ Look's\n" - "intensity_schedule. Re-style with diffusion? Insert a Feedback/Live Sampler\n" - "before the VJ Look.", -} - - -def main(): - out = { - "difforum_qrcode_illusion.json": build_qrcode_illusion(), - "difforum_ipadapter_coherent.json": build_ipadapter_coherent(), - "difforum_audio_reactive_video.json": build_audio_reactive_video(), - "difforum_animatediff_sd15.json": build_animatediff(), - "difforum_turbo_live.json": build_turbo_live(), - "difforum_intuitive_controls.json": build_intuitive(), - "difforum_schedule_basic.json": build_basic(), - "difforum_audio_reactive.json": build_audio(), - "difforum_feedback_classic.json": build_feedback(), - "difforum_camera_warp.json": build_camera_warp(), - "difforum_hybrid_wan_guides.json": build_hybrid_guides(), - "difforum_models_info.json": build_models_info(), - "difforum_deluxe_travel_controlnet_video.json": build_deluxe(), - "difforum_mesmerize_kaleidoscope.json": build_mesmerize(), - "difforum_realtime_live.json": build_realtime(), - "difforum_vj_footage.json": build_vj_footage(), - "difforum_fast_sdxl.json": build_fast_sdxl(), - } - for name, data in out.items(): - if name in NOTES: - add_note(data, NOTES[name]) - tier = TIERS.get(name, "advanced") - folder = HERE / tier - folder.mkdir(parents=True, exist_ok=True) - path = folder / name - path.write_text(json.dumps(data, indent=2), encoding="utf-8") - print(f"wrote {tier}/{path.name} " - f"({len(data['nodes'])} nodes, {len(data['links'])} links)") - print("OK") - - -if __name__ == "__main__": - main() diff --git a/examples/_guide_demo.py b/examples/_guide_demo.py deleted file mode 100644 index 9bed8ad..0000000 --- a/examples/_guide_demo.py +++ /dev/null @@ -1,50 +0,0 @@ -""" -Offline validation of the hybrid bridge: run the real Difforum nodes -(AnimSetup -> Camera -> GuideBuilder) on a real anchor image and save the guide -sequence + a GIF. No Wan model needed - this proves the guide batch the Wan -VACE graph would receive is coherent (camera-driven warp of the anchor). -""" - -import glob -import sys -from pathlib import Path - -import numpy as np -import torch -from PIL import Image - -sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) - -from difforum.nodes.guide_nodes import DifforumGuideBuilder # noqa: E402 -from difforum.nodes.hybrid_nodes import DifforumCamera # noqa: E402 -from difforum.nodes.schedule_nodes import DifforumAnimSetup # noqa: E402 - -HERE = Path(__file__).resolve().parent -OUT = HERE / "_render_out" - -# pick any rendered frame as the anchor -cands = sorted(glob.glob(str(OUT / "Difforum_pt_*.png"))) or sorted(glob.glob(str(OUT / "*.png"))) -if not cands: - raise SystemExit("no anchor image found in _render_out") -anchor_path = cands[-1] -img = Image.open(anchor_path).convert("RGB") -anchor = torch.from_numpy(np.asarray(img, dtype=np.float32) / 255.0).unsqueeze(0) -print(f"anchor: {anchor_path} shape={tuple(anchor.shape)}") - -params = DifforumAnimSetup().build(img.width, img.height, 16, 24, 0)[0] -cam = DifforumCamera().run( - params, "2d", 40.0, - translation_x="0:(0)", translation_y="0:(0)", translation_z="0:(0)", - rotation_3d_x="0:(0)", rotation_3d_y="0:(0)", rotation_3d_z="0:(0.8)", - zoom="0:(1.03)", -)[0] - -guides, masks, info = DifforumGuideBuilder().run(anchor, cam, params, "force_2d") -print(info) -print(f"guides shape={tuple(guides.shape)} masks shape={tuple(masks.shape)}") - -frames = [Image.fromarray((guides[i].clamp(0, 1).numpy() * 255).astype(np.uint8)) for i in range(guides.shape[0])] -frames = [f.resize((256, 256)) for f in frames] -gif = HERE / "difforum_guide_demo.gif" -frames[0].save(gif, save_all=True, append_images=frames[1:] + frames[::-1], duration=120, loop=0, optimize=True) -print(f"GIF: {gif} ({gif.stat().st_size // 1024} KB, {len(frames)} frames)") diff --git a/examples/_smoke_live.py b/examples/_smoke_live.py deleted file mode 100644 index 3abc2aa..0000000 --- a/examples/_smoke_live.py +++ /dev/null @@ -1,78 +0,0 @@ -"""Smoke test for DifforumLiveSampler: runs a short internal-loop live session -through a running ComfyUI and checks it returns frames. Usage: python _smoke_live.py [port]""" -import json -import sys -import time -import urllib.request - -PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 8199 -BASE = f"http://127.0.0.1:{PORT}" -CKPT = "juggernautXL_v9.safetensors" - - -def _post(p, d): - r = urllib.request.Request(BASE + p, data=json.dumps(d).encode(), headers={"Content-Type": "application/json"}) - return json.loads(urllib.request.urlopen(r, timeout=30).read()) - - -def _get(p): - return json.loads(urllib.request.urlopen(BASE + p, timeout=30).read()) - - -def wait_ready(t=300): - t0 = time.time() - while time.time() - t0 < t: - try: - oi = _get("/object_info") - if "DifforumLiveSampler" in oi and CKPT in json.dumps(oi.get("CheckpointLoaderSimple", {})): - return print("server ready") - except Exception: - pass - time.sleep(2) - raise SystemExit("not ready") - - -def graph(): - return { - "1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": CKPT}}, - "2": {"class_type": "CLIPTextEncode", "inputs": {"text": "psychedelic mandala, vivid colors", "clip": ["1", 1]}}, - "3": {"class_type": "CLIPTextEncode", "inputs": {"text": "blurry, low quality", "clip": ["1", 1]}}, - "4": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}}, - "5": {"class_type": "KSampler", "inputs": {"model": ["1", 0], "positive": ["2", 0], "negative": ["3", 0], - "latent_image": ["4", 0], "seed": 1, "steps": 8, "cfg": 4.0, "sampler_name": "euler", "scheduler": "normal", "denoise": 1.0}}, - "6": {"class_type": "VAEDecode", "inputs": {"samples": ["5", 0], "vae": ["1", 2]}}, - "7": {"class_type": "DifforumAnimSetup", "inputs": {"width": 512, "height": 512, "fps": 12, "max_frames": 30, "seed": 1}}, - "8": {"class_type": "DifforumCameraMove", "inputs": {"params": ["7", 0], "preset": "spiral", "speed": 1.0, "intensity": 1.0, "mode": "2d", "fov": 40.0}}, - "9": {"class_type": "DifforumLiveSampler", "inputs": { - "model": ["1", 0], "positive": ["2", 0], "negative": ["3", 0], "vae": ["1", 2], - "params": ["7", 0], "camera": ["8", 0], "init_image": ["6", 0], - "duration_frames": 8, "strength": 0.5, "steps": 2, "cfg": 2.0, - "sampler_name": "euler", "scheduler": "normal", "color_coherence": 0.5, - "color_mode": "lab", "symmetry": "kaleidoscope", "symmetry_segments": 6, - "target_fps": 0.0, "live_preview": True}}, - "10": {"class_type": "SaveImage", "inputs": {"images": ["9", 0], "filename_prefix": "Difforum_livetest"}}, - } - - -def main(): - wait_ready() - pid = _post("/prompt", {"prompt": graph()})["prompt_id"] - print("submitted", pid) - t0 = time.time() - while time.time() - t0 < 300: - h = _get(f"/history/{pid}") - if pid in h: - st = h[pid].get("status", {}) - if "outputs" in h[pid] or st.get("completed"): - imgs = h[pid].get("outputs", {}).get("10", {}).get("images", []) - print(f"LIVE SAMPLER OK - returned {len(imgs)} frames in {time.time()-t0:.1f}s") - return - if st.get("status_str") == "error": - print("ERROR:", json.dumps(st)[:500]) - raise SystemExit(1) - time.sleep(2) - raise SystemExit("timeout") - - -if __name__ == "__main__": - main() diff --git a/examples/_smoke_render.py b/examples/_smoke_render.py deleted file mode 100644 index 1ea5348..0000000 --- a/examples/_smoke_render.py +++ /dev/null @@ -1,110 +0,0 @@ -""" -Smoke render: drive a running ComfyUI via its HTTP API to actually render a -short Difforum Classic+ animation end-to-end with the real SD1.5 checkpoint. - -Usage: python _smoke_render.py [port] -Assumes the ComfyUI server is already up on 127.0.0.1:. -""" - -import json -import sys -import time -import urllib.request - -PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 8199 -BASE = f"http://127.0.0.1:{PORT}" -CKPT = "v1-5-pruned-emaonly-fp16.safetensors" - - -def _post(path, payload): - data = json.dumps(payload).encode("utf-8") - req = urllib.request.Request(BASE + path, data=data, - headers={"Content-Type": "application/json"}) - return json.loads(urllib.request.urlopen(req, timeout=30).read()) - - -def _get(path): - return json.loads(urllib.request.urlopen(BASE + path, timeout=30).read()) - - -def wait_ready(timeout=180): - t0 = time.time() - while time.time() - t0 < timeout: - try: - oi = _get("/object_info") - if "DifforumFeedbackSampler" in oi and "CheckpointLoaderSimple" in oi: - print("server ready; Difforum nodes registered") - return True - except Exception: - pass - time.sleep(2) - raise SystemExit("server did not become ready in time") - - -def build_prompt(): - return { - "1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": CKPT}}, - "2": {"class_type": "CLIPTextEncode", - "inputs": {"text": "psychedelic cosmic tunnel, vivid neon colors, highly detailed", - "clip": ["1", 1]}}, - "3": {"class_type": "CLIPTextEncode", - "inputs": {"text": "blurry, low quality, watermark, text", "clip": ["1", 1]}}, - "4": {"class_type": "EmptyLatentImage", - "inputs": {"width": 512, "height": 512, "batch_size": 1}}, - "5": {"class_type": "KSampler", - "inputs": {"model": ["1", 0], "positive": ["2", 0], "negative": ["3", 0], - "latent_image": ["4", 0], "seed": 7, "steps": 15, "cfg": 7.0, - "sampler_name": "euler", "scheduler": "normal", "denoise": 1.0}}, - "6": {"class_type": "VAEDecode", "inputs": {"samples": ["5", 0], "vae": ["1", 2]}}, - "7": {"class_type": "DifforumAnimSetup", - "inputs": {"width": 512, "height": 512, "fps": 12, "max_frames": 12, "seed": 7}}, - "8": {"class_type": "DifforumCamera", - "inputs": {"params": ["7", 0], "mode": "2d", "fov": 40.0, - "translation_x": "0:(3)", "translation_y": "0:(0)", "translation_z": "0:(0)", - "rotation_3d_x": "0:(0)", "rotation_3d_y": "0:(0)", "rotation_3d_z": "0:(1.0)", - "zoom": "0:(1.02)"}}, - "9": {"class_type": "DifforumSchedule", - "inputs": {"params": ["7", 0], "schedule": "0:(0.5)", "easing": "linear"}}, - "10": {"class_type": "DifforumFeedbackSampler", - "inputs": {"model": ["1", 0], "positive": ["2", 0], "negative": ["3", 0], - "vae": ["1", 2], "params": ["7", 0], "camera": ["8", 0], - "init_image": ["6", 0], "strength_schedule": ["9", 0], - "steps": 15, "cfg": 7.0, "sampler_name": "euler", - "scheduler": "normal", "color_coherence": 0.8}}, - "11": {"class_type": "SaveImage", - "inputs": {"images": ["10", 0], "filename_prefix": "Difforum_smoke"}}, - } - - -def main(): - wait_ready() - prompt = build_prompt() - res = _post("/prompt", {"prompt": prompt}) - pid = res["prompt_id"] - print(f"submitted prompt {pid}; rendering 12 frames...") - - t0 = time.time() - while True: - hist = _get(f"/history/{pid}") - if pid in hist: - entry = hist[pid] - status = entry.get("status", {}) - if status.get("completed") or "outputs" in entry: - outs = entry.get("outputs", {}) - imgs = outs.get("11", {}).get("images", []) - print(f"DONE in {time.time()-t0:.1f}s - {len(imgs)} frames saved:") - for im in imgs: - print(f" {im.get('subfolder','')}/{im['filename']}") - if status.get("status_str") == "error" or not imgs: - print("status:", json.dumps(status, indent=2)[:2000]) - return - if status.get("status_str") == "error": - print("ERROR:", json.dumps(status, indent=2)[:3000]) - return - if time.time() - t0 > 600: - raise SystemExit("render timed out") - time.sleep(2) - - -if __name__ == "__main__": - main() diff --git a/examples/_smoke_render_169.py b/examples/_smoke_render_169.py deleted file mode 100644 index 13c94e5..0000000 --- a/examples/_smoke_render_169.py +++ /dev/null @@ -1,76 +0,0 @@ -"""16:9 showcase render: drives a running ComfyUI to render a widescreen -Classic+ animation (768x432) for the README. Usage: python _smoke_render_169.py [port]""" -import json, sys, time, urllib.request - -PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 8199 -BASE = f"http://127.0.0.1:{PORT}" -CKPT = "v1-5-pruned-emaonly-fp16.safetensors" -PROMPTS = ("0: a luminous alien jungle, giant glowing flora, volumetric light\n" - "10: a crystalline ice canyon, aurora overhead\n" - "19: a golden desert of dunes under twin suns") -FRAME0 = "a luminous alien jungle, giant glowing flora, volumetric light" -W, H, N = 768, 432, 20 - - -def _post(p, d): - r = urllib.request.Request(BASE + p, data=json.dumps(d).encode(), headers={"Content-Type": "application/json"}) - return json.loads(urllib.request.urlopen(r, timeout=30).read()) - - -def _get(p): - return json.loads(urllib.request.urlopen(BASE + p, timeout=30).read()) - - -def wait_ready(t=180): - t0 = time.time() - while time.time() - t0 < t: - try: - if "DifforumFeedbackSampler" in _get("/object_info"): - return print("server ready") - except Exception: - pass - time.sleep(2) - raise SystemExit("server not ready") - - -def graph(): - return { - "1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": CKPT}}, - "2": {"class_type": "CLIPTextEncode", "inputs": {"text": FRAME0, "clip": ["1", 1]}}, - "3": {"class_type": "CLIPTextEncode", "inputs": {"text": "blurry, low quality, watermark", "clip": ["1", 1]}}, - "4": {"class_type": "EmptyLatentImage", "inputs": {"width": W, "height": H, "batch_size": 1}}, - "5": {"class_type": "KSampler", "inputs": {"model": ["1", 0], "positive": ["2", 0], "negative": ["3", 0], - "latent_image": ["4", 0], "seed": 5, "steps": 20, "cfg": 7.0, "sampler_name": "euler", "scheduler": "normal", "denoise": 1.0}}, - "6": {"class_type": "VAEDecode", "inputs": {"samples": ["5", 0], "vae": ["1", 2]}}, - "7": {"class_type": "DifforumAnimSetup", "inputs": {"width": W, "height": H, "fps": 12, "max_frames": N, "seed": 5}}, - "8": {"class_type": "DifforumCamera", "inputs": {"params": ["7", 0], "mode": "2d", "fov": 40.0, - "translation_x": "0:(3)", "translation_y": "0:(0)", "translation_z": "0:(0)", - "rotation_3d_x": "0:(0)", "rotation_3d_y": "0:(0)", "rotation_3d_z": "0:(0.4)", "zoom": "0:(1.02)"}}, - "9": {"class_type": "DifforumSchedule", "inputs": {"params": ["7", 0], "schedule": "0:(0.5)", "easing": "linear"}}, - "12": {"class_type": "DifforumPromptSchedule", "inputs": {"params": ["7", 0], "clip": ["1", 1], "prompts": PROMPTS, "easing": "ease_in_out"}}, - "10": {"class_type": "DifforumFeedbackSampler", "inputs": {"model": ["1", 0], "positive": ["2", 0], "negative": ["3", 0], - "vae": ["1", 2], "params": ["7", 0], "camera": ["8", 0], "init_image": ["6", 0], - "strength_schedule": ["9", 0], "positive_schedule": ["12", 0], - "steps": 20, "cfg": 7.0, "sampler_name": "euler", "scheduler": "normal", "color_coherence": 0.8, "color_mode": "lab"}}, - "11": {"class_type": "SaveImage", "inputs": {"images": ["10", 0], "filename_prefix": "Difforum_169"}}, - } - - -def main(): - wait_ready() - pid = _post("/prompt", {"prompt": graph()})["prompt_id"] - print(f"submitted {pid}; rendering {N} frames @ {W}x{H}...") - t0 = time.time() - while True: - h = _get(f"/history/{pid}") - if pid in h and ("outputs" in h[pid] or h[pid].get("status", {}).get("completed")): - imgs = h[pid].get("outputs", {}).get("11", {}).get("images", []) - print(f"DONE in {time.time()-t0:.1f}s - {len(imgs)} frames") - return - if time.time() - t0 > 600: - raise SystemExit("timeout") - time.sleep(2) - - -if __name__ == "__main__": - main() diff --git a/examples/_smoke_render_ab.py b/examples/_smoke_render_ab.py deleted file mode 100644 index 0344530..0000000 --- a/examples/_smoke_render_ab.py +++ /dev/null @@ -1,144 +0,0 @@ -"""A/B proof render for the detail guard: two identical SDXL feedback runs -(same seed, camera, prompt), A with the old settings (zeros border, no sharpen, -no noise) and B with the detail guard on. Exports a side-by-side GIF + final -frame PNG. Usage: python _smoke_render_ab.py [port]""" -import io -import json -import sys -import time -import urllib.request -from pathlib import Path - -PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 8199 -BASE = f"http://127.0.0.1:{PORT}" -CKPT = "juggernautXL_v9.safetensors" -W, H, N = 1024, 576, 48 -STEPS, CFG, SAMPLER, SCHED = 22, 6.5, "dpmpp_2m", "karras" -PROMPT = ("intricate ancient stone temple wall covered in detailed carvings, moss, " - "roots, volumetric light, highly detailed, sharp focus, cinematic") -NEG = "blurry, low quality, watermark, text, jpeg artifacts, washed out" -HERE = Path(__file__).resolve().parent - -VARIANTS = { - "off": {"border": "zeros", "sharpen": 0.0, "noise": 0.0}, - "guard": {"border": "reflection", "sharpen": 0.3, "noise": 0.03}, -} - - -def _post(p, d): - r = urllib.request.Request(BASE + p, data=json.dumps(d).encode(), headers={"Content-Type": "application/json"}) - return json.loads(urllib.request.urlopen(r, timeout=30).read()) - - -def _get(p): - return json.loads(urllib.request.urlopen(BASE + p, timeout=30).read()) - - -def _getbin(p): - return urllib.request.urlopen(BASE + p, timeout=60).read() - - -def wait_ready(t=300): - t0 = time.time() - while time.time() - t0 < t: - try: - oi = _get("/object_info") - fb = oi.get("DifforumFeedbackSampler", {}) - if "border" in json.dumps(fb) and CKPT in json.dumps(oi.get("CheckpointLoaderSimple", {})): - return print("server ready (new sampler visible)") - except Exception: - pass - time.sleep(2) - raise SystemExit("server/sampler not ready") - - -def graph(v): - return { - "1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": CKPT}}, - "2": {"class_type": "CLIPTextEncode", "inputs": {"text": PROMPT, "clip": ["1", 1]}}, - "3": {"class_type": "CLIPTextEncode", "inputs": {"text": NEG, "clip": ["1", 1]}}, - "4": {"class_type": "EmptyLatentImage", "inputs": {"width": W, "height": H, "batch_size": 1}}, - "5": {"class_type": "KSampler", "inputs": {"model": ["1", 0], "positive": ["2", 0], "negative": ["3", 0], - "latent_image": ["4", 0], "seed": 42, "steps": 26, "cfg": CFG, "sampler_name": SAMPLER, "scheduler": SCHED, "denoise": 1.0}}, - "6": {"class_type": "VAEDecode", "inputs": {"samples": ["5", 0], "vae": ["1", 2]}}, - "7": {"class_type": "DifforumAnimSetup", "inputs": {"width": W, "height": H, "fps": 12, "max_frames": N, "seed": 42}}, - "8": {"class_type": "DifforumCameraMove", "inputs": {"params": ["7", 0], "preset": "spiral", - "speed": 1.2, "intensity": 1.2, "mode": "2d", "fov": 40.0}}, - "9": {"class_type": "DifforumSchedule", "inputs": {"params": ["7", 0], "schedule": "0:(0.5)", "easing": "linear"}}, - "10": {"class_type": "DifforumFeedbackSampler", "inputs": {"model": ["1", 0], "positive": ["2", 0], "negative": ["3", 0], - "vae": ["1", 2], "params": ["7", 0], "camera": ["8", 0], "init_image": ["6", 0], - "strength_schedule": ["9", 0], - "steps": STEPS, "cfg": CFG, "sampler_name": SAMPLER, "scheduler": SCHED, - "color_coherence": 0.8, "color_mode": "lab", "symmetry": "none", "symmetry_segments": 6, - "border": v["border"], "sharpen": v["sharpen"], "noise": v["noise"]}}, - "11": {"class_type": "SaveImage", "inputs": {"images": ["10", 0], "filename_prefix": f"Difforum_ab_{v['border']}"}}, - } - - -def run(name, v): - pid = _post("/prompt", {"prompt": graph(v)})["prompt_id"] - print(f"[{name}] submitted {pid}; {N} frames...") - t0 = time.time() - while True: - h = _get(f"/history/{pid}") - if pid in h and ("outputs" in h[pid] or h[pid].get("status", {}).get("completed")): - imgs = h[pid].get("outputs", {}).get("11", {}).get("images", []) - print(f"[{name}] done in {time.time()-t0:.1f}s - {len(imgs)} frames") - return imgs - if time.time() - t0 > 2400: - raise SystemExit(f"[{name}] timeout") - time.sleep(3) - - -def fetch(images): - from PIL import Image - out = [] - for im in images: - q = f"/view?filename={im['filename']}&subfolder={im.get('subfolder','')}&type={im.get('type','output')}" - out.append(Image.open(io.BytesIO(_getbin(q))).convert("RGB")) - return out - - -def label(img, text): - from PIL import ImageDraw - d = ImageDraw.Draw(img) - d.rectangle([0, 0, 150, 22], fill=(0, 0, 0)) - d.text((6, 4), text, fill=(255, 255, 255)) - return img - - -def export(a, b): - from PIL import Image - tw, th = 480, 270 - frames = [] - for fa, fb in zip(a, b): - left = label(fa.resize((tw, th), Image.LANCZOS), "detail guard OFF") - right = label(fb.resize((tw, th), Image.LANCZOS), "detail guard ON") - sheet = Image.new("RGB", (tw * 2 + 4, th), (16, 16, 16)) - sheet.paste(left, (0, 0)) - sheet.paste(right, (tw + 4, 0)) - frames.append(sheet) - gif = HERE / "difforum_detail_ab.gif" - q = [f.quantize(colors=128, dither=Image.Dither.NONE) for f in frames] - q[0].save(gif, save_all=True, append_images=q[1:], duration=110, loop=0, optimize=True) - print(f"GIF -> {gif.name} ({gif.stat().st_size//1024} KB)") - # final-frame still, full res halves - fw, fh = 640, 360 - still = Image.new("RGB", (fw * 2 + 4, fh), (16, 16, 16)) - still.paste(label(a[-1].resize((fw, fh), Image.LANCZOS), "detail guard OFF"), (0, 0)) - still.paste(label(b[-1].resize((fw, fh), Image.LANCZOS), "detail guard ON"), (fw + 4, 0)) - png = HERE / "difforum_detail_ab_final.png" - still.save(png) - print(f"PNG -> {png.name} ({png.stat().st_size//1024} KB)") - - -def main(): - wait_ready() - a = fetch(run("off", VARIANTS["off"])) - b = fetch(run("guard", VARIANTS["guard"])) - export(a, b) - print("A/B DONE") - - -if __name__ == "__main__": - main() diff --git a/examples/_smoke_render_extras.py b/examples/_smoke_render_extras.py deleted file mode 100644 index 4103afb..0000000 --- a/examples/_smoke_render_extras.py +++ /dev/null @@ -1,222 +0,0 @@ -"""SDXL extras: renders the audio-reactive showcase (bass pumps zoom, beats -pulse strength + spin) and refreshes the kaleidoscope effect GIF, both at -1024x576 with Juggernaut XL. Synthesizes a 120 BPM test beat first. -Drives a running ComfyUI. Usage: python _smoke_render_extras.py [port]""" -import io -import json -import struct -import subprocess -import sys -import time -import urllib.request -import wave -from pathlib import Path - -import numpy as np - -PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 8199 -BASE = f"http://127.0.0.1:{PORT}" -CKPT = "juggernautXL_v9.safetensors" -W, H = 1024, 576 -STEPS, CFG, SAMPLER, SCHED = 28, 6.5, "dpmpp_2m", "karras" -QUAL = ", highly detailed, sharp focus, intricate, cinematic lighting, vivid colors, masterpiece" -NEG = ("blurry, low quality, watermark, text, jpeg artifacts, deformed, washed out, " - "oversaturated, plain background, low contrast") -HERE = Path(__file__).resolve().parent -COMFY_INPUT = Path(r"D:\ComfyUI-victor\input") -AUDIO_NAME = "difforum_beat.wav" - - -def make_beat(path, bpm=120, seconds=4.0, sr=44100): - """A simple 4-on-the-floor kick + bass + offbeat hat, for the demo.""" - n = int(seconds * sr) - t = np.arange(n) / sr - out = np.zeros(n, dtype=np.float32) - beat = 60.0 / bpm - # kick on every beat: 55 Hz sine with a fast exponential decay - k = 0.0 - while k < seconds: - s = int(k * sr) - env = np.exp(-np.linspace(0, 1, int(0.18 * sr)) * 18.0) - tone = np.sin(2 * np.pi * 55 * np.arange(len(env)) / sr) - e = min(len(out), s + len(env)) - out[s:e] += (tone * env)[: e - s] * 0.9 - k += beat - # offbeat hi-hat: short noise burst - h = beat / 2 - while h < seconds: - s = int(h * sr) - env = np.exp(-np.linspace(0, 1, int(0.05 * sr)) * 40.0) - noise = np.random.randn(len(env)).astype(np.float32) - e = min(len(out), s + len(env)) - out[s:e] += (noise * env)[: e - s] * 0.15 - h += beat - # sustained sub bass - out += np.sin(2 * np.pi * 82 * t).astype(np.float32) * 0.12 - out = np.clip(out / (np.abs(out).max() + 1e-6), -1, 1) - pcm = (out * 32767).astype(np.int16) - path.parent.mkdir(parents=True, exist_ok=True) - with wave.open(str(path), "wb") as wf: - wf.setnchannels(1) - wf.setsampwidth(2) - wf.setframerate(sr) - wf.writeframes(b"".join(struct.pack(" {path} ({seconds}s, {bpm} BPM)") - - -def _post(p, d): - r = urllib.request.Request(BASE + p, data=json.dumps(d).encode(), headers={"Content-Type": "application/json"}) - return json.loads(urllib.request.urlopen(r, timeout=30).read()) - - -def _get(p): - return json.loads(urllib.request.urlopen(BASE + p, timeout=30).read()) - - -def _getbin(p): - return urllib.request.urlopen(BASE + p, timeout=60).read() - - -def wait_ready(t=300): - t0 = time.time() - while time.time() - t0 < t: - try: - if CKPT in json.dumps(_get("/object_info").get("CheckpointLoaderSimple", {})): - return print("server ready") - except Exception: - pass - time.sleep(2) - raise SystemExit("server/checkpoint not ready") - - -def base_nodes(frame0, n): - return { - "1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": CKPT}}, - "2": {"class_type": "CLIPTextEncode", "inputs": {"text": frame0 + QUAL, "clip": ["1", 1]}}, - "3": {"class_type": "CLIPTextEncode", "inputs": {"text": NEG, "clip": ["1", 1]}}, - "4": {"class_type": "EmptyLatentImage", "inputs": {"width": W, "height": H, "batch_size": 1}}, - "5": {"class_type": "KSampler", "inputs": {"model": ["1", 0], "positive": ["2", 0], "negative": ["3", 0], - "latent_image": ["4", 0], "seed": 33, "steps": STEPS, "cfg": CFG, "sampler_name": SAMPLER, "scheduler": SCHED, "denoise": 1.0}}, - "6": {"class_type": "VAEDecode", "inputs": {"samples": ["5", 0], "vae": ["1", 2]}}, - "7": {"class_type": "DifforumAnimSetup", "inputs": {"width": W, "height": H, "fps": 12, "max_frames": n, "seed": 33}}, - } - - -def graph_audio(): - n = 48 - frame0 = "abstract liquid light show, vivid neon colors, glowing particles, kinetic, reactive" - prompts = ( - "0: abstract liquid light show, vivid neon colors, glowing particles, kinetic\n" - "16: pulsing energy waves, neon bloom, rhythmic concentric patterns\n" - "32: explosive bursts of color, fractal bloom, dynamic streaks\n" - "47: a vortex of sound made visible, vibrant, swirling light") - g = base_nodes(frame0, n) - g.update({ - "20": {"class_type": "LoadAudio", "inputs": {"audio": AUDIO_NAME}}, - "21": {"class_type": "DifforumAudioAnalyzer", "inputs": {"params": ["7", 0], "audio": ["20", 0], - "smoothing": 0.2, "normalize": True, "beat_sensitivity": 1.5}}, - "8": {"class_type": "DifforumCamera", "inputs": {"params": ["7", 0], "audio": ["21", 0], "mode": "2d", "fov": 40.0, - "translation_x": "0:(0)", "translation_y": "0:(0)", "translation_z": "0:(0)", - "rotation_3d_x": "0:(0)", "rotation_3d_y": "0:(0)", - "rotation_3d_z": "0:(0.2 + 3.0*beat)", "zoom": "0:(1.0 + 0.5*low)"}}, - "22": {"class_type": "DifforumAudioSchedule", "inputs": {"params": ["7", 0], "audio": ["21", 0], - "source": "beat", "mode": "add", "base": 0.4, "amount": 0.3, "smoothing": 0.2}}, - "12": {"class_type": "DifforumPromptSchedule", "inputs": {"params": ["7", 0], "clip": ["1", 1], "prompts": prompts, "easing": "ease_in_out"}}, - "10": {"class_type": "DifforumFeedbackSampler", "inputs": {"model": ["1", 0], "positive": ["2", 0], "negative": ["3", 0], - "vae": ["1", 2], "params": ["7", 0], "camera": ["8", 0], "init_image": ["6", 0], - "strength_schedule": ["22", 0], "positive_schedule": ["12", 0], - "steps": STEPS, "cfg": CFG, "sampler_name": SAMPLER, "scheduler": SCHED, - "color_coherence": 0.8, "color_mode": "lab", "symmetry": "none", "symmetry_segments": 6}}, - "13": {"class_type": "DifforumEchoTrails", "inputs": {"frames": ["10", 0], "decay": 0.5, "mix": 0.25}}, - "11": {"class_type": "SaveImage", "inputs": {"images": ["13", 0], "filename_prefix": "Difforum_sdxl_audio"}}, - }) - return g, n - - -def graph_kaleido(): - n = 40 - frame0 = "ornate symmetric mandala, stained glass, iridescent jewel tones, intricate fractal detail, glowing" - g = base_nodes(frame0, n) - g.update({ - "8": {"class_type": "DifforumCamera", "inputs": {"params": ["7", 0], "mode": "2d", "fov": 40.0, - "translation_x": "0:(0)", "translation_y": "0:(0)", "translation_z": "0:(0)", - "rotation_3d_x": "0:(0)", "rotation_3d_y": "0:(0)", "rotation_3d_z": "0:(0.7)", "zoom": "0:(1.012)"}}, - "9": {"class_type": "DifforumSchedule", "inputs": {"params": ["7", 0], "schedule": "0:(0.44)", "easing": "linear"}}, - "10": {"class_type": "DifforumFeedbackSampler", "inputs": {"model": ["1", 0], "positive": ["2", 0], "negative": ["3", 0], - "vae": ["1", 2], "params": ["7", 0], "camera": ["8", 0], "init_image": ["6", 0], - "strength_schedule": ["9", 0], - "steps": STEPS, "cfg": CFG, "sampler_name": SAMPLER, "scheduler": SCHED, - "color_coherence": 0.8, "color_mode": "lab", "symmetry": "kaleidoscope", "symmetry_segments": 6}}, - "13": {"class_type": "DifforumEchoTrails", "inputs": {"frames": ["10", 0], "decay": 0.5, "mix": 0.3}}, - "11": {"class_type": "SaveImage", "inputs": {"images": ["13", 0], "filename_prefix": "Difforum_sdxl_kaleido"}}, - }) - return g, n - - -def _ffmpeg(): - import imageio_ffmpeg - return imageio_ffmpeg.get_ffmpeg_exe() - - -def fetch_frames(name, images): - from PIL import Image - tmp = HERE / f"_extra_{name}" - tmp.mkdir(exist_ok=True) - for f in tmp.glob("*.png"): - f.unlink() - for i, im in enumerate(images): - q = f"/view?filename={im['filename']}&subfolder={im.get('subfolder','')}&type={im.get('type','output')}" - Image.open(io.BytesIO(_getbin(q))).convert("RGB").save(tmp / f"f_{i:04d}.png") - return tmp - - -def make_gif(tmp, dst, size, colors=128): - ff = _ffmpeg() - pal = HERE / "_pal.png" - subprocess.run([ff, "-y", "-i", str(tmp / "f_%04d.png"), "-update", "1", - "-vf", f"scale={size}:flags=lanczos,palettegen=max_colors={colors}", str(pal)], check=True) - subprocess.run([ff, "-y", "-framerate", "12", "-i", str(tmp / "f_%04d.png"), "-i", str(pal), - "-lavfi", f"scale={size}:flags=lanczos[x];[x][1:v]paletteuse", str(dst)], check=True) - pal.unlink(missing_ok=True) - print(f"GIF -> {dst.name} ({dst.stat().st_size//1024} KB)") - - -def make_mp4(tmp, dst): - ff = _ffmpeg() - subprocess.run([ff, "-y", "-framerate", "12", "-i", str(tmp / "f_%04d.png"), - "-vf", "scale=1280:720:flags=lanczos,minterpolate=fps=24:mi_mode=blend", - "-c:v", "libx264", "-crf", "17", "-pix_fmt", "yuv420p", str(dst)], check=True) - print(f"MP4 -> {dst.name} ({dst.stat().st_size//1024} KB)") - - -def run(name, builder): - g, n = builder() - pid = _post("/prompt", {"prompt": g})["prompt_id"] - print(f"[{name}] submitted {pid}; {n} frames @ {W}x{H}...") - t0 = time.time() - while True: - h = _get(f"/history/{pid}") - if pid in h and ("outputs" in h[pid] or h[pid].get("status", {}).get("completed")): - imgs = h[pid].get("outputs", {}).get("11", {}).get("images", []) - print(f"[{name}] render done in {time.time()-t0:.1f}s - {len(imgs)} frames") - return fetch_frames(name, imgs) if imgs else None - if time.time() - t0 > 2400: - raise SystemExit(f"[{name}] timeout") - time.sleep(3) - - -def main(): - make_beat(COMFY_INPUT / AUDIO_NAME) - wait_ready() - a = run("audio", graph_audio) - if a: - make_mp4(a, HERE / "difforum_audio.mp4") - make_gif(a, HERE / "difforum_audio.gif", "600:338", 128) - k = run("kaleido", graph_kaleido) - if k: - make_gif(k, HERE / "difforum_kaleidoscope.gif", "600:338", 128) - print("ALL EXTRAS DONE") - - -if __name__ == "__main__": - main() diff --git a/examples/_smoke_render_kaleido.py b/examples/_smoke_render_kaleido.py deleted file mode 100644 index 2f83c9e..0000000 --- a/examples/_smoke_render_kaleido.py +++ /dev/null @@ -1,103 +0,0 @@ -"""16:9 kaleidoscope showcase: drives a running ComfyUI to render the living -kaleidoscope (in-loop symmetry + Echo Trails) and assemble a GIF for the README. -Usage: python _smoke_render_kaleido.py [port]""" -import io -import json -import sys -import time -import urllib.request -from pathlib import Path - -PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 8199 -BASE = f"http://127.0.0.1:{PORT}" -CKPT = "v1-5-pruned-emaonly-fp16.safetensors" -FRAME0 = ("ornate symmetric mandala, stained glass, iridescent, " - "intricate fractal detail, glowing, kaleidoscope") -NEG = "blurry, low quality, watermark, text, asymmetric, jpeg artifacts" -W, H, N = 768, 432, 24 -OUT_GIF = Path(__file__).resolve().parent / "difforum_kaleidoscope.gif" - - -def _post(p, d): - r = urllib.request.Request(BASE + p, data=json.dumps(d).encode(), - headers={"Content-Type": "application/json"}) - return json.loads(urllib.request.urlopen(r, timeout=30).read()) - - -def _get(p): - return json.loads(urllib.request.urlopen(BASE + p, timeout=30).read()) - - -def _getbin(p): - return urllib.request.urlopen(BASE + p, timeout=30).read() - - -def wait_ready(t=240): - t0 = time.time() - while time.time() - t0 < t: - try: - if "DifforumEchoTrails" in _get("/object_info"): - return print("server ready") - except Exception: - pass - time.sleep(2) - raise SystemExit("server not ready") - - -def graph(): - return { - "1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": CKPT}}, - "2": {"class_type": "CLIPTextEncode", "inputs": {"text": FRAME0, "clip": ["1", 1]}}, - "3": {"class_type": "CLIPTextEncode", "inputs": {"text": NEG, "clip": ["1", 1]}}, - "4": {"class_type": "EmptyLatentImage", "inputs": {"width": W, "height": H, "batch_size": 1}}, - "5": {"class_type": "KSampler", "inputs": {"model": ["1", 0], "positive": ["2", 0], "negative": ["3", 0], - "latent_image": ["4", 0], "seed": 7, "steps": 22, "cfg": 7.5, "sampler_name": "euler", "scheduler": "normal", "denoise": 1.0}}, - "6": {"class_type": "VAEDecode", "inputs": {"samples": ["5", 0], "vae": ["1", 2]}}, - "7": {"class_type": "DifforumAnimSetup", "inputs": {"width": W, "height": H, "fps": 12, "max_frames": N, "seed": 7}}, - "8": {"class_type": "DifforumCamera", "inputs": {"params": ["7", 0], "mode": "2d", "fov": 40.0, - "translation_x": "0:(0)", "translation_y": "0:(0)", "translation_z": "0:(0)", - "rotation_3d_x": "0:(0)", "rotation_3d_y": "0:(0)", "rotation_3d_z": "0:(0.7)", "zoom": "0:(1.012)"}}, - "9": {"class_type": "DifforumSchedule", "inputs": {"params": ["7", 0], "schedule": "0:(0.42)", "easing": "linear"}}, - "10": {"class_type": "DifforumFeedbackSampler", "inputs": {"model": ["1", 0], "positive": ["2", 0], "negative": ["3", 0], - "vae": ["1", 2], "params": ["7", 0], "camera": ["8", 0], "init_image": ["6", 0], - "strength_schedule": ["9", 0], - "steps": 22, "cfg": 7.5, "sampler_name": "euler", "scheduler": "normal", - "color_coherence": 0.85, "color_mode": "lab", - "symmetry": "kaleidoscope", "symmetry_segments": 6}}, - "13": {"class_type": "DifforumEchoTrails", "inputs": {"frames": ["10", 0], "decay": 0.55, "mix": 0.35}}, - "11": {"class_type": "SaveImage", "inputs": {"images": ["13", 0], "filename_prefix": "Difforum_kaleido"}}, - } - - -def build_gif(images): - from PIL import Image - frames = [] - for im in images: - q = f"/view?filename={im['filename']}&subfolder={im.get('subfolder','')}&type={im.get('type','output')}" - frames.append(Image.open(io.BytesIO(_getbin(q))).convert("RGB")) - # ping-pong so the loop is seamless - seq = frames + frames[-2:0:-1] - seq[0].save(OUT_GIF, save_all=True, append_images=seq[1:], duration=90, loop=0, optimize=True) - print(f"GIF -> {OUT_GIF} ({len(seq)} frames, {OUT_GIF.stat().st_size//1024} KB)") - - -def main(): - wait_ready() - pid = _post("/prompt", {"prompt": graph()})["prompt_id"] - print(f"submitted {pid}; rendering {N} frames @ {W}x{H} (kaleidoscope)...") - t0 = time.time() - while True: - h = _get(f"/history/{pid}") - if pid in h and ("outputs" in h[pid] or h[pid].get("status", {}).get("completed")): - imgs = h[pid].get("outputs", {}).get("11", {}).get("images", []) - print(f"render DONE in {time.time()-t0:.1f}s - {len(imgs)} frames") - if imgs: - build_gif(imgs) - return - if time.time() - t0 > 900: - raise SystemExit("timeout") - time.sleep(2) - - -if __name__ == "__main__": - main() diff --git a/examples/_smoke_render_promo.py b/examples/_smoke_render_promo.py deleted file mode 100644 index 9100c78..0000000 --- a/examples/_smoke_render_promo.py +++ /dev/null @@ -1,131 +0,0 @@ -"""Promo reel for Difforum: a 16:9 cosmic prompt-travel journey with camera -motion, exported to MP4 (social) and GIF (README). Drives a running ComfyUI. -Usage: python _smoke_render_promo.py [port]""" -import io -import json -import subprocess -import sys -import time -import urllib.request -from pathlib import Path - -PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 8199 -BASE = f"http://127.0.0.1:{PORT}" -CKPT = "v1-5-pruned-emaonly-fp16.safetensors" -PROMPTS = ( - "0: ornate symmetric mandala, stained glass, iridescent jewel tones, intricate fractal detail, glowing\n" - "12: glowing sacred geometry, golden spirals, luminous neon lines, deep detail\n" - "24: a blooming fractal flower of life, emerald and gold, ornate filigree\n" - "36: a radiant rose window cathedral mandala, ruby and sapphire light\n" - "47: a cosmic mandala of fire and light, vivid blooming symmetry, hypnotic" -) -FRAME0 = "ornate symmetric mandala, stained glass, iridescent jewel tones, intricate fractal detail, glowing" -NEG = "blurry, low quality, watermark, text, jpeg artifacts, deformed, asymmetric, washed out" -W, H, N = 768, 432, 48 -HERE = Path(__file__).resolve().parent -TMP = HERE / "_promo_frames" -MP4 = HERE / "difforum_promo.mp4" -GIF = HERE / "difforum_promo.gif" - - -def _post(p, d): - r = urllib.request.Request(BASE + p, data=json.dumps(d).encode(), - headers={"Content-Type": "application/json"}) - return json.loads(urllib.request.urlopen(r, timeout=30).read()) - - -def _get(p): - return json.loads(urllib.request.urlopen(BASE + p, timeout=30).read()) - - -def _getbin(p): - return urllib.request.urlopen(BASE + p, timeout=30).read() - - -def wait_ready(t=240): - t0 = time.time() - while time.time() - t0 < t: - try: - if "DifforumFeedbackSampler" in _get("/object_info"): - return print("server ready") - except Exception: - pass - time.sleep(2) - raise SystemExit("server not ready") - - -def graph(): - return { - "1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": CKPT}}, - "2": {"class_type": "CLIPTextEncode", "inputs": {"text": FRAME0, "clip": ["1", 1]}}, - "3": {"class_type": "CLIPTextEncode", "inputs": {"text": NEG, "clip": ["1", 1]}}, - "4": {"class_type": "EmptyLatentImage", "inputs": {"width": W, "height": H, "batch_size": 1}}, - "5": {"class_type": "KSampler", "inputs": {"model": ["1", 0], "positive": ["2", 0], "negative": ["3", 0], - "latent_image": ["4", 0], "seed": 11, "steps": 24, "cfg": 7.0, "sampler_name": "euler", "scheduler": "normal", "denoise": 1.0}}, - "6": {"class_type": "VAEDecode", "inputs": {"samples": ["5", 0], "vae": ["1", 2]}}, - "7": {"class_type": "DifforumAnimSetup", "inputs": {"width": W, "height": H, "fps": 12, "max_frames": N, "seed": 11}}, - "8": {"class_type": "DifforumCamera", "inputs": {"params": ["7", 0], "mode": "2d", "fov": 40.0, - "translation_x": "0:(0)", "translation_y": "0:(0)", "translation_z": "0:(0)", - "rotation_3d_x": "0:(0)", "rotation_3d_y": "0:(0)", "rotation_3d_z": "0:(0.6)", "zoom": "0:(1.012)"}}, - "9": {"class_type": "DifforumSchedule", "inputs": {"params": ["7", 0], "schedule": "0:(0.44)", "easing": "linear"}}, - "12": {"class_type": "DifforumPromptSchedule", "inputs": {"params": ["7", 0], "clip": ["1", 1], "prompts": PROMPTS, "easing": "ease_in_out"}}, - "10": {"class_type": "DifforumFeedbackSampler", "inputs": {"model": ["1", 0], "positive": ["2", 0], "negative": ["3", 0], - "vae": ["1", 2], "params": ["7", 0], "camera": ["8", 0], "init_image": ["6", 0], - "strength_schedule": ["9", 0], "positive_schedule": ["12", 0], - "steps": 24, "cfg": 7.5, "sampler_name": "euler", "scheduler": "normal", - "color_coherence": 0.8, "color_mode": "lab", - "symmetry": "kaleidoscope", "symmetry_segments": 6}}, - "13": {"class_type": "DifforumEchoTrails", "inputs": {"frames": ["10", 0], "decay": 0.5, "mix": 0.3}}, - "11": {"class_type": "SaveImage", "inputs": {"images": ["13", 0], "filename_prefix": "Difforum_promo"}}, - } - - -def _ffmpeg(): - import imageio_ffmpeg - return imageio_ffmpeg.get_ffmpeg_exe() - - -def export(images): - from PIL import Image - TMP.mkdir(exist_ok=True) - for f in TMP.glob("*.png"): - f.unlink() - for i, im in enumerate(images): - q = f"/view?filename={im['filename']}&subfolder={im.get('subfolder','')}&type={im.get('type','output')}" - Image.open(io.BytesIO(_getbin(q))).convert("RGB").save(TMP / f"f_{i:04d}.png") - ff = _ffmpeg() - # MP4: upscale to 720p, blend-interpolate 12 -> 24 fps for smoothness, h264 - subprocess.run([ff, "-y", "-framerate", "12", "-i", str(TMP / "f_%04d.png"), - "-vf", "scale=1280:720:flags=lanczos,minterpolate=fps=24:mi_mode=blend", - "-c:v", "libx264", "-crf", "18", "-pix_fmt", "yuv420p", str(MP4)], check=True) - print(f"MP4 -> {MP4} ({MP4.stat().st_size//1024} KB)") - # GIF: ping-pong, palette, for the README - pal = HERE / "_palette.png" - subprocess.run([ff, "-y", "-i", str(TMP / "f_%04d.png"), "-update", "1", - "-vf", "scale=512:288:flags=lanczos,palettegen=max_colors=96", str(pal)], check=True) - subprocess.run([ff, "-y", "-framerate", "12", "-i", str(TMP / "f_%04d.png"), "-i", str(pal), - "-lavfi", "scale=512:288:flags=lanczos[x];[x][1:v]paletteuse", str(GIF)], check=True) - pal.unlink(missing_ok=True) - print(f"GIF -> {GIF} ({GIF.stat().st_size//1024} KB)") - - -def main(): - wait_ready() - pid = _post("/prompt", {"prompt": graph()})["prompt_id"] - print(f"submitted {pid}; rendering {N} frames @ {W}x{H} (promo)...") - t0 = time.time() - while True: - h = _get(f"/history/{pid}") - if pid in h and ("outputs" in h[pid] or h[pid].get("status", {}).get("completed")): - imgs = h[pid].get("outputs", {}).get("11", {}).get("images", []) - print(f"render DONE in {time.time()-t0:.1f}s - {len(imgs)} frames") - if imgs: - export(imgs) - return - if time.time() - t0 > 1200: - raise SystemExit("timeout") - time.sleep(2) - - -if __name__ == "__main__": - main() diff --git a/examples/_smoke_render_pt.py b/examples/_smoke_render_pt.py deleted file mode 100644 index 4c10dfb..0000000 --- a/examples/_smoke_render_pt.py +++ /dev/null @@ -1,113 +0,0 @@ -""" -Smoke render WITH prompt travel: drives a running ComfyUI to render a short -Classic+ animation whose prompt morphs across keyframes (forest -> cave -> -galaxy) while the camera moves. Validates DifforumPromptSchedule end-to-end. - -Usage: python _smoke_render_pt.py [port] -""" - -import json -import sys -import time -import urllib.request - -PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 8199 -BASE = f"http://127.0.0.1:{PORT}" -CKPT = "v1-5-pruned-emaonly-fp16.safetensors" - -PROMPTS = ("0: a serene misty forest, soft volumetric light, lush\n" - "8: a glowing crystal cave, bioluminescent, deep blues\n" - "15: a vast starry galaxy, swirling nebula, vivid colors") -FRAME0 = "a serene misty forest, soft volumetric light, lush" - - -def _post(path, payload): - data = json.dumps(payload).encode("utf-8") - req = urllib.request.Request(BASE + path, data=data, - headers={"Content-Type": "application/json"}) - return json.loads(urllib.request.urlopen(req, timeout=30).read()) - - -def _get(path): - return json.loads(urllib.request.urlopen(BASE + path, timeout=30).read()) - - -def wait_ready(timeout=180): - t0 = time.time() - while time.time() - t0 < timeout: - try: - oi = _get("/object_info") - if "DifforumPromptSchedule" in oi and "DifforumFeedbackSampler" in oi: - print("server ready; Difforum prompt-travel node registered") - return True - except Exception: - pass - time.sleep(2) - raise SystemExit("server did not become ready in time") - - -def build_prompt(): - return { - "1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": CKPT}}, - "2": {"class_type": "CLIPTextEncode", "inputs": {"text": FRAME0, "clip": ["1", 1]}}, - "3": {"class_type": "CLIPTextEncode", - "inputs": {"text": "blurry, low quality, watermark, text", "clip": ["1", 1]}}, - "4": {"class_type": "EmptyLatentImage", "inputs": {"width": 512, "height": 512, "batch_size": 1}}, - "5": {"class_type": "KSampler", - "inputs": {"model": ["1", 0], "positive": ["2", 0], "negative": ["3", 0], - "latent_image": ["4", 0], "seed": 11, "steps": 16, "cfg": 7.0, - "sampler_name": "euler", "scheduler": "normal", "denoise": 1.0}}, - "6": {"class_type": "VAEDecode", "inputs": {"samples": ["5", 0], "vae": ["1", 2]}}, - "7": {"class_type": "DifforumAnimSetup", - "inputs": {"width": 512, "height": 512, "fps": 12, "max_frames": 16, "seed": 11}}, - "8": {"class_type": "DifforumCamera", - "inputs": {"params": ["7", 0], "mode": "2d", "fov": 40.0, - "translation_x": "0:(2)", "translation_y": "0:(0)", "translation_z": "0:(0)", - "rotation_3d_x": "0:(0)", "rotation_3d_y": "0:(0)", "rotation_3d_z": "0:(0.7)", - "zoom": "0:(1.025)"}}, - "9": {"class_type": "DifforumSchedule", - "inputs": {"params": ["7", 0], "schedule": "0:(0.55)", "easing": "linear"}}, - "12": {"class_type": "DifforumPromptSchedule", - "inputs": {"params": ["7", 0], "clip": ["1", 1], "prompts": PROMPTS, - "easing": "ease_in_out"}}, - "10": {"class_type": "DifforumFeedbackSampler", - "inputs": {"model": ["1", 0], "positive": ["2", 0], "negative": ["3", 0], - "vae": ["1", 2], "params": ["7", 0], "camera": ["8", 0], - "init_image": ["6", 0], "strength_schedule": ["9", 0], - "positive_schedule": ["12", 0], - "steps": 16, "cfg": 7.0, "sampler_name": "euler", - "scheduler": "normal", "color_coherence": 0.7}}, - "11": {"class_type": "SaveImage", - "inputs": {"images": ["10", 0], "filename_prefix": "Difforum_pt"}}, - } - - -def main(): - wait_ready() - res = _post("/prompt", {"prompt": build_prompt()}) - pid = res["prompt_id"] - print(f"submitted {pid}; rendering 16 frames with prompt travel...") - t0 = time.time() - while True: - hist = _get(f"/history/{pid}") - if pid in hist: - entry = hist[pid] - status = entry.get("status", {}) - if status.get("completed") or "outputs" in entry: - imgs = entry.get("outputs", {}).get("11", {}).get("images", []) - print(f"DONE in {time.time()-t0:.1f}s - {len(imgs)} frames") - for im in imgs: - print(f" {im['filename']}") - if not imgs: - print("status:", json.dumps(status, indent=2)[:2000]) - return - if status.get("status_str") == "error": - print("ERROR:", json.dumps(status, indent=2)[:3000]) - return - if time.time() - t0 > 600: - raise SystemExit("render timed out") - time.sleep(2) - - -if __name__ == "__main__": - main() diff --git a/examples/_smoke_render_sdxl.py b/examples/_smoke_render_sdxl.py deleted file mode 100644 index 40ab0e4..0000000 --- a/examples/_smoke_render_sdxl.py +++ /dev/null @@ -1,156 +0,0 @@ -"""SDXL high-quality re-render of the showcase visuals (1024x576, dpmpp_2m -karras). Renders two clips: the kaleidoscope promo (sacred-geometry travel) and -a clean cosmic prompt-travel showcase. Exports MP4 + GIF for each. -Drives a running ComfyUI. Usage: python _smoke_render_sdxl.py [port]""" -import io -import json -import subprocess -import sys -import time -import urllib.request -from pathlib import Path - -PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 8199 -BASE = f"http://127.0.0.1:{PORT}" -CKPT = "juggernautXL_v9.safetensors" -W, H = 1024, 576 -STEPS, CFG, SAMPLER, SCHED = 28, 6.5, "dpmpp_2m", "karras" -QUAL = ", highly detailed, sharp focus, intricate, cinematic lighting, vivid colors, masterpiece" -NEG = ("blurry, low quality, watermark, text, jpeg artifacts, deformed, washed out, " - "oversaturated, plain background, low contrast") -HERE = Path(__file__).resolve().parent - -JOBS = [ - { - "name": "promo", "n": 40, "symmetry": "kaleidoscope", "segments": 6, - "strength": "0:(0.44)", "mode": "2d", "rot_z": "0:(0.6)", "zoom": "0:(1.012)", - "tz": "0:(0)", "ty": "0:(0)", "echo": (0.5, 0.3), - "frame0": "ornate symmetric mandala, stained glass, iridescent jewel tones, intricate fractal detail, glowing", - "prompts": ( - "0: ornate symmetric mandala, stained glass, iridescent jewel tones, intricate fractal detail, glowing\n" - "10: glowing sacred geometry, golden spirals, luminous neon lines, deep detail\n" - "20: a blooming fractal flower of life, emerald and gold, ornate filigree\n" - "30: a radiant rose window cathedral mandala, ruby and sapphire light\n" - "39: a cosmic mandala of fire and light, vivid blooming symmetry, hypnotic"), - }, - { - "name": "showcase", "n": 40, "symmetry": "none", "segments": 6, - "strength": "0:(0.45)", "mode": "3d", "rot_z": "0:(0.2)", "zoom": "0:(1.02)", - "tz": "0:(1.2)", "ty": "0:(0)", "echo": (0.5, 0.15), - "frame0": "a vast purple nebula, distant stars, volumetric cosmic dust, deep space, cinematic", - "prompts": ( - "0: a vast purple nebula, distant stars, volumetric cosmic dust, deep space, cinematic\n" - "10: a swirling spiral galaxy edge on, glowing core, dense star clusters\n" - "20: the bright accretion disk of a black hole, gravitational lensing, intense light\n" - "30: a newborn star igniting, brilliant rays piercing the dark, lens flare\n" - "39: a radiant cosmic vista, glowing nebulae and stars, awe, epic scale"), - }, -] - - -def _post(p, d): - r = urllib.request.Request(BASE + p, data=json.dumps(d).encode(), headers={"Content-Type": "application/json"}) - return json.loads(urllib.request.urlopen(r, timeout=30).read()) - - -def _get(p): - return json.loads(urllib.request.urlopen(BASE + p, timeout=30).read()) - - -def _getbin(p): - return urllib.request.urlopen(BASE + p, timeout=60).read() - - -def wait_ready(t=300): - t0 = time.time() - while time.time() - t0 < t: - try: - if CKPT in json.dumps(_get("/object_info").get("CheckpointLoaderSimple", {})): - return print("server ready (checkpoint visible)") - except Exception: - pass - time.sleep(2) - raise SystemExit("server/checkpoint not ready") - - -def graph(j): - return { - "1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": CKPT}}, - "2": {"class_type": "CLIPTextEncode", "inputs": {"text": j["frame0"] + QUAL, "clip": ["1", 1]}}, - "3": {"class_type": "CLIPTextEncode", "inputs": {"text": NEG, "clip": ["1", 1]}}, - "4": {"class_type": "EmptyLatentImage", "inputs": {"width": W, "height": H, "batch_size": 1}}, - "5": {"class_type": "KSampler", "inputs": {"model": ["1", 0], "positive": ["2", 0], "negative": ["3", 0], - "latent_image": ["4", 0], "seed": 21, "steps": STEPS, "cfg": CFG, "sampler_name": SAMPLER, "scheduler": SCHED, "denoise": 1.0}}, - "6": {"class_type": "VAEDecode", "inputs": {"samples": ["5", 0], "vae": ["1", 2]}}, - "7": {"class_type": "DifforumAnimSetup", "inputs": {"width": W, "height": H, "fps": 12, "max_frames": j["n"], "seed": 21}}, - "8": {"class_type": "DifforumCamera", "inputs": {"params": ["7", 0], "mode": j["mode"], "fov": 45.0, - "translation_x": "0:(0)", "translation_y": j["ty"], "translation_z": j["tz"], - "rotation_3d_x": "0:(0)", "rotation_3d_y": "0:(0)", "rotation_3d_z": j["rot_z"], "zoom": j["zoom"]}}, - "9": {"class_type": "DifforumSchedule", "inputs": {"params": ["7", 0], "schedule": j["strength"], "easing": "linear"}}, - "12": {"class_type": "DifforumPromptSchedule", "inputs": {"params": ["7", 0], "clip": ["1", 1], "prompts": j["prompts"], "easing": "ease_in_out"}}, - "10": {"class_type": "DifforumFeedbackSampler", "inputs": {"model": ["1", 0], "positive": ["2", 0], "negative": ["3", 0], - "vae": ["1", 2], "params": ["7", 0], "camera": ["8", 0], "init_image": ["6", 0], - "strength_schedule": ["9", 0], "positive_schedule": ["12", 0], - "steps": STEPS, "cfg": CFG, "sampler_name": SAMPLER, "scheduler": SCHED, - "color_coherence": 0.8, "color_mode": "lab", - "symmetry": j["symmetry"], "symmetry_segments": j["segments"]}}, - "13": {"class_type": "DifforumEchoTrails", "inputs": {"frames": ["10", 0], "decay": j["echo"][0], "mix": j["echo"][1]}}, - "11": {"class_type": "SaveImage", "inputs": {"images": ["13", 0], "filename_prefix": "Difforum_sdxl_" + j["name"]}}, - } - - -def _ffmpeg(): - import imageio_ffmpeg - return imageio_ffmpeg.get_ffmpeg_exe() - - -def export(name, images): - from PIL import Image - tmp = HERE / f"_sdxl_{name}" - tmp.mkdir(exist_ok=True) - for f in tmp.glob("*.png"): - f.unlink() - for i, im in enumerate(images): - q = f"/view?filename={im['filename']}&subfolder={im.get('subfolder','')}&type={im.get('type','output')}" - Image.open(io.BytesIO(_getbin(q))).convert("RGB").save(tmp / f"f_{i:04d}.png") - ff = _ffmpeg() - mp4 = HERE / f"difforum_{name}.mp4" - gif = HERE / f"difforum_{name}.gif" - pal = HERE / "_pal.png" - subprocess.run([ff, "-y", "-framerate", "12", "-i", str(tmp / "f_%04d.png"), - "-vf", "scale=1280:720:flags=lanczos,minterpolate=fps=24:mi_mode=blend", - "-c:v", "libx264", "-crf", "17", "-pix_fmt", "yuv420p", str(mp4)], check=True) - subprocess.run([ff, "-y", "-i", str(tmp / "f_%04d.png"), "-update", "1", - "-vf", "scale=600:338:flags=lanczos,palettegen=max_colors=128", str(pal)], check=True) - subprocess.run([ff, "-y", "-framerate", "12", "-i", str(tmp / "f_%04d.png"), "-i", str(pal), - "-lavfi", "scale=600:338:flags=lanczos[x];[x][1:v]paletteuse", str(gif)], check=True) - pal.unlink(missing_ok=True) - print(f" {name}: MP4 {mp4.stat().st_size//1024}KB GIF {gif.stat().st_size//1024}KB") - - -def run(j): - pid = _post("/prompt", {"prompt": graph(j)})["prompt_id"] - print(f"[{j['name']}] submitted {pid}; {j['n']} frames @ {W}x{H}...") - t0 = time.time() - while True: - h = _get(f"/history/{pid}") - if pid in h and ("outputs" in h[pid] or h[pid].get("status", {}).get("completed")): - imgs = h[pid].get("outputs", {}).get("11", {}).get("images", []) - print(f"[{j['name']}] render done in {time.time()-t0:.1f}s - {len(imgs)} frames") - if imgs: - export(j["name"], imgs) - return - if time.time() - t0 > 2400: - raise SystemExit(f"[{j['name']}] timeout") - time.sleep(3) - - -def main(): - wait_ready() - for j in JOBS: - run(j) - print("ALL SDXL RENDERS DONE") - - -if __name__ == "__main__": - main() diff --git a/examples/_validate_workflows.py b/examples/_validate_workflows.py deleted file mode 100644 index d938375..0000000 --- a/examples/_validate_workflows.py +++ /dev/null @@ -1,97 +0,0 @@ -""" -Validate every example workflow against the REAL node definitions. - -For each *.json: confirm every node type exists, and every *required connection* -input (MODEL/CLIP/VAE/CONDITIONING/LATENT/IMAGE/DIFFORUM_* etc.) is actually -linked. Widget params (INT/FLOAT/STRING/BOOLEAN/combo) are not connections and -are skipped. Also report which external assets each workflow needs. - -Run with the ComfyUI venv python (needs ComfyUI on path for core nodes). -""" - -import glob -import json -import sys -from pathlib import Path - -sys.path.insert(0, r"D:\ComfyUI-victor") -sys.path.insert(0, r"D:\ComfyUI-victor\custom_nodes") - -import asyncio # noqa: E402 - -import nodes as comfy_nodes # ComfyUI core # noqa: E402 -import difforum # noqa: E402 - -# load comfy_extras (LoadAudio, etc.) so the check sees every built-in node -try: - asyncio.run(comfy_nodes.init_builtin_extra_nodes()) -except Exception as e: # noqa: BLE001 - print(f"warn: could not load extra nodes: {e}") - -MAPPING = dict(comfy_nodes.NODE_CLASS_MAPPINGS) -MAPPING.update(difforum.NODE_CLASS_MAPPINGS) - -WIDGET_TYPES = {"INT", "FLOAT", "STRING", "BOOLEAN", "COMBO"} -ASSET_NODES = { - "CheckpointLoaderSimple": "an SD checkpoint", - "LoadImage": "an input image", - "LoadAudio": "an audio file", - "UNETLoader": "a UNet/diffusion model", - "UnetLoaderGGUF": "a GGUF model", - "VAELoader": "a VAE", -} - -HERE = Path(__file__).resolve().parent -problems_total = 0 - -for path in sorted(glob.glob(str(HERE / "*.json"))): - name = Path(path).name - wf = json.loads(Path(path).read_text(encoding="utf-8")) - nodes_by_id = {n["id"]: n for n in wf["nodes"]} - issues = [] - assets = set() - - external = set() - for node in wf["nodes"]: - ntype = node["type"] - if ntype in ASSET_NODES: - assets.add(ASSET_NODES[ntype]) - if ntype in ("Note", "MarkdownNote", "Reroute", "PrimitiveNode"): - continue # built-in frontend-only nodes, always present - cls = MAPPING.get(ntype) - if cls is None: - # not in this dev env: a third-party dep (RIFE, VHS, etc.), not a - # hard error - just note it must be installed where the wf runs. - external.add(ntype) - continue - try: - spec = cls.INPUT_TYPES() - except Exception as e: # noqa: BLE001 - issues.append(f"{ntype}: INPUT_TYPES failed ({e})") - continue - required = spec.get("required", {}) - # map this node's input slots by name -> link - slot_link = {inp["name"]: inp.get("link") for inp in node.get("inputs", [])} - for iname, ispec in required.items(): - itype = ispec[0] - if isinstance(itype, list) or itype in WIDGET_TYPES: - continue # widget, not a connection - # required connection -> must have a linked input slot - if iname not in slot_link: - issues.append(f"{ntype}: required input '{iname}' ({itype}) has no slot") - elif slot_link[iname] is None: - issues.append(f"{ntype}: required input '{iname}' ({itype}) NOT connected") - - status = "OK" if not issues else f"{len(issues)} ISSUE(S)" - print(f"\n=== {name} === [{status}]") - print(f" nodes: {len(wf['nodes'])} links: {len(wf['links'])}") - print(f" needs: {', '.join(sorted(assets)) if assets else 'nothing (runs anywhere)'}") - if external: - print(f" external nodes (must be installed): {', '.join(sorted(external))}") - for i in issues: - print(f" ! {i}") - problems_total += len(issues) - -print("\n" + "=" * 50) -print("ALL WORKFLOWS VALID" if problems_total == 0 else f"TOTAL ISSUES: {problems_total}") -sys.exit(1 if problems_total else 0) diff --git a/examples/advanced/difforum_animatediff_sd15.json b/examples/advanced/difforum_animatediff_sd15.json deleted file mode 100644 index 19fc3a2..0000000 --- a/examples/advanced/difforum_animatediff_sd15.json +++ /dev/null @@ -1,614 +0,0 @@ -{ - "last_node_id": 12, - "last_link_id": 13, - "nodes": [ - { - "id": 1, - "type": "CheckpointLoaderSimple", - "pos": [ - 40, - 40 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 5 - ], - "slot_index": 0 - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [ - 2, - 3 - ], - "slot_index": 1 - }, - { - "name": "VAE", - "type": "VAE", - "links": [ - 12 - ], - "slot_index": 2 - } - ], - "properties": { - "Node name for S&R": "CheckpointLoaderSimple" - }, - "widgets_values": [ - "dreamshaper_8.safetensors" - ] - }, - { - "id": 2, - "type": "CLIPTextEncode", - "pos": [ - 360, - 220 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 3 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 9 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "blurry, low quality, watermark, text" - ] - }, - { - "id": 3, - "type": "DifforumAnimSetup", - "pos": [ - 40, - 360 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "links": [ - 1 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumAnimSetup" - }, - "widgets_values": [ - 512, - 512, - 12, - 16, - 0, - "fixed" - ] - }, - { - "id": 4, - "type": "DifforumPromptScenes", - "pos": [ - 360, - 380 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 1 - }, - { - "name": "clip", - "type": "CLIP", - "link": 2 - } - ], - "outputs": [ - { - "name": "positive_schedule", - "type": "DIFFORUM_PROMPT", - "links": [ - 4 - ], - "slot_index": 0 - }, - { - "name": "first_frame_cond", - "type": "CONDITIONING", - "links": [], - "slot_index": 1 - }, - { - "name": "info", - "type": "STRING", - "links": [], - "slot_index": 2 - } - ], - "properties": { - "Node name for S&R": "DifforumPromptScenes" - }, - "widgets_values": [ - "a serene koi pond, lily pads", - "an autumn forest path, falling leaves", - "a snowy mountain peak at dawn", - "", - "ease_in_out" - ] - }, - { - "id": 5, - "type": "DifforumPromptBatch", - "pos": [ - 720, - 380 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "positive_schedule", - "type": "DIFFORUM_PROMPT", - "link": 4 - } - ], - "outputs": [ - { - "name": "conditioning", - "type": "CONDITIONING", - "links": [ - 8 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumPromptBatch" - }, - "widgets_values": [] - }, - { - "id": 6, - "type": "ADE_AnimateDiffUniformContextOptions", - "pos": [ - 40, - 560 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "CONTEXT_OPTIONS", - "type": "CONTEXT_OPTIONS", - "links": [ - 6 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "ADE_AnimateDiffUniformContextOptions" - }, - "widgets_values": [ - 16, - 1, - 4, - false, - "flat" - ] - }, - { - "id": 7, - "type": "ADE_AnimateDiffLoaderGen1", - "pos": [ - 360, - 560 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 5 - }, - { - "name": "context_options", - "type": "CONTEXT_OPTIONS", - "link": 6 - }, - { - "name": "motion_lora", - "type": "MOTION_LORA", - "link": null - } - ], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 7 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "ADE_AnimateDiffLoaderGen1" - }, - "widgets_values": [ - "v3_sd15_mm.ckpt", - "autoselect" - ] - }, - { - "id": 8, - "type": "EmptyLatentImage", - "pos": [ - 360, - 760 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 10 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "EmptyLatentImage" - }, - "widgets_values": [ - 512, - 512, - 16 - ] - }, - { - "id": 9, - "type": "KSampler", - "pos": [ - 760, - 560 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 7 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 8 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 9 - }, - { - "name": "latent_image", - "type": "LATENT", - "link": 10 - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 11 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "KSampler" - }, - "widgets_values": [ - 0, - "fixed", - 26, - 6.5, - "dpmpp_2m", - "karras", - 1.0 - ] - }, - { - "id": 10, - "type": "VAEDecode", - "pos": [ - 1120, - 560 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 9, - "mode": 0, - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 11 - }, - { - "name": "vae", - "type": "VAE", - "link": 12 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 13 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "VAEDecode" - }, - "widgets_values": [] - }, - { - "id": 11, - "type": "VHS_VideoCombine", - "pos": [ - 1400, - 560 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 10, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 13 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "VHS_VideoCombine" - }, - "widgets_values": [ - 12, - 0, - "Difforum_ADE", - "video/h264-mp4", - false, - true - ] - }, - { - "id": 12, - "type": "Note", - "pos": [ - 40, - -210 - ], - "size": [ - 460, - 300 - ], - "flags": {}, - "order": 11, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "READ ME - models needed", - "properties": { - "text": "" - }, - "widgets_values": [ - "Difforum: AnimateDiff bridge (native motion).\n\nModels:\n- SD1.5 checkpoint -> models/checkpoints\n- motion module mm_sd15_v3.ckpt -> models/animatediff_models (hf: guoyww/animatediff)\n- node: ComfyUI-AnimateDiff-Evolved" - ], - "color": "#432", - "bgcolor": "#653" - } - ], - "links": [ - [ - 1, - 3, - 0, - 4, - 0, - "DIFFORUM_PARAMS" - ], - [ - 2, - 1, - 1, - 4, - 1, - "CLIP" - ], - [ - 3, - 1, - 1, - 2, - 0, - "CLIP" - ], - [ - 4, - 4, - 0, - 5, - 0, - "DIFFORUM_PROMPT" - ], - [ - 5, - 1, - 0, - 7, - 0, - "MODEL" - ], - [ - 6, - 6, - 0, - 7, - 1, - "CONTEXT_OPTIONS" - ], - [ - 7, - 7, - 0, - 9, - 0, - "MODEL" - ], - [ - 8, - 5, - 0, - 9, - 1, - "CONDITIONING" - ], - [ - 9, - 2, - 0, - 9, - 2, - "CONDITIONING" - ], - [ - 10, - 8, - 0, - 9, - 3, - "LATENT" - ], - [ - 11, - 9, - 0, - 10, - 0, - "LATENT" - ], - [ - 12, - 1, - 2, - 10, - 1, - "VAE" - ], - [ - 13, - 10, - 0, - 11, - 0, - "IMAGE" - ] - ], - "groups": [], - "config": {}, - "extra": {}, - "version": 0.4 -} \ No newline at end of file diff --git a/examples/advanced/difforum_audio_reactive_video.json b/examples/advanced/difforum_audio_reactive_video.json deleted file mode 100644 index ad53d4a..0000000 --- a/examples/advanced/difforum_audio_reactive_video.json +++ /dev/null @@ -1,913 +0,0 @@ -{ - "last_node_id": 15, - "last_link_id": 24, - "nodes": [ - { - "id": 1, - "type": "CheckpointLoaderSimple", - "pos": [ - 40, - 40 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 1, - 15 - ], - "slot_index": 0 - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [ - 2, - 3 - ], - "slot_index": 1 - }, - { - "name": "VAE", - "type": "VAE", - "links": [ - 8, - 18 - ], - "slot_index": 2 - } - ], - "properties": { - "Node name for S&R": "CheckpointLoaderSimple" - }, - "widgets_values": [ - "dreamshaper_8.safetensors" - ] - }, - { - "id": 2, - "type": "CLIPTextEncode", - "pos": [ - 340, - 20 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 2 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 4, - 16 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "neon synthwave city, glowing grid, vivid" - ] - }, - { - "id": 3, - "type": "CLIPTextEncode", - "pos": [ - 340, - 200 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 3 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 5, - 17 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "blurry, low quality" - ] - }, - { - "id": 4, - "type": "EmptyLatentImage", - "pos": [ - 340, - 360 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 6 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "EmptyLatentImage" - }, - "widgets_values": [ - 768, - 432, - 1 - ] - }, - { - "id": 5, - "type": "KSampler", - "pos": [ - 660, - 120 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 1 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 4 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 5 - }, - { - "name": "latent_image", - "type": "LATENT", - "link": 6 - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 7 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "KSampler" - }, - "widgets_values": [ - 0, - "fixed", - 26, - 6.5, - "dpmpp_2m", - "karras", - 1.0 - ] - }, - { - "id": 6, - "type": "VAEDecode", - "pos": [ - 980, - 120 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 7 - }, - { - "name": "vae", - "type": "VAE", - "link": 8 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 21 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "VAEDecode" - }, - "widgets_values": [] - }, - { - "id": 7, - "type": "DifforumAnimSetup", - "pos": [ - 40, - 420 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "links": [ - 9, - 11, - 13, - 19 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumAnimSetup" - }, - "widgets_values": [ - 768, - 432, - 24, - 96, - 0, - "fixed" - ] - }, - { - "id": 8, - "type": "LoadAudio", - "pos": [ - 40, - 640 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "AUDIO", - "type": "AUDIO", - "links": [ - 10 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "LoadAudio" - }, - "widgets_values": [ - "your_track.mp3" - ] - }, - { - "id": 9, - "type": "DifforumAudioAnalyzer", - "pos": [ - 340, - 640 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 9 - }, - { - "name": "audio", - "type": "AUDIO", - "link": 10 - } - ], - "outputs": [ - { - "name": "audio_curves", - "type": "DIFFORUM_AUDIO", - "links": [ - 12, - 14 - ], - "slot_index": 0 - }, - { - "name": "amp", - "type": "FLOAT", - "links": [], - "slot_index": 1 - }, - { - "name": "low", - "type": "FLOAT", - "links": [], - "slot_index": 2 - }, - { - "name": "mid", - "type": "FLOAT", - "links": [], - "slot_index": 3 - }, - { - "name": "high", - "type": "FLOAT", - "links": [], - "slot_index": 4 - }, - { - "name": "onset", - "type": "FLOAT", - "links": [], - "slot_index": 5 - }, - { - "name": "beat", - "type": "FLOAT", - "links": [], - "slot_index": 6 - } - ], - "properties": { - "Node name for S&R": "DifforumAudioAnalyzer" - }, - "widgets_values": [ - 0.25, - true, - 1.5 - ] - }, - { - "id": 10, - "type": "DifforumAudioSchedule", - "pos": [ - 700, - 640 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 9, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 11 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": 12 - } - ], - "outputs": [ - { - "name": "schedule", - "type": "DIFFORUM_SCHEDULE", - "links": [ - 22 - ], - "slot_index": 0 - }, - { - "name": "values", - "type": "FLOAT", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumAudioSchedule" - }, - "widgets_values": [ - "beat", - "add", - 0.4, - 0.35, - 0.2 - ] - }, - { - "id": 11, - "type": "DifforumCamera", - "pos": [ - 340, - 860 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 10, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 13 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": 14 - } - ], - "outputs": [ - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "links": [ - 20 - ], - "slot_index": 0 - }, - { - "name": "info", - "type": "STRING", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumCamera" - }, - "widgets_values": [ - "2d", - 40.0, - "0:(0)", - "0:(0)", - "0:(0)", - "0:(0)", - "0:(0.3)", - "0:(1.0 + 0.06*low)" - ] - }, - { - "id": 12, - "type": "DifforumFeedbackSampler", - "pos": [ - 1280, - 360 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 11, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 15 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 16 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 17 - }, - { - "name": "vae", - "type": "VAE", - "link": 18 - }, - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 19 - }, - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "link": 20 - }, - { - "name": "init_image", - "type": "IMAGE", - "link": 21 - }, - { - "name": "strength_schedule", - "type": "DIFFORUM_SCHEDULE", - "link": 22 - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 23 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumFeedbackSampler" - }, - "widgets_values": [ - 24, - 6.5, - "dpmpp_2m", - "karras", - 0.8, - "lab", - 1.0, - 100.0, - false, - 1.0, - 0.6 - ] - }, - { - "id": 13, - "type": "VHS_VideoCombine", - "pos": [ - 1640, - 360 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 24 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "VHS_VideoCombine" - }, - "widgets_values": [ - 24, - 0, - "Difforum_audio", - "video/h264-mp4", - false, - true - ] - }, - { - "id": 14, - "type": "DifforumFlowStabilize", - "pos": [ - 1600, - 360 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 13, - "mode": 0, - "inputs": [ - { - "name": "frames", - "type": "IMAGE", - "link": 23 - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 24 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumFlowStabilize" - }, - "widgets_values": [ - 0.5, - 0.5, - 0.15 - ] - }, - { - "id": 15, - "type": "Note", - "pos": [ - 40, - -210 - ], - "size": [ - 460, - 300 - ], - "flags": {}, - "order": 14, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "READ ME - models needed", - "properties": { - "text": "" - }, - "widgets_values": [ - "Difforum: Audio-reactive video (bass->zoom, beat->strength).\n\nModels:\n- SD1.5 checkpoint -> models/checkpoints\n- an audio file (.mp3/.wav) -> Load Audio\n- node: ComfyUI-VideoHelperSuite (for the MP4)" - ], - "color": "#432", - "bgcolor": "#653" - } - ], - "links": [ - [ - 1, - 1, - 0, - 5, - 0, - "MODEL" - ], - [ - 2, - 1, - 1, - 2, - 0, - "CLIP" - ], - [ - 3, - 1, - 1, - 3, - 0, - "CLIP" - ], - [ - 4, - 2, - 0, - 5, - 1, - "CONDITIONING" - ], - [ - 5, - 3, - 0, - 5, - 2, - "CONDITIONING" - ], - [ - 6, - 4, - 0, - 5, - 3, - "LATENT" - ], - [ - 7, - 5, - 0, - 6, - 0, - "LATENT" - ], - [ - 8, - 1, - 2, - 6, - 1, - "VAE" - ], - [ - 9, - 7, - 0, - 9, - 0, - "DIFFORUM_PARAMS" - ], - [ - 10, - 8, - 0, - 9, - 1, - "AUDIO" - ], - [ - 11, - 7, - 0, - 10, - 0, - "DIFFORUM_PARAMS" - ], - [ - 12, - 9, - 0, - 10, - 1, - "DIFFORUM_AUDIO" - ], - [ - 13, - 7, - 0, - 11, - 0, - "DIFFORUM_PARAMS" - ], - [ - 14, - 9, - 0, - 11, - 1, - "DIFFORUM_AUDIO" - ], - [ - 15, - 1, - 0, - 12, - 0, - "MODEL" - ], - [ - 16, - 2, - 0, - 12, - 1, - "CONDITIONING" - ], - [ - 17, - 3, - 0, - 12, - 2, - "CONDITIONING" - ], - [ - 18, - 1, - 2, - 12, - 3, - "VAE" - ], - [ - 19, - 7, - 0, - 12, - 4, - "DIFFORUM_PARAMS" - ], - [ - 20, - 11, - 0, - 12, - 5, - "DIFFORUM_CAMERA" - ], - [ - 21, - 6, - 0, - 12, - 6, - "IMAGE" - ], - [ - 22, - 10, - 0, - 12, - 7, - "DIFFORUM_SCHEDULE" - ], - [ - 23, - 12, - 0, - 14, - 0, - "IMAGE" - ], - [ - 24, - 14, - 0, - 13, - 0, - "IMAGE" - ] - ], - "groups": [], - "config": {}, - "extra": {}, - "version": 0.4 -} \ No newline at end of file diff --git a/examples/advanced/difforum_deluxe_travel_controlnet_video.json b/examples/advanced/difforum_deluxe_travel_controlnet_video.json deleted file mode 100644 index 19f096b..0000000 --- a/examples/advanced/difforum_deluxe_travel_controlnet_video.json +++ /dev/null @@ -1,952 +0,0 @@ -{ - "last_node_id": 16, - "last_link_id": 25, - "nodes": [ - { - "id": 1, - "type": "CheckpointLoaderSimple", - "pos": [ - 40, - 40 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 1, - 13 - ], - "slot_index": 0 - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [ - 2, - 3, - 12 - ], - "slot_index": 1 - }, - { - "name": "VAE", - "type": "VAE", - "links": [ - 8, - 16 - ], - "slot_index": 2 - } - ], - "properties": { - "Node name for S&R": "CheckpointLoaderSimple" - }, - "widgets_values": [ - "dreamshaper_8.safetensors" - ] - }, - { - "id": 2, - "type": "CLIPTextEncode", - "pos": [ - 340, - 20 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 2 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 4, - 14 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "psychedelic mandala, vivid colors, intricate, highly detailed" - ] - }, - { - "id": 3, - "type": "CLIPTextEncode", - "pos": [ - 340, - 200 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 3 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 5, - 15 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "blurry, low quality, watermark, text" - ] - }, - { - "id": 4, - "type": "EmptyLatentImage", - "pos": [ - 340, - 380 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 6 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "EmptyLatentImage" - }, - "widgets_values": [ - 768, - 768, - 1 - ] - }, - { - "id": 5, - "type": "KSampler", - "pos": [ - 660, - 120 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 1 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 4 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 5 - }, - { - "name": "latent_image", - "type": "LATENT", - "link": 6 - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 7 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "KSampler" - }, - "widgets_values": [ - 0, - "fixed", - 26, - 6.5, - "dpmpp_2m", - "karras", - 1.0 - ] - }, - { - "id": 6, - "type": "VAEDecode", - "pos": [ - 980, - 120 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 7 - }, - { - "name": "vae", - "type": "VAE", - "link": 8 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 19 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "VAEDecode" - }, - "widgets_values": [] - }, - { - "id": 7, - "type": "ControlNetLoader", - "pos": [ - 40, - 280 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "CONTROL_NET", - "type": "CONTROL_NET", - "links": [ - 22 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "ControlNetLoader" - }, - "widgets_values": [ - "control_v11f1e_sd15_tile.pth" - ] - }, - { - "id": 8, - "type": "DifforumAnimSetup", - "pos": [ - 40, - 520 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "links": [ - 9, - 10, - 11, - 17 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumAnimSetup" - }, - "widgets_values": [ - 768, - 768, - 24, - 48, - 0, - "fixed" - ] - }, - { - "id": 9, - "type": "DifforumCamera", - "pos": [ - 340, - 560 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 9 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - } - ], - "outputs": [ - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "links": [ - 18 - ], - "slot_index": 0 - }, - { - "name": "info", - "type": "STRING", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumCamera" - }, - "widgets_values": [ - "2d", - 40.0, - "0:(2)", - "0:(0)", - "0:(0)", - "0:(0)", - "0:(0.6)", - "0:(0)", - "0:(1.02)" - ] - }, - { - "id": 10, - "type": "DifforumSchedule", - "pos": [ - 340, - 820 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 9, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 10 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - } - ], - "outputs": [ - { - "name": "schedule", - "type": "DIFFORUM_SCHEDULE", - "links": [ - 20 - ], - "slot_index": 0 - }, - { - "name": "values", - "type": "FLOAT", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumSchedule" - }, - "widgets_values": [ - "0:(0.5)", - "linear" - ] - }, - { - "id": 11, - "type": "DifforumPromptSchedule", - "pos": [ - 340, - 1060 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 10, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 11 - }, - { - "name": "clip", - "type": "CLIP", - "link": 12 - } - ], - "outputs": [ - { - "name": "positive_schedule", - "type": "DIFFORUM_PROMPT", - "links": [ - 21 - ], - "slot_index": 0 - }, - { - "name": "first_frame_cond", - "type": "CONDITIONING", - "links": [], - "slot_index": 1 - }, - { - "name": "info", - "type": "STRING", - "links": [], - "slot_index": 2 - } - ], - "properties": { - "Node name for S&R": "DifforumPromptSchedule" - }, - "widgets_values": [ - "0: psychedelic mandala, vivid colors\n24: cosmic fractal temple, golden light\n47: deep space nebula, glowing stars", - "ease_in_out" - ] - }, - { - "id": 12, - "type": "DifforumFeedbackSampler", - "pos": [ - 1000, - 480 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 11, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 13 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 14 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 15 - }, - { - "name": "vae", - "type": "VAE", - "link": 16 - }, - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 17 - }, - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "link": 18 - }, - { - "name": "init_image", - "type": "IMAGE", - "link": 19 - }, - { - "name": "strength_schedule", - "type": "DIFFORUM_SCHEDULE", - "link": 20 - }, - { - "name": "depth", - "type": "IMAGE", - "link": null - }, - { - "name": "cfg_schedule", - "type": "DIFFORUM_SCHEDULE", - "link": null - }, - { - "name": "positive_schedule", - "type": "DIFFORUM_PROMPT", - "link": 21 - }, - { - "name": "control_net", - "type": "CONTROL_NET", - "link": 22 - }, - { - "name": "control_image", - "type": "IMAGE", - "link": null - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 23, - 25 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumFeedbackSampler" - }, - "widgets_values": [ - 24, - 6.5, - "dpmpp_2m", - "karras", - 0.85, - "lab", - 1.0, - 100.0, - false, - 1.0, - 0.6 - ] - }, - { - "id": 13, - "type": "RIFE VFI", - "pos": [ - 1360, - 360 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [ - { - "name": "frames", - "type": "IMAGE", - "link": 23 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 24 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "RIFE VFI" - }, - "widgets_values": [ - "rife47.pth", - 10, - 2, - true, - true, - 1.0 - ] - }, - { - "id": 14, - "type": "VHS_VideoCombine", - "pos": [ - 1700, - 360 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 13, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 24 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "VHS_VideoCombine" - }, - "widgets_values": [ - 16, - 0, - "Difforum", - "video/h264-mp4", - false, - true - ] - }, - { - "id": 15, - "type": "SaveImage", - "pos": [ - 1360, - 620 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 14, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 25 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "SaveImage" - }, - "widgets_values": [ - "Difforum" - ] - }, - { - "id": 16, - "type": "Note", - "pos": [ - 40, - -210 - ], - "size": [ - 460, - 300 - ], - "flags": {}, - "order": 15, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "READ ME - models needed", - "properties": { - "text": "" - }, - "widgets_values": [ - "Difforum: Deluxe (travel + ControlNet + interpolation + MP4).\n\nModels:\n- SD1.5 checkpoint -> models/checkpoints (DreamShaper: civitai.com/models/4384)\n- tile ControlNet control_v11f1e_sd15_tile -> models/controlnet\n (hf: comfyanonymous/ControlNet-v1-1_fp16_safetensors)\n- nodes: ComfyUI-Frame-Interpolation (RIFE auto-downloads) + ComfyUI-VideoHelperSuite\nControlNet must match the base family (SD1.5 here)." - ], - "color": "#432", - "bgcolor": "#653" - } - ], - "links": [ - [ - 1, - 1, - 0, - 5, - 0, - "MODEL" - ], - [ - 2, - 1, - 1, - 2, - 0, - "CLIP" - ], - [ - 3, - 1, - 1, - 3, - 0, - "CLIP" - ], - [ - 4, - 2, - 0, - 5, - 1, - "CONDITIONING" - ], - [ - 5, - 3, - 0, - 5, - 2, - "CONDITIONING" - ], - [ - 6, - 4, - 0, - 5, - 3, - "LATENT" - ], - [ - 7, - 5, - 0, - 6, - 0, - "LATENT" - ], - [ - 8, - 1, - 2, - 6, - 1, - "VAE" - ], - [ - 9, - 8, - 0, - 9, - 0, - "DIFFORUM_PARAMS" - ], - [ - 10, - 8, - 0, - 10, - 0, - "DIFFORUM_PARAMS" - ], - [ - 11, - 8, - 0, - 11, - 0, - "DIFFORUM_PARAMS" - ], - [ - 12, - 1, - 1, - 11, - 1, - "CLIP" - ], - [ - 13, - 1, - 0, - 12, - 0, - "MODEL" - ], - [ - 14, - 2, - 0, - 12, - 1, - "CONDITIONING" - ], - [ - 15, - 3, - 0, - 12, - 2, - "CONDITIONING" - ], - [ - 16, - 1, - 2, - 12, - 3, - "VAE" - ], - [ - 17, - 8, - 0, - 12, - 4, - "DIFFORUM_PARAMS" - ], - [ - 18, - 9, - 0, - 12, - 5, - "DIFFORUM_CAMERA" - ], - [ - 19, - 6, - 0, - 12, - 6, - "IMAGE" - ], - [ - 20, - 10, - 0, - 12, - 7, - "DIFFORUM_SCHEDULE" - ], - [ - 21, - 11, - 0, - 12, - 10, - "DIFFORUM_PROMPT" - ], - [ - 22, - 7, - 0, - 12, - 11, - "CONTROL_NET" - ], - [ - 23, - 12, - 0, - 13, - 0, - "IMAGE" - ], - [ - 24, - 13, - 0, - 14, - 0, - "IMAGE" - ], - [ - 25, - 12, - 0, - 15, - 0, - "IMAGE" - ] - ], - "groups": [], - "config": {}, - "extra": {}, - "version": 0.4 -} \ No newline at end of file diff --git a/examples/advanced/difforum_fast_sdxl.json b/examples/advanced/difforum_fast_sdxl.json deleted file mode 100644 index 22c67d4..0000000 --- a/examples/advanced/difforum_fast_sdxl.json +++ /dev/null @@ -1,980 +0,0 @@ -{ - "last_node_id": 15, - "last_link_id": 25, - "nodes": [ - { - "id": 1, - "type": "CheckpointLoaderSimple", - "pos": [ - 40, - 40 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 1, - 6 - ], - "slot_index": 0 - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [ - 2, - 3, - 4 - ], - "slot_index": 1 - }, - { - "name": "VAE", - "type": "VAE", - "links": [ - 11, - 17 - ], - "slot_index": 2 - } - ], - "properties": { - "Node name for S&R": "CheckpointLoaderSimple" - }, - "widgets_values": [ - "juggernautXL_v9.safetensors" - ] - }, - { - "id": 2, - "type": "LoraLoader", - "pos": [ - 40, - 240 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 1 - }, - { - "name": "clip", - "type": "CLIP", - "link": 2 - } - ], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 14 - ], - "slot_index": 0 - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "LoraLoader" - }, - "widgets_values": [ - "dmd2_sdxl_4step_lora.safetensors", - 1.0, - 1.0 - ] - }, - { - "id": 3, - "type": "CLIPTextEncode", - "pos": [ - 340, - 200 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 3 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 8, - 16 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "blurry, low quality, watermark, text, washed out" - ] - }, - { - "id": 4, - "type": "EmptyLatentImage", - "pos": [ - 340, - 360 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 9 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "EmptyLatentImage" - }, - "widgets_values": [ - 768, - 432, - 1 - ] - }, - { - "id": 5, - "type": "DifforumPromptScenes", - "pos": [ - 340, - 520 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 5 - }, - { - "name": "clip", - "type": "CLIP", - "link": 4 - } - ], - "outputs": [ - { - "name": "positive_schedule", - "type": "DIFFORUM_PROMPT", - "links": [ - 22 - ], - "slot_index": 0 - }, - { - "name": "first_frame_cond", - "type": "CONDITIONING", - "links": [ - 7, - 15 - ], - "slot_index": 1 - }, - { - "name": "info", - "type": "STRING", - "links": [], - "slot_index": 2 - } - ], - "properties": { - "Node name for S&R": "DifforumPromptScenes" - }, - "widgets_values": [ - "a luminous coral canyon, volumetric light, highly detailed", - "a crystalline ice cathedral, refracted glow", - "a golden desert storm at dusk, cinematic", - "", - "ease_in_out" - ] - }, - { - "id": 6, - "type": "KSampler", - "pos": [ - 660, - 60 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 6 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 7 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 8 - }, - { - "name": "latent_image", - "type": "LATENT", - "link": 9 - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 10 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "KSampler" - }, - "widgets_values": [ - 0, - "fixed", - 26, - 6.5, - "dpmpp_2m", - "karras", - 1.0 - ] - }, - { - "id": 7, - "type": "VAEDecode", - "pos": [ - 980, - 60 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 10 - }, - { - "name": "vae", - "type": "VAE", - "link": 11 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 20 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "VAEDecode" - }, - "widgets_values": [] - }, - { - "id": 8, - "type": "DifforumAnimSetup", - "pos": [ - 40, - 480 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "links": [ - 5, - 12, - 13, - 18 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumAnimSetup" - }, - "widgets_values": [ - 768, - 432, - 24, - 96, - 0, - "fixed" - ] - }, - { - "id": 9, - "type": "DifforumCameraShots", - "pos": [ - 340, - 860 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 12 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - } - ], - "outputs": [ - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "links": [ - 19 - ], - "slot_index": 0 - }, - { - "name": "info", - "type": "STRING", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumCameraShots" - }, - "widgets_values": [ - "0: dolly_in 1.0 1.0\n48: spiral 1.0 1.2", - "2d", - 40.0 - ] - }, - { - "id": 10, - "type": "DifforumSchedule", - "pos": [ - 340, - 1060 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 9, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 13 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - } - ], - "outputs": [ - { - "name": "schedule", - "type": "DIFFORUM_SCHEDULE", - "links": [ - 21 - ], - "slot_index": 0 - }, - { - "name": "values", - "type": "FLOAT", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumSchedule" - }, - "widgets_values": [ - "0:(0.45)", - "linear" - ] - }, - { - "id": 11, - "type": "DifforumFeedbackSampler", - "pos": [ - 1000, - 460 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 10, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 14 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 15 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 16 - }, - { - "name": "vae", - "type": "VAE", - "link": 17 - }, - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 18 - }, - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "link": 19 - }, - { - "name": "init_image", - "type": "IMAGE", - "link": 20 - }, - { - "name": "strength_schedule", - "type": "DIFFORUM_SCHEDULE", - "link": 21 - }, - { - "name": "depth", - "type": "IMAGE", - "link": null - }, - { - "name": "cfg_schedule", - "type": "DIFFORUM_SCHEDULE", - "link": null - }, - { - "name": "positive_schedule", - "type": "DIFFORUM_PROMPT", - "link": 22 - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 23 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumFeedbackSampler" - }, - "widgets_values": [ - 5, - 1.2, - "lcm", - "sgm_uniform", - 0.8, - "lab", - 1.0, - 100.0, - false, - 1.0, - 0.6, - "none", - 6, - "reflection", - 0.25, - 0.02, - 2, - 0, - 0 - ] - }, - { - "id": 12, - "type": "DifforumFlowStabilize", - "pos": [ - 1340, - 460 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 11, - "mode": 0, - "inputs": [ - { - "name": "frames", - "type": "IMAGE", - "link": 23 - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 24 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumFlowStabilize" - }, - "widgets_values": [ - 0.5, - 0.5, - 0.15 - ] - }, - { - "id": 13, - "type": "SaveImage", - "pos": [ - 1680, - 460 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 25 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "SaveImage" - }, - "widgets_values": [ - "Difforum_fast" - ] - }, - { - "id": 14, - "type": "DifforumLoopBlend", - "pos": [ - 1520, - 660 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 13, - "mode": 0, - "inputs": [ - { - "name": "frames", - "type": "IMAGE", - "link": 24 - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 25 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumLoopBlend" - }, - "widgets_values": [ - 12, - true - ] - }, - { - "id": 15, - "type": "Note", - "pos": [ - 40, - -210 - ], - "size": [ - 460, - 300 - ], - "flags": {}, - "order": 14, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "READ ME - models needed", - "properties": { - "text": "" - }, - "widgets_values": [ - "Difforum: FAST SDXL (DMD2 distill + cadence). ~8-10x less diffusion cost\nthan the classic recipe, same checkpoint look. Best template for Macs.\n\nModels:\n- SDXL checkpoint -> models/checkpoints (Juggernaut XL: civitai)\n- DMD2 4-step LoRA -> models/loras (hf: tianweiy/DMD2,\n file dmd2_sdxl_4step_lora.safetensors)\n\nHow it works: frame 0 renders on the BASE model at 26 steps (full\nquality), the loop runs the DMD2 LoRA at 5 steps cfg 1.2 (lcm sampler)\nwith cadence 2. Group 4 = anti-flicker (Ctrl+B to bypass)." - ], - "color": "#432", - "bgcolor": "#653" - } - ], - "links": [ - [ - 1, - 1, - 0, - 2, - 0, - "MODEL" - ], - [ - 2, - 1, - 1, - 2, - 1, - "CLIP" - ], - [ - 3, - 1, - 1, - 3, - 0, - "CLIP" - ], - [ - 4, - 1, - 1, - 5, - 1, - "CLIP" - ], - [ - 5, - 8, - 0, - 5, - 0, - "DIFFORUM_PARAMS" - ], - [ - 6, - 1, - 0, - 6, - 0, - "MODEL" - ], - [ - 7, - 5, - 1, - 6, - 1, - "CONDITIONING" - ], - [ - 8, - 3, - 0, - 6, - 2, - "CONDITIONING" - ], - [ - 9, - 4, - 0, - 6, - 3, - "LATENT" - ], - [ - 10, - 6, - 0, - 7, - 0, - "LATENT" - ], - [ - 11, - 1, - 2, - 7, - 1, - "VAE" - ], - [ - 12, - 8, - 0, - 9, - 0, - "DIFFORUM_PARAMS" - ], - [ - 13, - 8, - 0, - 10, - 0, - "DIFFORUM_PARAMS" - ], - [ - 14, - 2, - 0, - 11, - 0, - "MODEL" - ], - [ - 15, - 5, - 1, - 11, - 1, - "CONDITIONING" - ], - [ - 16, - 3, - 0, - 11, - 2, - "CONDITIONING" - ], - [ - 17, - 1, - 2, - 11, - 3, - "VAE" - ], - [ - 18, - 8, - 0, - 11, - 4, - "DIFFORUM_PARAMS" - ], - [ - 19, - 9, - 0, - 11, - 5, - "DIFFORUM_CAMERA" - ], - [ - 20, - 7, - 0, - 11, - 6, - "IMAGE" - ], - [ - 21, - 10, - 0, - 11, - 7, - "DIFFORUM_SCHEDULE" - ], - [ - 22, - 5, - 0, - 11, - 10, - "DIFFORUM_PROMPT" - ], - [ - 23, - 11, - 0, - 12, - 0, - "IMAGE" - ], - [ - 24, - 12, - 0, - 14, - 0, - "IMAGE" - ], - [ - 25, - 14, - 0, - 13, - 0, - "IMAGE" - ] - ], - "groups": [ - { - "title": "1 - FIRST FRAME: full quality, base model (26 steps)", - "bounding": [ - 10, - -60, - 1270, - 440 - ], - "color": "#33415a", - "font_size": 22, - "locked": false - }, - { - "title": "2 - DIRECTOR + SCHEDULES", - "bounding": [ - 10, - 450, - 900, - 900 - ], - "color": "#33503f", - "font_size": 22, - "locked": false - }, - { - "title": "3 - FAST LOOP: DMD2 4-step + cadence 2", - "bounding": [ - 960, - 380, - 350, - 480 - ], - "color": "#5a3a33", - "font_size": 22, - "locked": false - }, - { - "title": "4 - QUALITY: anti-flicker (Ctrl+B to bypass)", - "bounding": [ - 1310, - 380, - 350, - 300 - ], - "color": "#4a3d5a", - "font_size": 22, - "locked": false - } - ], - "config": {}, - "extra": {}, - "version": 0.4 -} \ No newline at end of file diff --git a/examples/advanced/difforum_film_director.json b/examples/advanced/difforum_film_director.json deleted file mode 100644 index 15abc40..0000000 --- a/examples/advanced/difforum_film_director.json +++ /dev/null @@ -1,1258 +0,0 @@ -{ - "last_node_id": 21, - "last_link_id": 32, - "nodes": [ - { - "id": 16, - "type": "DifforumFilmDirector", - "pos": [ - 40, - 760 - ], - "size": [ - 560, - 340 - ], - "flags": {}, - "order": 16, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 21 - } - ], - "outputs": [ - { - "name": "camera_keys", - "type": "STRING", - "links": [ - 26 - ], - "slot_index": 0, - "shape": 3 - }, - { - "name": "prompt_schedule", - "type": "STRING", - "links": [ - 27 - ], - "slot_index": 1, - "shape": 3 - }, - { - "name": "strength_schedule", - "type": "STRING", - "links": [ - 28 - ], - "slot_index": 2, - "shape": 3 - }, - { - "name": "camera_shots", - "type": "STRING", - "links": null, - "slot_index": 3, - "shape": 3 - }, - { - "name": "info", - "type": "STRING", - "links": null, - "slot_index": 4, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DifforumFilmDirector" - }, - "widgets_values": [ - "[\n {\n \"frame_start\": 0,\n \"mood\": \"calm\",\n \"camera\": \"zoom_in\",\n \"prompt\": \"a luminous coral canyon, volumetric god rays, cinematic wide shot\"\n },\n {\n \"frame_start\": 30,\n \"mood\": \"build\",\n \"camera\": \"pan_right\",\n \"prompt\": \"crystalline ice formations growing, refracted glow, cold rim light\"\n },\n {\n \"frame_start\": 60,\n \"mood\": \"climax\",\n \"camera\": \"spiral\",\n \"prompt\": \"golden desert storm at dusk, swirling light, high contrast\"\n },\n {\n \"frame_start\": 90,\n \"mood\": \"resolve\",\n \"camera\": \"zoom_out\",\n \"prompt\": \"calm starfield, embers drifting, vast cosmic scale\"\n }\n]", - "cinematic", - 0.0, - 1.0, - 0.0, - 0 - ], - "title": "Film Director - drag the shots", - "color": "#323", - "bgcolor": "#535" - }, - { - "id": 1, - "type": "CheckpointLoaderSimple", - "pos": [ - 40, - 40 - ], - "size": [ - 310, - 98 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 1, - 8 - ], - "slot_index": 0, - "shape": 3 - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [ - 2, - 3, - 4 - ], - "slot_index": 1, - "shape": 3 - }, - { - "name": "VAE", - "type": "VAE", - "links": [ - 5, - 6 - ], - "slot_index": 2, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "CheckpointLoaderSimple" - }, - "widgets_values": [ - "juggernautXL_v9.safetensors" - ] - }, - { - "id": 2, - "type": "LoraLoader", - "pos": [ - 40, - 190 - ], - "size": [ - 310, - 126 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 1 - }, - { - "name": "clip", - "type": "CLIP", - "link": 2 - } - ], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 7 - ], - "slot_index": 0, - "shape": 3 - }, - { - "name": "CLIP", - "type": "CLIP", - "links": null, - "slot_index": 1, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "LoraLoader" - }, - "widgets_values": [ - "dmd2_sdxl_4step_lora.safetensors", - 1.0, - 1.0 - ], - "title": "DMD2 4-step distill" - }, - { - "id": 8, - "type": "DifforumAnimSetup", - "pos": [ - 40, - 370 - ], - "size": [ - 310, - 154 - ], - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "links": [ - 12, - 18, - 19, - 20, - 21, - 22 - ], - "slot_index": 0, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DifforumAnimSetup" - }, - "widgets_values": [ - 768, - 432, - 24.0, - 120, - 0 - ], - "title": "Anim Setup - 120f", - "color": "#223", - "bgcolor": "#335" - }, - { - "id": 3, - "type": "CLIPTextEncode", - "pos": [ - 400, - 40 - ], - "size": [ - 350, - 90 - ], - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 3 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 9, - 10 - ], - "slot_index": 0, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "blurry, low quality, watermark, text, washed out, jpeg artifacts" - ], - "title": "Negative" - }, - { - "id": 4, - "type": "EmptyLatentImage", - "pos": [ - 400, - 180 - ], - "size": [ - 350, - 106 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 11 - ], - "slot_index": 0, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "EmptyLatentImage" - }, - "widgets_values": [ - 768, - 432, - 1 - ] - }, - { - "id": 5, - "type": "DifforumPromptSchedule", - "pos": [ - 660, - 760 - ], - "size": [ - 430, - 130 - ], - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 12 - }, - { - "name": "clip", - "type": "CLIP", - "link": 4 - }, - { - "name": "prompts", - "type": "STRING", - "link": 27, - "widget": { - "name": "prompts" - } - } - ], - "outputs": [ - { - "name": "positive_schedule", - "type": "DIFFORUM_PROMPT", - "links": [ - 15 - ], - "slot_index": 0, - "shape": 3 - }, - { - "name": "first_frame_cond", - "type": "CONDITIONING", - "links": [ - 13, - 14 - ], - "slot_index": 1, - "shape": 3 - }, - { - "name": "info", - "type": "STRING", - "links": null, - "slot_index": 2, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DifforumPromptSchedule" - }, - "widgets_values": [ - "0: placeholder", - "ease_in_out" - ], - "color": "#323", - "bgcolor": "#535" - }, - { - "id": 9, - "type": "DifforumCameraKeys", - "pos": [ - 660, - 940 - ], - "size": [ - 430, - 240 - ], - "flags": {}, - "order": 9, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 18 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - }, - { - "name": "keys", - "type": "STRING", - "link": 26, - "widget": { - "name": "keys" - } - } - ], - "outputs": [ - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "links": [ - 23, - 24 - ], - "slot_index": 0, - "shape": 3 - }, - { - "name": "cycle_frames", - "type": "INT", - "links": null, - "slot_index": 1, - "shape": 3 - }, - { - "name": "info", - "type": "STRING", - "links": null, - "slot_index": 2, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DifforumCameraKeys" - }, - "widgets_values": [ - "0: still 1 1 40 linear", - 1.0, - 1.0, - "2d", - "off", - 0, - 3 - ], - "title": "Camera Keys - blend + lens", - "color": "#323", - "bgcolor": "#535" - }, - { - "id": 10, - "type": "DifforumSchedule", - "pos": [ - 660, - 1230 - ], - "size": [ - 430, - 130 - ], - "flags": {}, - "order": 10, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 19 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - }, - { - "name": "schedule", - "type": "STRING", - "link": 28, - "widget": { - "name": "schedule" - } - } - ], - "outputs": [ - { - "name": "schedule", - "type": "DIFFORUM_SCHEDULE", - "links": [ - 25 - ], - "slot_index": 0, - "shape": 3 - }, - { - "name": "values", - "type": "FLOAT", - "links": null, - "slot_index": 1, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DifforumSchedule" - }, - "widgets_values": [ - "0:(0.45)", - "linear" - ], - "title": "Strength (denoise)", - "color": "#323", - "bgcolor": "#535" - }, - { - "id": 6, - "type": "KSampler", - "pos": [ - 800, - 40 - ], - "size": [ - 310, - 262 - ], - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 8 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 13 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 9 - }, - { - "name": "latent_image", - "type": "LATENT", - "link": 11 - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 16 - ], - "slot_index": 0, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "KSampler" - }, - "widgets_values": [ - 0, - "fixed", - 26, - 6.5, - "dpmpp_2m", - "karras", - 1.0 - ], - "title": "Frame 0 (full quality)" - }, - { - "id": 7, - "type": "VAEDecode", - "pos": [ - 1160, - 40 - ], - "size": [ - 230, - 46 - ], - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 16 - }, - { - "name": "vae", - "type": "VAE", - "link": 5 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 17, - 29 - ], - "slot_index": 0, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "VAEDecode" - }, - "widgets_values": [] - }, - { - "id": 20, - "type": "DifforumStoryboard", - "pos": [ - 1160, - 900 - ], - "size": [ - 330, - 300 - ], - "flags": {}, - "order": 20, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 22 - }, - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "link": 24 - }, - { - "name": "init_image", - "type": "IMAGE", - "link": 29 - }, - { - "name": "depth", - "type": "IMAGE", - "link": null - } - ], - "outputs": [ - { - "name": "sheet", - "type": "IMAGE", - "links": [ - 30 - ], - "slot_index": 0, - "shape": 3 - }, - { - "name": "frames", - "type": "IMAGE", - "links": null, - "slot_index": 1, - "shape": 3 - }, - { - "name": "info", - "type": "STRING", - "links": null, - "slot_index": 2, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DifforumStoryboard" - }, - "widgets_values": [ - 8, - 6, - 192, - 0.5, - "reflection", - "none", - 6, - 1.0 - ], - "title": "Storyboard - no diffusion", - "color": "#232", - "bgcolor": "#353" - }, - { - "id": 21, - "type": "PreviewImage", - "pos": [ - 1520, - 900 - ], - "size": [ - 420, - 340 - ], - "flags": {}, - "order": 21, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 30 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "PreviewImage" - }, - "widgets_values": [], - "title": "Storyboard (run this first)" - }, - { - "id": 11, - "type": "DifforumFeedbackSampler", - "pos": [ - 1560, - 180 - ], - "size": [ - 400, - 470 - ], - "flags": {}, - "order": 11, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 7 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 14 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 10 - }, - { - "name": "vae", - "type": "VAE", - "link": 6 - }, - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 20 - }, - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "link": 23 - }, - { - "name": "init_image", - "type": "IMAGE", - "link": 17 - }, - { - "name": "strength_schedule", - "type": "DIFFORUM_SCHEDULE", - "link": 25 - }, - { - "name": "depth", - "type": "IMAGE", - "link": null - }, - { - "name": "cfg_schedule", - "type": "DIFFORUM_SCHEDULE", - "link": null - }, - { - "name": "positive_schedule", - "type": "DIFFORUM_PROMPT", - "link": 15 - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 31 - ], - "slot_index": 0, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DifforumFeedbackSampler" - }, - "widgets_values": [ - 5, - 1.2, - "lcm", - "sgm_uniform", - 0.8, - "lab", - 1.0, - 100.0, - false, - 1.0, - 0.6, - "none", - 6, - "reflection", - 0.25, - 0.025, - 2, - 0, - 0 - ] - }, - { - "id": 12, - "type": "DifforumFlowStabilize", - "pos": [ - 2010, - 180 - ], - "size": [ - 290, - 106 - ], - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [ - { - "name": "frames", - "type": "IMAGE", - "link": 31 - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 32 - ], - "slot_index": 0, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DifforumFlowStabilize" - }, - "widgets_values": [ - 0.5, - 0.5, - 0.15 - ] - }, - { - "id": 13, - "type": "SaveImage", - "pos": [ - 2350, - 180 - ], - "size": [ - 430, - 400 - ], - "flags": {}, - "order": 13, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 32 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "SaveImage" - }, - "widgets_values": [ - "Difforum_director" - ] - }, - { - "id": 15, - "type": "Note", - "pos": [ - 40, - 1120 - ], - "size": [ - 560, - 420 - ], - "flags": {}, - "order": 15, - "mode": 0, - "inputs": [], - "outputs": [], - "properties": { - "Node name for S&R": "Note" - }, - "widgets_values": [ - "DIFFORUM - FILM DIRECTOR\n\nRECOMMENDED FLOW\n1. Run once and look at the Storyboard (green). It shows the whole clip\n with camera only, no diffusion, in under a second. Fix the movement\n there.\n2. Only then let the Feedback Sampler run.\n\nThe purple node feeds three nodes at once:\n camera_keys -> Camera Keys (blend + lens)\n prompt_schedule -> Prompt Schedule (travel)\n strength_schedule -> Schedule (denoise)\n\nCAMERA KEYS vs CAMERA SHOTS\n blend=1.0 the camera accelerates out of one move and into the next\n (crane move, no jolt)\n blend=0.0 hard cut, same as the older node\n Lens (fov) is its own channel and interpolates between keys, so\n pushing in while the lens widens gives a dolly-zoom.\n\nTIMELINE EDITOR\n Click a shot to edit prompt / mood / move / lens / ease.\n Lens on \"auto\" lets the mood choose; ease is how the camera\n arrives from the previous shot.\n\nVARIATION\n 0 the timeline comes out exactly as written\n >0 nudges the moods deterministically from the seed\n Returning to 0 restores the original - it is not a drifting state.\n\nMOOD sets rate / amplitude / lens / denoise:\n calm .44 sp0.6 46d | build .52 sp1.0 40d | tense .57 sp1.2 32d\n climax .63 sp1.6 62d | resolve .44 sp0.5 44d | dream .48 sp0.4 52d\n\nMODELS\n- SDXL -> models/checkpoints (Juggernaut XL)\n- DMD2 4-step LoRA -> models/loras (hf: tianweiy/DMD2)\n\nAPPLE SILICON: no fp8 (MPS has no fp8 kernel). Use bf16/fp16 or GGUF.\n PYTORCH_MPS_HIGH_WATERMARK_RATIO=0.0 python main.py --use-pytorch-cross-attention\nSee PERFORMANCE.md for the full cost model.\n\nMOVES THAT NEED A DEPTH MAP\n dolly_in, dolly_out, orbit_left, orbit_right and sway drive camera\n motion THROUGH space. The 2D affine warp cannot express that, and\n the Feedback Sampler silently falls back to the 2D warp whenever\n `depth` is not connected - even with mode set to 3d. Those moves\n then render as a frozen frame, with nothing in the log to say why.\n\n Flat moves that always work:\n pan_left/right, pan_up/down, zoom_in/out, roll_cw/ccw,\n spiral, dolly_zoom, rise, shake\n\n For real parallax (near pixels moving faster than far ones) use\n examples/advanced/difforum_parallax_fluid.json - it wires\n Depth Anything V2 into the sampler.\n" - ], - "title": "READ THIS", - "color": "#432", - "bgcolor": "#653" - } - ], - "links": [ - [ - 1, - 1, - 0, - 2, - 0, - "MODEL" - ], - [ - 2, - 1, - 1, - 2, - 1, - "CLIP" - ], - [ - 3, - 1, - 1, - 3, - 0, - "CLIP" - ], - [ - 4, - 1, - 1, - 5, - 1, - "CLIP" - ], - [ - 5, - 1, - 2, - 7, - 1, - "VAE" - ], - [ - 6, - 1, - 2, - 11, - 3, - "VAE" - ], - [ - 7, - 2, - 0, - 11, - 0, - "MODEL" - ], - [ - 8, - 1, - 0, - 6, - 0, - "MODEL" - ], - [ - 9, - 3, - 0, - 6, - 2, - "CONDITIONING" - ], - [ - 10, - 3, - 0, - 11, - 2, - "CONDITIONING" - ], - [ - 11, - 4, - 0, - 6, - 3, - "LATENT" - ], - [ - 12, - 8, - 0, - 5, - 0, - "DIFFORUM_PARAMS" - ], - [ - 13, - 5, - 1, - 6, - 1, - "CONDITIONING" - ], - [ - 14, - 5, - 1, - 11, - 1, - "CONDITIONING" - ], - [ - 15, - 5, - 0, - 11, - 10, - "DIFFORUM_PROMPT" - ], - [ - 16, - 6, - 0, - 7, - 0, - "LATENT" - ], - [ - 17, - 7, - 0, - 11, - 6, - "IMAGE" - ], - [ - 18, - 8, - 0, - 9, - 0, - "DIFFORUM_PARAMS" - ], - [ - 19, - 8, - 0, - 10, - 0, - "DIFFORUM_PARAMS" - ], - [ - 20, - 8, - 0, - 11, - 4, - "DIFFORUM_PARAMS" - ], - [ - 21, - 8, - 0, - 16, - 0, - "DIFFORUM_PARAMS" - ], - [ - 22, - 8, - 0, - 20, - 0, - "DIFFORUM_PARAMS" - ], - [ - 23, - 9, - 0, - 11, - 5, - "DIFFORUM_CAMERA" - ], - [ - 24, - 9, - 0, - 20, - 1, - "DIFFORUM_CAMERA" - ], - [ - 25, - 10, - 0, - 11, - 7, - "DIFFORUM_SCHEDULE" - ], - [ - 26, - 16, - 0, - 9, - 2, - "STRING" - ], - [ - 27, - 16, - 1, - 5, - 2, - "STRING" - ], - [ - 28, - 16, - 2, - 10, - 2, - "STRING" - ], - [ - 29, - 7, - 0, - 20, - 2, - "IMAGE" - ], - [ - 30, - 20, - 0, - 21, - 0, - "IMAGE" - ], - [ - 31, - 11, - 0, - 12, - 0, - "IMAGE" - ], - [ - 32, - 12, - 0, - 13, - 0, - "IMAGE" - ] - ], - "groups": [ - { - "title": "1 - DIRECTION (edit here)", - "bounding": [ - 20, - 690, - 1080, - 620 - ], - "color": "#8A8", - "font_size": 24, - "flags": {} - }, - { - "title": "2 - Model + frame 0", - "bounding": [ - 20, - 0, - 1380, - 330 - ], - "color": "#3f789e", - "font_size": 24, - "flags": {} - }, - { - "title": "3 - Cheap preview", - "bounding": [ - 1140, - 850, - 820, - 400 - ], - "color": "#4a8", - "font_size": 24, - "flags": {} - }, - { - "title": "4 - Render", - "bounding": [ - 1530, - 110, - 1280, - 560 - ], - "color": "#a1309b", - "font_size": 24, - "flags": {} - } - ], - "config": {}, - "extra": { - "ds": { - "scale": 0.55, - "offset": [ - 60, - 40 - ] - } - }, - "version": 0.4 -} \ No newline at end of file diff --git a/examples/advanced/difforum_hybrid_wan_guides.json b/examples/advanced/difforum_hybrid_wan_guides.json deleted file mode 100644 index cfec48d..0000000 --- a/examples/advanced/difforum_hybrid_wan_guides.json +++ /dev/null @@ -1,306 +0,0 @@ -{ - "last_node_id": 6, - "last_link_id": 5, - "nodes": [ - { - "id": 1, - "type": "LoadImage", - "pos": [ - 40, - 60 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 2 - ], - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "anchor.png" - ] - }, - { - "id": 2, - "type": "DifforumAnimSetup", - "pos": [ - 40, - 360 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "links": [ - 1, - 4 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumAnimSetup" - }, - "widgets_values": [ - 832, - 480, - 16, - 81, - 0, - "fixed" - ] - }, - { - "id": 3, - "type": "DifforumCamera", - "pos": [ - 360, - 360 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 1 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - } - ], - "outputs": [ - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "links": [ - 3 - ], - "slot_index": 0 - }, - { - "name": "info", - "type": "STRING", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumCamera" - }, - "widgets_values": [ - "3d", - 40.0, - "0:(0)", - "0:(0)", - "0:(1.2)", - "0:(0)", - "0:(0.3*sin(2*pi*t/160))", - "0:(0)", - "0:(1.0)" - ] - }, - { - "id": 4, - "type": "DifforumGuideBuilder", - "pos": [ - 760, - 120 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [ - { - "name": "anchor_image", - "type": "IMAGE", - "link": 2 - }, - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "link": 3 - }, - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 4 - }, - { - "name": "depth", - "type": "IMAGE", - "link": null - } - ], - "outputs": [ - { - "name": "guide_frames", - "type": "IMAGE", - "links": [ - 5 - ], - "slot_index": 0 - }, - { - "name": "occlusion_masks", - "type": "MASK", - "links": [], - "slot_index": 1 - }, - { - "name": "info", - "type": "STRING", - "links": [], - "slot_index": 2 - } - ], - "properties": { - "Node name for S&R": "DifforumGuideBuilder" - }, - "widgets_values": [ - "force_2d", - 1.0, - 100.0, - false - ] - }, - { - "id": 5, - "type": "PreviewImage", - "pos": [ - 1140, - 120 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 5 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "PreviewImage" - }, - "widgets_values": [] - }, - { - "id": 6, - "type": "Note", - "pos": [ - 40, - -210 - ], - "size": [ - 460, - 300 - ], - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "READ ME - models needed", - "properties": { - "text": "" - }, - "widgets_values": [ - "Difforum: Hybrid bridge -> Wan 2.2 VACE.\n\nThis builds camera-warped GUIDE frames. To actually render, wire `guide_frames` into a Wan 2.2 VACE graph.\n\nModels:\n- anchor image -> Load Image\n- Wan 2.2 5B GGUF -> models/unet (hf: QuantStack/Wan2.2-TI2V-5B-GGUF)\n- umt5 text encoder -> models/text_encoders (hf: Comfy-Org/Wan_2.1_ComfyUI_repackaged)\n- wan_2.1_vae.safetensors -> models/vae\n- nodes: ComfyUI-WanVideoWrapper + ComfyUI-GGUF" - ], - "color": "#432", - "bgcolor": "#653" - } - ], - "links": [ - [ - 1, - 2, - 0, - 3, - 0, - "DIFFORUM_PARAMS" - ], - [ - 2, - 1, - 0, - 4, - 0, - "IMAGE" - ], - [ - 3, - 3, - 0, - 4, - 1, - "DIFFORUM_CAMERA" - ], - [ - 4, - 2, - 0, - 4, - 2, - "DIFFORUM_PARAMS" - ], - [ - 5, - 4, - 0, - 5, - 0, - "IMAGE" - ] - ], - "groups": [], - "config": {}, - "extra": {}, - "version": 0.4 -} \ No newline at end of file diff --git a/examples/advanced/difforum_ipadapter_coherent.json b/examples/advanced/difforum_ipadapter_coherent.json deleted file mode 100644 index 760d6a7..0000000 --- a/examples/advanced/difforum_ipadapter_coherent.json +++ /dev/null @@ -1,904 +0,0 @@ -{ - "last_node_id": 15, - "last_link_id": 23, - "nodes": [ - { - "id": 1, - "type": "CheckpointLoaderSimple", - "pos": [ - 40, - 40 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 1 - ], - "slot_index": 0 - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [ - 5, - 6 - ], - "slot_index": 1 - }, - { - "name": "VAE", - "type": "VAE", - "links": [ - 12, - 18 - ], - "slot_index": 2 - } - ], - "properties": { - "Node name for S&R": "CheckpointLoaderSimple" - }, - "widgets_values": [ - "dreamshaper_8.safetensors" - ] - }, - { - "id": 2, - "type": "LoadImage", - "pos": [ - 40, - 260 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 4 - ], - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "style.png" - ] - }, - { - "id": 3, - "type": "IPAdapterUnifiedLoader", - "pos": [ - 360, - 40 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 1 - }, - { - "name": "ipadapter", - "type": "IPADAPTER", - "link": null - } - ], - "outputs": [ - { - "name": "model", - "type": "MODEL", - "links": [ - 2 - ], - "slot_index": 0 - }, - { - "name": "ipadapter", - "type": "IPADAPTER", - "links": [ - 3 - ], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "IPAdapterUnifiedLoader" - }, - "widgets_values": [ - "PLUS (high strength)" - ] - }, - { - "id": 4, - "type": "IPAdapter", - "pos": [ - 660, - 40 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 2 - }, - { - "name": "ipadapter", - "type": "IPADAPTER", - "link": 3 - }, - { - "name": "image", - "type": "IMAGE", - "link": 4 - } - ], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 7, - 15 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "IPAdapter" - }, - "widgets_values": [ - 0.7, - "standard", - 0.0, - 1.0 - ] - }, - { - "id": 5, - "type": "CLIPTextEncode", - "pos": [ - 360, - 260 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 5 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 8, - 16 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "a dreamlike landscape, painterly, vivid" - ] - }, - { - "id": 6, - "type": "CLIPTextEncode", - "pos": [ - 360, - 420 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 6 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 9, - 17 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "blurry, low quality" - ] - }, - { - "id": 7, - "type": "EmptyLatentImage", - "pos": [ - 360, - 580 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 10 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "EmptyLatentImage" - }, - "widgets_values": [ - 768, - 432, - 1 - ] - }, - { - "id": 8, - "type": "KSampler", - "pos": [ - 960, - 40 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 7 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 8 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 9 - }, - { - "name": "latent_image", - "type": "LATENT", - "link": 10 - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 11 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "KSampler" - }, - "widgets_values": [ - 0, - "fixed", - 26, - 6.5, - "dpmpp_2m", - "karras", - 1.0 - ] - }, - { - "id": 9, - "type": "VAEDecode", - "pos": [ - 1260, - 40 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 11 - }, - { - "name": "vae", - "type": "VAE", - "link": 12 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 21 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "VAEDecode" - }, - "widgets_values": [] - }, - { - "id": 10, - "type": "DifforumAnimSetup", - "pos": [ - 40, - 600 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 9, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "links": [ - 13, - 14, - 19 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumAnimSetup" - }, - "widgets_values": [ - 768, - 432, - 24, - 48, - 0, - "fixed" - ] - }, - { - "id": 11, - "type": "DifforumCameraMove", - "pos": [ - 360, - 760 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 10, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 13 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - } - ], - "outputs": [ - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "links": [ - 20 - ], - "slot_index": 0 - }, - { - "name": "info", - "type": "STRING", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumCameraMove" - }, - "widgets_values": [ - "dolly_in", - 1.0, - 1.0, - "2d", - 40.0 - ] - }, - { - "id": 12, - "type": "DifforumSchedule", - "pos": [ - 360, - 980 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 11, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 14 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - } - ], - "outputs": [ - { - "name": "schedule", - "type": "DIFFORUM_SCHEDULE", - "links": [ - 22 - ], - "slot_index": 0 - }, - { - "name": "values", - "type": "FLOAT", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumSchedule" - }, - "widgets_values": [ - "0:(0.45)", - "linear" - ] - }, - { - "id": 13, - "type": "DifforumFeedbackSampler", - "pos": [ - 1260, - 300 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 15 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 16 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 17 - }, - { - "name": "vae", - "type": "VAE", - "link": 18 - }, - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 19 - }, - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "link": 20 - }, - { - "name": "init_image", - "type": "IMAGE", - "link": 21 - }, - { - "name": "strength_schedule", - "type": "DIFFORUM_SCHEDULE", - "link": 22 - }, - { - "name": "depth", - "type": "IMAGE", - "link": null - }, - { - "name": "cfg_schedule", - "type": "DIFFORUM_SCHEDULE", - "link": null - }, - { - "name": "positive_schedule", - "type": "DIFFORUM_PROMPT", - "link": null - }, - { - "name": "control_net", - "type": "CONTROL_NET", - "link": null - }, - { - "name": "control_image", - "type": "IMAGE", - "link": null - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 23 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumFeedbackSampler" - }, - "widgets_values": [ - 24, - 6.5, - "dpmpp_2m", - "karras", - 0.85, - "lab", - 1.0, - 100.0, - false, - 1.0, - 0.6 - ] - }, - { - "id": 14, - "type": "SaveImage", - "pos": [ - 1620, - 300 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 13, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 23 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "SaveImage" - }, - "widgets_values": [ - "Difforum_ipa" - ] - }, - { - "id": 15, - "type": "Note", - "pos": [ - 40, - -210 - ], - "size": [ - 460, - 300 - ], - "flags": {}, - "order": 14, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "READ ME - models needed", - "properties": { - "text": "" - }, - "widgets_values": [ - "Difforum: IPAdapter style-lock (coherence).\n\nModels:\n- SD1.5 checkpoint -> models/checkpoints\n- IP-Adapter ip-adapter-plus_sd15.safetensors -> models/ipadapter (hf: h94/IP-Adapter)\n- CLIP-Vision (ViT-H) -> models/clip_vision (hf: h94/IP-Adapter, image_encoder)\n- a style/reference image -> Load Image\n- node: ComfyUI_IPAdapter_plus" - ], - "color": "#432", - "bgcolor": "#653" - } - ], - "links": [ - [ - 1, - 1, - 0, - 3, - 0, - "MODEL" - ], - [ - 2, - 3, - 0, - 4, - 0, - "MODEL" - ], - [ - 3, - 3, - 1, - 4, - 1, - "IPADAPTER" - ], - [ - 4, - 2, - 0, - 4, - 2, - "IMAGE" - ], - [ - 5, - 1, - 1, - 5, - 0, - "CLIP" - ], - [ - 6, - 1, - 1, - 6, - 0, - "CLIP" - ], - [ - 7, - 4, - 0, - 8, - 0, - "MODEL" - ], - [ - 8, - 5, - 0, - 8, - 1, - "CONDITIONING" - ], - [ - 9, - 6, - 0, - 8, - 2, - "CONDITIONING" - ], - [ - 10, - 7, - 0, - 8, - 3, - "LATENT" - ], - [ - 11, - 8, - 0, - 9, - 0, - "LATENT" - ], - [ - 12, - 1, - 2, - 9, - 1, - "VAE" - ], - [ - 13, - 10, - 0, - 11, - 0, - "DIFFORUM_PARAMS" - ], - [ - 14, - 10, - 0, - 12, - 0, - "DIFFORUM_PARAMS" - ], - [ - 15, - 4, - 0, - 13, - 0, - "MODEL" - ], - [ - 16, - 5, - 0, - 13, - 1, - "CONDITIONING" - ], - [ - 17, - 6, - 0, - 13, - 2, - "CONDITIONING" - ], - [ - 18, - 1, - 2, - 13, - 3, - "VAE" - ], - [ - 19, - 10, - 0, - 13, - 4, - "DIFFORUM_PARAMS" - ], - [ - 20, - 11, - 0, - 13, - 5, - "DIFFORUM_CAMERA" - ], - [ - 21, - 9, - 0, - 13, - 6, - "IMAGE" - ], - [ - 22, - 12, - 0, - 13, - 7, - "DIFFORUM_SCHEDULE" - ], - [ - 23, - 13, - 0, - 14, - 0, - "IMAGE" - ] - ], - "groups": [], - "config": {}, - "extra": {}, - "version": 0.4 -} \ No newline at end of file diff --git a/examples/advanced/difforum_mesmerize_kaleidoscope.json b/examples/advanced/difforum_mesmerize_kaleidoscope.json deleted file mode 100644 index 55a25cd..0000000 --- a/examples/advanced/difforum_mesmerize_kaleidoscope.json +++ /dev/null @@ -1,904 +0,0 @@ -{ - "last_node_id": 15, - "last_link_id": 22, - "nodes": [ - { - "id": 1, - "type": "CheckpointLoaderSimple", - "pos": [ - 40, - 40 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 1, - 11 - ], - "slot_index": 0 - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [ - 2, - 3 - ], - "slot_index": 1 - }, - { - "name": "VAE", - "type": "VAE", - "links": [ - 8, - 14 - ], - "slot_index": 2 - } - ], - "properties": { - "Node name for S&R": "CheckpointLoaderSimple" - }, - "widgets_values": [ - "dreamshaper_8.safetensors" - ] - }, - { - "id": 2, - "type": "CLIPTextEncode", - "pos": [ - 340, - 20 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 2 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 4, - 12 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "ornate symmetric mandala, stained glass, iridescent, intricate fractal detail, glowing" - ] - }, - { - "id": 3, - "type": "CLIPTextEncode", - "pos": [ - 340, - 200 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 3 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 5, - 13 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "blurry, low quality, watermark, text, asymmetric" - ] - }, - { - "id": 4, - "type": "EmptyLatentImage", - "pos": [ - 340, - 380 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 6 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "EmptyLatentImage" - }, - "widgets_values": [ - 768, - 768, - 1 - ] - }, - { - "id": 5, - "type": "KSampler", - "pos": [ - 660, - 120 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 1 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 4 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 5 - }, - { - "name": "latent_image", - "type": "LATENT", - "link": 6 - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 7 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "KSampler" - }, - "widgets_values": [ - 0, - "fixed", - 26, - 6.5, - "dpmpp_2m", - "karras", - 1.0 - ] - }, - { - "id": 6, - "type": "VAEDecode", - "pos": [ - 980, - 120 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 7 - }, - { - "name": "vae", - "type": "VAE", - "link": 8 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 17 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "VAEDecode" - }, - "widgets_values": [] - }, - { - "id": 7, - "type": "DifforumAnimSetup", - "pos": [ - 40, - 520 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "links": [ - 9, - 10, - 15 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumAnimSetup" - }, - "widgets_values": [ - 768, - 768, - 24, - 72, - 0, - "fixed" - ] - }, - { - "id": 8, - "type": "DifforumCamera", - "pos": [ - 340, - 560 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 9 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - } - ], - "outputs": [ - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "links": [ - 16 - ], - "slot_index": 0 - }, - { - "name": "info", - "type": "STRING", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumCamera" - }, - "widgets_values": [ - "2d", - 40.0, - "0:(0)", - "0:(0)", - "0:(1.0)", - "0:(0.6)", - "0:(0)", - "0:(0)", - "0:(1.01)" - ] - }, - { - "id": 9, - "type": "DifforumSchedule", - "pos": [ - 340, - 820 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 10 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - } - ], - "outputs": [ - { - "name": "schedule", - "type": "DIFFORUM_SCHEDULE", - "links": [ - 18 - ], - "slot_index": 0 - }, - { - "name": "values", - "type": "FLOAT", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumSchedule" - }, - "widgets_values": [ - "0:(0.4)", - "linear" - ] - }, - { - "id": 10, - "type": "DifforumFeedbackSampler", - "pos": [ - 1000, - 480 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 9, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 11 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 12 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 13 - }, - { - "name": "vae", - "type": "VAE", - "link": 14 - }, - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 15 - }, - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "link": 16 - }, - { - "name": "init_image", - "type": "IMAGE", - "link": 17 - }, - { - "name": "strength_schedule", - "type": "DIFFORUM_SCHEDULE", - "link": 18 - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 19 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumFeedbackSampler" - }, - "widgets_values": [ - 24, - 6.5, - "dpmpp_2m", - "karras", - 0.85, - "lab", - 1.0, - 100.0, - false, - 1.0, - 0.6, - "kaleidoscope", - 6 - ] - }, - { - "id": 11, - "type": "DifforumEchoTrails", - "pos": [ - 1360, - 360 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 10, - "mode": 0, - "inputs": [ - { - "name": "frames", - "type": "IMAGE", - "link": 20 - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 21 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumEchoTrails" - }, - "widgets_values": [ - 0.6, - 0.4 - ] - }, - { - "id": 12, - "type": "SaveImage", - "pos": [ - 1680, - 360 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 11, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 22 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "SaveImage" - }, - "widgets_values": [ - "Difforum_mesmerize" - ] - }, - { - "id": 13, - "type": "DifforumFlowStabilize", - "pos": [ - 1330, - 480 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [ - { - "name": "frames", - "type": "IMAGE", - "link": 19 - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 20 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumFlowStabilize" - }, - "widgets_values": [ - 0.5, - 0.5, - 0.15 - ] - }, - { - "id": 14, - "type": "DifforumLoopBlend", - "pos": [ - 1860, - 480 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 13, - "mode": 0, - "inputs": [ - { - "name": "frames", - "type": "IMAGE", - "link": 21 - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 22 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumLoopBlend" - }, - "widgets_values": [ - 12, - true - ] - }, - { - "id": 15, - "type": "Note", - "pos": [ - 40, - -210 - ], - "size": [ - 460, - 300 - ], - "flags": {}, - "order": 14, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "READ ME - models needed", - "properties": { - "text": "" - }, - "widgets_values": [ - "Difforum: Living kaleidoscope (in-loop symmetry + Echo Trails).\n\nModels:\n- SD1.5 checkpoint -> models/checkpoints (DreamShaper: civitai.com/models/4384)\nSwap to SDXL/Flux for higher quality.\n\nHow it works: the Feedback Sampler's `symmetry` = kaleidoscope folds each\nwarped frame, then the diffusion heals the seams, so the pattern grows and\nstays symmetric. Echo Trails adds smooth motion-blur trails.\nTry symmetry mirror_h/mirror_v/mirror_quad, or raise symmetry_segments.\nLOOP tip: rotation-only camera (no net zoom/dolly) + ping-pong export =\nseamless loop. Group 4 = anti-flicker + trails (Ctrl+B to bypass)." - ], - "color": "#432", - "bgcolor": "#653" - } - ], - "links": [ - [ - 1, - 1, - 0, - 5, - 0, - "MODEL" - ], - [ - 2, - 1, - 1, - 2, - 0, - "CLIP" - ], - [ - 3, - 1, - 1, - 3, - 0, - "CLIP" - ], - [ - 4, - 2, - 0, - 5, - 1, - "CONDITIONING" - ], - [ - 5, - 3, - 0, - 5, - 2, - "CONDITIONING" - ], - [ - 6, - 4, - 0, - 5, - 3, - "LATENT" - ], - [ - 7, - 5, - 0, - 6, - 0, - "LATENT" - ], - [ - 8, - 1, - 2, - 6, - 1, - "VAE" - ], - [ - 9, - 7, - 0, - 8, - 0, - "DIFFORUM_PARAMS" - ], - [ - 10, - 7, - 0, - 9, - 0, - "DIFFORUM_PARAMS" - ], - [ - 11, - 1, - 0, - 10, - 0, - "MODEL" - ], - [ - 12, - 2, - 0, - 10, - 1, - "CONDITIONING" - ], - [ - 13, - 3, - 0, - 10, - 2, - "CONDITIONING" - ], - [ - 14, - 1, - 2, - 10, - 3, - "VAE" - ], - [ - 15, - 7, - 0, - 10, - 4, - "DIFFORUM_PARAMS" - ], - [ - 16, - 8, - 0, - 10, - 5, - "DIFFORUM_CAMERA" - ], - [ - 17, - 6, - 0, - 10, - 6, - "IMAGE" - ], - [ - 18, - 9, - 0, - 10, - 7, - "DIFFORUM_SCHEDULE" - ], - [ - 19, - 10, - 0, - 13, - 0, - "IMAGE" - ], - [ - 20, - 13, - 0, - 11, - 0, - "IMAGE" - ], - [ - 21, - 11, - 0, - 14, - 0, - "IMAGE" - ], - [ - 22, - 14, - 0, - 12, - 0, - "IMAGE" - ] - ], - "groups": [ - { - "title": "1 - FIRST FRAME (txt2img)", - "bounding": [ - 10, - -60, - 1270, - 500 - ], - "color": "#33415a", - "font_size": 22, - "locked": false - }, - { - "title": "2 - ANIMATION BRAIN", - "bounding": [ - 10, - 470, - 900, - 700 - ], - "color": "#33503f", - "font_size": 22, - "locked": false - }, - { - "title": "3 - RENDER LOOP (kaleidoscope inside)", - "bounding": [ - 960, - 390, - 350, - 480 - ], - "color": "#5a3a33", - "font_size": 22, - "locked": false - }, - { - "title": "4 - QUALITY: anti-flicker + trails (Ctrl+B to bypass)", - "bounding": [ - 1310, - 300, - 700, - 320 - ], - "color": "#4a3d5a", - "font_size": 22, - "locked": false - } - ], - "config": {}, - "extra": {}, - "version": 0.4 -} \ No newline at end of file diff --git a/examples/advanced/difforum_parallax_fluid.json b/examples/advanced/difforum_parallax_fluid.json deleted file mode 100644 index 41bdbe1..0000000 --- a/examples/advanced/difforum_parallax_fluid.json +++ /dev/null @@ -1,1548 +0,0 @@ -{ - "last_node_id": 42, - "last_link_id": 39, - "nodes": [ - { - "id": 16, - "type": "DifforumFilmDirector", - "pos": [ - 40, - 780 - ], - "size": [ - 600, - 470 - ], - "flags": {}, - "order": 16, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 27 - } - ], - "outputs": [ - { - "name": "camera_keys", - "type": "STRING", - "links": [ - 32 - ], - "slot_index": 0, - "shape": 3 - }, - { - "name": "prompt_schedule", - "type": "STRING", - "links": [ - 33 - ], - "slot_index": 1, - "shape": 3 - }, - { - "name": "strength_schedule", - "type": "STRING", - "links": [ - 34 - ], - "slot_index": 2, - "shape": 3 - }, - { - "name": "camera_shots", - "type": "STRING", - "links": null, - "slot_index": 3, - "shape": 3 - }, - { - "name": "info", - "type": "STRING", - "links": null, - "slot_index": 4, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DifforumFilmDirector" - }, - "widgets_values": [ - "[\n {\n \"frame_start\": 0,\n \"mood\": \"calm\",\n \"camera\": \"dolly_in\",\n \"lens\": 65,\n \"prompt\": \"vast canyon interior, layered rock walls receding into depth, shafts of light, strong foreground silhouettes\"\n },\n {\n \"frame_start\": 30,\n \"mood\": \"build\",\n \"camera\": \"pan_right\",\n \"lens\": 50,\n \"prompt\": \"carved corridors opening past the near wall, warm bounce light, dust in the air\"\n },\n {\n \"frame_start\": 60,\n \"mood\": \"tense\",\n \"camera\": \"dolly_in\",\n \"lens\": 35,\n \"prompt\": \"narrow passage, close rock texture rushing past, deep shadow beyond\"\n },\n {\n \"frame_start\": 90,\n \"mood\": \"climax\",\n \"camera\": \"orbit_right\",\n \"lens\": 75,\n \"prompt\": \"breaking into an open cavern, distant scale, god rays, foreground pillars sweeping by\"\n }\n]", - "cinematic", - 0.08, - 1.6, - 0.0, - 0 - ], - "title": "Film Director - drag the shots", - "color": "#323", - "bgcolor": "#535" - }, - { - "id": 1, - "type": "CheckpointLoaderSimple", - "pos": [ - 40, - 40 - ], - "size": [ - 320, - 98 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 1, - 8 - ], - "slot_index": 0, - "shape": 3 - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [ - 2, - 3, - 4 - ], - "slot_index": 1, - "shape": 3 - }, - { - "name": "VAE", - "type": "VAE", - "links": [ - 5, - 6 - ], - "slot_index": 2, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "CheckpointLoaderSimple" - }, - "widgets_values": [ - "juggernautXL_ragnarok.safetensors" - ] - }, - { - "id": 2, - "type": "LoraLoader", - "pos": [ - 40, - 190 - ], - "size": [ - 320, - 126 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 1 - }, - { - "name": "clip", - "type": "CLIP", - "link": 2 - } - ], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 7 - ], - "slot_index": 0, - "shape": 3 - }, - { - "name": "CLIP", - "type": "CLIP", - "links": null, - "slot_index": 1, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "LoraLoader" - }, - "widgets_values": [ - "dmd2_sdxl_4step_lora.safetensors", - 1.0, - 1.0 - ], - "title": "DMD2 4-step distill" - }, - { - "id": 8, - "type": "DifforumAnimSetup", - "pos": [ - 40, - 370 - ], - "size": [ - 320, - 154 - ], - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "links": [ - 12, - 24, - 25, - 26, - 27, - 28 - ], - "slot_index": 0, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DifforumAnimSetup" - }, - "widgets_values": [ - 768, - 432, - 24.0, - 120, - 0 - ], - "title": "Anim Setup - 120f @ 24fps", - "color": "#223", - "bgcolor": "#335" - }, - { - "id": 3, - "type": "CLIPTextEncode", - "pos": [ - 400, - 40 - ], - "size": [ - 350, - 90 - ], - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 3 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 9, - 10 - ], - "slot_index": 0, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "flat, washed out, blurry, low quality, watermark, text, jpeg artifacts" - ], - "title": "Negative" - }, - { - "id": 4, - "type": "EmptyLatentImage", - "pos": [ - 400, - 180 - ], - "size": [ - 350, - 106 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 11 - ], - "slot_index": 0, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "EmptyLatentImage" - }, - "widgets_values": [ - 768, - 432, - 1 - ] - }, - { - "id": 5, - "type": "DifforumPromptSchedule", - "pos": [ - 700, - 780 - ], - "size": [ - 430, - 130 - ], - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 12 - }, - { - "name": "clip", - "type": "CLIP", - "link": 4 - }, - { - "name": "prompts", - "type": "STRING", - "link": 33, - "widget": { - "name": "prompts" - } - } - ], - "outputs": [ - { - "name": "positive_schedule", - "type": "DIFFORUM_PROMPT", - "links": [ - 15 - ], - "slot_index": 0, - "shape": 3 - }, - { - "name": "first_frame_cond", - "type": "CONDITIONING", - "links": [ - 13, - 14 - ], - "slot_index": 1, - "shape": 3 - }, - { - "name": "info", - "type": "STRING", - "links": null, - "slot_index": 2, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DifforumPromptSchedule" - }, - "widgets_values": [ - "0: placeholder", - "ease_in_out" - ], - "color": "#323", - "bgcolor": "#535" - }, - { - "id": 9, - "type": "DifforumCameraKeys", - "pos": [ - 700, - 960 - ], - "size": [ - 430, - 240 - ], - "flags": {}, - "order": 9, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 24 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - }, - { - "name": "keys", - "type": "STRING", - "link": 32, - "widget": { - "name": "keys" - } - } - ], - "outputs": [ - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "links": [ - 29, - 30 - ], - "slot_index": 0, - "shape": 3 - }, - { - "name": "cycle_frames", - "type": "INT", - "links": null, - "slot_index": 1, - "shape": 3 - }, - { - "name": "info", - "type": "STRING", - "links": null, - "slot_index": 2, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DifforumCameraKeys" - }, - "widgets_values": [ - "0: still 1 1 40 linear", - 1.0, - 1.0, - "3d", - "off", - 0, - 3 - ], - "title": "Camera Keys - 3d (parallax)", - "color": "#323", - "bgcolor": "#535" - }, - { - "id": 10, - "type": "DifforumSchedule", - "pos": [ - 700, - 1250 - ], - "size": [ - 430, - 130 - ], - "flags": {}, - "order": 10, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 25 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - }, - { - "name": "schedule", - "type": "STRING", - "link": 34, - "widget": { - "name": "schedule" - } - } - ], - "outputs": [ - { - "name": "schedule", - "type": "DIFFORUM_SCHEDULE", - "links": [ - 31 - ], - "slot_index": 0, - "shape": 3 - }, - { - "name": "values", - "type": "FLOAT", - "links": null, - "slot_index": 1, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DifforumSchedule" - }, - "widgets_values": [ - "0:(0.5)", - "linear" - ], - "title": "Strength (denoise)", - "color": "#323", - "bgcolor": "#535" - }, - { - "id": 6, - "type": "KSampler", - "pos": [ - 800, - 40 - ], - "size": [ - 310, - 262 - ], - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 8 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 13 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 9 - }, - { - "name": "latent_image", - "type": "LATENT", - "link": 11 - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 16 - ], - "slot_index": 0, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "KSampler" - }, - "widgets_values": [ - 0, - "fixed", - 28, - 6.5, - "dpmpp_2m", - "karras", - 1.0 - ], - "title": "Frame 0 (full quality)" - }, - { - "id": 7, - "type": "VAEDecode", - "pos": [ - 1160, - 40 - ], - "size": [ - 230, - 46 - ], - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 16 - }, - { - "name": "vae", - "type": "VAE", - "link": 5 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 17, - 18, - 19 - ], - "slot_index": 0, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "VAEDecode" - }, - "widgets_values": [] - }, - { - "id": 30, - "type": "DownloadAndLoadDepthAnythingV2Model", - "pos": [ - 1160, - 140 - ], - "size": [ - 330, - 82 - ], - "flags": {}, - "order": 30, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "da_v2_model", - "type": "DAMODEL", - "links": [ - 20 - ], - "slot_index": 0, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DownloadAndLoadDepthAnythingV2Model" - }, - "widgets_values": [ - "depth_anything_v2_vitb_fp16.safetensors", - "auto" - ], - "title": "Depth model (auto-downloads)", - "color": "#432", - "bgcolor": "#653" - }, - { - "id": 31, - "type": "DepthAnything_V2", - "pos": [ - 1160, - 270 - ], - "size": [ - 330, - 58 - ], - "flags": {}, - "order": 31, - "mode": 0, - "inputs": [ - { - "name": "da_model", - "type": "DAMODEL", - "link": 20 - }, - { - "name": "images", - "type": "IMAGE", - "link": 18 - } - ], - "outputs": [ - { - "name": "image", - "type": "IMAGE", - "links": [ - 21, - 22, - 23 - ], - "slot_index": 0, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DepthAnything_V2" - }, - "widgets_values": [], - "title": "Depth map -> parallax", - "color": "#432", - "bgcolor": "#653" - }, - { - "id": 32, - "type": "PreviewImage", - "pos": [ - 1160, - 370 - ], - "size": [ - 330, - 270 - ], - "flags": {}, - "order": 32, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 23 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "PreviewImage" - }, - "widgets_values": [], - "title": "Depth check" - }, - { - "id": 20, - "type": "DifforumStoryboard", - "pos": [ - 1540, - 780 - ], - "size": [ - 330, - 300 - ], - "flags": {}, - "order": 20, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 28 - }, - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "link": 30 - }, - { - "name": "init_image", - "type": "IMAGE", - "link": 19 - }, - { - "name": "depth", - "type": "IMAGE", - "link": 22 - } - ], - "outputs": [ - { - "name": "sheet", - "type": "IMAGE", - "links": [ - 35 - ], - "slot_index": 0, - "shape": 3 - }, - { - "name": "frames", - "type": "IMAGE", - "links": null, - "slot_index": 1, - "shape": 3 - }, - { - "name": "info", - "type": "STRING", - "links": null, - "slot_index": 2, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DifforumStoryboard" - }, - "widgets_values": [ - 8, - 6, - 192, - 0.5, - "reflection", - "none", - 6, - 1.5 - ], - "title": "Storyboard - run this first", - "color": "#232", - "bgcolor": "#353" - }, - { - "id": 21, - "type": "PreviewImage", - "pos": [ - 1900, - 780 - ], - "size": [ - 430, - 340 - ], - "flags": {}, - "order": 21, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 35 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "PreviewImage" - }, - "widgets_values": [], - "title": "Storyboard sheet" - }, - { - "id": 11, - "type": "DifforumFeedbackSampler", - "pos": [ - 1560, - 120 - ], - "size": [ - 400, - 470 - ], - "flags": {}, - "order": 11, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 7 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 14 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 10 - }, - { - "name": "vae", - "type": "VAE", - "link": 6 - }, - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 26 - }, - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "link": 29 - }, - { - "name": "init_image", - "type": "IMAGE", - "link": 17 - }, - { - "name": "strength_schedule", - "type": "DIFFORUM_SCHEDULE", - "link": 31 - }, - { - "name": "depth", - "type": "IMAGE", - "link": 21 - }, - { - "name": "cfg_schedule", - "type": "DIFFORUM_SCHEDULE", - "link": null - }, - { - "name": "positive_schedule", - "type": "DIFFORUM_PROMPT", - "link": 15 - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 36 - ], - "slot_index": 0, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DifforumFeedbackSampler" - }, - "widgets_values": [ - 6, - 1.2, - "lcm", - "sgm_uniform", - 0.6, - "lab", - 0.6, - 40.0, - false, - 1.6, - 0.6, - "none", - 6, - "reflection", - 0.3, - 0.035, - 2, - 0, - 0 - ] - }, - { - "id": 12, - "type": "DifforumFlowStabilize", - "pos": [ - 2010, - 120 - ], - "size": [ - 290, - 106 - ], - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [ - { - "name": "frames", - "type": "IMAGE", - "link": 36 - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 37 - ], - "slot_index": 0, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DifforumFlowStabilize" - }, - "widgets_values": [ - 0.35, - 0.5, - 0.15 - ] - }, - { - "id": 40, - "type": "RIFE VFI", - "pos": [ - 2010, - 280 - ], - "size": [ - 330, - 200 - ], - "flags": {}, - "order": 40, - "mode": 0, - "inputs": [ - { - "name": "frames", - "type": "IMAGE", - "link": 37 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 38, - 39 - ], - "slot_index": 0, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "RIFE VFI" - }, - "widgets_values": [ - "rife47.pth", - 10, - 2, - true, - true, - 1.0 - ], - "title": "RIFE 2x - fluidity", - "color": "#432", - "bgcolor": "#653" - }, - { - "id": 41, - "type": "VHS_VideoCombine", - "pos": [ - 2390, - 120 - ], - "size": [ - 400, - 480 - ], - "flags": {}, - "order": 41, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 38 - }, - { - "name": "audio", - "type": "AUDIO", - "link": null - }, - { - "name": "meta_batch", - "type": "VHS_BatchManager", - "link": null - }, - { - "name": "vae", - "type": "VAE", - "link": null - } - ], - "outputs": [ - { - "name": "Filenames", - "type": "VHS_FILENAMES", - "links": null, - "slot_index": 0, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "VHS_VideoCombine" - }, - "widgets_values": { - "frame_rate": 48.0, - "loop_count": 0, - "filename_prefix": "Difforum_fluid", - "format": "video/h264-mp4", - "pix_fmt": "yuv420p", - "crf": 17, - "save_metadata": true, - "pingpong": false, - "save_output": true - }, - "title": "MP4 @ 48fps" - }, - { - "id": 42, - "type": "SaveImage", - "pos": [ - 2390, - 640 - ], - "size": [ - 400, - 300 - ], - "flags": {}, - "order": 42, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 39 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "SaveImage" - }, - "widgets_values": [ - "Difforum_fluid" - ] - }, - { - "id": 15, - "type": "Note", - "pos": [ - 40, - 1290 - ], - "size": [ - 600, - 430 - ], - "flags": {}, - "order": 15, - "mode": 0, - "inputs": [], - "outputs": [], - "properties": { - "Node name for S&R": "Note" - }, - "widgets_values": [ - "DIFFORUM - PARALLAX + FLUIDITY\n\nTwo fixes for \"static and choppy\", wired together.\n\n1. PARALLAX (fixes static)\n Without a depth map the Feedback Sampler silently falls back to a 2D\n affine warp - every pixel moves identically and the frame reads as a\n photo being pushed around. Depth Anything V2 supplies the depth, so\n warp_3d runs and near pixels travel faster than far ones.\n - Camera Keys mode must stay 3d\n - translation_scale 1.6 on the sampler exaggerates the parallax\n - near 0.6 / far 40 tightens the depth range onto the actual scene\n Write prompts with layers: something close, something far. Depth has\n nothing to work with in a flat image.\n\n2. FLUIDITY (fixes choppy)\n cadence 2 means content really changes at 12fps. RIFE interpolates\n 2x, so the MP4 runs at 48fps with motion-compensated in-betweens\n rather than warp slides. Raise the multiplier to 3-4 for projection.\n\n3. ANTI-CONVERGENCE\n The old settings settled into a fixed pattern. Changed:\n - color_coherence 0.80 -> 0.60 (less pull back to the anchor)\n - strength_bias +0.08 (the loop keeps evolving)\n - camera_scale 1.6 (gives the sampler something to bite)\n - noise 0.025 -> 0.035, sharpen 0.30\n - flow_align strength 0.5 -> 0.35 (over-stabilising also freezes it)\n\nFIRST RUN downloads the depth model (~50MB vitb) and the RIFE weights\n(~50MB). Both are automatic.\n\nWORKFLOW\n Run once and read the Storyboard + Depth check before anything else.\n Storyboard info reports \"drift avg\" - below 0.004 means it will\n stagnate, so raise camera_scale. The Depth check should show clear\n near/far separation; a flat grey map means the prompt has no depth.\n\nAPPLE SILICON\n RIFE runs on MPS. If it fails, set scale_factor 0.5.\n No fp8 anywhere - MPS has no fp8 kernel." - ], - "title": "READ THIS", - "color": "#432", - "bgcolor": "#653" - } - ], - "links": [ - [ - 1, - 1, - 0, - 2, - 0, - "MODEL" - ], - [ - 2, - 1, - 1, - 2, - 1, - "CLIP" - ], - [ - 3, - 1, - 1, - 3, - 0, - "CLIP" - ], - [ - 4, - 1, - 1, - 5, - 1, - "CLIP" - ], - [ - 5, - 1, - 2, - 7, - 1, - "VAE" - ], - [ - 6, - 1, - 2, - 11, - 3, - "VAE" - ], - [ - 7, - 2, - 0, - 11, - 0, - "MODEL" - ], - [ - 8, - 1, - 0, - 6, - 0, - "MODEL" - ], - [ - 9, - 3, - 0, - 6, - 2, - "CONDITIONING" - ], - [ - 10, - 3, - 0, - 11, - 2, - "CONDITIONING" - ], - [ - 11, - 4, - 0, - 6, - 3, - "LATENT" - ], - [ - 12, - 8, - 0, - 5, - 0, - "DIFFORUM_PARAMS" - ], - [ - 13, - 5, - 1, - 6, - 1, - "CONDITIONING" - ], - [ - 14, - 5, - 1, - 11, - 1, - "CONDITIONING" - ], - [ - 15, - 5, - 0, - 11, - 10, - "DIFFORUM_PROMPT" - ], - [ - 16, - 6, - 0, - 7, - 0, - "LATENT" - ], - [ - 17, - 7, - 0, - 11, - 6, - "IMAGE" - ], - [ - 18, - 7, - 0, - 31, - 1, - "IMAGE" - ], - [ - 19, - 7, - 0, - 20, - 2, - "IMAGE" - ], - [ - 20, - 30, - 0, - 31, - 0, - "DAMODEL" - ], - [ - 21, - 31, - 0, - 11, - 8, - "IMAGE" - ], - [ - 22, - 31, - 0, - 20, - 3, - "IMAGE" - ], - [ - 23, - 31, - 0, - 32, - 0, - "IMAGE" - ], - [ - 24, - 8, - 0, - 9, - 0, - "DIFFORUM_PARAMS" - ], - [ - 25, - 8, - 0, - 10, - 0, - "DIFFORUM_PARAMS" - ], - [ - 26, - 8, - 0, - 11, - 4, - "DIFFORUM_PARAMS" - ], - [ - 27, - 8, - 0, - 16, - 0, - "DIFFORUM_PARAMS" - ], - [ - 28, - 8, - 0, - 20, - 0, - "DIFFORUM_PARAMS" - ], - [ - 29, - 9, - 0, - 11, - 5, - "DIFFORUM_CAMERA" - ], - [ - 30, - 9, - 0, - 20, - 1, - "DIFFORUM_CAMERA" - ], - [ - 31, - 10, - 0, - 11, - 7, - "DIFFORUM_SCHEDULE" - ], - [ - 32, - 16, - 0, - 9, - 2, - "STRING" - ], - [ - 33, - 16, - 1, - 5, - 2, - "STRING" - ], - [ - 34, - 16, - 2, - 10, - 2, - "STRING" - ], - [ - 35, - 20, - 0, - 21, - 0, - "IMAGE" - ], - [ - 36, - 11, - 0, - 12, - 0, - "IMAGE" - ], - [ - 37, - 12, - 0, - 40, - 0, - "IMAGE" - ], - [ - 38, - 40, - 0, - 41, - 0, - "IMAGE" - ], - [ - 39, - 40, - 0, - 42, - 0, - "IMAGE" - ] - ], - "groups": [ - { - "title": "1 - DIRECTION (edit here)", - "bounding": [ - 20, - 710, - 1120, - 700 - ], - "color": "#8A8", - "font_size": 24, - "flags": {} - }, - { - "title": "2 - Model + frame 0", - "bounding": [ - 20, - 0, - 1380, - 330 - ], - "color": "#3f789e", - "font_size": 24, - "flags": {} - }, - { - "title": "3 - Depth (parallax)", - "bounding": [ - 1140, - 90, - 380, - 570 - ], - "color": "#b58b2a", - "font_size": 24, - "flags": {} - }, - { - "title": "4 - Cheap preview", - "bounding": [ - 1520, - 720, - 830, - 420 - ], - "color": "#4a8", - "font_size": 24, - "flags": {} - }, - { - "title": "5 - Render + fluidity", - "bounding": [ - 1540, - 50, - 1280, - 620 - ], - "color": "#a1309b", - "font_size": 24, - "flags": {} - } - ], - "config": {}, - "extra": { - "ds": { - "scale": 0.5, - "offset": [ - 60, - 40 - ] - } - }, - "version": 0.4 -} \ No newline at end of file diff --git a/examples/advanced/difforum_qrcode_illusion.json b/examples/advanced/difforum_qrcode_illusion.json deleted file mode 100644 index a8d6a9d..0000000 --- a/examples/advanced/difforum_qrcode_illusion.json +++ /dev/null @@ -1,818 +0,0 @@ -{ - "last_node_id": 14, - "last_link_id": 21, - "nodes": [ - { - "id": 1, - "type": "CheckpointLoaderSimple", - "pos": [ - 40, - 40 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 3, - 11 - ], - "slot_index": 0 - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [ - 1, - 2 - ], - "slot_index": 1 - }, - { - "name": "VAE", - "type": "VAE", - "links": [ - 8, - 14 - ], - "slot_index": 2 - } - ], - "properties": { - "Node name for S&R": "CheckpointLoaderSimple" - }, - "widgets_values": [ - "dreamshaper_8.safetensors" - ] - }, - { - "id": 2, - "type": "LoadImage", - "pos": [ - 40, - 260 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 20 - ], - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "pattern.png" - ] - }, - { - "id": 3, - "type": "ControlNetLoader", - "pos": [ - 40, - 480 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "CONTROL_NET", - "type": "CONTROL_NET", - "links": [ - 19 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "ControlNetLoader" - }, - "widgets_values": [ - "control_v1p_sd15_qrcode_monster.safetensors" - ] - }, - { - "id": 4, - "type": "CLIPTextEncode", - "pos": [ - 360, - 20 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 1 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 4, - 12 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "ornate fractal mandala, molten gold and emerald, intricate" - ] - }, - { - "id": 5, - "type": "CLIPTextEncode", - "pos": [ - 360, - 200 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 2 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 5, - 13 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "blurry, low quality, flat" - ] - }, - { - "id": 6, - "type": "EmptyLatentImage", - "pos": [ - 360, - 360 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 6 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "EmptyLatentImage" - }, - "widgets_values": [ - 768, - 432, - 1 - ] - }, - { - "id": 7, - "type": "KSampler", - "pos": [ - 660, - 120 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 3 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 4 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 5 - }, - { - "name": "latent_image", - "type": "LATENT", - "link": 6 - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 7 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "KSampler" - }, - "widgets_values": [ - 0, - "fixed", - 26, - 6.5, - "dpmpp_2m", - "karras", - 1.0 - ] - }, - { - "id": 8, - "type": "VAEDecode", - "pos": [ - 980, - 120 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 7 - }, - { - "name": "vae", - "type": "VAE", - "link": 8 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 17 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "VAEDecode" - }, - "widgets_values": [] - }, - { - "id": 9, - "type": "DifforumAnimSetup", - "pos": [ - 40, - 660 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "links": [ - 9, - 10, - 15 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumAnimSetup" - }, - "widgets_values": [ - 768, - 432, - 24, - 48, - 0, - "fixed" - ] - }, - { - "id": 10, - "type": "DifforumCameraMove", - "pos": [ - 360, - 560 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 9, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 9 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - } - ], - "outputs": [ - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "links": [ - 16 - ], - "slot_index": 0 - }, - { - "name": "info", - "type": "STRING", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumCameraMove" - }, - "widgets_values": [ - "zoom_in", - 0.5, - 0.6, - "2d", - 40.0 - ] - }, - { - "id": 11, - "type": "DifforumSchedule", - "pos": [ - 360, - 780 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 10, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 10 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - } - ], - "outputs": [ - { - "name": "schedule", - "type": "DIFFORUM_SCHEDULE", - "links": [ - 18 - ], - "slot_index": 0 - }, - { - "name": "values", - "type": "FLOAT", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumSchedule" - }, - "widgets_values": [ - "0:(0.4)", - "linear" - ] - }, - { - "id": 12, - "type": "DifforumFeedbackSampler", - "pos": [ - 1280, - 300 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 11, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 11 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 12 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 13 - }, - { - "name": "vae", - "type": "VAE", - "link": 14 - }, - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 15 - }, - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "link": 16 - }, - { - "name": "init_image", - "type": "IMAGE", - "link": 17 - }, - { - "name": "strength_schedule", - "type": "DIFFORUM_SCHEDULE", - "link": 18 - }, - { - "name": "depth", - "type": "IMAGE", - "link": null - }, - { - "name": "cfg_schedule", - "type": "DIFFORUM_SCHEDULE", - "link": null - }, - { - "name": "positive_schedule", - "type": "DIFFORUM_PROMPT", - "link": null - }, - { - "name": "control_net", - "type": "CONTROL_NET", - "link": 19 - }, - { - "name": "control_image", - "type": "IMAGE", - "link": 20 - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 21 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumFeedbackSampler" - }, - "widgets_values": [ - 24, - 6.5, - "dpmpp_2m", - "karras", - 0.85, - "lab", - 1.0, - 100.0, - false, - 1.0, - 1.2 - ] - }, - { - "id": 13, - "type": "SaveImage", - "pos": [ - 1640, - 300 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 21 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "SaveImage" - }, - "widgets_values": [ - "Difforum_illusion" - ] - }, - { - "id": 14, - "type": "Note", - "pos": [ - 40, - -210 - ], - "size": [ - 460, - 300 - ], - "flags": {}, - "order": 13, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "READ ME - models needed", - "properties": { - "text": "" - }, - "widgets_values": [ - "Difforum: Hidden-pattern illusion (QR Monster).\n\nModels:\n- SD1.5 checkpoint -> models/checkpoints\n- control_v1p_sd15_qrcode_monster -> models/controlnet (hf: monster-labs/control_v1p_sd15_qrcode_monster)\n- a grayscale pattern image (spiral/logo/mask) -> Load Image\ncontrol_strength ~1.2 = strong illusion; lower for subtle." - ], - "color": "#432", - "bgcolor": "#653" - } - ], - "links": [ - [ - 1, - 1, - 1, - 4, - 0, - "CLIP" - ], - [ - 2, - 1, - 1, - 5, - 0, - "CLIP" - ], - [ - 3, - 1, - 0, - 7, - 0, - "MODEL" - ], - [ - 4, - 4, - 0, - 7, - 1, - "CONDITIONING" - ], - [ - 5, - 5, - 0, - 7, - 2, - "CONDITIONING" - ], - [ - 6, - 6, - 0, - 7, - 3, - "LATENT" - ], - [ - 7, - 7, - 0, - 8, - 0, - "LATENT" - ], - [ - 8, - 1, - 2, - 8, - 1, - "VAE" - ], - [ - 9, - 9, - 0, - 10, - 0, - "DIFFORUM_PARAMS" - ], - [ - 10, - 9, - 0, - 11, - 0, - "DIFFORUM_PARAMS" - ], - [ - 11, - 1, - 0, - 12, - 0, - "MODEL" - ], - [ - 12, - 4, - 0, - 12, - 1, - "CONDITIONING" - ], - [ - 13, - 5, - 0, - 12, - 2, - "CONDITIONING" - ], - [ - 14, - 1, - 2, - 12, - 3, - "VAE" - ], - [ - 15, - 9, - 0, - 12, - 4, - "DIFFORUM_PARAMS" - ], - [ - 16, - 10, - 0, - 12, - 5, - "DIFFORUM_CAMERA" - ], - [ - 17, - 8, - 0, - 12, - 6, - "IMAGE" - ], - [ - 18, - 11, - 0, - 12, - 7, - "DIFFORUM_SCHEDULE" - ], - [ - 19, - 3, - 0, - 12, - 11, - "CONTROL_NET" - ], - [ - 20, - 2, - 0, - 12, - 12, - "IMAGE" - ], - [ - 21, - 12, - 0, - 13, - 0, - "IMAGE" - ] - ], - "groups": [], - "config": {}, - "extra": {}, - "version": 0.4 -} \ No newline at end of file diff --git a/examples/advanced/difforum_realtime_live.json b/examples/advanced/difforum_realtime_live.json deleted file mode 100644 index 2f02cf5..0000000 --- a/examples/advanced/difforum_realtime_live.json +++ /dev/null @@ -1,635 +0,0 @@ -{ - "last_node_id": 11, - "last_link_id": 17, - "nodes": [ - { - "id": 1, - "type": "CheckpointLoaderSimple", - "pos": [ - 40, - 40 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 1, - 10 - ], - "slot_index": 0 - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [ - 2, - 3 - ], - "slot_index": 1 - }, - { - "name": "VAE", - "type": "VAE", - "links": [ - 8, - 13 - ], - "slot_index": 2 - } - ], - "properties": { - "Node name for S&R": "CheckpointLoaderSimple" - }, - "widgets_values": [ - "sd_turbo.safetensors" - ] - }, - { - "id": 2, - "type": "CLIPTextEncode", - "pos": [ - 360, - 20 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 2 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 4, - 11 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "psychedelic stained glass mandala, vivid colors, intricate, glowing" - ] - }, - { - "id": 3, - "type": "CLIPTextEncode", - "pos": [ - 360, - 160 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 3 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 5, - 12 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "blurry, low quality" - ] - }, - { - "id": 4, - "type": "EmptyLatentImage", - "pos": [ - 360, - 300 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 6 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "EmptyLatentImage" - }, - "widgets_values": [ - 512, - 512, - 1 - ] - }, - { - "id": 5, - "type": "KSampler", - "pos": [ - 680, - 60 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 1 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 4 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 5 - }, - { - "name": "latent_image", - "type": "LATENT", - "link": 6 - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 7 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "KSampler" - }, - "widgets_values": [ - 0, - "fixed", - 4, - 1.0, - "euler", - "sgm_uniform", - 1.0 - ] - }, - { - "id": 6, - "type": "VAEDecode", - "pos": [ - 1000, - 60 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 7 - }, - { - "name": "vae", - "type": "VAE", - "link": 8 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 16 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "VAEDecode" - }, - "widgets_values": [] - }, - { - "id": 7, - "type": "DifforumAnimSetup", - "pos": [ - 40, - 360 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "links": [ - 9, - 14 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumAnimSetup" - }, - "widgets_values": [ - 512, - 512, - 24, - 120, - 0, - "fixed" - ] - }, - { - "id": 8, - "type": "DifforumCameraMove", - "pos": [ - 360, - 460 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 9 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - } - ], - "outputs": [ - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "links": [ - 15 - ], - "slot_index": 0 - }, - { - "name": "info", - "type": "STRING", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumCameraMove" - }, - "widgets_values": [ - "spiral", - 1.0, - 1.0, - "2d", - 40.0 - ] - }, - { - "id": 9, - "type": "DifforumLiveSampler", - "pos": [ - 1000, - 320 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 10 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 11 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 12 - }, - { - "name": "vae", - "type": "VAE", - "link": 13 - }, - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 14 - }, - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "link": 15 - }, - { - "name": "init_image", - "type": "IMAGE", - "link": 16 - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 17 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumLiveSampler" - }, - "widgets_values": [ - 240, - 0.5, - 1, - 1.0, - "lcm", - "sgm_uniform", - 0.5, - "lab", - "kaleidoscope", - 6, - 0.0 - ] - }, - { - "id": 10, - "type": "SaveImage", - "pos": [ - 1360, - 320 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 9, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 17 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "SaveImage" - }, - "widgets_values": [ - "Difforum_live" - ] - }, - { - "id": 11, - "type": "Note", - "pos": [ - 40, - -210 - ], - "size": [ - 460, - 300 - ], - "flags": {}, - "order": 10, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "READ ME - models needed", - "properties": { - "text": "" - }, - "widgets_values": [ - "Difforum: Native realtime (Live Sampler internal loop).\n\nQueue ONCE and watch the node stream a live preview as it generates - the\nLive Sampler keeps the model resident and loops warp + kaleidoscope +\n1-step re-diffuse internally. No external runtime.\n\nModels:\n- SD-Turbo -> models/checkpoints (hf: stabilityai/sd-turbo)\n or any SD1.5 + LCM-LoRA (hf: latent-consistency/lcm-lora-sdv1-5)\nUse 512x512, 1-2 steps, cfg 1. Add TensorRT + TAESD for max FPS.\n\nMagic mirror: set the Live Sampler's live_source to 0 (webcam) or a video\npath to stylize a live feed (kaleidoscope your camera). Needs opencv.\nLive output: set stream_dir (folder for OBS/Resolume) or spout_name\n(needs SpoutGL). See REALTIME.md." - ], - "color": "#432", - "bgcolor": "#653" - } - ], - "links": [ - [ - 1, - 1, - 0, - 5, - 0, - "MODEL" - ], - [ - 2, - 1, - 1, - 2, - 0, - "CLIP" - ], - [ - 3, - 1, - 1, - 3, - 0, - "CLIP" - ], - [ - 4, - 2, - 0, - 5, - 1, - "CONDITIONING" - ], - [ - 5, - 3, - 0, - 5, - 2, - "CONDITIONING" - ], - [ - 6, - 4, - 0, - 5, - 3, - "LATENT" - ], - [ - 7, - 5, - 0, - 6, - 0, - "LATENT" - ], - [ - 8, - 1, - 2, - 6, - 1, - "VAE" - ], - [ - 9, - 7, - 0, - 8, - 0, - "DIFFORUM_PARAMS" - ], - [ - 10, - 1, - 0, - 9, - 0, - "MODEL" - ], - [ - 11, - 2, - 0, - 9, - 1, - "CONDITIONING" - ], - [ - 12, - 3, - 0, - 9, - 2, - "CONDITIONING" - ], - [ - 13, - 1, - 2, - 9, - 3, - "VAE" - ], - [ - 14, - 7, - 0, - 9, - 4, - "DIFFORUM_PARAMS" - ], - [ - 15, - 8, - 0, - 9, - 5, - "DIFFORUM_CAMERA" - ], - [ - 16, - 6, - 0, - 9, - 6, - "IMAGE" - ], - [ - 17, - 9, - 0, - 10, - 0, - "IMAGE" - ] - ], - "groups": [], - "config": {}, - "extra": {}, - "version": 0.4 -} \ No newline at end of file diff --git a/examples/advanced/difforum_seamless_installation.json b/examples/advanced/difforum_seamless_installation.json deleted file mode 100644 index 99c2881..0000000 --- a/examples/advanced/difforum_seamless_installation.json +++ /dev/null @@ -1,1334 +0,0 @@ -{ - "last_node_id": 21, - "last_link_id": 34, - "nodes": [ - { - "id": 16, - "type": "DifforumFilmDirector", - "pos": [ - 40, - 760 - ], - "size": [ - 560, - 340 - ], - "flags": {}, - "order": 16, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 21 - } - ], - "outputs": [ - { - "name": "camera_keys", - "type": "STRING", - "links": [ - 26 - ], - "slot_index": 0, - "shape": 3 - }, - { - "name": "prompt_schedule", - "type": "STRING", - "links": [ - 27 - ], - "slot_index": 1, - "shape": 3 - }, - { - "name": "strength_schedule", - "type": "STRING", - "links": [ - 28 - ], - "slot_index": 2, - "shape": 3 - }, - { - "name": "camera_shots", - "type": "STRING", - "links": null, - "slot_index": 3, - "shape": 3 - }, - { - "name": "info", - "type": "STRING", - "links": null, - "slot_index": 4, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DifforumFilmDirector" - }, - "widgets_values": [ - "[\n {\n \"frame_start\": 0,\n \"mood\": \"calm\",\n \"camera\": \"pan_right\",\n \"prompt\": \"deep ocean bioluminescence, drifting particles, dark teal, volumetric\"\n },\n {\n \"frame_start\": 30,\n \"mood\": \"dream\",\n \"camera\": \"spiral\",\n \"prompt\": \"coral architecture unfolding, soft glow, organic geometry\"\n },\n {\n \"frame_start\": 60,\n \"mood\": \"build\",\n \"camera\": \"pan_right\",\n \"prompt\": \"currents of light threading through, luminous filaments\"\n },\n {\n \"frame_start\": 90,\n \"mood\": \"resolve\",\n \"camera\": \"rise\",\n \"prompt\": \"settling into stillness, deep blue, suspended motes\"\n }\n]", - "cinematic", - 0.0, - 1.0, - 0.0, - 0 - ], - "title": "Film Director - drag the shots", - "color": "#323", - "bgcolor": "#535" - }, - { - "id": 1, - "type": "CheckpointLoaderSimple", - "pos": [ - 40, - 40 - ], - "size": [ - 310, - 98 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 1, - 8 - ], - "slot_index": 0, - "shape": 3 - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [ - 2, - 3, - 4 - ], - "slot_index": 1, - "shape": 3 - }, - { - "name": "VAE", - "type": "VAE", - "links": [ - 5, - 6 - ], - "slot_index": 2, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "CheckpointLoaderSimple" - }, - "widgets_values": [ - "juggernautXL_v9.safetensors" - ] - }, - { - "id": 2, - "type": "LoraLoader", - "pos": [ - 40, - 190 - ], - "size": [ - 310, - 126 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 1 - }, - { - "name": "clip", - "type": "CLIP", - "link": 2 - } - ], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 7 - ], - "slot_index": 0, - "shape": 3 - }, - { - "name": "CLIP", - "type": "CLIP", - "links": null, - "slot_index": 1, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "LoraLoader" - }, - "widgets_values": [ - "dmd2_sdxl_4step_lora.safetensors", - 1.0, - 1.0 - ], - "title": "DMD2 4-step distill" - }, - { - "id": 8, - "type": "DifforumAnimSetup", - "pos": [ - 40, - 370 - ], - "size": [ - 310, - 154 - ], - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "links": [ - 12, - 18, - 19, - 20, - 21, - 22 - ], - "slot_index": 0, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DifforumAnimSetup" - }, - "widgets_values": [ - 768, - 432, - 24.0, - 360, - 0 - ], - "title": "Anim Setup - 360f (3 laps of 120)", - "color": "#223", - "bgcolor": "#335" - }, - { - "id": 3, - "type": "CLIPTextEncode", - "pos": [ - 400, - 40 - ], - "size": [ - 350, - 90 - ], - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 3 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 9, - 10 - ], - "slot_index": 0, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "blurry, low quality, watermark, text, washed out, jpeg artifacts" - ], - "title": "Negative" - }, - { - "id": 4, - "type": "EmptyLatentImage", - "pos": [ - 400, - 180 - ], - "size": [ - 350, - 106 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 11 - ], - "slot_index": 0, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "EmptyLatentImage" - }, - "widgets_values": [ - 768, - 432, - 1 - ] - }, - { - "id": 5, - "type": "DifforumPromptSchedule", - "pos": [ - 660, - 760 - ], - "size": [ - 430, - 130 - ], - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 12 - }, - { - "name": "clip", - "type": "CLIP", - "link": 4 - }, - { - "name": "prompts", - "type": "STRING", - "link": 27, - "widget": { - "name": "prompts" - } - } - ], - "outputs": [ - { - "name": "positive_schedule", - "type": "DIFFORUM_PROMPT", - "links": [ - 15 - ], - "slot_index": 0, - "shape": 3 - }, - { - "name": "first_frame_cond", - "type": "CONDITIONING", - "links": [ - 13, - 14 - ], - "slot_index": 1, - "shape": 3 - }, - { - "name": "info", - "type": "STRING", - "links": null, - "slot_index": 2, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DifforumPromptSchedule" - }, - "widgets_values": [ - "0: placeholder", - "ease_in_out" - ], - "color": "#323", - "bgcolor": "#535" - }, - { - "id": 9, - "type": "DifforumCameraKeys", - "pos": [ - 660, - 940 - ], - "size": [ - 430, - 240 - ], - "flags": {}, - "order": 9, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 18 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - }, - { - "name": "keys", - "type": "STRING", - "link": 26, - "widget": { - "name": "keys" - } - } - ], - "outputs": [ - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "links": [ - 23, - 24 - ], - "slot_index": 0, - "shape": 3 - }, - { - "name": "cycle_frames", - "type": "INT", - "links": [ - 33 - ], - "slot_index": 1, - "shape": 3 - }, - { - "name": "info", - "type": "STRING", - "links": null, - "slot_index": 2, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DifforumCameraKeys" - }, - "widgets_values": [ - "0: still 1 1 40 linear", - 1.0, - 1.0, - "2d", - "harmonic", - 120, - 3 - ], - "title": "Camera Keys - blend + lens + loop", - "color": "#323", - "bgcolor": "#535" - }, - { - "id": 10, - "type": "DifforumSchedule", - "pos": [ - 660, - 1230 - ], - "size": [ - 430, - 130 - ], - "flags": {}, - "order": 10, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 19 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - }, - { - "name": "schedule", - "type": "STRING", - "link": 28, - "widget": { - "name": "schedule" - } - } - ], - "outputs": [ - { - "name": "schedule", - "type": "DIFFORUM_SCHEDULE", - "links": [ - 25 - ], - "slot_index": 0, - "shape": 3 - }, - { - "name": "values", - "type": "FLOAT", - "links": null, - "slot_index": 1, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DifforumSchedule" - }, - "widgets_values": [ - "0:(0.45)", - "linear" - ], - "title": "Strength (denoise)", - "color": "#323", - "bgcolor": "#535" - }, - { - "id": 6, - "type": "KSampler", - "pos": [ - 800, - 40 - ], - "size": [ - 310, - 262 - ], - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 8 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 13 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 9 - }, - { - "name": "latent_image", - "type": "LATENT", - "link": 11 - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 16 - ], - "slot_index": 0, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "KSampler" - }, - "widgets_values": [ - 0, - "fixed", - 26, - 6.5, - "dpmpp_2m", - "karras", - 1.0 - ], - "title": "Frame 0 (full quality)" - }, - { - "id": 7, - "type": "VAEDecode", - "pos": [ - 1160, - 40 - ], - "size": [ - 230, - 46 - ], - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 16 - }, - { - "name": "vae", - "type": "VAE", - "link": 5 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 17, - 29 - ], - "slot_index": 0, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "VAEDecode" - }, - "widgets_values": [] - }, - { - "id": 20, - "type": "DifforumStoryboard", - "pos": [ - 1160, - 900 - ], - "size": [ - 330, - 300 - ], - "flags": {}, - "order": 20, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 22 - }, - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "link": 24 - }, - { - "name": "init_image", - "type": "IMAGE", - "link": 29 - }, - { - "name": "depth", - "type": "IMAGE", - "link": null - } - ], - "outputs": [ - { - "name": "sheet", - "type": "IMAGE", - "links": [ - 30 - ], - "slot_index": 0, - "shape": 3 - }, - { - "name": "frames", - "type": "IMAGE", - "links": null, - "slot_index": 1, - "shape": 3 - }, - { - "name": "info", - "type": "STRING", - "links": null, - "slot_index": 2, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DifforumStoryboard" - }, - "widgets_values": [ - 8, - 6, - 192, - 0.5, - "reflection", - "none", - 6, - 1.0 - ], - "title": "Storyboard - no diffusion", - "color": "#232", - "bgcolor": "#353" - }, - { - "id": 21, - "type": "PreviewImage", - "pos": [ - 1520, - 900 - ], - "size": [ - 420, - 340 - ], - "flags": {}, - "order": 21, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 30 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "PreviewImage" - }, - "widgets_values": [], - "title": "Storyboard (run this first)" - }, - { - "id": 11, - "type": "DifforumFeedbackSampler", - "pos": [ - 1560, - 180 - ], - "size": [ - 400, - 470 - ], - "flags": {}, - "order": 11, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 7 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 14 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 10 - }, - { - "name": "vae", - "type": "VAE", - "link": 6 - }, - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 20 - }, - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "link": 23 - }, - { - "name": "init_image", - "type": "IMAGE", - "link": 17 - }, - { - "name": "strength_schedule", - "type": "DIFFORUM_SCHEDULE", - "link": 25 - }, - { - "name": "depth", - "type": "IMAGE", - "link": null - }, - { - "name": "cfg_schedule", - "type": "DIFFORUM_SCHEDULE", - "link": null - }, - { - "name": "positive_schedule", - "type": "DIFFORUM_PROMPT", - "link": 15 - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 31 - ], - "slot_index": 0, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DifforumFeedbackSampler" - }, - "widgets_values": [ - 5, - 1.2, - "lcm", - "sgm_uniform", - 0.8, - "lab", - 1.0, - 100.0, - false, - 1.0, - 0.6, - "none", - 6, - "reflection", - 0.25, - 0.025, - 2, - 0, - 0 - ] - }, - { - "id": 12, - "type": "DifforumFlowStabilize", - "pos": [ - 2010, - 180 - ], - "size": [ - 290, - 106 - ], - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [ - { - "name": "frames", - "type": "IMAGE", - "link": 31 - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 32 - ], - "slot_index": 0, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DifforumFlowStabilize" - }, - "widgets_values": [ - 0.5, - 0.5, - 0.15 - ] - }, - { - "id": 14, - "type": "DifforumLoopTake", - "pos": [ - 2010, - 340 - ], - "size": [ - 290, - 110 - ], - "flags": {}, - "order": 14, - "mode": 0, - "inputs": [ - { - "name": "frames", - "type": "IMAGE", - "link": 32 - }, - { - "name": "cycle_frames", - "type": "INT", - "link": 33, - "widget": { - "name": "cycle_frames" - } - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 34 - ], - "slot_index": 0, - "shape": 3 - }, - { - "name": "seam_info", - "type": "STRING", - "links": null, - "slot_index": 1, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "DifforumLoopTake" - }, - "widgets_values": [ - 120, - "last" - ], - "title": "Loop Take - settled lap", - "color": "#232", - "bgcolor": "#353" - }, - { - "id": 13, - "type": "SaveImage", - "pos": [ - 2350, - 180 - ], - "size": [ - 430, - 400 - ], - "flags": {}, - "order": 13, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 34 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "SaveImage" - }, - "widgets_values": [ - "Difforum_loop" - ] - }, - { - "id": 15, - "type": "Note", - "pos": [ - 40, - 1120 - ], - "size": [ - 560, - 420 - ], - "flags": {}, - "order": 15, - "mode": 0, - "inputs": [], - "outputs": [], - "properties": { - "Node name for S&R": "Note" - }, - "widgets_values": [ - "DIFFORUM - SEAMLESS LOOP (installations)\n\nLoop Blend crossfades, and on a projection that runs for hours the eye\nfinds that dissolve every cycle. Nothing is blended here - the clip is\ngenuinely periodic instead.\n\nHOW IT WORKS\n1. Camera Keys with loop_mode=harmonic projects the motion onto\n components that are periodic over cycle_frames. The path returns\n exactly to its origin AND the velocity matches across the wrap.\n2. Anim Setup renders 3 LAPS (360f) of the same 120f cycle. The first\n two exist only so the feedback image can settle onto its cycle.\n3. Loop Take discards the early laps and returns the last one, with\n no pixel touched.\n\nREAD seam_info ON LOOP TAKE\n seam < 0.02 excellent 0.02-0.05 good\n 0.05-0.10 visible > 0.10 poor\n \"not settled yet\" -> raise max_frames to 4 laps (480)\n \"settled\" but seam shows -> lower harmonics to 2\n\nharmonics controls how much of the original move survives:\n 1 perfect seam, very simple motion\n 3 balanced (default)\n 6+ faithful to the shot list, seam may show\n\nCOST: 3 laps is 3x the render. You render once and project forever.\n\nMOVES THAT NEED A DEPTH MAP\n dolly_in, dolly_out, orbit_left, orbit_right and sway drive camera\n motion THROUGH space. The 2D affine warp cannot express that, and\n the Feedback Sampler silently falls back to the 2D warp whenever\n `depth` is not connected - even with mode set to 3d. Those moves\n then render as a frozen frame, with nothing in the log to say why.\n\n Flat moves that always work:\n pan_left/right, pan_up/down, zoom_in/out, roll_cw/ccw,\n spiral, dolly_zoom, rise, shake\n\n For real parallax (near pixels moving faster than far ones) use\n examples/advanced/difforum_parallax_fluid.json - it wires\n Depth Anything V2 into the sampler.\n" - ], - "title": "READ THIS", - "color": "#432", - "bgcolor": "#653" - } - ], - "links": [ - [ - 1, - 1, - 0, - 2, - 0, - "MODEL" - ], - [ - 2, - 1, - 1, - 2, - 1, - "CLIP" - ], - [ - 3, - 1, - 1, - 3, - 0, - "CLIP" - ], - [ - 4, - 1, - 1, - 5, - 1, - "CLIP" - ], - [ - 5, - 1, - 2, - 7, - 1, - "VAE" - ], - [ - 6, - 1, - 2, - 11, - 3, - "VAE" - ], - [ - 7, - 2, - 0, - 11, - 0, - "MODEL" - ], - [ - 8, - 1, - 0, - 6, - 0, - "MODEL" - ], - [ - 9, - 3, - 0, - 6, - 2, - "CONDITIONING" - ], - [ - 10, - 3, - 0, - 11, - 2, - "CONDITIONING" - ], - [ - 11, - 4, - 0, - 6, - 3, - "LATENT" - ], - [ - 12, - 8, - 0, - 5, - 0, - "DIFFORUM_PARAMS" - ], - [ - 13, - 5, - 1, - 6, - 1, - "CONDITIONING" - ], - [ - 14, - 5, - 1, - 11, - 1, - "CONDITIONING" - ], - [ - 15, - 5, - 0, - 11, - 10, - "DIFFORUM_PROMPT" - ], - [ - 16, - 6, - 0, - 7, - 0, - "LATENT" - ], - [ - 17, - 7, - 0, - 11, - 6, - "IMAGE" - ], - [ - 18, - 8, - 0, - 9, - 0, - "DIFFORUM_PARAMS" - ], - [ - 19, - 8, - 0, - 10, - 0, - "DIFFORUM_PARAMS" - ], - [ - 20, - 8, - 0, - 11, - 4, - "DIFFORUM_PARAMS" - ], - [ - 21, - 8, - 0, - 16, - 0, - "DIFFORUM_PARAMS" - ], - [ - 22, - 8, - 0, - 20, - 0, - "DIFFORUM_PARAMS" - ], - [ - 23, - 9, - 0, - 11, - 5, - "DIFFORUM_CAMERA" - ], - [ - 24, - 9, - 0, - 20, - 1, - "DIFFORUM_CAMERA" - ], - [ - 25, - 10, - 0, - 11, - 7, - "DIFFORUM_SCHEDULE" - ], - [ - 26, - 16, - 0, - 9, - 2, - "STRING" - ], - [ - 27, - 16, - 1, - 5, - 2, - "STRING" - ], - [ - 28, - 16, - 2, - 10, - 2, - "STRING" - ], - [ - 29, - 7, - 0, - 20, - 2, - "IMAGE" - ], - [ - 30, - 20, - 0, - 21, - 0, - "IMAGE" - ], - [ - 31, - 11, - 0, - 12, - 0, - "IMAGE" - ], - [ - 32, - 12, - 0, - 14, - 0, - "IMAGE" - ], - [ - 33, - 9, - 1, - 14, - 1, - "INT" - ], - [ - 34, - 14, - 0, - 13, - 0, - "IMAGE" - ] - ], - "groups": [ - { - "title": "1 - DIRECTION (edit here)", - "bounding": [ - 20, - 690, - 1080, - 620 - ], - "color": "#8A8", - "font_size": 24, - "flags": {} - }, - { - "title": "2 - Model + frame 0", - "bounding": [ - 20, - 0, - 1380, - 330 - ], - "color": "#3f789e", - "font_size": 24, - "flags": {} - }, - { - "title": "3 - Cheap preview", - "bounding": [ - 1140, - 850, - 820, - 400 - ], - "color": "#4a8", - "font_size": 24, - "flags": {} - }, - { - "title": "4 - Render", - "bounding": [ - 1530, - 110, - 1280, - 560 - ], - "color": "#a1309b", - "font_size": 24, - "flags": {} - } - ], - "config": {}, - "extra": { - "ds": { - "scale": 0.55, - "offset": [ - 60, - 40 - ] - } - }, - "version": 0.4 -} \ No newline at end of file diff --git a/examples/basic/difforum_audio_reactive.json b/examples/basic/difforum_audio_reactive.json deleted file mode 100644 index 6c1efeb..0000000 --- a/examples/basic/difforum_audio_reactive.json +++ /dev/null @@ -1,423 +0,0 @@ -{ - "last_node_id": 8, - "last_link_id": 8, - "nodes": [ - { - "id": 1, - "type": "DifforumAnimSetup", - "pos": [ - 80, - 120 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "links": [ - 1, - 3, - 5 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumAnimSetup" - }, - "widgets_values": [ - 768, - 768, - 24, - 240, - 0, - "fixed" - ] - }, - { - "id": 2, - "type": "LoadAudio", - "pos": [ - 80, - 420 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "AUDIO", - "type": "AUDIO", - "links": [ - 2 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "LoadAudio" - }, - "widgets_values": [ - "your_track.mp3" - ] - }, - { - "id": 3, - "type": "DifforumAudioAnalyzer", - "pos": [ - 460, - 360 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 1 - }, - { - "name": "audio", - "type": "AUDIO", - "link": 2 - } - ], - "outputs": [ - { - "name": "audio_curves", - "type": "DIFFORUM_AUDIO", - "links": [ - 4, - 6 - ], - "slot_index": 0 - }, - { - "name": "amp", - "type": "FLOAT", - "links": [], - "slot_index": 1 - }, - { - "name": "low", - "type": "FLOAT", - "links": [], - "slot_index": 2 - }, - { - "name": "mid", - "type": "FLOAT", - "links": [], - "slot_index": 3 - }, - { - "name": "high", - "type": "FLOAT", - "links": [], - "slot_index": 4 - }, - { - "name": "onset", - "type": "FLOAT", - "links": [], - "slot_index": 5 - }, - { - "name": "beat", - "type": "FLOAT", - "links": [], - "slot_index": 6 - } - ], - "properties": { - "Node name for S&R": "DifforumAudioAnalyzer" - }, - "widgets_values": [ - 0.25, - true, - 1.5 - ] - }, - { - "id": 4, - "type": "DifforumSchedule", - "pos": [ - 880, - 120 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 3 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": 4 - } - ], - "outputs": [ - { - "name": "schedule", - "type": "DIFFORUM_SCHEDULE", - "links": [ - 7 - ], - "slot_index": 0 - }, - { - "name": "values", - "type": "FLOAT", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumSchedule" - }, - "widgets_values": [ - "0:(1.0 + 0.6*low)", - "linear" - ] - }, - { - "id": 5, - "type": "DifforumSchedule", - "pos": [ - 880, - 380 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 5 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": 6 - } - ], - "outputs": [ - { - "name": "schedule", - "type": "DIFFORUM_SCHEDULE", - "links": [ - 8 - ], - "slot_index": 0 - }, - { - "name": "values", - "type": "FLOAT", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumSchedule" - }, - "widgets_values": [ - "0:(beat*30)", - "linear" - ] - }, - { - "id": 6, - "type": "DifforumScheduleInfo", - "pos": [ - 1280, - 120 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "schedule", - "type": "DIFFORUM_SCHEDULE", - "link": 7 - } - ], - "outputs": [ - { - "name": "info", - "type": "STRING", - "links": [], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumScheduleInfo" - }, - "widgets_values": [] - }, - { - "id": 7, - "type": "DifforumScheduleInfo", - "pos": [ - 1280, - 380 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [ - { - "name": "schedule", - "type": "DIFFORUM_SCHEDULE", - "link": 8 - } - ], - "outputs": [ - { - "name": "info", - "type": "STRING", - "links": [], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumScheduleInfo" - }, - "widgets_values": [] - }, - { - "id": 8, - "type": "Note", - "pos": [ - 40, - -210 - ], - "size": [ - 460, - 300 - ], - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "READ ME - models needed", - "properties": { - "text": "" - }, - "widgets_values": [ - "Difforum: Audio analysis (no render).\n\nNeeds:\n- an audio file (.mp3/.wav) -> dropped in the Load Audio node\nNo checkpoint required." - ], - "color": "#432", - "bgcolor": "#653" - } - ], - "links": [ - [ - 1, - 1, - 0, - 3, - 0, - "DIFFORUM_PARAMS" - ], - [ - 2, - 2, - 0, - 3, - 1, - "AUDIO" - ], - [ - 3, - 1, - 0, - 4, - 0, - "DIFFORUM_PARAMS" - ], - [ - 4, - 3, - 0, - 4, - 1, - "DIFFORUM_AUDIO" - ], - [ - 5, - 1, - 0, - 5, - 0, - "DIFFORUM_PARAMS" - ], - [ - 6, - 3, - 0, - 5, - 1, - "DIFFORUM_AUDIO" - ], - [ - 7, - 4, - 0, - 6, - 0, - "DIFFORUM_SCHEDULE" - ], - [ - 8, - 5, - 0, - 7, - 0, - "DIFFORUM_SCHEDULE" - ] - ], - "groups": [], - "config": {}, - "extra": {}, - "version": 0.4 -} \ No newline at end of file diff --git a/examples/basic/difforum_camera_warp.json b/examples/basic/difforum_camera_warp.json deleted file mode 100644 index fef42ae..0000000 --- a/examples/basic/difforum_camera_warp.json +++ /dev/null @@ -1,283 +0,0 @@ -{ - "last_node_id": 6, - "last_link_id": 4, - "nodes": [ - { - "id": 1, - "type": "LoadImage", - "pos": [ - 40, - 60 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 2 - ], - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "example.png" - ] - }, - { - "id": 2, - "type": "DifforumAnimSetup", - "pos": [ - 40, - 360 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "links": [ - 1 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumAnimSetup" - }, - "widgets_values": [ - 768, - 768, - 24, - 48, - 0, - "fixed" - ] - }, - { - "id": 3, - "type": "DifforumCameraMove", - "pos": [ - 360, - 360 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 1 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - } - ], - "outputs": [ - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "links": [ - 3 - ], - "slot_index": 0 - }, - { - "name": "info", - "type": "STRING", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumCameraMove" - }, - "widgets_values": [ - "spiral", - 1.0, - 1.0, - "2d", - 40.0 - ] - }, - { - "id": 4, - "type": "DifforumWarp", - "pos": [ - 760, - 120 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [ - { - "name": "image", - "type": "IMAGE", - "link": 2 - }, - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "link": 3 - }, - { - "name": "depth", - "type": "IMAGE", - "link": null - } - ], - "outputs": [ - { - "name": "warped", - "type": "IMAGE", - "links": [ - 4 - ], - "slot_index": 0 - }, - { - "name": "occlusion_mask", - "type": "MASK", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumWarp" - }, - "widgets_values": [ - 12, - "follow_camera", - 1.0, - 100.0, - false - ] - }, - { - "id": 5, - "type": "PreviewImage", - "pos": [ - 1120, - 120 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 4 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "PreviewImage" - }, - "widgets_values": [] - }, - { - "id": 6, - "type": "Note", - "pos": [ - 40, - -210 - ], - "size": [ - 460, - 300 - ], - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "READ ME - models needed", - "properties": { - "text": "" - }, - "widgets_values": [ - "Difforum: Warp preview.\n\nNeeds:\n- an input image -> Load Image\nNo checkpoint (this only previews the 2D/3D warp + occlusion mask).\nTip: uses the intuitive Camera Move presets." - ], - "color": "#432", - "bgcolor": "#653" - } - ], - "links": [ - [ - 1, - 2, - 0, - 3, - 0, - "DIFFORUM_PARAMS" - ], - [ - 2, - 1, - 0, - 4, - 0, - "IMAGE" - ], - [ - 3, - 3, - 0, - 4, - 1, - "DIFFORUM_CAMERA" - ], - [ - 4, - 4, - 0, - 5, - 0, - "IMAGE" - ] - ], - "groups": [], - "config": {}, - "extra": {}, - "version": 0.4 -} \ No newline at end of file diff --git a/examples/basic/difforum_feedback_classic.json b/examples/basic/difforum_feedback_classic.json deleted file mode 100644 index 9ab73fa..0000000 --- a/examples/basic/difforum_feedback_classic.json +++ /dev/null @@ -1,960 +0,0 @@ -{ - "last_node_id": 15, - "last_link_id": 24, - "nodes": [ - { - "id": 1, - "type": "CheckpointLoaderSimple", - "pos": [ - 40, - 40 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 1, - 13 - ], - "slot_index": 0 - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [ - 2, - 3, - 12 - ], - "slot_index": 1 - }, - { - "name": "VAE", - "type": "VAE", - "links": [ - 8, - 16 - ], - "slot_index": 2 - } - ], - "properties": { - "Node name for S&R": "CheckpointLoaderSimple" - }, - "widgets_values": [ - "dreamshaper_8.safetensors" - ] - }, - { - "id": 2, - "type": "CLIPTextEncode", - "pos": [ - 340, - 20 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 2 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 4, - 14 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "psychedelic cosmic tunnel, vivid colors, highly detailed" - ] - }, - { - "id": 3, - "type": "CLIPTextEncode", - "pos": [ - 340, - 200 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 3 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 5, - 15 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "blurry, low quality, watermark, text" - ] - }, - { - "id": 4, - "type": "EmptyLatentImage", - "pos": [ - 340, - 380 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 6 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "EmptyLatentImage" - }, - "widgets_values": [ - 768, - 768, - 1 - ] - }, - { - "id": 5, - "type": "KSampler", - "pos": [ - 660, - 120 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 1 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 4 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 5 - }, - { - "name": "latent_image", - "type": "LATENT", - "link": 6 - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 7 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "KSampler" - }, - "widgets_values": [ - 0, - "fixed", - 26, - 6.5, - "dpmpp_2m", - "karras", - 1.0 - ] - }, - { - "id": 6, - "type": "VAEDecode", - "pos": [ - 980, - 120 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 7 - }, - { - "name": "vae", - "type": "VAE", - "link": 8 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 19 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "VAEDecode" - }, - "widgets_values": [] - }, - { - "id": 7, - "type": "DifforumAnimSetup", - "pos": [ - 40, - 520 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "links": [ - 9, - 10, - 11, - 17 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumAnimSetup" - }, - "widgets_values": [ - 768, - 768, - 24, - 48, - 0, - "fixed" - ] - }, - { - "id": 8, - "type": "DifforumCamera", - "pos": [ - 340, - 560 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 9 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - } - ], - "outputs": [ - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "links": [ - 18 - ], - "slot_index": 0 - }, - { - "name": "info", - "type": "STRING", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumCamera" - }, - "widgets_values": [ - "3d", - 40.0, - "0:(0)", - "0:(0)", - "0:(1.5)", - "0:(0)", - "0:(0.5*sin(2*pi*t/120))", - "0:(0)", - "0:(1.0)" - ] - }, - { - "id": 9, - "type": "DifforumSchedule", - "pos": [ - 340, - 820 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 10 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - } - ], - "outputs": [ - { - "name": "schedule", - "type": "DIFFORUM_SCHEDULE", - "links": [ - 20 - ], - "slot_index": 0 - }, - { - "name": "values", - "type": "FLOAT", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumSchedule" - }, - "widgets_values": [ - "0:(0.45)", - "linear" - ] - }, - { - "id": 10, - "type": "DifforumPromptSchedule", - "pos": [ - 340, - 1060 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 9, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 11 - }, - { - "name": "clip", - "type": "CLIP", - "link": 12 - } - ], - "outputs": [ - { - "name": "positive_schedule", - "type": "DIFFORUM_PROMPT", - "links": [ - 21 - ], - "slot_index": 0 - }, - { - "name": "first_frame_cond", - "type": "CONDITIONING", - "links": [], - "slot_index": 1 - }, - { - "name": "info", - "type": "STRING", - "links": [], - "slot_index": 2 - } - ], - "properties": { - "Node name for S&R": "DifforumPromptSchedule" - }, - "widgets_values": [ - "0: psychedelic cosmic tunnel, vivid colors\n24: deep space nebula, glowing stars\n47: intricate fractal mandala, vivid", - "ease_in_out" - ] - }, - { - "id": 11, - "type": "DifforumFeedbackSampler", - "pos": [ - 1000, - 480 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 10, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 13 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 14 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 15 - }, - { - "name": "vae", - "type": "VAE", - "link": 16 - }, - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 17 - }, - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "link": 18 - }, - { - "name": "init_image", - "type": "IMAGE", - "link": 19 - }, - { - "name": "strength_schedule", - "type": "DIFFORUM_SCHEDULE", - "link": 20 - }, - { - "name": "depth", - "type": "IMAGE", - "link": null - }, - { - "name": "cfg_schedule", - "type": "DIFFORUM_SCHEDULE", - "link": null - }, - { - "name": "positive_schedule", - "type": "DIFFORUM_PROMPT", - "link": 21 - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 22 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumFeedbackSampler" - }, - "widgets_values": [ - 24, - 6.5, - "dpmpp_2m", - "karras", - 0.8, - "lab", - 1.0, - 100.0, - false, - 1.0, - 0.6, - "none", - 6, - "reflection", - 0.25, - 0.02, - 1, - 0, - 0 - ] - }, - { - "id": 12, - "type": "SaveImage", - "pos": [ - 1700, - 480 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 11, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 24 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "SaveImage" - }, - "widgets_values": [ - "Difforum" - ] - }, - { - "id": 13, - "type": "DifforumFlowStabilize", - "pos": [ - 1340, - 480 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [ - { - "name": "frames", - "type": "IMAGE", - "link": 22 - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 23 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumFlowStabilize" - }, - "widgets_values": [ - 0.5, - 0.5, - 0.15 - ] - }, - { - "id": 14, - "type": "DifforumLoopBlend", - "pos": [ - 1520, - 660 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 13, - "mode": 0, - "inputs": [ - { - "name": "frames", - "type": "IMAGE", - "link": 23 - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 24 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumLoopBlend" - }, - "widgets_values": [ - 12, - true - ] - }, - { - "id": 15, - "type": "Note", - "pos": [ - 40, - -210 - ], - "size": [ - 460, - 300 - ], - "flags": {}, - "order": 14, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "READ ME - models needed", - "properties": { - "text": "" - }, - "widgets_values": [ - "Difforum: Classic+ feedback video.\n\nModels:\n- SD1.5 checkpoint -> models/checkpoints (DreamShaper: civitai.com/models/4384)\n- (optional) better VAE vae-ft-mse-840000 -> models/vae (hf: stabilityai/sd-vae-ft-mse)\n\nQuality upgrade: swap the checkpoint for SDXL or Flux (model-agnostic).\nGroups: 4 = anti-flicker (Flow Stabilize) - select it and Ctrl+B to bypass.\nIn the sampler: sharpen/noise = detail guard, cadence 2-3 = faster long videos.\nTip: try Camera Shots + Prompt Scenes for an easier setup." - ], - "color": "#432", - "bgcolor": "#653" - } - ], - "links": [ - [ - 1, - 1, - 0, - 5, - 0, - "MODEL" - ], - [ - 2, - 1, - 1, - 2, - 0, - "CLIP" - ], - [ - 3, - 1, - 1, - 3, - 0, - "CLIP" - ], - [ - 4, - 2, - 0, - 5, - 1, - "CONDITIONING" - ], - [ - 5, - 3, - 0, - 5, - 2, - "CONDITIONING" - ], - [ - 6, - 4, - 0, - 5, - 3, - "LATENT" - ], - [ - 7, - 5, - 0, - 6, - 0, - "LATENT" - ], - [ - 8, - 1, - 2, - 6, - 1, - "VAE" - ], - [ - 9, - 7, - 0, - 8, - 0, - "DIFFORUM_PARAMS" - ], - [ - 10, - 7, - 0, - 9, - 0, - "DIFFORUM_PARAMS" - ], - [ - 11, - 7, - 0, - 10, - 0, - "DIFFORUM_PARAMS" - ], - [ - 12, - 1, - 1, - 10, - 1, - "CLIP" - ], - [ - 13, - 1, - 0, - 11, - 0, - "MODEL" - ], - [ - 14, - 2, - 0, - 11, - 1, - "CONDITIONING" - ], - [ - 15, - 3, - 0, - 11, - 2, - "CONDITIONING" - ], - [ - 16, - 1, - 2, - 11, - 3, - "VAE" - ], - [ - 17, - 7, - 0, - 11, - 4, - "DIFFORUM_PARAMS" - ], - [ - 18, - 8, - 0, - 11, - 5, - "DIFFORUM_CAMERA" - ], - [ - 19, - 6, - 0, - 11, - 6, - "IMAGE" - ], - [ - 20, - 9, - 0, - 11, - 7, - "DIFFORUM_SCHEDULE" - ], - [ - 21, - 10, - 0, - 11, - 10, - "DIFFORUM_PROMPT" - ], - [ - 22, - 11, - 0, - 13, - 0, - "IMAGE" - ], - [ - 23, - 13, - 0, - 14, - 0, - "IMAGE" - ], - [ - 24, - 14, - 0, - 12, - 0, - "IMAGE" - ] - ], - "groups": [ - { - "title": "1 - FIRST FRAME (txt2img)", - "bounding": [ - 10, - -60, - 1270, - 500 - ], - "color": "#33415a", - "font_size": 22, - "locked": false - }, - { - "title": "2 - ANIMATION BRAIN (camera + schedules + prompt travel)", - "bounding": [ - 10, - 470, - 900, - 920 - ], - "color": "#33503f", - "font_size": 22, - "locked": false - }, - { - "title": "3 - RENDER LOOP (detail guard inside)", - "bounding": [ - 960, - 390, - 350, - 480 - ], - "color": "#5a3a33", - "font_size": 22, - "locked": false - }, - { - "title": "4 - QUALITY: anti-flicker (select + Ctrl+B to bypass)", - "bounding": [ - 1310, - 390, - 350, - 300 - ], - "color": "#4a3d5a", - "font_size": 22, - "locked": false - } - ], - "config": {}, - "extra": {}, - "version": 0.4 -} \ No newline at end of file diff --git a/examples/basic/difforum_intuitive_controls.json b/examples/basic/difforum_intuitive_controls.json deleted file mode 100644 index 18b5dcc..0000000 --- a/examples/basic/difforum_intuitive_controls.json +++ /dev/null @@ -1,1039 +0,0 @@ -{ - "last_node_id": 17, - "last_link_id": 26, - "nodes": [ - { - "id": 1, - "type": "CheckpointLoaderSimple", - "pos": [ - 40, - 40 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 1, - 15 - ], - "slot_index": 0 - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [ - 2, - 3, - 14 - ], - "slot_index": 1 - }, - { - "name": "VAE", - "type": "VAE", - "links": [ - 8, - 18 - ], - "slot_index": 2 - } - ], - "properties": { - "Node name for S&R": "CheckpointLoaderSimple" - }, - "widgets_values": [ - "dreamshaper_8.safetensors" - ] - }, - { - "id": 2, - "type": "CLIPTextEncode", - "pos": [ - 340, - 20 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 2 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 4, - 16 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "a serene misty forest, soft volumetric light" - ] - }, - { - "id": 3, - "type": "CLIPTextEncode", - "pos": [ - 340, - 200 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 3 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 5, - 17 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "blurry, low quality, watermark, text" - ] - }, - { - "id": 4, - "type": "EmptyLatentImage", - "pos": [ - 340, - 380 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 6 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "EmptyLatentImage" - }, - "widgets_values": [ - 768, - 768, - 1 - ] - }, - { - "id": 5, - "type": "KSampler", - "pos": [ - 660, - 120 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 1 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 4 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 5 - }, - { - "name": "latent_image", - "type": "LATENT", - "link": 6 - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 7 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "KSampler" - }, - "widgets_values": [ - 0, - "fixed", - 26, - 6.5, - "dpmpp_2m", - "karras", - 1.0 - ] - }, - { - "id": 6, - "type": "VAEDecode", - "pos": [ - 980, - 120 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 7 - }, - { - "name": "vae", - "type": "VAE", - "link": 8 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 21 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "VAEDecode" - }, - "widgets_values": [] - }, - { - "id": 7, - "type": "DifforumAnimSetup", - "pos": [ - 40, - 460 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "links": [ - 9, - 12, - 13, - 19 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumAnimSetup" - }, - "widgets_values": [ - 768, - 768, - 24, - 48, - 0, - "fixed" - ] - }, - { - "id": 8, - "type": "DifforumCameraShots", - "pos": [ - 340, - 500 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 9 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - } - ], - "outputs": [ - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "links": [ - 10, - 20 - ], - "slot_index": 0 - }, - { - "name": "info", - "type": "STRING", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumCameraShots" - }, - "widgets_values": [ - "0: dolly_in 1.0 1.0\n24: orbit_right 1.0 0.8\n36: spiral 1.0 1.2", - "2d", - 40.0 - ] - }, - { - "id": 9, - "type": "DifforumCameraPreview", - "pos": [ - 660, - 540 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "link": 10 - } - ], - "outputs": [ - { - "name": "path_image", - "type": "IMAGE", - "links": [ - 11 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumCameraPreview" - }, - "widgets_values": [ - 512 - ] - }, - { - "id": 10, - "type": "PreviewImage", - "pos": [ - 660, - 700 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 9, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 11 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "PreviewImage" - }, - "widgets_values": [] - }, - { - "id": 11, - "type": "DifforumSchedule", - "pos": [ - 340, - 720 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 10, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 12 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - } - ], - "outputs": [ - { - "name": "schedule", - "type": "DIFFORUM_SCHEDULE", - "links": [ - 22 - ], - "slot_index": 0 - }, - { - "name": "values", - "type": "FLOAT", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumSchedule" - }, - "widgets_values": [ - "0:(0.5)", - "linear" - ] - }, - { - "id": 12, - "type": "DifforumPromptScenes", - "pos": [ - 340, - 920 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 11, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 13 - }, - { - "name": "clip", - "type": "CLIP", - "link": 14 - } - ], - "outputs": [ - { - "name": "positive_schedule", - "type": "DIFFORUM_PROMPT", - "links": [ - 23 - ], - "slot_index": 0 - }, - { - "name": "first_frame_cond", - "type": "CONDITIONING", - "links": [], - "slot_index": 1 - }, - { - "name": "info", - "type": "STRING", - "links": [], - "slot_index": 2 - } - ], - "properties": { - "Node name for S&R": "DifforumPromptScenes" - }, - "widgets_values": [ - "a serene misty forest, soft light", - "a glowing crystal cave, bioluminescent", - "a vast starry galaxy, swirling nebula", - "", - "ease_in_out" - ] - }, - { - "id": 13, - "type": "DifforumFeedbackSampler", - "pos": [ - 1000, - 460 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 15 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 16 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 17 - }, - { - "name": "vae", - "type": "VAE", - "link": 18 - }, - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 19 - }, - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "link": 20 - }, - { - "name": "init_image", - "type": "IMAGE", - "link": 21 - }, - { - "name": "strength_schedule", - "type": "DIFFORUM_SCHEDULE", - "link": 22 - }, - { - "name": "depth", - "type": "IMAGE", - "link": null - }, - { - "name": "cfg_schedule", - "type": "DIFFORUM_SCHEDULE", - "link": null - }, - { - "name": "positive_schedule", - "type": "DIFFORUM_PROMPT", - "link": 23 - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 24 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumFeedbackSampler" - }, - "widgets_values": [ - 24, - 6.5, - "dpmpp_2m", - "karras", - 0.85, - "lab", - 1.0, - 100.0, - false, - 1.0, - 0.6, - "none", - 6, - "reflection", - 0.25, - 0.02, - 1, - 0, - 0 - ] - }, - { - "id": 14, - "type": "SaveImage", - "pos": [ - 1700, - 460 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 13, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 26 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "SaveImage" - }, - "widgets_values": [ - "Difforum" - ] - }, - { - "id": 15, - "type": "DifforumFlowStabilize", - "pos": [ - 1340, - 460 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 14, - "mode": 0, - "inputs": [ - { - "name": "frames", - "type": "IMAGE", - "link": 24 - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 25 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumFlowStabilize" - }, - "widgets_values": [ - 0.5, - 0.5, - 0.15 - ] - }, - { - "id": 16, - "type": "DifforumLoopBlend", - "pos": [ - 1520, - 660 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 15, - "mode": 0, - "inputs": [ - { - "name": "frames", - "type": "IMAGE", - "link": 25 - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 26 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumLoopBlend" - }, - "widgets_values": [ - 12, - true - ] - }, - { - "id": 17, - "type": "Note", - "pos": [ - 40, - -210 - ], - "size": [ - 460, - 300 - ], - "flags": {}, - "order": 16, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "READ ME - models needed", - "properties": { - "text": "" - }, - "widgets_values": [ - "Difforum: Intuitive controls (Camera Move presets + Prompt Scenes).\n\nModels:\n- SD1.5 checkpoint -> models/checkpoints (DreamShaper: civitai.com/models/4384)\nSwap to SDXL/Flux for higher quality.\n\nCamera Shots = direct like an edit (frame: preset speed intensity), and the\nPath Preview shows the trajectory BEFORE you render. Group 4 = anti-flicker\n(select + Ctrl+B to bypass). cadence 2-3 in the sampler = faster long videos." - ], - "color": "#432", - "bgcolor": "#653" - } - ], - "links": [ - [ - 1, - 1, - 0, - 5, - 0, - "MODEL" - ], - [ - 2, - 1, - 1, - 2, - 0, - "CLIP" - ], - [ - 3, - 1, - 1, - 3, - 0, - "CLIP" - ], - [ - 4, - 2, - 0, - 5, - 1, - "CONDITIONING" - ], - [ - 5, - 3, - 0, - 5, - 2, - "CONDITIONING" - ], - [ - 6, - 4, - 0, - 5, - 3, - "LATENT" - ], - [ - 7, - 5, - 0, - 6, - 0, - "LATENT" - ], - [ - 8, - 1, - 2, - 6, - 1, - "VAE" - ], - [ - 9, - 7, - 0, - 8, - 0, - "DIFFORUM_PARAMS" - ], - [ - 10, - 8, - 0, - 9, - 0, - "DIFFORUM_CAMERA" - ], - [ - 11, - 9, - 0, - 10, - 0, - "IMAGE" - ], - [ - 12, - 7, - 0, - 11, - 0, - "DIFFORUM_PARAMS" - ], - [ - 13, - 7, - 0, - 12, - 0, - "DIFFORUM_PARAMS" - ], - [ - 14, - 1, - 1, - 12, - 1, - "CLIP" - ], - [ - 15, - 1, - 0, - 13, - 0, - "MODEL" - ], - [ - 16, - 2, - 0, - 13, - 1, - "CONDITIONING" - ], - [ - 17, - 3, - 0, - 13, - 2, - "CONDITIONING" - ], - [ - 18, - 1, - 2, - 13, - 3, - "VAE" - ], - [ - 19, - 7, - 0, - 13, - 4, - "DIFFORUM_PARAMS" - ], - [ - 20, - 8, - 0, - 13, - 5, - "DIFFORUM_CAMERA" - ], - [ - 21, - 6, - 0, - 13, - 6, - "IMAGE" - ], - [ - 22, - 11, - 0, - 13, - 7, - "DIFFORUM_SCHEDULE" - ], - [ - 23, - 12, - 0, - 13, - 10, - "DIFFORUM_PROMPT" - ], - [ - 24, - 13, - 0, - 15, - 0, - "IMAGE" - ], - [ - 25, - 15, - 0, - 16, - 0, - "IMAGE" - ], - [ - 26, - 16, - 0, - 14, - 0, - "IMAGE" - ] - ], - "groups": [ - { - "title": "1 - FIRST FRAME (txt2img)", - "bounding": [ - 10, - -60, - 1270, - 480 - ], - "color": "#33415a", - "font_size": 22, - "locked": false - }, - { - "title": "2 - DIRECTOR (shot list + path preview + scenes)", - "bounding": [ - 10, - 440, - 940, - 900 - ], - "color": "#33503f", - "font_size": 22, - "locked": false - }, - { - "title": "3 - RENDER (detail guard inside)", - "bounding": [ - 960, - 380, - 350, - 460 - ], - "color": "#5a3a33", - "font_size": 22, - "locked": false - }, - { - "title": "4 - QUALITY: anti-flicker (Ctrl+B to bypass)", - "bounding": [ - 1310, - 380, - 350, - 300 - ], - "color": "#4a3d5a", - "font_size": 22, - "locked": false - } - ], - "config": {}, - "extra": {}, - "version": 0.4 -} \ No newline at end of file diff --git a/examples/basic/difforum_models_info.json b/examples/basic/difforum_models_info.json deleted file mode 100644 index a18e164..0000000 --- a/examples/basic/difforum_models_info.json +++ /dev/null @@ -1,314 +0,0 @@ -{ - "last_node_id": 7, - "last_link_id": 3, - "nodes": [ - { - "id": 1, - "type": "DifforumModelProfile", - "pos": [ - 40, - 60 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "profile", - "type": "DIFFORUM_PROFILE", - "links": [], - "slot_index": 0 - }, - { - "name": "summary", - "type": "STRING", - "links": [], - "slot_index": 1 - }, - { - "name": "width", - "type": "INT", - "links": [], - "slot_index": 2 - }, - { - "name": "height", - "type": "INT", - "links": [], - "slot_index": 3 - }, - { - "name": "segment_frames", - "type": "INT", - "links": [], - "slot_index": 4 - } - ], - "properties": { - "Node name for S&R": "DifforumModelProfile" - }, - "widgets_values": [ - "wan22", - "balanced", - true, - 12, - "sdpa" - ] - }, - { - "id": 2, - "type": "DifforumModelCatalog", - "pos": [ - 40, - 360 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "recipe", - "type": "DIFFORUM_RECIPE", - "links": [], - "slot_index": 0 - }, - { - "name": "guide", - "type": "STRING", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumModelCatalog" - }, - "widgets_values": [ - "wan22_12gb" - ] - }, - { - "id": 3, - "type": "DifforumAnimSetup", - "pos": [ - 440, - 360 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "links": [ - 1 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumAnimSetup" - }, - "widgets_values": [ - 768, - 768, - 24, - 60, - 0, - "fixed" - ] - }, - { - "id": 4, - "type": "DifforumSchedule", - "pos": [ - 440, - 560 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 1 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - } - ], - "outputs": [ - { - "name": "schedule", - "type": "DIFFORUM_SCHEDULE", - "links": [ - 2, - 3 - ], - "slot_index": 0 - }, - { - "name": "values", - "type": "FLOAT", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumSchedule" - }, - "widgets_values": [ - "0:(0.2), 30:(0.8), 59:(0.2)", - "ease_in_out" - ] - }, - { - "id": 5, - "type": "DifforumSampleSchedule", - "pos": [ - 840, - 560 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "schedule", - "type": "DIFFORUM_SCHEDULE", - "link": 2 - } - ], - "outputs": [ - { - "name": "value", - "type": "FLOAT", - "links": [], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumSampleSchedule" - }, - "widgets_values": [ - 30 - ] - }, - { - "id": 6, - "type": "DifforumScheduleInfo", - "pos": [ - 840, - 360 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "schedule", - "type": "DIFFORUM_SCHEDULE", - "link": 3 - } - ], - "outputs": [ - { - "name": "info", - "type": "STRING", - "links": [], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumScheduleInfo" - }, - "widgets_values": [] - }, - { - "id": 7, - "type": "Note", - "pos": [ - 40, - -210 - ], - "size": [ - 460, - 300 - ], - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "READ ME - models needed", - "properties": { - "text": "" - }, - "widgets_values": [ - "Difforum: Reference card.\n\nNo models needed. Model Profile auto-detects your VRAM; Model Catalog lists recipes + download links." - ], - "color": "#432", - "bgcolor": "#653" - } - ], - "links": [ - [ - 1, - 3, - 0, - 4, - 0, - "DIFFORUM_PARAMS" - ], - [ - 2, - 4, - 0, - 5, - 0, - "DIFFORUM_SCHEDULE" - ], - [ - 3, - 4, - 0, - 6, - 0, - "DIFFORUM_SCHEDULE" - ] - ], - "groups": [], - "config": {}, - "extra": {}, - "version": 0.4 -} \ No newline at end of file diff --git a/examples/basic/difforum_schedule_basic.json b/examples/basic/difforum_schedule_basic.json deleted file mode 100644 index 54f1237..0000000 --- a/examples/basic/difforum_schedule_basic.json +++ /dev/null @@ -1,258 +0,0 @@ -{ - "last_node_id": 6, - "last_link_id": 4, - "nodes": [ - { - "id": 1, - "type": "DifforumAnimSetup", - "pos": [ - 80, - 200 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "links": [ - 1 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumAnimSetup" - }, - "widgets_values": [ - 768, - 768, - 24, - 120, - 0, - "fixed" - ] - }, - { - "id": 2, - "type": "DifforumSchedule", - "pos": [ - 460, - 200 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 1 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - } - ], - "outputs": [ - { - "name": "schedule", - "type": "DIFFORUM_SCHEDULE", - "links": [ - 2, - 3 - ], - "slot_index": 0 - }, - { - "name": "values", - "type": "FLOAT", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumSchedule" - }, - "widgets_values": [ - "0:(0), 60:(0.5*sin(2*pi*t/30)), 120:(1.0)", - "linear" - ] - }, - { - "id": 3, - "type": "DifforumScheduleInfo", - "pos": [ - 860, - 120 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "schedule", - "type": "DIFFORUM_SCHEDULE", - "link": 2 - } - ], - "outputs": [ - { - "name": "info", - "type": "STRING", - "links": [], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumScheduleInfo" - }, - "widgets_values": [] - }, - { - "id": 4, - "type": "DifforumSchedulePlot", - "pos": [ - 860, - 320 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [ - { - "name": "schedule", - "type": "DIFFORUM_SCHEDULE", - "link": 3 - } - ], - "outputs": [ - { - "name": "plot", - "type": "IMAGE", - "links": [ - 4 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumSchedulePlot" - }, - "widgets_values": [ - 512, - 256 - ] - }, - { - "id": 5, - "type": "PreviewImage", - "pos": [ - 1200, - 320 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 4 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "PreviewImage" - }, - "widgets_values": [] - }, - { - "id": 6, - "type": "Note", - "pos": [ - 40, - -210 - ], - "size": [ - 460, - 300 - ], - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "READ ME - models needed", - "properties": { - "text": "" - }, - "widgets_values": [ - "Difforum: Schedule basics.\n\nNo models needed - this runs anywhere (schedules + curve plot only)." - ], - "color": "#432", - "bgcolor": "#653" - } - ], - "links": [ - [ - 1, - 1, - 0, - 2, - 0, - "DIFFORUM_PARAMS" - ], - [ - 2, - 2, - 0, - 3, - 0, - "DIFFORUM_SCHEDULE" - ], - [ - 3, - 2, - 0, - 4, - 0, - "DIFFORUM_SCHEDULE" - ], - [ - 4, - 4, - 0, - 5, - 0, - "IMAGE" - ] - ], - "groups": [], - "config": {}, - "extra": {}, - "version": 0.4 -} \ No newline at end of file diff --git a/examples/basic/difforum_turbo_live.json b/examples/basic/difforum_turbo_live.json deleted file mode 100644 index 51de76e..0000000 --- a/examples/basic/difforum_turbo_live.json +++ /dev/null @@ -1,815 +0,0 @@ -{ - "last_node_id": 13, - "last_link_id": 22, - "nodes": [ - { - "id": 1, - "type": "CheckpointLoaderSimple", - "pos": [ - 40, - 40 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "MODEL", - "type": "MODEL", - "links": [ - 1, - 13 - ], - "slot_index": 0 - }, - { - "name": "CLIP", - "type": "CLIP", - "links": [ - 2, - 3, - 12 - ], - "slot_index": 1 - }, - { - "name": "VAE", - "type": "VAE", - "links": [ - 8, - 16 - ], - "slot_index": 2 - } - ], - "properties": { - "Node name for S&R": "CheckpointLoaderSimple" - }, - "widgets_values": [ - "sd_turbo.safetensors" - ] - }, - { - "id": 2, - "type": "CLIPTextEncode", - "pos": [ - 340, - 20 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 2 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 4, - 14 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "psychedelic mandala, vivid colors, intricate" - ] - }, - { - "id": 3, - "type": "CLIPTextEncode", - "pos": [ - 340, - 200 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "clip", - "type": "CLIP", - "link": 3 - } - ], - "outputs": [ - { - "name": "CONDITIONING", - "type": "CONDITIONING", - "links": [ - 5, - 15 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "CLIPTextEncode" - }, - "widgets_values": [ - "blurry, low quality" - ] - }, - { - "id": 4, - "type": "EmptyLatentImage", - "pos": [ - 340, - 380 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 6 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "EmptyLatentImage" - }, - "widgets_values": [ - 512, - 512, - 1 - ] - }, - { - "id": 5, - "type": "KSampler", - "pos": [ - 660, - 120 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 1 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 4 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 5 - }, - { - "name": "latent_image", - "type": "LATENT", - "link": 6 - } - ], - "outputs": [ - { - "name": "LATENT", - "type": "LATENT", - "links": [ - 7 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "KSampler" - }, - "widgets_values": [ - 0, - "fixed", - 4, - 1.0, - "euler", - "sgm_uniform", - 1.0 - ] - }, - { - "id": 6, - "type": "VAEDecode", - "pos": [ - 980, - 120 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "samples", - "type": "LATENT", - "link": 7 - }, - { - "name": "vae", - "type": "VAE", - "link": 8 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 19 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "VAEDecode" - }, - "widgets_values": [] - }, - { - "id": 7, - "type": "DifforumAnimSetup", - "pos": [ - 40, - 460 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "links": [ - 9, - 10, - 11, - 17 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumAnimSetup" - }, - "widgets_values": [ - 512, - 512, - 24, - 64, - 0, - "fixed" - ] - }, - { - "id": 8, - "type": "DifforumCameraMove", - "pos": [ - 340, - 500 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 9 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - } - ], - "outputs": [ - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "links": [ - 18 - ], - "slot_index": 0 - }, - { - "name": "info", - "type": "STRING", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumCameraMove" - }, - "widgets_values": [ - "spiral", - 1.0, - 1.0, - "2d", - 40.0 - ] - }, - { - "id": 9, - "type": "DifforumSchedule", - "pos": [ - 340, - 720 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 10 - }, - { - "name": "audio", - "type": "DIFFORUM_AUDIO", - "link": null - } - ], - "outputs": [ - { - "name": "schedule", - "type": "DIFFORUM_SCHEDULE", - "links": [ - 20 - ], - "slot_index": 0 - }, - { - "name": "values", - "type": "FLOAT", - "links": [], - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "DifforumSchedule" - }, - "widgets_values": [ - "0:(0.5)", - "linear" - ] - }, - { - "id": 10, - "type": "DifforumPromptScenes", - "pos": [ - 340, - 920 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 9, - "mode": 0, - "inputs": [ - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 11 - }, - { - "name": "clip", - "type": "CLIP", - "link": 12 - } - ], - "outputs": [ - { - "name": "positive_schedule", - "type": "DIFFORUM_PROMPT", - "links": [ - 21 - ], - "slot_index": 0 - }, - { - "name": "first_frame_cond", - "type": "CONDITIONING", - "links": [], - "slot_index": 1 - }, - { - "name": "info", - "type": "STRING", - "links": [], - "slot_index": 2 - } - ], - "properties": { - "Node name for S&R": "DifforumPromptScenes" - }, - "widgets_values": [ - "psychedelic mandala, vivid colors", - "cosmic fractal temple, golden light", - "deep space nebula, glowing stars", - "", - "ease_in_out" - ] - }, - { - "id": 11, - "type": "DifforumFeedbackSampler", - "pos": [ - 1000, - 460 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 10, - "mode": 0, - "inputs": [ - { - "name": "model", - "type": "MODEL", - "link": 13 - }, - { - "name": "positive", - "type": "CONDITIONING", - "link": 14 - }, - { - "name": "negative", - "type": "CONDITIONING", - "link": 15 - }, - { - "name": "vae", - "type": "VAE", - "link": 16 - }, - { - "name": "params", - "type": "DIFFORUM_PARAMS", - "link": 17 - }, - { - "name": "camera", - "type": "DIFFORUM_CAMERA", - "link": 18 - }, - { - "name": "init_image", - "type": "IMAGE", - "link": 19 - }, - { - "name": "strength_schedule", - "type": "DIFFORUM_SCHEDULE", - "link": 20 - }, - { - "name": "depth", - "type": "IMAGE", - "link": null - }, - { - "name": "cfg_schedule", - "type": "DIFFORUM_SCHEDULE", - "link": null - }, - { - "name": "positive_schedule", - "type": "DIFFORUM_PROMPT", - "link": 21 - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 22 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumFeedbackSampler" - }, - "widgets_values": [ - 4, - 1.0, - "euler", - "sgm_uniform", - 0.7, - "lab", - 1.0, - 100.0, - false, - 1.0, - 0.6, - "none", - 6, - "reflection", - 0.15, - 0.02, - 2, - 0, - 0 - ] - }, - { - "id": 12, - "type": "SaveImage", - "pos": [ - 1360, - 460 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 11, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 22 - } - ], - "outputs": [], - "properties": { - "Node name for S&R": "SaveImage" - }, - "widgets_values": [ - "Difforum_turbo" - ] - }, - { - "id": 13, - "type": "Note", - "pos": [ - 40, - -210 - ], - "size": [ - 460, - 300 - ], - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "READ ME - models needed", - "properties": { - "text": "" - }, - "widgets_values": [ - "Difforum: Turbo/LCM fast feedback (4 steps, cfg 1).\n\nModels:\n- SD-Turbo -> models/checkpoints (hf: stabilityai/sd-turbo)\n or any SD1.5 + LCM-LoRA (hf: latent-consistency/lcm-lora-sdv1-5 -> models/loras)\nThis is the path toward realtime." - ], - "color": "#432", - "bgcolor": "#653" - } - ], - "links": [ - [ - 1, - 1, - 0, - 5, - 0, - "MODEL" - ], - [ - 2, - 1, - 1, - 2, - 0, - "CLIP" - ], - [ - 3, - 1, - 1, - 3, - 0, - "CLIP" - ], - [ - 4, - 2, - 0, - 5, - 1, - "CONDITIONING" - ], - [ - 5, - 3, - 0, - 5, - 2, - "CONDITIONING" - ], - [ - 6, - 4, - 0, - 5, - 3, - "LATENT" - ], - [ - 7, - 5, - 0, - 6, - 0, - "LATENT" - ], - [ - 8, - 1, - 2, - 6, - 1, - "VAE" - ], - [ - 9, - 7, - 0, - 8, - 0, - "DIFFORUM_PARAMS" - ], - [ - 10, - 7, - 0, - 9, - 0, - "DIFFORUM_PARAMS" - ], - [ - 11, - 7, - 0, - 10, - 0, - "DIFFORUM_PARAMS" - ], - [ - 12, - 1, - 1, - 10, - 1, - "CLIP" - ], - [ - 13, - 1, - 0, - 11, - 0, - "MODEL" - ], - [ - 14, - 2, - 0, - 11, - 1, - "CONDITIONING" - ], - [ - 15, - 3, - 0, - 11, - 2, - "CONDITIONING" - ], - [ - 16, - 1, - 2, - 11, - 3, - "VAE" - ], - [ - 17, - 7, - 0, - 11, - 4, - "DIFFORUM_PARAMS" - ], - [ - 18, - 8, - 0, - 11, - 5, - "DIFFORUM_CAMERA" - ], - [ - 19, - 6, - 0, - 11, - 6, - "IMAGE" - ], - [ - 20, - 9, - 0, - 11, - 7, - "DIFFORUM_SCHEDULE" - ], - [ - 21, - 10, - 0, - 11, - 10, - "DIFFORUM_PROMPT" - ], - [ - 22, - 11, - 0, - 12, - 0, - "IMAGE" - ] - ], - "groups": [], - "config": {}, - "extra": {}, - "version": 0.4 -} \ No newline at end of file diff --git a/examples/basic/difforum_vj_footage.json b/examples/basic/difforum_vj_footage.json deleted file mode 100644 index 819e9c4..0000000 --- a/examples/basic/difforum_vj_footage.json +++ /dev/null @@ -1,241 +0,0 @@ -{ - "last_node_id": 5, - "last_link_id": 4, - "nodes": [ - { - "id": 1, - "type": "DifforumLoadVideo", - "pos": [ - 40, - 40 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 0, - "mode": 0, - "inputs": [], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 1 - ], - "slot_index": 0 - }, - { - "name": "frame_count", - "type": "INT", - "links": [], - "slot_index": 1 - }, - { - "name": "fps", - "type": "FLOAT", - "links": [ - 4 - ], - "slot_index": 2 - } - ], - "properties": { - "Node name for S&R": "DifforumLoadVideo" - }, - "widgets_values": [ - "footage.mp4", - 0, - 1, - 0 - ] - }, - { - "id": 2, - "type": "DifforumLook", - "pos": [ - 400, - 40 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [ - { - "name": "image", - "type": "IMAGE", - "link": 1 - } - ], - "outputs": [ - { - "name": "image", - "type": "IMAGE", - "links": [ - 2 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumLook" - }, - "widgets_values": [ - "neon", - 1.2, - 0 - ] - }, - { - "id": 3, - "type": "DifforumEchoTrails", - "pos": [ - 720, - 40 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 2, - "mode": 0, - "inputs": [ - { - "name": "frames", - "type": "IMAGE", - "link": 2 - } - ], - "outputs": [ - { - "name": "frames", - "type": "IMAGE", - "links": [ - 3 - ], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumEchoTrails" - }, - "widgets_values": [ - 0.4, - 0.3 - ] - }, - { - "id": 4, - "type": "DifforumSaveVideo", - "pos": [ - 1040, - 40 - ], - "size": [ - 320, - 200 - ], - "flags": {}, - "order": 3, - "mode": 0, - "inputs": [ - { - "name": "frames", - "type": "IMAGE", - "link": 3 - }, - { - "name": "fps_in", - "type": "FLOAT", - "link": 4 - } - ], - "outputs": [ - { - "name": "path", - "type": "STRING", - "links": [], - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "DifforumSaveVideo" - }, - "widgets_values": [ - "Difforum_vj", - 24.0, - 8 - ] - }, - { - "id": 5, - "type": "Note", - "pos": [ - 40, - -210 - ], - "size": [ - 460, - 300 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [], - "outputs": [], - "title": "READ ME - models needed", - "properties": { - "text": "" - }, - "widgets_values": [ - "Difforum: VJ look for video footage. Complete + self-contained, NO model\nand NO external nodes.\n\nLoad Video -> VJ Look (grade + glow + chroma + grain) -> Echo Trails ->\nSave Video (MP4). Pure post, runs fast.\n\nSetup:\n- put your clip in ComfyUI/input, pick it in Load Video\n- Load Video / Save Video need opencv-python + imageio-ffmpeg (usually\n already in ComfyUI). Set resize_to (e.g. 720) to cap resolution.\n\nVJ Look presets: neon / cinematic / vaporwave / film / noir / psychedelic.\nBeat-reactive: add Anim Setup + Load Audio + Audio Analyzer + Audio Schedule\n(source=beat, base=1.0, amount=0.5) and wire its schedule into the VJ Look's\nintensity_schedule. Re-style with diffusion? Insert a Feedback/Live Sampler\nbefore the VJ Look." - ], - "color": "#432", - "bgcolor": "#653" - } - ], - "links": [ - [ - 1, - 1, - 0, - 2, - 0, - "IMAGE" - ], - [ - 2, - 2, - 0, - 3, - 0, - "IMAGE" - ], - [ - 3, - 3, - 0, - 4, - 0, - "IMAGE" - ], - [ - 4, - 1, - 2, - 4, - 1, - "FLOAT" - ] - ], - "groups": [], - "config": {}, - "extra": {}, - "version": 0.4 -} \ No newline at end of file diff --git a/examples/difforum_audio.gif b/examples/difforum_audio.gif deleted file mode 100644 index 8608677..0000000 Binary files a/examples/difforum_audio.gif and /dev/null differ diff --git a/examples/difforum_audio.mp4 b/examples/difforum_audio.mp4 deleted file mode 100644 index fb8f155..0000000 Binary files a/examples/difforum_audio.mp4 and /dev/null differ diff --git a/examples/difforum_detail_ab.gif b/examples/difforum_detail_ab.gif deleted file mode 100644 index ef93c1d..0000000 Binary files a/examples/difforum_detail_ab.gif and /dev/null differ diff --git a/examples/difforum_detail_ab_final.png b/examples/difforum_detail_ab_final.png deleted file mode 100644 index c4bc5e7..0000000 Binary files a/examples/difforum_detail_ab_final.png and /dev/null differ diff --git a/examples/difforum_guide_demo.gif b/examples/difforum_guide_demo.gif deleted file mode 100644 index 9b08ec5..0000000 Binary files a/examples/difforum_guide_demo.gif and /dev/null differ diff --git a/examples/difforum_kaleidoscope.gif b/examples/difforum_kaleidoscope.gif deleted file mode 100644 index 12d4e25..0000000 Binary files a/examples/difforum_kaleidoscope.gif and /dev/null differ diff --git a/examples/difforum_preview.gif b/examples/difforum_preview.gif deleted file mode 100644 index 2daa202..0000000 Binary files a/examples/difforum_preview.gif and /dev/null differ diff --git a/examples/difforum_promo.gif b/examples/difforum_promo.gif deleted file mode 100644 index ab4250a..0000000 Binary files a/examples/difforum_promo.gif and /dev/null differ diff --git a/examples/difforum_promo.mp4 b/examples/difforum_promo.mp4 deleted file mode 100644 index 8fde785..0000000 Binary files a/examples/difforum_promo.mp4 and /dev/null differ diff --git a/examples/difforum_prompt_travel.gif b/examples/difforum_prompt_travel.gif deleted file mode 100644 index c6bf15e..0000000 Binary files a/examples/difforum_prompt_travel.gif and /dev/null differ diff --git a/examples/difforum_showcase.gif b/examples/difforum_showcase.gif deleted file mode 100644 index fb96e7d..0000000 Binary files a/examples/difforum_showcase.gif and /dev/null differ diff --git a/examples/difforum_showcase.mp4 b/examples/difforum_showcase.mp4 deleted file mode 100644 index 9692069..0000000 Binary files a/examples/difforum_showcase.mp4 and /dev/null differ diff --git a/js/director_timeline.js b/js/director_timeline.js new file mode 100644 index 0000000..983b1a7 --- /dev/null +++ b/js/director_timeline.js @@ -0,0 +1,828 @@ +/** + * Difforum · Director - multi-track timeline editor + * ================================================== + * Scenes (prompt + mood), Camera (visual move picker) and Energy (denoise + * curve) on one timeline, with an animated camera preview computed by the + * same Python engine that renders (POST /difforum/preview). + * + * The node's hidden `timeline` STRING widget stays the source of truth, so + * workflows serialise exactly as ComfyUI expects. + */ + +import { app } from "../../scripts/app.js"; +import { api } from "../../scripts/api.js"; + +const NODE_ID = "Difforum_Director"; +const SETUP_ID = "Difforum_Setup"; + +// Fallback catalogue, replaced by GET /difforum/catalog when the server answers. +let CATALOG = { + moves: [ + "still", "zoom_in", "zoom_out", "pan_left", "pan_right", "pan_up", "pan_down", + "roll_cw", "roll_ccw", "dolly_in", "dolly_out", "orbit_left", "orbit_right", + "tilt_up", "tilt_down", "rise", "crane_up", "dolly_zoom", "spiral", "vortex", + "sway", "breathe", "drift", "handheld", "shake", + ].map((id) => ({ id, label: id.replace(/_/g, " "), group: "basic", hint: "", depth: false })), + moods: [ + { id: "calm", color: "#4a90d9", strength: 0.44 }, + { id: "build", color: "#e0a020", strength: 0.52 }, + { id: "tense", color: "#e2703a", strength: 0.57 }, + { id: "climax", color: "#a45ec4", strength: 0.63 }, + { id: "resolve", color: "#37a86b", strength: 0.44 }, + { id: "dream", color: "#1f9c8c", strength: 0.48 }, + ], + easings: ["linear", "ease_in", "ease_out", "ease_in_out", "step"], + reactions: [{ id: "none", hint: "" }, { id: "beat_pulse", hint: "" }, { id: "onset_shake", hint: "" }, + { id: "bass_speed", hint: "" }, { id: "mid_sway", hint: "" }], +}; +let catalogLoaded = null; +function loadCatalog() { + if (!catalogLoaded) { + catalogLoaded = api.fetchApi("/difforum/catalog") + .then((r) => (r.ok ? r.json() : null)) + .then((c) => { if (c && c.moves) CATALOG = c; }) + .catch(() => {}); + } + return catalogLoaded; +} +const moodOf = (id) => CATALOG.moods.find((m) => m.id === id) || CATALOG.moods[0]; +const moveOf = (id) => CATALOG.moves.find((m) => m.id === id) || CATALOG.moves[0]; + +const LENSES = [[0, "auto (mood)"], [16, "16° ultra tele"], [24, "24° tele"], [32, "32° long"], + [40, "40° normal"], [50, "50° wide"], [65, "65° very wide"], [85, "85° ultra wide"], + [110, "110° fisheye"]]; +const REACT_LABEL = { none: "no audio", beat_pulse: "pulse on beat", onset_shake: "shake on hits", + bass_speed: "bass drives speed", mid_sway: "mids rock roll" }; + +// --------------------------------------------------------------------------- +// move glyphs (drawn, so they scale and theme cleanly) +// --------------------------------------------------------------------------- +function arrow(ctx, x0, y0, x1, y1, h = 4) { + ctx.beginPath(); ctx.moveTo(x0, y0); ctx.lineTo(x1, y1); ctx.stroke(); + const a = Math.atan2(y1 - y0, x1 - x0); + ctx.beginPath(); + ctx.moveTo(x1, y1); + ctx.lineTo(x1 - h * Math.cos(a - 0.5), y1 - h * Math.sin(a - 0.5)); + ctx.moveTo(x1, y1); + ctx.lineTo(x1 - h * Math.cos(a + 0.5), y1 - h * Math.sin(a + 0.5)); + ctx.stroke(); +} +function arc(ctx, cx, cy, r, a0, a1, head = true) { + ctx.beginPath(); ctx.arc(cx, cy, r, a0, a1, a1 < a0); ctx.stroke(); + if (!head) return; + const ex = cx + r * Math.cos(a1), ey = cy + r * Math.sin(a1); + const t = a1 + (a1 > a0 ? Math.PI / 2 : -Math.PI / 2); + arrow(ctx, ex - 2 * Math.cos(t), ey - 2 * Math.sin(t), ex, ey, 4); +} +function drawGlyph(ctx, id, x, y, s, color) { + ctx.save(); + ctx.translate(x, y); + ctx.scale(s / 24, s / 24); + ctx.strokeStyle = color; ctx.fillStyle = color; + ctx.lineWidth = 1.8; ctx.lineCap = "round"; ctx.lineJoin = "round"; + const R = (w, h) => ctx.strokeRect(12 - w / 2, 12 - h / 2, w, h); + switch (id) { + case "still": R(14, 10); ctx.beginPath(); ctx.arc(12, 12, 1.5, 0, 7); ctx.fill(); break; + case "zoom_in": R(18, 13); R(8, 6); arrow(ctx, 5, 6, 8, 9, 3); arrow(ctx, 19, 18, 16, 15, 3); break; + case "zoom_out": R(8, 6); R(18, 13); arrow(ctx, 8, 9, 4, 5, 3); arrow(ctx, 16, 15, 20, 19, 3); break; + case "pan_left": R(12, 10); arrow(ctx, 20, 12, 3, 12); break; + case "pan_right": R(12, 10); arrow(ctx, 4, 12, 21, 12); break; + case "pan_up": R(12, 10); arrow(ctx, 12, 21, 12, 3); break; + case "pan_down": R(12, 10); arrow(ctx, 12, 3, 12, 21); break; + case "roll_cw": arc(ctx, 12, 12, 8, -2.4, 1.6); break; + case "roll_ccw": arc(ctx, 12, 12, 8, 1.6, -2.4); break; + case "dolly_in": + ctx.beginPath(); ctx.moveTo(3, 20); ctx.lineTo(10, 9); ctx.moveTo(21, 20); ctx.lineTo(14, 9); ctx.stroke(); + arrow(ctx, 12, 21, 12, 7); break; + case "dolly_out": + ctx.beginPath(); ctx.moveTo(3, 20); ctx.lineTo(10, 9); ctx.moveTo(21, 20); ctx.lineTo(14, 9); ctx.stroke(); + arrow(ctx, 12, 8, 12, 21); break; + case "orbit_left": + ctx.beginPath(); ctx.ellipse(12, 13, 9, 4, 0, 0, 7); ctx.stroke(); + ctx.beginPath(); ctx.arc(12, 9, 2.4, 0, 7); ctx.fill(); arrow(ctx, 8, 17, 4, 15, 3); break; + case "orbit_right": + ctx.beginPath(); ctx.ellipse(12, 13, 9, 4, 0, 0, 7); ctx.stroke(); + ctx.beginPath(); ctx.arc(12, 9, 2.4, 0, 7); ctx.fill(); arrow(ctx, 16, 17, 20, 15, 3); break; + case "tilt_up": arc(ctx, 6, 12, 10, 0.7, -0.7); break; + case "tilt_down": arc(ctx, 6, 12, 10, -0.7, 0.7); break; + case "rise": case "crane_up": + ctx.beginPath(); ctx.moveTo(4, 21); ctx.lineTo(20, 21); ctx.stroke(); + arrow(ctx, 12, 19, 12, 4); + if (id === "crane_up") arc(ctx, 12, 8, 5, -0.2, 1.2); break; + case "dolly_zoom": + ctx.beginPath(); ctx.moveTo(3, 5); ctx.lineTo(12, 12); ctx.lineTo(3, 19); ctx.stroke(); + ctx.beginPath(); ctx.moveTo(21, 8); ctx.lineTo(14, 12); ctx.lineTo(21, 16); ctx.stroke(); break; + case "spiral": case "vortex": { + ctx.beginPath(); + for (let t = 0; t < 12.5; t += 0.2) { + const r = 1 + t * 0.75, px = 12 + r * Math.cos(t), py = 12 + r * Math.sin(t); + t === 0 ? ctx.moveTo(px, py) : ctx.lineTo(px, py); + } + ctx.stroke(); + if (id === "vortex") arrow(ctx, 17, 17, 21, 21, 3); else arrow(ctx, 16, 8, 13, 11, 3); + break; + } + case "sway": arc(ctx, 12, 24, 14, -2.2, -0.9); arc(ctx, 12, 24, 14, -0.9, -2.2); break; + case "breathe": R(16, 11); ctx.setLineDash([2, 2]); R(10, 7); ctx.setLineDash([]); break; + case "drift": + ctx.beginPath(); ctx.moveTo(3, 14); + ctx.bezierCurveTo(8, 4, 12, 22, 21, 9); ctx.stroke(); break; + case "handheld": + ctx.beginPath(); ctx.moveTo(3, 12); + for (let i = 1; i <= 9; i++) ctx.lineTo(3 + i * 2, 12 + (i % 2 ? -2 : 2) * (i % 3 ? 0.6 : 1)); + ctx.stroke(); break; + case "shake": + ctx.beginPath(); ctx.moveTo(3, 12); + for (let i = 1; i <= 9; i++) ctx.lineTo(3 + i * 2, 12 + (i % 2 ? -5 : 5)); + ctx.stroke(); break; + default: R(14, 10); + } + ctx.restore(); +} + +// --------------------------------------------------------------------------- +// styles +// --------------------------------------------------------------------------- +function injectStyles() { + if (document.getElementById("dfx-css")) return; + const el = document.createElement("style"); + el.id = "dfx-css"; + el.textContent = ` +.dfx { --bg:#141414; --panel:#1c1c1c; --line:#2c2c2c; --txt:#d4d4d4; --dim:#7a7a7a; --acc:#4f8ef7; + font: 11px/1.3 -apple-system, BlinkMacSystemFont, "Segoe UI", Inter, sans-serif; color: var(--txt); + background: var(--bg); border-radius: 8px; display: flex; flex-direction: column; height: 100%; + box-sizing: border-box; overflow: hidden; pointer-events: none; user-select: none; } +.dfx > * { pointer-events: auto; } +.dfx-bar { display: flex; align-items: center; gap: 6px; padding: 6px 8px; background: var(--panel); + border-bottom: 1px solid var(--line); } +.dfx-btn { background: #2a2a2a; border: 1px solid #3c3c3c; color: var(--txt); border-radius: 5px; + padding: 3px 9px; cursor: pointer; font: inherit; white-space: nowrap; } +.dfx-btn:hover { background: #343434; } +.dfx-btn.on, .dfx-btn.pri { background: #22406f; border-color: var(--acc); } +.dfx-time { font-variant-numeric: tabular-nums; color: var(--dim); min-width: 96px; } +.dfx-sp { flex: 1; } +.dfx-main { display: flex; gap: 0; flex: 0 0 250px; } +.dfx-prev { width: 38%; min-width: 180px; max-width: 320px; padding: 8px; box-sizing: border-box; + border-right: 1px solid var(--line); display: flex; flex-direction: column; gap: 6px; } +.dfx-prev canvas { width: 100%; background: #0b0b0b; border-radius: 5px; } +.dfx-cap { color: var(--dim); font-size: 10px; min-height: 26px; overflow: hidden; } +.dfx-tl { flex: 1; position: relative; } +.dfx-tl canvas { position: absolute; inset: 0; width: 100%; height: 100%; cursor: default; } +.dfx-ins { border-top: 1px solid var(--line); background: var(--panel); padding: 8px; display: flex; flex: 1 1 auto; + flex-direction: column; gap: 7px; min-height: 84px; overflow-y: auto; } +.dfx-row { display: flex; gap: 8px; align-items: center; flex-wrap: wrap; } +.dfx-lbl { color: var(--dim); font-size: 10px; text-transform: uppercase; letter-spacing: .05em; } +.dfx-chip { padding: 3px 9px; border-radius: 12px; cursor: pointer; border: 1px solid transparent; + font-size: 10px; color: #fff; opacity: .55; } +.dfx-chip.on { opacity: 1; border-color: #fff; } +.dfx-moves { display: grid; grid-template-columns: repeat(auto-fill, minmax(58px, 1fr)); gap: 3px; width: 100%; } +.dfx-mv { display: flex; flex-direction: column; align-items: center; gap: 1px; padding: 3px 2px; + border-radius: 6px; background: #232323; border: 1px solid #2f2f2f; cursor: pointer; position: relative; } +.dfx-mv:hover { background: #2c2c2c; } +.dfx-mv.on { border-color: var(--acc); background: #1f2f4a; } +.dfx-mv span { font-size: 9px; color: #bdbdbd; text-align: center; } +.dfx-mv i { position: absolute; top: 2px; right: 4px; font-style: normal; font-size: 8px; color: #e0a020; } +.dfx-grp { font-size: 9px; color: var(--dim); grid-column: 1 / -1; margin-top: 2px; } +.dfx-sl { display: flex; align-items: center; gap: 6px; min-width: 150px; flex: 1; } +.dfx-sl input[type=range] { flex: 1; accent-color: var(--acc); } +.dfx-sl b { font-weight: 500; min-width: 30px; text-align: right; font-variant-numeric: tabular-nums; } +.dfx select, .dfx textarea { background: #242424; color: var(--txt); border: 1px solid #3a3a3a; + border-radius: 5px; padding: 3px 6px; font: inherit; } +.dfx textarea { width: 100%; box-sizing: border-box; resize: vertical; min-height: 44px; user-select: text; } +.dfx-warn { color: #e0a020; font-size: 10px; padding: 0 8px 6px; } +.dfx-hint { color: var(--dim); font-size: 10px; } +.dfx-head b { font-weight: 600; font-size: 12px; } +`; + document.head.appendChild(el); +} + +// --------------------------------------------------------------------------- +// helpers +// --------------------------------------------------------------------------- +const el = (tag, cls, text) => { + const e = document.createElement(tag); + if (cls) e.className = cls; + if (text != null) e.textContent = text; + return e; +}; +const clamp = (v, a, b) => Math.max(a, Math.min(b, v)); + +function emptyTimeline() { + return { version: 2, scenes: [], camera: [], energy: [], guidance: [] }; +} +function normalise(v) { + if (Array.isArray(v)) { // 0.x list -> v2 + const t = emptyTimeline(); + for (const s of v) { + const start = s.frame_start ?? s.start ?? 0; + t.scenes.push({ start, mood: s.mood || "calm", prompt: s.prompt || "" }); + t.camera.push({ start, move: s.camera || "still", speed: 1, intensity: 1, + lens: s.lens || 0, ease: s.ease || "ease_in_out", react: "none" }); + } + return t; + } + const t = Object.assign(emptyTimeline(), v || {}); + for (const k of ["scenes", "camera", "energy", "guidance"]) if (!Array.isArray(t[k])) t[k] = []; + t.scenes = t.scenes.map((s) => ({ start: 0, mood: "calm", prompt: "", ...s })); + t.camera = t.camera.map((c) => ({ start: 0, move: "still", speed: 1, intensity: 1, lens: 0, + ease: "ease_in_out", react: "none", ...c })); + return t; +} + +/** frames / fps / size: read the upstream Setup node when there is one. */ +function readSetup(node) { + const out = { frames: 120, fps: 24, width: 768, height: 432, known: false }; + try { + const slot = node.inputs?.findIndex((i) => i.name === "params") ?? -1; + const src = slot >= 0 && node.getInputNode ? node.getInputNode(slot) : null; + if (src && src.type === SETUP_ID) { + const w = (n) => src.widgets?.find((x) => x.name === n)?.value; + let fps = Number(w("fps") ?? 24); + const target = String(w("target") ?? ""); + if (target.startsWith("MiniMax")) fps = 24; + const dur = Number(w("duration") ?? 5); + out.fps = fps; + out.frames = Math.max(1, Math.round(w("duration_mode") === "frames" ? dur : dur * fps)); + const le = Number(w("long_edge") ?? 768); + const asp = String(w("aspect") ?? "16:9"); + const m = asp.match(/([\d.]+):([\d.]+)/); + const [aw, ah] = m ? [Number(m[1]), Number(m[2])] : [16, 9]; + out.width = aw >= ah ? le : Math.round(le * aw / ah); + out.height = aw >= ah ? Math.round(le * ah / aw) : le; + out.known = true; + } + } catch (_) { /* graph not ready */ } + return out; +} + +// --------------------------------------------------------------------------- +// the editor +// --------------------------------------------------------------------------- +function buildEditor(node, tw) { + let tl = emptyTimeline(); + let setup = readSetup(node); + let sel = null; // {track:"scenes"|"camera", i} | {track:"energy", i} + let drag = null; + let playing = false, playhead = 0, lastT = 0; + let preview = null, previewErr = ""; + let previewTimer = null; + + const root = el("div", "dfx"); + // toolbar + const bar = el("div", "dfx-bar"); + const playBtn = el("button", "dfx-btn", "▶"); + const time = el("span", "dfx-time", ""); + const addScene = el("button", "dfx-btn", "+ Scene"); + const addCam = el("button", "dfx-btn", "+ Camera"); + const autoE = el("button", "dfx-btn", "Energy: auto"); + const del = el("button", "dfx-btn", "Delete"); + bar.append(playBtn, time, el("span", "dfx-sp"), addScene, addCam, autoE, del); + for (const b of [playBtn, addScene, addCam, autoE, del]) b.type = "button"; + + // preview + tracks + const main = el("div", "dfx-main"); + const prev = el("div", "dfx-prev"); + const pc = el("canvas"); + const cap = el("div", "dfx-cap"); + prev.append(pc, cap); + const tlBox = el("div", "dfx-tl"); + const tc = el("canvas"); + tlBox.append(tc); + main.append(prev, tlBox); + + const warn = el("div", "dfx-warn"); + const ins = el("div", "dfx-ins"); + root.append(bar, main, warn, ins); + + // ---- data ----------------------------------------------------------- + function load() { + try { tl = normalise(JSON.parse(tw.value || "{}")); } catch { tl = emptyTimeline(); } + sortAll(); + } + function sortAll() { + tl.scenes.sort((a, b) => a.start - b.start); + tl.camera.sort((a, b) => a.start - b.start); + tl.energy.sort((a, b) => a[0] - b[0]); + } + function save() { + sortAll(); + tw.value = JSON.stringify(tl); + if (tw.callback) try { tw.callback(tw.value); } catch (_) { /* noop */ } + node.setDirtyCanvas?.(true, true); + schedulePreview(); + } + const N = () => setup.frames; + const secs = (f) => (f / setup.fps).toFixed(2) + "s"; + + function schedulePreview() { + clearTimeout(previewTimer); + previewTimer = setTimeout(fetchPreview, 220); + } + async function fetchPreview() { + const w = (n) => node.widgets?.find((x) => x.name === n)?.value; + try { + const r = await api.fetchApi("/difforum/preview", { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + timeline: tl, frames: N(), fps: setup.fps, width: setup.width, height: setup.height, + mode: w("camera_mode"), transition: w("transition"), camera_scale: w("camera_scale"), + energy_bias: w("energy_bias"), variation: w("variation"), variation_seed: w("variation_seed"), + }), + }); + const j = await r.json(); + if (j.error) { previewErr = j.error; } else { preview = j; previewErr = ""; } + } catch (e) { previewErr = "preview offline (server restart needed?)"; } + warn.textContent = previewErr || (preview?.warnings || []).map((x) => "⚠ " + x).join(" "); + draw(); + } + + // ---- geometry -------------------------------------------------------- + const RULER = 18, LANE = 34, PAD = 8; + let tlH = 160; + const lanes = () => ({ + scenes: [RULER + 4, LANE], + camera: [RULER + 8 + LANE, LANE], + energy: [RULER + 12 + 2 * LANE, Math.max(40, tlH - (RULER + 12 + 2 * LANE) - 6)], + }); + function fx(frame, W) { return PAD + (frame / Math.max(1, N())) * (W - 2 * PAD); } + function frameAt(x, W) { return clamp(Math.round(((x - PAD) / (W - 2 * PAD)) * N()), 0, N() - 1); } + function blockEnd(list, i) { return i + 1 < list.length ? list[i + 1].start : N(); } + function energyCurve() { + if (tl.energy.length) return tl.energy; + if (preview?.strength) { + const st = preview.strength_step || 1; + return preview.strength.map((v, k) => [k * st, v]); + } + return tl.scenes.map((s) => [s.start, moodOf(s.mood).strength ?? 0.5]); + } + + // ---- drawing --------------------------------------------------------- + function sizeCanvas(c) { + const dpr = window.devicePixelRatio || 1; + const r = c.getBoundingClientRect(); + const w = Math.max(10, Math.round(r.width * dpr)), h = Math.max(10, Math.round(r.height * dpr)); + if (c.width !== w || c.height !== h) { c.width = w; c.height = h; } + const ctx = c.getContext("2d"); + ctx.setTransform(dpr, 0, 0, dpr, 0, 0); + return [ctx, r.width, r.height]; + } + + function drawTracks() { + const [ctx, W, H] = sizeCanvas(tc); + tlH = H; + ctx.clearRect(0, 0, W, H); + const L = lanes(); + // ruler + ctx.fillStyle = "#1a1a1a"; ctx.fillRect(0, 0, W, RULER); + ctx.fillStyle = "#6c6c6c"; ctx.font = "9px sans-serif"; + const total = N() / setup.fps; + const stepS = total > 40 ? 5 : total > 16 ? 2 : total > 6 ? 1 : 0.5; + for (let s = 0; s <= total + 1e-6; s += stepS) { + const x = fx(s * setup.fps, W); + ctx.fillRect(x, RULER - 6, 1, 6); + ctx.fillText(s % 1 ? s.toFixed(1) + "s" : s + "s", x + 2, 11); + } + // lane backgrounds + labels + for (const [name, [y, h]] of Object.entries(L)) { + ctx.fillStyle = "#181818"; ctx.fillRect(PAD, y, W - 2 * PAD, h); + if (name === "energy" || !tl[name]?.length) { + ctx.fillStyle = "#4a4a4a"; ctx.font = "9px sans-serif"; + ctx.fillText(name === "energy" ? "ENERGY (denoise)" : `${name.toUpperCase()} · double-click to add`, + PAD + 4, name === "energy" ? y + h - 4 : y + h / 2 + 3); + } + } + // scenes + tl.scenes.forEach((s, i) => { + const [y, h] = L.scenes; + const x0 = fx(s.start, W), x1 = fx(blockEnd(tl.scenes, i), W); + const m = moodOf(s.mood); + ctx.fillStyle = m.color + "cc"; + roundRect(ctx, x0 + 1, y + 2, Math.max(4, x1 - x0 - 2), h - 4, 5, true); + if (sel?.track === "scenes" && sel.i === i) outline(ctx, x0 + 1, y + 2, x1 - x0 - 2, h - 4); + ctx.fillStyle = "#fff"; ctx.font = "600 10px sans-serif"; + clipText(ctx, s.prompt || "(empty prompt)", x0 + 6, y + 15, x1 - x0 - 12); + ctx.fillStyle = "#ffffffaa"; ctx.font = "9px sans-serif"; + clipText(ctx, s.mood, x0 + 6, y + 27, x1 - x0 - 12); + grip(ctx, x0, y, h); + }); + // camera + tl.camera.forEach((c, i) => { + const [y, h] = L.camera; + const x0 = fx(c.start, W), x1 = fx(blockEnd(tl.camera, i), W); + const mv = moveOf(c.move); + ctx.fillStyle = mv.depth ? "#3a3222" : "#23324a"; + roundRect(ctx, x0 + 1, y + 2, Math.max(4, x1 - x0 - 2), h - 4, 5, true); + if (sel?.track === "camera" && sel.i === i) outline(ctx, x0 + 1, y + 2, x1 - x0 - 2, h - 4); + drawGlyph(ctx, c.move, x0 + 5, y + 6, 22, "#e6e6e6"); + ctx.fillStyle = "#e6e6e6"; ctx.font = "600 10px sans-serif"; + clipText(ctx, mv.label, x0 + 31, y + 15, x1 - x0 - 36); + ctx.fillStyle = "#ffffff88"; ctx.font = "9px sans-serif"; + const sub = `x${(+c.speed).toFixed(2)} · ${c.ease.replace(/_/g, " ")}` + + (c.react && c.react !== "none" ? " · ♪" : ""); + clipText(ctx, sub, x0 + 31, y + 27, x1 - x0 - 36); + grip(ctx, x0, y, h); + }); + // energy + { + const [y, h] = L.energy; + const pts = energyCurve(); + const auto = !tl.energy.length; + const vy = (v) => y + h - 4 - clamp(v, 0, 1) * (h - 8); + ctx.strokeStyle = "#2a2a2a"; + for (const q of [0.25, 0.5, 0.75]) { ctx.beginPath(); ctx.moveTo(PAD, vy(q)); ctx.lineTo(W - PAD, vy(q)); ctx.stroke(); } + if (pts.length) { + ctx.strokeStyle = auto ? "#6f6f6f" : "#f0b53a"; + ctx.setLineDash(auto ? [4, 3] : []); + ctx.lineWidth = 1.6; + ctx.beginPath(); + ctx.moveTo(PAD, vy(pts[0][1])); + for (const [f, v] of pts) ctx.lineTo(fx(f, W), vy(v)); + ctx.lineTo(W - PAD, vy(pts[pts.length - 1][1])); + ctx.stroke(); + ctx.setLineDash([]); ctx.lineWidth = 1; + if (!auto) tl.energy.forEach(([f, v], i) => { + ctx.fillStyle = sel?.track === "energy" && sel.i === i ? "#fff" : "#f0b53a"; + ctx.beginPath(); ctx.arc(fx(f, W), vy(v), 4, 0, 7); ctx.fill(); + }); + } + ctx.fillStyle = "#5a5a5a"; ctx.font = "9px sans-serif"; + ctx.fillText(auto ? "auto from moods · click to draw" : "click: add · drag: move · alt-click: remove", + W - PAD - 190, y + 10); + } + // playhead + const px = fx(playhead, W); + ctx.fillStyle = "#4f8ef7"; ctx.fillRect(px, 0, 1.5, H); + ctx.beginPath(); ctx.moveTo(px - 4, 0); ctx.lineTo(px + 5.5, 0); ctx.lineTo(px + 0.75, 6); ctx.fill(); + } + + function drawPreview() { + const r = pc.getBoundingClientRect(); + pc.style.height = Math.round(r.width * (setup.height / setup.width)) + "px"; + const [ctx, W, H] = sizeCanvas(pc); + ctx.fillStyle = "#0b0b0b"; ctx.fillRect(0, 0, W, H); + const sx = W / setup.width, sy = H / setup.height; + // the "world": a grid of the first frame, carried by the camera + let a = [1, 0, 0, 0, 1, 0]; + if (preview?.affines?.length) { + let best = preview.affines[0]; + for (const row of preview.affines) { if (row[0] <= playhead) best = row; else break; } + a = best.slice(1); + } + // canvas = scale(view) . affine(camera) ; a = [a00 a01 a02 a10 a11 a12] + const dpr = window.devicePixelRatio || 1; + ctx.save(); + ctx.setTransform(a[0] * sx * dpr, a[3] * sy * dpr, a[1] * sx * dpr, + a[4] * sy * dpr, a[2] * sx * dpr, a[5] * sy * dpr); + const gw = setup.width, gh = setup.height; + const g = ctx.createLinearGradient(0, 0, gw, gh); + g.addColorStop(0, "#1e3a5f"); g.addColorStop(0.5, "#2b1e4a"); g.addColorStop(1, "#4a2a1e"); + ctx.fillStyle = g; ctx.fillRect(0, 0, gw, gh); + ctx.strokeStyle = "#ffffff30"; ctx.lineWidth = Math.max(1, gw / 300); + for (let i = 0; i <= 12; i++) { + ctx.beginPath(); ctx.moveTo((i / 12) * gw, 0); ctx.lineTo((i / 12) * gw, gh); ctx.stroke(); + ctx.beginPath(); ctx.moveTo(0, (i / 12) * gh); ctx.lineTo(gw, (i / 12) * gh); ctx.stroke(); + } + ctx.strokeStyle = "#ffffffa0"; ctx.lineWidth = Math.max(1.5, gw / 200); + ctx.beginPath(); ctx.arc(gw / 2, gh / 2, gh * 0.18, 0, 7); ctx.stroke(); + ctx.restore(); + // frame guides + ctx.strokeStyle = "#ffffff40"; ctx.strokeRect(0.5, 0.5, W - 1, H - 1); + ctx.strokeStyle = "#ffffff18"; + ctx.beginPath(); ctx.moveTo(W / 3, 0); ctx.lineTo(W / 3, H); ctx.moveTo(2 * W / 3, 0); ctx.lineTo(2 * W / 3, H); + ctx.moveTo(0, H / 3); ctx.lineTo(W, H / 3); ctx.moveTo(0, 2 * H / 3); ctx.lineTo(W, 2 * H / 3); ctx.stroke(); + + const sc = [...tl.scenes].reverse().find((s) => s.start <= playhead); + const cm = [...tl.camera].reverse().find((c) => c.start <= playhead); + cap.textContent = `${cm ? moveOf(cm.move).label : "still"} - ${sc ? sc.prompt : ""}`; + time.textContent = `${secs(playhead)} / ${secs(N())} (${playhead}f)`; + } + + function roundRect(ctx, x, y, w, h, r, fill) { + ctx.beginPath(); + ctx.moveTo(x + r, y); ctx.arcTo(x + w, y, x + w, y + h, r); ctx.arcTo(x + w, y + h, x, y + h, r); + ctx.arcTo(x, y + h, x, y, r); ctx.arcTo(x, y, x + w, y, r); ctx.closePath(); + fill ? ctx.fill() : ctx.stroke(); + } + function outline(ctx, x, y, w, h) { ctx.strokeStyle = "#fff"; ctx.lineWidth = 1.5; roundRect(ctx, x, y, w, h, 5, false); ctx.lineWidth = 1; } + function grip(ctx, x, y, h) { ctx.fillStyle = "#ffffff30"; ctx.fillRect(x + 1, y + 6, 3, h - 12); } + function clipText(ctx, t, x, y, maxW) { + if (maxW < 8) return; + let s = String(t); + while (s.length > 1 && ctx.measureText(s).width > maxW) s = s.slice(0, -2) + "…"; + ctx.fillText(s, x, y); + } + function draw() { drawTracks(); drawPreview(); } + + // ---- hit testing & mouse ------------------------------------------- + function hit(x, y, W) { + const L = lanes(); + for (const track of ["scenes", "camera"]) { + const [ly, lh] = L[track]; + if (y < ly || y > ly + lh) continue; + const list = tl[track]; + for (let i = list.length - 1; i >= 0; i--) { + const x0 = fx(list[i].start, W), x1 = fx(blockEnd(list, i), W); + if (x >= x0 && x <= x1) return { track, i, edge: x - x0 < 7 && i > 0 }; + } + return { track, i: -1 }; + } + const [ey, eh] = L.energy; + if (y >= ey && y <= ey + eh) { + const vy = (v) => ey + eh - 4 - clamp(v, 0, 1) * (eh - 8); + for (let i = 0; i < tl.energy.length; i++) { + const [f, v] = tl.energy[i]; + if (Math.hypot(fx(f, W) - x, vy(v) - y) < 7) return { track: "energy", i }; + } + return { track: "energy", i: -1, value: clamp((ey + eh - 4 - y) / (eh - 8), 0, 1) }; + } + if (y < RULER) return { track: "ruler" }; + return null; + } + + tc.addEventListener("pointerdown", (e) => { + const r = tc.getBoundingClientRect(); + const x = e.clientX - r.left, y = e.clientY - r.top, W = r.width; + const h = hit(x, y, W); + const f = frameAt(x, W); + tc.setPointerCapture(e.pointerId); + if (!h || h.track === "ruler") { playhead = f; drag = { kind: "scrub" }; draw(); return; } + if (h.track === "energy") { + if (h.i >= 0 && e.altKey) { tl.energy.splice(h.i, 1); sel = null; save(); renderInspector(); return; } + if (h.i < 0) { + if (!tl.energy.length) tl.energy = energyCurve().map(([ff, v]) => [ff, +(+v).toFixed(3)]) + .filter((_, k, arr) => arr.length < 24 || k % Math.ceil(arr.length / 12) === 0); + tl.energy.push([f, +h.value.toFixed(3)]); + sortAll(); + h.i = tl.energy.findIndex((p) => p[0] === f); + } + sel = { track: "energy", i: h.i }; + drag = { kind: "energy", i: h.i }; + renderInspector(); draw(); return; + } + if (h.i < 0) { sel = null; renderInspector(); playhead = f; draw(); return; } + sel = { track: h.track, i: h.i }; + drag = { kind: h.edge ? "edge" : "move", track: h.track, i: h.i, f0: f, + start0: tl[h.track][h.i].start }; + playhead = tl[h.track][h.i].start; + renderInspector(); draw(); + }); + tc.addEventListener("pointermove", (e) => { + if (!drag) return; + const r = tc.getBoundingClientRect(); + const W = r.width, f = frameAt(e.clientX - r.left, W); + if (drag.kind === "scrub") { playhead = f; draw(); return; } + if (drag.kind === "energy") { + const [ey, eh] = lanes().energy; + const v = clamp((ey + eh - 4 - (e.clientY - r.top)) / (eh - 8), 0.02, 0.98); + tl.energy[drag.i] = [f, +v.toFixed(3)]; + draw(); return; + } + const list = tl[drag.track]; + const i = drag.i; + const lo = i > 0 ? list[i - 1].start + 2 : 0; + const hi = i + 1 < list.length ? list[i + 1].start - 2 : N() - 2; + if (i === 0) return; // the first block always starts at 0 + list[i].start = clamp(drag.start0 + (f - drag.f0), lo, hi); + playhead = list[i].start; + draw(); + }); + tc.addEventListener("pointerup", () => { + if (drag && drag.kind !== "scrub") save(); + drag = null; + renderInspector(); + }); + tc.addEventListener("dblclick", (e) => { + const r = tc.getBoundingClientRect(); + const h = hit(e.clientX - r.left, e.clientY - r.top, r.width); + const f = frameAt(e.clientX - r.left, r.width); + if (h?.track === "scenes") addBlock("scenes", f); + else if (h?.track === "camera") addBlock("camera", f); + }); + + function addBlock(track, f) { + f = clamp(Math.round(f), 0, N() - 2); + const list = tl[track]; + const start = list.length ? f : 0; + if (list.some((b) => Math.abs(b.start - start) < 2)) return; // one block per spot + let blk; + if (track === "scenes") { + const prevS = [...list].reverse().find((s) => s.start <= f); + blk = { start, mood: prevS?.mood || "calm", prompt: prevS?.prompt || "" }; + } else { + const prevC = [...list].reverse().find((c) => c.start <= f); + blk = { start, move: "zoom_in", speed: prevC?.speed ?? 1, intensity: prevC?.intensity ?? 1, + lens: 0, ease: "ease_in_out", react: "none" }; + } + list.push(blk); + sortAll(); + sel = { track, i: list.indexOf(blk) }; + save(); renderInspector(); draw(); + } + + // ---- inspector --------------------------------------------------------- + function slider(label, value, min, max, step, onInput) { + const wrap = el("label", "dfx-sl"); + const lab = el("span", "dfx-lbl", label); + const r = el("input"); r.type = "range"; r.min = min; r.max = max; r.step = step; r.value = value; + const b = el("b", null, (+value).toFixed(2)); + r.addEventListener("input", () => { b.textContent = (+r.value).toFixed(2); onInput(+r.value); draw(); }); + r.addEventListener("change", save); + wrap.append(lab, r, b); + return wrap; + } + function select(options, value, onChange) { + const s = el("select"); + for (const [v, t] of options) s.add(new Option(t, v)); + s.value = value; + s.addEventListener("change", () => { onChange(s.value); save(); renderInspector(); draw(); }); + return s; + } + + function renderInspector() { + ins.replaceChildren(); + if (!sel || (sel.track !== "energy" && !tl[sel.track]?.[sel.i])) { + ins.append(el("div", "dfx-hint", + "Click a block to edit it · double-click a lane to add a block · drag a block's left edge to retime · " + + "click the ruler to scrub · ▶ plays the camera the renderer will use.")); + return; + } + if (sel.track === "energy") { + const p = tl.energy[sel.i]; + if (!p) return; + ins.append(el("div", "dfx-hint", + `Energy point at ${secs(p[0])}: ${p[1].toFixed(2)} denoise. Higher = the model re-imagines more ` + + "each frame (more morphing); lower = steadier. Alt-click removes a point.")); + return; + } + if (sel.track === "scenes") { + const s = tl.scenes[sel.i]; + const r1 = el("div", "dfx-row"); + r1.append(el("span", "dfx-lbl", `Scene @ ${secs(s.start)}`)); + for (const m of CATALOG.moods) { + const c = el("span", "dfx-chip" + (m.id === s.mood ? " on" : ""), m.id); + c.style.background = m.color; + c.addEventListener("click", () => { s.mood = m.id; save(); renderInspector(); draw(); }); + r1.append(c); + } + const ta = el("textarea"); + ta.placeholder = "What is on screen in this scene…"; + ta.value = s.prompt; + ta.addEventListener("input", () => { s.prompt = ta.value; draw(); }); + ta.addEventListener("change", save); + ta.addEventListener("keydown", (e) => e.stopPropagation()); + ins.append(r1, ta); + return; + } + // camera block + const c = tl.camera[sel.i]; + const grid = el("div", "dfx-moves"); + const groups = { basic: "FLAT - works in 2D", space: "SPACE - real parallax with 3D + depth", fx: "MOTION FX" }; + for (const [gid, gname] of Object.entries(groups)) { + grid.append(el("div", "dfx-grp", gname)); + for (const mv of CATALOG.moves.filter((m) => (m.group || "basic") === gid)) { + const b = el("div", "dfx-mv" + (mv.id === c.move ? " on" : "")); + b.title = mv.hint || mv.label; + const cv = el("canvas"); cv.width = 44; cv.height = 44; cv.style.width = "22px"; cv.style.height = "22px"; + const g = cv.getContext("2d"); g.scale(2, 2); drawGlyph(g, mv.id, 0, 0, 22, mv.id === c.move ? "#fff" : "#bdbdbd"); + b.append(cv, el("span", null, mv.label)); + if (mv.depth) b.append(el("i", null, "3D")); + b.addEventListener("click", () => { c.move = mv.id; save(); renderInspector(); draw(); }); + grid.append(b); + } + } + const r2 = el("div", "dfx-row"); + r2.append( + slider("Speed", c.speed, 0.1, 4, 0.05, (v) => { c.speed = v; }), + slider("Amount", c.intensity, 0, 3, 0.05, (v) => { c.intensity = v; }), + ); + const r3 = el("div", "dfx-row"); + r3.append( + el("span", "dfx-lbl", "Lens"), select(LENSES.map(([v, t]) => [String(v), t]), String(c.lens || 0), + (v) => { c.lens = Number(v); }), + el("span", "dfx-lbl", "Ease in"), select(CATALOG.easings.map((e) => [e, e.replace(/_/g, " ")]), c.ease, + (v) => { c.ease = v; }), + el("span", "dfx-lbl", "Audio"), select(CATALOG.reactions.map((r) => [r.id, REACT_LABEL[r.id] || r.id]), + c.react || "none", (v) => { c.react = v; }), + ); + const head = el("div", "dfx-head"); + head.append(el("b", null, `${moveOf(c.move).label} @ ${secs(c.start)}`), + el("span", "dfx-hint", " " + (moveOf(c.move).hint || ""))); + ins.append(head, grid, r2, r3); + } + + // ---- toolbar ----------------------------------------------------------- + addScene.onclick = () => addBlock("scenes", playhead); + addCam.onclick = () => addBlock("camera", playhead); + autoE.onclick = () => { tl.energy = []; sel = null; save(); renderInspector(); draw(); }; + del.onclick = () => { + if (!sel) return; + if (sel.track === "energy") tl.energy.splice(sel.i, 1); + else if (tl[sel.track].length > 1) { + tl[sel.track].splice(sel.i, 1); + if (tl[sel.track][0]) tl[sel.track][0].start = 0; + } + sel = null; save(); renderInspector(); draw(); + }; + playBtn.onclick = () => { + playing = !playing; + playBtn.textContent = playing ? "❚❚" : "▶"; + playBtn.classList.toggle("on", playing); + if (playing) { + if (playhead >= N() - 1) playhead = 0; + phF = playhead; lastT = performance.now(); requestAnimationFrame(tick); + } + }; + let phF = 0; + function tick(t) { + if (!playing || !root.isConnected) { playing = false; playBtn.textContent = "▶"; return; } + phF = (playhead === Math.floor(phF) ? phF : playhead) + ((t - lastT) / 1000) * setup.fps; + lastT = t; + if (phF >= N() - 1) phF = 0; + playhead = Math.floor(phF); + draw(); + requestAnimationFrame(tick); + } + + // keep in sync with Setup / widget changes + const syncTimer = setInterval(() => { + if (!root.isConnected) return; + const s = readSetup(node); + if (s.frames !== setup.frames || s.fps !== setup.fps || s.width !== setup.width || s.height !== setup.height) { + setup = s; schedulePreview(); draw(); + } + }, 800); + const ro = new ResizeObserver(() => draw()); + ro.observe(root); + + loadCatalog().then(() => { renderInspector(); draw(); }); + load(); + renderInspector(); + schedulePreview(); + + return { + root, + reload: () => { load(); sel = null; renderInspector(); schedulePreview(); draw(); }, + refresh: () => schedulePreview(), + dispose: () => { playing = false; clearInterval(syncTimer); ro.disconnect(); }, + }; +} + +// --------------------------------------------------------------------------- +app.registerExtension({ + name: "Difforum.DirectorTimeline", + + async beforeRegisterNodeDef(nodeType, nodeData) { + if (nodeData.name !== NODE_ID) return; + + const onCreated = nodeType.prototype.onNodeCreated; + nodeType.prototype.onNodeCreated = function () { + onCreated?.apply(this, arguments); + injectStyles(); + const tw = this.widgets?.find((w) => w.name === "timeline"); + if (!tw) return; + tw.computeSize = () => [0, -4]; + tw.hidden = true; + if (tw.inputEl) tw.inputEl.style.display = "none"; + if (tw.element) tw.element.style.display = "none"; + + const ed = buildEditor(this, tw); + this.addDOMWidget("difforum_timeline", "div", ed.root, { + serialize: false, + hideOnZoom: false, + getHeight: () => 640, + getMinHeight: () => 560, + }); + // other widgets (camera_mode, transition...) refresh the preview + for (const w of this.widgets || []) { + if (w === tw || w.name === "difforum_timeline") continue; + const cb = w.callback; + w.callback = function () { const r = cb?.apply(this, arguments); ed.refresh(); return r; }; + } + const onDraw = this.onDrawForeground; + this.onDrawForeground = function (...a) { + if (tw.inputEl) tw.inputEl.style.display = "none"; + if (tw.element) tw.element.style.display = "none"; + return onDraw?.apply(this, a); + }; + this.__dfx = ed; + this.setSize([Math.max(this.size[0], 800), Math.max(this.size[1], 880)]); + }; + + const onConfigure = nodeType.prototype.onConfigure; + nodeType.prototype.onConfigure = function () { + onConfigure?.apply(this, arguments); + setTimeout(() => this.__dfx?.reload(), 30); + }; + const onConn = nodeType.prototype.onConnectionsChange; + nodeType.prototype.onConnectionsChange = function () { + const r = onConn?.apply(this, arguments); + setTimeout(() => this.__dfx?.refresh(), 30); + return r; + }; + const onRemoved = nodeType.prototype.onRemoved; + nodeType.prototype.onRemoved = function () { + this.__dfx?.dispose(); + return onRemoved?.apply(this, arguments); + }; + }, +}); diff --git a/js/director.js b/js/legacy_director.js similarity index 100% rename from js/director.js rename to js/legacy_director.js diff --git a/legacy/__init__.py b/legacy/__init__.py new file mode 100644 index 0000000..4a6e1fe --- /dev/null +++ b/legacy/__init__.py @@ -0,0 +1,88 @@ +""" +Difforum 0.x nodes, kept so existing workflows still open and run. + +Every class here is flagged ``DEPRECATED`` (hidden from the node search in +current ComfyUI frontends) and moved to the ``Difforum/legacy`` category. They +will be removed in Difforum 2.0. See docs/MIGRATION.md for the v1 replacement +of each one. +""" + +from __future__ import annotations + +import importlib +import logging + +log = logging.getLogger("difforum") + +_MODULES = ( + "schedule_nodes", "audio_nodes", "hybrid_nodes", "catalog_nodes", + "warp_nodes", "sampler_nodes", "guide_nodes", "prompt_nodes", + "live_nodes", "effects_nodes", "look_nodes", "video_nodes", + "glitch_nodes", "director_nodes", "storyboard_nodes", "loop_nodes", + "setup_nodes", +) + +# old class id -> the v1 node that replaces it (shown in the node description) +REPLACED_BY = { + "DifforumAnimSetup": "Difforum · Setup", + "DifforumAnimSetupPlus": "Difforum · Setup", + "DifforumSchedule": "Difforum · Schedule", + "DifforumSampleSchedule": "Difforum · Schedule (values output)", + "DifforumScheduleInfo": "Difforum · Schedule Plot", + "DifforumSchedulePlot": "Difforum · Schedule Plot", + "DifforumAudioAnalyzer": "Difforum · Audio Analyzer", + "DifforumAudioSchedule": "Difforum · Audio Curve", + "DifforumCamera": "Difforum · Camera (expressions)", + "DifforumCameraMove": "Difforum · Camera (one key)", + "DifforumCameraShots": "Difforum · Camera (blend=0)", + "DifforumCameraKeys": "Difforum · Camera", + "DifforumSeamlessCamera": "Difforum · Camera (loop_mode)", + "DifforumCameraPreview": "Difforum · Storyboard", + "DifforumWarp": "Difforum · Storyboard", + "DifforumStoryboard": "Difforum · Storyboard", + "DifforumFilmDirector": "Difforum · Director", + "DifforumPromptSchedule": "Difforum · Prompt Travel", + "DifforumPromptScenes": "Difforum · Director / Prompt Travel", + "DifforumPromptBatch": "Difforum · Prompt Travel (batched output)", + "DifforumFeedbackSampler": "Difforum · Feedback Sampler", + "DifforumLiveSampler": "Difforum · Live Sampler", + "DifforumLiveStep": "Difforum · Live Sampler", + "DifforumGuideBuilder": "Difforum · Guide Frames", + "DifforumLoopTake": "Difforum · Loop", + "DifforumLoopBlend": "Difforum · Loop", + "DifforumPingPong": "Difforum · Loop", + "DifforumSymmetry": "Difforum · Symmetry", + "DifforumEchoTrails": "Difforum · Echo Trails", + "DifforumDetailGuard": "Difforum · Detail Guard", + "DifforumFlowStabilize": "Difforum · Flow Stabilize", + "DifforumModelProfile": "docs/MODELS.md", + "DifforumModelCatalog": "docs/MODELS.md", + "DifforumLoadVideo": "core Load Video / VHS", + "DifforumSaveVideo": "core Save Video (keeps audio)", + "DifforumLook": "a colour-grading pack", + "DifforumColorGrade": "a colour-grading pack", + "DifforumGlow": "a colour-grading pack", + "DifforumGlitch": "a glitch pack", + "DifforumDatamosh": "a glitch pack", +} + +NODE_CLASS_MAPPINGS: dict = {} +NODE_DISPLAY_NAME_MAPPINGS: dict = {} + +for _name in _MODULES: + try: + _mod = importlib.import_module(f".{_name}", __name__) + except Exception as exc: # a broken legacy module must never block v1 + log.warning("[Difforum] legacy module %s not loaded: %s", _name, exc) + continue + for _cid, _cls in _mod.NODE_CLASS_MAPPINGS.items(): + _cls.DEPRECATED = True + _cls.CATEGORY = "Difforum/legacy" + _new = REPLACED_BY.get(_cid, "a v1 node") + _cls.DESCRIPTION = ( + f"Difforum 0.x node, kept so old workflows open. Replaced by {_new}. " + "Removed in Difforum 2.0." + ) + NODE_CLASS_MAPPINGS[_cid] = _cls + _disp = _mod.NODE_DISPLAY_NAME_MAPPINGS.get(_cid, _cid) + NODE_DISPLAY_NAME_MAPPINGS[_cid] = f"{_disp} [legacy]" diff --git a/nodes/audio_nodes.py b/legacy/audio_nodes.py similarity index 93% rename from nodes/audio_nodes.py rename to legacy/audio_nodes.py index 13f5f19..6ddae8e 100644 --- a/nodes/audio_nodes.py +++ b/legacy/audio_nodes.py @@ -9,17 +9,12 @@ from __future__ import annotations -import sys -from pathlib import Path import numpy as np -_PKG_ROOT = Path(__file__).resolve().parent.parent -if str(_PKG_ROOT) not in sys.path: - sys.path.insert(0, str(_PKG_ROOT)) -from core.audio import REACTIVE_MODES, REACTIVE_SOURCES, analyze, reactive_curve # noqa: E402 -from core.schedule import Schedule # noqa: E402 +from ..core.audio import REACTIVE_MODES, REACTIVE_SOURCES, analyze, reactive_curve # noqa: E402 +from ..core.schedule import Schedule # noqa: E402 CATEGORY = "Difforum/audio" diff --git a/nodes/catalog_nodes.py b/legacy/catalog_nodes.py similarity index 85% rename from nodes/catalog_nodes.py rename to legacy/catalog_nodes.py index 92ad045..1d9ea07 100644 --- a/nodes/catalog_nodes.py +++ b/legacy/catalog_nodes.py @@ -9,14 +9,9 @@ from __future__ import annotations -import sys -from pathlib import Path -_PKG_ROOT = Path(__file__).resolve().parent.parent -if str(_PKG_ROOT) not in sys.path: - sys.path.insert(0, str(_PKG_ROOT)) -from core import models as M # noqa: E402 +from ..core import models as M # noqa: E402 CATEGORY = "Difforum/models" diff --git a/nodes/director_nodes.py b/legacy/director_nodes.py similarity index 100% rename from nodes/director_nodes.py rename to legacy/director_nodes.py diff --git a/nodes/effects_nodes.py b/legacy/effects_nodes.py similarity index 94% rename from nodes/effects_nodes.py rename to legacy/effects_nodes.py index d7ef2bd..b826273 100644 --- a/nodes/effects_nodes.py +++ b/legacy/effects_nodes.py @@ -9,16 +9,11 @@ from __future__ import annotations -import sys -from pathlib import Path -_PKG_ROOT = Path(__file__).resolve().parent.parent -if str(_PKG_ROOT) not in sys.path: - sys.path.insert(0, str(_PKG_ROOT)) -from core.detail import NOISE_MODES, detail_guard # noqa: E402 -from core.effects import echo_trails, loop_blend, pingpong # noqa: E402 -from core.symmetry import SYMMETRY_MODES, apply_symmetry # noqa: E402 +from ..core.detail import NOISE_MODES, detail_guard # noqa: E402 +from ..core.effects import echo_trails, loop_blend, pingpong # noqa: E402 +from ..core.symmetry import SYMMETRY_MODES, apply_symmetry # noqa: E402 CATEGORY = "Difforum/effects" @@ -127,7 +122,7 @@ def INPUT_TYPES(cls): CATEGORY = CATEGORY def run(self, frames, strength, flow_scale, error_gate): - from core.flow import stabilize + from ..core.flow import stabilize return (stabilize(frames, strength=float(strength), flow_scale=float(flow_scale), error_gate=float(error_gate)),) diff --git a/nodes/glitch_nodes.py b/legacy/glitch_nodes.py similarity index 92% rename from nodes/glitch_nodes.py rename to legacy/glitch_nodes.py index 6d5d0e2..401069e 100644 --- a/nodes/glitch_nodes.py +++ b/legacy/glitch_nodes.py @@ -5,14 +5,9 @@ from __future__ import annotations -import sys -from pathlib import Path -_PKG_ROOT = Path(__file__).resolve().parent.parent -if str(_PKG_ROOT) not in sys.path: - sys.path.insert(0, str(_PKG_ROOT)) -from core.glitch import KERNELS, apply_glitch # noqa: E402 +from ..core.glitch import KERNELS, apply_glitch # noqa: E402 CATEGORY = "Difforum/effects" @@ -74,7 +69,7 @@ def INPUT_TYPES(cls): CATEGORY = CATEGORY def run(self, frames, intensity, mode, block_size, flow_scale): - from core.flow import datamosh + from ..core.flow import datamosh return (datamosh(frames, intensity=float(intensity), mode=mode, block_size=int(block_size), flow_scale=float(flow_scale)),) diff --git a/nodes/guide_nodes.py b/legacy/guide_nodes.py similarity index 95% rename from nodes/guide_nodes.py rename to legacy/guide_nodes.py index aecefc6..8f5aa66 100644 --- a/nodes/guide_nodes.py +++ b/legacy/guide_nodes.py @@ -14,16 +14,11 @@ from __future__ import annotations import math -import sys -from pathlib import Path import torch -_PKG_ROOT = Path(__file__).resolve().parent.parent -if str(_PKG_ROOT) not in sys.path: - sys.path.insert(0, str(_PKG_ROOT)) -from core.warp import warp_2d, warp_3d # noqa: E402 +from ..core.warp import warp_2d, warp_3d # noqa: E402 CATEGORY = "Difforum/hybrid" diff --git a/nodes/hybrid_nodes.py b/legacy/hybrid_nodes.py similarity index 93% rename from nodes/hybrid_nodes.py rename to legacy/hybrid_nodes.py index b745824..7c86405 100644 --- a/nodes/hybrid_nodes.py +++ b/legacy/hybrid_nodes.py @@ -11,19 +11,14 @@ from __future__ import annotations -import sys -from pathlib import Path - -_PKG_ROOT = Path(__file__).resolve().parent.parent -if str(_PKG_ROOT) not in sys.path: - sys.path.insert(0, str(_PKG_ROOT)) - -from core import build_schedule # noqa: E402 -from core.camera import CAMERA_MODES, build_camera # noqa: E402 -from core.camera_presets import CAMERA_PRESETS, preset_schedules # noqa: E402 -from core.plot import render_camera_path # noqa: E402 -from core.profiles import FAMILIES, QUALITIES, detect_device, resolve_profile, summarize # noqa: E402 -from core.shots import shots_to_axis_values # noqa: E402 + + +from ..core import build_schedule # noqa: E402 +from ..core.camera import CAMERA_MODES, build_camera # noqa: E402 +from ..core.camera_presets import CAMERA_PRESETS, preset_schedules # noqa: E402 +from ..core.plot import render_camera_path # noqa: E402 +from ..core.profiles import FAMILIES, QUALITIES, detect_device, resolve_profile, summarize # noqa: E402 +from ..core.shots import shots_to_axis_values # noqa: E402 CATEGORY = "Difforum/hybrid" diff --git a/nodes/live_nodes.py b/legacy/live_nodes.py similarity index 97% rename from nodes/live_nodes.py rename to legacy/live_nodes.py index b7ffbe8..987141f 100644 --- a/nodes/live_nodes.py +++ b/legacy/live_nodes.py @@ -15,21 +15,16 @@ from __future__ import annotations import math -import sys -from pathlib import Path import torch -_PKG_ROOT = Path(__file__).resolve().parent.parent -if str(_PKG_ROOT) not in sys.path: - sys.path.insert(0, str(_PKG_ROOT)) -from core.color import COLOR_MODES, match_color # noqa: E402 -from core.detail import add_noise as _add_noise # noqa: E402 -from core.detail import sharpen as _sharpen # noqa: E402 -from core.fx import FxRunner # noqa: E402 -from core.symmetry import SYMMETRY_MODES, apply_symmetry # noqa: E402 -from core.warp import warp_2d, warp_3d # noqa: E402 +from ..core.color import COLOR_MODES, match_color # noqa: E402 +from ..core.detail import add_noise as _add_noise # noqa: E402 +from ..core.detail import sharpen as _sharpen # noqa: E402 +from ..core.fx import FxRunner # noqa: E402 +from ..core.symmetry import SYMMETRY_MODES, apply_symmetry # noqa: E402 +from ..core.warp import warp_2d, warp_3d # noqa: E402 CATEGORY = "Difforum/live" @@ -167,7 +162,6 @@ def send(self, image): if not self.ok: return try: - import SpoutGL from OpenGL import GL arr = (image[0].clamp(0, 1).cpu().numpy() * 255).astype("uint8") h, w = arr.shape[:2] diff --git a/nodes/look_nodes.py b/legacy/look_nodes.py similarity index 95% rename from nodes/look_nodes.py rename to legacy/look_nodes.py index ad76f5d..6a966f2 100644 --- a/nodes/look_nodes.py +++ b/legacy/look_nodes.py @@ -8,16 +8,11 @@ from __future__ import annotations -import sys -from pathlib import Path import torch -_PKG_ROOT = Path(__file__).resolve().parent.parent -if str(_PKG_ROOT) not in sys.path: - sys.path.insert(0, str(_PKG_ROOT)) -from core.look import ( # noqa: E402 +from ..core.look import ( # noqa: E402 LOOK_PRESETS, apply_look, color_grade, diff --git a/nodes/loop_nodes.py b/legacy/loop_nodes.py similarity index 95% rename from nodes/loop_nodes.py rename to legacy/loop_nodes.py index 618bf15..59b36d6 100644 --- a/nodes/loop_nodes.py +++ b/legacy/loop_nodes.py @@ -8,20 +8,17 @@ from __future__ import annotations -import os -import sys -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from core.camera import CAMERA_MODES, build_camera # noqa: E402 -from core.loop import ( # noqa: E402 +from ..core.camera import CAMERA_MODES, build_camera # noqa: E402 +from ..core.loop import ( # noqa: E402 LOOP_MODES, close_axis_values, seam_report, suggest_harmonics, tile_laps, ) -from core.shots import shots_to_axis_values # noqa: E402 +from ..core.shots import shots_to_axis_values # noqa: E402 CATEGORY = "Difforum/loop" diff --git a/nodes/prompt_nodes.py b/legacy/prompt_nodes.py similarity index 95% rename from nodes/prompt_nodes.py rename to legacy/prompt_nodes.py index 69b103c..0a72e3c 100644 --- a/nodes/prompt_nodes.py +++ b/legacy/prompt_nodes.py @@ -8,17 +8,12 @@ from __future__ import annotations -import sys -from pathlib import Path -_PKG_ROOT = Path(__file__).resolve().parent.parent -if str(_PKG_ROOT) not in sys.path: - sys.path.insert(0, str(_PKG_ROOT)) -from core.prompt import ( # noqa: E402 +from ..core.prompt import ( # noqa: E402 blend_conditioning, parse_prompt_schedule, plan_blend, scenes_to_keyframes, ) -from core.schedule import EASINGS # noqa: E402 +from ..core.schedule import EASINGS # noqa: E402 CATEGORY = "Difforum/schedule" diff --git a/nodes/sampler_nodes.py b/legacy/sampler_nodes.py similarity index 95% rename from nodes/sampler_nodes.py rename to legacy/sampler_nodes.py index 5c43cc2..1943e41 100644 --- a/nodes/sampler_nodes.py +++ b/legacy/sampler_nodes.py @@ -13,21 +13,16 @@ from __future__ import annotations import math -import sys -from pathlib import Path import torch -_PKG_ROOT = Path(__file__).resolve().parent.parent -if str(_PKG_ROOT) not in sys.path: - sys.path.insert(0, str(_PKG_ROOT)) -from core.color import COLOR_MODES, match_color # noqa: E402 -from core.detail import add_noise as _add_noise # noqa: E402 -from core.detail import sharpen as _sharpen # noqa: E402 -from core.fx import FxRunner # noqa: E402 -from core.symmetry import SYMMETRY_MODES, apply_symmetry # noqa: E402 -from core.warp import warp_2d, warp_3d # noqa: E402 +from ..core.color import COLOR_MODES, match_color # noqa: E402 +from ..core.detail import add_noise as _add_noise # noqa: E402 +from ..core.detail import sharpen as _sharpen # noqa: E402 +from ..core.fx import FxRunner # noqa: E402 +from ..core.symmetry import SYMMETRY_MODES, apply_symmetry # noqa: E402 +from ..core.warp import warp_2d, warp_3d # noqa: E402 BORDER_MODES = ("reflection", "zeros", "border") diff --git a/nodes/schedule_nodes.py b/legacy/schedule_nodes.py similarity index 94% rename from nodes/schedule_nodes.py rename to legacy/schedule_nodes.py index 19db278..e2b3965 100644 --- a/nodes/schedule_nodes.py +++ b/legacy/schedule_nodes.py @@ -12,15 +12,10 @@ from __future__ import annotations -import sys -from pathlib import Path -# allow `from core import ...` whether loaded as a package or standalone -_PKG_ROOT = Path(__file__).resolve().parent.parent -if str(_PKG_ROOT) not in sys.path: - sys.path.insert(0, str(_PKG_ROOT)) +# allow `from ..core import ...` whether loaded as a package or standalone -from core import EASINGS, build_schedule # noqa: E402 +from ..core import EASINGS, build_schedule # noqa: E402 CATEGORY = "Difforum/schedule" @@ -170,7 +165,7 @@ def INPUT_TYPES(cls): def run(self, schedule, width, height): import torch - from core.plot import render_curve + from ..core.plot import render_curve arr = render_curve(schedule.as_list(), width=int(width), height=int(height)) return (torch.from_numpy(arr).unsqueeze(0),) diff --git a/nodes/setup_nodes.py b/legacy/setup_nodes.py similarity index 100% rename from nodes/setup_nodes.py rename to legacy/setup_nodes.py diff --git a/nodes/storyboard_nodes.py b/legacy/storyboard_nodes.py similarity index 95% rename from nodes/storyboard_nodes.py rename to legacy/storyboard_nodes.py index 601cfdf..76389ed 100644 --- a/nodes/storyboard_nodes.py +++ b/legacy/storyboard_nodes.py @@ -7,23 +7,20 @@ from __future__ import annotations -import os -import sys -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import torch # noqa: E402 -from core.camera import CAMERA_MODES, build_camera # noqa: E402 -from core.camera_keys import ( # noqa: E402 +from ..core.camera import CAMERA_MODES, build_camera # noqa: E402 +from ..core.camera_keys import ( # noqa: E402 keys_to_axis_values, lens_note, parse_camera_keys, ) -from core.camera_presets import flat_presets, needs_depth # noqa: E402 -from core.loop import LOOP_MODES, close_axis_values, tile_laps # noqa: E402 -from core.storyboard import contact_sheet, drift_curve, simulate # noqa: E402 -from core.symmetry import SYMMETRY_MODES # noqa: E402 +from ..core.camera_presets import flat_presets, needs_depth # noqa: E402 +from ..core.loop import LOOP_MODES, close_axis_values, tile_laps # noqa: E402 +from ..core.storyboard import contact_sheet, drift_curve, simulate # noqa: E402 +from ..core.symmetry import SYMMETRY_MODES # noqa: E402 CATEGORY = "Difforum/camera" diff --git a/nodes/video_nodes.py b/legacy/video_nodes.py similarity index 93% rename from nodes/video_nodes.py rename to legacy/video_nodes.py index 73e5984..23d0ae2 100644 --- a/nodes/video_nodes.py +++ b/legacy/video_nodes.py @@ -7,14 +7,9 @@ from __future__ import annotations import os -import sys -from pathlib import Path -_PKG_ROOT = Path(__file__).resolve().parent.parent -if str(_PKG_ROOT) not in sys.path: - sys.path.insert(0, str(_PKG_ROOT)) -from core.video import VIDEO_EXTS, read_video, safe_input_path, write_video # noqa: E402 +from ..core.video import VIDEO_EXTS, read_video, safe_input_path, write_video # noqa: E402 CATEGORY = "Difforum/video" diff --git a/nodes/warp_nodes.py b/legacy/warp_nodes.py similarity index 94% rename from nodes/warp_nodes.py rename to legacy/warp_nodes.py index 7306c17..76af7cf 100644 --- a/nodes/warp_nodes.py +++ b/legacy/warp_nodes.py @@ -14,16 +14,11 @@ from __future__ import annotations -import sys -from pathlib import Path import torch -_PKG_ROOT = Path(__file__).resolve().parent.parent -if str(_PKG_ROOT) not in sys.path: - sys.path.insert(0, str(_PKG_ROOT)) -from core.warp import warp_2d, warp_3d # noqa: E402 +from ..core.warp import warp_2d, warp_3d # noqa: E402 CATEGORY = "Difforum/warp" diff --git a/nodes/__init__.py b/nodes/__init__.py index e69de29..b571de1 100644 --- a/nodes/__init__.py +++ b/nodes/__init__.py @@ -0,0 +1,12 @@ +"""Difforum v1 nodes.""" + +from __future__ import annotations + +from . import bridges, curves, direction, export, post, render, setup + +NODE_CLASS_MAPPINGS: dict = {} +NODE_DISPLAY_NAME_MAPPINGS: dict = {} + +for _m in (setup, direction, curves, render, bridges, export, post): + NODE_CLASS_MAPPINGS.update(_m.NODE_CLASS_MAPPINGS) + NODE_DISPLAY_NAME_MAPPINGS.update(_m.NODE_DISPLAY_NAME_MAPPINGS) diff --git a/nodes/_common.py b/nodes/_common.py new file mode 100644 index 0000000..1a2bb3a --- /dev/null +++ b/nodes/_common.py @@ -0,0 +1,74 @@ +"""Shared plumbing for the Difforum v1 nodes.""" + +from __future__ import annotations + +import logging + +log = logging.getLogger("difforum") + +# Socket types. Kept identical to 0.x where the payload is compatible, so a +# legacy node can still feed a v1 node during migration. +PARAMS = "DIFFORUM_PARAMS" +SCHEDULE = "DIFFORUM_SCHEDULE" +CAMERA = "DIFFORUM_CAMERA" +AUDIO = "DIFFORUM_AUDIO" +PROMPT = "DIFFORUM_PROMPT" + +ROOT = "Difforum" +CAT_SETUP = f"{ROOT}/1 · Setup" +CAT_DIRECT = f"{ROOT}/2 · Direction" +CAT_CURVES = f"{ROOT}/3 · Curves & Prompts" +CAT_RENDER = f"{ROOT}/4 · Render" +CAT_BRIDGE = f"{ROOT}/5 · Video model bridges" +CAT_EXPORT = f"{ROOT}/6 · Export" +CAT_POST = f"{ROOT}/7 · Post" + + +def audio_vars(audio) -> dict | None: + """Per-frame audio curves for expressions, or None.""" + return audio.get("curves") if isinstance(audio, dict) else None + + +def call_comfy_node(class_id: str, **kwargs): + """Call another ComfyUI node by class id, v1 or v3 schema, and return its + outputs as a tuple. Used for bridges onto core nodes (e.g. LTXVAddGuide) so + Difforum never imports another pack's internals by path.""" + import nodes as comfy_nodes + + cls = comfy_nodes.NODE_CLASS_MAPPINGS.get(class_id) + if cls is None: + raise RuntimeError( + f"ComfyUI node {class_id!r} is not available - update ComfyUI " + "(it ships with core) or install the pack that provides it." + ) + execute = getattr(cls, "execute", None) + if callable(execute): + out = execute(**kwargs) # v3 schema (io.ComfyNode): classmethod + else: + out = getattr(cls(), cls.FUNCTION)(**kwargs) + res = getattr(out, "result", out) + if isinstance(res, dict): + res = res.get("result", ()) + return tuple(res) + + +def progress_bar(total: int): + try: + import comfy.utils + return comfy.utils.ProgressBar(int(total)) + except Exception: # outside ComfyUI (tests) + class _Null: + def update(self, *_a, **_k): + pass + + def update_absolute(self, *_a, **_k): + pass + return _Null() + + +def check_interrupt(): + try: + import comfy.model_management as mm + mm.throw_exception_if_processing_interrupted() + except ImportError: + pass diff --git a/nodes/bridges.py b/nodes/bridges.py new file mode 100644 index 0000000..de0975a --- /dev/null +++ b/nodes/bridges.py @@ -0,0 +1,419 @@ +"""Bridges from Difforum direction to modern video models. + +Difforum stays the director (camera, timing, prompts); the video model does +the temporally coherent rendering. Nothing here imports another pack's +internals: outputs are plain IMAGE / MASK / INT / STRING, and the LTX node +drives ComfyUI's own core `LTXVAddGuide`. +""" + +from __future__ import annotations + +import torch + +from ..core.direction import blocks_in_range, describe_blocks, describe_timed, describe_track +from ..core.engine import EngineConfig, FeedbackEngine, resize_bhwc +from ._common import CAMERA, CAT_BRIDGE, PARAMS, call_comfy_node, progress_bar +from .direction import DIRECTION +from .setup import TARGETS, snap, snap_frames + +GRIDS = { + "LTX-2 / 2.5 (8k+1)": (8, 1), + "MiniMax H3 (17k+5)": (17, 5), + "Wan 2.x (4k+1)": (4, 1), + "any": (1, 0), +} + + +def _params_camera(direction, params, camera): + if direction is not None: + params = params or direction.params + camera = camera or direction.camera + if params is None or camera is None: + raise ValueError("Connect a Director `direction`, or params + camera.") + return params, camera + + +def _parse_indices(text: str, n: int) -> list[int]: + out = [] + for tok in str(text or "").replace(";", ",").split(","): + tok = tok.strip() + if tok.lstrip("-").isdigit(): + i = int(tok) + out.append(i + n if i < 0 else i) + return sorted({i for i in out if 0 <= i < n}) + + +class DifforumGuideFrames: + """Warp one anchor image along the Director's camera path. + + Produces a guide video whose motion is exactly the camera you drew, for + video models that take control/guide frames (LTX guides, Wan VACE, H3 + first/last frame). Revealed areas are filled with neutral gray and marked + in the mask, using the convention the target expects. + """ + + DESCRIPTION = __doc__ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "anchor_image": ("IMAGE",), + "mask_convention": (["1 = generate (VACE / LTX / H3)", "1 = keep"], + {"default": "1 = generate (VACE / LTX / H3)"}), + "hole_fill": (["gray", "stretch edge", "black"], {"default": "gray"}), + }, + "optional": { + "direction": (DIRECTION,), + "params": (PARAMS,), + "camera": (CAMERA,), + "depth": ("IMAGE", {"tooltip": "Depth of the anchor, for real parallax in 3d mode."}), + "translation_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 20.0, "step": 0.05}), + }, + } + + RETURN_TYPES = ("IMAGE", "MASK", "STRING") + RETURN_NAMES = ("guide_frames", "masks", "info") + FUNCTION = "run" + CATEGORY = CAT_BRIDGE + + def run(self, anchor_image, mask_convention, hole_fill, direction=None, params=None, + camera=None, depth=None, translation_scale=1.0): + params, camera = _params_camera(direction, params, camera) + n = min(int(params["max_frames"]), len(camera.deltas)) + w, h = int(params["width"]), int(params["height"]) + border = "border" if hole_fill == "stretch edge" else "zeros" + engine = FeedbackEngine(camera, EngineConfig(width=w, height=h, border=border, + translation_scale=float(translation_scale)), depth=depth) + pbar = progress_bar(n) + imgs, masks = [], [] + for f, (img, valid) in enumerate(engine.iter_from_anchor(anchor_image, n)): + if hole_fill == "gray": + img = img * valid + 0.5 * (1.0 - valid) + imgs.append(img) + masks.append(valid[..., 0]) + pbar.update_absolute(f + 1, n) + mask = torch.cat(masks, dim=0) + if mask_convention.startswith("1 = generate"): + mask = 1.0 - mask + info = (f"{n} guide frames {w}x{h} ({engine.report.mode}); " + f"revealed area avg {float((1 - torch.cat(masks)).mean()) * 100:.1f}%") + return (torch.cat(imgs, dim=0), mask, info) + + +class DifforumKeyframes: + """Pick keyframes out of any frame batch (a Storyboard, a Feedback render, + guide frames) for a video model, on that model's frame grid. + + Outputs the keyframes, their indices, a full-length *sparse* batch (black + except at keyframes - what `LTXVAddGuidesFromBatch` expects), the first and + last frame (for first-last-frame models) and the snapped clip length. + """ + + DESCRIPTION = __doc__ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "frames": ("IMAGE",), + "grid": (list(GRIDS), {"default": "LTX-2 / 2.5 (8k+1)"}), + "every_seconds": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 60.0, "step": 0.05, + "tooltip": "Spacing between keyframes. 0 = first and last only."}), + "fps": ("FLOAT", {"default": 24.0, "min": 1.0, "max": 120.0}), + }, + "optional": { + "indices": ("STRING", {"default": "", "tooltip": "Explicit frame list, e.g. 0, 48, 96, -1 (overrides spacing)."}), + }, + } + + RETURN_TYPES = ("IMAGE", "STRING", "IMAGE", "IMAGE", "IMAGE", "INT", "STRING") + RETURN_NAMES = ("keyframes", "indices", "sparse_batch", "first_frame", "last_frame", "length", "info") + FUNCTION = "run" + CATEGORY = CAT_BRIDGE + + def run(self, frames, grid, every_seconds, fps, indices=""): + rule = GRIDS.get(grid, (1, 0)) + total = int(frames.shape[0]) + length = snap_frames(total, rule) + if length > total: # never ask for frames we do not have + a, b = rule + length = max(b if b else 1, total if a <= 1 else ((total - b) // a) * a + b) + frames = frames[:length] + + if indices.strip(): + idx = _parse_indices(indices, length) + else: + step = int(round(float(every_seconds) * float(fps))) + if rule[0] > 1 and step > 0: # align to the model's latent stride + step = max(rule[0], int(round(step / rule[0])) * rule[0]) + idx = list(range(0, length, step)) if step > 0 else [0] + if idx[-1] != length - 1: + idx.append(length - 1) + idx = idx or [0] + + keys = frames[idx] + sparse = torch.zeros_like(frames) + sparse[idx] = frames[idx] + info = (f"{len(idx)} keyframes on {grid}, length {length} " + f"({length / fps:.2f}s): {', '.join(map(str, idx))}") + return (keys, ",".join(map(str, idx)), sparse, frames[:1], frames[-1:], int(length), info) + + +class DifforumCameraPrompt: + """The camera, in words - for prompt-driven video models (MiniMax H3, + LTX, Seedance, Veo...). Uses the Director's blocks when available, or + reads any camera track and names its moves.""" + + DESCRIPTION = __doc__ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "format": (["sentence", "timed lines", "prompt suffix"], {"default": "sentence"}), + }, + "optional": { + "direction": (DIRECTION,), + "params": (PARAMS,), + "camera": (CAMERA,), + "prefix": ("STRING", {"default": "", "multiline": True, + "tooltip": "Your shot description; the camera sentence is appended."}), + }, + } + + RETURN_TYPES = ("STRING",) + RETURN_NAMES = ("text",) + FUNCTION = "run" + CATEGORY = CAT_BRIDGE + + def run(self, format, direction=None, params=None, camera=None, prefix=""): + if direction is not None and camera is None: + d = direction.direction + sentence = d.camera_text + timed = describe_timed(d.camera_blocks, d.frames, d.fps) + else: + params, camera = _params_camera(direction, params, camera) + sentence, timed = describe_track(camera, float(params["fps"])) + if format == "timed lines": + body = timed + elif format == "prompt suffix": + body = "Camera: " + sentence[len("The camera "):] if sentence.startswith("The camera ") else sentence + else: + body = sentence + text = (prefix.strip() + " " + body).strip() if prefix.strip() else body + return (text,) + + +class DifforumLTXGuides: + """Put Difforum keyframes into an LTX-2 / 2.5 latent as guides. + + Wraps ComfyUI's core `LTXVAddGuide` once per keyframe, so the camera you + drew becomes the LTX shot. Feed keyframes + indices from the Keyframes + node (from a Storyboard, Guide Frames or a Feedback render), and an empty + LTX latent whose length matches (Setup target = LTX). + """ + + DESCRIPTION = __doc__ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "positive": ("CONDITIONING",), + "negative": ("CONDITIONING",), + "vae": ("VAE",), + "latent": ("LATENT",), + "keyframes": ("IMAGE",), + "indices": ("STRING", {"default": "0", "forceInput": True}), + "strength": ("FLOAT", {"default": 0.7, "min": 0.0, "max": 1.0, "step": 0.01, + "tooltip": "Guide strength for in-between keyframes."}), + "first_strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01, + "tooltip": "Frame 0 usually locks the look: keep it high."}), + "last_strength": ("FLOAT", {"default": 0.7, "min": 0.0, "max": 1.0, "step": 0.01}), + }, + } + + RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT", "STRING") + RETURN_NAMES = ("positive", "negative", "latent", "info") + FUNCTION = "run" + CATEGORY = CAT_BRIDGE + + def run(self, positive, negative, vae, latent, keyframes, indices, strength, + first_strength, last_strength): + idx = _parse_indices(indices, 10**9) + if len(idx) != keyframes.shape[0]: + raise ValueError(f"{keyframes.shape[0]} keyframes but {len(idx)} indices ({indices!r})") + lines = [] + for k, (fi, img) in enumerate(zip(idx, keyframes)): + s = first_strength if k == 0 else (last_strength if k == len(idx) - 1 else strength) + if s <= 0: + continue + positive, negative, latent = call_comfy_node( + "LTXVAddGuide", positive=positive, negative=negative, vae=vae, latent=latent, + image=img.unsqueeze(0), frame_idx=int(fi), strength=float(s))[:3] + lines.append(f" guide @ {fi} strength {s:g}") + return (positive, negative, latent, "LTX guides:\n" + "\n".join(lines)) + + +class DifforumH3Guides: + """Anchor Difforum keyframes inside a MiniMax H3 generation. + + Wraps ComfyUI's core `MiniMaxH3AddGuide` once per keyframe, so the camera + you drew becomes the H3 shot: feed keyframes + indices from the Keyframes + node (grid H3) and the positive + AV latent from `MiniMax H3 Reference to + Video` (or `Image to Video`). Optionally anchor a soundtrack at frame 0, so + an audio-reactive direction and H3's own audio stay in sync. + + H3 is trained with a few guides per clip: `max_guides` keeps the first, + the last and evenly spaced ones in between. + """ + + DESCRIPTION = __doc__ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "positive": ("CONDITIONING",), + "latent": ("LATENT", {"tooltip": "The MiniMax H3 AV latent."}), + "vae": ("VAE", {"tooltip": "MiniMax H3 video VAE."}), + "keyframes": ("IMAGE",), + "indices": ("STRING", {"default": "0", "forceInput": True}), + "max_guides": ("INT", {"default": 4, "min": 1, "max": 16}), + "skip_first": ("BOOLEAN", {"default": False, + "tooltip": "Enable when frame 0 is already set (e.g. Image to Video first_frame)."}), + }, + "optional": { + "audio_vae": ("VAE", {"tooltip": "MiniMax H3 audio VAE, needed with audio."}), + "audio": ("AUDIO", {"tooltip": "Soundtrack anchored at frame 0."}), + }, + } + + RETURN_TYPES = ("CONDITIONING", "STRING") + RETURN_NAMES = ("positive", "info") + FUNCTION = "run" + CATEGORY = CAT_BRIDGE + + def run(self, positive, latent, vae, keyframes, indices, max_guides, skip_first, + audio_vae=None, audio=None): + idx = _parse_indices(indices, 10**9) + if len(idx) != keyframes.shape[0]: + raise ValueError(f"{keyframes.shape[0]} keyframes but {len(idx)} indices ({indices!r})") + pairs = list(zip(idx, keyframes)) + if skip_first and pairs and pairs[0][0] == 0: + pairs = pairs[1:] + if len(pairs) > max_guides: # keep the keyframes nearest to evenly spaced times + f0, f1 = pairs[0][0], pairs[-1][0] + keep = set() + for i in range(max_guides): + t = f0 + (f1 - f0) * i / max(1, max_guides - 1) + free = [k for k in range(len(pairs)) if k not in keep] + keep.add(min(free, key=lambda k: abs(pairs[k][0] - t))) + pairs = [pairs[k] for k in sorted(keep)] + lines = [] + for k, (fi, img) in enumerate(pairs): + kw = {"positive": positive, "latent": latent, "frame_idx": int(fi), "vae": vae, + "image": img.unsqueeze(0)} + if k == 0 and audio is not None and fi == 0: + kw.update(audio=audio, audio_vae=audio_vae) + positive = call_comfy_node("MiniMaxH3AddGuide", **kw)[0] + lines.append(f" guide @ frame {fi}" + (" + audio" if "audio" in kw else "")) + if audio is not None and not any("audio" in x for x in lines): + positive = call_comfy_node("MiniMaxH3AddGuide", positive=positive, latent=latent, + frame_idx=0, audio=audio, audio_vae=audio_vae)[0] + lines.append(" audio @ frame 0") + return (positive, "H3 guides:\n" + "\n".join(lines)) + + +H3_LENGTHS = {"auto (from frames)": 0, "124 (~5s)": 124, "243 (~10s)": 243, + "362 (~15s)": 362, "481 (~20s)": 481} + + +class DifforumH3Shot: + """Everything a MiniMax H3 first-last-frame shot needs, from Difforum. + + Give it the frames of a Storyboard / Feedback render / Guide Frames and it + returns the first and last frame of the chosen segment, the length on the + H3 17k+5 grid, the size on the 32 px grid, and the camera move of that + segment in words for the prompt. Clips longer than one H3 generation are + split into segments: render segment 0, then feed its last frame as the + next segment's first frame. + """ + + DESCRIPTION = __doc__ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "frames": ("IMAGE",), + "segment_length": (list(H3_LENGTHS), {"default": "auto (from frames)"}), + "segment": ("INT", {"default": 0, "min": 0, "max": 999}), + }, + "optional": { + "direction": (DIRECTION,), + "params": (PARAMS,), + "camera": (CAMERA,), + "shot_description": ("STRING", {"default": "", "multiline": True}), + }, + } + + RETURN_TYPES = ("IMAGE", "IMAGE", "INT", "INT", "INT", "STRING", "INT", "STRING") + RETURN_NAMES = ("first_frame", "last_frame", "length", "width", "height", "prompt", + "segments", "info") + FUNCTION = "run" + CATEGORY = CAT_BRIDGE + + def run(self, frames, segment_length, segment, direction=None, params=None, camera=None, + shot_description=""): + total = int(frames.shape[0]) + rule = TARGETS["MiniMax H3"][0] + seg_len = H3_LENGTHS.get(segment_length, 0) or min(481, snap_frames(total, rule)) + seg_len = min(seg_len, max(5, total)) + seg_len = max(5, ((seg_len - 5) // 17) * 17 + 5) + stride = seg_len - 1 # segments share their boundary frame + segments = max(1, -(-(total - 1) // stride)) + s = min(int(segment), segments - 1) + start = s * stride + end = min(total - 1, start + seg_len - 1) + + h, w = int(frames.shape[1]), int(frames.shape[2]) + w32, h32 = snap(w, 32), snap(h, 32) + first = resize_bhwc(frames[start:start + 1], w32, h32) + last = resize_bhwc(frames[end:end + 1], w32, h32) + + cam_text = "" + if direction is not None: + d = direction.direction + cam_text = describe_blocks(blocks_in_range(d.camera_blocks, start, end + 1), + end + 1 - start, d.fps) + elif camera is not None and params is not None: + from ..core.camera import CameraTrack + sub = CameraTrack(deltas=camera.deltas[start:end + 1], poses=camera.poses[start:end + 1], + zoom=camera.zoom[start:end + 1], fov=camera.fov[start:end + 1], + mode=camera.mode) + cam_text = describe_track(sub, float(params["fps"]))[0] + prompt = " ".join(x for x in (shot_description.strip(), cam_text) if x) + info = (f"H3 segment {s + 1}/{segments}: frames {start}-{end} " + f"(length {seg_len}, {seg_len / 24:.2f}s @ 24fps), {w32}x{h32}\n" + f" mode fl2va - wire first/last into MiniMax H3 Image to Video") + return (first, last, int(seg_len), int(w32), int(h32), prompt, int(segments), info) + + +NODE_CLASS_MAPPINGS = { + "Difforum_GuideFrames": DifforumGuideFrames, + "Difforum_Keyframes": DifforumKeyframes, + "Difforum_CameraPrompt": DifforumCameraPrompt, + "Difforum_LTXGuides": DifforumLTXGuides, + "Difforum_H3Shot": DifforumH3Shot, + "Difforum_H3Guides": DifforumH3Guides, +} +NODE_DISPLAY_NAME_MAPPINGS = { + "Difforum_GuideFrames": "Difforum · Guide Frames", + "Difforum_Keyframes": "Difforum · Keyframes", + "Difforum_CameraPrompt": "Difforum · Camera → Prompt", + "Difforum_LTXGuides": "Difforum · LTX Guides", + "Difforum_H3Shot": "Difforum · H3 Shot", + "Difforum_H3Guides": "Difforum · H3 Guides", +} diff --git a/nodes/curves.py b/nodes/curves.py new file mode 100644 index 0000000..cf125d2 --- /dev/null +++ b/nodes/curves.py @@ -0,0 +1,212 @@ +"""Curves & prompts: expression schedules, audio analysis, prompt travel.""" + +from __future__ import annotations + +import numpy as np + +from ..core import EASINGS, build_schedule +from ..core.audio import REACTIVE_MODES, REACTIVE_SOURCES, analyze, reactive_curve +from ..core.prompt import parse_prompt_schedule +from ..core.schedule import Schedule +from ._common import AUDIO, CAT_CURVES, PARAMS, PROMPT, SCHEDULE, audio_vars +from .direction import encode_prompt_track + + +class DifforumSchedule: + """A per-frame curve from Deforum keyframe syntax: `0:(0.4), 48:(0.6+0.1*sin(t/8))`. + + Variables: t/f (frame), s (seconds), fps, max_f, and every audio band when + an Audio Analyzer is connected. Easing applies between keyframes.""" + + DESCRIPTION = __doc__ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "params": (PARAMS,), + "schedule": ("STRING", {"multiline": True, "default": "0:(0.45), 60:(0.6), 119:(0.45)"}), + "easing": (list(EASINGS), {"default": "ease_in_out"}), + }, + "optional": {"audio": (AUDIO,)}, + } + + RETURN_TYPES = (SCHEDULE, "STRING") + RETURN_NAMES = ("schedule", "info") + FUNCTION = "run" + CATEGORY = CAT_CURVES + + def run(self, params, schedule, easing, audio=None): + sched = build_schedule(schedule, max_frames=params["max_frames"], fps=params["fps"], + easing=easing, extra_vars=audio_vars(audio)) + v = sched.as_list() + return (sched, f"{len(v)} frames min {min(v):.4g} max {max(v):.4g}") + + +class DifforumSchedulePlot: + """Draw any schedule as an image (and a one-line summary).""" + + DESCRIPTION = __doc__ + + @classmethod + def INPUT_TYPES(cls): + return {"required": { + "schedule": (SCHEDULE,), + "width": ("INT", {"default": 512, "min": 64, "max": 4096}), + "height": ("INT", {"default": 200, "min": 64, "max": 4096}), + }} + + RETURN_TYPES = ("IMAGE", "STRING") + RETURN_NAMES = ("plot", "info") + FUNCTION = "run" + CATEGORY = CAT_CURVES + + def run(self, schedule, width, height): + import torch + + from ..core.plot import render_curve + vals = schedule.as_list() + arr = render_curve(vals, width=int(width), height=int(height)) + info = (f"{len(vals)} frames min {min(vals):.4g} max {max(vals):.4g} " + f"first {vals[0]:.4g} last {vals[-1]:.4g} ({schedule.source})") if vals else "empty" + return (torch.from_numpy(arr).unsqueeze(0), info) + + +class DifforumAudioAnalyzer: + """Audio -> per-frame bands: amp, low, mid, high, onset, beat (0..1). + + Use the bands in any expression (`1 + 0.05*low`), in a Director camera + block's audio reaction, or turn one into a curve with Audio Curve.""" + + DESCRIPTION = __doc__ + + @classmethod + def INPUT_TYPES(cls): + return {"required": { + "params": (PARAMS,), + "audio": ("AUDIO",), + "smoothing": ("FLOAT", {"default": 0.25, "min": 0.0, "max": 0.99, "step": 0.01}), + "beat_sensitivity": ("FLOAT", {"default": 1.5, "min": 0.1, "max": 5.0, "step": 0.1}), + "offset_seconds": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 36000.0, "step": 0.1, + "tooltip": "Start reading the track here (sync to an edit)."}), + }} + + RETURN_TYPES = (AUDIO, "IMAGE", "STRING") + RETURN_NAMES = ("audio_curves", "bands_plot", "info") + FUNCTION = "run" + CATEGORY = CAT_CURVES + + def run(self, params, audio, smoothing, beat_sensitivity, offset_seconds): + import torch + + from ..core.plot import render_curve + sr = int(audio["sample_rate"]) + samples = np.asarray(audio["waveform"].detach().cpu().numpy(), dtype=np.float32) + start = int(float(offset_seconds) * sr) + if start: + samples = samples[..., start:] + curves = analyze(samples, sample_rate=sr, fps=params["fps"], + max_frames=params["max_frames"], smoothing=float(smoothing), + normalize=True, beat_sensitivity=float(beat_sensitivity)) + rows = [render_curve(curves[k], width=512, height=64) for k in ("low", "mid", "high", "beat")] + plot = torch.from_numpy(np.concatenate(rows, axis=0)).unsqueeze(0) + beats = int(sum(1 for v in curves["beat"] if v > 0.5)) + bundle = {"curves": curves, "fps": params["fps"], "frames": params["max_frames"]} + return (bundle, plot, f"audio: {params['max_frames']} frames, ~{beats} beats " + f"(plot rows: low / mid / high / beat)") + + +class DifforumAudioCurve: + """One audio band -> a ready curve: e.g. bass-pumped energy + (`low`, add, base 0.45, amount 0.2) or a beat-pulsed cfg.""" + + DESCRIPTION = __doc__ + + @classmethod + def INPUT_TYPES(cls): + return {"required": { + "params": (PARAMS,), + "audio_curves": (AUDIO,), + "source": (list(REACTIVE_SOURCES), {"default": "low"}), + "mode": (list(REACTIVE_MODES), {"default": "add"}), + "base": ("FLOAT", {"default": 0.45, "min": -100.0, "max": 100.0, "step": 0.01}), + "amount": ("FLOAT", {"default": 0.2, "min": -100.0, "max": 100.0, "step": 0.01}), + "smoothing": ("FLOAT", {"default": 0.2, "min": 0.0, "max": 0.99, "step": 0.01}), + }} + + RETURN_TYPES = (SCHEDULE,) + RETURN_NAMES = ("schedule",) + FUNCTION = "run" + CATEGORY = CAT_CURVES + + def run(self, params, audio_curves, source, mode, base, amount, smoothing): + vals = reactive_curve(audio_vars(audio_curves) or {}, source, base, amount, mode=mode, + smoothing=float(smoothing), max_frames=params["max_frames"]) + return (Schedule(values=vals, fps=params["fps"], + source=f"audio {source} {mode} {base:g}+{amount:g}"),) + + +class DifforumPromptTravel: + """Prompt travel from `frame: prompt` lines (or `2.5s: prompt`), blended + per frame. Encodes each prompt once and blends lazily, so long clips on big + text encoders stay light. `build_batched` adds one batched conditioning + (frame i = prompt i) for batch samplers such as AnimateDiff.""" + + DESCRIPTION = __doc__ + + @classmethod + def INPUT_TYPES(cls): + return {"required": { + "params": (PARAMS,), + "clip": ("CLIP",), + "prompts": ("STRING", {"multiline": True, "default": + "0: a serene misty forest, soft light\n" + "2s: a glowing crystal cave, bioluminescent\n" + "4s: a vast starry galaxy, swirling nebula"}), + "easing": (list(EASINGS), {"default": "ease_in_out"}), + }, "optional": { + "build_batched": ("BOOLEAN", {"default": False, + "tooltip": "Also build the batched output (costs RAM on long clips)."}), + }} + + RETURN_TYPES = (PROMPT, "CONDITIONING", "CONDITIONING", "STRING") + RETURN_NAMES = ("prompts", "first_frame_cond", "batched", "info") + FUNCTION = "run" + CATEGORY = CAT_CURVES + + def run(self, params, clip, prompts, easing, build_batched=False): + fps = float(params["fps"]) + n = int(params["max_frames"]) + lines = [] + for raw in prompts.splitlines(): # allow "2.5s: ..." timestamps + head, sep, rest = raw.partition(":") + h = head.strip().lower() + if sep and h.endswith("s") and h[:-1].replace(".", "", 1).isdigit(): + lines.append(f"{int(round(float(h[:-1]) * fps))}:{rest}") + else: + lines.append(raw) + kfs = parse_prompt_schedule("\n".join(lines)) + track = encode_prompt_track(clip, kfs, n, easing) + info = f"{len(kfs)} prompts over {n} frames\n" + "\n".join( + f" {f / fps:6.2f}s {t[:60]}" for f, t in kfs) + batched = None + if build_batched: + from ..core.prompt import batch_conditioning + batched = batch_conditioning(track) + return (track, track[0], batched, info) + + +NODE_CLASS_MAPPINGS = { + "Difforum_Schedule": DifforumSchedule, + "Difforum_SchedulePlot": DifforumSchedulePlot, + "Difforum_AudioAnalyzer": DifforumAudioAnalyzer, + "Difforum_AudioCurve": DifforumAudioCurve, + "Difforum_PromptTravel": DifforumPromptTravel, +} +NODE_DISPLAY_NAME_MAPPINGS = { + "Difforum_Schedule": "Difforum · Schedule", + "Difforum_SchedulePlot": "Difforum · Schedule Plot", + "Difforum_AudioAnalyzer": "Difforum · Audio Analyzer", + "Difforum_AudioCurve": "Difforum · Audio Curve", + "Difforum_PromptTravel": "Difforum · Prompt Travel", +} diff --git a/nodes/direction.py b/nodes/direction.py new file mode 100644 index 0000000..e03bff7 --- /dev/null +++ b/nodes/direction.py @@ -0,0 +1,380 @@ +"""Direction nodes: the Director timeline, keyframed/expression cameras and the +Storyboard dry run.""" + +from __future__ import annotations + +import json + +from ..core.camera import CAMERA_MODES, build_camera +from ..core.camera_keys import keys_to_axis_values, lens_note, parse_camera_keys +from ..core.camera_presets import flat_presets, needs_depth +from ..core.direction import ( + STYLES, DirectionBundle, build_direction, default_timeline, describe_timed, +) +from ..core.loop import LOOP_MODES, close_axis_values, tile_laps +from ..core.schedule import Schedule, build_schedule +from ._common import ( + AUDIO, CAMERA, CAT_DIRECT, PARAMS, PROMPT, SCHEDULE, audio_vars, +) + +DIRECTION = "DIFFORUM_DIRECTION" + + +def encode_prompt_track(clip, keyframes, frames, easing="ease_in_out"): + from ..core.prompt import PromptTrack + encoded = [clip.encode_from_tokens_scheduled(clip.tokenize(t)) for _, t in keyframes] + return PromptTrack(encoded, keyframes, frames, easing) + + +def _track(axes: dict, lens: list, n: int, mode: str, moves: list): + """Camera track for the chosen mode. Moves through space in a 2d shot are + kept as a 3D track rendered flat, so they become pseudo-3D (dolly -> zoom, + orbit -> pan) instead of being dropped.""" + values = dict(axes) + values["fov"] = lens + if mode == "2d" and any(needs_depth(m) for m in moves): + cam = build_camera(values, max_frames=n, mode="3d", fov=lens[0]) + cam.flat = True + return cam + return build_camera(values, max_frames=n, mode=mode, fov=lens[0]) + + +def _fit(series: list, n: int, pad): + series = list(series) + if len(series) < n: + series.extend([series[-1] if series else pad] * (n - len(series))) + return series[:n] + + +# --------------------------------------------------------------------------- +# Director +# --------------------------------------------------------------------------- + +class DifforumDirector: + """Direct the whole clip on a visual timeline. + + Three tracks, edited with the mouse: **Scenes** (prompt + mood), **Camera** + (pick a move, set speed / amplitude / lens / easing / audio reaction) and + **Energy** (the denoise curve - how much each moment is re-imagined). The + preview panel plays the camera back with the same engine that renders it. + + One `direction` wire carries camera, strength, cfg and prompt travel into + the Feedback Sampler or a video-model bridge. The separate outputs are + there for custom graphs. + """ + + DESCRIPTION = __doc__ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "params": (PARAMS,), + "timeline": ("STRING", {"multiline": True, + "default": json.dumps(default_timeline(120))}), + "camera_mode": (list(CAMERA_MODES), {"default": "2d", + "tooltip": "3d = real parallax when a depth map reaches the sampler " + "(pseudo-3D otherwise)."}), + "style": (list(STYLES), {"default": "cinematic", + "tooltip": "Colour lock / detail defaults the sampler uses in 'from director' mode."}), + "transition": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.05, + "tooltip": "How much of each camera block is spent easing into the next. 0 = hard cuts."}), + "camera_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 4.0, "step": 0.05, + "tooltip": "Master multiplier on every move."}), + "energy_bias": ("FLOAT", {"default": 0.0, "min": -0.3, "max": 0.3, "step": 0.01, + "tooltip": "Shifts the whole denoise curve up or down."}), + "variation": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01, + "tooltip": "0 = exactly as drawn. Above 0, seeded nudges per block."}), + "variation_seed": ("INT", {"default": 0, "min": 0, "max": 0xFFFFFFFF}), + }, + "optional": { + "clip": ("CLIP", {"tooltip": "Connect to encode the scene prompts (prompt travel)."}), + "audio": (AUDIO, {"tooltip": "From Audio Analyzer - enables per-block audio reactions."}), + }, + } + + RETURN_TYPES = (DIRECTION, CAMERA, SCHEDULE, PROMPT, "STRING", "STRING") + RETURN_NAMES = ("direction", "camera", "strength", "prompts", "camera_text", "info") + FUNCTION = "run" + CATEGORY = CAT_DIRECT + + def run(self, params, timeline, camera_mode, style, transition, camera_scale, + energy_bias, variation, variation_seed, clip=None, audio=None): + n = int(params["max_frames"]) + fps = float(params["fps"]) + d = build_direction( + timeline, n, fps, mode=camera_mode, camera_scale=camera_scale, + strength_bias=energy_bias, blend=transition, variation=variation, + variation_seed=variation_seed, audio_curves=audio_vars(audio), + ) + camera = _track(d.axes, d.lens, n, camera_mode, [b["move"] for b in d.camera_blocks]) + strength = Schedule(values=d.strength, fps=fps, source="director energy") + cfg = Schedule(values=d.cfg, fps=fps, source="director guidance") if d.cfg else None + prompts = encode_prompt_track(clip, d.prompts, n) if clip is not None else None + + bundle = DirectionBundle(params=params, camera=camera, strength=strength, cfg=cfg, + prompts=prompts, direction=d, style=style) + info = "\n".join([ + f"[Difforum Director] {n} frames {n / fps:.2f}s camera {camera_mode} style {style}", + *d.summary, + *([""] + [f" ! {w}" for w in d.warnings] if d.warnings else []), + "" if clip is not None else " (connect a CLIP to get prompt travel on the direction wire)", + ]) + timed = describe_timed(d.camera_blocks, n, fps) + return { + "ui": {"text": [info]}, + "result": (bundle, camera, strength, prompts, d.camera_text + "\n\n" + timed, info), + } + + +# --------------------------------------------------------------------------- +# Camera (keys, text) and Camera (expressions) +# --------------------------------------------------------------------------- + +_DEFAULT_KEYS = ( + "0: zoom_in 0.6 0.5 40 ease_in_out\n" + "30: pan_right 1.0 0.8 40 ease_in_out\n" + "60: spiral 1.4 1.1 55 ease_in\n" + "90: zoom_out 0.5 0.45 40 ease_out" +) + + +class DifforumCamera: + """Keyframed camera as text, one key per line: + + frame: move [speed] [intensity] [lens] [ease] + + Moves blend into each other (`transition` 1.0 = continuous crane move, + 0.0 = hard cuts). `loop_mode` makes the path periodic over `cycle_frames` + so the clip loops without a crossfade. The Director timeline writes the + same thing visually - use this node when you prefer typing. + """ + + DESCRIPTION = __doc__ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "params": (PARAMS,), + "keys": ("STRING", {"multiline": True, "default": _DEFAULT_KEYS}), + "mode": (list(CAMERA_MODES), {"default": "2d"}), + "transition": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.05}), + "speed": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 5.0, "step": 0.05}), + "loop_mode": (list(LOOP_MODES), {"default": "off"}), + "cycle_frames": ("INT", {"default": 0, "min": 0, "max": 100000, + "tooltip": "0 = whole clip is one cycle."}), + "harmonics": ("INT", {"default": 3, "min": 1, "max": 16}), + }, + "optional": {"audio": (AUDIO,)}, + } + + RETURN_TYPES = (CAMERA, "INT", "STRING") + RETURN_NAMES = ("camera", "cycle_frames", "info") + FUNCTION = "run" + CATEGORY = CAT_DIRECT + + def run(self, params, keys, mode, transition, speed, loop_mode, cycle_frames, + harmonics, audio=None): + total = int(params["max_frames"]) + fps = float(params["fps"]) + looping = loop_mode != "off" + cyc = int(cycle_frames) if int(cycle_frames) > 0 else total + cyc = max(8, min(cyc, total)) + span = cyc if looping else total + + values, lens, summary = keys_to_axis_values( + keys, max_frames=span, fps=fps, extra_vars=audio_vars(audio), blend=transition) + if abs(speed - 1.0) > 1e-6: + for ax, series in values.items(): + values[ax] = ([1.0 + (v - 1.0) * speed for v in series] if ax == "zoom" + else [v * speed for v in series]) + + notes = [] + if looping: + values, residual = close_axis_values(values, mode=loop_mode, + keep_harmonics=int(harmonics)) + laps = max(1, total // cyc) + values = tile_laps(values, laps) + lens = list(lens) * laps + err = max((abs(v - (1.0 if a == "zoom" else 0.0)) for a, v in residual.items()), + default=0.0) + notes.append(f" loop: {laps} lap(s) of {cyc}f, closure error {err:.1e}") + if laps < 2: + notes.append(f" ! one lap cannot settle - set duration to {cyc * 3} frames " + "and keep the last lap with the Loop node") + + values = {ax: _fit(s, total, 1.0 if ax == "zoom" else 0.0) for ax, s in values.items()} + moves = [k["move"] for k in parse_camera_keys(keys)] + cam = _track(values, _fit(lens, total, 40.0), total, mode, moves) + + depth_moves = sorted({m for m in moves if needs_depth(m)}) + if depth_moves: + notes.append(f" {', '.join(depth_moves)}: real parallax needs mode=3d + a depth map; " + "otherwise they run as pseudo-3D (dolly->zoom, orbit->pan).") + notes.append(f" flat moves: {', '.join(flat_presets()[:10])}...") + lo, hi = min(cam.fov), max(cam.fov) + info = "\n".join([f"camera ({mode}, transition {transition:g})", + f" lens {lo:.0f}-{hi:.0f} deg ({lens_note(lo)})", summary, *notes]) + return (cam, cyc if looping else total, info) + + +_AXES = ("translation_x", "translation_y", "translation_z", + "rotation_3d_x", "rotation_3d_y", "rotation_3d_z", "zoom") +_EXPR_DEFAULTS = { + "translation_x": "0:(0)", "translation_y": "0:(0)", "translation_z": "0:(0)", + "rotation_3d_x": "0:(0)", "rotation_3d_y": "0:(0)", "rotation_3d_z": "0:(0.3)", + "zoom": "0:(1.0 + 0.004*sin(2*pi*t/48))", +} + + +class DifforumCameraExpr: + """Deforum-style camera from math expressions, per axis. + + `0:(0), 60:(2*sin(2*pi*t/30))`, with `t` (frame), `s` (seconds) and any audio + band (`low`, `beat`...) when audio is connected. Each axis also has an + optional schedule socket (e.g. an Audio Curve) that overrides its text. + Values are per-frame increments; zoom is a per-frame scale factor. + """ + + DESCRIPTION = __doc__ + + @classmethod + def INPUT_TYPES(cls): + req = {"params": (PARAMS,), "mode": (list(CAMERA_MODES), {"default": "2d"}), + "fov": ("FLOAT", {"default": 40.0, "min": 1.0, "max": 170.0, "step": 1.0})} + for ax in _AXES: + req[ax] = ("STRING", {"multiline": False, "default": _EXPR_DEFAULTS[ax]}) + opt = {"audio": (AUDIO,)} + for ax in _AXES: + opt[f"{ax}_curve"] = (SCHEDULE,) + return {"required": req, "optional": opt} + + RETURN_TYPES = (CAMERA, "STRING") + RETURN_NAMES = ("camera", "info") + FUNCTION = "run" + CATEGORY = CAT_DIRECT + + def run(self, params, mode, fov, audio=None, **kw): + n = int(params["max_frames"]) + extra = audio_vars(audio) + values, used = {}, [] + for ax in _AXES: + curve = kw.get(f"{ax}_curve") + if curve is not None: + values[ax] = _fit(curve.as_list(), n, 1.0 if ax == "zoom" else 0.0) + used.append(f"{ax} <- curve") + else: + values[ax] = build_schedule(kw.get(ax, _EXPR_DEFAULTS[ax]), max_frames=n, + fps=params["fps"], extra_vars=extra).as_list() + cam = build_camera(values, max_frames=n, mode=mode, fov=fov) + info = f"camera expressions ({mode}, fov {fov:g})" + ( + "\n " + "\n ".join(used) if used else "") + return (cam, info) + + +# --------------------------------------------------------------------------- +# Storyboard +# --------------------------------------------------------------------------- + +class DifforumStoryboard: + """The whole clip warped by the camera, no diffusion - in about a second. + + Same engine as the Feedback Sampler with the sampler switched off, so what + you see is exactly the motion the render will get: contact sheet, the + frames themselves, the camera path from above, and a pacing verdict. + """ + + DESCRIPTION = __doc__ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "init_image": ("IMAGE",), + "every_nth": ("INT", {"default": 8, "min": 1, "max": 240}), + "columns": ("INT", {"default": 6, "min": 1, "max": 16}), + "preview_scale": ("FLOAT", {"default": 0.5, "min": 0.1, "max": 1.0, "step": 0.05}), + }, + "optional": { + "direction": (DIRECTION,), + "params": (PARAMS,), + "camera": (CAMERA,), + "depth": ("IMAGE",), + "symmetry": (["none", "mirror_h", "mirror_v", "mirror_quad", "kaleidoscope"], + {"default": "none"}), + }, + } + + RETURN_TYPES = ("IMAGE", "IMAGE", "IMAGE", "STRING") + RETURN_NAMES = ("sheet", "frames", "camera_path", "info") + FUNCTION = "run" + CATEGORY = CAT_DIRECT + + def run(self, init_image, every_nth, columns, preview_scale, direction=None, + params=None, camera=None, depth=None, symmetry="none"): + import torch + + from ..core.engine import EngineConfig, FeedbackEngine, iter_feedback + from ..core.plot import render_camera_path + from ..core.storyboard import contact_sheet, drift_curve + + if direction is not None: + params = params or direction.params + camera = camera or direction.camera + if params is None or camera is None: + raise ValueError("Storyboard needs a direction, or params + camera.") + + n = min(int(params["max_frames"]), len(camera.deltas)) + fps = float(params["fps"]) + w = max(64, int(params["width"] * preview_scale) // 8 * 8) + h = max(64, int(params["height"] * preview_scale) // 8 * 8) + cfg = EngineConfig(width=w, height=h, symmetry=symmetry, sharpen=0.0, noise=0.0, + hole_noise=0.0, color_mode="none", anchor_mode="none") + engine = FeedbackEngine(camera, cfg, depth=depth) + frames = [img for _f, img in iter_feedback(engine, init_image, n)] + + drift = drift_curve(frames) + step = max(1, int(every_nth)) + picks = list(range(0, len(frames), step)) + if picks[-1] != len(frames) - 1: + picks.append(len(frames) - 1) + sheet = contact_sheet([frames[i] for i in picks], columns=int(columns), cell_width=192, + labels=[f"{i / fps:.1f}s" for i in picks]) + path = torch.from_numpy(render_camera_path(camera.poses, camera.zoom, mode=camera.mode, + width=384, height=384)).unsqueeze(0) + + body = drift[1:] or [0.0] + avg = sum(body) / len(body) + peak = max(body) + peak_at = body.index(peak) + 1 + verdict = [] + if avg < 0.004: + verdict.append(" camera is nearly still - the feedback will stagnate; add motion") + elif avg > 0.05: + verdict.append(" camera is very fast - expect smearing unless energy is high") + if avg > 0 and peak > avg * 4: + verdict.append(f" spike at {peak_at / fps:.2f}s ({peak / avg:.1f}x average) - " + "raise the Director's transition to ease that cut") + if not verdict: + verdict.append(" motion is evenly paced") + info = "\n".join([ + f"storyboard: {len(frames)} frames at {w}x{h} ({engine.report.mode}), " + f"drift avg {avg:.4f} peak {peak:.4f}", + *verdict, *[f" {x}" for x in engine.report.notes], + ]) + return (sheet, torch.cat(frames, dim=0), path, info) + + +NODE_CLASS_MAPPINGS = { + "Difforum_Director": DifforumDirector, + "Difforum_Camera": DifforumCamera, + "Difforum_CameraExpr": DifforumCameraExpr, + "Difforum_Storyboard": DifforumStoryboard, +} +NODE_DISPLAY_NAME_MAPPINGS = { + "Difforum_Director": "Difforum · Director (timeline)", + "Difforum_Camera": "Difforum · Camera (keys)", + "Difforum_CameraExpr": "Difforum · Camera (expressions)", + "Difforum_Storyboard": "Difforum · Storyboard", +} diff --git a/nodes/export.py b/nodes/export.py new file mode 100644 index 0000000..4873bc9 --- /dev/null +++ b/nodes/export.py @@ -0,0 +1,187 @@ +"""Camera export / import: After Effects, Blender and JSON.""" + +from __future__ import annotations + +import json +import os + +from ..core import export as X +from ._common import CAMERA, CAT_EXPORT, PARAMS +from .direction import DIRECTION + + +def _output_dir() -> str: + try: + import folder_paths + base = folder_paths.get_output_directory() + except Exception: + base = "." + d = os.path.join(base, "difforum") + os.makedirs(d, exist_ok=True) + return d + + +def _unique(path: str) -> str: + root, ext = os.path.splitext(path) + i, out = 1, path + while os.path.exists(out): + out = f"{root}_{i:03d}{ext}" + i += 1 + return out + + +class DifforumCameraExport: + """Send the Difforum camera to After Effects, Blender or any tool (JSON). + + Writes to `output/difforum/`: + * `.jsx` - run in After Effects: a 3D camera (3d mode) or a transform + null that reproduces the 2D move exactly (2d mode). + * `.py` - run in Blender's Text Editor: an animated camera, fps, + resolution and frame range matched. + * `.json` - camera-to-world matrices + focal length per frame. + Composite titles, 3D elements or a relight pass on top of the AI render + with the exact same camera. + """ + + DESCRIPTION = __doc__ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "formats": (["all", "after effects", "blender", "json"], {"default": "all"}), + "filename_prefix": ("STRING", {"default": "difforum_camera"}), + "blender_unit_scale": ("FLOAT", {"default": 0.1, "min": 0.0001, "max": 100.0, "step": 0.01, + "tooltip": "Metres per Difforum scene unit."}), + }, + "optional": { + "direction": (DIRECTION,), + "params": (PARAMS,), + "camera": (CAMERA,), + "translation_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 20.0, "step": 0.05, + "tooltip": "Match the sampler's value so the exported move matches the render."}), + }, + } + + RETURN_TYPES = ("STRING", "STRING") + RETURN_NAMES = ("paths", "json") + FUNCTION = "run" + OUTPUT_NODE = True + CATEGORY = CAT_EXPORT + + def run(self, formats, filename_prefix, blender_unit_scale, direction=None, params=None, + camera=None, translation_scale=1.0): + if direction is not None: + params = params or direction.params + camera = camera or direction.camera + if params is None or camera is None: + raise ValueError("Connect a Director `direction`, or params + camera.") + if abs(translation_scale - 1.0) > 1e-9: + camera = _scaled(camera, translation_scale) + w, h, fps = int(params["width"]), int(params["height"]), float(params["fps"]) + prefix = os.path.basename(str(filename_prefix)) or "difforum_camera" + out = _output_dir() + paths = [] + data = X.to_json(camera, w, h, fps) + text = json.dumps(data) + if formats in ("all", "json"): + p = _unique(os.path.join(out, f"{prefix}.json")) + with open(p, "w", encoding="utf-8") as fh: + fh.write(text) + paths.append(p) + if formats in ("all", "after effects"): + p = _unique(os.path.join(out, f"{prefix}.jsx")) + with open(p, "w", encoding="utf-8") as fh: + fh.write(X.to_ae_jsx(camera, w, h, fps)) + paths.append(p) + if formats in ("all", "blender"): + p = _unique(os.path.join(out, f"{prefix}_blender.py")) + with open(p, "w", encoding="utf-8") as fh: + fh.write(X.to_blender_script(camera, w, h, fps, unit_scale=blender_unit_scale)) + paths.append(p) + listing = "\n".join(paths) + return {"ui": {"text": [listing]}, "result": (listing, text)} + + +def _scaled(camera, k): + import numpy as np + + from ..core.camera import CameraTrack + deltas = [] + for d in camera.deltas: + d = np.array(d, dtype=np.float64).copy() + d[:3, 3] *= k + deltas.append(d) + return CameraTrack(deltas=deltas, poses=camera.poses, zoom=camera.zoom, fov=camera.fov, + mode=camera.mode) + + +class DifforumCameraImport: + """Bring a camera in from Blender, After Effects or JSON. + + Accepts the files written by Camera Export and by the helper scripts in + `tools/` (export the active Blender camera, or the selected After Effects + camera). The result drives the Feedback Sampler, Storyboard or Guide + Frames like any Difforum camera, so a move blocked in 3D becomes the AI + shot. + """ + + DESCRIPTION = __doc__ + + @classmethod + def INPUT_TYPES(cls): + files = [] + try: + import folder_paths + d = folder_paths.get_input_directory() + files = sorted(f for f in os.listdir(d) if f.lower().endswith(".json")) + except Exception: + pass + return { + "required": { + "file": (files or ["(put a camera .json in ComfyUI/input)"],), + "params": (PARAMS,), + "retime": (["match frames", "keep source length"], {"default": "match frames"}), + }, + "optional": {"json_text": ("STRING", {"multiline": True, "default": "", + "tooltip": "Paste JSON here instead of choosing a file."})}, + } + + RETURN_TYPES = (CAMERA, "STRING") + RETURN_NAMES = ("camera", "info") + FUNCTION = "run" + CATEGORY = CAT_EXPORT + + def run(self, file, params, retime, json_text=""): + if json_text.strip(): + data = json_text + else: + import folder_paths + + from ..core.video import safe_input_path + path = safe_input_path(folder_paths.get_input_directory(), file) + if not os.path.isfile(path): + raise ValueError("Camera Import: put a camera .json in ComfyUI/input " + "(or paste it into json_text).") + with open(path, encoding="utf-8") as fh: + data = fh.read() + c2w, focals, w, h, fps = X.from_json(data) + n_target = int(params["max_frames"]) + if retime == "match frames" and len(c2w) != n_target and len(c2w) > 1: + idx = [round(i * (len(c2w) - 1) / max(1, n_target - 1)) for i in range(n_target)] + c2w = [c2w[i] for i in idx] + focals = [focals[i] for i in idx] + # focal lengths are relative to the source width; rescale to the render width + k = int(params["width"]) / float(w) + cam = X.world_to_camera_track(c2w, [fl * k for fl in focals], int(params["width"])) + return (cam, f"imported {len(c2w)} frames from a {w}x{h} @ {fps:g} fps camera") + + +NODE_CLASS_MAPPINGS = { + "Difforum_CameraExport": DifforumCameraExport, + "Difforum_CameraImport": DifforumCameraImport, +} +NODE_DISPLAY_NAME_MAPPINGS = { + "Difforum_CameraExport": "Difforum · Camera Export (AE / Blender / JSON)", + "Difforum_CameraImport": "Difforum · Camera Import", +} diff --git a/nodes/post.py b/nodes/post.py new file mode 100644 index 0000000..7e9224f --- /dev/null +++ b/nodes/post.py @@ -0,0 +1,173 @@ +"""Post: loops, symmetry, trails, anti-flicker, detail.""" + +from __future__ import annotations + +from ..core.detail import NOISE_MODES, detail_guard +from ..core.effects import echo_trails, loop_blend, pingpong +from ..core.loop import seam_report +from ..core.symmetry import SYMMETRY_MODES, apply_symmetry +from ._common import CAT_POST + +LOOP_METHODS = ("keep settled lap", "flow crossfade", "ping-pong") + + +class DifforumLoop: + """Make a clip loop. + + * keep settled lap - for renders made with a looping camera (Director / + Camera loop_mode) over several laps: keeps the last lap, no blending. + * flow crossfade - morphs the tail into the head along optical flow. + * ping-pong - forward then backward (motion reverses). + Reports how visible the seam is. + """ + + DESCRIPTION = __doc__ + + @classmethod + def INPUT_TYPES(cls): + return {"required": { + "frames": ("IMAGE",), + "method": (list(LOOP_METHODS), {"default": "flow crossfade"}), + "cycle_frames": ("INT", {"default": 0, "min": 0, "max": 100000, + "tooltip": "keep settled lap: frames per lap (the Camera node outputs it)."}), + "blend_frames": ("INT", {"default": 12, "min": 1, "max": 240, + "tooltip": "flow crossfade: overlap length."}), + }} + + RETURN_TYPES = ("IMAGE", "STRING") + RETURN_NAMES = ("frames", "seam_info") + FUNCTION = "run" + CATEGORY = CAT_POST + + def run(self, frames, method, cycle_frames, blend_frames): + total = int(frames.shape[0]) + if method == "keep settled lap": + cyc = int(cycle_frames) if int(cycle_frames) > 0 else total + cyc = max(2, min(cyc, total)) + out = frames[total - cyc:] + elif method == "ping-pong": + out = pingpong(frames) + else: + out = loop_blend(frames, blend=min(int(blend_frames), max(1, total // 2)), flow=True) + report = seam_report([out[i] for i in range(out.shape[0])], int(out.shape[0])) + return (out, f"{method}: {total} -> {out.shape[0]} frames\n{report}") + + +class DifforumSymmetry: + """Mirror or kaleidoscope a frame or batch (the Feedback Sampler can also + do it inside the loop, where it compounds into a living pattern).""" + + DESCRIPTION = __doc__ + + @classmethod + def INPUT_TYPES(cls): + return {"required": { + "image": ("IMAGE",), + "mode": (list(SYMMETRY_MODES), {"default": "kaleidoscope"}), + "segments": ("INT", {"default": 6, "min": 2, "max": 64}), + "mix": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.05}), + }, "optional": { + "flip": ("BOOLEAN", {"default": False}), + "center_x": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), + "center_y": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), + "angle": ("FLOAT", {"default": 0.0, "min": -360.0, "max": 360.0, "step": 1.0}), + }} + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES = ("image",) + FUNCTION = "run" + CATEGORY = CAT_POST + + def run(self, image, mode, segments, mix, flip=False, center_x=0.5, center_y=0.5, angle=0.0): + return (apply_symmetry(image, mode=mode, segments=int(segments), flip=bool(flip), + mix=float(mix), center_x=float(center_x), + center_y=float(center_y), angle=float(angle)),) + + +class DifforumEchoTrails: + """Long-exposure trails across a frame batch.""" + + DESCRIPTION = __doc__ + + @classmethod + def INPUT_TYPES(cls): + return {"required": { + "frames": ("IMAGE",), + "decay": ("FLOAT", {"default": 0.6, "min": 0.0, "max": 0.99, "step": 0.01}), + "mix": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.05}), + }} + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES = ("frames",) + FUNCTION = "run" + CATEGORY = CAT_POST + + def run(self, frames, decay, mix): + return (echo_trails(frames, decay=float(decay), mix=float(mix)),) + + +class DifforumFlowStabilize: + """Anti-flicker: blends history along optical flow, only where it matches, + so texture stops boiling but motion never ghosts.""" + + DESCRIPTION = __doc__ + + @classmethod + def INPUT_TYPES(cls): + return {"required": { + "frames": ("IMAGE",), + "strength": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 0.95, "step": 0.05}), + "flow_scale": ("FLOAT", {"default": 0.5, "min": 0.25, "max": 1.0, "step": 0.25}), + "error_gate": ("FLOAT", {"default": 0.15, "min": 0.02, "max": 0.5, "step": 0.01}), + }} + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES = ("frames",) + FUNCTION = "run" + CATEGORY = CAT_POST + + def run(self, frames, strength, flow_scale, error_gate): + from ..core.flow import stabilize + return (stabilize(frames, strength=float(strength), flow_scale=float(flow_scale), + error_gate=float(error_gate)),) + + +class DifforumDetailGuard: + """Unsharp + contrast + grain, for frames that went soft.""" + + DESCRIPTION = __doc__ + + @classmethod + def INPUT_TYPES(cls): + return {"required": { + "image": ("IMAGE",), + "sharpen": ("FLOAT", {"default": 0.3, "min": 0.0, "max": 2.0, "step": 0.05}), + "contrast": ("FLOAT", {"default": 1.0, "min": 0.5, "max": 2.0, "step": 0.01}), + "grain": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 0.5, "step": 0.005}), + "grain_mode": (list(NOISE_MODES), {"default": "gaussian"}), + }} + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES = ("image",) + FUNCTION = "run" + CATEGORY = CAT_POST + + def run(self, image, sharpen, contrast, grain, grain_mode): + return (detail_guard(image, sharpen_amount=float(sharpen), noise_amount=float(grain), + contrast=float(contrast), noise_mode=grain_mode),) + + +NODE_CLASS_MAPPINGS = { + "Difforum_Loop": DifforumLoop, + "Difforum_Symmetry": DifforumSymmetry, + "Difforum_EchoTrails": DifforumEchoTrails, + "Difforum_FlowStabilize": DifforumFlowStabilize, + "Difforum_DetailGuard": DifforumDetailGuard, +} +NODE_DISPLAY_NAME_MAPPINGS = { + "Difforum_Loop": "Difforum · Loop", + "Difforum_Symmetry": "Difforum · Symmetry", + "Difforum_EchoTrails": "Difforum · Echo Trails", + "Difforum_FlowStabilize": "Difforum · Flow Stabilize", + "Difforum_DetailGuard": "Difforum · Detail Guard", +} diff --git a/nodes/render.py b/nodes/render.py new file mode 100644 index 0000000..f7c40b3 --- /dev/null +++ b/nodes/render.py @@ -0,0 +1,499 @@ +"""Render nodes: the Feedback Sampler (Deforum look, any SD/SDXL/Flux model) +and the Live Sampler (realtime loop with live preview and VJ outputs).""" + +from __future__ import annotations + +import time + +import torch + +from ..core.color import COLOR_MODES +from ..core.engine import ( + ANCHOR_MODES, BORDER_MODES, DEPTH_TRACKING, EngineConfig, FeedbackEngine, + iter_feedback, resize_bhwc, +) +from ..core.schedule import Schedule +from ..core.symmetry import SYMMETRY_MODES +from ._common import ( + CAMERA, CAT_RENDER, PARAMS, PROMPT, SCHEDULE, check_interrupt, progress_bar, +) +from .direction import DIRECTION + +OPTIONS = "DIFFORUM_OPTIONS" + + +def _samplers(): + try: + import comfy.samplers + return comfy.samplers.KSampler.SAMPLERS, comfy.samplers.KSampler.SCHEDULERS + except Exception: # outside ComfyUI + return ["euler", "lcm"], ["normal", "sgm_uniform"] + + +def _resolve(direction, params, camera, strength, cfg_curve, prompts): + """Explicit sockets win over the direction wire.""" + if direction is not None: + params = params or direction.params + camera = camera or direction.camera + strength = strength or direction.strength + cfg_curve = cfg_curve or direction.cfg + prompts = prompts if prompts is not None else direction.prompts + if params is None or camera is None: + raise ValueError("Connect a Director `direction` wire, or both `params` and `camera`.") + return params, camera, strength, cfg_curve, prompts + + +def make_diffuser(model, vae, positive, negative, steps, cfg, sampler_name, scheduler, + strength_at, cfg_at, prompts, seed, seed_mode, + control_net=None, control_image=None, control_strength=0.6): + """The img2img step the engine calls on key frames.""" + from nodes import common_ksampler + + cn_apply = None + if control_net is not None and control_strength > 0.0: + from nodes import ControlNetApplyAdvanced + cn_apply = ControlNetApplyAdvanced().apply_controlnet + + def diffuse(img: torch.Tensor, f: int) -> torch.Tensor: + check_interrupt() + denoise = max(0.0, min(1.0, float(strength_at(f)))) + pos = prompts[f] if prompts is not None and len(prompts) else positive + neg = negative + if cn_apply is not None: + if control_image is not None: + hint = control_image[f % control_image.shape[0]].unsqueeze(0) + else: + hint = img[..., :3] + pos, neg = cn_apply(pos, neg, control_net, hint, float(control_strength), + 0.0, 1.0, vae=vae) + latent = {"samples": vae.encode(img[..., :3])} + seed_f = int(seed) if seed_mode == "fixed" else int(seed) + f + out = common_ksampler(model, seed_f, int(steps), float(cfg_at(f)), sampler_name, + scheduler, pos, neg, latent, denoise=denoise)[0] + image = vae.decode(out["samples"]) + if image.dim() == 5: # video VAEs return [B,T,H,W,C] + image = image.reshape(-1, *image.shape[-3:]) + return image[:1] + + return diffuse + + +def _options(direction, options) -> dict: + """Engine settings: a Render Options node wins; otherwise the Director's + style sets the look; otherwise the defaults.""" + opts = {k: spec.get("default") for k, (_t, spec) in _COMMON_OPTIONAL.items()} + if options is not None: + opts.update(options) + elif direction is not None: + opts.update(direction.look) + return opts + + +_COMMON_OPTIONAL = { + "color_coherence": ("FLOAT", {"default": 0.8, "min": 0.0, "max": 1.0, "step": 0.05, + "tooltip": "How hard colours are held to the anchor."}), + "color_mode": (list(COLOR_MODES), {"default": "lab"}), + "anchor_mode": (list(ANCHOR_MODES), {"default": "scene", + "tooltip": "scene = re-anchor colour at each prompt scene; first = hold " + "frame 0; rolling = previous key; none = free."}), + "sharpen": ("FLOAT", {"default": 0.2, "min": 0.0, "max": 2.0, "step": 0.05}), + "noise": ("FLOAT", {"default": 0.02, "min": 0.0, "max": 0.5, "step": 0.005}), + "hole_noise": ("FLOAT", {"default": 0.25, "min": 0.0, "max": 1.0, "step": 0.05, + "tooltip": "Extra noise where the camera reveals new area, so it is repainted."}), + "symmetry": (list(SYMMETRY_MODES), {"default": "none"}), + "symmetry_segments": ("INT", {"default": 6, "min": 2, "max": 64}), + "border": (list(BORDER_MODES), {"default": "reflection"}), + "depth_tracking": (list(DEPTH_TRACKING), {"default": "follow", + "tooltip": "follow = depth is re-projected with the image every frame."}), + "translation_scale": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 20.0, "step": 0.05, + "tooltip": "3D motion strength vs. the depth map's range."}), + "near": ("FLOAT", {"default": 1.0, "min": 0.01, "max": 1000.0}), + "far": ("FLOAT", {"default": 100.0, "min": 0.02, "max": 10000.0}), + "invert_depth": ("BOOLEAN", {"default": False, "tooltip": "Enable if near things are dark in your depth map."}), + "seed_mode": (["fixed", "increment"], {"default": "fixed", + "tooltip": "fixed = same noise every frame (calmer texture)."}), + "start_frame": ("INT", {"default": 0, "min": 0, "max": 1000000, + "tooltip": "Feedback Sampler: render a chunk; init_image is the frame at start_frame."}), + "end_frame": ("INT", {"default": 0, "min": 0, "max": 1000000, "tooltip": "0 = to the end."}), +} + + +class DifforumRenderOptions: + """Everything the samplers can fine-tune, kept off the sampler itself. + + Without this node the look (colour lock, sharpen, grain) comes from the + Director's style and the rest uses sensible defaults. Connect it to take + manual control: colour anchoring, detail guard, in-loop symmetry, 3D depth + calibration, noise seeding and chunked rendering. + """ + + DESCRIPTION = __doc__ + + @classmethod + def INPUT_TYPES(cls): + return {"required": dict(_COMMON_OPTIONAL)} + + RETURN_TYPES = (OPTIONS,) + RETURN_NAMES = ("options",) + FUNCTION = "run" + CATEGORY = CAT_RENDER + + def run(self, **kw): + return (dict(kw),) + + +class DifforumFeedbackSampler: + """The Deforum look on any image model: every frame is the last one, + moved by the camera and re-imagined by the sampler. + + Plug a Director `direction` wire and a first frame; that is the whole + setup. Works with SD1.5, SDXL, Flux, SD3.5 and turbo/LCM/DMD2 distills + (plain MODEL / VAE / CONDITIONING). Connect a depth map (e.g. Depth + Anything V2 on the first frame) and set the Director to 3d for real + parallax - the depth then follows the image frame by frame. + + Outputs the frames, the tracked depth per frame (for comp / relight / + video-model guides) and a run report. + """ + + DESCRIPTION = __doc__ + + @classmethod + def INPUT_TYPES(cls): + samplers, schedulers = _samplers() + return { + "required": { + "model": ("MODEL",), + "positive": ("CONDITIONING", {"tooltip": "Used when no prompt travel is connected."}), + "negative": ("CONDITIONING",), + "vae": ("VAE",), + "init_image": ("IMAGE", {"tooltip": "The first frame."}), + "steps": ("INT", {"default": 20, "min": 1, "max": 200}), + "cfg": ("FLOAT", {"default": 6.0, "min": 0.0, "max": 30.0, "step": 0.1}), + "sampler_name": (samplers, {"default": "euler"}), + "scheduler": (schedulers, {"default": "normal"}), + "cadence": ("INT", {"default": 1, "min": 1, "max": 12, + "tooltip": "Diffuse every Nth frame; the rest are crossfaded warps. ~N x faster."}), + }, + "optional": { + "direction": (DIRECTION,), + "options": (OPTIONS, {"tooltip": "Difforum · Render Options, for manual control."}), + "params": (PARAMS,), + "camera": (CAMERA,), + "strength": (SCHEDULE, {"tooltip": "Overrides the Director's energy curve."}), + "cfg_curve": (SCHEDULE,), + "prompts": (PROMPT,), + "depth": ("IMAGE", {"tooltip": "Depth of the first frame (white = near), or a per-frame batch."}), + "control_net": ("CONTROL_NET",), + "control_image": ("IMAGE",), + "control_strength": ("FLOAT", {"default": 0.6, "min": 0.0, "max": 3.0, "step": 0.05}), + "energy": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01, + "tooltip": "Denoise when neither a Director nor a strength curve is connected."}), + }, + } + + RETURN_TYPES = ("IMAGE", "IMAGE", "STRING") + RETURN_NAMES = ("frames", "depth", "report") + FUNCTION = "run" + CATEGORY = CAT_RENDER + + def run(self, model, positive, negative, vae, init_image, steps, cfg, sampler_name, + scheduler, cadence, direction=None, options=None, params=None, camera=None, + strength=None, cfg_curve=None, prompts=None, depth=None, control_net=None, + control_image=None, control_strength=0.6, energy=0.5): + kw = _options(direction, options) + start_frame, end_frame = int(kw["start_frame"]), int(kw["end_frame"]) + params, camera, strength, cfg_curve, prompts = _resolve( + direction, params, camera, strength, cfg_curve, prompts) + n = int(params["max_frames"]) + w, h = int(params["width"]), int(params["height"]) + seed = int(params.get("seed", 0)) + strength = strength or Schedule(values=[float(energy)] * n, fps=params["fps"]) + cfg_at = cfg_curve.at if cfg_curve is not None else (lambda _f: float(cfg)) + + ctrl = resize_bhwc(control_image, w, h) if control_image is not None else None + diffuse = make_diffuser(model, vae, positive, negative, steps, cfg, sampler_name, + scheduler, strength.at, cfg_at, prompts, seed, + kw["seed_mode"], control_net, ctrl, control_strength) + + cfg_e = EngineConfig( + width=w, height=h, border=kw["border"], symmetry=kw["symmetry"], + symmetry_segments=int(kw["symmetry_segments"]), hole_noise=float(kw["hole_noise"]), + cadence=int(cadence), anchor_mode=kw["anchor_mode"], near=float(kw["near"]), + far=float(kw["far"]), invert_depth=bool(kw["invert_depth"]), + translation_scale=float(kw["translation_scale"]), + depth_tracking=kw["depth_tracking"], seed=seed, + color_coherence=float(kw["color_coherence"]), color_mode=kw["color_mode"], + sharpen=float(kw["sharpen"]), noise=float(kw["noise"]), + ) + engine = FeedbackEngine(camera, cfg_e, depth=depth) + pbar = progress_bar(n) + frames = [] + for f, img in iter_feedback(engine, init_image, n, diffuse=diffuse, prompt_track=prompts, + start_frame=start_frame, end_frame=end_frame): + frames.append(img) + pbar.update_absolute(f + 1, n) + + out = torch.cat(frames, dim=0) + depth_out = self._depth_batch(engine, start_frame, len(frames), w, h) + report = engine.report.text() + if direction is not None and direction.direction.warnings: + report += "\n" + "\n".join(f" ! {x}" for x in direction.direction.warnings) + return (out, depth_out, report) + + @staticmethod + def _depth_batch(engine, start, count, w, h): + if not engine.depth_log: + return torch.full((1, h, w, 3), 0.5) + keys = sorted(engine.depth_log) + seq = [] + last = engine.depth_log[keys[0]] + for f in range(start, start + count): + last = engine.depth_log.get(f, last) + seq.append(last.reshape(1, h, w)) + d = torch.cat(seq, dim=0).clamp(0, 1) + return d.unsqueeze(-1).expand(-1, -1, -1, 3).contiguous() + + +# --------------------------------------------------------------------------- +# Live +# --------------------------------------------------------------------------- + +def _to_preview(image, max_size=512): + from PIL import Image + arr = (image[0].clamp(0, 1).cpu().numpy() * 255).astype("uint8") + return ("JPEG", Image.fromarray(arr), max_size) + + +class _SpoutSink: + def __init__(self, name): + self.sender = None + if not name: + return + try: + import SpoutGL + self.sender = SpoutGL.SpoutSender() + self.sender.setSenderName(name) + except Exception as e: + print(f"[Difforum] Spout output disabled ({e}); pip install SpoutGL to enable.") + + def send(self, image): + if self.sender is None: + return + try: + from OpenGL import GL + arr = (image[0].clamp(0, 1).cpu().numpy() * 255).astype("uint8") + self.sender.sendImage(arr.tobytes(), arr.shape[1], arr.shape[0], GL.GL_RGB, False, 0) + self.sender.setFrameSync(self.sender.getName()) + except Exception: + self.sender = None + + def close(self): + try: + if self.sender is not None: + self.sender.releaseSender() + except Exception: + pass + + +class _LiveSource: + """Webcam index ("0") or a video path (looped) via OpenCV.""" + + def __init__(self, spec): + self.cap = None + self.is_file = False + spec = (spec or "").strip() + if not spec: + return + try: + import cv2 + src = int(spec) if spec.lstrip("-").isdigit() else spec + self.is_file = not isinstance(src, int) + self.cap = cv2.VideoCapture(src) + if not self.cap.isOpened(): + raise RuntimeError(f"cannot open {spec!r}") + except Exception as e: + print(f"[Difforum] live source disabled ({e})") + self.cap = None + + def read(self, w, h): + if self.cap is None: + return None + import cv2 + ok, frame = self.cap.read() + if not ok and self.is_file: + self.cap.set(cv2.CAP_PROP_POS_FRAMES, 0) + ok, frame = self.cap.read() + if not ok: + return None + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + return resize_bhwc(torch.from_numpy(frame).float().div(255.0).unsqueeze(0), w, h) + + def close(self): + if self.cap is not None: + self.cap.release() + + +class DifforumLiveSampler: + """Realtime Difforum: queue once and watch it play inside the node. + + Pair with a 1-4 step model (SD-Turbo, SDXL-Turbo, LCM, DMD2) at ~512 px. + `live_source` turns it into a magic mirror (webcam "0" or a video path); + `stream_dir` / `spout_name` feed OBS, Resolume or TouchDesigner. Only the + last `keep_frames` frames are returned, so long sessions do not fill RAM. + """ + + DESCRIPTION = __doc__ + + @classmethod + def INPUT_TYPES(cls): + samplers, schedulers = _samplers() + return { + "required": { + "model": ("MODEL",), + "positive": ("CONDITIONING",), + "negative": ("CONDITIONING",), + "vae": ("VAE",), + "init_image": ("IMAGE",), + "run_frames": ("INT", {"default": 480, "min": 1, "max": 10_000_000}), + "steps": ("INT", {"default": 2, "min": 1, "max": 50}), + "cfg": ("FLOAT", {"default": 1.2, "min": 0.0, "max": 15.0, "step": 0.1}), + "sampler_name": (samplers, {"default": "lcm" if "lcm" in samplers else samplers[0]}), + "scheduler": (schedulers, {"default": "sgm_uniform" if "sgm_uniform" in schedulers else schedulers[0]}), + "cadence": ("INT", {"default": 1, "min": 1, "max": 12}), + "target_fps": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 60.0, "step": 0.5, + "tooltip": "0 = as fast as possible."}), + }, + "optional": { + "direction": (DIRECTION,), + "options": (OPTIONS,), + "params": (PARAMS,), + "camera": (CAMERA,), + "strength": (SCHEDULE,), + "prompts": (PROMPT,), + "depth": ("IMAGE",), + "energy": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), + "loop_camera": ("BOOLEAN", {"default": True}), + "live_source": ("STRING", {"default": "", "tooltip": "'' off, '0' webcam, or a video path."}), + "source_blend": ("FLOAT", {"default": 0.9, "min": 0.0, "max": 1.0, "step": 0.05}), + "stream_dir": ("STRING", {"default": "", "tooltip": "Folder to write live PNG frames to."}), + "spout_name": ("STRING", {"default": ""}), + "keep_frames": ("INT", {"default": 240, "min": 1, "max": 100000}), + "live_preview": ("BOOLEAN", {"default": True}), + }, + } + + RETURN_TYPES = ("IMAGE", "STRING") + RETURN_NAMES = ("frames", "report") + FUNCTION = "run" + CATEGORY = CAT_RENDER + + def run(self, model, positive, negative, vae, init_image, run_frames, steps, cfg, + sampler_name, scheduler, cadence, target_fps, direction=None, options=None, + params=None, camera=None, strength=None, prompts=None, depth=None, energy=0.5, + loop_camera=True, live_source="", source_blend=0.9, stream_dir="", + spout_name="", keep_frames=240, live_preview=True): + from collections import deque + from pathlib import Path + + from ..core.camera import CameraTrack + + kw = _options(direction, options) + params, camera, strength, _cfgc, prompts = _resolve( + direction, params, camera, strength, None, prompts) + w, h = int(params["width"]), int(params["height"]) + n_cam = max(1, len(camera.deltas)) + total = int(run_frames) + seed = int(params.get("seed", 0)) + + # stretch the camera track to the run length (looped or held) + idx = [(i % n_cam) if loop_camera else min(i, n_cam - 1) for i in range(total)] + cam = CameraTrack(deltas=[camera.deltas[i] for i in idx], poses=[camera.poses[i] for i in idx], + zoom=[camera.zoom[i] for i in idx], fov=[camera.fov[i] for i in idx], + mode=camera.mode) + sched = strength or Schedule(values=[float(energy)], fps=params["fps"]) + + def strength_at(f): + return sched.at(idx[f] if f < total else f) + + def prompt_view(): + if prompts is None: + return None + + class _P: + def __len__(self): + return total + + def __getitem__(self, f): + return prompts[idx[f]] + return _P() + + diffuse = make_diffuser(model, vae, positive, negative, steps, cfg, sampler_name, + scheduler, strength_at, lambda _f: float(cfg), prompt_view(), + seed, kw["seed_mode"]) + cfg_e = EngineConfig( + width=w, height=h, border=kw["border"], symmetry=kw["symmetry"], + symmetry_segments=int(kw["symmetry_segments"]), hole_noise=float(kw["hole_noise"]), + cadence=int(cadence), anchor_mode="first" if kw["anchor_mode"] == "scene" else kw["anchor_mode"], + near=float(kw["near"]), far=float(kw["far"]), invert_depth=bool(kw["invert_depth"]), + translation_scale=float(kw["translation_scale"]), depth_tracking=kw["depth_tracking"], + seed=seed, color_coherence=float(kw["color_coherence"]), color_mode=kw["color_mode"], + sharpen=float(kw["sharpen"]), noise=float(kw["noise"]), + ) + engine = FeedbackEngine(cam, cfg_e, depth=depth) + + src = _LiveSource(live_source) + sink = _SpoutSink(spout_name) + out_dir = Path(stream_dir) if stream_dir else None + if out_dir is not None: + out_dir.mkdir(parents=True, exist_ok=True) + + def pre_warp(prev, f): + cam_frame = src.read(w, h) + if cam_frame is None: + return prev + b = float(source_blend) + return cam_frame.to(prev.dtype) * b + prev * (1.0 - b) + + pbar = progress_bar(total) + kept = deque(maxlen=int(keep_frames)) + min_dt = 1.0 / target_fps if target_fps and target_fps > 0 else 0.0 + t_last = time.perf_counter() + t0 = t_last + try: + for f, img in iter_feedback(engine, init_image, total, diffuse=diffuse, + pre_warp=pre_warp if src.cap is not None else None): + check_interrupt() + kept.append(img) + if live_preview: + pbar.update_absolute(f + 1, total, _to_preview(img)) + else: + pbar.update_absolute(f + 1, total) + if out_dir is not None: + from PIL import Image + arr = (img[0].clamp(0, 1).cpu().numpy() * 255).astype("uint8") + Image.fromarray(arr).save(out_dir / f"live_{f:07d}.png") + sink.send(img) + if min_dt: + dt = time.perf_counter() - t_last + if dt < min_dt: + time.sleep(min_dt - dt) + t_last = time.perf_counter() + finally: + src.close() + sink.close() + fps_real = total / max(1e-6, time.perf_counter() - t0) + return (torch.cat(list(kept), dim=0), + engine.report.text() + f"\n {fps_real:.2f} fps measured") + + +NODE_CLASS_MAPPINGS = { + "Difforum_FeedbackSampler": DifforumFeedbackSampler, + "Difforum_LiveSampler": DifforumLiveSampler, + "Difforum_RenderOptions": DifforumRenderOptions, +} +NODE_DISPLAY_NAME_MAPPINGS = { + "Difforum_FeedbackSampler": "Difforum · Feedback Sampler", + "Difforum_LiveSampler": "Difforum · Live Sampler", + "Difforum_RenderOptions": "Difforum · Render Options", +} diff --git a/nodes/routes.py b/nodes/routes.py new file mode 100644 index 0000000..3e5a250 --- /dev/null +++ b/nodes/routes.py @@ -0,0 +1,74 @@ +"""HTTP endpoints for the Director timeline widget. + +GET /difforum/catalog -> moves, moods, easings, reactions (one source of truth) +POST /difforum/preview -> the camera path the renderer will actually use, + as per-frame 2x3 affines for the animated preview +""" + +from __future__ import annotations + +import logging + +log = logging.getLogger("difforum") + + +def compute_preview(body: dict) -> dict: + import torch + + from ..core.direction import build_direction + from ..core.engine import EngineConfig, FeedbackEngine + + frames = max(1, min(int(body.get("frames", 120)), 20000)) + fps = float(body.get("fps", 24.0)) + mode = body.get("mode", "2d") if body.get("mode") in ("2d", "3d") else "2d" + w = max(64, min(int(body.get("width", 768)), 8192)) + h = max(64, min(int(body.get("height", 432)), 8192)) + d = build_direction(body.get("timeline", ""), frames, fps, mode=mode, + camera_scale=float(body.get("camera_scale", 1.0)), + strength_bias=float(body.get("energy_bias", 0.0)), + blend=float(body.get("transition", 1.0)), + variation=float(body.get("variation", 0.0)), + variation_seed=int(body.get("variation_seed", 0))) + from .direction import _track + cam = _track(d.axes, d.lens, frames, mode, [b["move"] for b in d.camera_blocks]) + eng = FeedbackEngine(cam, EngineConfig(width=w, height=h)) + acc = torch.eye(3, dtype=torch.float64) + step = max(1, frames // 480) + affines = [] + for f in range(frames): + if f > 0: + acc = eng._affine_for(f) @ acc + if f % step == 0 or f == frames - 1: + a = acc[:2].flatten().tolist() + affines.append([f] + [round(x, 5) for x in a]) + return { + "frames": frames, "fps": fps, "width": w, "height": h, + "affines": affines, + "strength": [round(v, 4) for v in d.strength[::step]], + "strength_step": step, + "warnings": d.warnings, + "camera_text": d.camera_text, + } + + +def register_routes(): + try: + from aiohttp import web + from server import PromptServer + except Exception: # outside ComfyUI + return + routes = PromptServer.instance.routes + + @routes.get("/difforum/catalog") + async def _catalog(_request): + from ..core.direction import ui_catalog + return web.json_response(ui_catalog()) + + @routes.post("/difforum/preview") + async def _preview(request): + try: + body = await request.json() + return web.json_response(compute_preview(body)) + except Exception as exc: # the widget shows the message + log.warning("[Difforum] preview failed: %s", exc) + return web.json_response({"error": str(exc)}, status=400) diff --git a/nodes/setup.py b/nodes/setup.py new file mode 100644 index 0000000..9d224c5 --- /dev/null +++ b/nodes/setup.py @@ -0,0 +1,130 @@ +"""Difforum · Setup - one place for duration, framing and the target model grid.""" + +from __future__ import annotations + +from ._common import CAT_SETUP, PARAMS + +ASPECTS = { + "16:9 landscape": (16, 9), + "9:16 vertical": (9, 16), + "1:1 square": (1, 1), + "4:5 social portrait": (4, 5), + "4:3 classic": (4, 3), + "3:4 portrait": (3, 4), + "21:9 ultrawide": (21, 9), + "2.39:1 anamorphic": (239, 100), + "2:1 univisium": (2, 1), + "custom": (0, 0), +} + +# target -> (frame rule, spatial multiple, native fps or None) +# frame rule (a, b): valid lengths are a*k + b +TARGETS = { + "feedback (SD/SDXL/Flux)": ((1, 0), 8, None), + "LTX-2 / 2.5": ((8, 1), 32, None), + "MiniMax H3": ((17, 5), 32, 24.0), + "Wan 2.x": ((4, 1), 16, None), +} + + +def snap_frames(n: int, rule: tuple[int, int]) -> int: + """Nearest valid length a*k+b (never below b, never 0).""" + a, b = rule + if a <= 1: + return max(1, int(n)) + k = max(0, round((int(n) - b) / a)) + return max(b if b > 0 else a, a * k + b) + + +def snap(v: float, multiple: int) -> int: + return max(multiple, int(round(v / multiple)) * multiple) + + +def resolve_size(aspect: tuple[float, float], long_edge: int, multiple: int) -> tuple[int, int]: + aw, ah = aspect + if aw <= 0 or ah <= 0: + return snap(long_edge, multiple), snap(long_edge, multiple) + if aw >= ah: + w, h = long_edge, long_edge * ah / aw + else: + w, h = long_edge * aw / ah, long_edge + return snap(w, multiple), snap(h, multiple) + + +class DifforumSetup: + """Duration, framing and the grid of the model you will render with. + + Say how long the clip is (seconds or frames) and how it is framed (aspect + + long edge); the resolution is derived, snapped to what the target model's + VAE accepts, and the frame count is snapped to the model's length grid + (LTX 8k+1, MiniMax H3 17k+5 at 24 fps, Wan 4k+1). Every other Difforum node + reads these params, so the whole graph agrees on one timeline. + """ + + DESCRIPTION = __doc__ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "target": (list(TARGETS), {"default": "feedback (SD/SDXL/Flux)", + "tooltip": "Snaps size and length to what this model family accepts."}), + "duration_mode": (["seconds", "frames"], {"default": "seconds"}), + "duration": ("FLOAT", {"default": 5.0, "min": 0.1, "max": 100000.0, "step": 0.1}), + "fps": ("FLOAT", {"default": 24.0, "min": 1.0, "max": 120.0, "step": 1.0}), + "aspect": (list(ASPECTS), {"default": "16:9 landscape"}), + "long_edge": ("INT", {"default": 768, "min": 128, "max": 8192, "step": 32, + "tooltip": "Pixels on the longer side before snapping."}), + "seed": ("INT", {"default": 0, "min": 0, "max": 0xFFFFFFFFFFFFFFFF}), + }, + "optional": { + "custom_aspect_w": ("FLOAT", {"default": 16.0, "min": 0.1, "max": 100.0, "step": 0.1}), + "custom_aspect_h": ("FLOAT", {"default": 9.0, "min": 0.1, "max": 100.0, "step": 0.1}), + "max_megapixels": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 64.0, "step": 0.05, + "tooltip": "0 = no cap. Otherwise shrink to fit (e.g. 0.35 for a 16 GB Mac)."}), + }, + } + + RETURN_TYPES = (PARAMS, "INT", "INT", "INT", "FLOAT", "STRING") + RETURN_NAMES = ("params", "width", "height", "frames", "fps", "info") + FUNCTION = "build" + CATEGORY = CAT_SETUP + + def build(self, target, duration_mode, duration, fps, aspect, long_edge, seed, + custom_aspect_w=16.0, custom_aspect_h=9.0, max_megapixels=0.0): + rule, multiple, native_fps = TARGETS.get(target, TARGETS["feedback (SD/SDXL/Flux)"]) + notes = [] + fps = max(1.0, float(fps)) + if native_fps and abs(fps - native_fps) > 1e-6: + notes.append(f" {target} renders at {native_fps:g} fps - fps set to {native_fps:g}") + fps = native_fps + + raw = float(duration) * fps if duration_mode == "seconds" else float(duration) + frames = snap_frames(int(round(raw)), rule) + if frames != int(round(raw)): + notes.append(f" length snapped {int(round(raw))} -> {frames} frames " + f"({rule[0]}k+{rule[1]} grid)") + + ratio = ASPECTS.get(aspect, (16, 9)) + if ratio == (0, 0): + ratio = (max(0.1, float(custom_aspect_w)), max(0.1, float(custom_aspect_h))) + w, h = resolve_size(ratio, int(long_edge), multiple) + + if max_megapixels and w * h > max_megapixels * 1e6: + s = (max_megapixels * 1e6 / (w * h)) ** 0.5 + w2, h2 = snap(w * s, multiple), snap(h * s, multiple) + notes.append(f" capped {w}x{h} -> {w2}x{h2} ({max_megapixels:g} MP)") + w, h = w2, h2 + + params = {"width": int(w), "height": int(h), "fps": float(fps), + "max_frames": int(frames), "seed": int(seed), "target": target} + info = "\n".join([ + f"setup {w}x{h} {fps:g} fps {frames} frames = {frames / fps:.2f}s", + f" target {target} (size multiple {multiple})", + *notes, + ]) + return (params, int(w), int(h), int(frames), float(fps), info) + + +NODE_CLASS_MAPPINGS = {"Difforum_Setup": DifforumSetup} +NODE_DISPLAY_NAME_MAPPINGS = {"Difforum_Setup": "Difforum · Setup"} diff --git a/pyproject.toml b/pyproject.toml index 0de5cc1..1bdb378 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,25 +1,34 @@ [project] name = "difforum" -description = "Modern Deforum-style animation toolkit for ComfyUI: math-driven keyframe schedules, a visual shot-timeline director, audio reactivity, depth-aware camera warping, seamless loops, and guide bridges to Wan 2.2 and LTX-2. Runs on CUDA and Apple Silicon (MPS)." -version = "0.6.0" -license = { text = "MIT" } +description = "Camera direction and timeline orchestration for ComfyUI: a visual Director timeline (scenes, camera moves, energy, audio reactions) that drives a Deforum-style feedback sampler on any image model, guides LTX-2 / MiniMax H3, and exports the camera to After Effects and Blender." +version = "1.0.0" +license = { file = "LICENSE" } readme = "README.md" requires-python = ">=3.10" -# Intentionally minimal: numpy ships with ComfyUI. No librosa/numexpr/pandas -# so the node installs cleanly on ComfyCloud and Python 3.12+. -dependencies = [ - "numpy", -] +# numpy and torch ship with ComfyUI. OpenCV (optical flow, webcam) is optional +# and imported lazily by the nodes that need it. +dependencies = ["numpy"] +classifiers = ["Operating System :: OS Independent"] + +[project.optional-dependencies] +dev = ["pytest", "ruff", "opencv-python-headless"] [project.urls] Repository = "https://github.com/chillithebillis/Difforum" +Documentation = "https://github.com/chillithebillis/Difforum/tree/main/docs" +"Bug Tracker" = "https://github.com/chillithebillis/Difforum/issues" [tool.comfy] -# Set PublisherId to your Comfy Registry publisher handle before `comfy node publish`. PublisherId = "chillithebillis" DisplayName = "Difforum" -Icon = "" +Icon = "https://raw.githubusercontent.com/chillithebillis/Difforum/main/docs/media/icon.png" +includes = [] + +[tool.ruff] +line-length = 110 +target-version = "py310" +extend-exclude = ["legacy", "tests/scripts"] -[tool.setuptools] -# Pure-source custom node; no build step required. -packages = ["core", "nodes"] +[tool.ruff.lint] +select = ["E", "F", "W", "I"] +ignore = ["E501", "E731", "E741", "I001"] diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..4d95e82 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +testpaths = tests +addopts = --import-mode=importlib -q diff --git a/requirements.txt b/requirements.txt index 55da10b..8b3f1f0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ -# Difforum runtime deps (kept minimal for ComfyCloud / Python 3.12 compatibility). -# numpy is already provided by ComfyUI; listed for standalone installs. +# Runtime: numpy (ships with ComfyUI). Everything else is optional and lazy: +# opencv-python Flow Stabilize, Loop flow crossfade, Live webcam input +# SpoutGL Live Sampler Spout output (Windows) numpy diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..7d0c975 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,69 @@ +"""Make the pack importable as `difforum` whatever its folder is called, and +provide stub ComfyUI modules so node code runs without ComfyUI or a GPU.""" + +import sys +import types +from pathlib import Path + +import pytest +import torch + +PACK = Path(__file__).resolve().parent.parent +if str(Path(__file__).resolve().parent) not in sys.path: + sys.path.insert(0, str(Path(__file__).resolve().parent)) + +if "difforum" not in sys.modules: # load the real package, whatever the folder is called + import importlib.util + + _spec = importlib.util.spec_from_file_location( + "difforum", PACK / "__init__.py", submodule_search_locations=[str(PACK)]) + _pkg = importlib.util.module_from_spec(_spec) + sys.modules["difforum"] = _pkg + _spec.loader.exec_module(_pkg) + + +class StubVAE: + """encode/decode are identity-ish: latent = image, as channels-first.""" + + def encode(self, img): + return img[..., :3].permute(0, 3, 1, 2).clone() + + def decode(self, lat): + return lat.permute(0, 2, 3, 1).clamp(0, 1) + + +def _stub_ksampler(model, seed, steps, cfg, sampler, scheduler, pos, neg, latent, denoise=1.0): + x = latent["samples"] + g = torch.Generator().manual_seed(int(seed) & 0x7FFFFFFF) + noise = torch.rand(x.shape, generator=g) + model.calls.append({"denoise": denoise, "cfg": cfg, "pos": pos}) + return ({"samples": x * (1 - 0.2 * denoise) + noise * 0.2 * denoise},) + + +class StubModel: + def __init__(self): + self.calls = [] + + +@pytest.fixture(autouse=True, scope="session") +def comfy_stubs(): + nodes_mod = types.ModuleType("nodes") + nodes_mod.common_ksampler = _stub_ksampler + nodes_mod.NODE_CLASS_MAPPINGS = {} + sys.modules.setdefault("nodes", nodes_mod) + yield + + +@pytest.fixture +def stub_model(): + return StubModel() + + +@pytest.fixture +def stub_vae(): + return StubVAE() + + +def gradient(h=64, w=96): + ys, xs = torch.meshgrid(torch.linspace(0, 1, h), torch.linspace(0, 1, w), indexing="ij") + return torch.stack([xs, ys, (xs * ys)], dim=-1).unsqueeze(0) diff --git a/tests/test_audio.py b/tests/scripts/check_audio.py similarity index 98% rename from tests/test_audio.py rename to tests/scripts/check_audio.py index b41509d..e166e72 100644 --- a/tests/test_audio.py +++ b/tests/scripts/check_audio.py @@ -5,7 +5,7 @@ import numpy as np -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from core.audio import analyze, reactive_curve, to_mono # noqa: E402 diff --git a/tests/test_color.py b/tests/scripts/check_color.py similarity index 97% rename from tests/test_color.py rename to tests/scripts/check_color.py index 029c9eb..1f0484b 100644 --- a/tests/test_color.py +++ b/tests/scripts/check_color.py @@ -5,7 +5,7 @@ import torch -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from core.color import _lab_to_rgb, _rgb_to_lab, match_color # noqa: E402 diff --git a/tests/test_core.py b/tests/scripts/check_core.py similarity index 98% rename from tests/test_core.py rename to tests/scripts/check_core.py index 8b9e414..209226e 100644 --- a/tests/test_core.py +++ b/tests/scripts/check_core.py @@ -10,7 +10,7 @@ import sys from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from core import build_schedule, compile_expr # noqa: E402 from core.expr import ExprError # noqa: E402 diff --git a/tests/test_detail.py b/tests/scripts/check_detail.py similarity index 97% rename from tests/test_detail.py rename to tests/scripts/check_detail.py index f732cb9..e8fb16e 100644 --- a/tests/test_detail.py +++ b/tests/scripts/check_detail.py @@ -5,7 +5,7 @@ import torch -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from core.detail import ( # noqa: E402 NOISE_MODES, diff --git a/tests/test_effects.py b/tests/scripts/check_effects.py similarity index 98% rename from tests/test_effects.py rename to tests/scripts/check_effects.py index d5c0c02..16859dd 100644 --- a/tests/test_effects.py +++ b/tests/scripts/check_effects.py @@ -5,7 +5,7 @@ import torch -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from core.effects import echo_trails # noqa: E402 from core.symmetry import SYMMETRY_MODES, apply_symmetry # noqa: E402 diff --git a/tests/test_flow.py b/tests/scripts/check_flow.py similarity index 96% rename from tests/test_flow.py rename to tests/scripts/check_flow.py index 1dde49c..302e1e9 100644 --- a/tests/test_flow.py +++ b/tests/scripts/check_flow.py @@ -5,7 +5,7 @@ import torch -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) try: import cv2 # noqa: F401 diff --git a/tests/test_glitch.py b/tests/scripts/check_glitch.py similarity index 97% rename from tests/test_glitch.py rename to tests/scripts/check_glitch.py index 20d2d20..28cb87e 100644 --- a/tests/test_glitch.py +++ b/tests/scripts/check_glitch.py @@ -5,7 +5,7 @@ import torch -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from core.glitch import KERNELS, apply_glitch, bitcrush, convolve, dsp_delay, vhs # noqa: E402 diff --git a/tests/test_hybrid.py b/tests/scripts/check_hybrid.py similarity index 98% rename from tests/test_hybrid.py rename to tests/scripts/check_hybrid.py index 6ab92a4..3929cc2 100644 --- a/tests/test_hybrid.py +++ b/tests/scripts/check_hybrid.py @@ -5,7 +5,7 @@ import numpy as np -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from core.camera import build_camera, euler_to_matrix # noqa: E402 from core.camera_presets import CAMERA_PRESETS, preset_schedules # noqa: E402 diff --git a/tests/test_integration.py b/tests/scripts/check_integration.py similarity index 92% rename from tests/test_integration.py rename to tests/scripts/check_integration.py index 0c8e524..3fcb409 100644 --- a/tests/test_integration.py +++ b/tests/scripts/check_integration.py @@ -19,10 +19,13 @@ import torch -# import Difforum as a package (difforum.nodes.*), like the real ComfyUI runtime, +# import Difforum as a package (difforum.legacy.*), like the real ComfyUI runtime, # so the stub top-level `nodes` module below does not collide with difforum's own # internal `nodes` subpackage. -sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) +import types as _t # noqa: E402 +_pk = _t.ModuleType("difforum") +_pk.__path__ = [str(Path(__file__).resolve().parents[2])] +sys.modules.setdefault("difforum", _pk) # --- inject stub comfy + nodes BEFORE the sampler's lazy imports run -------- _comfy = types.ModuleType("comfy") @@ -75,12 +78,12 @@ def decode(self, samples): # --- import the real Difforum nodes ------------------------------------------ -from difforum.nodes.audio_nodes import DifforumAudioAnalyzer # noqa: E402 -from difforum.nodes.guide_nodes import DifforumGuideBuilder # noqa: E402 -from difforum.nodes.hybrid_nodes import DifforumCamera # noqa: E402 -from difforum.nodes.sampler_nodes import DifforumFeedbackSampler # noqa: E402 -from difforum.nodes.live_nodes import DifforumLiveStep # noqa: E402 -from difforum.nodes.schedule_nodes import DifforumAnimSetup, DifforumSchedule # noqa: E402 +from difforum.legacy.audio_nodes import DifforumAudioAnalyzer # noqa: E402 +from difforum.legacy.guide_nodes import DifforumGuideBuilder # noqa: E402 +from difforum.legacy.hybrid_nodes import DifforumCamera # noqa: E402 +from difforum.legacy.sampler_nodes import DifforumFeedbackSampler # noqa: E402 +from difforum.legacy.live_nodes import DifforumLiveStep # noqa: E402 +from difforum.legacy.schedule_nodes import DifforumAnimSetup, DifforumSchedule # noqa: E402 _failures = [] diff --git a/tests/test_look.py b/tests/scripts/check_look.py similarity index 97% rename from tests/test_look.py rename to tests/scripts/check_look.py index 35b6fe7..f239d39 100644 --- a/tests/test_look.py +++ b/tests/scripts/check_look.py @@ -5,7 +5,7 @@ import torch -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from core.look import ( # noqa: E402 LOOK_PRESETS, diff --git a/tests/test_models.py b/tests/scripts/check_models.py similarity index 96% rename from tests/test_models.py rename to tests/scripts/check_models.py index e2a5fb6..9dcb1be 100644 --- a/tests/test_models.py +++ b/tests/scripts/check_models.py @@ -3,7 +3,7 @@ import sys from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from core import models as M # noqa: E402 diff --git a/tests/test_plot.py b/tests/scripts/check_plot.py similarity index 95% rename from tests/test_plot.py rename to tests/scripts/check_plot.py index 4fc0967..eb551be 100644 --- a/tests/test_plot.py +++ b/tests/scripts/check_plot.py @@ -6,7 +6,7 @@ import numpy as np -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from core.plot import render_curve # noqa: E402 diff --git a/tests/test_prompt.py b/tests/scripts/check_prompt.py similarity index 93% rename from tests/test_prompt.py rename to tests/scripts/check_prompt.py index 40088ce..3f75422 100644 --- a/tests/test_prompt.py +++ b/tests/scripts/check_prompt.py @@ -5,7 +5,7 @@ import torch -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from core.prompt import ( # noqa: E402 blend_conditioning, parse_prompt_schedule, plan_blend, scenes_to_keyframes, @@ -91,8 +91,11 @@ def check(name, cond, detail=""): for i in range(8) ] # inline the node logic (import the class) -sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) -from difforum.nodes.prompt_nodes import DifforumPromptBatch # noqa: E402 +import types as _t # noqa: E402 +_pk = _t.ModuleType("difforum") +_pk.__path__ = [str(Path(__file__).resolve().parents[2])] +sys.modules.setdefault("difforum", _pk) +from difforum.legacy.prompt_nodes import DifforumPromptBatch # noqa: E402 out = DifforumPromptBatch().run(per_frame)[0] check("batch single entry", len(out) == 1 and len(out[0]) == 2) diff --git a/tests/test_shots.py b/tests/scripts/check_shots.py similarity index 97% rename from tests/test_shots.py rename to tests/scripts/check_shots.py index 4a6c291..3d8b645 100644 --- a/tests/test_shots.py +++ b/tests/scripts/check_shots.py @@ -5,7 +5,7 @@ import numpy as np -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from core.camera import build_camera # noqa: E402 from core.plot import render_camera_path # noqa: E402 diff --git a/tests/test_video.py b/tests/scripts/check_video.py similarity index 97% rename from tests/test_video.py rename to tests/scripts/check_video.py index 6925c85..026ebef 100644 --- a/tests/test_video.py +++ b/tests/scripts/check_video.py @@ -7,7 +7,7 @@ import torch -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) try: import cv2 # noqa: F401 diff --git a/tests/test_warp.py b/tests/scripts/check_warp.py similarity index 98% rename from tests/test_warp.py rename to tests/scripts/check_warp.py index 9a66fa6..1928a45 100644 --- a/tests/test_warp.py +++ b/tests/scripts/check_warp.py @@ -5,7 +5,7 @@ import torch -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from core.warp import warp_2d, warp_3d, _focal # noqa: E402 diff --git a/tests/test_direction.py b/tests/test_direction.py new file mode 100644 index 0000000..c4817e2 --- /dev/null +++ b/tests/test_direction.py @@ -0,0 +1,82 @@ +"""Timeline model: parsing, building, words.""" + +import json + +from difforum.core.camera_presets import CAMERA_PRESETS, MOVE_INFO, preset_schedules +from difforum.core.direction import ( + build_direction, default_timeline, describe_blocks, parse_timeline, ui_catalog, +) + + +def test_every_preset_has_ui_metadata_and_builds(): + assert set(CAMERA_PRESETS) == set(MOVE_INFO) + for p in CAMERA_PRESETS: + assert len(preset_schedules(p, 1.0, 1.0)) == 7 + + +def test_default_timeline_is_depth_free_and_fits(): + d = build_direction(default_timeline(120), 120, 24.0, mode="2d") + assert d.warnings == [] + assert len(d.strength) == 120 and len(d.lens) == 120 + assert all(len(v) == 120 for v in d.axes.values()) + assert d.prompts[0][0] == 0 and len(d.prompts) == 4 + + +def test_legacy_scene_list_is_converted(): + old = [{"frame_start": 0, "mood": "calm", "camera": "dolly_in", "prompt": "a"}, + {"frame_start": 40, "mood": "climax", "camera": "spiral", "prompt": "b"}] + tl = parse_timeline(json.dumps(old)) + assert [c["move"] for c in tl["camera"]] == ["dolly_in", "spiral"] + assert [s["start"] for s in tl["scenes"]] == [0, 40] + + +def test_energy_points_override_moods_and_blocks_past_end_warn(): + tl = default_timeline(120) + tl["energy"] = [[0, 0.3], [60, 0.7]] + tl["camera"].append({"start": 500, "move": "shake"}) + d = build_direction(tl, 120, 24.0) + assert abs(d.strength[0] - 0.3) < 1e-6 and abs(d.strength[-1] - 0.7) < 1e-6 + assert any("after the clip ends" in w for w in d.warnings) + + +def test_depth_moves_warn_in_2d(): + tl = default_timeline(96) + tl["camera"][0]["move"] = "orbit_left" + d = build_direction(tl, 96, 24.0, mode="2d") + assert any("Orbit left" in w for w in d.warnings) + + +def test_audio_reaction_changes_the_camera(): + tl = default_timeline(48) + base = build_direction(tl, 48, 24.0) + tl["camera"][0]["react"] = "beat_pulse" + curves = {k: [0.0] * 48 for k in ("amp", "low", "mid", "high", "onset", "beat")} + curves["beat"][5] = 1.0 + d = build_direction(tl, 48, 24.0, audio_curves=curves) + assert d.axes["zoom"][5] > base.axes["zoom"][5] + assert d.axes["zoom"][6] == base.axes["zoom"][6] + + +def test_variation_zero_is_exact_and_seeded(): + tl = default_timeline(60) + a = build_direction(tl, 60, 24.0) + b = build_direction(tl, 60, 24.0, variation=0.0, variation_seed=7) + assert a.axes == b.axes + c1 = build_direction(tl, 60, 24.0, variation=0.5, variation_seed=3) + c2 = build_direction(tl, 60, 24.0, variation=0.5, variation_seed=3) + assert c1.axes == c2.axes and c1.axes != a.axes + + +def test_words(): + text = describe_blocks([{"start": 0, "move": "dolly_in", "speed": 0.4, "intensity": 0.8}, + {"start": 30, "move": "orbit_right", "speed": 1, "intensity": 1}, + {"start": 60, "move": "crane_up", "speed": 2, "intensity": 1.2}], 90, 24) + assert text.startswith("The camera pushes in very slowly") + assert "and finally cranes up" in text + + +def test_catalog_for_the_ui(): + cat = ui_catalog() + ids = [m["id"] for m in cat["moves"]] + assert ids == list(CAMERA_PRESETS) + assert {m["id"] for m in cat["moods"]} >= {"calm", "climax"} diff --git a/tests/test_engine.py b/tests/test_engine.py new file mode 100644 index 0000000..be07812 --- /dev/null +++ b/tests/test_engine.py @@ -0,0 +1,100 @@ +"""The shared feedback engine: geometry, depth tracking, cadence, anchoring.""" + +import torch +from conftest import gradient + +from difforum.core.camera import build_camera +from difforum.core.engine import EngineConfig, FeedbackEngine, iter_feedback +from difforum.core.prompt import PromptTrack +from difforum.core.warp import affine_2d, pseudo_3d_params, warp_2d, warp_3d, warp_affine + + +def cam(n=24, mode="2d", **axes): + vals = {k: [v] * n for k, v in axes.items()} + return build_camera(vals, n, mode=mode) + + +def test_warp_3d_is_deterministic_and_tracks_depth(): + img = gradient(32, 48)[0] + depth = torch.linspace(0, 1, 48).expand(32, 48).clone() + t = torch.eye(4) + t[0, 3] = 2.0 + t[2, 3] = -3.0 + a, ma, da = warp_3d(img, depth, t, return_depth=True) + b, mb, db = warp_3d(img, depth, t, return_depth=True) + assert torch.equal(a, b) and torch.equal(ma, mb) and torch.equal(da, db) + assert da.shape == depth.shape + assert 0.0 <= float(da.min()) and float(da.max()) <= 1.0 + assert not torch.allclose(da, depth) # depth moved with the image + + +def test_affine_matches_warp_2d(): + img = gradient(24, 32) + ref, _ = warp_2d(img, 3.0, -2.0, 7.0, 1.08, padding_mode="border") + out, _ = warp_affine(img, affine_2d(3.0, -2.0, 7.0, 1.08, 32, 24), padding_mode="border") + assert torch.allclose(ref, out, atol=1e-4) + + +def test_pseudo3d_dolly_zooms_and_orbit_pans(): + t = torch.eye(4) + t[2, 3] = -1.5 + _tx, _ty, _a, zoom = pseudo_3d_params(t, 40.0, 256) + assert zoom > 1.0 + from difforum.core.camera import euler_to_matrix + r = torch.eye(4) + r[:3, :3] = torch.as_tensor(euler_to_matrix(0.0, 1.0, 0.0)) + tx, _ty, _a, _z = pseudo_3d_params(r, 40.0, 256) + assert abs(tx) > 1.0 + + +def test_3d_without_depth_is_not_frozen(): + c = cam(12, mode="3d", translation_z=-1.5) + eng = FeedbackEngine(c, EngineConfig(width=48, height=32, sharpen=0, noise=0, hole_noise=0)) + frames = [f for _i, f in iter_feedback(eng, gradient(32, 48), 12)] + assert eng.report.mode == "pseudo3d" + assert float((frames[-1] - frames[0]).abs().mean()) > 0.01 + + +def test_3d_with_depth_logs_depth_per_frame(): + c = cam(8, mode="3d", translation_z=-1.0, rotation_3d_y=0.5) + depth = torch.linspace(0, 1, 48).expand(32, 48).clone() + eng = FeedbackEngine(c, EngineConfig(width=48, height=32), depth=depth) + list(iter_feedback(eng, gradient(32, 48), 8)) + assert eng.report.mode == "3d" + assert sorted(eng.depth_log) == list(range(8)) + + +def test_cadence_crossfade_orders_frames_and_counts(): + c = cam(13, translation_x=1.0) + eng = FeedbackEngine(c, EngineConfig(width=48, height=32, cadence=4, noise=0, hole_noise=0)) + calls = [] + + def diffuse(img, f): + calls.append(f) + return (img * 0.9 + 0.05).clamp(0, 1) + + out = list(iter_feedback(eng, gradient(32, 48), 13, diffuse=diffuse)) + assert [f for f, _ in out] == list(range(13)) + assert calls == [4, 8, 12] + assert eng.report.keys == 3 and eng.report.tweens == 9 + # no pop at the key: the tween before a key is close to the key itself + d_tween = float((out[7][1] - out[8][1]).abs().mean()) + d_far = float((out[5][1] - out[8][1]).abs().mean()) + assert d_tween <= d_far + 1e-6 + + +def _cond(v): + return [[torch.full((1, 4, 8), float(v)), {}]] + + +def test_scene_anchor_releases_during_travel(): + track = PromptTrack([_cond(0), _cond(1)], [(0, "a"), (10, "b")], 20, "linear") + assert track.scene_index(3) == 0 and track.scene_index(12) == 1 + assert 0.0 < track.transition_weight(5) < 1.0 + assert track.transition_weight(15) == 0.0 + mid = track[5][0][0] + assert torch.allclose(mid, torch.full_like(mid, 0.5), atol=1e-6) + from difforum.core.engine import _anchor_strength + cfg = EngineConfig(width=8, height=8, color_coherence=0.8, anchor_mode="scene") + assert _anchor_strength(cfg, track, 0) == 0.8 + assert _anchor_strength(cfg, track, 9) < 0.2 diff --git a/tests/test_export.py b/tests/test_export.py new file mode 100644 index 0000000..7c551b7 --- /dev/null +++ b/tests/test_export.py @@ -0,0 +1,85 @@ +"""Camera interchange round trips (AE / Blender / JSON) and the preview endpoint.""" + +import json + +import numpy as np +import pytest + +from difforum.core import export as X +from difforum.core.camera import build_camera +from difforum.nodes.routes import compute_preview + + +def cam3d(n=30): + vals = { + "translation_x": [0.8] * n, "translation_y": [-0.3] * n, "translation_z": [-1.2] * n, + "rotation_3d_x": [0.2] * n, "rotation_3d_y": [0.6] * n, "rotation_3d_z": [0.3] * n, + "zoom": [1.0] * n, + } + return build_camera(vals, n, mode="3d", fov=40.0) + + +def test_json_round_trip_recovers_the_move(): + c = cam3d() + data = X.to_json(c, 768, 432, 24.0) + c2w, focals, w, h, fps = X.from_json(json.dumps(data)) + back = X.world_to_camera_track(c2w, focals, w) + for a, b in zip(c.deltas[1:], back.deltas[1:]): + assert np.allclose(np.asarray(a), b, atol=1e-5) + + +def test_blender_round_trip(): + c = cam3d() + c2w, focals = X.camera_track_to_world(c, 768, 432) + frames = [] + for m, fl in zip(c2w, focals): + mb = X._CV_TO_BL_WORLD @ m @ X._CV_TO_BL_CAM + mb[:3, 3] *= 0.1 + frames.append({"matrix": mb.tolist(), "focal_px": fl}) + data = {"convention": "blender", "unit_scale": 0.1, "width": 768, "height": 432, "fps": 24, + "frames": frames} + c2w_b, *_ = X.from_json(data) + for a, b in zip(c2w, c2w_b): + assert np.allclose(a, b, atol=1e-6) + + +def test_ae_round_trip(): + c = cam3d() + jsx = X.to_ae_jsx(c, 768, 432, 24.0) + pos = json.loads(jsx.split("var POS = ")[1].split(";")[0]) + rot = json.loads(jsx.split("var ROT = ")[1].split(";")[0]) + zoom = json.loads(jsx.split("var ZOOM = ")[1].split(";")[0]) + data = {"convention": "ae", "width": 768, "height": 432, "fps": 24, "zoom0": zoom[0], + "reference_z": X.reference_z(), + "frames": [{"position": p, "rotation": r, "zoom": z} for p, r, z in zip(pos, rot, zoom)]} + c2w_ae, *_ = X.from_json(data) + c2w, _ = X.camera_track_to_world(c, 768, 432) + for a, b in zip(c2w, c2w_ae): + assert np.allclose(a, b, atol=1e-3) + + +@pytest.mark.parametrize("mode", ["2d", "3d"]) +def test_scripts_are_valid(mode): + n = 12 + vals = {"translation_x": [1.5] * n, "rotation_3d_z": [0.4] * n, "zoom": [1.01] * n} + c = build_camera(vals, n, mode=mode) + compile(X.to_blender_script(c, 640, 360, 24.0), "blender.py", "exec") + jsx = X.to_ae_jsx(c, 640, 360, 24.0) + assert "setValueAtTime" in jsx and jsx.count("{") == jsx.count("}") + + +def test_2d_export_is_a_nodal_camera(): + n = 10 + c = build_camera({"translation_x": [4.0] * n}, n, mode="2d") + c2w, focals = X.camera_track_to_world(c, 640, 360) + assert np.allclose(c2w[-1][:3, 3], 0.0) # no travel, only rotation + assert not np.allclose(c2w[-1][:3, :3], np.eye(3)) + + +def test_preview_endpoint_payload(): + from difforum.core.direction import default_timeline + out = compute_preview({"timeline": default_timeline(96), "frames": 96, "fps": 24, + "width": 640, "height": 360}) + assert out["affines"][0][0] == 0 and len(out["affines"][0]) == 7 + assert out["affines"][-1][0] == 95 + assert len(out["strength"]) == 96 diff --git a/tests/test_nodes.py b/tests/test_nodes.py new file mode 100644 index 0000000..14c7e78 --- /dev/null +++ b/tests/test_nodes.py @@ -0,0 +1,207 @@ +"""v1 nodes end to end, with stub MODEL / VAE / CLIP.""" + +import json + +import pytest +import torch +from conftest import StubModel, StubVAE, gradient + +import difforum +from difforum.core.direction import default_timeline +from difforum.nodes.bridges import ( + DifforumCameraPrompt, DifforumGuideFrames, DifforumH3Shot, DifforumKeyframes, +) +from difforum.nodes.curves import DifforumPromptTravel, DifforumSchedule +from difforum.nodes.direction import DifforumCamera, DifforumDirector, DifforumStoryboard +from difforum.nodes.post import DifforumLoop +from difforum.nodes.render import DifforumFeedbackSampler, DifforumRenderOptions +from difforum.nodes.setup import DifforumSetup + + +def setup(target="feedback (SD/SDXL/Flux)", seconds=2.0, fps=24.0, edge=128): + return DifforumSetup().build(target, "seconds", seconds, fps, "16:9 landscape", edge, 0) + + +class StubClip: + def tokenize(self, text): + return text + + def encode_from_tokens_scheduled(self, tokens): + v = float(len(tokens) % 7) + return [[torch.full((1, 4, 8), v), {"pooled_output": torch.full((1, 8), v)}]] + + +def director(params, tl=None, mode="2d", clip=None): + tl = tl or default_timeline(params["max_frames"]) + return DifforumDirector().run(params, json.dumps(tl), mode, "cinematic", 1.0, 1.0, 0.0, + 0.0, 0, clip=clip)["result"] + + +def test_registry(): + v1 = [k for k in difforum.NODE_CLASS_MAPPINGS if k.startswith("Difforum_")] + assert len(v1) == 26 + legacy = difforum.NODE_CLASS_MAPPINGS["DifforumFeedbackSampler"] + assert legacy.DEPRECATED and legacy.CATEGORY == "Difforum/legacy" + for k in v1: + cls = difforum.NODE_CLASS_MAPPINGS[k] + cls.INPUT_TYPES() + assert cls.CATEGORY.startswith("Difforum/") + assert len(cls.RETURN_TYPES) == len(getattr(cls, "RETURN_NAMES", cls.RETURN_TYPES)) + + +@pytest.mark.parametrize("target,rule,mult", [ + ("LTX-2 / 2.5", (8, 1), 32), ("MiniMax H3", (17, 5), 32), ("Wan 2.x", (4, 1), 16), +]) +def test_setup_snaps_to_model_grid(target, rule, mult): + params, w, h, frames, fps, _ = setup(target, seconds=5.0, fps=30.0, edge=1000) + a, b = rule + assert (frames - b) % a == 0 + assert w % mult == 0 and h % mult == 0 + if target == "MiniMax H3": + assert fps == 24.0 + + +def test_director_storyboard_and_sampler(): + params = setup()[0] + bundle, camera, strength, prompts, text, info = director(params, clip=StubClip()) + assert len(camera.deltas) == params["max_frames"] and len(strength) == params["max_frames"] + assert text.startswith("The camera") + assert len(prompts) == params["max_frames"] + + sheet, frames, path, sb_info = DifforumStoryboard().run(gradient(72, 128), 8, 6, 1.0, + direction=bundle) + assert frames.shape[1:] == (72, 128, 3) and path.shape[-1] == 3 + + model = StubModel() + out, depth, report = DifforumFeedbackSampler().run( + model, _c(), _c(), StubVAE(), gradient(72, 128), 4, 5.0, "euler", "normal", 2, + direction=bundle) + assert out.shape == (params["max_frames"], 72, 128, 3) + assert depth.shape[-1] == 3 + assert "keys=" in report + # the energy curve reached the sampler, and prompt travel conditioning too + assert len({round(c["denoise"], 3) for c in model.calls}) > 1 + assert all(c["pos"] is not None for c in model.calls) + + +def test_sampler_without_director(): + params = setup(seconds=0.5)[0] + cam = DifforumCamera().run(params, "0: zoom_in", "2d", 1.0, 1.0, "off", 0, 3)[0] + sched = DifforumSchedule().run(params, "0:(0.4)", "linear")[0] + opts = DifforumRenderOptions().run(**{k: s[1]["default"] for k, s in + DifforumRenderOptions.INPUT_TYPES()["required"].items()})[0] + opts.update(anchor_mode="first", end_frame=6) + out = DifforumFeedbackSampler().run(StubModel(), _c(), _c(), StubVAE(), gradient(72, 128), 2, + 5.0, "euler", "normal", 1, options=opts, params=params, + camera=cam, strength=sched)[0] + assert out.shape[0] == 6 + + +def test_bridges(): + params = setup("MiniMax H3", seconds=6.0)[0] + bundle = director(params)[0] + guides, masks, _ = DifforumGuideFrames().run(gradient(72, 128), "1 = generate (VACE / LTX / H3)", + "gray", direction=bundle) + n = params["max_frames"] + assert guides.shape[0] == n and masks.shape == (n, guides.shape[1], guides.shape[2]) + assert float(masks[0].max()) == 0.0 # frame 0 has nothing to generate + + keys, idx, sparse, first, last, length, _ = DifforumKeyframes().run( + guides, "LTX-2 / 2.5 (8k+1)", 1.0, 24.0) + ids = [int(i) for i in idx.split(",")] + assert (length - 1) % 8 == 0 and ids[0] == 0 and ids[-1] == length - 1 + assert all(i % 8 == 0 for i in ids[:-1]) + assert keys.shape[0] == len(ids) and sparse.shape[0] == length + assert float(sparse[1].abs().sum()) == 0.0 + + first, last, length, w, h, prompt, segments, info = DifforumH3Shot().run( + guides, "124 (~5s)", 0, direction=bundle, shot_description="A forest.") + assert (length - 5) % 17 == 0 and w % 32 == 0 and h % 32 == 0 + assert segments >= 2 and prompt.startswith("A forest. The camera") + + text = DifforumCameraPrompt().run("prompt suffix", direction=bundle)[0] + assert text.startswith("Camera:") + + +def test_prompt_travel_seconds_syntax(): + params = setup(seconds=4.0)[0] + track, first, batched, info = DifforumPromptTravel().run( + params, StubClip(), "0: a\n2s: bbb\n3.5s: cc", "linear", build_batched=True) + assert [f for f, _ in track.keyframes] == [0, 48, 84] + assert batched[0][0].shape[0] == params["max_frames"] + + +def test_loop_methods(): + frames = torch.rand(24, 16, 16, 3) + assert DifforumLoop().run(frames, "keep settled lap", 8, 4)[0].shape[0] == 8 + assert DifforumLoop().run(frames, "ping-pong", 0, 4)[0].shape[0] > 24 + assert DifforumLoop().run(frames, "flow crossfade", 0, 4)[0].shape[0] == 20 + + +def _c(): + return [[torch.zeros(1, 4, 8), {}]] + + +def test_live_sampler_ring_buffer(): + from difforum.nodes.render import DifforumLiveSampler + params = setup(seconds=0.5)[0] + bundle = director(params)[0] + frames, report = DifforumLiveSampler().run( + StubModel(), _c(), _c(), StubVAE(), gradient(72, 128), 30, 1, 1.0, "euler", "normal", 2, + 0.0, direction=bundle, keep_frames=10, live_preview=False) + assert frames.shape[0] == 10 and "fps measured" in report + + +def test_guide_frames_3d_with_depth_and_export(tmp_path, monkeypatch): + from difforum.nodes.export import DifforumCameraExport, DifforumCameraImport + params = setup(seconds=1.0)[0] + tl = default_timeline(params["max_frames"]) + tl["camera"][0]["move"] = "dolly_in" + bundle = director(params, tl=tl, mode="3d")[0] + depth = torch.linspace(0, 1, 128).expand(72, 128)[None, ..., None].expand(1, 72, 128, 3) + guides, masks, info = DifforumGuideFrames().run(gradient(72, 128), "1 = keep", "stretch edge", + direction=bundle, depth=depth) + assert "(3d)" in info and float(masks[-1].mean()) < 1.0 + + monkeypatch.chdir(tmp_path) + res = DifforumCameraExport().run("all", "shot", 0.1, direction=bundle)["result"] + paths = res[0].splitlines() + assert len(paths) == 3 and all(p.endswith((".json", ".jsx", "_blender.py")) for p in paths) + cam, _ = DifforumCameraImport().run("", params, "match frames", json_text=res[1]) + assert len(cam.deltas) == params["max_frames"] + + +def test_2d_director_keeps_depth_moves_alive(): + params = setup(seconds=1.0)[0] + tl = default_timeline(params["max_frames"]) + tl["camera"] = [{"start": 0, "move": "dolly_in", "speed": 1.0, "intensity": 1.0}] + bundle, camera = director(params, tl=tl, mode="2d")[:2] + assert camera.mode == "3d" and camera.flat + frames = DifforumStoryboard().run(gradient(72, 128), 8, 6, 1.0, direction=bundle)[1] + assert float((frames[-1] - frames[0]).abs().mean()) > 0.01 + depth = torch.full((1, 72, 128, 3), 0.5) + info = DifforumStoryboard().run(gradient(72, 128), 8, 6, 1.0, direction=bundle, depth=depth)[3] + assert "(pseudo3d)" in info # flat = 2d stays 2d even with a depth map + + +def test_h3_guides_chain_and_limit(monkeypatch): + import nodes as stub_nodes + + from difforum.nodes.bridges import DifforumH3Guides + + calls = [] + + class FakeAddGuide: + @classmethod + def execute(cls, positive, latent, frame_idx, vae=None, audio_vae=None, image=None, audio=None): + calls.append((frame_idx, audio is not None)) + return (positive + [frame_idx],) + + monkeypatch.setitem(stub_nodes.NODE_CLASS_MAPPINGS, "MiniMaxH3AddGuide", FakeAddGuide) + keys = torch.rand(6, 32, 32, 3) + pos, info = DifforumH3Guides().run([], {"samples": None}, object(), keys, "0,17,34,51,68,123", + 4, False, audio_vae=object(), audio={"waveform": 1}) + assert [c[0] for c in calls] == [0, 34, 68, 123] and calls[0][1] and not calls[1][1] + calls.clear() + DifforumH3Guides().run([], {"samples": None}, object(), keys, "0,17,34,51,68,123", 8, True) + assert [c[0] for c in calls] == [17, 34, 51, 68, 123] diff --git a/tests/test_scripts.py b/tests/test_scripts.py new file mode 100644 index 0000000..ae594a8 --- /dev/null +++ b/tests/test_scripts.py @@ -0,0 +1,15 @@ +"""Run the 0.x script-style engine checks (tests/scripts/check_*.py) as tests.""" + +import subprocess +import sys +from pathlib import Path + +import pytest + +SCRIPTS = sorted((Path(__file__).parent / "scripts").glob("check_*.py")) + + +@pytest.mark.parametrize("script", SCRIPTS, ids=[s.stem for s in SCRIPTS]) +def test_script(script): + r = subprocess.run([sys.executable, str(script)], capture_output=True, text=True, timeout=600) + assert r.returncode == 0, r.stdout[-3000:] + r.stderr[-3000:] diff --git a/tools/aftereffects/Difforum Export Camera.jsx b/tools/aftereffects/Difforum Export Camera.jsx new file mode 100644 index 0000000..fa324b8 --- /dev/null +++ b/tools/aftereffects/Difforum Export Camera.jsx @@ -0,0 +1,39 @@ +// Export the selected After Effects camera layer to Difforum. +// File > Scripts > Run Script File..., with a camera layer selected in the +// active comp. Writes a .json you load with Difforum · Camera Import +// (put it in ComfyUI/input). +// +// Convention: AE pixels (x right, y down, z into the screen), X/Y/Z Rotation +// composed as Rz * Ry * Rx - the same order Camera Export writes. Orientation +// is added to the rotations (exact when one of the two is zero). +(function () { + var comp = app.project.activeItem; + if (!(comp && comp instanceof CompItem)) { alert("Open a comp first."); return; } + var cam = comp.selectedLayers.length ? comp.selectedLayers[0] : null; + if (!cam || !(cam instanceof CameraLayer)) { alert("Select a camera layer."); return; } + + var fps = comp.frameRate, n = Math.round(comp.workAreaDuration * fps); + var t0 = comp.workAreaStart; + var zoom0 = cam.zoom.valueAtTime(t0, false); + var rows = []; + function r5(v) { return Math.round(v * 100000) / 100000; } + for (var i = 0; i < n; i++) { + var t = t0 + i / fps; + var p = cam.position.valueAtTime(t, false); + var o = cam.orientation.valueAtTime(t, false); + var rx = cam.xRotation.valueAtTime(t, false) + o[0]; + var ry = cam.yRotation.valueAtTime(t, false) + o[1]; + var rz = cam.zRotation.valueAtTime(t, false) + o[2]; + var z = cam.zoom.valueAtTime(t, false); + rows.push('{"f":' + i + ',"position":[' + r5(p[0]) + ',' + r5(p[1]) + ',' + r5(p[2]) + + '],"rotation":[' + r5(rx) + ',' + r5(ry) + ',' + r5(rz) + '],"zoom":' + r5(z) + '}'); + } + var json = '{"schema":"difforum.camera/1","convention":"ae","fps":' + fps + + ',"width":' + comp.width + ',"height":' + comp.height + ',"zoom0":' + zoom0 + + ',"reference_z":50.5,"frames":[' + rows.join(",") + ']}'; + var f = File.saveDialog("Save Difforum camera", "JSON:*.json"); + if (!f) return; + f.encoding = "UTF-8"; + f.open("w"); f.write(json); f.close(); + alert("Difforum: " + n + " frames written."); +})(); diff --git a/tools/blender/difforum_export_camera.py b/tools/blender/difforum_export_camera.py new file mode 100644 index 0000000..e1b2837 --- /dev/null +++ b/tools/blender/difforum_export_camera.py @@ -0,0 +1,49 @@ +"""Export the active Blender camera to Difforum. + +Run from Blender's Text Editor (Alt+P) with the camera selected or set as the +scene camera. Writes `_difforum_camera.json` next to the .blend +(or to your home folder for an unsaved file). Load it in ComfyUI with +Difforum · Camera Import (put the file in ComfyUI/input). + +UNIT_SCALE must match Camera Export's `blender_unit_scale` if the move came +from Difforum originally (default 0.1 m per Difforum unit). +""" + +import json +import os + +import bpy + +UNIT_SCALE = 0.1 + +scn = bpy.context.scene +cam = bpy.context.active_object if (bpy.context.active_object and + bpy.context.active_object.type == "CAMERA") else scn.camera +if cam is None: + raise RuntimeError("Select a camera (or set a scene camera) first.") + +W = scn.render.resolution_x * scn.render.resolution_percentage // 100 +H = scn.render.resolution_y * scn.render.resolution_percentage // 100 +fps = scn.render.fps / scn.render.fps_base +frames = [] +current = scn.frame_current +for f in range(scn.frame_start, scn.frame_end + 1): + scn.frame_set(f) + data = cam.data + sensor = data.sensor_width if data.sensor_fit != "VERTICAL" else data.sensor_height * W / H + focal_px = data.lens / sensor * W + frames.append({ + "f": f - scn.frame_start, + "matrix": [list(row) for row in cam.matrix_world], + "focal_px": focal_px, + }) +scn.frame_set(current) + +out = {"schema": "difforum.camera/1", "convention": "blender", "unit_scale": UNIT_SCALE, + "fps": fps, "width": W, "height": H, "frames": frames} +base = bpy.path.abspath("//") or os.path.expanduser("~") +name = (bpy.path.basename(bpy.data.filepath).rsplit(".", 1)[0] or "untitled") + "_difforum_camera.json" +path = os.path.join(base, name) +with open(path, "w", encoding="utf-8") as fh: + json.dump(out, fh) +print(f"Difforum: wrote {len(frames)} frames -> {path}") diff --git a/tools/build_docs.py b/tools/build_docs.py new file mode 100644 index 0000000..b63cf61 --- /dev/null +++ b/tools/build_docs.py @@ -0,0 +1,86 @@ +""" +Generate docs/NODES.md from the nodes themselves (description, inputs, +tooltips, outputs), so the reference can never lag behind the code. + + python tools/build_docs.py # write + python tools/build_docs.py --check # fail if docs/NODES.md is stale +""" + +from __future__ import annotations + +import importlib.util +import inspect +import sys +from pathlib import Path + +PACK = Path(__file__).resolve().parents[1] +if "difforum" not in sys.modules: + spec = importlib.util.spec_from_file_location( + "difforum", PACK / "__init__.py", submodule_search_locations=[str(PACK)]) + mod = importlib.util.module_from_spec(spec) + sys.modules["difforum"] = mod + spec.loader.exec_module(mod) + +from difforum.nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS # noqa: E402 + +WIDGETS = {"INT", "FLOAT", "STRING", "BOOLEAN"} + + +def _type(spec) -> str: + t = spec[0] + if isinstance(t, (list, tuple)): + shown = ", ".join(map(str, t[:6])) + (", ..." if len(t) > 6 else "") + return f"choice ({shown})" + return str(t) + + +def _default(spec) -> str: + opts = spec[1] if len(spec) > 1 and isinstance(spec[1], dict) else {} + d = opts.get("default") + if d is None: + return "" + s = str(d).replace("\n", " ") + return s if len(s) < 40 else s[:37] + "..." + + +def render() -> str: + by_cat: dict[str, list] = {} + for cid, cls in NODE_CLASS_MAPPINGS.items(): + by_cat.setdefault(cls.CATEGORY, []).append((cid, cls)) + out = ["# Node reference", "", + "Generated from the code by `tools/build_docs.py`. Every node also shows this text " + "as its description inside ComfyUI.", ""] + for cat in sorted(by_cat): + out += [f"## {cat.split('/', 1)[1]}", ""] + for cid, cls in by_cat[cat]: + out += [f"### {NODE_DISPLAY_NAME_MAPPINGS[cid]}", "", f"`{cid}`", ""] + doc = inspect.cleandoc(cls.__doc__ or "") + if doc: + out += [doc, ""] + it = cls.INPUT_TYPES() + rows = [] + for section in ("required", "optional"): + for name, spec in it.get(section, {}).items(): + opts = spec[1] if len(spec) > 1 and isinstance(spec[1], dict) else {} + tip = str(opts.get("tooltip", "")).replace("|", "/") + opt = "" if section == "required" else " *(optional)*" + rows.append(f"| `{name}`{opt} | {_type(spec)} | {_default(spec)} | {tip} |") + if rows: + out += ["| input | type | default | notes |", "|---|---|---|---|", *rows, ""] + names = getattr(cls, "RETURN_NAMES", cls.RETURN_TYPES) + outs = ", ".join(f"`{n}` ({t})" for n, t in zip(names, cls.RETURN_TYPES)) + if outs: + out += [f"**Outputs:** {outs}", ""] + return "\n".join(out).rstrip() + "\n" + + +if __name__ == "__main__": + text = render() + path = PACK / "docs" / "NODES.md" + if "--check" in sys.argv: + if not path.exists() or path.read_text(encoding="utf-8") != text: + print("docs/NODES.md is stale - run python tools/build_docs.py") + sys.exit(1) + else: + path.write_text(text, encoding="utf-8") + print("wrote docs/NODES.md") diff --git a/tools/build_workflows.py b/tools/build_workflows.py new file mode 100644 index 0000000..90ba260 --- /dev/null +++ b/tools/build_workflows.py @@ -0,0 +1,615 @@ +""" +Build the example workflows in example_workflows/ from code. + +Widget order for Difforum nodes is read from the real INPUT_TYPES, so a +template can never drift out of sync with a node again (the 0.x templates +did: several shipped with camera values in the wrong fields). External nodes +are described in EXTERNAL below. + + python tools/build_workflows.py # write + python tools/build_workflows.py --check # verify committed files are current +""" + +from __future__ import annotations + +import json +import sys +import types +from pathlib import Path + +PACK = Path(__file__).resolve().parents[1] +OUT = PACK / "example_workflows" # ComfyUI lists these in its Templates browser + +if "difforum" not in sys.modules: + _alias = types.ModuleType("difforum") + _alias.__path__ = [str(PACK)] + sys.modules["difforum"] = _alias + +from difforum.core.direction import default_timeline, parse_timeline # noqa: E402 +from difforum.nodes import NODE_CLASS_MAPPINGS as DF # noqa: E402 + +WIDGET_TYPES = {"INT", "FLOAT", "STRING", "BOOLEAN"} + +# type -> (link inputs [(name, type)], widget names, outputs [(name, type)]) +EXTERNAL = { + "CheckpointLoaderSimple": ([], ["ckpt_name"], [("MODEL", "MODEL"), ("CLIP", "CLIP"), ("VAE", "VAE")]), + "LoraLoader": ([("model", "MODEL"), ("clip", "CLIP")], ["lora_name", "strength_model", "strength_clip"], + [("MODEL", "MODEL"), ("CLIP", "CLIP")]), + "CLIPTextEncode": ([("clip", "CLIP")], ["text"], [("CONDITIONING", "CONDITIONING")]), + "EmptyLatentImage": ([], ["width", "height", "batch_size"], [("LATENT", "LATENT")]), + "KSampler": ([("model", "MODEL"), ("positive", "CONDITIONING"), ("negative", "CONDITIONING"), + ("latent_image", "LATENT")], + ["seed", "control_after_generate", "steps", "cfg", "sampler_name", "scheduler", "denoise"], + [("LATENT", "LATENT")]), + "VAEDecode": ([("samples", "LATENT"), ("vae", "VAE")], [], [("IMAGE", "IMAGE")]), + "LoadImage": ([], ["image", "upload"], [("IMAGE", "IMAGE"), ("MASK", "MASK")]), + "LoadAudio": ([], ["audio", "audioUI", "upload"], [("AUDIO", "AUDIO")]), + "PreviewImage": ([("images", "IMAGE")], [], []), + "SaveImage": ([("images", "IMAGE")], ["filename_prefix"], []), + "PreviewAny": ([("source", "*")], [], [("STRING", "STRING")]), + "CreateVideo": ([("images", "IMAGE"), ("audio", "AUDIO")], ["fps"], [("VIDEO", "VIDEO")]), + "SaveVideo": ([("video", "VIDEO")], ["filename_prefix", "format", "codec"], []), + "DownloadAndLoadDepthAnythingV2Model": ([], ["model"], [("da_v2_model", "DAMODEL")]), + "DepthAnything_V2": ([("da_model", "DAMODEL"), ("images", "IMAGE")], [], [("image", "IMAGE")]), + "EmptyLTXVLatentVideo": ([], ["width", "height", "length", "batch_size"], [("LATENT", "LATENT")]), + "MarkdownNote": ([], ["text"], []), + # MiniMax H3 (ComfyUI core) - names match the official templates + "UNETLoader": ([], ["unet_name", "weight_dtype"], [("MODEL", "MODEL")]), + "LoraLoaderModelOnly": ([("model", "MODEL")], ["lora_name", "strength_model"], [("MODEL", "MODEL")]), + "CLIPLoader": ([], ["clip_name", "type", "device"], [("CLIP", "CLIP")]), + "VAELoader": ([], ["vae_name"], [("VAE", "VAE")]), + "MiniMaxH3ImageToVideo": ([("clip", "CLIP"), ("vae", "VAE"), ("first_frame", "IMAGE"), ("last_frame", "IMAGE")], + ["prompt", "width", "height", "length"], + [("positive", "CONDITIONING"), ("LATENT", "LATENT")]), + "MiniMaxH3ReferenceToVideo": ([("clip", "CLIP"), ("vae", "VAE"), ("audio_vae", "VAE"), + ("ref_images.ref_image_0", "IMAGE")], + ["prompt", "width", "height", "length", "ref_image_size"], + [("positive", "CONDITIONING"), ("LATENT", "LATENT")]), + "RandomNoise": ([], ["noise_seed", "control_after_generate"], [("NOISE", "NOISE")]), + "KSamplerSelect": ([], ["sampler_name"], [("SAMPLER", "SAMPLER")]), + "BasicScheduler": ([("model", "MODEL")], ["scheduler", "steps", "denoise"], [("SIGMAS", "SIGMAS")]), + "BasicGuider": ([("model", "MODEL"), ("conditioning", "CONDITIONING")], [], [("GUIDER", "GUIDER")]), + "SamplerCustomAdvanced": ([("noise", "NOISE"), ("guider", "GUIDER"), ("sampler", "SAMPLER"), + ("sigmas", "SIGMAS"), ("latent_image", "LATENT")], [], + [("output", "LATENT"), ("denoised_output", "LATENT")]), + "VAEDecodeAudio": ([("samples", "LATENT"), ("vae", "VAE")], [], [("AUDIO", "AUDIO")]), +} + + +def difforum_spec(type_): + cls = DF[type_] + it = cls.INPUT_TYPES() + links, widgets = [], [] + for section in ("required", "optional"): + for name, spec in it.get(section, {}).items(): + t = spec[0] + opts = spec[1] if len(spec) > 1 and isinstance(spec[1], dict) else {} + if isinstance(t, (list, tuple)) or (t in WIDGET_TYPES and not opts.get("forceInput")): + widgets.append(name) + if t == "INT" and "seed" in name: + widgets.append(name + ".control") + else: + links.append((name, t)) + outs = list(zip(getattr(cls, "RETURN_NAMES", cls.RETURN_TYPES), cls.RETURN_TYPES)) + defaults = {} + for section in ("required", "optional"): + for name, spec in it.get(section, {}).items(): + t = spec[0] + opts = spec[1] if len(spec) > 1 and isinstance(spec[1], dict) else {} + if "default" in opts: + defaults[name] = opts["default"] + elif isinstance(t, (list, tuple)) and t: + defaults[name] = t[0] + return links, widgets, outs, defaults + + +class Graph: + def __init__(self, title): + self.title = title + self.nodes, self.links = [], [] + + def add(self, type_, pos, size=(340, 200), title=None, mode=0, **values): + if type_ in DF: + links, widgets, outs, defaults = difforum_spec(type_) + else: + links, widgets, outs = EXTERNAL[type_] + defaults = {} + wv = [] + for w in widgets: + if w.endswith(".control") or w == "control_after_generate": + wv.append("fixed") + else: + if w not in values and w not in defaults and type_ not in EXTERNAL: + raise KeyError(f"{type_}.{w} has no default") + wv.append(values.pop(w, defaults.get(w))) + if values: + raise KeyError(f"{type_}: unknown widgets {sorted(values)}") + node = { + "id": len(self.nodes) + 1, "type": type_, "pos": list(pos), "size": list(size), + "flags": {}, "order": len(self.nodes), "mode": mode, + "inputs": [{"name": n, "type": t, "link": None} for n, t in links], + "outputs": [{"name": n, "type": t, "links": [], "slot_index": i} + for i, (n, t) in enumerate(outs)], + "properties": {"Node name for S&R": type_}, + "widgets_values": wv, + } + if title: + node["title"] = title + self.nodes.append(node) + return node["id"] + + def note(self, pos, text, size=(420, 260)): + return self.add("MarkdownNote", pos, size=size, text=text) + + def link(self, src, out, dst, inp): + s, d = self.nodes[src - 1], self.nodes[dst - 1] + oi = next(i for i, o in enumerate(s["outputs"]) if o["name"] == out) + ii = next((i for i, x in enumerate(d["inputs"]) if x["name"] == inp), None) + if ii is None: # link into a widget (convert-to-input) + d["inputs"].append({"name": inp, "type": s["outputs"][oi]["type"], "link": None, + "widget": {"name": inp}}) + ii = len(d["inputs"]) - 1 + lid = len(self.links) + 1 + self.links.append([lid, src, oi, dst, ii, s["outputs"][oi]["type"]]) + s["outputs"][oi]["links"].append(lid) + d["inputs"][ii]["link"] = lid + + def json(self): + return { + "last_node_id": len(self.nodes), "last_link_id": len(self.links), + "nodes": self.nodes, "links": self.links, "groups": [], "config": {}, + "extra": {"ds": {"scale": 0.65, "offset": [40, 40]}, "difforum": "1.0", + "title": self.title}, + "version": 0.4, + } + + +def tl_json(frames, **patch): + tl = default_timeline(frames) + for k, v in patch.items(): + tl[k] = v + return json.dumps(parse_timeline(tl)) + + +# --------------------------------------------------------------------------- +# workflows +# --------------------------------------------------------------------------- + +def sd_front(g, x=0, y=0, ckpt="sd_xl_base_1.0.safetensors", pos_text="", steps=24, cfg=6.0, + sampler="dpmpp_2m", scheduler="karras"): + """Checkpoint + prompts + first frame. Returns ids.""" + ck = g.add("CheckpointLoaderSimple", (x, y), size=(320, 100), ckpt_name=ckpt) + pos = g.add("CLIPTextEncode", (x + 360, y), size=(380, 140), title="Look / style (positive)", + text=pos_text or "cinematic film still, rich detail, volumetric light, 35mm") + neg = g.add("CLIPTextEncode", (x + 360, y + 180), size=(380, 120), title="Negative", + text="blurry, low quality, watermark, text, frame, border") + g.link(ck, "CLIP", pos, "clip") + g.link(ck, "CLIP", neg, "clip") + return ck, pos, neg + + +def first_frame(g, ck, pos, neg, setup, x, y, steps=24, cfg=6.0, sampler="dpmpp_2m", scheduler="karras"): + lat = g.add("EmptyLatentImage", (x, y), size=(260, 110), width=1024, height=576, batch_size=1) + g.link(setup, "width", lat, "width") + g.link(setup, "height", lat, "height") + ks = g.add("KSampler", (x + 290, y), size=(300, 260), title="First frame", seed=7, steps=steps, + cfg=cfg, sampler_name=sampler, scheduler=scheduler, denoise=1.0) + g.link(ck, "MODEL", ks, "model") + g.link(pos, "CONDITIONING", ks, "positive") + g.link(neg, "CONDITIONING", ks, "negative") + g.link(lat, "LATENT", ks, "latent_image") + dec = g.add("VAEDecode", (x + 620, y), size=(200, 60)) + g.link(ks, "LATENT", dec, "samples") + g.link(ck, "VAE", dec, "vae") + return dec + + +def wf_storyboard(): + g = Graph("01 · Storyboard - direct a shot, no model needed") + g.note((0, -300), "## Start here - no model needed\n\n" + "1. Load any still in **Load Image**.\n2. Edit the **Director** timeline: drag blocks, " + "pick camera moves, press ▶ to preview.\n3. Queue: the **Storyboard** renders the whole " + "camera move in about a second, and **Camera → Prompt** writes it in words for " + "prompt-driven video models.\n\nWhen the motion feels right, move to 02 (render) or " + "07/08 (LTX / H3).", size=(520, 260)) + s = g.add("Difforum_Setup", (0, 0), size=(320, 300), duration=5.0) + d = g.add("Difforum_Director", (360, 0), size=(800, 880), timeline=tl_json(120)) + g.link(s, "params", d, "params") + img = g.add("LoadImage", (0, 360), size=(320, 360), image="example.png") + sb = g.add("Difforum_Storyboard", (1200, 0), size=(320, 220)) + g.link(img, "IMAGE", sb, "init_image") + g.link(d, "direction", sb, "direction") + p1 = g.add("PreviewImage", (1560, 0), size=(620, 380), title="Contact sheet") + g.link(sb, "sheet", p1, "images") + p2 = g.add("PreviewImage", (1560, 420), size=(300, 300), title="Camera path (top view)") + g.link(sb, "camera_path", p2, "images") + cp = g.add("Difforum_CameraPrompt", (1200, 280), size=(320, 160)) + g.link(d, "direction", cp, "direction") + t = g.add("PreviewAny", (1200, 480), size=(320, 200), title="Camera in words") + g.link(cp, "text", t, "source") + return g + + +def wf_feedback(depth=False, title="02 · Feedback render (SDXL)", camera_mode="2d", tl=None, + duration=5.0): + g = Graph(title) + ck, pos, neg = sd_front(g, 0, 0) + s = g.add("Difforum_Setup", (0, 360), size=(320, 300), duration=duration, long_edge=1024) + d = g.add("Difforum_Director", (420, 360), size=(800, 880), camera_mode=camera_mode, + timeline=tl or tl_json(120)) + g.link(s, "params", d, "params") + g.link(ck, "CLIP", d, "clip") + dec = first_frame(g, ck, pos, neg, s, 820, 0) + fb = g.add("Difforum_FeedbackSampler", (1260, 360), size=(360, 460), steps=20, cfg=6.0, + sampler_name="dpmpp_2m", scheduler="karras", cadence=2) + for a, b in (("MODEL", "model"), ("VAE", "vae")): + g.link(ck, a, fb, b) + g.link(pos, "CONDITIONING", fb, "positive") + g.link(neg, "CONDITIONING", fb, "negative") + g.link(dec, "IMAGE", fb, "init_image") + g.link(d, "direction", fb, "direction") + if depth: + dm = g.add("DownloadAndLoadDepthAnythingV2Model", (1260, 0), size=(360, 80), + model="depth_anything_v2_vitl_fp32.safetensors") + da = g.add("DepthAnything_V2", (1260, 120), size=(360, 80)) + g.link(dm, "da_v2_model", da, "da_model") + g.link(dec, "IMAGE", da, "images") + g.link(da, "image", fb, "depth") + st = g.add("Difforum_FlowStabilize", (1660, 360), size=(300, 130), strength=0.45) + g.link(fb, "frames", st, "frames") + cv = g.add("CreateVideo", (2000, 360), size=(260, 100), fps=24.0) + g.link(st, "frames", cv, "images") + g.link(s, "fps", cv, "fps") + sv = g.add("SaveVideo", (2000, 500), size=(300, 320), filename_prefix="video/difforum", + format="auto", codec="auto") + g.link(cv, "VIDEO", sv, "video") + rp = g.add("PreviewAny", (1660, 540), size=(300, 200), title="Run report") + g.link(fb, "report", rp, "source") + return g, s, d, fb, cv + + +def wf_feedback_2d(): + g, *_ = wf_feedback() + g.note((0, -320), "## Feedback render (SDXL)\n\nThe classic Deforum look on a modern model. " + "**Director** drives camera, energy and prompt travel through one `direction` wire.\n\n" + "- Any SD1.5 / SDXL / Flux checkpoint works (plain MODEL/VAE/CONDITIONING).\n" + "- `cadence 2` diffuses every other frame and crossfades the rest (~2x faster).\n" + "- Faster still: a DMD2 / Lightning / Turbo LoRA at 4-8 steps, cfg 1-2.\n" + "- Director prompts replace the positive prompt once CLIP is connected; the positive " + "box still sets the first frame.", size=(560, 260)) + return g + + +def wf_parallax(): + tl = tl_json(120, camera=[ + {"start": 0, "move": "dolly_in", "speed": 0.9, "intensity": 0.8, "ease": "ease_in_out"}, + {"start": 40, "move": "orbit_right", "speed": 1.0, "intensity": 0.9, "ease": "ease_in_out"}, + {"start": 80, "move": "crane_up", "speed": 0.8, "intensity": 0.8, "ease": "ease_out"}, + ]) + g, *_ = wf_feedback(depth=True, title="03 · Real parallax (3D + depth)", camera_mode="3d", tl=tl) + g.note((0, -320), "## Real parallax\n\nDepth Anything V2 reads the first frame, the Director " + "is in **3d**, and the sampler re-projects the depth with the image every frame - " + "near things move faster than far ones for the whole clip, not just the start.\n\n" + "Needs **ComfyUI-DepthAnythingV2** (Kijai). Too strong? Lower the sampler's " + "`translation_scale`. The `depth` output is the tracked depth per frame, ready for " + "compositing.", size=(560, 240)) + return g + + +def wf_audio(): + g, s, d, fb, cv = wf_feedback(title="04 · Audio reactive", duration=10.0, tl=tl_json(240, camera=[ + {"start": 0, "move": "zoom_in", "speed": 0.7, "intensity": 0.8, "react": "beat_pulse"}, + {"start": 96, "move": "spiral", "speed": 1.0, "intensity": 1.0, "react": "bass_speed"}, + {"start": 180, "move": "handheld", "speed": 1.0, "intensity": 1.2, "react": "onset_shake"}, + ])) + au = g.add("LoadAudio", (0, 720), size=(320, 140), audio="track.mp3") + an = g.add("Difforum_AudioAnalyzer", (0, 900), size=(320, 200)) + g.link(s, "params", an, "params") + g.link(au, "AUDIO", an, "audio") + g.link(an, "audio_curves", d, "audio") + g.link(au, "AUDIO", cv, "audio") + pv = g.add("PreviewImage", (0, 1140), size=(320, 260), title="Bands: low / mid / high / beat") + g.link(an, "bands_plot", pv, "images") + g.note((0, -320), "## Audio reactive, no expressions\n\nEach camera block in the Director has " + "an **Audio** reaction: pulse on beat, shake on hits, bass drives speed, mids rock the " + "roll. The analyzer feeds them; **Create Video** muxes the track back in.\n\nFor custom " + "curves use **Audio Curve** or type bands (`low`, `beat`...) in a Schedule.", size=(560, 220)) + return g + + +def wf_live(): + g = Graph("05 · Live (turbo models)") + ck, pos, neg = sd_front(g, 0, 0, ckpt="sd_xl_turbo_1.0_fp16.safetensors", + pos_text="liquid chrome sculpture, iridescent, studio light") + s = g.add("Difforum_Setup", (0, 360), size=(320, 300), duration=10.0, long_edge=512) + d = g.add("Difforum_Director", (420, 360), size=(800, 880), timeline=tl_json(240)) + g.link(s, "params", d, "params") + dec = first_frame(g, ck, pos, neg, s, 820, 0, steps=2, cfg=1.0, sampler="euler_ancestral", + scheduler="sgm_uniform") + lv = g.add("Difforum_LiveSampler", (1260, 360), size=(360, 560), steps=2, cfg=1.0, + sampler_name="euler_ancestral", scheduler="sgm_uniform", run_frames=1200) + op = g.add("Difforum_RenderOptions", (1260, 960), size=(340, 560), title="Render Options (kaleidoscope)", + symmetry="kaleidoscope", color_coherence=0.5) + g.link(op, "options", lv, "options") + for a, b in (("MODEL", "model"), ("VAE", "vae")): + g.link(ck, a, lv, b) + g.link(pos, "CONDITIONING", lv, "positive") + g.link(neg, "CONDITIONING", lv, "negative") + g.link(dec, "IMAGE", lv, "init_image") + g.link(d, "direction", lv, "direction") + pv = g.add("PreviewImage", (1660, 360), size=(400, 300)) + g.link(lv, "frames", pv, "images") + g.note((0, -320), "## Live\n\nQueue once: the node plays in place. `live_source` = `0` for a " + "webcam magic mirror; `stream_dir` / `spout_name` feed OBS, Resolume or TouchDesigner. " + "Use a 1-4 step model at ~512 px.", size=(560, 160)) + return g + + +def wf_loop(): + g = Graph("06 · Seamless loop (installations)") + ck, pos, neg = sd_front(g, 0, 0) + s = g.add("Difforum_Setup", (0, 360), size=(320, 300), duration_mode="frames", duration=360.0) + cam = g.add("Difforum_Camera", (420, 360), size=(380, 360), keys="0: orbit_right 1.0 0.8\n60: spiral 1.2 1.0", + loop_mode="harmonic", cycle_frames=120) + g.link(s, "params", cam, "params") + sch = g.add("Difforum_Schedule", (420, 760), size=(380, 160), schedule="0:(0.5)") + g.link(s, "params", sch, "params") + dec = first_frame(g, ck, pos, neg, s, 820, 0) + fb = g.add("Difforum_FeedbackSampler", (860, 360), size=(360, 460)) + for a, b in (("MODEL", "model"), ("VAE", "vae")): + g.link(ck, a, fb, b) + g.link(pos, "CONDITIONING", fb, "positive") + g.link(neg, "CONDITIONING", fb, "negative") + g.link(dec, "IMAGE", fb, "init_image") + g.link(s, "params", fb, "params") + g.link(cam, "camera", fb, "camera") + g.link(sch, "schedule", fb, "strength") + lp = g.add("Difforum_Loop", (1260, 360), size=(300, 160), method="keep settled lap") + g.link(fb, "frames", lp, "frames") + g.link(cam, "cycle_frames", lp, "cycle_frames") + cv = g.add("CreateVideo", (1600, 360), size=(260, 100), fps=24.0) + g.link(lp, "frames", cv, "images") + sv = g.add("SaveVideo", (1600, 500), size=(300, 320), filename_prefix="video/difforum_loop", + format="auto", codec="auto") + g.link(cv, "VIDEO", sv, "video") + g.note((0, -320), "## Loop without a crossfade\n\nThe Camera path is made periodic over 120 frames " + "and rendered for 3 laps; the feedback settles onto its cycle and **Loop** keeps the last " + "lap. For projections that run for hours.", size=(560, 180)) + return g + + +def wf_ltx(): + g = Graph("07 · LTX-2 / 2.5 guides from the Director") + s = g.add("Difforum_Setup", (0, 0), size=(320, 300), target="LTX-2 / 2.5", duration=5.0, + long_edge=1216) + d = g.add("Difforum_Director", (360, 0), size=(800, 880), timeline=tl_json(121)) + g.link(s, "params", d, "params") + img = g.add("LoadImage", (0, 360), size=(320, 360), image="example.png", title="Anchor / first frame") + gf = g.add("Difforum_GuideFrames", (1200, 0), size=(340, 200)) + g.link(img, "IMAGE", gf, "anchor_image") + g.link(d, "direction", gf, "direction") + kf = g.add("Difforum_Keyframes", (1200, 240), size=(340, 220), every_seconds=1.0) + g.link(gf, "guide_frames", kf, "frames") + g.link(s, "fps", kf, "fps") + lat = g.add("EmptyLTXVLatentVideo", (1200, 500), size=(300, 160), width=1216, height=704, + length=121, batch_size=1) + g.link(s, "width", lat, "width") + g.link(s, "height", lat, "height") + g.link(kf, "length", lat, "length") + lg = g.add("Difforum_LTXGuides", (1580, 240), size=(340, 260)) + g.link(kf, "keyframes", lg, "keyframes") + g.link(kf, "indices", lg, "indices") + g.link(lat, "LATENT", lg, "latent") + cp = g.add("Difforum_CameraPrompt", (1580, 0), size=(340, 160), format="prompt suffix") + g.link(d, "direction", cp, "direction") + pv = g.add("PreviewImage", (1580, 540), size=(340, 260), title="Keyframes") + g.link(kf, "keyframes", pv, "images") + g.note((1960, 0), "## Wire into your LTX graph\n\nConnect your LTX **positive / negative** " + "(text encoded with the Camera → Prompt text appended), and the LTX **VAE** into " + "**LTX Guides**. Its outputs replace the conditioning and latent going into your LTX " + "sampler.\n\n- Setup target = LTX keeps length on 8k+1 and size on 32 px.\n" + "- Guides every 1 s at strength 0.7; frame 0 at 1.0 locks the look.\n" + "- Swap Guide Frames for a Feedback render or a Storyboard to guide with other " + "images.", size=(520, 300)) + return g + + +def h3_render(g, x, y, conditioning_node, cond_out, latent_node, latent_out, model_node, setup, audio_vae, + video_vae, lora, steps=20): + """The official MiniMax H3 sampling tail: guider -> custom sampler -> AV decode -> video.""" + lo = g.add("LoraLoaderModelOnly", (x, y - 120), size=(360, 90), title="Turbo LoRA (Ctrl+B to enable, then steps 4-8)", + mode=4, lora_name=lora, strength_model=1.0) + g.link(model_node, "MODEL", lo, "model") + gd = g.add("BasicGuider", (x, y), size=(220, 50)) + g.link(lo, "MODEL", gd, "model") + g.link(conditioning_node, cond_out, gd, "conditioning") + nz = g.add("RandomNoise", (x, y + 90), size=(260, 90), noise_seed=7) + ks = g.add("KSamplerSelect", (x, y + 210), size=(260, 60), sampler_name="res_multistep") + sc = g.add("BasicScheduler", (x, y + 300), size=(260, 110), scheduler="simple", steps=steps, denoise=1.0) + g.link(lo, "MODEL", sc, "model") + sa = g.add("SamplerCustomAdvanced", (x + 300, y), size=(260, 120)) + g.link(nz, "NOISE", sa, "noise") + g.link(gd, "GUIDER", sa, "guider") + g.link(ks, "SAMPLER", sa, "sampler") + g.link(sc, "SIGMAS", sa, "sigmas") + g.link(latent_node, latent_out, sa, "latent_image") + dv = g.add("VAEDecode", (x + 600, y), size=(200, 60)) + g.link(sa, "output", dv, "samples") + g.link(video_vae, "VAE", dv, "vae") + da = g.add("VAEDecodeAudio", (x + 600, y + 100), size=(200, 60)) + g.link(sa, "output", da, "samples") + g.link(audio_vae, "VAE", da, "vae") + cv = g.add("CreateVideo", (x + 840, y), size=(240, 100), fps=24.0) + g.link(dv, "IMAGE", cv, "images") + g.link(da, "AUDIO", cv, "audio") + sv = g.add("SaveVideo", (x + 840, y + 140), size=(420, 420), filename_prefix="video/difforum_h3", + format="auto", codec="auto") + g.link(cv, "VIDEO", sv, "video") + + +def h3_loaders(g, x, y, unet): + un = g.add("UNETLoader", (x, y), size=(420, 90), unet_name=unet, weight_dtype="default") + cl = g.add("CLIPLoader", (x, y + 120), size=(420, 110), clip_name="qwen3vl_32b_minimax_h3_nvfp4_awq.safetensors", + type="minimax", device="default") + vv = g.add("VAELoader", (x, y + 260), size=(420, 60), vae_name="minimax_h3_video_vae_int8_convrot.safetensors") + va = g.add("VAELoader", (x, y + 350), size=(420, 60), vae_name="minimax_h3_audio_vae_fp32.safetensors") + return un, cl, vv, va + + +H3_TIMELINE = dict(scenes=[ + {"start": 0, "mood": "calm", "prompt": "misty ancient forest at dawn, light shafts through the canopy"}, + {"start": 62, "mood": "build", "prompt": "the mist lifts, glowing moss and roots in the foreground"}, +], camera=[ + {"start": 0, "move": "dolly_in", "speed": 0.8, "intensity": 0.8, "ease": "ease_in_out"}, + {"start": 62, "move": "orbit_right", "speed": 0.9, "intensity": 0.8, "ease": "ease_in_out"}, +]) + + +def wf_h3(): + g = Graph("08 · MiniMax H3 first/last frame (FL2VA) from the Director") + s = g.add("Difforum_Setup", (0, 0), size=(320, 300), target="MiniMax H3", duration=5.0, long_edge=832) + d = g.add("Difforum_Director", (360, 0), size=(800, 880), camera_mode="3d", timeline=tl_json(124, **H3_TIMELINE)) + g.link(s, "params", d, "params") + img = g.add("LoadImage", (0, 360), size=(320, 360), image="example.png", title="First frame") + gf = g.add("Difforum_GuideFrames", (1200, 0), size=(340, 200), hole_fill="stretch edge") + g.link(img, "IMAGE", gf, "anchor_image") + g.link(d, "direction", gf, "direction") + h3 = g.add("Difforum_H3Shot", (1200, 240), size=(340, 300), segment_length="124 (~5s)", + shot_description="A misty ancient forest at dawn; light shafts cut through the canopy. " + "Soft ambient birdsong, a low wind.") + g.link(gf, "guide_frames", h3, "frames") + g.link(d, "direction", h3, "direction") + un, cl, vv, va = h3_loaders(g, 1580, -200, "minimax_h3_fl2va_pruned_int8_convrot.safetensors") + i2v = g.add("MiniMaxH3ImageToVideo", (1580, 300), size=(420, 260), prompt="", width=832, height=480, length=124) + g.link(cl, "CLIP", i2v, "clip") + g.link(vv, "VAE", i2v, "vae") + g.link(h3, "first_frame", i2v, "first_frame") + g.link(h3, "last_frame", i2v, "last_frame") + g.link(h3, "prompt", i2v, "prompt") + g.link(h3, "width", i2v, "width") + g.link(h3, "height", i2v, "height") + g.link(h3, "length", i2v, "length") + h3_render(g, 2060, 120, i2v, "positive", i2v, "LATENT", un, s, va, vv, + "minimax_h3_fl2v_turbo_8step_v1.0_comfyui_bf16.safetensors") + p1 = g.add("PreviewImage", (1200, 580), size=(340, 240), title="first / last frame") + g.link(h3, "last_frame", p1, "images") + g.note((0, -330), "## MiniMax H3 · first / last frame\n\nThe **Director** draws the move, **Guide Frames** " + "carries your first frame along it, and **H3 Shot** hands H3 the first frame, the frame where the " + "camera ends, the length (17k+5), the size and a prompt with the camera move in words. H3 then " + "performs *your* move.\n\n- Model files: the names of the official MiniMax H3 templates - swap for " + "yours.\n- Faster: enable the Turbo LoRA (Ctrl+B) and set steps 6-8.\n- Longer than 20 s: raise " + "`segment` on H3 Shot and feed each render's last frame forward.", size=(620, 280)) + return g + + +def wf_h3_guides(): + g = Graph("10 · MiniMax H3 multi-keyframe guides from the Director") + s = g.add("Difforum_Setup", (0, 0), size=(320, 300), target="MiniMax H3", duration=5.0, long_edge=832) + d = g.add("Difforum_Director", (360, 0), size=(800, 880), camera_mode="3d", timeline=tl_json(124, **H3_TIMELINE)) + g.link(s, "params", d, "params") + img = g.add("LoadImage", (0, 360), size=(320, 360), image="example.png", title="Anchor image (also the H3 reference)") + gf = g.add("Difforum_GuideFrames", (1200, 0), size=(340, 200), hole_fill="stretch edge") + g.link(img, "IMAGE", gf, "anchor_image") + g.link(d, "direction", gf, "direction") + kf = g.add("Difforum_Keyframes", (1200, 240), size=(340, 220), grid="MiniMax H3 (17k+5)", every_seconds=2.0) + g.link(gf, "guide_frames", kf, "frames") + g.link(s, "fps", kf, "fps") + cp = g.add("Difforum_CameraPrompt", (1200, 500), size=(340, 220), format="sentence", + prefix=" shows a misty ancient forest at dawn. Keep its look, light and " + "composition. Soft ambient birdsong, a low wind.") + g.link(d, "direction", cp, "direction") + un, cl, vv, va = h3_loaders(g, 1580, -200, "minimax_h3_ref2va_pruned_int8_convrot.safetensors") + r2v = g.add("MiniMaxH3ReferenceToVideo", (1580, 300), size=(420, 300), prompt="", width=832, height=480, + length=124, ref_image_size="match") + g.link(cl, "CLIP", r2v, "clip") + g.link(vv, "VAE", r2v, "vae") + g.link(va, "VAE", r2v, "audio_vae") + g.link(img, "IMAGE", r2v, "ref_images.ref_image_0") + g.link(cp, "text", r2v, "prompt") + g.link(s, "width", r2v, "width") + g.link(s, "height", r2v, "height") + g.link(kf, "length", r2v, "length") + hg = g.add("Difforum_H3Guides", (1580, 640), size=(420, 200), max_guides=4) + g.link(r2v, "positive", hg, "positive") + g.link(r2v, "LATENT", hg, "latent") + g.link(vv, "VAE", hg, "vae") + g.link(kf, "keyframes", hg, "keyframes") + g.link(kf, "indices", hg, "indices") + h3_render(g, 2060, 120, hg, "positive", r2v, "LATENT", un, s, va, vv, + "minimax_h3_ref2v_turbo_4step_v0.1_comfyui_bf16.safetensors") + pv = g.add("PreviewImage", (1200, 760), size=(340, 240), title="Keyframes sent to H3") + g.link(kf, "keyframes", pv, "images") + g.note((0, -330), "## MiniMax H3 · keyframes along your camera\n\nThe **Director** draws the move. " + "**Guide Frames** carries the anchor image along it, **Keyframes** picks one every 2 s on the " + "H3 grid, and **H3 Guides** anchors them inside the generation with the core " + "`MiniMaxH3AddGuide` - the same mechanism as the official *Multiframe Reference* template, " + "driven by a camera instead of hand-placed images.\n\n- The anchor is also ``, " + "which holds identity and style.\n- **Camera → Prompt** adds the move in words.\n" + "- Swap Guide Frames for a Feedback render or a Storyboard to guide with other images; " + "connect an **audio** to H3 Guides to anchor a soundtrack at frame 0.\n- Model files follow " + "the official template - swap for yours. Turbo LoRA: Ctrl+B, steps 4.", size=(620, 330)) + return g + + +def wf_export(): + g = Graph("09 · Camera to After Effects / Blender (and back)") + s = g.add("Difforum_Setup", (0, 0), size=(320, 300)) + d = g.add("Difforum_Director", (360, 0), size=(800, 880), camera_mode="3d", timeline=tl_json(120)) + g.link(s, "params", d, "params") + ex = g.add("Difforum_CameraExport", (1200, 0), size=(360, 200)) + g.link(d, "direction", ex, "direction") + t = g.add("PreviewAny", (1600, 0), size=(360, 160), title="Written files") + g.link(ex, "paths", t, "source") + im = g.add("Difforum_CameraImport", (1200, 260), size=(360, 200)) + g.link(s, "params", im, "params") + img = g.add("LoadImage", (0, 360), size=(320, 360), image="example.png") + sb = g.add("Difforum_Storyboard", (1600, 260), size=(320, 220)) + g.link(img, "IMAGE", sb, "init_image") + g.link(s, "params", sb, "params") + g.link(im, "camera", sb, "camera") + pv = g.add("PreviewImage", (1960, 260), size=(520, 320)) + g.link(sb, "sheet", pv, "images") + g.note((0, -320), "## Camera interchange\n\n**Camera Export** writes to `output/difforum/`: a " + "`.jsx` for After Effects (File > Scripts > Run Script File), a `.py` for Blender " + "(Text Editor > Run) and a `.json`. Composite titles or 3D on top of the render with " + "the same camera.\n\n**Camera Import** reads a `.json` from `input/` - from Camera " + "Export, or from `tools/blender/difforum_export_camera.py` / `tools/aftereffects/" + "Difforum Export Camera.jsx` - so a move blocked in Blender or AE drives the AI shot. " + "(The import branch errors until a file is in `input/`: bypass it with Ctrl+B.)", + size=(620, 260)) + return g + + +WORKFLOWS = { + "01_storyboard_no_model.json": wf_storyboard, + "02_feedback_sdxl.json": wf_feedback_2d, + "03_parallax_3d_depth.json": wf_parallax, + "04_audio_reactive.json": wf_audio, + "05_live_turbo.json": wf_live, + "06_seamless_loop.json": wf_loop, + "07_ltx_guides.json": wf_ltx, + "08_h3_first_last_frame.json": wf_h3, + "09_camera_to_ae_blender.json": wf_export, + "10_h3_multikeyframe_guides.json": wf_h3_guides, +} + + +def main(check=False): + OUT.mkdir(parents=True, exist_ok=True) + stale = [] + for name, fn in WORKFLOWS.items(): + text = json.dumps(fn().json(), indent=1, ensure_ascii=False) + "\n" + path = OUT / name + if check: + if not path.exists() or path.read_text(encoding="utf-8") != text: + stale.append(name) + else: + path.write_text(text, encoding="utf-8") + print("wrote", path.relative_to(PACK)) + if check and stale: + print("stale workflows:", ", ".join(stale)) + sys.exit(1) + + +if __name__ == "__main__": + main(check="--check" in sys.argv)