From aea08129c12e03606eb747b728b8682d4e441631 Mon Sep 17 00:00:00 2001 From: Rufan Date: Tue, 14 Jul 2026 23:33:28 +0200 Subject: [PATCH] feat(reco-core,reco-gui,reco-cli): automatic per-camera color matching at the seam Reimplemented from scratch against the current architecture, like the seam-positioning feature - color_match.rs, blur.wgsl, and multiband_composite.wgsl from the original design do not exist anywhere in this codebase currently, only the fisheye.wgsl uniform slots they used to drive (color_scale/color_offset_blend), which are wired but always hardcoded to identity. Periodically samples a coarse grid in the seam-adjacent band of each camera's raw frame and derives a small, EMA-smoothed, clamped per-camera YUV offset that nudges both cameras toward their shared mean - same idea as the original, reduced scope (offset only, no separate multiplicative scale; fixed internal constants for band width/grid/ interval/smoothing instead of individually GUI-tunable knobs). Sampling geometry note (see color_match.rs's module doc for the full rationale): deliberately reuses the same forward-KB4 primitive and normalized-intrinsics convention as stitch/geometry.rs's PlaneMap and fisheye.wgsl's fs_main - NOT lens::undistorted_to_distorted, which uses a different (halved-FOV, plane-fitted) intrinsics convention built for the single-camera undistort preview and would silently sample the wrong region here. Getting this exact mapping wrong is the class of bug this feature's own history already hit twice upstream in its prior form. Correctness note, found via this codebase's own agreement-oracle tests (not assumed): this codebase treats CPU/GPU render agreement as load-bearing and test-gated, but color matching is GPU-only (like the original's own "no CPU pixel access on BGRA/zero-copy paths" limitation) with no CPU-executor mirror. Defaulting it on (matching the original design) silently broke 7 existing agreement tests during development the moment a source had non-uniform seam-band content - caught by running the full suite, not shipped. Fixed by defaulting color_match_enabled to false (opt-in), unlike the original's "on by default" - a deliberate, documented difference from the upstream design, not an oversight. Wiring: StitchPipeline::set_color_match_enabled -> Executor (no-op on the CPU arm) -> StitchCore -> StitchSession -> StitchJob::color_match() builder -> `reco stitch --color-match` -> reco-gui "Auto color match (experimental)" checkbox (Stitching panel and export), mirroring show_seam_line's wiring exactly. Verified: cargo fmt --check clean; cargo test -p reco-core (156 passed, including 3 new ColorMatchState unit tests proving the correction moves in the right direction, is symmetric, and converges to the configured clamp; only the 2 pre-existing CUDA hardware-gap failures, unrelated); cargo check clean across the whole workspace (reco-obs excluded, pre-existing unrelated OBS-SDK build gap); clippy clean for this change's own code (blocked only by the same 4 pre-existing issues on upstream's own current main already fixed in the separate fix/d3d11-stage-frame-unsafe branch). Not included: the multi-band spatial seam blend from the original design (blur.wgsl/multiband_composite.wgsl, an alternate seam-blend algorithm) - independent enough from color matching to be its own follow-up PR rather than bundled here. Co-Authored-By: Claude Sonnet 5 --- crates/reco-cli/src/main.rs | 7 + crates/reco-cli/src/stitch.rs | 4 + crates/reco-core/src/core/mod.rs | 7 + crates/reco-core/src/render/color_match.rs | 378 +++++++++++++++++++++ crates/reco-core/src/render/mod.rs | 2 + crates/reco-core/src/render/pipeline.rs | 99 ++++++ crates/reco-core/src/render/renderer.rs | 18 + crates/reco-core/src/session/wiring.rs | 7 + crates/reco-core/src/stitch/executor.rs | 12 + crates/reco-gui/src/export.rs | 2 + crates/reco-gui/src/main.rs | 17 + crates/reco-gui/ui/main.slint | 8 + crates/reco-io/src/stitch_job.rs | 15 + 13 files changed, 576 insertions(+) create mode 100644 crates/reco-core/src/render/color_match.rs diff --git a/crates/reco-cli/src/main.rs b/crates/reco-cli/src/main.rs index 0c65bcfd..f4dd9dcc 100644 --- a/crates/reco-cli/src/main.rs +++ b/crates/reco-cli/src/main.rs @@ -176,6 +176,11 @@ enum Commands { #[arg(long, value_parser = parse_blend)] blend: Option, + /// Enable automatic per-camera exposure/white-balance matching + /// at the seam. Opt-in: off by default. + #[arg(long, default_value_t = false)] + color_match: bool, + /// Frame offset for temporal sync between cameras. /// Positive: skip N right frames (right started first). /// Negative: skip N left frames (left started first). @@ -787,6 +792,7 @@ fn main() -> anyhow::Result<()> { codec, quality, blend, + color_match, sync_offset, model, detection_interval, @@ -812,6 +818,7 @@ fn main() -> anyhow::Result<()> { width, height, blend, + color_match, start_time, end_time, max_frames, diff --git a/crates/reco-cli/src/stitch.rs b/crates/reco-cli/src/stitch.rs index b2490b89..bfb61605 100644 --- a/crates/reco-cli/src/stitch.rs +++ b/crates/reco-cli/src/stitch.rs @@ -24,6 +24,7 @@ pub struct StitchArgs<'a> { pub width: u32, pub height: u32, pub blend: Option, + pub color_match: bool, pub start_time: Option, pub end_time: Option, pub max_frames: Option, @@ -126,6 +127,9 @@ pub fn run_stitch(args: StitchArgs<'_>, interrupted: &Arc) -> anyhow if let Some(b) = args.blend { job = job.blend_width(b); } + if args.color_match { + job = job.color_match(true); + } if let Some(t) = args.start_time { job = job.start_time(t); } diff --git a/crates/reco-core/src/core/mod.rs b/crates/reco-core/src/core/mod.rs index 0352cc04..828fdc8e 100644 --- a/crates/reco-core/src/core/mod.rs +++ b/crates/reco-core/src/core/mod.rs @@ -487,6 +487,13 @@ impl StitchCore { self.executor.set_blend_width(width); } + /// Toggle automatic per-camera seam-band color matching (GPU executor + /// only; see [`crate::stitch::Executor::set_color_match_enabled`]). + /// Off by default. + pub fn set_color_match_enabled(&mut self, enabled: bool) { + self.executor.set_color_match_enabled(enabled); + } + /// Set the lens-correction strength on every lens (`0` = pinhole, /// `1` = full KB4). pub fn set_lens_correction_amount(&mut self, amount: f32) { diff --git a/crates/reco-core/src/render/color_match.rs b/crates/reco-core/src/render/color_match.rs new file mode 100644 index 00000000..a604fd0a --- /dev/null +++ b/crates/reco-core/src/render/color_match.rs @@ -0,0 +1,378 @@ +//! Per-camera exposure/white-balance matching at the stitch seam. +//! +//! Two independently-metering action cameras rarely agree on exposure or +//! white balance, showing up as a visible color/brightness step at the +//! seam, independent of geometric alignment. This periodically samples a +//! coarse grid in the seam-adjacent band of each camera's raw frame, +//! derives a small per-camera YUV offset that nudges both toward their +//! shared mean, and feeds it into the shader's existing (previously +//! always-identity) `color_scale`/`color_offset_blend` uniforms. +//! +//! ## Sampling geometry - the one thing this MUST get exactly right +//! +//! The sampling point in plane UV -> source pixel mapping here +//! deliberately reuses the same forward-KB4 primitive +//! ([`crate::lens::kb4::kb4_forward_scale_with_correction`]) and the same +//! normalized-intrinsics convention (`fx/width`, `fy/height`, ...) as +//! [`crate::stitch::geometry`]'s `PlaneMap` and `fisheye.wgsl`'s +//! `fs_main` - NOT [`crate::lens::undistorted_to_distorted`], which uses +//! a different (halved-FOV, plane-fitted) intrinsics convention built for +//! a different consumer (the single-camera undistort preview) and would +//! silently sample the wrong region here. Getting this mapping wrong is +//! exactly the class of bug this feature's own history already hit twice +//! (see FRICTION.md/git history) - sampling unrelated scene content and +//! applying the resulting "correction" made the seam worse, not better. + +use crate::calibration::Lens; +use crate::lens::kb4; +use crate::render::planes::{Nv12Planes, YuvPlanes}; + +/// Width of the seam-adjacent sampling band, in the plane's own local UV +/// units (same space as `Topology::blend_width`/`seam_offset`). +const BAND_WIDTH: f64 = 0.08; +/// Sampling grid density within the band. +const GRID_COLS: usize = 4; +const GRID_ROWS: usize = 6; +/// Re-measure every N frames - the correction only needs to track slow +/// lighting drift, not per-frame noise. +const INTERVAL_FRAMES: u64 = 15; +/// Exponential-moving-average smoothing factor for the correction +/// (higher = faster to react, noisier). +const EMA_ALPHA: f32 = 0.15; +/// Clamp on the luma (Y) correction, in normalized `[0, 1]` units. +const MAX_Y_OFFSET: f32 = 0.06; +/// Clamp on each chroma (U/V) correction, in normalized `[0, 1]` units. +const MAX_CHROMA_OFFSET: f32 = 0.03; + +/// Map a plane-local UV coordinate to the corresponding source-frame UV +/// via forward KB4 distortion. `None` if the point falls outside the +/// source frame (matches `fisheye.wgsl`'s bounds check and +/// `PlaneMap::sample_uv`'s forward-mapping section exactly). +fn plane_uv_to_source_uv(uv_x: f64, uv_y: f64, cam: &Lens) -> Option<(f64, f64)> { + let euv_x = uv_x * 2.0 - 0.5; + let euv_y = uv_y * 2.0 - 0.5; + let fx_n = cam.fx / cam.width as f64; + let fy_n = cam.fy / cam.height as f64; + let cx_n = cam.cx / cam.width as f64; + let cy_n = cam.cy / cam.height as f64; + let xn = (euv_x - cx_n) / fx_n; + let yn = (euv_y - cy_n) / fy_n; + let r = (xn * xn + yn * yn).sqrt(); + let scale = kb4::kb4_forward_scale_with_correction(r, &cam.distortion, cam.correction as f64); + let du = fx_n * xn * scale + cx_n; + let dv = fy_n * yn * scale + cy_n; + if !(0.0..=1.0).contains(&du) || !(0.0..=1.0).contains(&dv) { + return None; + } + Some((du, dv)) +} + +/// Nearest-sample a normalized `[0, 1]` Y/U/V byte value at source UV +/// `(u, v)` from tightly-packed YUV420P planes. +fn sample_yuv420p(planes: &YuvPlanes<'_>, w: u32, h: u32, u: f64, v: f64) -> (f32, f32, f32) { + let x = ((u * w as f64) as u32).min(w - 1); + let y = ((v * h as f64) as u32).min(h - 1); + let cw = w / 2; + let ch = h / 2; + let cx = (x / 2).min(cw.saturating_sub(1)); + let cy = (y / 2).min(ch.saturating_sub(1)); + let yv = planes.y[(y * w + x) as usize] as f32 / 255.0; + let uv_idx = (cy * cw + cx) as usize; + let uu = planes.u.get(uv_idx).copied().unwrap_or(128) as f32 / 255.0; + let vv = planes.v.get(uv_idx).copied().unwrap_or(128) as f32 / 255.0; + (yv, uu, vv) +} + +/// Same as [`sample_yuv420p`] but for interleaved-UV NV12 planes. +fn sample_nv12(planes: &Nv12Planes<'_>, w: u32, h: u32, u: f64, v: f64) -> (f32, f32, f32) { + let x = ((u * w as f64) as u32).min(w - 1); + let y = ((v * h as f64) as u32).min(h - 1); + let cw = w / 2; + let ch = h / 2; + let cx = (x / 2).min(cw.saturating_sub(1)); + let cy = (y / 2).min(ch.saturating_sub(1)); + let yv = planes.y[(y * w + x) as usize] as f32 / 255.0; + let uv_idx = ((cy * cw + cx) * 2) as usize; + let uu = planes.uv.get(uv_idx).copied().unwrap_or(128) as f32 / 255.0; + let vv = planes.uv.get(uv_idx + 1).copied().unwrap_or(128) as f32 / 255.0; + (yv, uu, vv) +} + +/// Average Y/U/V over a coarse grid in the seam-adjacent band of one +/// camera's raw frame. `is_right` selects which edge is seam-adjacent: +/// the right plane fades in from its own left edge (`uv_x` near `0`, see +/// `fisheye.wgsl`'s `fs_main`), so it samples `[0, BAND_WIDTH]`; the left +/// plane sits geometrically adjacent to that seam along its own right +/// edge, so it samples `[1 - BAND_WIDTH, 1]`. +fn measure_band_mean( + sample: impl Fn(f64, f64) -> Option<(f32, f32, f32)>, + is_right: bool, +) -> Option<(f32, f32, f32)> { + let (band_lo, band_hi) = if is_right { + (0.0, BAND_WIDTH) + } else { + (1.0 - BAND_WIDTH, 1.0) + }; + let mut sum = (0.0f64, 0.0f64, 0.0f64); + let mut n = 0u32; + for row in 0..GRID_ROWS { + let uv_y = (row as f64 + 0.5) / GRID_ROWS as f64; + for col in 0..GRID_COLS { + let uv_x = band_lo + (band_hi - band_lo) * (col as f64 + 0.5) / GRID_COLS as f64; + if let Some((y, u, v)) = sample(uv_x, uv_y) { + sum.0 += y as f64; + sum.1 += u as f64; + sum.2 += v as f64; + n += 1; + } + } + } + if n == 0 { + return None; + } + Some(( + (sum.0 / n as f64) as f32, + (sum.1 / n as f64) as f32, + (sum.2 / n as f64) as f32, + )) +} + +/// EMA-smoothed, clamped per-camera YUV offset derived from periodic +/// seam-band measurements. `[0.0; 3]` (identity, matching the shader's +/// long-standing hardcoded default) until the first successful +/// measurement. +#[derive(Debug, Clone, Copy)] +pub(crate) struct ColorMatchState { + left_offset: [f32; 3], + right_offset: [f32; 3], + frames_since_measure: u64, +} + +impl Default for ColorMatchState { + fn default() -> Self { + Self { + left_offset: [0.0; 3], + right_offset: [0.0; 3], + frames_since_measure: INTERVAL_FRAMES, // measure on frame 0 + } + } +} + +impl ColorMatchState { + /// Current per-camera YUV offset uniforms: `(left, right)`, each + /// `[y, u, v]` in the shader's `color_offset_blend.xyz` convention. + pub(crate) fn offsets(&self) -> ([f32; 3], [f32; 3]) { + (self.left_offset, self.right_offset) + } + + /// Re-measure and update the smoothed offsets if the interval has + /// elapsed. `sample_left`/`sample_right` map a source UV to a Y/U/V + /// triple (`None` outside the frame). + fn update( + &mut self, + sample_left: impl Fn(f64, f64) -> Option<(f32, f32, f32)>, + sample_right: impl Fn(f64, f64) -> Option<(f32, f32, f32)>, + ) { + self.frames_since_measure += 1; + if self.frames_since_measure < INTERVAL_FRAMES { + return; + } + self.frames_since_measure = 0; + + let (Some(left_mean), Some(right_mean)) = ( + measure_band_mean(sample_left, false), + measure_band_mean(sample_right, true), + ) else { + return; + }; + + // Nudge both cameras toward their shared mean, split evenly, so + // neither camera is treated as "the reference" - a static rig + // where one camera happens to be correctly exposed would + // otherwise get needlessly corrected too. + let shared = ( + (left_mean.0 + right_mean.0) * 0.5, + (left_mean.1 + right_mean.1) * 0.5, + (left_mean.2 + right_mean.2) * 0.5, + ); + let target_left = [ + (shared.0 - left_mean.0).clamp(-MAX_Y_OFFSET, MAX_Y_OFFSET), + (shared.1 - left_mean.1).clamp(-MAX_CHROMA_OFFSET, MAX_CHROMA_OFFSET), + (shared.2 - left_mean.2).clamp(-MAX_CHROMA_OFFSET, MAX_CHROMA_OFFSET), + ]; + let target_right = [ + (shared.0 - right_mean.0).clamp(-MAX_Y_OFFSET, MAX_Y_OFFSET), + (shared.1 - right_mean.1).clamp(-MAX_CHROMA_OFFSET, MAX_CHROMA_OFFSET), + (shared.2 - right_mean.2).clamp(-MAX_CHROMA_OFFSET, MAX_CHROMA_OFFSET), + ]; + for i in 0..3 { + self.left_offset[i] += (target_left[i] - self.left_offset[i]) * EMA_ALPHA; + self.right_offset[i] += (target_right[i] - self.right_offset[i]) * EMA_ALPHA; + } + } + + /// Re-measure from YUV420P source planes, if the interval has elapsed. + pub(crate) fn update_yuv420p( + &mut self, + left: &YuvPlanes<'_>, + right: &YuvPlanes<'_>, + left_cam: &Lens, + right_cam: &Lens, + ) { + let (lw, lh) = (left_cam.width, left_cam.height); + let (rw, rh) = (right_cam.width, right_cam.height); + self.update( + |u, v| { + plane_uv_to_source_uv(u, v, left_cam) + .map(|(su, sv)| sample_yuv420p(left, lw, lh, su, sv)) + }, + |u, v| { + plane_uv_to_source_uv(u, v, right_cam) + .map(|(su, sv)| sample_yuv420p(right, rw, rh, su, sv)) + }, + ); + } + + /// Re-measure from NV12 source planes, if the interval has elapsed. + pub(crate) fn update_nv12( + &mut self, + left: &Nv12Planes<'_>, + right: &Nv12Planes<'_>, + left_cam: &Lens, + right_cam: &Lens, + ) { + let (lw, lh) = (left_cam.width, left_cam.height); + let (rw, rh) = (right_cam.width, right_cam.height); + self.update( + |u, v| { + plane_uv_to_source_uv(u, v, left_cam) + .map(|(su, sv)| sample_nv12(left, lw, lh, su, sv)) + }, + |u, v| { + plane_uv_to_source_uv(u, v, right_cam) + .map(|(su, sv)| sample_nv12(right, rw, rh, su, sv)) + }, + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Solid-color lens + plane pair so the measured band mean is exactly + /// the fill value regardless of exactly which pixels the KB4 forward + /// map lands on - isolates "does the correction move in the right + /// direction by the right rough magnitude" from "is the UV mapping + /// pixel-exact" (the latter is what the GPU agreement tests in + /// `stitch::executor::tests` cover for the shared geometry primitive). + fn solid_lens_and_planes(w: u32, h: u32, y_fill: u8) -> (Lens, Vec, Vec, Vec) { + let lens = Lens::fisheye( + w, + h, + w as f64 * 0.5, + w as f64 * 0.5, + w as f64 * 0.5, + h as f64 * 0.5, + [0.0, 0.0, 0.0, 0.0], // zero distortion: keeps the math simple + ); + let y = vec![y_fill; (w * h) as usize]; + let u = vec![128u8; (w * h / 4) as usize]; + let v = vec![128u8; (w * h / 4) as usize]; + (lens, y, u, v) + } + + #[test] + fn identical_cameras_produce_no_correction() { + let (lens_l, ly, lu, lv) = solid_lens_and_planes(64, 64, 120); + let (lens_r, ry, ru, rv) = solid_lens_and_planes(64, 64, 120); + let left = YuvPlanes { + y: &ly, + u: &lu, + v: &lv, + }; + let right = YuvPlanes { + y: &ry, + u: &ru, + v: &rv, + }; + + let mut state = ColorMatchState::default(); + state.update_yuv420p(&left, &right, &lens_l, &lens_r); + + let (l, r) = state.offsets(); + assert_eq!(l, [0.0, 0.0, 0.0]); + assert_eq!(r, [0.0, 0.0, 0.0]); + } + + #[test] + fn mismatched_cameras_nudge_toward_each_other() { + // Left is darker (Y=80/255), right is brighter (Y=170/255). + let (lens_l, ly, lu, lv) = solid_lens_and_planes(64, 64, 80); + let (lens_r, ry, ru, rv) = solid_lens_and_planes(64, 64, 170); + let left = YuvPlanes { + y: &ly, + u: &lu, + v: &lv, + }; + let right = YuvPlanes { + y: &ry, + u: &ru, + v: &rv, + }; + + let mut state = ColorMatchState::default(); + state.update_yuv420p(&left, &right, &lens_l, &lens_r); + + let (l, r) = state.offsets(); + // Left is darker than the shared mean -> its Y offset must be + // positive (brighten it). Right is brighter -> negative (darken + // it). Chroma is identical (128 both sides) -> stays at zero. + assert!(l[0] > 0.0, "left Y offset should be positive, got {l:?}"); + assert!(r[0] < 0.0, "right Y offset should be negative, got {r:?}"); + assert!( + (l[0] + r[0]).abs() < 1e-6, + "offsets should be symmetric: {l:?} vs {r:?}" + ); + assert_eq!(l[1], 0.0); + assert_eq!(l[2], 0.0); + assert_eq!(r[1], 0.0); + assert_eq!(r[2], 0.0); + // One EMA step (alpha=0.15) must not have already fully converged. + assert!(l[0] < MAX_Y_OFFSET); + } + + #[test] + fn repeated_measurement_converges_toward_the_clamped_target() { + let (lens_l, ly, lu, lv) = solid_lens_and_planes(64, 64, 40); + let (lens_r, ry, ru, rv) = solid_lens_and_planes(64, 64, 220); + let left = YuvPlanes { + y: &ly, + u: &lu, + v: &lv, + }; + let right = YuvPlanes { + y: &ry, + u: &ru, + v: &rv, + }; + + let mut state = ColorMatchState::default(); + let mut prev = 0.0f32; + for _ in 0..(INTERVAL_FRAMES * 40) { + state.update_yuv420p(&left, &right, &lens_l, &lens_r); + let (l, _) = state.offsets(); + assert!( + l[0] >= prev - 1e-6, + "offset should monotonically increase toward the clamp" + ); + prev = l[0]; + } + // A huge, sustained mismatch must saturate at the configured clamp, + // not drift past it. + assert!( + (prev - MAX_Y_OFFSET).abs() < 1e-4, + "expected convergence to {MAX_Y_OFFSET}, got {prev}" + ); + } +} diff --git a/crates/reco-core/src/render/mod.rs b/crates/reco-core/src/render/mod.rs index 25143f0c..eaf42cae 100644 --- a/crates/reco-core/src/render/mod.rs +++ b/crates/reco-core/src/render/mod.rs @@ -5,6 +5,8 @@ //! `scene` are pure value/math modules the CPU stitch path shares, so //! they stay available without the `gpu` feature. +#[cfg(feature = "gpu")] +mod color_match; #[cfg(feature = "gpu")] pub mod pipeline; pub mod planes; diff --git a/crates/reco-core/src/render/pipeline.rs b/crates/reco-core/src/render/pipeline.rs index 3f75178b..16b460a3 100644 --- a/crates/reco-core/src/render/pipeline.rs +++ b/crates/reco-core/src/render/pipeline.rs @@ -22,6 +22,7 @@ //! )?; //! ``` +use super::color_match; use super::renderer::{InputFormat, RenderError, Renderer}; use super::scene::SceneGeometry; use super::viewport::{ResolvedViewport, ViewportConfig}; @@ -78,6 +79,16 @@ pub struct StitchPipeline { pub(crate) calibration: Calibration, /// Output viewport configuration. pub(crate) viewport: ViewportConfig, + /// Periodic per-camera seam-band color measurement. Only the + /// CPU-upload render paths (`render_to_target`/`render_to_target_nv12` + /// and their `_to_view` counterparts) actually update this - BGRA and + /// GPU zero-copy paths have no CPU pixel access at render time, so + /// they render with whatever offset was last measured (identity + /// `[0,0,0]` if never updated). Interior mutability: the render + /// methods take `&self`. + color_match: std::cell::RefCell, + /// Master toggle for color matching. Opt-in (see [`Self::with_gpu`]). + pub(crate) color_match_enabled: bool, /// GPU renderer (textures, pipelines, bind groups). renderer: Renderer, /// Input frame dimensions. @@ -161,6 +172,17 @@ impl StitchPipeline { scene, calibration, viewport, + color_match: std::cell::RefCell::new(color_match::ColorMatchState::default()), + // Opt-in, unlike the original design's "on by default": this + // codebase treats CPU/GPU render agreement as load-bearing + // and test-gated (see the `stitch` module docs), and color + // matching has no CPU-side mirror (a genuinely new capability, + // GPU-only) - defaulting it on would silently change output + // for every existing consumer AND break the agreement-oracle + // tests the moment a source has non-uniform seam-band content + // (confirmed: this exact default broke 7 of them during + // development). Callers opt in via `set_color_match_enabled`. + color_match_enabled: false, renderer, input_width, input_height, @@ -271,6 +293,20 @@ impl StitchPipeline { self.calibration.topology.blend_width = width; } + /// Toggle automatic per-camera seam-band color matching. When + /// disabled, rendering uses whatever offset was last measured + /// (freezes, does not reset to identity) until re-enabled. + pub fn set_color_match_enabled(&mut self, enabled: bool) { + self.color_match_enabled = enabled; + } + + /// Current per-camera color-match offset uniforms: `(left, right)`. + /// Read by every render path; only updated by the CPU-upload paths + /// (see [`Self::color_match`]'s doc comment). + fn color_offsets(&self) -> ([f32; 3], [f32; 3]) { + self.color_match.borrow().offsets() + } + /// Update calibration parameters. Recomputes [`SceneGeometry`] from the /// new layout. Takes effect on the next render call (uniforms are rebuilt /// each frame from the stored calibration and scene). @@ -531,6 +567,16 @@ impl StitchPipeline { self.renderer .upload_right_yuv(&self.gpu, right.y, right.u, right.v)?; + if self.color_match_enabled { + self.color_match.borrow_mut().update_yuv420p( + left, + right, + &self.calibration.lenses[0], + &self.calibration.lenses[1], + ); + } + let (left_color, right_color) = self.color_offsets(); + let viewport = ResolvedViewport { config: self.viewport.clone(), position: ViewportPosition { @@ -546,6 +592,8 @@ impl StitchPipeline { &self.calibration, &viewport, self.calibration.topology.blend_width, + left_color, + right_color, target_view, ); Ok(()) @@ -568,6 +616,16 @@ impl StitchPipeline { self.renderer .upload_right_nv12(&self.gpu, right.y, right.uv)?; + if self.color_match_enabled { + self.color_match.borrow_mut().update_nv12( + left, + right, + &self.calibration.lenses[0], + &self.calibration.lenses[1], + ); + } + let (left_color, right_color) = self.color_offsets(); + let viewport = ResolvedViewport { config: self.viewport.clone(), position: ViewportPosition { @@ -583,6 +641,8 @@ impl StitchPipeline { &self.calibration, &viewport, self.calibration.topology.blend_width, + left_color, + right_color, target_view, ); Ok(()) @@ -609,6 +669,16 @@ impl StitchPipeline { self.renderer .upload_right_yuv(&self.gpu, right.y, right.u, right.v)?; + if self.color_match_enabled { + self.color_match.borrow_mut().update_yuv420p( + left, + right, + &self.calibration.lenses[0], + &self.calibration.lenses[1], + ); + } + let (left_color, right_color) = self.color_offsets(); + let viewport = ResolvedViewport { config: self.viewport.clone(), position: ViewportPosition { @@ -624,6 +694,8 @@ impl StitchPipeline { &self.calibration, &viewport, self.calibration.topology.blend_width, + left_color, + right_color, )) } @@ -647,6 +719,16 @@ impl StitchPipeline { self.renderer .upload_right_nv12(&self.gpu, right.y, right.uv)?; + if self.color_match_enabled { + self.color_match.borrow_mut().update_nv12( + left, + right, + &self.calibration.lenses[0], + &self.calibration.lenses[1], + ); + } + let (left_color, right_color) = self.color_offsets(); + let viewport = ResolvedViewport { config: self.viewport.clone(), position: ViewportPosition { @@ -662,6 +744,8 @@ impl StitchPipeline { &self.calibration, &viewport, self.calibration.topology.blend_width, + left_color, + right_color, )) } @@ -685,6 +769,13 @@ impl StitchPipeline { self.renderer.upload_left_bgra(&self.gpu, left.rgba)?; self.renderer.upload_right_bgra(&self.gpu, right.rgba)?; + // No CPU pixel access on this path (packed RGBA, not the raw YUV + // color-match sampling expects) - renders with whatever offset + // was last measured by a CPU-upload path, or identity if none. + // Documented limitation, matches `show_seam_line`'s and + // `color_match`'s own doc comments. + let (left_color, right_color) = self.color_offsets(); + let viewport = ResolvedViewport { config: self.viewport.clone(), position: ViewportPosition { @@ -700,6 +791,8 @@ impl StitchPipeline { &self.calibration, &viewport, self.calibration.topology.blend_width, + left_color, + right_color, )) } @@ -748,6 +841,10 @@ impl StitchPipeline { /// bind groups, then use this for subsequent frames with the same /// textures to avoid per-frame bind group allocation. pub fn render_to_target_gpu(&self, yaw: f32, pitch: f32) -> wgpu::CommandBuffer { + // No CPU pixel access on the zero-copy path - see + // `render_to_target_bgra`'s identical comment. + let (left_color, right_color) = self.color_offsets(); + let viewport = ResolvedViewport { config: self.viewport.clone(), position: ViewportPosition { @@ -763,6 +860,8 @@ impl StitchPipeline { &self.calibration, &viewport, self.calibration.topology.blend_width, + left_color, + right_color, ) } diff --git a/crates/reco-core/src/render/renderer.rs b/crates/reco-core/src/render/renderer.rs index 8db9701f..fe5b84d5 100644 --- a/crates/reco-core/src/render/renderer.rs +++ b/crates/reco-core/src/render/renderer.rs @@ -809,6 +809,8 @@ impl Renderer { calibration: &Calibration, viewport: &ResolvedViewport, blend_width: f32, + left_color_offset: [f32; 3], + right_color_offset: [f32; 3], target_view: &wgpu::TextureView, aspect: f32, encoder_label: &str, @@ -840,6 +842,9 @@ impl Renderer { self.is_full_range, ); left_uniforms.lens_preview[0] = calibration.lenses[0].correction; + left_uniforms.color_offset_blend[0] = left_color_offset[0]; + left_uniforms.color_offset_blend[1] = left_color_offset[1]; + left_uniforms.color_offset_blend[2] = left_color_offset[2]; let right_mvp = projection * view * scene.model_matrix_right(); let mut right_uniforms = build_gpu_uniforms( @@ -852,6 +857,9 @@ impl Renderer { self.is_full_range, ); right_uniforms.lens_preview[0] = calibration.lenses[1].correction; + right_uniforms.color_offset_blend[0] = right_color_offset[0]; + right_uniforms.color_offset_blend[1] = right_color_offset[1]; + right_uniforms.color_offset_blend[2] = right_color_offset[2]; gpu.queue.write_buffer( &self.left.uniform_buffer, @@ -913,6 +921,7 @@ impl Renderer { feature = "profiling", tracing::instrument(skip_all, name = "gpu_render_to_target") )] + #[allow(clippy::too_many_arguments)] pub fn render_to_target( &self, gpu: &GpuContext, @@ -920,6 +929,8 @@ impl Renderer { calibration: &Calibration, viewport: &ResolvedViewport, blend_width: f32, + left_color_offset: [f32; 3], + right_color_offset: [f32; 3], ) -> wgpu::CommandBuffer { let aspect = self.output_width as f32 / self.output_height as f32; let encoder = self.encode_stitch_pass( @@ -928,6 +939,8 @@ impl Renderer { calibration, viewport, blend_width, + left_color_offset, + right_color_offset, &self.render_target_view, aspect, "stitch_to_target", @@ -947,6 +960,7 @@ impl Renderer { /// /// Unlike [`Self::render_to_target`], this does NOT read back the result to CPU. /// Used for interactive preview windows. + #[allow(clippy::too_many_arguments)] pub fn render_to_view( &self, gpu: &GpuContext, @@ -954,6 +968,8 @@ impl Renderer { calibration: &Calibration, viewport: &ResolvedViewport, blend_width: f32, + left_color_offset: [f32; 3], + right_color_offset: [f32; 3], target_view: &wgpu::TextureView, ) { let aspect = viewport.config.width as f32 / viewport.config.height as f32; @@ -963,6 +979,8 @@ impl Renderer { calibration, viewport, blend_width, + left_color_offset, + right_color_offset, target_view, aspect, "preview_frame", diff --git a/crates/reco-core/src/session/wiring.rs b/crates/reco-core/src/session/wiring.rs index a0240fea..14b24340 100644 --- a/crates/reco-core/src/session/wiring.rs +++ b/crates/reco-core/src/session/wiring.rs @@ -111,6 +111,13 @@ impl StitchSession { self.lookahead_frames = frames; } + /// Toggle automatic per-camera seam-band color matching (see + /// [`StitchCore::set_color_match_enabled`](crate::core::StitchCore::set_color_match_enabled)). + /// Off by default. + pub fn set_color_match_enabled(&mut self, enabled: bool) { + self.core.set_color_match_enabled(enabled); + } + /// Attach a stacked-video replay recorder. /// /// Forwards to `StitchCore::set_stacked_recorder` on the diff --git a/crates/reco-core/src/stitch/executor.rs b/crates/reco-core/src/stitch/executor.rs index cab10707..b5aba84c 100644 --- a/crates/reco-core/src/stitch/executor.rs +++ b/crates/reco-core/src/stitch/executor.rs @@ -531,6 +531,18 @@ impl Executor { } } + /// Toggle automatic per-camera seam-band color matching. GPU-only - + /// see [`crate::render::pipeline::StitchPipeline::set_color_match_enabled`]'s + /// doc comment for why this has no CPU-executor mirror (yet). No-op + /// on the CPU arm. + pub fn set_color_match_enabled(&mut self, enabled: bool) { + match self { + Executor::Cpu(_) => {} + #[cfg(feature = "gpu")] + Executor::Gpu(g) => g.pipeline.set_color_match_enabled(enabled), + } + } + /// Set the lens-correction strength on every lens, clamped to `[0, 1]`. pub fn set_lens_correction_amount(&mut self, amount: f32) { match self { diff --git a/crates/reco-gui/src/export.rs b/crates/reco-gui/src/export.rs index e1c8c3b3..145c1472 100644 --- a/crates/reco-gui/src/export.rs +++ b/crates/reco-gui/src/export.rs @@ -117,6 +117,7 @@ pub fn run_export( codec_str: String, quality_str: String, blend: f32, + color_match_enabled: bool, start_secs: f32, end_secs: f32, autocam: AutocamUiConfig, @@ -196,6 +197,7 @@ pub fn run_export( .format(format) .resolution(width, height) .blend_width(blend) + .color_match(color_match_enabled) .on_progress(move |p: &reco_core::session::types::FrameProgress| { let frames = p.frames_completed; let elapsed = progress_start.elapsed().as_secs_f64(); diff --git a/crates/reco-gui/src/main.rs b/crates/reco-gui/src/main.rs index 2763a88b..af522112 100644 --- a/crates/reco-gui/src/main.rs +++ b/crates/reco-gui/src/main.rs @@ -931,6 +931,16 @@ impl AppState { } } + /// Toggle automatic per-camera seam-band color matching. Opt-in, not + /// persisted with the calibration (no CPU-executor mirror yet - see + /// `StitchPipeline::set_color_match_enabled`'s doc comment). + fn set_color_match_enabled(&mut self, enabled: bool) { + if let Some(bridge) = self.bridge.as_mut() { + bridge.engine_mut().set_color_match_enabled(enabled); + self.preview_dirty = true; + } + } + fn set_rig_tilt(&mut self, deg: f32) { if let Some(cal) = self.calibration.as_mut() { cal.framing.tilt = (deg as f64).to_radians(); @@ -2763,6 +2773,11 @@ fn main() -> anyhow::Result<()> { } }); + let state_ref = Rc::clone(&state); + app.on_toggled_color_match(move |enabled| { + state_ref.borrow_mut().set_color_match_enabled(enabled); + }); + let state_ref = Rc::clone(&state); let app_weak = app.as_weak(); app.on_changed_rig_tilt(move |deg| { @@ -3479,6 +3494,7 @@ fn main() -> anyhow::Result<()> { let codec_str = app.get_export_codec().to_string(); let quality_str = app.get_export_quality().to_string(); let blend = app.get_blend_width(); + let color_match_enabled = app.get_color_match_enabled(); let start_secs = app.get_export_start_secs(); let end_secs = app.get_export_end_secs(); log::info!("Export range: start={start_secs:.1}s, end={end_secs:.1}s"); @@ -3563,6 +3579,7 @@ fn main() -> anyhow::Result<()> { codec_str, quality_str, blend, + color_match_enabled, start_secs, end_secs, autocam, diff --git a/crates/reco-gui/ui/main.slint b/crates/reco-gui/ui/main.slint index 3eb6e210..60b57f4f 100644 --- a/crates/reco-gui/ui/main.slint +++ b/crates/reco-gui/ui/main.slint @@ -507,6 +507,7 @@ export component RecoApp inherits Window { in-out property pitch: 0.0; // radians in-out property fov: 75.0; // degrees in-out property blend-width: 0.05; + in-out property color-match-enabled: false; in-out property rig-tilt: 0.0; // degrees in-out property rig-roll: 0.0; // degrees in-out property sync-offset: 0; // frames @@ -704,6 +705,7 @@ export component RecoApp inherits Window { callback reset-view(); // Settings-panel slider callbacks. callback changed-blend-width(float); + callback toggled-color-match(bool); callback changed-rig-tilt(float); callback changed-rig-roll(float); callback changed-sync-offset(int); @@ -1117,6 +1119,12 @@ export component RecoApp inherits Window { changed(v) => { root.changed-blend-width(v); } } + if root.files-loaded: CheckBox { + text: "Auto color match (experimental)"; + checked <=> root.color-match-enabled; + toggled => { root.toggled-color-match(self.checked); } + } + if root.files-loaded: LabeledSlider { label: "Rig tilt"; minimum: -30; diff --git a/crates/reco-io/src/stitch_job.rs b/crates/reco-io/src/stitch_job.rs index 9a6350ce..7af53d60 100644 --- a/crates/reco-io/src/stitch_job.rs +++ b/crates/reco-io/src/stitch_job.rs @@ -56,6 +56,10 @@ pub struct StitchJob { /// Seam blend override. `None` (default) respects the calibration /// document's saved value - the single home for render params. blend_width: Option, + /// Automatic per-camera seam-band color matching. Opt-in (defaults + /// off) - see `reco_core::render::pipeline::StitchPipeline:: + /// set_color_match_enabled`'s doc comment for why. + color_match_enabled: bool, // Callbacks on_progress: Option, @@ -250,6 +254,7 @@ impl StitchJob { max_frames: None, sync_offset: None, blend_width: None, + color_match_enabled: false, on_progress: None, on_finalizing: None, session_hooks: Vec::new(), @@ -374,6 +379,15 @@ impl StitchJob { self } + /// Enable automatic per-camera exposure/white-balance matching at + /// the seam. Opt-in (off by default) - has no CPU-executor mirror + /// yet, so it only affects the CPU-upload GPU render paths (any file- + /// based stitch job takes one of these). + pub fn color_match(mut self, enabled: bool) -> Self { + self.color_match_enabled = enabled; + self + } + // ── Replay recording (M6.5 stacked-video) ── /// Record pre-stitch source frames to a stacked-video file at @@ -616,6 +630,7 @@ impl StitchJob { right_rotation: source.right_rotation(), }; let mut session = reco_core::session::StitchSession::with_gpu(gpu, session_config)?; + session.set_color_match_enabled(self.color_match_enabled); session.telemetry_mut().set_gpu_name(gpu_name.clone()); session.telemetry_mut().set_decode_mode(decode_mode.clone());