diff --git a/crates/reco-cli/src/main.rs b/crates/reco-cli/src/main.rs index 0c65bcfde..ed14d9a30 100644 --- a/crates/reco-cli/src/main.rs +++ b/crates/reco-cli/src/main.rs @@ -202,6 +202,15 @@ enum Commands { #[arg(long, default_value_t = 1.5)] lookahead: f64, + /// Downconvert the lookahead pool to 8-bit before buffering, + /// roughly halving its VRAM cost on 10-bit sources (e.g. DJI + /// Action 4 HEVC) - use this if export fails with a "not enough + /// VRAM for a Ns lookahead" error. No effect on 8-bit sources. + /// The same buffered frames feed the final render too, so this + /// trades a little gradient smoothness for memory. + #[arg(long)] + lookahead_reduced_bit_depth: bool, + /// Tracking mode: "field" (ball + players, default), "ball" /// (ball only), "sweep" (no AI, debug pan). field is robust with /// COCO models; ball-only follows the weak COCO ball alone. @@ -791,6 +800,7 @@ fn main() -> anyhow::Result<()> { model, detection_interval, lookahead, + lookahead_reduced_bit_depth, tracking, quality_value, preset, @@ -822,6 +832,7 @@ fn main() -> anyhow::Result<()> { model_path: model.as_deref(), detection_interval, lookahead, + lookahead_reduced_bit_depth, tracking_mode: &tracking, quality_value, preset, diff --git a/crates/reco-cli/src/stitch.rs b/crates/reco-cli/src/stitch.rs index b2490b89c..2a49052c7 100644 --- a/crates/reco-cli/src/stitch.rs +++ b/crates/reco-cli/src/stitch.rs @@ -34,6 +34,7 @@ pub struct StitchArgs<'a> { pub model_path: Option<&'a str>, pub detection_interval: u64, pub lookahead: f64, + pub lookahead_reduced_bit_depth: bool, pub tracking_mode: &'a str, pub quality_value: Option, pub preset: Option, @@ -153,6 +154,12 @@ pub fn run_stitch(args: StitchArgs<'_>, interrupted: &Arc) -> anyhow "Lookahead: {:.1}s buffer enabled (AI tracking active)", args.lookahead ); + if args.lookahead_reduced_bit_depth { + job = job.lookahead_reduced_bit_depth(true); + log::info!( + "Lookahead bit depth: reduced to 8-bit (halved VRAM, 10-bit sources only)" + ); + } } else { log::debug!( "Lookahead {:.1}s ignored: no AI tracking (needs --model, non-sweep); \ diff --git a/crates/reco-core/src/interop/d3d11.rs b/crates/reco-core/src/interop/d3d11.rs index 4b6f73138..e07f7fdea 100644 --- a/crates/reco-core/src/interop/d3d11.rs +++ b/crates/reco-core/src/interop/d3d11.rs @@ -80,13 +80,32 @@ struct StagingState { device: ID3D11Device, context: ID3D11DeviceContext, staging: Vec, - _wgpu_textures: Vec, + /// The imported multi-planar (NV12/P010) wgpu texture per slot. Read + /// via [`D3d11StagingPool::plane_source`] for a plane-aspect-selected + /// `copy_texture_to_texture` into a same-format destination (see + /// `session::vram_pool::VramPool::copy_from_d3d11`) - `y_views`/ + /// `uv_views` below are already aspect-selected and cannot themselves + /// be the source of a texture-to-texture copy (that needs the raw + /// texture + an aspect, not a view). + wgpu_textures: Vec, y_views: Vec, uv_views: Vec, event_query: ID3D11Query, cuda_nv12: Option>, } +/// Borrowed handles for one D3D11 staging slot - see +/// [`D3d11StagingPool::plane_source`]. +pub struct D3d11PlaneSource<'a> { + /// The raw multi-planar (NV12/P010) imported texture. Select + /// `TextureAspect::Plane0` (Y) / `Plane1` (UV) for a same-format copy. + pub texture: &'a wgpu::Texture, + /// Pre-built Y plane view (`Plane0`), for sampling in a render pass. + pub y_view: &'a wgpu::TextureView, + /// Pre-built UV plane view (`Plane1`), for sampling in a render pass. + pub uv_view: &'a wgpu::TextureView, +} + /// Double-buffered NV12 staging pool for D3D11VA -> wgpu zero-copy. /// /// Allocates 4 staging textures (2 per camera, ping-pong) with shared @@ -360,7 +379,7 @@ impl D3d11StagingPool { device, context, staging: staging_textures, - _wgpu_textures: wgpu_textures, + wgpu_textures, y_views, uv_views, cuda_nv12, @@ -492,6 +511,28 @@ impl D3d11StagingPool { .uv_views[slot] } + /// Borrowed handles for one staging slot, for copying it into a + /// longer-lived pool (`session::vram_pool::VramPool::copy_from_d3d11`). + /// + /// Includes both the raw multi-planar `texture` (for a plane-aspect- + /// selected `copy_texture_to_texture` when the destination format + /// matches this pool's `pixel_format` - bit-exact, no shader) and the + /// pre-built plane views (for a downconvert render pass when the + /// destination is a different, lower bit depth - a raw copy cannot + /// change bit depth, and a render pass only needs a view, not the + /// aspect-selected texture). + /// + /// # Panics + /// Panics if called before the first `stage_frame` (pool not initialized). + pub fn plane_source(&self, slot: usize) -> D3d11PlaneSource<'_> { + let state = self.state.as_ref().expect("staging pool not initialized"); + D3d11PlaneSource { + texture: &state.wgpu_textures[slot], + y_view: &state.y_views[slot], + uv_view: &state.uv_views[slot], + } + } + /// Number of staging slots. pub fn n_slots(&self) -> usize { self.n_slots diff --git a/crates/reco-core/src/render/lookahead_downconvert.rs b/crates/reco-core/src/render/lookahead_downconvert.rs new file mode 100644 index 000000000..9adf4dad8 --- /dev/null +++ b/crates/reco-core/src/render/lookahead_downconvert.rs @@ -0,0 +1,313 @@ +//! GPU-resident bit-depth downconversion for the lookahead pool. +//! +//! See `shaders/lookahead_downconvert.wgsl` for the "why" (lookahead pool +//! VRAM cost scales with source bit depth) and the shader itself. This +//! module owns the two small render pipelines (one per plane, since the Y +//! and UV planes differ in channel count and resolution) and the single +//! entry point, [`LookaheadDownconverter::convert_plane`], that runs one. +//! +//! Opt-in, off by default - see [`crate::session::vram_pool::LookaheadBitDepth`]. +//! Currently wired into [`crate::session::vram_pool::VramPool`] (Linux/ +//! macOS). Not yet wired into the Windows `D3d11StagingPool` path - see +//! `FRICTION.md` "Lookahead pool VRAM cost scales with source bit depth" +//! for why that side needs separate, careful follow-up (the pool there +//! also feeds CUDA-resident AI detection via D3D11 shared-handle import, +//! which this pass's plain wgpu-native destination textures don't +//! support without additional interop work). + +use crate::gpu::GpuContext; + +/// Fullscreen-triangle render pipelines that copy one plane's pixel values +/// from a source texture into a same-resolution, lower-bit-depth +/// destination texture (e.g. `R16Unorm` -> `R8Unorm` for a Y plane, or +/// `Rg16Unorm` -> `Rg8Unorm` for a UV plane). +/// +/// Two pipelines are needed (not one) because a wgpu render pipeline is +/// tied to a fixed output format at creation time, and the Y/UV planes use +/// different formats (single- vs dual-channel). Both pipelines share the +/// same shader module and bind group layout - only the color target format +/// differs. +pub struct LookaheadDownconverter { + bind_group_layout: wgpu::BindGroupLayout, + y_pipeline: wgpu::RenderPipeline, + uv_pipeline: wgpu::RenderPipeline, +} + +impl LookaheadDownconverter { + /// Build the downconversion pipelines. + /// + /// Cheap one-time setup (two small pipeline objects); construct once + /// per `GpuContext` and reuse across frames, mirroring how + /// [`crate::render::renderer::Renderer`] owns its pipelines. + pub fn new(gpu: &GpuContext) -> Self { + let device = &gpu.device; + + let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("lookahead_downconvert"), + source: wgpu::ShaderSource::Wgsl( + include_str!("../shaders/lookahead_downconvert.wgsl").into(), + ), + }); + + // Non-filterable: the shader uses textureLoad (exact texel copy, + // same resolution in and out), not textureSample, so no sampler is + // bound and the source format does not need to support linear + // filtering (16-bit Unorm formats are not guaranteed filterable on + // every wgpu backend). + let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("lookahead_downconvert_bind_group_layout"), + entries: &[wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + sample_type: wgpu::TextureSampleType::Float { filterable: false }, + view_dimension: wgpu::TextureViewDimension::D2, + multisampled: false, + }, + count: None, + }], + }); + + let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("lookahead_downconvert_pipeline_layout"), + bind_group_layouts: &[&bind_group_layout], + immediate_size: 0, + }); + + let make_pipeline = |label: &str, format: wgpu::TextureFormat| { + device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some(label), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &shader, + entry_point: Some("vs_main"), + compilation_options: Default::default(), + buffers: &[], + }, + fragment: Some(wgpu::FragmentState { + module: &shader, + entry_point: Some("fs_main"), + compilation_options: Default::default(), + targets: &[Some(wgpu::ColorTargetState { + format, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + }), + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleList, + ..Default::default() + }, + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview_mask: None, + cache: None, + }) + }; + + let y_pipeline = make_pipeline("lookahead_downconvert_y", wgpu::TextureFormat::R8Unorm); + let uv_pipeline = make_pipeline("lookahead_downconvert_uv", wgpu::TextureFormat::Rg8Unorm); + + Self { + bind_group_layout, + y_pipeline, + uv_pipeline, + } + } + + /// Downconvert one plane: `src` is sampled with `textureLoad` and its + /// values written into `dst` at `dst`'s (lower) bit depth. + /// + /// `dst` must be `R8Unorm` when `is_uv` is `false` (Y plane) or + /// `Rg8Unorm` when `is_uv` is `true` (UV plane), with + /// `RENDER_ATTACHMENT` usage, and the same width/height as `src`. `src` + /// must have `TEXTURE_BINDING` usage. Appends one render pass to + /// `encoder`; does not submit. + pub fn convert_plane( + &self, + device: &wgpu::Device, + encoder: &mut wgpu::CommandEncoder, + src: &wgpu::TextureView, + dst: &wgpu::TextureView, + is_uv: bool, + ) { + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("lookahead_downconvert_bind_group"), + layout: &self.bind_group_layout, + entries: &[wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(src), + }], + }); + + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("lookahead_downconvert_pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: dst, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::BLACK), + store: wgpu::StoreOp::Store, + }, + depth_slice: None, + })], + depth_stencil_attachment: None, + occlusion_query_set: None, + timestamp_writes: None, + multiview_mask: None, + }); + pass.set_pipeline(if is_uv { + &self.uv_pipeline + } else { + &self.y_pipeline + }); + pass.set_bind_group(0, &bind_group, &[]); + pass.draw(0..3, 0..1); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// End-to-end correctness check on a real (or software) adapter: known + /// 16-bit values in a source texture must round-trip through the + /// downconvert pass to the expected 8-bit values, within the rounding + /// wgpu's own hardware Unorm normalization introduces at each end + /// (1 LSB in the 8-bit output, i.e. +/-1/255). + /// + /// Runs against whatever adapter `GpuContext::new` selects in this + /// environment (falls back to software rendering under CI/headless + /// conditions per wgpu's normal adapter selection) - skips instead of + /// failing if no adapter is available at all, matching this crate's + /// other GPU-dependent tests (see `interop::cuda::tests`). + #[test] + fn y_plane_downconvert_matches_expected_8bit_values() { + let gpu = match pollster::block_on(GpuContext::new()) { + Ok(gpu) => gpu, + Err(e) => { + eprintln!("skipping: no GPU adapter available ({e})"); + return; + } + }; + let downconverter = LookaheadDownconverter::new(&gpu); + + const W: u32 = 4; + const H: u32 = 1; + // R16Unorm source values chosen to land on exact 8-bit boundaries + // after normalization: 0x0000 -> 0, 0x8080 -> 128, 0xFFFF -> 255, + // 0x4040 -> 64 (0x4040 / 0xFFFF * 255 = 64.0 exactly). + let src_u16: [u16; W as usize] = [0x0000, 0x8080, 0xFFFF, 0x4040]; + + let src_texture = gpu.device.create_texture(&wgpu::TextureDescriptor { + label: Some("test_src_y"), + size: wgpu::Extent3d { + width: W, + height: H, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::R16Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + gpu.queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture: &src_texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + bytemuck::cast_slice(&src_u16), + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(W * 2), + rows_per_image: Some(H), + }, + wgpu::Extent3d { + width: W, + height: H, + depth_or_array_layers: 1, + }, + ); + + let dst_texture = gpu.device.create_texture(&wgpu::TextureDescriptor { + label: Some("test_dst_y"), + size: wgpu::Extent3d { + width: W, + height: H, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::R8Unorm, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }); + + let src_view = src_texture.create_view(&wgpu::TextureViewDescriptor::default()); + let dst_view = dst_texture.create_view(&wgpu::TextureViewDescriptor::default()); + + let mut encoder = gpu + .device + .create_command_encoder(&wgpu::CommandEncoderDescriptor { label: None }); + downconverter.convert_plane(&gpu.device, &mut encoder, &src_view, &dst_view, false); + + // Readback: copy to a mapped buffer. + let bytes_per_row = 256; // wgpu COPY_BYTES_PER_ROW_ALIGNMENT + let readback = gpu.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("test_readback"), + size: (bytes_per_row * H) as u64, + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }); + encoder.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { + texture: &dst_texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyBufferInfo { + buffer: &readback, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(bytes_per_row), + rows_per_image: Some(H), + }, + }, + wgpu::Extent3d { + width: W, + height: H, + depth_or_array_layers: 1, + }, + ); + gpu.queue.submit(std::iter::once(encoder.finish())); + + let (tx, rx) = std::sync::mpsc::channel(); + let slice = readback.slice(..); + slice.map_async(wgpu::MapMode::Read, move |result| { + let _ = tx.send(result); + }); + gpu.device + .poll(wgpu::PollType::wait_indefinitely()) + .expect("device poll failed"); + rx.recv() + .expect("map_async never signaled") + .expect("buffer map failed"); + + let data = slice.get_mapped_range(); + let got = [data[0], data[1], data[2], data[3]]; + drop(data); + readback.unmap(); + + assert_eq!( + got, + [0, 128, 255, 64], + "unexpected downconverted Y values: {got:?}" + ); + } +} diff --git a/crates/reco-core/src/render/mod.rs b/crates/reco-core/src/render/mod.rs index 25143f0c6..7a37f03eb 100644 --- a/crates/reco-core/src/render/mod.rs +++ b/crates/reco-core/src/render/mod.rs @@ -5,6 +5,8 @@ //! `scene` are pure value/math modules the CPU stitch path shares, so //! they stay available without the `gpu` feature. +#[cfg(feature = "gpu")] +pub mod lookahead_downconvert; #[cfg(feature = "gpu")] pub mod pipeline; pub mod planes; diff --git a/crates/reco-core/src/session/frame_processing.rs b/crates/reco-core/src/session/frame_processing.rs index 8e4168810..b4fba4f7b 100644 --- a/crates/reco-core/src/session/frame_processing.rs +++ b/crates/reco-core/src/session/frame_processing.rs @@ -210,15 +210,25 @@ impl StitchSession { right_texture, right_slice, } => { - if let Some(staging_slot) = self.current_vram_slot { - // Buffered path: staging was done during produce. - // Render from the pre-staged slot. - self.render_d3d11_from_slot( - staging_slot, - staging_slot + 1, - pos.yaw, - pos.pitch, - )?; + if let Some(vram_idx) = self.current_vram_slot { + // Buffered path: the frame was already staged and + // copied into the VramPool during produce (see + // `copy_to_vram_pool_platform`). Render from the pool's + // bind groups - mirrors Linux's `render_gpu_resident` + // buffered-path branch exactly (same pool type, same + // bind-group shape, just reached via a different + // producer). + let pool = self + .vram_pool + .as_ref() + .expect("vram_pool must exist when current_vram_slot is set"); + let left_bg = pool.left_bind_group(vram_idx); + let right_bg = pool.right_bind_group(vram_idx); + let render_buf = self + .core + .pipeline_mut() + .render_with_bind_groups(left_bg, right_bg, pos.yaw, pos.pitch); + self.submit_render_output(render_buf)?; } else { // Immediate path: stage and render now. let _ = self @@ -542,23 +552,26 @@ impl StitchSession { if first_frame { let (w, h) = self.core.source_info(); let needs_cuda = self.core.detector_needs_cuda_frames(); - // For lookahead, size slots to the max frames simultaneously - // in flight (decoded but not yet rendered), x2 for left+right. - // Peak occupancy is n + post_smooth_half + 1 (buffer hits n+1 - // right after a produce while the pose queue holds - // post_smooth_half). Slots are assigned by produce_index modulo - // n_slots with no occupancy check, so the pool must exceed peak - // occupancy or a producer would overwrite a frame still queued - // for render. +4 keeps a few frames of slack above the exact - // fit (the Linux VramPool uses ref-counted acquire/release; this - // path relies on the sizing margin instead). Without lookahead, - // 4 slots (double-buffered stereo) suffice. - let n_slots = if self.lookahead_frames > 0 { - let post_smooth_half = (self.lookahead_frames / 2).max(1); - (self.lookahead_frames + post_smooth_half + 4) * 2 - } else { - 4 - }; + // This pool only bridges the D3D11 shared-handle import: each + // staged frame is read (by detection, and by `copy_from_d3d11` + // into the long-lived `VramPool`) and explicitly polled to + // completion before the next produce reuses the same slot (see + // `copy_to_vram_pool_platform` below) - so a small, fixed, + // double-buffered-stereo pool suffices regardless of lookahead + // depth. The long-lived buffering that used to size this pool + // to the full lookahead window now lives in `VramPool` instead + // (mirrors Linux/macOS, where the D3D11-equivalent handoff is + // similarly short-lived). + // + // CUDA caveat: `needs_cuda`/`cuda_nv12_ptrs()` is currently + // dead code on Windows (nothing calls `cuda_nv12_ptrs`, see + // that method's doc comment) - if a future Windows CUDA + // detection path is wired up, it MUST add its own explicit + // `cuStreamSynchronize` (or equivalent) before this pool's next + // `stage_frame` reuses the same slot, since a small pool no + // longer provides the large implicit margin the old + // lookahead-scaled sizing gave for free. + let n_slots = 4; match crate::interop::d3d11::D3d11StagingPool::new( self.core.gpu(), w, @@ -590,27 +603,6 @@ impl StitchSession { Ok(first_frame) } - /// Render from specific D3D11 staging slots (buffered lookahead path). - #[cfg(target_os = "windows")] - fn render_d3d11_from_slot( - &mut self, - left_slot: usize, - right_slot: usize, - yaw: f32, - pitch: f32, - ) -> Result<(), SessionError> { - let pool = self.d3d11_staging_pool.as_ref().unwrap(); - let render_buf = self.core.render_imported_views_at_pose( - pool.y_view(left_slot), - pool.uv_view(left_slot), - pool.y_view(right_slot), - pool.uv_view(right_slot), - yaw, - pitch, - ); - self.submit_render_output(render_buf) - } - /// Render from already-staged D3D11VA views (immediate path). #[cfg(target_os = "windows")] fn render_d3d11_staged(&mut self, yaw: f32, pitch: f32) -> Result<(), SessionError> { @@ -883,6 +875,13 @@ impl StitchSession { Ok(Some(slot)) } + /// Stage a D3D11VA frame and copy it into the long-lived `VramPool`. + /// + /// Returns the `VramPool` slot (not the D3D11 staging slot - that pool + /// is now just a short-lived bridge, see `stage_d3d11_frames`'s doc + /// comment). Mirrors the Linux/macOS platform arms of this method: + /// stage into a small transient buffer, immediately copy into the + /// pool, return the pool's slot index. #[cfg(target_os = "windows")] fn copy_to_vram_pool_platform( &mut self, @@ -906,7 +905,34 @@ impl StitchSession { let right_slot = (produce_index as usize * 2 + 1) % pool.n_slots(); pool.stage_frame(*left_texture, *left_slice, left_slot)?; pool.stage_frame(*right_texture, *right_slice, right_slot)?; - return Ok(Some(left_slot)); + + if self.vram_pool.is_none() { + return Ok(Some(left_slot)); + } + let d3d11_pool = self.d3d11_staging_pool.as_ref().unwrap(); + let src_left = d3d11_pool.plane_source(left_slot); + let src_right = d3d11_pool.plane_source(right_slot); + let vram_pool = self.vram_pool.as_mut().unwrap(); + let vram_slot = vram_pool.acquire().ok_or_else(|| { + SessionError::Config(format!( + "VRAM pool exhausted ({} slots, {} available)", + vram_pool.capacity(), + vram_pool.available() + )) + })?; + let gpu = self.core.pipeline().gpu(); + vram_pool.copy_from_d3d11(gpu, vram_slot, src_left, src_right); + // Explicit poll before returning: the D3D11 staging slots just + // used are reused (overwritten via CopySubresourceRegion) by a + // future produce as soon as this call returns. That D3D11-side + // write has no automatic ordering against this wgpu-side read + // of the same NT-shared-handle memory (D3D11 and DX12/wgpu are + // different device/queue objects - shared VRAM does not imply + // shared scheduling). Mirrors the same pattern already used + // after every Linux/macOS `copy_from_textures` call for the + // identical reason (see `copy_nvmm_to_vram_pool` above). + let _ = gpu.device.poll(wgpu::PollType::wait_indefinitely()); + return Ok(Some(vram_slot)); } Ok(None) } diff --git a/crates/reco-core/src/session/mod.rs b/crates/reco-core/src/session/mod.rs index 84059ad36..6db0c23c7 100644 --- a/crates/reco-core/src/session/mod.rs +++ b/crates/reco-core/src/session/mod.rs @@ -31,7 +31,7 @@ mod run_loop; pub(crate) mod vram_pool; /// Lookahead VRAM fit estimate, for the export UI risk slider and the /// pre-flight budget check. -pub use vram_pool::{LookaheadFit, lookahead_budget_bytes, lookahead_fit}; +pub use vram_pool::{LookaheadBitDepth, LookaheadFit, lookahead_budget_bytes, lookahead_fit}; /// Configuration wiring (set/clear/attach methods). mod wiring; @@ -89,6 +89,10 @@ pub struct StitchSession { pub(crate) skip_detection: bool, /// Number of lookahead frames (0 = disabled). pub(crate) lookahead_frames: usize, + /// Whether the lookahead pool buffers frames at the source's native + /// bit depth or downconverts to 8-bit first. Opt-in, default `Native` + /// (no behavior change). See [`vram_pool::LookaheadBitDepth`]. + pub(crate) lookahead_bit_depth: vram_pool::LookaheadBitDepth, pub(crate) frame_count: u64, /// Session start time for metrics computation. session_start: Option, @@ -251,6 +255,7 @@ impl StitchSession { encoder: None, skip_detection: false, lookahead_frames: 0, + lookahead_bit_depth: vram_pool::LookaheadBitDepth::default(), frame_count: 0, extra_encoders: Vec::new(), session_start: None, diff --git a/crates/reco-core/src/session/run_loop.rs b/crates/reco-core/src/session/run_loop.rs index 9b5bf7705..9f8bd3f70 100644 --- a/crates/reco-core/src/session/run_loop.rs +++ b/crates/reco-core/src/session/run_loop.rs @@ -276,8 +276,13 @@ impl StitchSession { // frame_processing.rs (peak occupancy + slack). let pool_size = n + post_smooth_half + 4; let (w, h) = self.core.pipeline().source_info(); + // The pool's actual per-slot cost depends on lookahead_bit_depth, + // not just the source's native format - Reduced8Bit stores 8-bit + // NV12 regardless of source format, so the budget check must use + // the same (pool) format the allocation below will actually use. + let pool_pixel_format = self.lookahead_bit_depth.pool_format(self.gpu_pixel_format); let per_slot = - super::vram_pool::estimate_vram(w, h, 1, self.gpu_pixel_format.bytes_per_sample()); + super::vram_pool::estimate_vram(w, h, 1, pool_pixel_format.bytes_per_sample()); let required = per_slot * pool_size; // Pre-flight VRAM budget check: fail fast with a fit suggestion. @@ -310,10 +315,27 @@ impl StitchSession { let max_n = super::vram_pool::max_lookahead_frames(per_slot, budget); let max_secs = max_n as f64 / fps; let req_secs = n as f64 / fps; + // A 10-bit source not already using Reduced8Bit has a + // real fourth lever: halving the pool's per-slot cost + // by dropping to 8-bit before buffering (see + // `LookaheadBitDepth`). Only surfaced when it would + // actually change anything, so 8-bit sources and + // already-reduced sessions get the original message. + let bit_depth_suggestion = if self.lookahead_bit_depth + == super::vram_pool::LookaheadBitDepth::Native + && self.gpu_pixel_format + != super::vram_pool::LookaheadBitDepth::Reduced8Bit + .pool_format(self.gpu_pixel_format) + { + ", reduce the lookahead pool's bit depth (roughly halves its VRAM \ + cost for 10-bit sources, at some cost to gradient smoothness)" + } else { + "" + }; return Err(SessionError::Config(format!( "not enough VRAM for a {req_secs:.1}s lookahead: reduce the lookahead \ - to <= {max_secs:.1}s, use lower-resolution source footage, or free \ - GPU memory. The frame pool needs ~{:.1} GB ({pool_size} slots @ \ + to <= {max_secs:.1}s, use lower-resolution source footage{bit_depth_suggestion}, \ + or free GPU memory. The frame pool needs ~{:.1} GB ({pool_size} slots @ \ {w}x{h}); usable budget is ~{:.1} GB of {:.1} GB total.", required as f64 / 1e9, budget as f64 / 1e9, @@ -331,26 +353,30 @@ impl StitchSession { } } - // The VramPool is only consumed on Linux (CUDA/Vulkan copy) and - // macOS (Metal CVPixelBuffer import). On Windows the lookahead - // frames live in the separate D3D11 staging pool, so a VramPool - // here would be allocated-but-unused VRAM (it roughly doubles the - // footprint). The pre-flight budget check above still runs on all - // platforms; on Windows it sizes the D3D11 staging pool, whose - // total VRAM equals estimate_vram(w,h,1,bps)*pool_size. - #[cfg(not(target_os = "windows"))] - { - let pool = super::vram_pool::VramPool::new( - self.core.pipeline().gpu(), - self.core.pipeline(), - w, - h, - pool_size, - self.gpu_pixel_format, - ) - .map_err(SessionError::Config)?; - self.vram_pool = Some(pool); - } + // The long-lived lookahead buffer is always a VramPool, on + // every platform. On Linux/macOS it is fed directly from + // already-separate shared CUDA/Vulkan or CVPixelBuffer plane + // textures (`copy_from_textures`). On Windows the D3D11VA + // decode path still stages each frame into a small, fixed-size + // `D3d11StagingPool` first (see `frame_processing.rs`'s + // `stage_d3d11_frames`) - that pool only bridges the D3D11 + // shared-handle import, it is not itself the long-lived + // buffer - then copies from there into this VramPool + // (`copy_from_d3d11`). This mirrors Linux/macOS's own + // stage-then-buffer shape instead of diverging from it, and is + // what lets `LookaheadBitDepth::Reduced8Bit` apply uniformly + // across platforms. + let pool = super::vram_pool::VramPool::new( + self.core.pipeline().gpu(), + self.core.pipeline(), + w, + h, + pool_size, + self.gpu_pixel_format, + self.lookahead_bit_depth, + ) + .map_err(SessionError::Config)?; + self.vram_pool = Some(pool); } let produce_one = |session: &mut StitchSession, diff --git a/crates/reco-core/src/session/vram_pool.rs b/crates/reco-core/src/session/vram_pool.rs index 33294e84b..17dbbf91d 100644 --- a/crates/reco-core/src/session/vram_pool.rs +++ b/crates/reco-core/src/session/vram_pool.rs @@ -10,7 +10,47 @@ use std::collections::VecDeque; -/// A single stereo NV12 frame in VRAM. +/// Whether the lookahead pool buffers frames at the source's native pixel +/// format or downconverts to 8-bit NV12 first. +/// +/// The lookahead pool's job is to hold N future decoded stereo frames so +/// the AI panner/tracker can smooth camera trajectory using future world- +/// state - but the *same* buffered frames are also what the final stitch +/// render consumes once it catches up (this is not an AI-only side +/// buffer). For 10-bit sources (`GpuPixelFormat::P010`, e.g. DJI Action 4 +/// HEVC), each pool slot costs 2x the bytes of an 8-bit source, which is +/// what makes deep lookahead windows VRAM-expensive on lower-VRAM cards. +/// `Reduced8Bit` roughly halves that cost by discarding the source's extra +/// bit depth before it enters the pool - a genuine (if subtle) whole- +/// export quality tradeoff (more banding risk in smooth gradients: sky, +/// pitch grass, floodlit surfaces), not a free lunch. See FRICTION.md +/// "Lookahead pool VRAM cost scales with source bit depth". +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum LookaheadBitDepth { + /// Buffer frames at the source's native bit depth (current/default + /// behavior, unchanged from before this option existed). + #[default] + Native, + /// Downconvert to 8-bit NV12 before buffering. No effect when the + /// source is already 8-bit. + Reduced8Bit, +} + +impl LookaheadBitDepth { + /// The pixel format the pool's own textures should be allocated in, + /// given the source's actual format. + pub fn pool_format( + self, + source_format: crate::render::renderer::GpuPixelFormat, + ) -> crate::render::renderer::GpuPixelFormat { + match self { + Self::Native => source_format, + Self::Reduced8Bit => crate::render::renderer::GpuPixelFormat::Nv12, + } + } +} + +/// A single stereo frame in VRAM, at the pool's `pool_format`. struct VramSlot { left_y: wgpu::Texture, left_uv: wgpu::Texture, @@ -20,12 +60,23 @@ struct VramSlot { right_bind_group: wgpu::BindGroup, } -/// Pool of VRAM-resident stereo NV12 textures for frame buffering. +/// Pool of VRAM-resident stereo NV12/P010 textures for frame buffering. pub(crate) struct VramPool { slots: Vec, free: VecDeque, width: u32, height: u32, + /// Pixel format of the source textures passed to `copy_from_textures`. + source_format: crate::render::renderer::GpuPixelFormat, + /// Pixel format the pool's own slot textures are allocated in. Equal to + /// `source_format` unless `LookaheadBitDepth::Reduced8Bit` was + /// requested for a 10-bit source, in which case `copy_from_textures` + /// runs a GPU downconvert pass (see `downconverter`) instead of a raw + /// texture-to-texture copy. + pool_format: crate::render::renderer::GpuPixelFormat, + /// Built only when `pool_format != source_format` - a plain same- + /// format copy needs no shader pass. + downconverter: Option, } impl VramPool { @@ -41,19 +92,32 @@ impl VramPool { /// the wgpu OOM panic into a descriptive error. (Error scopes are /// avoided here: they deadlock once the driver is in a bad post-OOM /// state.) + /// + /// `source_format` is the pixel format of the textures that will be + /// passed to `copy_from_textures` (the decode/import format). `bit_depth` + /// selects whether the pool stores frames at that format unchanged + /// (`Native`) or downconverts to 8-bit NV12 first (`Reduced8Bit`) - see + /// [`LookaheadBitDepth`]. pub fn new( gpu: &crate::gpu::GpuContext, pipeline: &crate::render::pipeline::StitchPipeline, width: u32, height: u32, n_slots: usize, - pixel_format: crate::render::renderer::GpuPixelFormat, + source_format: crate::render::renderer::GpuPixelFormat, + bit_depth: LookaheadBitDepth, ) -> Result { - let vram_bytes = estimate_vram(width, height, n_slots, pixel_format.bytes_per_sample()); + let pool_format = bit_depth.pool_format(source_format); + let vram_bytes = estimate_vram(width, height, n_slots, pool_format.bytes_per_sample()); let vram_mb = vram_bytes as f64 / (1024.0 * 1024.0); - let y_format = pixel_format.y_format(); - let uv_format = pixel_format.uv_format(); + let y_format = pool_format.y_format(); + let uv_format = pool_format.uv_format(); + let downconverter = if pool_format != source_format { + Some(crate::render::lookahead_downconvert::LookaheadDownconverter::new(gpu)) + } else { + None + }; let mut slots = Vec::with_capacity(n_slots); let mut free = VecDeque::with_capacity(n_slots); @@ -75,7 +139,13 @@ impl VramPool { sample_count: 1, dimension: wgpu::TextureDimension::D2, format: fmt, - usage: wgpu::TextureUsages::COPY_DST | wgpu::TextureUsages::TEXTURE_BINDING, + // RENDER_ATTACHMENT is only exercised when + // `downconverter` is Some (the downconvert render + // pass writes into these textures); harmless to + // request unconditionally otherwise. + usage: wgpu::TextureUsages::COPY_DST + | wgpu::TextureUsages::TEXTURE_BINDING + | wgpu::TextureUsages::RENDER_ATTACHMENT, view_formats: &[], }) }; @@ -114,7 +184,13 @@ impl VramPool { } log::info!( - "VramPool: {n_slots} stereo NV12 slots at {width}x{height}, ~{vram_mb:.0} MB VRAM" + "VramPool: {n_slots} stereo {pool_format:?} slots at {width}x{height}, \ + ~{vram_mb:.0} MB VRAM{}", + if downconverter.is_some() { + format!(" (downconverted from source {source_format:?})") + } else { + String::new() + } ); Ok(Self { @@ -122,6 +198,9 @@ impl VramPool { free, width, height, + source_format, + pool_format, + downconverter, }) } @@ -137,6 +216,14 @@ impl VramPool { } /// Copy from source textures (Y + UV per camera) into a pool slot. + /// + /// `src_*` must be in `self.source_format`. When the pool was built + /// with `LookaheadBitDepth::Native` (the common case), this is a plain + /// GPU-to-GPU `copy_texture_to_texture` - bit-exact, no shader + /// involved, identical to this method's behavior before + /// `LookaheadBitDepth` existed. When built with `Reduced8Bit` for a + /// 10-bit source, it instead runs the `LookaheadDownconverter` render + /// pass per plane (see `render::lookahead_downconvert`). pub fn copy_from_textures( &self, gpu: &crate::gpu::GpuContext, @@ -153,60 +240,218 @@ impl VramPool { label: Some("vram_pool_copy"), }); - let copy_tex = |enc: &mut wgpu::CommandEncoder, - src: &wgpu::Texture, - dst: &wgpu::Texture, - w: u32, - h: u32| { - enc.copy_texture_to_texture( - wgpu::TexelCopyTextureInfo { - texture: src, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - wgpu::TexelCopyTextureInfo { - texture: dst, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - wgpu::Extent3d { - width: w, - height: h, - depth_or_array_layers: 1, - }, - ); - }; + match &self.downconverter { + None => { + let copy_tex = |enc: &mut wgpu::CommandEncoder, + src: &wgpu::Texture, + dst: &wgpu::Texture, + w: u32, + h: u32| { + enc.copy_texture_to_texture( + wgpu::TexelCopyTextureInfo { + texture: src, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyTextureInfo { + texture: dst, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::Extent3d { + width: w, + height: h, + depth_or_array_layers: 1, + }, + ); + }; - copy_tex( - &mut encoder, - src_left_y, - &dst.left_y, - self.width, - self.height, - ); - copy_tex( - &mut encoder, - src_left_uv, - &dst.left_uv, - self.width / 2, - self.height / 2, - ); - copy_tex( - &mut encoder, - src_right_y, - &dst.right_y, - self.width, - self.height, - ); - copy_tex( - &mut encoder, - src_right_uv, - &dst.right_uv, - self.width / 2, - self.height / 2, - ); + copy_tex( + &mut encoder, + src_left_y, + &dst.left_y, + self.width, + self.height, + ); + copy_tex( + &mut encoder, + src_left_uv, + &dst.left_uv, + self.width / 2, + self.height / 2, + ); + copy_tex( + &mut encoder, + src_right_y, + &dst.right_y, + self.width, + self.height, + ); + copy_tex( + &mut encoder, + src_right_uv, + &dst.right_uv, + self.width / 2, + self.height / 2, + ); + } + Some(downconverter) => { + let view = + |t: &wgpu::Texture| t.create_view(&wgpu::TextureViewDescriptor::default()); + let (src_ly, src_luv, src_ry, src_ruv) = ( + view(src_left_y), + view(src_left_uv), + view(src_right_y), + view(src_right_uv), + ); + let (dst_ly, dst_luv, dst_ry, dst_ruv) = ( + view(&dst.left_y), + view(&dst.left_uv), + view(&dst.right_y), + view(&dst.right_uv), + ); + downconverter.convert_plane(&gpu.device, &mut encoder, &src_ly, &dst_ly, false); + downconverter.convert_plane(&gpu.device, &mut encoder, &src_luv, &dst_luv, true); + downconverter.convert_plane(&gpu.device, &mut encoder, &src_ry, &dst_ry, false); + downconverter.convert_plane(&gpu.device, &mut encoder, &src_ruv, &dst_ruv, true); + } + } + + gpu.queue.submit(std::iter::once(encoder.finish())); + } + + /// Copy a stereo frame from D3D11-imported multi-planar textures + /// (Windows only) into a pool slot. + /// + /// Unlike [`Self::copy_from_textures`] (Linux/macOS, where the source + /// is already two separate single-plane textures per camera), the + /// D3D11 source is one combined multi-planar texture per camera with + /// `TextureAspect::Plane0`/`Plane1` selecting Y/UV - see + /// [`crate::interop::d3d11::D3d11StagingPool::plane_source`]. When the + /// pool's format matches the source's, this does a plane-aspect- + /// selected `copy_texture_to_texture` (bit-exact, no shader - a raw + /// copy needs the source `texture` + aspect, not a view, which is why + /// this is a separate method from `copy_from_textures` rather than a + /// shared helper). When built with `LookaheadBitDepth::Reduced8Bit`, + /// runs the same `LookaheadDownconverter` render pass + /// `copy_from_textures` uses, via the pre-built plane views (a render + /// pass only needs a view). + pub fn copy_from_d3d11( + &self, + gpu: &crate::gpu::GpuContext, + slot: usize, + src_left: crate::interop::d3d11::D3d11PlaneSource<'_>, + src_right: crate::interop::d3d11::D3d11PlaneSource<'_>, + ) { + let dst = &self.slots[slot]; + let mut encoder = gpu + .device + .create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("vram_pool_copy_from_d3d11"), + }); + + match &self.downconverter { + None => { + let copy = |enc: &mut wgpu::CommandEncoder, + src_tex: &wgpu::Texture, + aspect: wgpu::TextureAspect, + dst_tex: &wgpu::Texture, + w: u32, + h: u32| { + enc.copy_texture_to_texture( + wgpu::TexelCopyTextureInfo { + texture: src_tex, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect, + }, + wgpu::TexelCopyTextureInfo { + texture: dst_tex, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::Extent3d { + width: w, + height: h, + depth_or_array_layers: 1, + }, + ); + }; + copy( + &mut encoder, + src_left.texture, + wgpu::TextureAspect::Plane0, + &dst.left_y, + self.width, + self.height, + ); + copy( + &mut encoder, + src_left.texture, + wgpu::TextureAspect::Plane1, + &dst.left_uv, + self.width / 2, + self.height / 2, + ); + copy( + &mut encoder, + src_right.texture, + wgpu::TextureAspect::Plane0, + &dst.right_y, + self.width, + self.height, + ); + copy( + &mut encoder, + src_right.texture, + wgpu::TextureAspect::Plane1, + &dst.right_uv, + self.width / 2, + self.height / 2, + ); + } + Some(downconverter) => { + let view = + |t: &wgpu::Texture| t.create_view(&wgpu::TextureViewDescriptor::default()); + let (dst_ly, dst_luv, dst_ry, dst_ruv) = ( + view(&dst.left_y), + view(&dst.left_uv), + view(&dst.right_y), + view(&dst.right_uv), + ); + downconverter.convert_plane( + &gpu.device, + &mut encoder, + src_left.y_view, + &dst_ly, + false, + ); + downconverter.convert_plane( + &gpu.device, + &mut encoder, + src_left.uv_view, + &dst_luv, + true, + ); + downconverter.convert_plane( + &gpu.device, + &mut encoder, + src_right.y_view, + &dst_ry, + false, + ); + downconverter.convert_plane( + &gpu.device, + &mut encoder, + src_right.uv_view, + &dst_ruv, + true, + ); + } + } gpu.queue.submit(std::iter::once(encoder.finish())); } @@ -347,6 +592,38 @@ pub fn lookahead_fit( mod tests { use super::*; + #[test] + fn bit_depth_native_keeps_source_format() { + use crate::render::renderer::GpuPixelFormat; + assert_eq!( + LookaheadBitDepth::Native.pool_format(GpuPixelFormat::P010), + GpuPixelFormat::P010 + ); + assert_eq!( + LookaheadBitDepth::Native.pool_format(GpuPixelFormat::Nv12), + GpuPixelFormat::Nv12 + ); + } + + #[test] + fn bit_depth_reduced_always_yields_nv12() { + use crate::render::renderer::GpuPixelFormat; + assert_eq!( + LookaheadBitDepth::Reduced8Bit.pool_format(GpuPixelFormat::P010), + GpuPixelFormat::Nv12 + ); + // No-op for sources that are already 8-bit. + assert_eq!( + LookaheadBitDepth::Reduced8Bit.pool_format(GpuPixelFormat::Nv12), + GpuPixelFormat::Nv12 + ); + } + + #[test] + fn bit_depth_default_is_native() { + assert_eq!(LookaheadBitDepth::default(), LookaheadBitDepth::Native); + } + #[test] fn pool_slots_match_buffered_sizing() { assert_eq!(lookahead_pool_slots(0), 1 + 4); // post_smooth_half clamps to 1 diff --git a/crates/reco-core/src/session/wiring.rs b/crates/reco-core/src/session/wiring.rs index a0240fea1..652e99a1a 100644 --- a/crates/reco-core/src/session/wiring.rs +++ b/crates/reco-core/src/session/wiring.rs @@ -111,6 +111,18 @@ impl StitchSession { self.lookahead_frames = frames; } + /// Opt into downconverting the lookahead pool to 8-bit NV12, even for + /// 10-bit sources. Off (`Native`) by default. Roughly halves the + /// pool's VRAM footprint for 10-bit sources, at the cost of losing + /// that extra bit depth in the final render too (the same buffered + /// frames feed both AI trajectory smoothing and the stitch render) - + /// see [`crate::session::vram_pool::LookaheadBitDepth`]. No effect on + /// 8-bit sources. Must be called before the first `run()`/produce + /// call that would otherwise lazily create the pool. + pub fn set_lookahead_bit_depth(&mut self, bit_depth: crate::session::LookaheadBitDepth) { + self.lookahead_bit_depth = bit_depth; + } + /// Attach a stacked-video replay recorder. /// /// Forwards to `StitchCore::set_stacked_recorder` on the diff --git a/crates/reco-core/src/shaders/lookahead_downconvert.wgsl b/crates/reco-core/src/shaders/lookahead_downconvert.wgsl new file mode 100644 index 000000000..2459ef396 --- /dev/null +++ b/crates/reco-core/src/shaders/lookahead_downconvert.wgsl @@ -0,0 +1,62 @@ +// Reco v2 -- lookahead pool bit-depth downconversion (P010 -> NV12) +// +// Copies one plane (Y or UV) of a decoded stereo frame from its source +// format into an 8-bit destination plane, as a GPU-resident render pass. +// +// Why this exists: the lookahead pool buffers N future decoded stereo +// frames so the AI panner/tracker can smooth camera trajectory using +// future world-state (see `session::vram_pool::VramPool` and +// `interop::d3d11::D3d11StagingPool`). Those buffered frames are also +// what the final stitch render consumes when it catches up to them - the +// pool is not an AI-only side buffer. For 10-bit sources (`GpuPixelFormat:: +// P010`, e.g. DJI Action 4 HEVC), each pool slot costs 2x the bytes of an +// 8-bit source, which is what makes deep lookahead windows VRAM-expensive +// on lower-VRAM cards (see FRICTION.md "Lookahead pool VRAM cost scales +// with source bit depth"). Downconverting to 8-bit (NV12) before the +// frame enters the long-lived pool roughly halves that cost. +// +// This is a genuine, opt-in quality/memory tradeoff, not a free lunch: +// the same buffered frame feeds the final render, so downconverting here +// discards the extra precision a 10-bit source captured (more banding +// risk in smooth gradients - sky, pitch grass, floodlit surfaces). It +// must stay opt-in and default-off; see `LookaheadBitDepth` in +// `session/vram_pool.rs`. +// +// One shader body handles both planes: the Y plane's source is a +// single-channel texture (`R8Unorm`/`R16Unorm`), the UV plane's source is +// two-channel (`Rg8Unorm`/`Rg16Unorm`). `textureLoad` always returns a +// `vec4` regardless of the source's channel count (missing channels +// read as 0.0/1.0), and the render target's format (`R8Unorm` for Y, +// `Rg8Unorm` for UV) determines which of the output's components actually +// get written - so no separate entry point is needed per plane. No manual +// bit-shift/rescale math is needed either: wgpu/WebGPU normalizes both +// 8-bit `[0,255]` and 16-bit `[0,65535]` Unorm source formats to `[0.0, +// 1.0]` on load, and quantizes the `[0.0,1.0]` fragment output back down +// to the target format's bit depth on write. +// +// Source and destination are always the same resolution (the Y plane is +// full-res in and out, the UV plane is half-res in and out - this shader +// only changes bit depth, never scale), so a direct `textureLoad` at the +// fragment's own integer pixel coordinate is used instead of a filtered +// `textureSample`. This is both more precise (an exact 1:1 texel copy, +// zero interpolation) and sidesteps needing the source format to support +// linear filtering (16-bit Unorm texture formats are not guaranteed +// filterable on every wgpu backend). +// +// Draws a fullscreen triangle (3 vertices, no vertex/index buffer) per +// invocation - see `render::lookahead_downconvert::LookaheadDownconverter`. + +// Standard oversized fullscreen triangle: covers the [-1,1] clip-space +// square with a single triangle (no diagonal seam, no vertex buffer). +@vertex +fn vs_main(@builtin(vertex_index) vertex_index: u32) -> @builtin(position) vec4 { + let uv = vec2(f32((vertex_index << 1u) & 2u), f32(vertex_index & 2u)); + return vec4(uv * 2.0 - 1.0, 0.0, 1.0); +} + +@group(0) @binding(0) var src: texture_2d; + +@fragment +fn fs_main(@builtin(position) pos: vec4) -> @location(0) vec4 { + return textureLoad(src, vec2(pos.xy), 0); +} diff --git a/crates/reco-gui/src/export.rs b/crates/reco-gui/src/export.rs index e1c8c3b35..8b24b6660 100644 --- a/crates/reco-gui/src/export.rs +++ b/crates/reco-gui/src/export.rs @@ -47,6 +47,13 @@ pub struct AutocamUiConfig { pub detection_interval: u32, /// Lookahead buffer depth in seconds (0 = off). pub lookahead_secs: f64, + /// Downconvert the lookahead pool to 8-bit NV12 even for 10-bit + /// sources, roughly halving its VRAM cost at some cost to gradient + /// smoothness in the final render (the same buffered frames feed + /// both AI tracking and the stitch render - see + /// `reco_core::session::vram_pool::LookaheadBitDepth`). Off by + /// default; no effect on already-8-bit sources. + pub lookahead_reduced_bit_depth: bool, /// Preset name used as the config base; visible knobs overlay it. pub preset: String, /// `"action"` or `"frame_all"`. @@ -271,6 +278,9 @@ pub fn run_export( #[cfg(feature = "autocam")] if autocam.enabled && autocam.lookahead_secs > 0.0 { job = job.lookahead(autocam.lookahead_secs); + if autocam.lookahead_reduced_bit_depth { + job = job.lookahead_reduced_bit_depth(true); + } } #[cfg(feature = "autocam")] diff --git a/crates/reco-gui/src/main.rs b/crates/reco-gui/src/main.rs index 2763a88b8..1aef21838 100644 --- a/crates/reco-gui/src/main.rs +++ b/crates/reco-gui/src/main.rs @@ -3488,6 +3488,7 @@ fn main() -> anyhow::Result<()> { tracking_mode: app.get_export_tracking_mode().to_string(), detection_interval: app.get_export_detection_interval() as u32, lookahead_secs: app.get_export_lookahead_secs() as f64, + lookahead_reduced_bit_depth: app.get_export_lookahead_reduced_bit_depth(), preset: app.get_export_panner_preset().to_string(), framing: app.get_export_framing().to_string(), lock_pitch: app.get_export_lock_pitch(), diff --git a/crates/reco-gui/ui/main.slint b/crates/reco-gui/ui/main.slint index 3eb6e2101..beb60bf89 100644 --- a/crates/reco-gui/ui/main.slint +++ b/crates/reco-gui/ui/main.slint @@ -549,6 +549,12 @@ export component RecoApp inherits Window { in-out property lookahead-green-max: 0; in-out property lookahead-red-min: 0; in-out property lookahead-risk-active: false; + // Downconvert the lookahead pool to 8-bit before buffering, roughly + // halving its VRAM cost for 10-bit sources (no effect on 8-bit + // sources) at some cost to gradient smoothness in the final render - + // the buffered frames feed both AI tracking and the stitch render. + // Off by default. + in-out property export-lookahead-reduced-bit-depth: false; in-out property export-framing: "action"; in-out property export-lock-pitch: false; in-out property export-cluster-mode: "density"; @@ -2615,6 +2621,16 @@ export component RecoApp inherits Window { risk-active: root.lookahead-risk-active; } + // Buffers the lookahead at 8-bit instead of the source's native bit + // depth, roughly halving its VRAM cost on 10-bit footage (e.g. DJI + // Action 4 HEVC) - lets a longer lookahead fit on lower-VRAM cards. + // No effect on already-8-bit sources. Leave off unless hitting a + // VRAM error, since it trades a little gradient smoothness for memory. + CheckBox { + text: "Reduce lookahead memory (8-bit)"; + checked <=> root.export-lookahead-reduced-bit-depth; + } + panner-advanced := SectionHeader { title: "Advanced panner"; expanded: false; diff --git a/crates/reco-io/src/stitch_job.rs b/crates/reco-io/src/stitch_job.rs index 9a6350ce9..bdd4e99ae 100644 --- a/crates/reco-io/src/stitch_job.rs +++ b/crates/reco-io/src/stitch_job.rs @@ -79,6 +79,10 @@ pub struct StitchJob { /// decodes N frames ahead to give the panner future context. lookahead_secs: f64, + /// Downconvert the lookahead pool to 8-bit before buffering. See + /// [`Self::lookahead_reduced_bit_depth`]. + lookahead_reduced_bit_depth: bool, + /// Path for pipeline event JSONL output. When set, attaches a /// `JsonlSink` to the session that records every detection, /// filter decision, and pan decision for offline analysis. @@ -257,6 +261,7 @@ impl StitchJob { replay_recording: None, force_cpu_decode: false, lookahead_secs: 0.0, + lookahead_reduced_bit_depth: false, events_path: None, } } @@ -462,6 +467,16 @@ impl StitchJob { self } + /// Downconvert the lookahead pool to 8-bit NV12 even for 10-bit + /// sources, roughly halving its VRAM cost at some cost to gradient + /// smoothness (the same buffered frames feed both AI tracking and the + /// final stitch render). Off by default; no effect on already-8-bit + /// sources. See `reco_core::session::vram_pool::LookaheadBitDepth`. + pub fn lookahead_reduced_bit_depth(mut self, reduced: bool) -> Self { + self.lookahead_reduced_bit_depth = reduced; + self + } + /// Record pipeline events (detections, filter decisions, pan /// decisions) to a JSONL file for offline analysis. pub fn events(mut self, path: impl AsRef) -> Self { @@ -633,6 +648,10 @@ impl StitchJob { .unwrap_or(30.0); let frames = (self.lookahead_secs * fps).round() as usize; session.set_lookahead(frames); + if self.lookahead_reduced_bit_depth { + session.set_lookahead_bit_depth(reco_core::session::LookaheadBitDepth::Reduced8Bit); + log::info!("Lookahead bit depth: Reduced8Bit (halved VRAM, 10-bit sources only)"); + } } for hook in self.session_hooks.drain(..) {