From 740c285768523e0e70aba90689fe76197e7435f1 Mon Sep 17 00:00:00 2001 From: Fabian Grajales Date: Tue, 6 Oct 2026 09:23:44 -0500 Subject: [PATCH] feat(relight): add AI relighting panel with movable lights Adds a Relight panel that lights the photo with up to six point lights placed on the image. Normal and depth maps come from MoGe-2 (small or large ONNX checkpoint, downloaded on first use like the other AI models) and are cached per image. The relight pass runs before lens blur in the preview, export and thumbnail pipelines. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Dwb4HkWbxjkmwpshwXd2zi --- src-tauri/src/adjustment_utils.rs | 3 +- src-tauri/src/ai_commands.rs | 272 +++++ src-tauri/src/ai_processing.rs | 351 +++++- src-tauri/src/cache_utils.rs | 35 + src-tauri/src/file_management.rs | 3 +- src-tauri/src/lens_blur.rs | 57 +- src-tauri/src/lib.rs | 9 +- src-tauri/src/relight.rs | 1050 +++++++++++++++++ src/App.tsx | 3 + src/components/panel/Editor.tsx | 5 +- src/components/panel/PanelSwitcher.tsx | 4 + src/components/panel/editor/ImageCanvas.tsx | 158 ++- .../panel/editor/overlays/RelightHandle.tsx | 168 +++ src/components/panel/right/RelightPanel.tsx | 431 +++++++ src/components/ui/AppProperties.tsx | 1 + src/i18n/locales/ca.json | 24 + src/i18n/locales/cs.json | 24 + src/i18n/locales/de.json | 24 + src/i18n/locales/en.json | 24 + src/i18n/locales/es.json | 24 + src/i18n/locales/fr.json | 24 + src/i18n/locales/it.json | 24 + src/i18n/locales/ja.json | 24 + src/i18n/locales/ko.json | 24 + src/i18n/locales/nl.json | 24 + src/i18n/locales/pl.json | 24 + src/i18n/locales/pt.json | 24 + src/i18n/locales/ru.json | 24 + src/i18n/locales/zh-CN.json | 24 + src/i18n/locales/zh-TW.json | 24 + src/store/useEditorStore.ts | 4 + src/store/useUIStore.ts | 6 +- src/utils/adjustments.ts | 89 ++ src/utils/relightUtils.ts | 91 ++ 34 files changed, 3065 insertions(+), 35 deletions(-) create mode 100644 src-tauri/src/relight.rs create mode 100644 src/components/panel/editor/overlays/RelightHandle.tsx create mode 100644 src/components/panel/right/RelightPanel.tsx create mode 100644 src/utils/relightUtils.ts diff --git a/src-tauri/src/adjustment_utils.rs b/src-tauri/src/adjustment_utils.rs index bacabf872d..0d6c8d1cd2 100644 --- a/src-tauri/src/adjustment_utils.rs +++ b/src-tauri/src/adjustment_utils.rs @@ -120,7 +120,8 @@ pub fn apply_all_transformations<'a, I: IntoCowImage<'a>>( let image = image.into_cow(); let warped_image = apply_geometry_warp(image, adjustments); - let blurred_image = crate::lens_blur::apply_lens_blur(warped_image, adjustments); + let relit_image = crate::relight::apply_relight(warped_image, adjustments); + let blurred_image = crate::lens_blur::apply_lens_blur(relit_image, adjustments); let (cropped_image, unscaled_crop_offset) = apply_spatial_transformations(blurred_image, adjustments); diff --git a/src-tauri/src/ai_commands.rs b/src-tauri/src/ai_commands.rs index dc2182802d..2f445d687d 100644 --- a/src-tauri/src/ai_commands.rs +++ b/src-tauri/src/ai_commands.rs @@ -301,6 +301,200 @@ pub async fn generate_full_image_depth_map( Ok(format!("data:image/png;base64,{}", base64_str)) } +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct RelightMapsPayload { + normal_map: String, + depth_map: String, + depth_scale: f32, +} + +fn encode_png_data_url(image: &image::DynamicImage) -> Result { + let mut buf = Cursor::new(Vec::new()); + image + .write_to(&mut buf, ImageFormat::Png) + .map_err(|e| e.to_string())?; + let base64_str = general_purpose::STANDARD.encode(buf.get_ref()); + Ok(format!("data:image/png;base64,{}", base64_str)) +} + +// Generated maps are kept on disk, one file per image, geometry and model quality, so that +// switching quality back and forth or reopening a photo does not run the model again. +const RELIGHT_MAPS_CACHE_VERSION: &str = "v1"; +const RELIGHT_MAPS_CACHE_MAX_FILES: usize = 60; + +fn relight_maps_cache_file( + cache_dir: &std::path::Path, + image_path: &str, + js_adjustments: &serde_json::Value, + quality: crate::ai_processing::NormalModelQuality, +) -> std::path::PathBuf { + let mut hasher = blake3::Hasher::new(); + hasher.update(RELIGHT_MAPS_CACHE_VERSION.as_bytes()); + hasher.update(image_path.as_bytes()); + hasher.update(format!("{:?}", quality).as_bytes()); + hasher.update(&crate::cache_utils::calculate_geometry_hash(js_adjustments).to_le_bytes()); + + // A file that was replaced on disk must not reuse the maps of the old one. + let (source_path, _) = crate::file_management::parse_virtual_path(image_path); + if let Ok(metadata) = std::fs::metadata(&source_path) { + hasher.update(&metadata.len().to_le_bytes()); + if let Ok(elapsed) = metadata.modified().and_then(|m| { + m.duration_since(std::time::UNIX_EPOCH) + .map_err(std::io::Error::other) + }) { + hasher.update(&elapsed.as_nanos().to_le_bytes()); + } + } + + cache_dir.join(format!("{}.json", hasher.finalize().to_hex())) +} + +fn read_relight_maps_cache(file: &std::path::Path) -> Option { + let bytes = std::fs::read(file).ok()?; + serde_json::from_slice(&bytes).ok() +} + +fn write_relight_maps_cache( + file: &std::path::Path, + maps: &RelightMapsPayload, +) -> Result<(), String> { + let dir = file.parent().ok_or("Invalid relight cache path")?; + std::fs::create_dir_all(dir).map_err(|e| e.to_string())?; + + // Written next to the target and renamed, so a crash never leaves half a file behind. + let tmp = file.with_extension("tmp"); + std::fs::write(&tmp, serde_json::to_vec(maps).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + std::fs::rename(&tmp, file).map_err(|e| e.to_string())?; + + // Drop the least recently written maps once there are too many. + let mut entries: Vec<(std::time::SystemTime, std::path::PathBuf)> = std::fs::read_dir(dir) + .map_err(|e| e.to_string())? + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "json")) + .filter_map(|entry| Some((entry.metadata().ok()?.modified().ok()?, entry.path()))) + .collect(); + if entries.len() > RELIGHT_MAPS_CACHE_MAX_FILES { + entries.sort(); + for (_, path) in &entries[..entries.len() - RELIGHT_MAPS_CACHE_MAX_FILES] { + let _ = std::fs::remove_file(path); + } + } + Ok(()) +} + +/// Returns the relight maps for the open image, generating them only when there is no saved +/// copy for this image and quality (or when `force` asks for a fresh run). +#[tauri::command] +pub async fn generate_relight_maps( + js_adjustments: serde_json::Value, + quality: Option, + force: Option, + state: tauri::State<'_, AppState>, + app_handle: tauri::AppHandle, +) -> Result { + use tauri::Manager; + + let quality = quality.unwrap_or_default(); + let image_path = state + .original_image + .lock() + .unwrap() + .as_ref() + .map(|loaded| loaded.path.clone()) + .ok_or("No original image loaded")?; + + let cache_file = app_handle.path().app_cache_dir().ok().map(|dir| { + relight_maps_cache_file( + &dir.join("relight_maps"), + &image_path, + &js_adjustments, + quality, + ) + }); + + if !force.unwrap_or(false) + && let Some(maps) = cache_file.as_deref().and_then(read_relight_maps_cache) + { + return Ok(maps); + } + + let normal_model = crate::ai_processing::get_or_init_normal_model( + &app_handle, + &state.ai_state, + &state.ai_init_lock, + quality, + ) + .await + .map_err(|e| e.to_string())?; + + let warped_image = crate::get_cached_full_warped_image(&state, &js_adjustments)?; + + let maps = crate::ai_processing::run_normal_model(warped_image.as_ref(), &normal_model) + .map_err(|e| e.to_string())?; + + let payload = RelightMapsPayload { + normal_map: encode_png_data_url(&image::DynamicImage::ImageRgba8(maps.normal))?, + depth_map: encode_png_data_url(&image::DynamicImage::ImageLuma16(maps.depth))?, + depth_scale: maps.depth_scale, + }; + + if let Some(file) = &cache_file + && let Err(error) = write_relight_maps_cache(file, &payload) + { + log::warn!( + "Could not save relight maps to {}: {}", + file.display(), + error + ); + } + + Ok(payload) +} + +/// Normal map as the relight pass uses it, laid out like the edited image (orientation, +/// rotation and crop applied) so it can be shown on top of it. +#[tauri::command] +pub async fn generate_relight_normal_preview( + js_adjustments: serde_json::Value, + state: tauri::State<'_, AppState>, +) -> Result { + let warped_image = crate::get_cached_full_warped_image(&state, &js_adjustments)?; + + let normals = crate::relight::render_normal_preview(warped_image.as_ref(), &js_adjustments) + .ok_or_else(|| "No normal map available".to_string())?; + + // The preview is smaller than the image, so the crop (stored in full-size pixels) has to + // shrink with it. + let scale = normals.width() as f64 / warped_image.width().max(1) as f64; + let mut layout = serde_json::json!({ + "orientationSteps": js_adjustments["orientationSteps"], + "rotation": js_adjustments["rotation"], + "flipHorizontal": js_adjustments["flipHorizontal"], + "flipVertical": js_adjustments["flipVertical"], + "crop": js_adjustments["crop"], + }); + if let Some(crop) = layout["crop"].as_object_mut() { + for key in ["x", "y", "width", "height"] { + if let Some(value) = crop.get(key).and_then(|v| v.as_f64()) { + crop.insert(key.to_string(), serde_json::json!(value * scale)); + } + } + } + + let (preview, _) = crate::adjustment_utils::apply_spatial_transformations( + image::DynamicImage::ImageRgb8(normals), + &layout, + ); + + let mut buf = Cursor::new(Vec::new()); + image::codecs::jpeg::JpegEncoder::new_with_quality(&mut buf, 90) + .encode_image(&preview.to_rgb8()) + .map_err(|e| e.to_string())?; + Ok(tauri::ipc::Response::new(buf.into_inner())) +} + #[allow(clippy::too_many_arguments)] #[tauri::command] pub async fn generate_ai_subject_mask( @@ -569,3 +763,81 @@ pub async fn test_ai_connector_connection(address: String) -> Result<(), String> Err(e) => Err(e.to_string()), } } + +#[cfg(test)] +mod relight_maps_cache_tests { + use super::*; + use crate::ai_processing::NormalModelQuality; + use serde_json::json; + + fn payload(tag: &str) -> RelightMapsPayload { + RelightMapsPayload { + normal_map: format!("normal-{tag}"), + depth_map: format!("depth-{tag}"), + depth_scale: 1.5, + } + } + + #[test] + fn saved_maps_are_read_back() { + let dir = tempfile::tempdir().unwrap(); + let file = relight_maps_cache_file( + dir.path(), + "photo.raw", + &json!({}), + NormalModelQuality::High, + ); + + assert_eq!(read_relight_maps_cache(&file), None); + write_relight_maps_cache(&file, &payload("a")).unwrap(); + assert_eq!(read_relight_maps_cache(&file), Some(payload("a"))); + } + + #[test] + fn each_image_quality_and_geometry_gets_its_own_file() { + let dir = std::path::Path::new("cache"); + let base = + relight_maps_cache_file(dir, "photo.raw", &json!({}), NormalModelQuality::Standard); + + let same = relight_maps_cache_file( + dir, + "photo.raw", + &json!({ "exposure": 2 }), + NormalModelQuality::Standard, + ); + let other_quality = + relight_maps_cache_file(dir, "photo.raw", &json!({}), NormalModelQuality::High); + let other_image = + relight_maps_cache_file(dir, "other.raw", &json!({}), NormalModelQuality::Standard); + let other_geometry = relight_maps_cache_file( + dir, + "photo.raw", + &json!({ "transformRotate": 5 }), + NormalModelQuality::Standard, + ); + + assert_eq!(base, same); + assert_ne!(base, other_quality); + assert_ne!(base, other_image); + assert_ne!(base, other_geometry); + } + + #[test] + fn oldest_maps_are_dropped_when_the_cache_is_full() { + let dir = tempfile::tempdir().unwrap(); + for i in 0..RELIGHT_MAPS_CACHE_MAX_FILES + 3 { + let file = dir.path().join(format!("{i:04}.json")); + write_relight_maps_cache(&file, &payload("x")).unwrap(); + let stamp = filetime::FileTime::from_unix_time(1_700_000_000 + i as i64, 0); + filetime::set_file_mtime(&file, stamp).unwrap(); + } + // One more write triggers the clean-up with every timestamp in place. + let newest = dir.path().join("newest.json"); + write_relight_maps_cache(&newest, &payload("x")).unwrap(); + + let remaining = std::fs::read_dir(dir.path()).unwrap().count(); + assert_eq!(remaining, RELIGHT_MAPS_CACHE_MAX_FILES); + assert!(!dir.path().join("0000.json").exists()); + assert!(newest.exists()); + } +} diff --git a/src-tauri/src/ai_processing.rs b/src-tauri/src/ai_processing.rs index 504e7263dc..96ae785e38 100644 --- a/src-tauri/src/ai_processing.rs +++ b/src-tauri/src/ai_processing.rs @@ -60,6 +60,45 @@ const DEPTH_FILENAME: &str = "depth_anything_v2_vits.onnx"; const DEPTH_INPUT_SIZE: u32 = 518; const DEPTH_SHA256: &str = "d2b11a11c1d4a12b47608fa65a17ee9a4c605b55ee1730c8e3b526304f2562be"; +/// MoGe-2 checkpoints used for the relight maps. Both take the same inputs and return the +/// same outputs; the large one resolves much finer geometry but is far heavier. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Default, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum NormalModelQuality { + #[default] + Standard, + High, +} + +struct NormalModelSpec { + url: &'static str, + filename: &'static str, + sha256: &'static str, + name: &'static str, +} + +impl NormalModelQuality { + fn spec(self) -> NormalModelSpec { + match self { + Self::Standard => NormalModelSpec { + url: "https://huggingface.co/Ruicheng/moge-2-vits-normal-onnx/resolve/e50ffda41565591092adea54c6ac83d6212e1e23/model.onnx?download=true", + filename: "moge2_vits_normal.onnx", + sha256: "24eacb5dc7a2c54c7bc98f7de085ffbed79ad006ea5b664c2c2cdc02ff3a52f0", + name: "Normal Map Model", + }, + Self::High => NormalModelSpec { + url: "https://huggingface.co/Ruicheng/moge-2-vitl-normal-onnx/resolve/1c02388b024dd34da413e1a934d7a941785f1460/model.onnx?download=true", + filename: "moge2_vitl_normal.onnx", + sha256: "afbc4ccc3450298f3afb35b90f015f4c4f552dea21dc6470d5f7b78b77e2d751", + name: "Normal Map Model (High Quality, 1.3 GB)", + }, + } + } +} + +const NORMAL_MAX_SIZE: u32 = 1024; +const NORMAL_NUM_TOKENS: i64 = 3600; + pub struct AiModels { pub sam_encoder: Mutex, pub sam_decoder: Mutex, @@ -92,6 +131,8 @@ pub struct AiState { pub denoise_model: Option>>, pub clip_models: Option>, pub lama_model: Option>>, + // Only one checkpoint is kept loaded: the large one alone takes well over 1 GB of RAM. + pub normal_model: Option<(NormalModelQuality, Arc>)>, pub embeddings: Option, pub depth_map: Option, } @@ -314,13 +355,8 @@ fn get_models_dir(app_handle: &tauri::AppHandle) -> Result { Ok(models_dir) } -fn persist_downloaded_asset(dest: &Path, bytes: &[u8]) -> Result<()> { - if bytes.is_empty() { - return Err(anyhow::anyhow!( - "Downloaded asset for {} was empty", - dest.display() - )); - } +async fn download_model(url: &str, dest: &Path) -> Result<()> { + let mut response = reqwest::get(url).await?.error_for_status()?; let parent = dest.parent().ok_or_else(|| { anyhow::anyhow!( @@ -336,10 +372,29 @@ fn persist_downloaded_asset(dest: &Path, bytes: &[u8]) -> Result<()> { .ok_or_else(|| anyhow::anyhow!("Invalid downloaded asset path {}", dest.display()))?; let tmp_path = dest.with_file_name(format!(".{}.download", file_name)); - { + // Written in chunks so that large models never have to fit in memory. + let mut total_bytes = 0usize; + let write_result: Result<()> = async { let mut file = fs::File::create(&tmp_path)?; - file.write_all(bytes)?; + while let Some(chunk) = response.chunk().await? { + file.write_all(&chunk)?; + total_bytes += chunk.len(); + } file.sync_all()?; + Ok(()) + } + .await; + + if let Err(error) = write_result { + let _ = fs::remove_file(&tmp_path); + return Err(error); + } + if total_bytes == 0 { + let _ = fs::remove_file(&tmp_path); + return Err(anyhow::anyhow!( + "Downloaded asset for {} was empty", + dest.display() + )); } fs::rename(&tmp_path, dest).or_else(|rename_error| -> std::io::Result<()> { @@ -354,12 +409,6 @@ fn persist_downloaded_asset(dest: &Path, bytes: &[u8]) -> Result<()> { Ok(()) } -async fn download_model(url: &str, dest: &Path) -> Result<()> { - let response = reqwest::get(url).await?.error_for_status()?; - let bytes = response.bytes().await?; - persist_downloaded_asset(dest, &bytes) -} - fn verify_sha256(path: &Path, expected_hash: &str) -> Result { if !path.exists() { return Ok(false); @@ -550,6 +599,7 @@ pub async fn get_or_init_ai_models( denoise_model: None, clip_models: None, lama_model: None, + normal_model: None, embeddings: None, depth_map: None, }); @@ -610,6 +660,7 @@ pub async fn get_or_init_denoise_model( denoise_model: Some(denoise_model.clone()), clip_models: None, lama_model: None, + normal_model: None, embeddings: None, depth_map: None, }); @@ -682,6 +733,7 @@ pub async fn get_or_init_clip_models( denoise_model: None, clip_models: Some(clip_models.clone()), lama_model: None, + normal_model: None, embeddings: None, depth_map: None, }); @@ -742,6 +794,7 @@ pub async fn get_or_init_lama_model( denoise_model: None, clip_models: None, lama_model: Some(lama_model.clone()), + normal_model: None, embeddings: None, depth_map: None, }); @@ -750,6 +803,74 @@ pub async fn get_or_init_lama_model( Ok(lama_model) } +pub async fn get_or_init_normal_model( + app_handle: &tauri::AppHandle, + ai_state_mutex: &Mutex>, + ai_init_lock: &TokioMutex<()>, + quality: NormalModelQuality, +) -> Result>> { + let loaded_model = || { + ai_state_mutex + .lock() + .unwrap() + .as_ref() + .and_then(|state| state.normal_model.clone()) + .filter(|(loaded_quality, _)| *loaded_quality == quality) + .map(|(_, model)| model) + }; + + if let Some(normal_model) = loaded_model() { + return Ok(normal_model); + } + + let _guard = ai_init_lock.lock().await; + + if let Some(normal_model) = loaded_model() { + return Ok(normal_model); + } + + let spec = quality.spec(); + let models_dir = get_models_dir(app_handle)?; + download_and_verify_model( + app_handle, + &models_dir, + spec.filename, + spec.url, + spec.sha256, + spec.name, + ) + .await?; + + // Free the other checkpoint before loading this one. + if let Some(state) = ai_state_mutex.lock().unwrap().as_mut() { + state.normal_model = None; + } + + let _ = ort::init().with_name("AI-Normal").commit(); + let model_path = models_dir.join(spec.filename); + let session = Session::builder()?.commit_from_file(model_path)?; + let normal_model = Arc::new(Mutex::new(session)); + + crate::register_exit_handler(); + + let mut ai_state_lock = ai_state_mutex.lock().unwrap(); + if let Some(state) = ai_state_lock.as_mut() { + state.normal_model = Some((quality, normal_model.clone())); + } else { + *ai_state_lock = Some(AiState { + models: None, + denoise_model: None, + clip_models: None, + lama_model: None, + normal_model: Some((quality, normal_model.clone())), + embeddings: None, + depth_map: None, + }); + } + + Ok(normal_model) +} + #[derive(Clone, Copy)] struct TileParams { cs: usize, @@ -1684,6 +1805,167 @@ pub fn run_depth_anything_model( Ok(depth_map) } +pub struct RelightMaps { + /// RGB = camera-space normal (X right, Y up, Z towards the viewer) packed as n * 0.5 + 0.5, + /// A = surface validity (0 where there is no geometry, e.g. sky). + pub normal: RgbaImage, + /// Distance from the nearest surface, 0 = nearest and 65535 = farthest. + pub depth: ImageBuffer, Vec>, + /// Depth covered by the full range of `depth`, in units of the longest image side. + pub depth_scale: f32, +} + +/// Turns the model's camera-space point map into a depth map whose scale matches +/// the image plane, so a light can be placed in front of or behind the subject. +fn build_depth_map(points: &[f32], mask: &[f32], w: usize, h: usize) -> (Vec, f32) { + let long_side = w.max(h) as f32; + + let mut depths = Vec::with_capacity(w * h); + // Least squares fit of X = s * Z * x_norm, where s is the sensor size over the focal length. + let mut fit_num = 0.0f64; + let mut fit_den = 0.0f64; + + for (i, (point, m)) in points.as_chunks::<3>().0.iter().zip(mask).enumerate() { + let z = point[2]; + if *m < 0.5 || !z.is_finite() || z <= 0.0 || !point[0].is_finite() || !point[1].is_finite() + { + continue; + } + depths.push(z); + + let xn = ((i % w) as f32 + 0.5 - w as f32 * 0.5) / long_side; + let yn = ((i / w) as f32 + 0.5 - h as f32 * 0.5) / long_side; + fit_num += (point[0] * z * xn + point[1] * z * yn) as f64; + fit_den += ((z * xn) * (z * xn) + (z * yn) * (z * yn)) as f64; + } + + if depths.len() < 16 { + return (vec![0; w * h], 0.0); + } + + depths.sort_unstable_by(|a, b| a.total_cmp(b)); + let percentile = |p: f32| depths[((depths.len() - 1) as f32 * p) as usize]; + let near = percentile(0.005); + let far = percentile(0.995); + let reference = percentile(0.5); + let range = far - near; + + if range <= 1e-6 || reference <= 1e-6 { + return (vec![0; w * h], 0.0); + } + + let sensor_over_focal = (fit_num / fit_den.max(1e-12)) as f32; + let sensor_over_focal = if sensor_over_focal.is_finite() && sensor_over_focal > 0.05 { + sensor_over_focal + } else { + 1.0 + }; + + let depth_data = points + .as_chunks::<3>() + .0 + .iter() + .zip(mask) + .map(|(point, m)| { + let z = point[2]; + if *m < 0.5 || !z.is_finite() { + u16::MAX + } else { + (((z - near) / range).clamp(0.0, 1.0) * 65535.0).round() as u16 + } + }) + .collect(); + + (depth_data, range / (sensor_over_focal * reference)) +} + +/// Runs MoGe-2 and returns the normal and depth maps used for relighting. +pub fn run_normal_model( + image: &DynamicImage, + normal_session: &Mutex, +) -> Result { + let resized_image = if image.width().max(image.height()) > NORMAL_MAX_SIZE { + image.resize(NORMAL_MAX_SIZE, NORMAL_MAX_SIZE, FilterType::Triangle) + } else { + image.clone() + }; + let (resized_w, resized_h) = resized_image.dimensions(); + let resized_rgb = resized_image.into_rgb8(); + let raw_pixels = resized_rgb.as_raw(); + + let rw = resized_w as usize; + let rh = resized_h as usize; + + let mut input_tensor: Array = Array::zeros((1, 3, rh, rw)); + for y in 0..rh { + for x in 0..rw { + let idx = (y * rw + x) * 3; + input_tensor[[0, 0, y, x]] = raw_pixels[idx] as f32 / 255.0; + input_tensor[[0, 1, y, x]] = raw_pixels[idx + 1] as f32 / 255.0; + input_tensor[[0, 2, y, x]] = raw_pixels[idx + 2] as f32 / 255.0; + } + } + + let t_image = Tensor::from_array(input_tensor.into_dyn())?; + let t_num_tokens = Tensor::from_array(ndarray::arr0(NORMAL_NUM_TOKENS))?; + + let mut session = normal_session.lock().unwrap(); + let outputs = session.run(ort::inputs![ + "image" => t_image, + "num_tokens" => t_num_tokens, + ])?; + + let normal_tensor = outputs["normal"].try_extract_array::()?; + let mask_tensor = outputs["mask"].try_extract_array::()?; + let points_tensor = outputs["points"].try_extract_array::()?; + + if normal_tensor.shape() != [1, rh, rw, 3] + || mask_tensor.shape() != [1, rh, rw] + || points_tensor.shape() != [1, rh, rw, 3] + { + return Err(anyhow::anyhow!( + "Unexpected normal model output shape: {:?} / {:?} / {:?}", + normal_tensor.shape(), + mask_tensor.shape(), + points_tensor.shape() + )); + } + + let normals = normal_tensor.as_standard_layout(); + let normals = normals.as_slice().unwrap(); + let mask = mask_tensor.as_standard_layout(); + let mask = mask.as_slice().unwrap(); + + let pack = |v: f32| ((v * 0.5 + 0.5).clamp(0.0, 1.0) * 255.0).round() as u8; + + let mut normal_data = Vec::with_capacity(rw * rh * 4); + for (n, m) in normals.as_chunks::<3>().0.iter().zip(mask) { + // The model outputs OpenCV camera space (Y down, Z away from the viewer). + let (nx, ny, nz) = (n[0], -n[1], -n[2]); + let len = (nx * nx + ny * ny + nz * nz).sqrt(); + if len.is_finite() && len > 1e-6 { + normal_data.extend_from_slice(&[pack(nx / len), pack(ny / len), pack(nz / len)]); + } else { + normal_data.extend_from_slice(&[128, 128, 255]); + } + normal_data.push((m.clamp(0.0, 1.0) * 255.0).round() as u8); + } + + let normal = RgbaImage::from_raw(resized_w, resized_h, normal_data) + .ok_or_else(|| anyhow::anyhow!("Failed to create normal map from model output"))?; + + let points = points_tensor.as_standard_layout(); + let (depth_data, depth_scale) = build_depth_map(points.as_slice().unwrap(), mask, rw, rh); + let depth = ImageBuffer::from_raw(resized_w, resized_h, depth_data) + .ok_or_else(|| anyhow::anyhow!("Failed to create depth map from model output"))?; + + Ok(RelightMaps { + normal, + depth, + depth_scale, + }) +} + #[derive(Serialize, Deserialize, Debug, Clone, Default)] #[serde(rename_all = "camelCase")] pub struct AiSubjectMaskParameters { @@ -1757,3 +2039,42 @@ pub struct AiDepthMaskParameters { #[serde(default)] pub orientation_steps: Option, } + +#[cfg(test)] +mod relight_map_tests { + use super::build_depth_map; + + #[test] + fn depth_map_spans_nearest_to_farthest_and_matches_image_scale() { + // A 32x32 view with a 90 degree field of view (sensor / focal = 2) of a plane + // tilted so that depth grows from 2 on the left to 4 on the right. + let (w, h) = (32usize, 32usize); + let mut points = Vec::new(); + for y in 0..h { + for x in 0..w { + let z = 2.0 + 2.0 * x as f32 / (w - 1) as f32; + let xn = (x as f32 + 0.5 - 16.0) / 32.0; + let yn = (y as f32 + 0.5 - 16.0) / 32.0; + points.extend_from_slice(&[2.0 * z * xn, 2.0 * z * yn, z]); + } + } + let mask = vec![1.0f32; w * h]; + + let (depth, scale) = build_depth_map(&points, &mask, w, h); + + assert!(depth[0] < 1000); + assert!(depth[w - 1] > 64000); + assert!(depth[w / 2] > depth[w / 4]); + // Depth range ~2 over an image that is 2 * 3 = 6 units wide at the median depth. + assert!((scale - 2.0 / 6.0).abs() < 0.03, "scale was {scale}"); + } + + #[test] + fn image_without_geometry_has_no_depth() { + let points = [[0.0f32, 0.0, 1.0]; 64].concat(); + let (depth, scale) = build_depth_map(&points, &[0.0; 64], 8, 8); + + assert_eq!(scale, 0.0); + assert!(depth.iter().all(|d| *d == 0)); + } +} diff --git a/src-tauri/src/cache_utils.rs b/src-tauri/src/cache_utils.rs index b099105b67..7f1cf1d730 100644 --- a/src-tauri/src/cache_utils.rs +++ b/src-tauri/src/cache_utils.rs @@ -77,9 +77,42 @@ pub fn calculate_patched_warped_hash(adjustments: &serde_json::Value) -> u64 { } } + hash_relight(adjustments, &mut hasher); + hasher.finish() } +fn hash_relight(adjustments: &serde_json::Value, hasher: &mut DefaultHasher) { + let relight_enabled = adjustments["relightEnabled"].as_bool().unwrap_or(false); + relight_enabled.hash(hasher); + + if relight_enabled { + let relight_keys = [ + "relightLights", + "relightIntensity", + "relightX", + "relightY", + "relightDepth", + "relightRange", + "relightColor", + "relightAmbient", + "relightSoftness", + "relightSpecular", + "relightDetail", + "relightNormalMap", + "relightDepthMap", + "relightDepthScale", + ]; + + for key in relight_keys { + if let Some(val) = adjustments.get(key) { + key.hash(hasher); + val.to_string().hash(hasher); + } + } + } +} + pub fn calculate_thumbnail_base_hash(adjustments: &serde_json::Value) -> u64 { let mut hasher = DefaultHasher::new(); @@ -166,6 +199,8 @@ pub fn calculate_transform_hash(adjustments: &serde_json::Value) -> u64 { } } + hash_relight(adjustments, &mut hasher); + if let Some(crop_val) = adjustments.get("crop") && !crop_val.is_null() { diff --git a/src-tauri/src/file_management.rs b/src-tauri/src/file_management.rs index 267334fb25..d6d878eb52 100644 --- a/src-tauri/src/file_management.rs +++ b/src-tauri/src/file_management.rs @@ -1592,7 +1592,8 @@ pub fn generate_thumbnail_data( let warped_image = apply_geometry_warp(Cow::Borrowed(&composite_image), &meta.adjustments); - let blurred_image = crate::lens_blur::apply_lens_blur(warped_image, &meta.adjustments); + let relit_image = crate::relight::apply_relight(warped_image, &meta.adjustments); + let blurred_image = crate::lens_blur::apply_lens_blur(relit_image, &meta.adjustments); let orientation_steps = meta.adjustments["orientationSteps"].as_u64().unwrap_or(0) as u8; diff --git a/src-tauri/src/lens_blur.rs b/src-tauri/src/lens_blur.rs index 6a411752c7..ca8e4b6afe 100644 --- a/src-tauri/src/lens_blur.rs +++ b/src-tauri/src/lens_blur.rs @@ -134,15 +134,16 @@ fn depth_to_signed_coc( } #[allow(clippy::too_many_arguments)] -fn build_coc_field( - src: &Rgb32FImage, - depth: &image::GrayImage, - min_depth: f32, - max_depth: f32, - min_fade: f32, - max_fade: f32, - max_radius: f32, -) -> Vec { +/// Edge guide for joint upsampling: a lightly smoothed luma plane at full resolution +/// plus a copy downscaled so that its longest side is at most 1024 px. +pub(crate) struct LumaGuide { + pub luma_full: Vec, + pub guide: Vec, + pub gw: usize, + pub gh: usize, +} + +pub(crate) fn build_luma_guide(src: &Rgb32FImage) -> LumaGuide { let w = src.width() as usize; let h = src.height() as usize; let raw = src.as_raw(); @@ -196,6 +197,33 @@ fn build_coc_field( } }); + LumaGuide { + luma_full, + guide, + gw, + gh, + } +} + +fn build_coc_field( + src: &Rgb32FImage, + depth: &image::GrayImage, + min_depth: f32, + max_depth: f32, + min_fade: f32, + max_fade: f32, + max_radius: f32, +) -> Vec { + let w = src.width() as usize; + let h = src.height() as usize; + + let LumaGuide { + luma_full, + guide, + gw, + gh, + } = build_luma_guide(src); + let depth_small = dof_depth_to_f32(depth, gw, gh); let radius = ((gw.max(gh) as f32) * 0.015).round().max(2.0) as usize; let packed = build_guided_model(&guide, &depth_small, gw, gh, radius); @@ -243,7 +271,14 @@ fn build_coc_field( coc } -fn build_guided_model(guide: &[f32], p: &[f32], w: usize, h: usize, radius: usize) -> Vec { +/// Returns, per pixel, the local linear model `a * guide + b` and the `[lo, hi]` band it must stay in. +pub(crate) fn build_guided_model( + guide: &[f32], + p: &[f32], + w: usize, + h: usize, + radius: usize, +) -> Vec { let n = w * h; let eps = 4.0e-3f32; @@ -881,7 +916,7 @@ fn blur_layer_bokeh( out } -fn dof_box_filter(buf: &mut [f32], w: usize, h: usize, ch: usize, radius: usize) { +pub(crate) fn dof_box_filter(buf: &mut [f32], w: usize, h: usize, ch: usize, radius: usize) { if radius == 0 || w < 2 || h < 2 { return; } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 8bd7ce3a84..091a2dea81 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -56,6 +56,7 @@ mod panorama_stitching; mod panorama_utils; mod preset_converter; mod raw_processing; +mod relight; mod tagging; mod tagging_utils; mod white_balance; @@ -269,7 +270,8 @@ fn compute_patched_and_warped( }; let warped = apply_geometry_warp(patched_image, adjustments); - let blurred = crate::lens_blur::apply_lens_blur(warped, adjustments); + let relit = crate::relight::apply_relight(warped, adjustments); + let blurred = crate::lens_blur::apply_lens_blur(relit, adjustments); Ok(Arc::new(blurred.into_owned())) } @@ -863,8 +865,9 @@ async fn generate_uncropped_preview( Cow::Borrowed(loaded_image.image.as_ref()) }; + let relit_image = crate::relight::apply_relight(patched_image, &adjustments_clone); let blurred_image = - crate::lens_blur::apply_lens_blur(patched_image, &adjustments_clone); + crate::lens_blur::apply_lens_blur(relit_image, &adjustments_clone); let settings = load_settings(app_handle.clone()).unwrap_or_default(); let target_dim = (settings.editor_preview_resolution.unwrap_or(1920) as f32) as u32; @@ -2279,6 +2282,8 @@ pub fn run() { ai_commands::check_ai_connector_status, ai_commands::test_ai_connector_connection, ai_commands::generate_full_image_depth_map, + ai_commands::generate_relight_maps, + ai_commands::generate_relight_normal_preview, ai_commands::cancel_ai_task, apple_raw::is_raw9_available, inpainting::invoke_generative_replace_with_mask_def, diff --git a/src-tauri/src/relight.rs b/src-tauri/src/relight.rs new file mode 100644 index 0000000000..9c31b78623 --- /dev/null +++ b/src-tauri/src/relight.rs @@ -0,0 +1,1050 @@ +use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; +use image::{DynamicImage, GenericImageView, ImageBuffer, Luma, Rgb32FImage, RgbaImage}; +use rayon::prelude::*; +use std::borrow::Cow; +use std::sync::{Arc, Mutex}; + +use crate::lens_blur::{LumaGuide, build_guided_model, build_luma_guide, dof_box_filter}; + +const SPECULAR_POWER: f32 = 24.0; +// How far in front of the nearest surface the light can be pulled, in units of the longest image side. +const MAX_FRONT_DISTANCE: f32 = 1.5; + +// Below this size there is nothing to gain from snapping the maps to the image edges. +const MIN_REFINE_SIZE: u32 = 64; +const MAX_LIGHTS: usize = 8; +// Brightness slope to surface tilt, for a detail amount of 100. +const DETAIL_GAIN: f32 = 2.0; + +type DepthMap = ImageBuffer, Vec>; + +/// Normal, coverage and depth for every pixel of the image being relit. +struct SurfaceMaps { + normals: Vec<[i8; 3]>, + coverage: Vec, + depth: Vec, + // Fine relief (hair, pores, fabric) read from the photo's own texture, as an X / Y tilt + // to add to the normal. The AI maps are too coarse to carry it. + detail: Vec<[i8; 2]>, +} + +impl SurfaceMaps { + #[inline(always)] + fn normal(&self, i: usize, detail_strength: f32) -> (f32, f32, f32) { + let n = self.normals[i]; + let (nx, ny, nz) = ( + n[0] as f32 / 127.0, + n[1] as f32 / 127.0, + n[2] as f32 / 127.0, + ); + if detail_strength <= 0.0 { + return (nx, ny, nz); + } + + let d = self.detail[i]; + let nx = nx + d[0] as f32 / 127.0 * detail_strength; + let ny = ny + d[1] as f32 / 127.0 * detail_strength; + let inv_len = 1.0 / (nx * nx + ny * ny + nz * nz).sqrt().max(1e-6); + (nx * inv_len, ny * inv_len, nz * inv_len) + } +} + +// Building the maps only depends on the photo and the AI maps, never on the light, so the +// last result is kept around while the light is being moved. +static SURFACE_MAPS_CACHE: Mutex)>> = Mutex::new(None); + +// The on-screen normal map view keeps its own, smaller maps so that looking at it never +// evicts the full-size ones the relight pass is using. +const PREVIEW_MAX_SIZE: u32 = 2048; +type PreviewMaps = (blake3::Hash, (u32, u32), Arc); +static PREVIEW_MAPS_CACHE: Mutex> = Mutex::new(None); + +struct Light { + // Position as a fraction of the image width / height. + x: f32, + y: f32, + // Depth in units of the longest image side: 0 is the nearest surface, positive values + // move towards the viewer and negative values go into the scene. + z: f32, + // Falloff radius, in units of the longest image side. + radius: f32, + color: [f32; 3], + intensity: f32, +} + +struct RelightParams { + lights: Vec, + // Depth covered by the full range of the depth map, in units of the longest image side. + depth_scale: f32, + ambient: f32, + wrap: f32, + specular: f32, + detail: f32, +} + +pub fn apply_relight<'a>( + image: Cow<'a, DynamicImage>, + adjustments: &serde_json::Value, +) -> Cow<'a, DynamicImage> { + if !adjustments["relightEnabled"].as_bool().unwrap_or(false) { + return image; + } + + let (w, h) = image.dimensions(); + if w < 2 || h < 2 { + return image; + } + + let start = std::time::Instant::now(); + let mut out = image.as_ref().to_rgb32f(); + + let Some(maps) = cached_surface_maps(&out, adjustments) else { + return image; + }; + + let has_depth_map = !adjustments["relightDepthMap"] + .as_str() + .unwrap_or("") + .is_empty(); + let params = read_params(adjustments, has_depth_map); + relight_in_place(&mut out, &maps, ¶ms); + + log::info!("relight ({}x{}) took {:.2?}", w, h, start.elapsed()); + + Cow::Owned(DynamicImage::ImageRgb32F(out)) +} + +/// Renders the normal map for inspection: scaled up, snapped to the photo's edges and with +/// the detail amount applied, like the relight pass does. It works on a reduced copy of the +/// image, which is plenty for the screen and several times faster than the full size. +pub fn render_normal_preview( + image: &DynamicImage, + adjustments: &serde_json::Value, +) -> Option { + let normal_b64 = adjustments["relightNormalMap"].as_str().unwrap_or(""); + if normal_b64.is_empty() || image.width() < 2 || image.height() < 2 { + return None; + } + let depth_b64 = adjustments["relightDepthMap"].as_str().unwrap_or(""); + let depth_scale = adjustments["relightDepthScale"].as_f64().unwrap_or(0.0) as f32; + + // Keyed on the full-size image so that a hit does not even need the reduced copy. + let mut hasher = blake3::Hasher::new(); + hasher.update(&image.width().to_le_bytes()); + hasher.update(&image.height().to_le_bytes()); + hasher.update(&depth_scale.to_le_bytes()); + hasher.update(&(normal_b64.len() as u64).to_le_bytes()); + hasher.update(normal_b64.as_bytes()); + hasher.update(depth_b64.as_bytes()); + let bytes = image.as_bytes(); + let step = (bytes.len() / 4096).max(1); + for chunk in bytes.chunks(16).step_by(step.div_ceil(16)) { + hasher.update(chunk); + } + let cache_key = hasher.finalize(); + + let cached = PREVIEW_MAPS_CACHE + .lock() + .unwrap_or_else(|e| e.into_inner()) + .as_ref() + .filter(|(key, _, _)| *key == cache_key) + .map(|(_, size, maps)| (*size, Arc::clone(maps))); + + let ((w, h), maps) = match cached { + Some(cached) => cached, + None => { + let start = std::time::Instant::now(); + let small = if image.width().max(image.height()) > PREVIEW_MAX_SIZE { + image + .thumbnail(PREVIEW_MAX_SIZE, PREVIEW_MAX_SIZE) + .to_rgb32f() + } else { + image.to_rgb32f() + }; + let size = small.dimensions(); + + let normal_map = decode_data_url(normal_b64)?.into_rgba8(); + if normal_map.width() < 2 || normal_map.height() < 2 { + return None; + } + let depth_map = decode_data_url(depth_b64) + .map(|img| img.into_luma16()) + .filter(|depth| depth.dimensions() == normal_map.dimensions()); + + let refine = size.0 >= MIN_REFINE_SIZE && size.1 >= MIN_REFINE_SIZE; + let maps = Arc::new(build_surface_maps( + &small, + &normal_map, + depth_map.as_ref(), + depth_scale, + refine, + )); + log::info!( + "relight preview maps ({}x{}) built in {:.2?}", + size.0, + size.1, + start.elapsed() + ); + + *PREVIEW_MAPS_CACHE.lock().unwrap_or_else(|e| e.into_inner()) = + Some((cache_key, size, Arc::clone(&maps))); + (size, maps) + } + }; + + let detail = read_params(adjustments, false).detail; + // Areas without geometry (sky) are shown dark instead of as a made-up normal. + const EMPTY: f32 = 32.0; + + let mut raw = vec![0u8; w as usize * h as usize * 3]; + raw.par_chunks_exact_mut(3).enumerate().for_each(|(i, px)| { + let (nx, ny, nz) = maps.normal(i, detail); + let coverage = maps.coverage[i] as f32 / 255.0; + for (c, n) in [nx, ny, nz].into_iter().enumerate() { + let packed = (n * 0.5 + 0.5).clamp(0.0, 1.0) * 255.0; + px[c] = (EMPTY + (packed - EMPTY) * coverage).round() as u8; + } + }); + + image::RgbImage::from_raw(w, h, raw) +} + +fn cached_surface_maps( + image: &Rgb32FImage, + adjustments: &serde_json::Value, +) -> Option> { + let normal_b64 = adjustments["relightNormalMap"].as_str().unwrap_or(""); + if normal_b64.is_empty() { + return None; + } + let depth_b64 = adjustments["relightDepthMap"].as_str().unwrap_or(""); + let depth_scale = adjustments["relightDepthScale"].as_f64().unwrap_or(0.0) as f32; + let cache_key = surface_maps_key(image, normal_b64, depth_b64, depth_scale); + + let cached = SURFACE_MAPS_CACHE + .lock() + .unwrap_or_else(|e| e.into_inner()) + .as_ref() + .filter(|(key, _)| *key == cache_key) + .map(|(_, maps)| Arc::clone(maps)); + if cached.is_some() { + return cached; + } + + let start = std::time::Instant::now(); + let normal_map = decode_data_url(normal_b64)?.into_rgba8(); + if normal_map.width() < 2 || normal_map.height() < 2 { + return None; + } + let depth_map = decode_data_url(depth_b64) + .map(|img| img.into_luma16()) + .filter(|depth| depth.dimensions() == normal_map.dimensions()); + + let (w, h) = image.dimensions(); + let refine = w >= MIN_REFINE_SIZE && h >= MIN_REFINE_SIZE; + let maps = Arc::new(build_surface_maps( + image, + &normal_map, + depth_map.as_ref(), + depth_scale, + refine, + )); + log::info!( + "relight maps ({}x{}) built in {:.2?}", + w, + h, + start.elapsed() + ); + + *SURFACE_MAPS_CACHE.lock().unwrap_or_else(|e| e.into_inner()) = + Some((cache_key, Arc::clone(&maps))); + Some(maps) +} + +fn decode_data_url(data_url: &str) -> Option { + if data_url.is_empty() { + return None; + } + let b64_data = match data_url.find(',') { + Some(idx) => &data_url[idx + 1..], + None => data_url, + }; + let decoded = BASE64.decode(b64_data).ok()?; + image::load_from_memory(&decoded).ok() +} + +fn surface_maps_key( + image: &Rgb32FImage, + normal_b64: &str, + depth_b64: &str, + depth_scale: f32, +) -> blake3::Hash { + let mut hasher = blake3::Hasher::new(); + hasher.update(&image.width().to_le_bytes()); + hasher.update(&image.height().to_le_bytes()); + hasher.update(&depth_scale.to_le_bytes()); + hasher.update(&(normal_b64.len() as u64).to_le_bytes()); + hasher.update(normal_b64.as_bytes()); + hasher.update(depth_b64.as_bytes()); + + // A sparse sample is enough to notice that the photo itself changed. + let raw = image.as_raw(); + let step = (raw.len() / 4096).max(1); + for value in raw.iter().step_by(step) { + hasher.update(&value.to_le_bytes()); + } + + hasher.finalize() +} + +fn smoothstep(e0: f32, e1: f32, x: f32) -> f32 { + let t = ((x - e0) / (e1 - e0)).clamp(0.0, 1.0); + t * t * (3.0 - 2.0 * t) +} + +/// Position of a destination pixel inside a smaller source grid, for bilinear sampling. +struct Tap { + i0: usize, + i1: usize, + t: f32, +} + +fn taps(dst: usize, src: usize) -> Vec { + let scale = src as f32 / dst as f32; + (0..dst) + .map(|d| { + let f = ((d as f32 + 0.5) * scale - 0.5).clamp(0.0, (src - 1) as f32); + let i0 = f as usize; + Tap { + i0, + i1: (i0 + 1).min(src - 1), + t: f - i0 as f32, + } + }) + .collect() +} + +fn resample_plane(src: &[f32], sw: usize, sh: usize, dw: usize, dh: usize) -> Vec { + if sw == dw && sh == dh { + return src.to_vec(); + } + let x_taps = taps(dw, sw); + let y_taps = taps(dh, sh); + let mut out = vec![0.0f32; dw * dh]; + out.par_chunks_exact_mut(dw) + .enumerate() + .for_each(|(y, row)| { + let ty = &y_taps[y]; + for (x, value) in row.iter_mut().enumerate() { + let tx = &x_taps[x]; + let top = src[ty.i0 * sw + tx.i0] * (1.0 - tx.t) + src[ty.i0 * sw + tx.i1] * tx.t; + let bot = src[ty.i1 * sw + tx.i0] * (1.0 - tx.t) + src[ty.i1 * sw + tx.i1] * tx.t; + *value = top + (bot - top) * ty.t; + } + }); + out +} + +/// Local standard deviation of a plane, used to find where it has a hard edge. +fn local_deviation(plane: &[f32], w: usize, h: usize, radius: usize) -> Vec { + let mut mean = plane.to_vec(); + dof_box_filter(&mut mean, w, h, 1, radius); + let mut mean_sq: Vec = plane.iter().map(|v| v * v).collect(); + dof_box_filter(&mut mean_sq, w, h, 1, radius); + mean.iter() + .zip(&mean_sq) + .map(|(m, sq)| (sq - m * m).max(0.0).sqrt()) + .collect() +} + +/// How well the photo explains a plane around each pixel: the squared correlation between +/// the two, from 0 (unrelated) to 1 (the photo's edges are the plane's edges). +fn guide_agreement(guide: &[f32], plane: &[f32], w: usize, h: usize, radius: usize) -> Vec { + let blurred = |values: Vec| { + let mut values = values; + dof_box_filter(&mut values, w, h, 1, radius); + values + }; + let mean_i = blurred(guide.to_vec()); + let mean_p = blurred(plane.to_vec()); + let mean_ii = blurred(guide.iter().map(|v| v * v).collect()); + let mean_pp = blurred(plane.iter().map(|v| v * v).collect()); + let mean_ip = blurred(guide.iter().zip(plane).map(|(i, p)| i * p).collect()); + + (0..w * h) + .map(|k| { + let cov = mean_ip[k] - mean_i[k] * mean_p[k]; + let var_i = (mean_ii[k] - mean_i[k] * mean_i[k]).max(0.0); + let var_p = (mean_pp[k] - mean_p[k] * mean_p[k]).max(0.0); + (cov * cov / (var_i * var_p + 1e-9)).min(1.0) + }) + .collect() +} + +/// Scales the AI maps up to the image. With `refine`, silhouettes (hair, object outlines) +/// are snapped to the edges of the photo instead of being left as a soft blur, while the +/// inside of each surface keeps the detail the model predicted. +fn build_surface_maps( + image: &Rgb32FImage, + normal_map: &RgbaImage, + depth_map: Option<&DepthMap>, + depth_scale: f32, + refine: bool, +) -> SurfaceMaps { + let w = image.width() as usize; + let h = image.height() as usize; + let nw = normal_map.width() as usize; + let nh = normal_map.height() as usize; + + // Planes 0-2: normal packed to 0..1, 3: coverage, 4: depth. + let normal_raw = normal_map.as_raw(); + let mut planes: Vec> = (0..4) + .map(|c| { + normal_raw + .iter() + .skip(c) + .step_by(4) + .map(|v| *v as f32 / 255.0) + .collect() + }) + .collect(); + planes.push(match depth_map { + Some(depth) => depth.as_raw().iter().map(|v| *v as f32 / 65535.0).collect(), + None => vec![0.0; nw * nh], + }); + + let (gw, gh, models, edge_weight, luma_full) = if refine { + let LumaGuide { + luma_full, + guide, + gw, + gh, + } = build_luma_guide(image); + + planes = planes + .iter() + .map(|plane| resample_plane(plane, nw, nh, gw, gh)) + .collect(); + + let radius = ((gw.max(gh) as f32) * 0.015).round().max(2.0) as usize; + + // Only trust the photo's edges where the geometry really breaks (a depth jump or the + // border of the valid area) and the photo actually shows that break. Anywhere else + // the filter has nothing to follow and would just smear the maps. + let has_depth = depth_map.is_some() && depth_scale > 0.0; + let geometry_plane = if has_depth { &planes[4] } else { &planes[2] }; + let coverage_dev = local_deviation(&planes[3], gw, gh, radius); + let geometry_dev = local_deviation(geometry_plane, gw, gh, radius); + let coverage_agreement = guide_agreement(&guide, &planes[3], gw, gh, radius); + let geometry_agreement = guide_agreement(&guide, geometry_plane, gw, gh, radius); + + let mut edge_weight: Vec = (0..gw * gh) + .map(|k| { + let geometry_edge = if has_depth { + smoothstep(0.01, 0.06, geometry_dev[k] * depth_scale) + } else { + smoothstep(0.05, 0.2, geometry_dev[k]) + }; + let coverage_edge = smoothstep(0.05, 0.25, coverage_dev[k]); + (coverage_edge * smoothstep(0.25, 0.6, coverage_agreement[k])) + .max(geometry_edge * smoothstep(0.25, 0.6, geometry_agreement[k])) + }) + .collect(); + dof_box_filter(&mut edge_weight, gw, gh, 1, (radius / 2).max(1)); + + let models: Vec> = planes + .par_iter() + .map(|plane| build_guided_model(&guide, plane, gw, gh, radius)) + .collect(); + + (gw, gh, models, edge_weight, luma_full) + } else { + (nw, nh, Vec::new(), Vec::new(), Vec::new()) + }; + + let x_taps = taps(w, gw); + let y_taps = taps(h, gh); + + let mut normals = vec![[0i8; 3]; w * h]; + let mut coverage = vec![0u8; w * h]; + let mut depth = vec![0u16; w * h]; + let mut detail = vec![[0i8; 2]; w * h]; + + // A feature that is one pixel wide in a small render spans several pixels in a large + // one, so slopes are measured against a fixed reference size. + let detail_gain = DETAIL_GAIN * w.max(h) as f32 / 1000.0; + + normals + .par_chunks_exact_mut(w) + .zip(coverage.par_chunks_exact_mut(w)) + .zip(depth.par_chunks_exact_mut(w)) + .zip(detail.par_chunks_exact_mut(w)) + .enumerate() + .for_each( + |(y, (((normal_row, coverage_row), depth_row), detail_row))| { + let ty = &y_taps[y]; + let (r0, r1) = (ty.i0 * gw, ty.i1 * gw); + + for x in 0..w { + let tx = &x_taps[x]; + let (i00, i10, i01, i11) = (r0 + tx.i0, r0 + tx.i1, r1 + tx.i0, r1 + tx.i1); + let bilerp = |v00: f32, v10: f32, v01: f32, v11: f32| { + let top = v00 + (v10 - v00) * tx.t; + let bot = v01 + (v11 - v01) * tx.t; + top + (bot - top) * ty.t + }; + + let mut values = [0.0f32; 5]; + for (c, value) in values.iter_mut().enumerate() { + let plane = &planes[c]; + *value = bilerp(plane[i00], plane[i10], plane[i01], plane[i11]); + } + + if refine { + let weight = bilerp( + edge_weight[i00], + edge_weight[i10], + edge_weight[i01], + edge_weight[i11], + ); + + // Treat brightness as height and take its slope (Sobel). A silhouette is + // a change of object rather than relief, so it is left out. + let (xl, xr) = (x.saturating_sub(1), (x + 1).min(w - 1)); + let (yu, yd) = (y.saturating_sub(1) * w, (y + 1).min(h - 1) * w); + let yc = y * w; + let l = |row: usize, col: usize| luma_full[row + col]; + let slope_x = (l(yu, xr) + 2.0 * l(yc, xr) + l(yd, xr) + - l(yu, xl) + - 2.0 * l(yc, xl) + - l(yd, xl)) + * 0.125; + let slope_y = (l(yd, xl) + 2.0 * l(yd, x) + l(yd, xr) + - l(yu, xl) + - 2.0 * l(yu, x) + - l(yu, xr)) + * 0.125; + // The surface leans away from the uphill direction; image Y points down. + let tilt_x = -slope_x * detail_gain; + let tilt_y = slope_y * detail_gain; + let soft_clip = (1.0 - weight).max(0.0) + / (1.0 + (tilt_x * tilt_x + tilt_y * tilt_y).sqrt()); + detail_row[x] = [ + (tilt_x * soft_clip * 127.0).round() as i8, + (tilt_y * soft_clip * 127.0).round() as i8, + ]; + if weight > 1e-3 { + let luma = luma_full[y * w + x]; + for (c, value) in values.iter_mut().enumerate() { + let model = &models[c]; + let m = |k: usize| { + bilerp( + model[i00 * 4 + k], + model[i10 * 4 + k], + model[i01 * 4 + k], + model[i11 * 4 + k], + ) + }; + let mut guided = + (m(0) * luma + m(1)).clamp(m(2), m(3)).clamp(0.0, 1.0); + if c == 3 { + // Coverage is a matte: push it back towards solid / empty so the + // photo's texture does not leak into it as a halo. + guided = smoothstep(0.25, 0.75, guided); + } + *value += (guided - *value) * weight; + } + } + } + + let nx = values[0] * 2.0 - 1.0; + let ny = values[1] * 2.0 - 1.0; + let nz = values[2] * 2.0 - 1.0; + let scale = 127.0 / (nx * nx + ny * ny + nz * nz).sqrt().max(1e-6); + normal_row[x] = [ + (nx * scale).round() as i8, + (ny * scale).round() as i8, + (nz * scale).round() as i8, + ]; + coverage_row[x] = (values[3].clamp(0.0, 1.0) * 255.0).round() as u8; + depth_row[x] = (values[4].clamp(0.0, 1.0) * 65535.0).round() as u16; + } + }, + ); + + SurfaceMaps { + normals, + coverage, + depth, + detail, + } +} + +fn srgb_to_linear(v: f32) -> f32 { + if v <= 0.04045 { + v / 12.92 + } else { + ((v + 0.055) / 1.055).powf(2.4) + } +} + +/// Parses "#rrggbb" into a linear light tint that keeps roughly the same brightness as white. +fn parse_light_color(hex: &str) -> [f32; 3] { + let hex = hex.trim().trim_start_matches('#'); + if hex.len() != 6 || !hex.is_ascii() { + return [1.0; 3]; + } + + let mut color = [1.0f32; 3]; + for (i, c) in color.iter_mut().enumerate() { + match u8::from_str_radix(&hex[i * 2..i * 2 + 2], 16) { + Ok(v) => *c = srgb_to_linear(v as f32 / 255.0), + Err(_) => return [1.0; 3], + } + } + + let luma = 0.2126 * color[0] + 0.7152 * color[1] + 0.0722 * color[2]; + let norm = luma.max(0.3); + color.map(|c| c / norm) +} + +fn read_light(light: &serde_json::Value, depth_scale: f32) -> Light { + let num = |key: &str, default: f64| light[key].as_f64().unwrap_or(default) as f32; + + let range = (num("range", 60.0) / 100.0).clamp(0.0, 1.0); + + // Forwards the light moves linearly away from the scene. Backwards it eases in, so + // small moves place it just behind the subject even when the background is far away. + let depth = (num("depth", 30.0) / 100.0).clamp(-1.0, 1.0); + let z = if depth >= 0.0 { + 0.02 + depth * MAX_FRONT_DISTANCE + } else { + 0.02 - depth * depth * depth_scale.clamp(0.5, 10.0) + }; + + Light { + x: num("x", 0.3), + y: num("y", 0.3), + z, + radius: 0.1 + 1.9 * range * range, + color: parse_light_color(light["color"].as_str().unwrap_or("#ffffff")), + intensity: (num("intensity", 50.0) / 100.0).clamp(0.0, 1.0) * 2.0, + } +} + +fn read_params(adjustments: &serde_json::Value, has_depth_map: bool) -> RelightParams { + let num = |key: &str, default: f64| adjustments[key].as_f64().unwrap_or(default) as f32; + + let depth_scale = if has_depth_map { + num("relightDepthScale", 0.0).max(0.0) + } else { + 0.0 + }; + + let lights = match adjustments["relightLights"].as_array() { + Some(lights) => lights + .iter() + .take(MAX_LIGHTS) + .map(|light| read_light(light, depth_scale)) + .collect(), + // Edits saved before multiple lights existed describe a single light at the top level. + None => vec![read_light( + &serde_json::json!({ + "x": adjustments["relightX"], + "y": adjustments["relightY"], + "depth": adjustments["relightDepth"], + "range": adjustments["relightRange"], + "color": adjustments["relightColor"], + "intensity": adjustments["relightIntensity"], + }), + depth_scale, + )], + }; + + RelightParams { + lights, + depth_scale, + ambient: (num("relightAmbient", 80.0) / 100.0).clamp(0.0, 1.0), + wrap: (num("relightSoftness", 30.0) / 100.0).clamp(0.0, 1.0), + specular: (num("relightSpecular", 0.0) / 100.0).clamp(0.0, 1.0), + detail: (num("relightDetail", 30.0) / 100.0).clamp(0.0, 1.0) * 2.0, + } +} + +fn relight_in_place(image: &mut Rgb32FImage, maps: &SurfaceMaps, p: &RelightParams) { + let w = image.width() as usize; + let h = image.height() as usize; + + let depth_to_z = -p.depth_scale / 65535.0; + let inv_long_side = 1.0 / w.max(h) as f32; + + // Per light: position in pixels, depth and inverse squared falloff radius. + let lights: Vec<(f32, f32, f32, f32, &Light)> = p + .lights + .iter() + .filter(|light| light.intensity > 0.0) + .map(|light| { + ( + light.x * w as f32, + light.y * h as f32, + light.z, + 1.0 / (light.radius * light.radius), + light, + ) + }) + .collect(); + + image + .as_mut() + .par_chunks_exact_mut(w * 3) + .enumerate() + .for_each(|(y, row)| { + for (x, px) in row.as_chunks_mut::<3>().0.iter_mut().enumerate() { + let i = y * w + x; + + let coverage = maps.coverage[i] as f32 / 255.0; + if coverage <= 0.0 { + continue; + } + + let (nx, ny, nz) = maps.normal(i, p.detail); + let surface_z = maps.depth[i] as f32 * depth_to_z; + + let mut lit = [0.0f32; 3]; + for (light_px, light_py, light_z, inv_radius_sq, light) in &lights { + let lx = (light_px - (x as f32 + 0.5)) * inv_long_side; + // Image rows grow downwards while the normal map uses Y up. + let ly = -(light_py - (y as f32 + 0.5)) * inv_long_side; + let lz = light_z - surface_z; + let dist_sq = lx * lx + ly * ly + lz * lz; + let inv_l_len = 1.0 / dist_sq.sqrt().max(1e-6); + let (dx, dy, dz) = (lx * inv_l_len, ly * inv_l_len, lz * inv_l_len); + + let attenuation = 1.0 / (1.0 + dist_sq * inv_radius_sq); + + let n_dot_l = nx * dx + ny * dy + nz * dz; + let diffuse = ((n_dot_l + p.wrap) / (1.0 + p.wrap)).max(0.0); + + let spec = if p.specular > 0.0 && n_dot_l > 0.0 { + // Half vector between the light and the viewer (0, 0, 1). + let hz = dz + 1.0; + let inv_h_len = 1.0 / (dx * dx + dy * dy + hz * hz).sqrt().max(1e-6); + let n_dot_h = (nx * dx + ny * dy + nz * hz) * inv_h_len; + n_dot_h.max(0.0).powf(SPECULAR_POWER) * p.specular + } else { + 0.0 + }; + + let amount = light.intensity * attenuation * (diffuse + spec); + for (value, color) in lit.iter_mut().zip(light.color) { + *value += amount * color; + } + } + + for c in 0..3 { + let gain = p.ambient + lit[c]; + px[c] *= 1.0 + (gain - 1.0) * coverage; + } + } + }); +} + +#[cfg(test)] +mod tests { + use super::*; + use image::{Rgb, Rgb32FImage, Rgba}; + use serde_json::json; + + fn flat_normal_map(normal: [u8; 3], alpha: u8) -> RgbaImage { + RgbaImage::from_pixel(4, 4, Rgba([normal[0], normal[1], normal[2], alpha])) + } + + fn relit_pixel(adjustments: serde_json::Value, normal_map: &RgbaImage, x: u32) -> [f32; 3] { + relit_pixel_with_depth(adjustments, normal_map, None, x) + } + + fn relit_pixel_with_depth( + adjustments: serde_json::Value, + normal_map: &RgbaImage, + depth_map: Option<&DepthMap>, + x: u32, + ) -> [f32; 3] { + let mut image = Rgb32FImage::from_pixel(8, 8, Rgb([0.5, 0.5, 0.5])); + let params = read_params(&adjustments, depth_map.is_some()); + let maps = build_surface_maps(&image, normal_map, depth_map, params.depth_scale, false); + relight_in_place(&mut image, &maps, ¶ms); + image.get_pixel(x, 3).0 + } + + #[test] + fn surface_facing_the_light_gets_brighter_than_one_facing_away() { + let adjustments = json!({ + "relightX": 1.0, "relightY": 0.5, "relightDepth": 0, "relightSoftness": 0, "relightRange": 100, + }); + let facing_right = relit_pixel( + adjustments.clone(), + &flat_normal_map([255, 128, 128], 255), + 3, + ); + let facing_left = relit_pixel(adjustments, &flat_normal_map([0, 128, 128], 255), 3); + + assert!(facing_right[1] > 0.5); + assert!(facing_left[1] < 0.5); + } + + #[test] + fn light_above_the_pixel_brightens_upward_facing_surfaces() { + let adjustments = json!({ + "relightX": 0.5, "relightY": 0.0, "relightDepth": 0, "relightSoftness": 0, "relightRange": 100, + }); + let facing_up = relit_pixel( + adjustments.clone(), + &flat_normal_map([128, 255, 128], 255), + 3, + ); + let facing_down = relit_pixel(adjustments, &flat_normal_map([128, 0, 128], 255), 3); + + assert!(facing_up[1] > facing_down[1]); + } + + #[test] + fn light_falls_off_with_distance() { + let adjustments = + json!({ "relightX": 0.0, "relightY": 0.5, "relightDepth": 50, "relightRange": 20 }); + let normal_map = flat_normal_map([128, 128, 255], 255); + let near = relit_pixel(adjustments.clone(), &normal_map, 0); + let far = relit_pixel(adjustments, &normal_map, 7); + + assert!(near[1] > far[1]); + } + + #[test] + fn light_behind_a_surface_does_not_light_its_front() { + let normal_map = flat_normal_map([128, 128, 255], 255); + let depth_map = DepthMap::from_pixel(4, 4, Luma([0])); + let adjustments = |depth: i32| { + json!({ + "relightX": 0.5, "relightY": 0.5, "relightDepth": depth, "relightSoftness": 0, + "relightAmbient": 100, "relightDepthScale": 2.0, + }) + }; + + let in_front = relit_pixel_with_depth(adjustments(40), &normal_map, Some(&depth_map), 3); + let behind = relit_pixel_with_depth(adjustments(-40), &normal_map, Some(&depth_map), 3); + + assert!(in_front[1] > 0.5); + assert_eq!(behind[1], 0.5); + } + + #[test] + fn light_behind_the_subject_still_reaches_the_background() { + let normal_map = flat_normal_map([128, 128, 255], 255); + let adjustments = json!({ + "relightX": 0.5, "relightY": 0.5, "relightDepth": -40, "relightSoftness": 0, + "relightAmbient": 100, "relightDepthScale": 2.0, + }); + + let subject = DepthMap::from_pixel(4, 4, Luma([0])); + let background = DepthMap::from_pixel(4, 4, Luma([u16::MAX])); + let on_subject = + relit_pixel_with_depth(adjustments.clone(), &normal_map, Some(&subject), 3); + let on_background = relit_pixel_with_depth(adjustments, &normal_map, Some(&background), 3); + + assert_eq!(on_subject[1], 0.5); + assert!(on_background[1] > 0.5); + } + + #[test] + fn refinement_snaps_a_soft_depth_edge_to_the_photo_edge() { + // Photo: dark subject on the left, bright background on the right, hard edge at x = 2048. + let image = Rgb32FImage::from_fn(4096, 96, |x, _| { + if x < 2048 { + Rgb([0.05, 0.05, 0.05]) + } else { + Rgb([0.8, 0.8, 0.8]) + } + }); + // Maps: the same split at 1/16 of the resolution, so the edge comes out blurry. + let normal_map = RgbaImage::from_pixel(256, 6, Rgba([128, 128, 255, 255])); + let depth_map = + DepthMap::from_fn(256, 6, |x, _| Luma([if x < 128 { 0 } else { u16::MAX }])); + + // Depth change over the few pixels around the photo edge. + let depth_step = |refine: bool| { + let maps = build_surface_maps(&image, &normal_map, Some(&depth_map), 2.0, refine); + let row = 48 * 4096; + maps.depth[row + 2051] as f32 - maps.depth[row + 2045] as f32 + }; + + let soft = depth_step(false); + let refined = depth_step(true); + + assert!(soft > 0.0); + assert!(refined > soft * 1.5, "refined {refined} vs bilinear {soft}"); + } + + #[test] + fn detail_tilts_the_surface_along_the_photo_texture() { + // A bright ridge down the middle of a flat, camera-facing surface. + let image = Rgb32FImage::from_fn(1000, 64, |x, _| { + let v = 0.2 + 0.6 * (-((x as f32 - 500.0) / 6.0).powi(2)).exp(); + Rgb([v, v, v]) + }); + let normal_map = RgbaImage::from_pixel(16, 16, Rgba([128, 128, 255, 255])); + let maps = build_surface_maps(&image, &normal_map, None, 0.0, true); + let row = 32 * 1000; + + // Left of the ridge the surface climbs to the right, so it faces left; and vice versa. + let (left_nx, _, _) = maps.normal(row + 494, 1.0); + let (right_nx, _, _) = maps.normal(row + 506, 1.0); + assert!(left_nx < -0.05, "left side nx {left_nx}"); + assert!(right_nx > 0.05, "right side nx {right_nx}"); + + // Without detail the AI normal is used as is. + let (plain_nx, _, plain_nz) = maps.normal(row + 494, 0.0); + assert!(plain_nx.abs() < 0.02 && plain_nz > 0.98); + } + + #[test] + fn normal_preview_shows_normals_and_darkens_empty_areas() { + let image = DynamicImage::ImageRgb32F(Rgb32FImage::from_pixel(8, 8, Rgb([0.5, 0.5, 0.5]))); + let encode = |alpha: u8| { + let mut buf = std::io::Cursor::new(Vec::new()); + DynamicImage::ImageRgba8(flat_normal_map([128, 128, 255], alpha)) + .write_to(&mut buf, image::ImageFormat::Png) + .unwrap(); + format!("data:image/png;base64,{}", BASE64.encode(buf.get_ref())) + }; + + let solid = render_normal_preview(&image, &json!({ "relightNormalMap": encode(255) })); + let empty = render_normal_preview(&image, &json!({ "relightNormalMap": encode(0) })); + + let solid = solid.unwrap().get_pixel(3, 3).0; + assert!(solid[2] > 250 && (solid[0] as i32 - 128).abs() <= 2); + assert_eq!(empty.unwrap().get_pixel(3, 3).0, [32, 32, 32]); + assert!(render_normal_preview(&image, &json!({})).is_none()); + } + + #[test] + fn normal_preview_is_rendered_from_a_reduced_copy_of_large_images() { + let image = DynamicImage::ImageRgb32F(Rgb32FImage::from_pixel(4096, 1024, Rgb([0.5; 3]))); + let mut buf = std::io::Cursor::new(Vec::new()); + DynamicImage::ImageRgba8(flat_normal_map([200, 128, 220], 255)) + .write_to(&mut buf, image::ImageFormat::Png) + .unwrap(); + let adjustments = json!({ + "relightNormalMap": format!("data:image/png;base64,{}", BASE64.encode(buf.get_ref())), + }); + + let preview = render_normal_preview(&image, &adjustments).unwrap(); + assert_eq!(preview.dimensions(), (2048, 512)); + let px = preview.get_pixel(1000, 200).0; + assert!( + px[0] > 180 && px[2] > 200, + "unexpected normal colour {px:?}" + ); + + // A second request for the same image is served from the cached maps. + let again = render_normal_preview(&image, &adjustments).unwrap(); + assert_eq!(preview, again); + } + + #[test] + fn refinement_leaves_edges_alone_when_the_photo_does_not_show_them() { + // The geometry has a hard edge (depth and normal change at the middle), but the photo + // is evenly lit noise there, so it gives no hint about where that edge really is. + let image = Rgb32FImage::from_fn(512, 128, |x, y| { + let v = 0.4 + 0.1 * (((x * 7 + y * 13) % 11) as f32 / 11.0); + Rgb([v, v, v]) + }); + let normal_map = RgbaImage::from_fn(256, 64, |x, _| { + if x < 128 { + Rgba([60, 128, 230, 255]) + } else { + Rgba([200, 128, 230, 255]) + } + }); + let depth_map = + DepthMap::from_fn(256, 64, |x, _| Luma([if x < 128 { 0 } else { u16::MAX }])); + + let plain = build_surface_maps(&image, &normal_map, Some(&depth_map), 2.0, false); + let refined = build_surface_maps(&image, &normal_map, Some(&depth_map), 2.0, true); + + // Guessing would smear the edge over a wide band; it has to stay as sharp as it was. + let row = 64 * 512; + for x in 236..276 { + let (a, b) = (plain.normals[row + x], refined.normals[row + x]); + assert!( + (a[0] as i32 - b[0] as i32).abs() <= 6, + "normal at x={x} changed from {a:?} to {b:?}" + ); + } + } + + #[test] + fn refinement_keeps_normals_away_from_edges() { + let image = Rgb32FImage::from_fn(128, 128, |x, y| { + let v = ((x / 8 + y / 8) % 2) as f32 * 0.5 + 0.2; + Rgb([v, v, v]) + }); + // A textured photo over a flat, evenly tilted surface: there is no geometric edge to snap to. + let normal_map = RgbaImage::from_pixel(16, 16, Rgba([200, 128, 220, 255])); + let depth_map = DepthMap::from_pixel(16, 16, Luma([1000])); + + let plain = build_surface_maps(&image, &normal_map, Some(&depth_map), 2.0, false); + let refined = build_surface_maps(&image, &normal_map, Some(&depth_map), 2.0, true); + + assert_eq!(plain.normals, refined.normals); + assert_eq!(plain.depth, refined.depth); + } + + #[test] + fn several_lights_add_up_with_their_own_colors() { + let normal_map = flat_normal_map([128, 128, 255], 255); + let red = json!({ "x": 0.0, "y": 0.5, "color": "#ff0000", "range": 20 }); + let blue = json!({ "x": 1.0, "y": 0.5, "color": "#0000ff", "range": 20 }); + let both = json!({ "relightLights": [red, blue] }); + + let near_red = relit_pixel(both.clone(), &normal_map, 0); + let near_blue = relit_pixel(both.clone(), &normal_map, 7); + assert!(near_red[0] > near_red[2]); + assert!(near_blue[2] > near_blue[0]); + + let only_red = relit_pixel(json!({ "relightLights": [red] }), &normal_map, 0); + assert!(near_red[2] > only_red[2]); + } + + #[test] + fn no_lights_leaves_only_the_ambient_level() { + let adjustments = json!({ "relightLights": [], "relightAmbient": 50 }); + let lit = relit_pixel(adjustments, &flat_normal_map([128, 128, 255], 255), 3); + + assert_eq!(lit, [0.25; 3]); + } + + #[test] + fn light_color_tints_the_lit_surface() { + let adjustments = json!({ "relightX": 0.5, "relightY": 0.5, "relightColor": "#ff8000" }); + let lit = relit_pixel(adjustments, &flat_normal_map([128, 128, 255], 255), 3); + + assert!(lit[0] > lit[1] && lit[1] > lit[2]); + } + + #[test] + fn invalid_light_color_falls_back_to_white() { + assert_eq!(parse_light_color("not-a-color"), [1.0; 3]); + assert!( + parse_light_color("#ffffff") + .iter() + .all(|c| (c - 1.0).abs() < 1e-4) + ); + } + + #[test] + fn pixels_without_geometry_are_left_untouched() { + let adjustments = json!({ "relightIntensity": 100, "relightAmbient": 0 }); + let lit = relit_pixel(adjustments, &flat_normal_map([128, 128, 255], 0), 3); + + assert_eq!(lit, [0.5; 3]); + } +} diff --git a/src/App.tsx b/src/App.tsx index 1043fcf581..d8ef2afb9d 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -31,6 +31,7 @@ import CropPanel from './components/panel/right/CropPanel'; import MasksPanel from './components/panel/right/MasksPanel'; import AIPanel from './components/panel/right/AIPanel'; import PresetsPanel from './components/panel/right/PresetsPanel'; +import RelightPanel from './components/panel/right/RelightPanel'; import TetheringPanel from './components/panel/right/TetheringPanel'; import EditorView from './components/views/EditorView'; @@ -754,6 +755,8 @@ function App() { return ; case Panel.Presets: return setUI({ activeView: 'community' })} />; + case Panel.Relight: + return ; case Panel.Tethering: return ; default: diff --git a/src/components/panel/Editor.tsx b/src/components/panel/Editor.tsx index fabbadc71c..c021406f78 100644 --- a/src/components/panel/Editor.tsx +++ b/src/components/panel/Editor.tsx @@ -871,7 +871,10 @@ export default function Editor({ onBackToLibrary, onContextMenu, onImageSelect, const handlePointerDown = useCallback( (e: React.PointerEvent) => { - wasPanningDisabledOnDown.current = isPanningDisabled; + // This runs in the capture phase, so the light handle cannot stop it by itself. + const isRelightHandle = !!(e.target as HTMLElement).closest('[data-relight-handle]'); + wasPanningDisabledOnDown.current = isPanningDisabled || isRelightHandle; + if (isRelightHandle) return; const isCropHandle = isCropping && e.button === 0 && !!(e.target as HTMLElement).closest('[data-ord]'); cropResizeStartRef.current = isCropHandle ? lastValidCropRef.current : null; diff --git a/src/components/panel/PanelSwitcher.tsx b/src/components/panel/PanelSwitcher.tsx index 4978036a63..8ad7e3e70d 100644 --- a/src/components/panel/PanelSwitcher.tsx +++ b/src/components/panel/PanelSwitcher.tsx @@ -10,6 +10,7 @@ import { Layers, Paintbrush, SwatchBook, + Lightbulb, FileInput, Camera, Folder as FolderIcon, @@ -25,6 +26,7 @@ export const PANEL_ICONS: Record = { [Panel.Masks]: Layers, [Panel.Ai]: Paintbrush, [Panel.Presets]: SwatchBook, + [Panel.Relight]: Lightbulb, [Panel.Export]: FileInput, [Panel.FolderTree]: FolderIcon, [Panel.Tethering]: Camera, @@ -37,6 +39,7 @@ const PANEL_TITLES: Record = { [Panel.Masks]: 'editor.switcher.tooltips.masks', [Panel.Ai]: 'editor.switcher.tooltips.inpaint', [Panel.Presets]: 'editor.switcher.tooltips.presets', + [Panel.Relight]: 'editor.switcher.tooltips.relight', [Panel.Export]: 'editor.switcher.tooltips.export', [Panel.FolderTree]: 'library.folders.sourcesTitle', [Panel.Tethering]: 'editor.switcher.tooltips.tethering', @@ -256,6 +259,7 @@ export function MobilePanelSwitcher({ Panel.Masks, Panel.Ai, Panel.Presets, + Panel.Relight, Panel.Export, ]; diff --git a/src/components/panel/editor/ImageCanvas.tsx b/src/components/panel/editor/ImageCanvas.tsx index 6cfd0729a1..8eae9e0472 100644 --- a/src/components/panel/editor/ImageCanvas.tsx +++ b/src/components/panel/editor/ImageCanvas.tsx @@ -5,15 +5,25 @@ import { Stage, Layer, Ellipse, Line, Transformer, Group, Circle, Rect, Arrow } import { PercentCrop, Crop } from 'react-image-crop'; import { Stamp, Bandage, Spline, BrushCleaning } from 'lucide-react'; import { invoke } from '@tauri-apps/api/core'; -import { Adjustments, AiPatch, Coord, MaskContainer, GuideLine, GuideOrientation } from '../../../utils/adjustments'; +import { + Adjustments, + AiPatch, + Coord, + MaskContainer, + GuideLine, + GuideOrientation, + RelightLight, +} from '../../../utils/adjustments'; import { Mask, SubMask, SubMaskMode, ToolType } from '../right/Masks'; -import { AppSettings, BrushSettings, Invokes, SelectedImage } from '../../ui/AppProperties'; +import { AppSettings, BrushSettings, Invokes, Panel, SelectedImage } from '../../ui/AppProperties'; import { RenderSize } from '../../../hooks/useImageRenderSize'; import { useOsPlatform } from '../../../hooks/useOsPlatform'; import { useTranslation } from 'react-i18next'; import { useEditorStore } from '../../../store/useEditorStore'; +import { useUIStore } from '../../../store/useUIStore'; import type { OverlayMode } from '../right/CropPanel'; import CompositionOverlays from './overlays/CompositionOverlays'; +import RelightHandle from './overlays/RelightHandle'; import { calculateStraightenAngle } from '../../../utils/cropUtils'; import { toast } from 'react-toastify'; import { @@ -23,6 +33,7 @@ import { withKelvinWhiteBalance, withRelativeWhiteBalance, } from '../../../utils/whiteBalance'; +import { RelightGeometry } from '../../../utils/relightUtils'; interface CursorPreview { visible: boolean; @@ -1739,6 +1750,118 @@ const ImageCanvas = memo( : activeCrop.y : 0; + const isRelightPanelActive = useUIStore((state) => Object.values(state.activePanels).includes(Panel.Relight)); + const activeRotation = + liveRotation !== null && liveRotation !== undefined ? liveRotation : adjustments.rotation || 0; + const relightGeometry = useMemo( + () => ({ + imageWidth: selectedImage.width || 0, + imageHeight: selectedImage.height || 0, + orientationSteps: adjustments.orientationSteps || 0, + flipHorizontal: !!adjustments.flipHorizontal, + flipVertical: !!adjustments.flipVertical, + rotation: activeRotation, + cropX, + cropY, + }), + [ + selectedImage.width, + selectedImage.height, + adjustments.orientationSteps, + adjustments.flipHorizontal, + adjustments.flipVertical, + activeRotation, + cropX, + cropY, + ], + ); + const relightLights = adjustments.relightLights ?? []; + const activeRelightLightId = useEditorStore((state) => state.activeRelightLightId); + const setEditor = useEditorStore((state) => state.setEditor); + const handleRelightLightSelect = useCallback((id: string) => setEditor({ activeRelightLightId: id }), [setEditor]); + const handleRelightLightChange = useCallback( + (id: string, changes: Partial) => { + setAdjustments((prev: Adjustments) => ({ + ...prev, + relightLights: (prev.relightLights ?? []).map((light) => + light.id === id ? { ...light, ...changes } : light, + ), + })); + }, + [setAdjustments], + ); + const isRelightHandleVisible = + !!adjustments.relightEnabled && + !!adjustments.relightNormalMap && + isRelightPanelActive && + !isCropping && + !isMasking && + !isAiEditing && + !isWbPickerActive && + !showOriginal && + relightGeometry.imageWidth > 0 && + imageRenderSize.width > 0; + + // Normal map view: the backend renders the map exactly as the relight pass uses it, + // already laid out like the edited image, so it can simply cover the photo. + const showRelightNormalMap = useEditorStore((state) => state.showRelightNormalMap); + const isRelightNormalMapVisible = isRelightHandleVisible && showRelightNormalMap; + const [relightNormalPreviewUrl, setRelightNormalPreviewUrl] = useState(null); + const relightNormalPreviewRequest = useRef(0); + const relightPreviewAdjustmentsRef = useRef(adjustments); + relightPreviewAdjustmentsRef.current = adjustments; + const relightPreviewLayoutKey = JSON.stringify([ + adjustments.crop, + adjustments.rotation, + adjustments.flipHorizontal, + adjustments.flipVertical, + adjustments.orientationSteps, + ]); + + const showRelightNormalPreview = useCallback((url: string | null) => { + setRelightNormalPreviewUrl((previous) => { + if (previous) URL.revokeObjectURL(previous); + return url; + }); + }, []); + + useEffect(() => () => showRelightNormalPreview(null), [showRelightNormalPreview]); + + useEffect(() => { + const request = ++relightNormalPreviewRequest.current; + if (!isRelightNormalMapVisible) { + showRelightNormalPreview(null); + return; + } + + // A short pause so that dragging the detail slider does not queue a render per step. + const timeout = window.setTimeout(async () => { + try { + // Masks, patches and LUT data play no part here and only make the request heavier. + const payload: Record = { ...relightPreviewAdjustmentsRef.current }; + for (const key of ['masks', 'aiPatches', 'lutData', 'lensBlurDepthMap']) delete payload[key]; + + const bytes: ArrayBuffer = await invoke('generate_relight_normal_preview', { jsAdjustments: payload }); + if (request !== relightNormalPreviewRequest.current) return; + showRelightNormalPreview(URL.createObjectURL(new Blob([new Uint8Array(bytes)], { type: 'image/jpeg' }))); + } catch (e) { + if (request === relightNormalPreviewRequest.current) { + showRelightNormalPreview(null); + toast.error(`Failed to render normal map: ${e}`); + } + } + }, 40); + return () => window.clearTimeout(timeout); + }, [ + isRelightNormalMapVisible, + adjustments.relightNormalMap, + adjustments.relightDepthMap, + adjustments.relightDepthScale, + adjustments.relightDetail, + relightPreviewLayoutKey, + showRelightNormalPreview, + ]); + const effectiveZoomScale = transformState.scale > 0 ? transformState.scale : 1; const brushStageSize = (brushSettings?.size ?? 0) / effectiveZoomScale; const brushImageSpaceSize = brushStageSize / (imageRenderSize.scale || 1); @@ -3320,6 +3443,23 @@ const ImageCanvas = memo( )} + {isRelightNormalMapVisible && relightNormalPreviewUrl && ( + {t('adjustments.effects.relightNormalMap')} + )} + {displayedMaskUrl && ( Mask Overlay
+ {isRelightHandleVisible && + relightLights.map((light) => ( + 1 && (activeRelightLightId ?? relightLights[0].id) === light.id} + geometry={relightGeometry} + imageRenderSize={imageRenderSize} + inverseScale={1 / maxSafeScale} + onSelect={handleRelightLightSelect} + onChange={handleRelightLightChange} + /> + ))} + {!isDrawing.current && directPatchMarkers.map((m) => { const left = (m.cx - cropX) * imageRenderSize.scale + imageRenderSize.offsetX; diff --git a/src/components/panel/editor/overlays/RelightHandle.tsx b/src/components/panel/editor/overlays/RelightHandle.tsx new file mode 100644 index 0000000000..6f8d31667e --- /dev/null +++ b/src/components/panel/editor/overlays/RelightHandle.tsx @@ -0,0 +1,168 @@ +import { useCallback, useEffect, useRef, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { Coord, RelightLight } from '../../../../utils/adjustments'; +import { RenderSize } from '../../../../hooks/useImageRenderSize'; +import { RelightGeometry, relightDisplayToUv, relightUvToDisplay } from '../../../../utils/relightUtils'; + +interface RelightHandleProps { + light: RelightLight; + isActive: boolean; + geometry: RelightGeometry; + imageRenderSize: RenderSize; + inverseScale: number; + onSelect(id: string): void; + onChange(id: string, changes: Partial): void; +} + +const HANDLE_SIZE = 30; +const WHEEL_DEPTH_STEP = 4; + +const clamp01 = (value: number) => Math.max(0, Math.min(1, value)); + +export default function RelightHandle({ + light, + isActive, + geometry, + imageRenderSize, + inverseScale, + onSelect, + onChange, +}: RelightHandleProps) { + const { t } = useTranslation(); + const handleRef = useRef(null); + const frameRef = useRef(null); + const pendingUvRef = useRef(null); + const depthRef = useRef(light.depth); + const [dragUv, setDragUv] = useState(null); + + depthRef.current = light.depth; + + useEffect( + () => () => { + if (frameRef.current !== null) cancelAnimationFrame(frameRef.current); + }, + [], + ); + + // The editor zooms with a native wheel listener, so this has to be native too to get in first. + useEffect(() => { + const handle = handleRef.current; + if (!handle) return; + + const handleWheel = (e: WheelEvent) => { + e.preventDefault(); + e.stopPropagation(); + const step = e.deltaY < 0 ? WHEEL_DEPTH_STEP : -WHEEL_DEPTH_STEP; + const depth = Math.max(-100, Math.min(100, depthRef.current + step)); + depthRef.current = depth; + onSelect(light.id); + onChange(light.id, { depth }); + }; + + handle.addEventListener('wheel', handleWheel, { passive: false }); + return () => handle.removeEventListener('wheel', handleWheel); + }, [light.id, onSelect, onChange]); + + const commitUv = useCallback( + (uv: Coord) => { + pendingUvRef.current = uv; + if (frameRef.current !== null) return; + frameRef.current = requestAnimationFrame(() => { + frameRef.current = null; + const next = pendingUvRef.current; + if (!next) return; + onChange(light.id, { x: next.x, y: next.y }); + }); + }, + [light.id, onChange], + ); + + const pointerToUv = useCallback( + (clientX: number, clientY: number): Coord | null => { + const container = handleRef.current?.parentElement; + if (!container || !container.offsetWidth || !imageRenderSize.scale) return null; + + // The container sits inside the zoom/pan transform, so undo its on-screen scale. + const rect = container.getBoundingClientRect(); + const zoom = rect.width / container.offsetWidth || 1; + const localX = (clientX - rect.left) / zoom; + const localY = (clientY - rect.top) / zoom; + + const uv = relightDisplayToUv( + { + x: (localX - imageRenderSize.offsetX) / imageRenderSize.scale, + y: (localY - imageRenderSize.offsetY) / imageRenderSize.scale, + }, + geometry, + ); + return { x: clamp01(uv.x), y: clamp01(uv.y) }; + }, + [geometry, imageRenderSize], + ); + + const handlePointerDown = (e: React.PointerEvent) => { + e.stopPropagation(); + e.preventDefault(); + e.currentTarget.setPointerCapture(e.pointerId); + onSelect(light.id); + setDragUv({ x: light.x, y: light.y }); + }; + + const handlePointerMove = (e: React.PointerEvent) => { + if (!dragUv) return; + e.stopPropagation(); + const uv = pointerToUv(e.clientX, e.clientY); + if (!uv) return; + setDragUv(uv); + commitUv(uv); + }; + + const handlePointerUp = (e: React.PointerEvent) => { + if (!dragUv) return; + e.stopPropagation(); + if (e.currentTarget.hasPointerCapture(e.pointerId)) { + e.currentTarget.releasePointerCapture(e.pointerId); + } + setDragUv(null); + }; + + const uv = dragUv ?? { x: light.x, y: light.y }; + const display = relightUvToDisplay(uv, geometry); + const color = light.color || '#ffffff'; + const depth = Math.max(-100, Math.min(100, light.depth)); + // Closer lights look bigger; a light behind the subject gets a dashed outline. + const depthScale = 0.7 + 0.6 * ((depth + 100) / 200); + const isBehind = depth < 0; + const outline = isActive ? '0 0 0 2.5px var(--color-accent, #ffffff), ' : ''; + + return ( +
e.stopPropagation()} + onTouchStart={(e) => e.stopPropagation()} + onClick={(e) => e.stopPropagation()} + style={{ + left: display.x * imageRenderSize.scale + imageRenderSize.offsetX, + top: display.y * imageRenderSize.scale + imageRenderSize.offsetY, + width: HANDLE_SIZE, + height: HANDLE_SIZE, + transform: `translate(-50%, -50%) scale(${inverseScale * depthScale})`, + transformOrigin: 'center', + cursor: dragUv ? 'grabbing' : 'grab', + touchAction: 'none', + zIndex: isActive ? 1 : 0, + background: `radial-gradient(circle at 35% 30%, #ffffff 0%, ${color} 38%, color-mix(in srgb, ${color} 45%, #000000) 100%)`, + border: `1.5px ${isBehind ? 'dashed' : 'solid'} rgba(255, 255, 255, 0.9)`, + opacity: isBehind ? 0.75 : 1, + boxShadow: `${outline}0 0 0 1px rgba(0, 0, 0, 0.45), 0 0 14px 4px color-mix(in srgb, ${color} 55%, transparent), 0 2px 6px rgba(0, 0, 0, 0.5)`, + }} + /> + ); +} diff --git a/src/components/panel/right/RelightPanel.tsx b/src/components/panel/right/RelightPanel.tsx new file mode 100644 index 0000000000..a65b909a6f --- /dev/null +++ b/src/components/panel/right/RelightPanel.tsx @@ -0,0 +1,431 @@ +import { useCallback, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { invoke } from '@tauri-apps/api/core'; +import { toast } from 'react-toastify'; +import { Loader2, Plus, RotateCcw, Trash2 } from 'lucide-react'; +import clsx from 'clsx'; +import { v4 as uuidv4 } from 'uuid'; +import Slider from '../../ui/Slider'; +import Switch from '../../ui/Switch'; +import Button from '../../ui/Button'; +import Text from '../../ui/Text'; +import { TextColors, TextVariants, TextWeights } from '../../../types/typography'; +import { + Adjustments, + Effect, + INITIAL_ADJUSTMENTS, + RelightLight, + RelightQuality, + DEFAULT_RELIGHT_LIGHT, + MAX_RELIGHT_LIGHTS, +} from '../../../utils/adjustments'; +import { RELIGHT_TEMPERATURE_PRESETS, kelvinToHex } from '../../../utils/relightUtils'; +import { useEditorStore } from '../../../store/useEditorStore'; +import { useProcessStore } from '../../../store/useProcessStore'; +import { useEditorActions } from '../../../hooks/useEditorActions'; + +const NEW_RELIGHT_LIGHT_TEMPERATURES = [5600, 3200, 8000, 2700, 10000, 4000]; + +const RELIGHT_ADJUSTMENT_KEYS: Array = [ + Effect.RelightEnabled, + Effect.RelightLights, + Effect.RelightAmbient, + Effect.RelightSoftness, + Effect.RelightSpecular, + Effect.RelightDetail, + Effect.RelightNormalMap, + Effect.RelightQuality, + Effect.RelightDepthMap, + Effect.RelightDepthScale, +]; + +export default function RelightPanel() { + const { t } = useTranslation(); + const { setAdjustments } = useEditorActions(); + const adjustments = useEditorStore((state) => state.adjustments); + const selectedImage = useEditorStore((state) => state.selectedImage); + const activeRelightLightId = useEditorStore((state) => state.activeRelightLightId); + const showRelightNormalMap = useEditorStore((state) => state.showRelightNormalMap); + const setEditor = useEditorStore((state) => state.setEditor); + const aiModelDownloadStatus = useProcessStore((state) => state.aiModelDownloadStatus); + const [isGeneratingNormal, setIsGeneratingNormal] = useState(false); + + const onDragStateChange = useCallback( + (isDragging: boolean) => setEditor({ isSliderDragging: isDragging }), + [setEditor], + ); + + const relightQuality: RelightQuality = adjustments.relightQuality ?? 'standard'; + + // Maps are saved per image and quality, so this only runs the model when there is no + // saved copy yet, or when `force` asks for a fresh one. + const handleGenerateRelightNormalMap = async (quality: RelightQuality = relightQuality, force = false) => { + setIsGeneratingNormal(true); + try { + const maps: { normalMap: string; depthMap: string; depthScale: number } = await invoke('generate_relight_maps', { + jsAdjustments: adjustments, + quality, + force, + }); + setAdjustments((prev: Adjustments) => ({ + ...prev, + relightNormalMap: maps.normalMap, + relightQuality: quality, + relightDepthMap: maps.depthMap, + relightDepthScale: maps.depthScale, + })); + } catch (e) { + toast.error(`Failed to generate normal map: ${e}`); + setAdjustments((prev: Adjustments) => ({ ...prev, relightEnabled: false })); + } finally { + setIsGeneratingNormal(false); + } + }; + + const handleSliderChange = (key: keyof Adjustments, value: string | number) => { + setAdjustments((prev: Adjustments) => ({ ...prev, [key]: Number(value) })); + }; + + const relightLights = adjustments.relightLights ?? []; + const activeRelightLight = relightLights.find((light) => light.id === activeRelightLightId) ?? relightLights[0]; + const relightColor = (activeRelightLight?.color || '#ffffff').toLowerCase(); + + const updateActiveRelightLight = (changes: Partial) => { + if (!activeRelightLight) return; + const id = activeRelightLight.id; + setAdjustments((prev: Adjustments) => ({ + ...prev, + relightLights: (prev.relightLights ?? []).map((light) => (light.id === id ? { ...light, ...changes } : light)), + })); + }; + + const handleRelightColorChange = (color: string) => { + updateActiveRelightLight({ color: color.toLowerCase() }); + }; + + const handleAddRelightLight = () => { + if (relightLights.length >= MAX_RELIGHT_LIGHTS) return; + // Each new light starts somewhere else and with another tint, so it is easy to tell apart. + const index = relightLights.length; + const light: RelightLight = { + ...DEFAULT_RELIGHT_LIGHT, + id: uuidv4(), + x: [0.7, 0.5, 0.3, 0.7, 0.5][index % 5], + y: [0.3, 0.7, 0.7, 0.7, 0.2][index % 5], + color: kelvinToHex(NEW_RELIGHT_LIGHT_TEMPERATURES[index % NEW_RELIGHT_LIGHT_TEMPERATURES.length]), + }; + setAdjustments((prev: Adjustments) => ({ + ...prev, + relightLights: [...(prev.relightLights ?? []), light], + })); + setEditor({ activeRelightLightId: light.id }); + }; + + const handleRemoveRelightLight = () => { + if (!activeRelightLight || relightLights.length <= 1) return; + const id = activeRelightLight.id; + setAdjustments((prev: Adjustments) => ({ + ...prev, + relightLights: (prev.relightLights ?? []).filter((light) => light.id !== id), + })); + setEditor({ activeRelightLightId: null }); + }; + + const handleRelightToggle = (enabled: boolean) => { + setAdjustments((prev: Adjustments) => ({ ...prev, relightEnabled: enabled })); + if (enabled && (!adjustments.relightNormalMap || !adjustments.relightDepthMap)) { + handleGenerateRelightNormalMap(); + } + }; + + const handleReset = () => { + setAdjustments((prev: Adjustments) => ({ + ...prev, + ...Object.fromEntries(RELIGHT_ADJUSTMENT_KEYS.map((key) => [key, INITIAL_ADJUSTMENTS[key]])), + })); + setEditor({ activeRelightLightId: null, showRelightNormalMap: false }); + }; + + return ( +
+
+ {t('adjustments.effects.relight')} + +
+ +
+ {selectedImage ? ( + <> + + +
+
+
+ {isGeneratingNormal ? ( +
+
+ + + {aiModelDownloadStatus + ? t('editor.masks.settings.aiModelDownloading') + : t('editor.ai.generatingNormalMap')} + +
+ {aiModelDownloadStatus && ( + + {aiModelDownloadStatus} + + )} +
+ ) : ( + <> +
+ + {t('adjustments.effects.relightLights')} + +
+ {relightLights.map((light) => ( + + )} + {relightLights.length > 1 && ( + + )} +
+
+ + updateActiveRelightLight({ intensity: Number(e.target.value) })} + step={1} + value={activeRelightLight?.intensity ?? 50} + onDragStateChange={onDragStateChange} + fillOrigin="min" + /> + + updateActiveRelightLight({ depth: Number(e.target.value) })} + step={1} + value={activeRelightLight?.depth ?? 30} + onDragStateChange={onDragStateChange} + /> + + updateActiveRelightLight({ range: Number(e.target.value) })} + step={1} + value={activeRelightLight?.range ?? 60} + onDragStateChange={onDragStateChange} + fillOrigin="min" + /> + +
+ + {t('adjustments.effects.relightColor')} + +
+ {RELIGHT_TEMPERATURE_PRESETS.map((kelvin) => { + const presetColor = kelvinToHex(kelvin); + return ( + + ); + })} +
+ +
+ + handleSliderChange(Effect.RelightAmbient, e.target.value)} + step={1} + value={adjustments.relightAmbient ?? 80} + onDragStateChange={onDragStateChange} + fillOrigin="min" + /> + + handleSliderChange(Effect.RelightSoftness, e.target.value)} + step={1} + value={adjustments.relightSoftness ?? 30} + onDragStateChange={onDragStateChange} + fillOrigin="min" + /> + + handleSliderChange(Effect.RelightSpecular, e.target.value)} + step={1} + value={adjustments.relightSpecular ?? 0} + onDragStateChange={onDragStateChange} + fillOrigin="min" + /> + + handleSliderChange(Effect.RelightDetail, e.target.value)} + step={1} + value={adjustments.relightDetail ?? 30} + onDragStateChange={onDragStateChange} + fillOrigin="min" + /> + + {adjustments.relightNormalMap && ( + setEditor({ showRelightNormalMap: checked })} + /> + )} + + {adjustments.relightNormalMap && ( + {t('adjustments.effects.relightNormalMap')} + )} + +
+ + {t('adjustments.effects.relightQuality')} + +
+ {(['standard', 'high'] as const).map((quality) => ( + + ))} +
+
+ + + + )} +
+
+
+ + ) : ( +
+ + {t('editor.ai.noImageSelected')} + +
+ )} +
+
+ ); +} diff --git a/src/components/ui/AppProperties.tsx b/src/components/ui/AppProperties.tsx index 2a3005c8b0..1b4e7d77b7 100644 --- a/src/components/ui/AppProperties.tsx +++ b/src/components/ui/AppProperties.tsx @@ -136,6 +136,7 @@ export enum Panel { Masks = 'masks', Metadata = 'metadata', Presets = 'presets', + Relight = 'relight', FolderTree = 'folderTree', Tethering = 'tethering', } diff --git a/src/i18n/locales/ca.json b/src/i18n/locales/ca.json index 1f30f31be5..02a768e3bc 100644 --- a/src/i18n/locales/ca.json +++ b/src/i18n/locales/ca.json @@ -128,6 +128,28 @@ "lightFlares": "Esclats de llum", "lut": "LUT", "midpoint": "Punt mitjà", + "relight": "Reil·luminar", + "relightAddLight": "Afegeix una llum", + "relightAmbient": "Ambient", + "relightColor": "Color de la llum", + "relightCustomColor": "Color personalitzat", + "relightDepth": "Profunditat", + "relightDetail": "Detall", + "relightDragLight": "Arrossega per moure la llum, fes servir la roda per acostar-la o allunyar-la", + "relightGenerateNormalMap": "Genera el mapa de normals", + "relightIntensity": "Intensitat", + "relightLights": "Llums", + "relightNormalMap": "Mapa de normals", + "relightQuality": "Qualitat del mapa", + "relightQualityHigh": "Alta", + "relightQualityHighTooltip": "Mapes més nítids i detallats. Baixa un model d'1,3 GB la primera vegada i triga unes tres vegades més.", + "relightQualityStandard": "Estàndard", + "relightRange": "Abast", + "relightRegenerateNormalMap": "Regenera el mapa de normals", + "relightRemoveLight": "Elimina la llum seleccionada", + "relightShowNormalMap": "Mostra el mapa de normals a la imatge", + "relightSoftness": "Suavitat", + "relightSpecular": "Especular", "roughness": "Rugositat", "roundness": "Rodonesa", "size": "Mida", @@ -377,6 +399,7 @@ "editSettingsTitle": "Configuració d'edició", "editsTitle": "Edicions", "generatingDepthMap": "S'està calculant el mapa de profunditat d'IA...", + "generatingNormalMap": "S'està calculant el mapa de normals d'IA...", "generativeEditTitle": "Edició generativa", "inpaintingTitle": "Inpainting", "liquify": { @@ -770,6 +793,7 @@ "inpaint": "Inpainting", "masks": "Màscares", "presets": "Ajustos preestablerts", + "relight": "Reil·luminar", "tethering": "Captura connectada" } }, diff --git a/src/i18n/locales/cs.json b/src/i18n/locales/cs.json index 2ac986f0e9..e4fccdfb7e 100644 --- a/src/i18n/locales/cs.json +++ b/src/i18n/locales/cs.json @@ -127,6 +127,28 @@ "lightFlares": "Světelné záblesky", "lut": "LUT", "midpoint": "Střed", + "relight": "Přesvětlení", + "relightAddLight": "Přidat světlo", + "relightAmbient": "Okolní světlo", + "relightColor": "Barva světla", + "relightCustomColor": "Vlastní barva", + "relightDepth": "Hloubka", + "relightDetail": "Detail", + "relightDragLight": "Tažením světlo přesunete, kolečkem jej přiblížíte nebo oddálíte", + "relightGenerateNormalMap": "Vytvořit normálovou mapu", + "relightIntensity": "Intenzita", + "relightLights": "Světla", + "relightNormalMap": "Normálová mapa", + "relightQuality": "Kvalita mapy", + "relightQualityHigh": "Vysoká", + "relightQualityHighTooltip": "Ostřejší a podrobnější mapy. Při prvním použití se stáhne model o velikosti 1,3 GB a výpočet trvá zhruba třikrát déle.", + "relightQualityStandard": "Standardní", + "relightRange": "Dosah", + "relightRegenerateNormalMap": "Znovu vytvořit normálovou mapu", + "relightRemoveLight": "Odebrat vybrané světlo", + "relightShowNormalMap": "Zobrazit normálovou mapu na snímku", + "relightSoftness": "Měkkost", + "relightSpecular": "Odlesky", "roughness": "Drsnost", "roundness": "Kulatost", "size": "Velikost", @@ -376,6 +398,7 @@ "editSettingsTitle": "Upravit nastavení", "editsTitle": "Úpravy", "generatingDepthMap": "Výpočet hloubkové mapy AI…", + "generatingNormalMap": "Výpočet AI normálové mapy...", "generativeEditTitle": "Generativní úprava", "inpaintingTitle": "AI retuš", "liquify": { @@ -769,6 +792,7 @@ "inpaint": "Retuš", "masks": "Masky", "presets": "Presety", + "relight": "Přesvětlení", "tethering": "Tetherované snímání" } }, diff --git a/src/i18n/locales/de.json b/src/i18n/locales/de.json index cfd6266e39..62f7a8bc90 100644 --- a/src/i18n/locales/de.json +++ b/src/i18n/locales/de.json @@ -128,6 +128,28 @@ "lightFlares": "Blendenflecke", "lut": "LUT", "midpoint": "Mittelpunkt", + "relight": "Neu beleuchten", + "relightAddLight": "Licht hinzufügen", + "relightAmbient": "Umgebungslicht", + "relightColor": "Lichtfarbe", + "relightCustomColor": "Eigene Farbe", + "relightDepth": "Tiefe", + "relightDetail": "Details", + "relightDragLight": "Ziehen, um das Licht zu bewegen, scrollen, um es näher oder weiter weg zu setzen", + "relightGenerateNormalMap": "Normal Map erzeugen", + "relightIntensity": "Intensität", + "relightLights": "Lichter", + "relightNormalMap": "Normal Map", + "relightQuality": "Kartenqualität", + "relightQualityHigh": "Hoch", + "relightQualityHighTooltip": "Schärfere, detailliertere Karten. Lädt beim ersten Mal ein 1,3 GB großes Modell und dauert etwa dreimal so lange.", + "relightQualityStandard": "Standard", + "relightRange": "Reichweite", + "relightRegenerateNormalMap": "Normal Map neu erzeugen", + "relightRemoveLight": "Ausgewähltes Licht entfernen", + "relightShowNormalMap": "Normal Map im Bild anzeigen", + "relightSoftness": "Weichheit", + "relightSpecular": "Glanzlicht", "roughness": "Rauheit", "roundness": "Rundheit", "size": "Größe", @@ -377,6 +399,7 @@ "editSettingsTitle": "Bearbeitungseinstellungen", "editsTitle": "Bearbeitungen", "generatingDepthMap": "KI-Tiefenkarte wird berechnet...", + "generatingNormalMap": "KI-Normal-Map wird berechnet...", "generativeEditTitle": "Generative Bearbeitung", "inpaintingTitle": "Reparieren", "liquify": { @@ -770,6 +793,7 @@ "inpaint": "Reparieren", "masks": "Masken", "presets": "Vorgaben", + "relight": "Neu beleuchten", "tethering": "Tethering" } }, diff --git a/src/i18n/locales/en.json b/src/i18n/locales/en.json index aad1e94e95..7e20c9d592 100644 --- a/src/i18n/locales/en.json +++ b/src/i18n/locales/en.json @@ -128,6 +128,28 @@ "lightFlares": "Light Flares", "lut": "LUT", "midpoint": "Midpoint", + "relight": "Relight", + "relightAddLight": "Add light", + "relightAmbient": "Ambient", + "relightColor": "Light Color", + "relightCustomColor": "Custom Color", + "relightDepth": "Depth", + "relightDetail": "Detail", + "relightDragLight": "Drag to move the light, scroll to move it closer or farther", + "relightGenerateNormalMap": "Generate Normal Map", + "relightIntensity": "Intensity", + "relightLights": "Lights", + "relightNormalMap": "Normal Map", + "relightQuality": "Map Quality", + "relightQualityHigh": "High", + "relightQualityHighTooltip": "Sharper, more detailed maps. Downloads a 1.3 GB model the first time and takes about three times longer.", + "relightQualityStandard": "Standard", + "relightRange": "Range", + "relightRegenerateNormalMap": "Regenerate Normal Map", + "relightRemoveLight": "Remove selected light", + "relightShowNormalMap": "Show normal map on image", + "relightSoftness": "Softness", + "relightSpecular": "Specular", "roughness": "Roughness", "roundness": "Roundness", "size": "Size", @@ -377,6 +399,7 @@ "editSettingsTitle": "Edit Settings", "editsTitle": "Edits", "generatingDepthMap": "Calculating AI Depth Map...", + "generatingNormalMap": "Calculating AI Normal Map...", "generativeEditTitle": "Generative Edit", "inpaintingTitle": "Inpainting", "liquify": { @@ -770,6 +793,7 @@ "inpaint": "Inpaint", "masks": "Masks", "presets": "Presets", + "relight": "Relight", "tethering": "Tethering" } }, diff --git a/src/i18n/locales/es.json b/src/i18n/locales/es.json index baa9232131..f1ca492b12 100644 --- a/src/i18n/locales/es.json +++ b/src/i18n/locales/es.json @@ -128,6 +128,28 @@ "lightFlares": "Destellos de luz", "lut": "LUT", "midpoint": "Punto medio", + "relight": "Reiluminar", + "relightAddLight": "Añadir luz", + "relightAmbient": "Ambiente", + "relightColor": "Color de la luz", + "relightCustomColor": "Color personalizado", + "relightDepth": "Profundidad", + "relightDetail": "Detalle", + "relightDragLight": "Arrastra para mover la luz, usa la rueda para acercarla o alejarla", + "relightGenerateNormalMap": "Generar mapa de normales", + "relightIntensity": "Intensidad", + "relightLights": "Luces", + "relightNormalMap": "Mapa de normales", + "relightQuality": "Calidad del mapa", + "relightQualityHigh": "Alta", + "relightQualityHighTooltip": "Mapas más nítidos y detallados. Descarga un modelo de 1,3 GB la primera vez y tarda unas tres veces más.", + "relightQualityStandard": "Estándar", + "relightRange": "Alcance", + "relightRegenerateNormalMap": "Regenerar mapa de normales", + "relightRemoveLight": "Quitar la luz seleccionada", + "relightShowNormalMap": "Ver mapa de normales en la imagen", + "relightSoftness": "Suavidad", + "relightSpecular": "Especular", "roughness": "Rugosidad", "roundness": "Redondez", "size": "Tamaño", @@ -393,6 +415,7 @@ "editSettingsTitle": "Ajustes de edición", "editsTitle": "Ediciones", "generatingDepthMap": "Calculando mapa de profundidad IA...", + "generatingNormalMap": "Calculando mapa de normales IA...", "generativeEditTitle": "Edición generativa", "inpaintingTitle": "Inpainting", "liquify": { @@ -791,6 +814,7 @@ "inpaint": "Inpainting", "masks": "Máscaras", "presets": "Ajustes preestablecidos", + "relight": "Reiluminar", "tethering": "Captura conectada" } }, diff --git a/src/i18n/locales/fr.json b/src/i18n/locales/fr.json index 24c5a0cf53..b019c51a1f 100644 --- a/src/i18n/locales/fr.json +++ b/src/i18n/locales/fr.json @@ -128,6 +128,28 @@ "lightFlares": "Reflets", "lut": "LUT", "midpoint": "Point médian", + "relight": "Rééclairage", + "relightAddLight": "Ajouter une lumière", + "relightAmbient": "Ambiance", + "relightColor": "Couleur de la lumière", + "relightCustomColor": "Couleur personnalisée", + "relightDepth": "Profondeur", + "relightDetail": "Détail", + "relightDragLight": "Faites glisser pour déplacer la lumière, utilisez la molette pour la rapprocher ou l'éloigner", + "relightGenerateNormalMap": "Générer la carte de normales", + "relightIntensity": "Intensité", + "relightLights": "Lumières", + "relightNormalMap": "Carte de normales", + "relightQuality": "Qualité de la carte", + "relightQualityHigh": "Élevée", + "relightQualityHighTooltip": "Cartes plus nettes et détaillées. Télécharge un modèle de 1,3 Go la première fois et prend environ trois fois plus de temps.", + "relightQualityStandard": "Standard", + "relightRange": "Portée", + "relightRegenerateNormalMap": "Régénérer la carte de normales", + "relightRemoveLight": "Supprimer la lumière sélectionnée", + "relightShowNormalMap": "Afficher la carte de normales sur l'image", + "relightSoftness": "Douceur", + "relightSpecular": "Spéculaire", "roughness": "Rugosité", "roundness": "Arrondi", "size": "Taille", @@ -393,6 +415,7 @@ "editSettingsTitle": "Paramètres de retouche", "editsTitle": "Retouches", "generatingDepthMap": "Calcul de la carte de profondeur IA...", + "generatingNormalMap": "Calcul de la carte de normales IA...", "generativeEditTitle": "Édition générative", "inpaintingTitle": "Inpainting", "liquify": { @@ -791,6 +814,7 @@ "inpaint": "Inpainting", "masks": "Masques", "presets": "Paramètres prédéfinis", + "relight": "Rééclairage", "tethering": "Prise de vue connectée" } }, diff --git a/src/i18n/locales/it.json b/src/i18n/locales/it.json index 3a171e9574..c4eb6e6e2a 100644 --- a/src/i18n/locales/it.json +++ b/src/i18n/locales/it.json @@ -128,6 +128,28 @@ "lightFlares": "Riflessi di Luce", "lut": "LUT", "midpoint": "Punto Medio", + "relight": "Reilluminazione", + "relightAddLight": "Aggiungi luce", + "relightAmbient": "Ambiente", + "relightColor": "Colore della luce", + "relightCustomColor": "Colore personalizzato", + "relightDepth": "Profondità", + "relightDetail": "Dettaglio", + "relightDragLight": "Trascina per spostare la luce, usa la rotellina per avvicinarla o allontanarla", + "relightGenerateNormalMap": "Genera mappa delle normali", + "relightIntensity": "Intensità", + "relightLights": "Luci", + "relightNormalMap": "Mappa delle normali", + "relightQuality": "Qualità della mappa", + "relightQualityHigh": "Alta", + "relightQualityHighTooltip": "Mappe più nitide e dettagliate. Scarica un modello da 1,3 GB la prima volta e richiede circa il triplo del tempo.", + "relightQualityStandard": "Standard", + "relightRange": "Portata", + "relightRegenerateNormalMap": "Rigenera mappa delle normali", + "relightRemoveLight": "Rimuovi la luce selezionata", + "relightShowNormalMap": "Mostra la mappa delle normali sull'immagine", + "relightSoftness": "Morbidezza", + "relightSpecular": "Speculare", "roughness": "Ruvidezza", "roundness": "Rotondità", "size": "Dimensione", @@ -393,6 +415,7 @@ "editSettingsTitle": "Impostazioni Modifica", "editsTitle": "Modifiche", "generatingDepthMap": "Calcolo mappa di profondità IA...", + "generatingNormalMap": "Calcolo mappa delle normali IA...", "generativeEditTitle": "Modifica generativa", "inpaintingTitle": "Inpainting", "liquify": { @@ -791,6 +814,7 @@ "inpaint": "Inpainting", "masks": "Maschere", "presets": "Predefiniti", + "relight": "Reilluminazione", "tethering": "Acquisizione diretta" } }, diff --git a/src/i18n/locales/ja.json b/src/i18n/locales/ja.json index 41f77b1cc6..d35a9191df 100644 --- a/src/i18n/locales/ja.json +++ b/src/i18n/locales/ja.json @@ -128,6 +128,28 @@ "lightFlares": "ライトフレア", "lut": "LUT", "midpoint": "中間点", + "relight": "リライト", + "relightAddLight": "ライトを追加", + "relightAmbient": "環境光", + "relightColor": "ライトの色", + "relightCustomColor": "カスタムカラー", + "relightDepth": "奥行き", + "relightDetail": "ディテール", + "relightDragLight": "ドラッグでライトを移動、スクロールで前後に移動", + "relightGenerateNormalMap": "法線マップを生成", + "relightIntensity": "強度", + "relightLights": "ライト", + "relightNormalMap": "法線マップ", + "relightQuality": "マップ品質", + "relightQualityHigh": "高", + "relightQualityHighTooltip": "よりシャープで詳細なマップ。初回は1.3GBのモデルをダウンロードし、約3倍の時間がかかります。", + "relightQualityStandard": "標準", + "relightRange": "範囲", + "relightRegenerateNormalMap": "法線マップを再生成", + "relightRemoveLight": "選択したライトを削除", + "relightShowNormalMap": "画像上に法線マップを表示", + "relightSoftness": "柔らかさ", + "relightSpecular": "スペキュラ", "roughness": "粗さ", "roundness": "丸み", "size": "サイズ", @@ -377,6 +399,7 @@ "editSettingsTitle": "編集設定", "editsTitle": "編集一覧", "generatingDepthMap": "AI深度マップを計算中...", + "generatingNormalMap": "AI法線マップを計算中...", "generativeEditTitle": "生成編集", "inpaintingTitle": "インペインティング", "liquify": { @@ -768,6 +791,7 @@ "inpaint": "インペイント", "masks": "マスク", "presets": "プリセット", + "relight": "リライト", "tethering": "テザー撮影" } }, diff --git a/src/i18n/locales/ko.json b/src/i18n/locales/ko.json index 6d16f6e46c..2471f7ae7b 100644 --- a/src/i18n/locales/ko.json +++ b/src/i18n/locales/ko.json @@ -128,6 +128,28 @@ "lightFlares": "빛 번짐", "lut": "LUT", "midpoint": "중간점", + "relight": "재조명", + "relightAddLight": "조명 추가", + "relightAmbient": "주변광", + "relightColor": "조명 색상", + "relightCustomColor": "사용자 지정 색상", + "relightDepth": "깊이", + "relightDetail": "디테일", + "relightDragLight": "드래그하여 조명 이동, 스크롤하여 앞뒤로 이동", + "relightGenerateNormalMap": "노멀 맵 생성", + "relightIntensity": "강도", + "relightLights": "조명", + "relightNormalMap": "노멀 맵", + "relightQuality": "맵 품질", + "relightQualityHigh": "높음", + "relightQualityHighTooltip": "더 선명하고 세밀한 맵. 처음에는 1.3GB 모델을 다운로드하며 약 3배 더 오래 걸립니다.", + "relightQualityStandard": "표준", + "relightRange": "범위", + "relightRegenerateNormalMap": "노멀 맵 다시 생성", + "relightRemoveLight": "선택한 조명 제거", + "relightShowNormalMap": "이미지에 노멀 맵 표시", + "relightSoftness": "부드러움", + "relightSpecular": "스페큘러", "roughness": "거칠기", "roundness": "둥글기", "size": "크기", @@ -377,6 +399,7 @@ "editSettingsTitle": "편집 설정", "editsTitle": "편집", "generatingDepthMap": "AI 심도 맵 계산 중...", + "generatingNormalMap": "AI 노멀 맵 계산 중...", "generativeEditTitle": "생성형 편집", "inpaintingTitle": "인페인팅", "liquify": { @@ -768,6 +791,7 @@ "inpaint": "인페인팅", "masks": "마스크", "presets": "프리셋", + "relight": "재조명", "tethering": "테더링" } }, diff --git a/src/i18n/locales/nl.json b/src/i18n/locales/nl.json index 41917e7284..ecb45a21a3 100644 --- a/src/i18n/locales/nl.json +++ b/src/i18n/locales/nl.json @@ -127,6 +127,28 @@ "lightFlares": "Lensflare", "lut": "LUT", "midpoint": "Middelpunt", + "relight": "Herbelichten", + "relightAddLight": "Licht toevoegen", + "relightAmbient": "Omgevingslicht", + "relightColor": "Lichtkleur", + "relightCustomColor": "Aangepaste kleur", + "relightDepth": "Diepte", + "relightDetail": "Detail", + "relightDragLight": "Sleep om het licht te verplaatsen, scrol om het dichterbij of verder weg te zetten", + "relightGenerateNormalMap": "Normal map genereren", + "relightIntensity": "Intensiteit", + "relightLights": "Lichten", + "relightNormalMap": "Normal map", + "relightQuality": "Kwaliteit van de map", + "relightQualityHigh": "Hoog", + "relightQualityHighTooltip": "Scherpere, gedetailleerdere maps. Downloadt de eerste keer een model van 1,3 GB en duurt ongeveer drie keer zo lang.", + "relightQualityStandard": "Standaard", + "relightRange": "Bereik", + "relightRegenerateNormalMap": "Normal map opnieuw genereren", + "relightRemoveLight": "Geselecteerd licht verwijderen", + "relightShowNormalMap": "Normal map op de afbeelding tonen", + "relightSoftness": "Zachtheid", + "relightSpecular": "Spiegeling", "roughness": "Ruwheid", "roundness": "Rondheid", "size": "Grootte", @@ -371,6 +393,7 @@ "editSettingsTitle": "Bewerkingsinstellingen", "editsTitle": "Bewerkingen", "generatingDepthMap": "AI-dieptekaart berekenen...", + "generatingNormalMap": "AI-normal map berekenen...", "generativeEditTitle": "Generatieve bewerking", "inpaintingTitle": "Inpainting", "liquify": { @@ -764,6 +787,7 @@ "inpaint": "Inpainting", "masks": "Maskers", "presets": "Voorinstellingen", + "relight": "Herbelichten", "tethering": "Kabelopname" } }, diff --git a/src/i18n/locales/pl.json b/src/i18n/locales/pl.json index b0d5b0508b..9eb06b5ebd 100644 --- a/src/i18n/locales/pl.json +++ b/src/i18n/locales/pl.json @@ -128,6 +128,28 @@ "lightFlares": "Flary świetlne", "lut": "LUT", "midpoint": "Punkt środkowy", + "relight": "Ponowne oświetlenie", + "relightAddLight": "Dodaj światło", + "relightAmbient": "Światło otoczenia", + "relightColor": "Kolor światła", + "relightCustomColor": "Własny kolor", + "relightDepth": "Głębia", + "relightDetail": "Szczegóły", + "relightDragLight": "Przeciągnij, aby przesunąć światło, przewiń, aby je przybliżyć lub oddalić", + "relightGenerateNormalMap": "Generuj mapę normalnych", + "relightIntensity": "Intensywność", + "relightLights": "Światła", + "relightNormalMap": "Mapa normalnych", + "relightQuality": "Jakość mapy", + "relightQualityHigh": "Wysoka", + "relightQualityHighTooltip": "Ostrzejsze, bardziej szczegółowe mapy. Za pierwszym razem pobiera model 1,3 GB i trwa około trzy razy dłużej.", + "relightQualityStandard": "Standardowa", + "relightRange": "Zasięg", + "relightRegenerateNormalMap": "Generuj ponownie mapę normalnych", + "relightRemoveLight": "Usuń wybrane światło", + "relightShowNormalMap": "Pokaż mapę normalnych na obrazie", + "relightSoftness": "Miękkość", + "relightSpecular": "Odbicia", "roughness": "Szorstkość", "roundness": "Zaokrąglenie", "size": "Rozmiar", @@ -409,6 +431,7 @@ "editSettingsTitle": "Ustawienia edycji", "editsTitle": "Edycje", "generatingDepthMap": "Obliczanie mapy głębi AI...", + "generatingNormalMap": "Obliczanie mapy normalnych AI...", "generativeEditTitle": "Edycja generatywna", "inpaintingTitle": "Inpainting", "liquify": { @@ -812,6 +835,7 @@ "inpaint": "Inpaint", "masks": "Maski", "presets": "Presety", + "relight": "Ponowne oświetlenie", "tethering": "Tethering (Połączenie przewodowe)" } }, diff --git a/src/i18n/locales/pt.json b/src/i18n/locales/pt.json index cec79f397e..8357ab0506 100644 --- a/src/i18n/locales/pt.json +++ b/src/i18n/locales/pt.json @@ -128,6 +128,28 @@ "lightFlares": "Efeitos de Luz", "lut": "LUT", "midpoint": "Ponto Médio", + "relight": "Reiluminar", + "relightAddLight": "Adicionar luz", + "relightAmbient": "Ambiente", + "relightColor": "Cor da luz", + "relightCustomColor": "Cor personalizada", + "relightDepth": "Profundidade", + "relightDetail": "Detalhe", + "relightDragLight": "Arraste para mover a luz, use a roda para a aproximar ou afastar", + "relightGenerateNormalMap": "Gerar mapa de normais", + "relightIntensity": "Intensidade", + "relightLights": "Luzes", + "relightNormalMap": "Mapa de normais", + "relightQuality": "Qualidade do mapa", + "relightQualityHigh": "Alta", + "relightQualityHighTooltip": "Mapas mais nítidos e detalhados. Transfere um modelo de 1,3 GB na primeira vez e demora cerca de três vezes mais.", + "relightQualityStandard": "Padrão", + "relightRange": "Alcance", + "relightRegenerateNormalMap": "Regenerar mapa de normais", + "relightRemoveLight": "Remover a luz selecionada", + "relightShowNormalMap": "Mostrar mapa de normais na imagem", + "relightSoftness": "Suavidade", + "relightSpecular": "Especular", "roughness": "Rugosidade", "roundness": "Arredondamento", "size": "Tamanho", @@ -393,6 +415,7 @@ "editSettingsTitle": "Configurações de Edição", "editsTitle": "Edições", "generatingDepthMap": "A calcular mapa de profundidade IA...", + "generatingNormalMap": "A calcular mapa de normais IA...", "generativeEditTitle": "Edição Generativa", "inpaintingTitle": "Preenchimento", "liquify": { @@ -791,6 +814,7 @@ "inpaint": "Preencher", "masks": "Máscaras", "presets": "Predefinições", + "relight": "Reiluminar", "tethering": "Captura Conectada (Tethering)" } }, diff --git a/src/i18n/locales/ru.json b/src/i18n/locales/ru.json index f13599a82c..56b0222297 100644 --- a/src/i18n/locales/ru.json +++ b/src/i18n/locales/ru.json @@ -128,6 +128,28 @@ "lightFlares": "Блики", "lut": "LUT", "midpoint": "Средняя точка", + "relight": "Переосвещение", + "relightAddLight": "Добавить свет", + "relightAmbient": "Окружающий свет", + "relightColor": "Цвет света", + "relightCustomColor": "Свой цвет", + "relightDepth": "Глубина", + "relightDetail": "Детали", + "relightDragLight": "Перетащите, чтобы переместить свет, прокрутите, чтобы приблизить или отдалить его", + "relightGenerateNormalMap": "Создать карту нормалей", + "relightIntensity": "Интенсивность", + "relightLights": "Источники света", + "relightNormalMap": "Карта нормалей", + "relightQuality": "Качество карты", + "relightQualityHigh": "Высокое", + "relightQualityHighTooltip": "Более чёткие и детальные карты. В первый раз загружает модель 1,3 ГБ и работает примерно в три раза дольше.", + "relightQualityStandard": "Стандартное", + "relightRange": "Дальность", + "relightRegenerateNormalMap": "Пересоздать карту нормалей", + "relightRemoveLight": "Удалить выбранный свет", + "relightShowNormalMap": "Показать карту нормалей на изображении", + "relightSoftness": "Мягкость", + "relightSpecular": "Блики", "roughness": "Шероховатость", "roundness": "Округлость", "size": "Размер", @@ -409,6 +431,7 @@ "editSettingsTitle": "Настройки редактирования", "editsTitle": "Правки", "generatingDepthMap": "Расчет карты глубины ИИ...", + "generatingNormalMap": "Расчет карты нормалей ИИ...", "generativeEditTitle": "Генеративное редактирование", "inpaintingTitle": "Восстановление", "liquify": { @@ -812,6 +835,7 @@ "inpaint": "Восстановление", "masks": "Маски", "presets": "Пресеты", + "relight": "Переосвещение", "tethering": "Съемка на ПК (Тетеринг)" } }, diff --git a/src/i18n/locales/zh-CN.json b/src/i18n/locales/zh-CN.json index e1342cd550..aaa761a368 100644 --- a/src/i18n/locales/zh-CN.json +++ b/src/i18n/locales/zh-CN.json @@ -128,6 +128,28 @@ "lightFlares": "光线耀斑", "lut": "LUT", "midpoint": "中点", + "relight": "重新打光", + "relightAddLight": "添加灯光", + "relightAmbient": "环境光", + "relightColor": "灯光颜色", + "relightCustomColor": "自定义颜色", + "relightDepth": "深度", + "relightDetail": "细节", + "relightDragLight": "拖动以移动灯光，滚动以前后移动", + "relightGenerateNormalMap": "生成法线贴图", + "relightIntensity": "强度", + "relightLights": "灯光", + "relightNormalMap": "法线贴图", + "relightQuality": "贴图质量", + "relightQualityHigh": "高", + "relightQualityHighTooltip": "更清晰、更细致的贴图。首次使用会下载 1.3 GB 的模型，耗时约为三倍。", + "relightQualityStandard": "标准", + "relightRange": "范围", + "relightRegenerateNormalMap": "重新生成法线贴图", + "relightRemoveLight": "移除所选灯光", + "relightShowNormalMap": "在图像上显示法线贴图", + "relightSoftness": "柔和度", + "relightSpecular": "高光", "roughness": "粗糙度", "roundness": "圆度", "size": "大小", @@ -377,6 +399,7 @@ "editSettingsTitle": "编辑设置", "editsTitle": "编辑", "generatingDepthMap": "正在计算 AI 深度图...", + "generatingNormalMap": "正在计算 AI 法线贴图...", "generativeEditTitle": "生成式编辑", "inpaintingTitle": "图像修复", "liquify": { @@ -768,6 +791,7 @@ "inpaint": "修复", "masks": "蒙版", "presets": "预设", + "relight": "重新打光", "tethering": "联机拍摄" } }, diff --git a/src/i18n/locales/zh-TW.json b/src/i18n/locales/zh-TW.json index 82df863a32..482d046c39 100644 --- a/src/i18n/locales/zh-TW.json +++ b/src/i18n/locales/zh-TW.json @@ -128,6 +128,28 @@ "lightFlares": "光線耀斑", "lut": "LUT", "midpoint": "中點", + "relight": "重新打光", + "relightAddLight": "新增燈光", + "relightAmbient": "環境光", + "relightColor": "燈光顏色", + "relightCustomColor": "自訂顏色", + "relightDepth": "深度", + "relightDetail": "細節", + "relightDragLight": "拖曳以移動燈光，捲動以前後移動", + "relightGenerateNormalMap": "產生法線貼圖", + "relightIntensity": "強度", + "relightLights": "燈光", + "relightNormalMap": "法線貼圖", + "relightQuality": "貼圖品質", + "relightQualityHigh": "高", + "relightQualityHighTooltip": "更清晰、更細緻的貼圖。首次使用會下載 1.3 GB 的模型，耗時約為三倍。", + "relightQualityStandard": "標準", + "relightRange": "範圍", + "relightRegenerateNormalMap": "重新產生法線貼圖", + "relightRemoveLight": "移除所選燈光", + "relightShowNormalMap": "在影像上顯示法線貼圖", + "relightSoftness": "柔和度", + "relightSpecular": "高光", "roughness": "粗糙度", "roundness": "圓度", "size": "大小", @@ -377,6 +399,7 @@ "editSettingsTitle": "編輯設定", "editsTitle": "編輯", "generatingDepthMap": "正在計算 AI 深度圖...", + "generatingNormalMap": "正在計算 AI 法線貼圖...", "generativeEditTitle": "生成式編輯", "inpaintingTitle": "影像修復", "liquify": { @@ -768,6 +791,7 @@ "inpaint": "修復", "masks": "遮色片", "presets": "預設", + "relight": "重新打光", "tethering": "連線拍攝" } }, diff --git a/src/store/useEditorStore.ts b/src/store/useEditorStore.ts index f05b5238c3..40fb2beb02 100644 --- a/src/store/useEditorStore.ts +++ b/src/store/useEditorStore.ts @@ -46,6 +46,8 @@ interface EditorState { // Interaction State isSliderDragging: boolean; + activeRelightLightId: string | null; + showRelightNormalMap: boolean; zoom: number; displaySize: ImageDimensions; previewSize: ImageDimensions; @@ -105,6 +107,8 @@ export const useEditorStore = create((set) => ({ waveformHeight: 220, isSliderDragging: false, + activeRelightLightId: null, + showRelightNormalMap: false, interactivePatch: null, activeMaskContainerId: null, activeMaskId: null, diff --git a/src/store/useUIStore.ts b/src/store/useUIStore.ts index b4a097af8a..4879271edd 100644 --- a/src/store/useUIStore.ts +++ b/src/store/useUIStore.ts @@ -93,6 +93,7 @@ const ALL_PANELS: Panel[] = [ Panel.Masks, Panel.Ai, Panel.Presets, + Panel.Relight, ]; const DEFAULT_PANEL_DEFAULT_REGIONS: Record = { @@ -105,6 +106,7 @@ const DEFAULT_PANEL_DEFAULT_REGIONS: Record = { [Panel.Masks]: 'rightTop', [Panel.Ai]: 'rightTop', [Panel.Presets]: 'rightTop', + [Panel.Relight]: 'rightTop', }; export const DEFAULT_PANEL_WIDTH = 350; @@ -125,7 +127,7 @@ export function reconcileWorkspace( panelLayout: { leftTop: [Panel.Metadata, Panel.FolderTree, Panel.Export, ...(isTetheringSupported ? [Panel.Tethering] : [])], leftBottom: [], - rightTop: [Panel.Adjustments, Panel.Crop, Panel.Masks, Panel.Ai, Panel.Presets], + rightTop: [Panel.Adjustments, Panel.Crop, Panel.Masks, Panel.Ai, Panel.Presets, Panel.Relight], rightBottom: [], }, activePanels: { @@ -292,7 +294,7 @@ export const useUIStore = create((set, get) => ({ panelLayout: { leftTop: [Panel.Metadata, Panel.FolderTree, Panel.Export], leftBottom: [], - rightTop: [Panel.Adjustments, Panel.Crop, Panel.Masks, Panel.Ai, Panel.Presets], + rightTop: [Panel.Adjustments, Panel.Crop, Panel.Masks, Panel.Ai, Panel.Presets, Panel.Relight], rightBottom: [], }, activePanels: { diff --git a/src/utils/adjustments.ts b/src/utils/adjustments.ts index 3f936e2ac4..ec6ee6d8a7 100644 --- a/src/utils/adjustments.ts +++ b/src/utils/adjustments.ts @@ -98,6 +98,16 @@ export enum Effect { LensBlurMaxDepth = 'lensBlurMaxDepth', LensBlurMinFade = 'lensBlurMinFade', LensBlurMaxFade = 'lensBlurMaxFade', + RelightEnabled = 'relightEnabled', + RelightLights = 'relightLights', + RelightAmbient = 'relightAmbient', + RelightSoftness = 'relightSoftness', + RelightSpecular = 'relightSpecular', + RelightNormalMap = 'relightNormalMap', + RelightQuality = 'relightQuality', + RelightDepthMap = 'relightDepthMap', + RelightDepthScale = 'relightDepthScale', + RelightDetail = 'relightDetail', } export enum CreativeAdjustment { @@ -245,6 +255,16 @@ export interface Adjustments { lutIsSceneReferred?: boolean; masks: Array; orientationSteps: number; + relightAmbient: number; + relightDepthMap: string | null; + relightDepthScale: number; + relightDetail: number; + relightEnabled: boolean; + relightLights: Array; + relightNormalMap: string | null; + relightQuality: RelightQuality; + relightSoftness: number; + relightSpecular: number; rotation: number; saturation: number; sectionVisibility: SectionVisibility; @@ -515,6 +535,55 @@ export const INITIAL_MASK_CONTAINER: MaskContainer = { visible: true, }; +export interface RelightLight { + id: string; + // Position as a fraction of the image before orientation, flip, rotation and crop. + x: number; + y: number; + // -100 (far behind the subject) to 100 (close to the camera). + depth: number; + intensity: number; + range: number; + color: string; +} + +export const MAX_RELIGHT_LIGHTS = 6; + +// Which AI model generated the relight maps: the high quality one is a 1.3 GB download. +export type RelightQuality = 'standard' | 'high'; + +export const DEFAULT_RELIGHT_LIGHT: RelightLight = { + id: 'light-1', + x: 0.3, + y: 0.3, + depth: 30, + intensity: 50, + range: 60, + color: '#ffffff', +}; + +// Edits saved before multiple lights existed describe a single light at the top level. +const normalizeRelightLights = (loaded: any): Array => { + if (Array.isArray(loaded.relightLights)) { + return loaded.relightLights.map((light: Partial, index: number) => ({ + ...DEFAULT_RELIGHT_LIGHT, + ...light, + id: light.id || `light-${index + 1}`, + })); + } + return [ + { + ...DEFAULT_RELIGHT_LIGHT, + x: loaded.relightX ?? DEFAULT_RELIGHT_LIGHT.x, + y: loaded.relightY ?? DEFAULT_RELIGHT_LIGHT.y, + depth: loaded.relightDepth ?? DEFAULT_RELIGHT_LIGHT.depth, + intensity: loaded.relightIntensity ?? DEFAULT_RELIGHT_LIGHT.intensity, + range: loaded.relightRange ?? DEFAULT_RELIGHT_LIGHT.range, + color: loaded.relightColor ?? DEFAULT_RELIGHT_LIGHT.color, + }, + ]; +}; + export const INITIAL_ADJUSTMENTS: Adjustments = { aiPatches: [], aspectRatio: null, @@ -584,6 +653,16 @@ export const INITIAL_ADJUSTMENTS: Adjustments = { lutIsSceneReferred: false, masks: [], orientationSteps: 0, + relightAmbient: 80, + relightDepthMap: null, + relightDepthScale: 0, + relightDetail: 30, + relightEnabled: false, + relightLights: [{ ...DEFAULT_RELIGHT_LIGHT }], + relightNormalMap: null, + relightQuality: 'standard', + relightSoftness: 30, + relightSpecular: 0, rotation: 0, saturation: 0, sectionVisibility: { @@ -732,6 +811,16 @@ export const normalizeLoadedAdjustments = (loadedAdjustments: Adjustments): any lensBlurMaxFade: loadedAdjustments.lensBlurMaxFade ?? INITIAL_ADJUSTMENTS.lensBlurMaxFade, lensBlurMinDepth: loadedAdjustments.lensBlurMinDepth ?? INITIAL_ADJUSTMENTS.lensBlurMinDepth, lensBlurMinFade: loadedAdjustments.lensBlurMinFade ?? INITIAL_ADJUSTMENTS.lensBlurMinFade, + relightAmbient: loadedAdjustments.relightAmbient ?? INITIAL_ADJUSTMENTS.relightAmbient, + relightDepthMap: loadedAdjustments.relightDepthMap ?? INITIAL_ADJUSTMENTS.relightDepthMap, + relightDepthScale: loadedAdjustments.relightDepthScale ?? INITIAL_ADJUSTMENTS.relightDepthScale, + relightDetail: loadedAdjustments.relightDetail ?? INITIAL_ADJUSTMENTS.relightDetail, + relightEnabled: loadedAdjustments.relightEnabled ?? INITIAL_ADJUSTMENTS.relightEnabled, + relightLights: normalizeRelightLights(loadedAdjustments), + relightNormalMap: loadedAdjustments.relightNormalMap ?? INITIAL_ADJUSTMENTS.relightNormalMap, + relightQuality: loadedAdjustments.relightQuality ?? INITIAL_ADJUSTMENTS.relightQuality, + relightSoftness: loadedAdjustments.relightSoftness ?? INITIAL_ADJUSTMENTS.relightSoftness, + relightSpecular: loadedAdjustments.relightSpecular ?? INITIAL_ADJUSTMENTS.relightSpecular, lensCorrectionMode: loadedAdjustments.lensCorrectionMode || 'manual', lensMaker: loadedAdjustments.lensMaker ?? INITIAL_ADJUSTMENTS.lensMaker, lensModel: loadedAdjustments.lensModel ?? INITIAL_ADJUSTMENTS.lensModel, diff --git a/src/utils/relightUtils.ts b/src/utils/relightUtils.ts new file mode 100644 index 0000000000..ed3b9300f3 --- /dev/null +++ b/src/utils/relightUtils.ts @@ -0,0 +1,91 @@ +import { Coord } from './adjustments'; + +export const RELIGHT_TEMPERATURE_PRESETS = [1900, 2700, 3200, 4000, 5600, 6500, 8000, 10000]; + +const toHexChannel = (value: number) => + Math.round(Math.max(0, Math.min(255, value))) + .toString(16) + .padStart(2, '0'); + +// Tanner Helland's black body approximation, good enough for picking a light tint. +export const kelvinToHex = (kelvin: number): string => { + const temp = Math.max(1000, Math.min(40000, kelvin)) / 100; + + const red = temp <= 66 ? 255 : 329.698727446 * Math.pow(temp - 60, -0.1332047592); + const green = + temp <= 66 ? 99.4708025861 * Math.log(temp) - 161.1195681661 : 288.1221695283 * Math.pow(temp - 60, -0.0755148492); + const blue = temp >= 66 ? 255 : temp <= 19 ? 0 : 138.5177312231 * Math.log(temp - 10) - 305.0447927307; + + return `#${toHexChannel(red)}${toHexChannel(green)}${toHexChannel(blue)}`; +}; + +export interface RelightGeometry { + imageWidth: number; + imageHeight: number; + orientationSteps: number; + flipHorizontal: boolean; + flipVertical: boolean; + rotation: number; + cropX: number; + cropY: number; +} + +const orientPoint = (x: number, y: number, w: number, h: number, steps: number): Coord => { + const s = ((steps % 4) + 4) % 4; + if (s === 0) return { x, y }; + if (s === 1) return { x: h - y, y: x }; + if (s === 2) return { x: w - x, y: h - y }; + return { x: y, y: w - x }; +}; + +const getOrientedSize = (g: RelightGeometry) => { + const swapped = (((g.orientationSteps % 4) + 4) % 4) % 2 !== 0; + return { + width: swapped ? g.imageHeight : g.imageWidth, + height: swapped ? g.imageWidth : g.imageHeight, + }; +}; + +const rotateAboutCenter = (x: number, y: number, width: number, height: number, degrees: number): Coord => { + if (Math.abs(degrees) < 1e-4) return { x, y }; + const rad = (degrees * Math.PI) / 180; + const cos = Math.cos(rad); + const sin = Math.sin(rad); + const cx = width / 2; + const cy = height / 2; + const dx = x - cx; + const dy = y - cy; + return { x: cx + dx * cos - dy * sin, y: cy + dx * sin + dy * cos }; +}; + +/** + * The light position is stored as a fraction of the image before orientation, flip, + * rotation and crop, because that is where the relight pass runs. These helpers convert + * between that space and pixels of the cropped image shown in the editor. + */ +export const relightUvToDisplay = (uv: Coord, g: RelightGeometry): Coord => { + const oriented = getOrientedSize(g); + let { x, y } = orientPoint( + uv.x * g.imageWidth, + uv.y * g.imageHeight, + g.imageWidth, + g.imageHeight, + g.orientationSteps, + ); + if (g.flipHorizontal) x = oriented.width - x; + if (g.flipVertical) y = oriented.height - y; + const rotated = rotateAboutCenter(x, y, oriented.width, oriented.height, g.rotation); + return { x: rotated.x - g.cropX, y: rotated.y - g.cropY }; +}; + +export const relightDisplayToUv = (point: Coord, g: RelightGeometry): Coord => { + const oriented = getOrientedSize(g); + let { x, y } = rotateAboutCenter(point.x + g.cropX, point.y + g.cropY, oriented.width, oriented.height, -g.rotation); + if (g.flipHorizontal) x = oriented.width - x; + if (g.flipVertical) y = oriented.height - y; + const original = orientPoint(x, y, oriented.width, oriented.height, 4 - (((g.orientationSteps % 4) + 4) % 4)); + return { + x: g.imageWidth > 0 ? original.x / g.imageWidth : 0, + y: g.imageHeight > 0 ? original.y / g.imageHeight : 0, + }; +};