diff --git a/crates/cli/CHANGELOG.md b/crates/cli/CHANGELOG.md index df22eb68..a114fc88 100644 --- a/crates/cli/CHANGELOG.md +++ b/crates/cli/CHANGELOG.md @@ -15,6 +15,15 @@ versioning through the workspace version in the root `Cargo.toml`. - MarmotKit/UniFFI now exposes the cached Nostr Kind 0 `website` field through a read-only profile accessor so app profile surfaces can display it without widening the writable profile record. +### Fixed + +- Encrypted media uploads now fail over, in order, across a default list of Blossom endpoints that accept opaque + `application/octet-stream` ciphertext, and encrypted group-image uploads use that list's primary endpoint, replacing + the media-only Primal default that returned HTTP 415. Blossom upload failures also retain bounded, privacy-filtered + server rejection reasons for actionable diagnostics. Group images store no endpoint in group state, so clients + compiled with different defaults resolve them against different endpoints until the group image is re-set on a + current build. + ## [0.9.3] - 2026-07-07 ### Added diff --git a/crates/cli/README.md b/crates/cli/README.md index a7a9e40d..b8148142 100644 --- a/crates/cli/README.md +++ b/crates/cli/README.md @@ -247,14 +247,19 @@ Media commands: ```sh wn --account media list wn --account media upload --send --message -wn --account media upload --server https://blossom.primal.net +wn --account media upload --server https://blossom.divine.video wn --account media download --output ./file.jpg ``` `media upload` encrypts the file with the group's current `MLS-Exporter("marmot", "encrypted-media", 32)` media secret, -uploads the ciphertext to Blossom, and optionally sends a kind-9 media message. Without `--server`, the upload targets -the endpoints in the group's `marmot.group.encrypted-media.v1` component (`https://blossom.primal.net` is the endpoint -baked into newly created groups' component, not a hardcoded CLI default). +uploads the ciphertext to Blossom, and optionally sends a kind-9 media message. Because the encrypted bytes are opaque, +the server must accept `application/octet-stream` uploads. Without `--server`, the upload targets the ordered endpoints +in the group's `marmot.group.encrypted-media.v1` component. Newly created groups use MDK's ciphertext-compatible +built-in endpoint list unless the application was compiled with `MARMOT_ENCRYPTED_MEDIA_BLOB_ENDPOINTS`. + +Endpoint policy is signed group state. Upgrading MDK changes the defaults for new groups only; an active group admin must +call `replace_encrypted_media_blob_endpoints` to migrate an existing group whose embedded endpoints reject encrypted +blobs. Upload JSON returns an `attachments` array with each attachment's `plaintext_sha256`, `ciphertext_sha256`, and locators. `media download` resolves a projected media reference by plaintext hash, fetches the encrypted blob, verifies it, decrypts it, and writes the plaintext file. diff --git a/crates/marmot-app/README.md b/crates/marmot-app/README.md index 96da16eb..ebbef06b 100644 --- a/crates/marmot-app/README.md +++ b/crates/marmot-app/README.md @@ -55,6 +55,22 @@ list helpers, and public re-exports. Runtime orchestration lives in the `src/run live in the `src/client/` module, group DTOs/component projection helpers live in `src/groups.rs`, and encrypted-media DTOs plus Blossom upload/download helpers live in the `src/media/` module. +## Encrypted media endpoints + +Encrypted media and encrypted group images are uploaded as opaque `application/octet-stream` blobs. A compatible +Blossom server must accept arbitrary binary data rather than only recognizable image, audio, or video payloads. New +groups use the ordered built-in ciphertext-compatible endpoint list unless the host build supplies +`MARMOT_ENCRYPTED_MEDIA_BLOB_ENDPOINTS`. Encrypted media uploads try those endpoints in order; encrypted group-image +uploads use only the primary (first) endpoint. + +The endpoint list is embedded in the signed `marmot.group.encrypted-media.v1` component. Changing application defaults +does not rewrite existing group state. An active group admin can migrate an existing group with +`replace_encrypted_media_blob_endpoints` through the app runtime or UniFFI API. + +Encrypted group images differ: no endpoint is stored in group state, so upload and fetch both resolve against the +build's primary endpoint. Clients compiled with different defaults therefore look for group images in different +places; re-setting the group image on a current build republishes it to the current primary endpoint. + ## Run the tests ```sh diff --git a/crates/marmot-app/src/client/mod.rs b/crates/marmot-app/src/client/mod.rs index d7ffdcbe..c7018cd6 100644 --- a/crates/marmot-app/src/client/mod.rs +++ b/crates/marmot-app/src/client/mod.rs @@ -27,7 +27,7 @@ use crate::groups::{ }; use crate::ids::{admin_pubkey_from_account_id_hex, admin_pubkey_from_member_id}; use crate::media::{ - DEFAULT_BLOSSOM_SERVER_URL, download_encrypted_media, fetch_group_image, + DEFAULT_BLOSSOM_SERVER_URLS, download_encrypted_media, fetch_group_image, is_loopback_http_endpoint, upload_encrypted_media, upload_group_image, }; use crate::messages::{AppMessageIntent, build_inner_event, encode_inner_event, tag_value}; @@ -1951,7 +1951,10 @@ impl AppClient { .encrypted_media_blob_endpoints .is_empty() { - vec![DEFAULT_BLOSSOM_SERVER_URL.to_owned()] + DEFAULT_BLOSSOM_SERVER_URLS + .iter() + .map(|endpoint| (*endpoint).to_owned()) + .collect() } else { self.app .service_endpoints() diff --git a/crates/marmot-app/src/lib.rs b/crates/marmot-app/src/lib.rs index 4ca0f96c..8a48258f 100644 --- a/crates/marmot-app/src/lib.rs +++ b/crates/marmot-app/src/lib.rs @@ -143,9 +143,10 @@ pub use ids::{ /// depending on `marmot-forensics` directly. pub use marmot_forensics::AuditDataMode; pub use media::{ - DEFAULT_BLOSSOM_SERVER_URL, ENCRYPTED_MEDIA_VERSION, MediaAttachmentReference, - MediaDownloadResult, MediaLocator, MediaUploadAttachmentRequest, MediaUploadAttachmentResult, - MediaUploadRequest, MediaUploadResult, media_attachment_from_imeta_tag, + DEFAULT_BLOSSOM_SERVER_URL, DEFAULT_BLOSSOM_SERVER_URLS, ENCRYPTED_MEDIA_VERSION, + MediaAttachmentReference, MediaDownloadResult, MediaLocator, MediaUploadAttachmentRequest, + MediaUploadAttachmentResult, MediaUploadRequest, MediaUploadResult, + media_attachment_from_imeta_tag, }; pub use messages::{is_stream_final_event, tag_value, tag_values}; pub use notifications::{ diff --git a/crates/marmot-app/src/media/blossom.rs b/crates/marmot-app/src/media/blossom.rs index 626a0334..94338a04 100644 --- a/crates/marmot-app/src/media/blossom.rs +++ b/crates/marmot-app/src/media/blossom.rs @@ -12,6 +12,7 @@ use crate::{AppError, unix_now_seconds}; const BLOSSOM_UPLOAD_AUTH_TTL: Duration = Duration::from_secs(10 * 60); const BLOSSOM_UPLOAD_CONTENT_TYPE: &str = "application/octet-stream"; +const MAX_BLOSSOM_ERROR_BODY_BYTES: u64 = 1024; const MEDIA_HTTP_CONNECT_TIMEOUT: Duration = Duration::from_secs(5); const MEDIA_HTTP_READ_TIMEOUT: Duration = Duration::from_secs(15); const MEDIA_HTTP_TOTAL_TIMEOUT: Duration = Duration::from_secs(60); @@ -45,10 +46,7 @@ pub(crate) async fn upload_blossom_blob( .await .map_err(reqwest_blob_error)?; if !response.status().is_success() { - return Err(AppError::BlobStore(format!( - "upload returned HTTP {}", - response.status().as_u16() - ))); + return Err(blossom_upload_status_error(response).await); } let descriptor = response .json::() @@ -76,6 +74,68 @@ pub(crate) async fn upload_blossom_blob( Ok(url) } +async fn blossom_upload_status_error(response: reqwest::Response) -> AppError { + let status = response.status().as_u16(); + let header_reason = response + .headers() + .get("X-Reason") + .and_then(|value| value.to_str().ok()) + .and_then(privacy_safe_server_reason); + let body = read_limited_blossom_body(response, MAX_BLOSSOM_ERROR_BODY_BYTES) + .await + .ok(); + let body_reason = body + .as_deref() + .and_then(blossom_error_body_reason) + .and_then(|reason| privacy_safe_server_reason(&reason)); + match header_reason.or(body_reason) { + Some(reason) => AppError::BlobStore(format!("upload returned HTTP {status}: {reason}")), + None => AppError::BlobStore(format!("upload returned HTTP {status}")), + } +} + +fn blossom_error_body_reason(body: &[u8]) -> Option { + if let Ok(value) = serde_json::from_slice::(body) { + return ["reason", "message", "error"] + .into_iter() + .find_map(|key| value.get(key).and_then(|value| value.as_str())) + .map(str::to_owned); + } + std::str::from_utf8(body).ok().map(str::to_owned) +} + +/// Keep server-provided diagnostics useful without allowing an untrusted +/// response to inject blob hashes, pubkeys, URLs, or unbounded text into app +/// errors that may later be logged. +fn privacy_safe_server_reason(reason: &str) -> Option { + let reason = reason.split_whitespace().collect::>().join(" "); + if reason.is_empty() + || reason.len() > 256 + || !reason + .chars() + .all(|character| character.is_ascii() && !character.is_ascii_control()) + { + return None; + } + let lowercase = reason.to_ascii_lowercase(); + if ["://", "nostr:", "npub1", "nsec1"] + .iter() + .any(|needle| lowercase.contains(needle)) + { + return None; + } + if reason.split_ascii_whitespace().any(|token| { + // Collapse punctuation so hyphenated hashes and UUIDs still register + // as one long identifier. + let token: String = token.chars().filter(char::is_ascii_alphanumeric).collect(); + token.len() >= 48 + || (token.len() >= 32 && token.bytes().all(|byte| byte.is_ascii_hexdigit())) + }) { + return None; + } + Some(reason) +} + pub(crate) async fn fetch_blossom_blob( url: &str, allow_loopback_http: bool, diff --git a/crates/marmot-app/src/media/mod.rs b/crates/marmot-app/src/media/mod.rs index 2049a7ed..9b608ba6 100644 --- a/crates/marmot-app/src/media/mod.rs +++ b/crates/marmot-app/src/media/mod.rs @@ -27,7 +27,21 @@ pub(crate) use crypto::canonical_media_type; pub(crate) use group_image::{fetch_group_image, upload_group_image}; pub(crate) use host_safety::is_loopback_http_endpoint; -pub const DEFAULT_BLOSSOM_SERVER_URL: &str = "https://blossom.primal.net"; +/// Built-in encrypted-media endpoints, in upload fallback order. +/// +/// Every endpoint in this list must accept opaque `application/octet-stream` +/// blobs. Encrypted media and encrypted group images are indistinguishable +/// from random bytes, so media-only Blossom servers are not compatible even +/// when the original plaintext was an image or video. +pub const DEFAULT_BLOSSOM_SERVER_URLS: &[&str] = &[ + "https://blossom.divine.video", + "https://blossom.ditto.pub", + "https://cdn.hzrd149.com", +]; + +/// Primary built-in Blossom endpoint used by single-endpoint APIs such as +/// encrypted group-image upload. +pub const DEFAULT_BLOSSOM_SERVER_URL: &str = DEFAULT_BLOSSOM_SERVER_URLS[0]; pub const ENCRYPTED_MEDIA_VERSION: &str = ENCRYPTED_MEDIA_FORMAT_V1; #[derive(Clone, Debug, PartialEq, Eq)] diff --git a/crates/marmot-app/src/media/tests.rs b/crates/marmot-app/src/media/tests.rs index 6f154dbc..dff6b1e0 100644 --- a/crates/marmot-app/src/media/tests.rs +++ b/crates/marmot-app/src/media/tests.rs @@ -113,6 +113,18 @@ fn http_status_response(status: u16, reason: &str) -> Vec { .into_bytes() } +fn http_error_response(status: u16, reason: &str, headers: &[(&str, &str)], body: &str) -> Vec { + let headers = headers + .iter() + .map(|(name, value)| format!("{name}: {value}\r\n")) + .collect::(); + format!( + "HTTP/1.1 {status} {reason}\r\n{headers}Content-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) + .into_bytes() +} + fn blossom_endpoint(base_url: String) -> BlobStoreEndpointV1 { BlobStoreEndpointV1 { locator_kind: BLOSSOM_LOCATOR_KIND_V1.to_owned(), @@ -222,6 +234,178 @@ async fn upload_encrypted_media_reports_all_blossom_endpoint_failures() { ); } +#[tokio::test] +async fn upload_encrypted_media_preserves_privacy_safe_blossom_rejection_reason() { + let rejecting = spawn_http_response(http_error_response( + 415, + "Unsupported Media Type", + &[], + "upload rejected: unsupported media type application/octet-stream", + )); + let endpoints = [blossom_endpoint(rejecting)]; + let secret = media_secret(); + let keys = signing_keys(); + + let error = upload_encrypted_media( + media_upload_request(None), + 42, + &secret, + &keys, + &endpoints, + &[], + true, + ) + .await + .expect_err("the server rejection should fail the upload"); + + assert_eq!( + error.to_string(), + "blob store request failed: upload failed for all Blossom servers: server 1: upload returned HTTP 415: upload rejected: unsupported media type application/octet-stream" + ); +} + +#[tokio::test] +async fn upload_encrypted_media_drops_sensitive_blossom_rejection_reason() { + let secret_value = "11".repeat(32); + let rejecting = spawn_http_response(http_error_response( + 403, + "Forbidden", + &[( + "X-Reason", + &format!("blob https://media.example/{secret_value} is forbidden"), + )], + "", + )); + let endpoints = [blossom_endpoint(rejecting)]; + let secret = media_secret(); + let keys = signing_keys(); + + let error = upload_encrypted_media( + media_upload_request(None), + 42, + &secret, + &keys, + &endpoints, + &[], + true, + ) + .await + .expect_err("the server rejection should fail the upload"); + let message = error.to_string(); + + assert!(message.contains("upload returned HTTP 403")); + assert!(!message.contains("media.example")); + assert!(!message.contains(&secret_value)); +} + +#[tokio::test] +async fn upload_encrypted_media_drops_punctuated_hash_rejection_reason() { + let secret_value = "11".repeat(32); + let rejecting = spawn_http_response(http_error_response( + 409, + "Conflict", + &[( + "X-Reason", + &format!("duplicate-blob-{secret_value}-already-exists"), + )], + "", + )); + let endpoints = [blossom_endpoint(rejecting)]; + let secret = media_secret(); + let keys = signing_keys(); + + let error = upload_encrypted_media( + media_upload_request(None), + 42, + &secret, + &keys, + &endpoints, + &[], + true, + ) + .await + .expect_err("the server rejection should fail the upload"); + let message = error.to_string(); + + assert!(message.contains("upload returned HTTP 409")); + assert!(!message.contains(&secret_value)); +} + +#[tokio::test] +async fn upload_encrypted_media_drops_uuid_rejection_reason() { + let rejecting = spawn_http_response(http_error_response( + 403, + "Forbidden", + &[( + "X-Reason", + "upload id 123e4567-e89b-12d3-a456-426614174000 already used", + )], + "", + )); + let endpoints = [blossom_endpoint(rejecting)]; + let secret = media_secret(); + let keys = signing_keys(); + + let error = upload_encrypted_media( + media_upload_request(None), + 42, + &secret, + &keys, + &endpoints, + &[], + true, + ) + .await + .expect_err("the server rejection should fail the upload"); + let message = error.to_string(); + + assert!(message.contains("upload returned HTTP 403")); + assert!(!message.contains("123e4567")); +} + +#[tokio::test] +async fn upload_encrypted_media_drops_non_http_url_scheme_rejection_reason() { + let rejecting = spawn_http_response(http_error_response( + 403, + "Forbidden", + &[("X-Reason", "denied, use relay wss://relay.example instead")], + "", + )); + let endpoints = [blossom_endpoint(rejecting)]; + let secret = media_secret(); + let keys = signing_keys(); + + let error = upload_encrypted_media( + media_upload_request(None), + 42, + &secret, + &keys, + &endpoints, + &[], + true, + ) + .await + .expect_err("the server rejection should fail the upload"); + let message = error.to_string(); + + assert!(message.contains("upload returned HTTP 403")); + assert!(!message.contains("relay.example")); +} + +#[test] +fn built_in_blossom_endpoints_are_ciphertext_compatible_fallbacks() { + assert_eq!(DEFAULT_BLOSSOM_SERVER_URL, DEFAULT_BLOSSOM_SERVER_URLS[0]); + assert_eq!( + DEFAULT_BLOSSOM_SERVER_URLS, + [ + "https://blossom.divine.video", + "https://blossom.ditto.pub", + "https://cdn.hzrd149.com", + ] + ); + assert!(!DEFAULT_BLOSSOM_SERVER_URLS.contains(&"https://blossom.primal.net")); +} + #[tokio::test] async fn explicit_blossom_server_override_skips_default_endpoint_failover() { let override_server = spawn_http_response(http_status_response(500, "Internal Server Error")); diff --git a/crates/marmot-app/tests/relay_runtime.rs b/crates/marmot-app/tests/relay_runtime.rs index 9d290ac2..bfd30775 100644 --- a/crates/marmot-app/tests/relay_runtime.rs +++ b/crates/marmot-app/tests/relay_runtime.rs @@ -4099,6 +4099,35 @@ async fn encrypted_media_endpoint_updates_are_full_replacement_and_admin_only() let group_id_hex = hex::encode(group_id.as_slice()); bob.sync().await.unwrap(); + let initial_group = app.group("alice", &group_id_hex).unwrap().unwrap(); + let initial_endpoints = initial_group + .encrypted_media + .default_blob_endpoints + .iter() + .map(|endpoint| endpoint.base_url.as_str()) + .collect::>(); + // Host builds may compile in MARMOT_ENCRYPTED_MEDIA_BLOB_ENDPOINTS, so + // expect whichever list the client actually embeds. + let configured = marmot_app::MarmotServiceEndpoints::compiled().encrypted_media_blob_endpoints; + let expected_endpoints = if configured.is_empty() { + marmot_app::DEFAULT_BLOSSOM_SERVER_URLS + .iter() + .map(|endpoint| (*endpoint).to_owned()) + .collect::>() + } else { + configured + }; + assert_eq!( + initial_endpoints + .iter() + .map(|endpoint| endpoint.trim_end_matches('/')) + .collect::>(), + expected_endpoints + .iter() + .map(|endpoint| endpoint.trim_end_matches('/')) + .collect::>() + ); + let bob_error = bob .replace_encrypted_media_blob_endpoints( &group_id,