From de7c3f9349b0c6d162590f937c6393e21c55a18a Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Wed, 27 May 2026 18:56:50 +0300 Subject: [PATCH 01/12] feat: smart HTTP file cache with parallel downloads and 20% pre-cache trigger MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add rockbox-cache crate: SHA-256 URL keying, parallel Range-request downloads, LRU eviction, disk-space guard, atomic rename, auto-skip for live streams (no Content-Length) and configurable no_cache_patterns - netstream: check cache on open, serve from BufReader on hit, start background fetch on miss - settings: wire all 6 cache fields (enabled, dir, max_size_mb, min_free_space_mb, parallel_parts, no_cache_patterns) from settings.toml - cli: spawn precache-monitor thread — at 20% through current track, pre-fetch the next HTTP track in the background - docs: add HTTP file cache section to mintlify/configuration.mdx and README --- Cargo.lock | 15 + README.md | 33 ++ crates/cache/Cargo.toml | 15 + crates/cache/src/lib.rs | 558 ++++++++++++++++++++++++++ crates/cli/Cargo.toml | 2 + crates/cli/src/lib.rs | 56 +++ crates/netstream/Cargo.toml | 1 + crates/netstream/src/lib.rs | 403 +++++++++---------- crates/settings/Cargo.toml | 1 + crates/settings/src/lib.rs | 36 ++ crates/sys/src/types/user_settings.rs | 20 + mintlify/configuration.mdx | 53 +++ 12 files changed, 984 insertions(+), 209 deletions(-) create mode 100644 crates/cache/Cargo.toml create mode 100644 crates/cache/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 2b1568ff8a8..95af9474d6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7375,6 +7375,7 @@ dependencies = [ "mockito", "once_cell", "reqwest", + "rockbox-cache", "tracing", ] @@ -9324,6 +9325,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "rockbox-cache" +version = "0.1.0" +dependencies = [ + "libc", + "once_cell", + "reqwest", + "sha2", + "tracing", +] + [[package]] name = "rockbox-chromecast" version = "0.1.0" @@ -9355,6 +9367,7 @@ dependencies = [ "owo-colors 4.1.0", "reqwest", "rockbox-airplay", + "rockbox-cache", "rockbox-chromecast", "rockbox-cpal-sink", "rockbox-fts5", @@ -9363,6 +9376,7 @@ dependencies = [ "rockbox-rocksky", "rockbox-settings", "rockbox-slim", + "rockbox-sys", "rockbox-typesense", "rockbox-upnp", "tokio", @@ -9787,6 +9801,7 @@ name = "rockbox-settings" version = "0.1.0" dependencies = [ "anyhow", + "rockbox-cache", "rockbox-sys", "rockbox-upnp", "toml 0.8.19", diff --git a/README.md b/README.md index 953e43b551f..e2200f298f8 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ and Squeezelite. - [x] [Chromecast](https://developers.google.com/cast) - [x] Gapless playback and crossfading - [x] Supports 20+ codecs: MP3, OGG, FLAC, WAV, AAC, Opus, and more +- [x] Smart HTTP file cache — parallel multi-part downloads, LRU eviction, auto-skips live streams ### APIs & integrations - [x] [gRPC API](https://buf.build/tsiry/rockboxapis/docs/main:rockbox.v1alpha1) @@ -452,6 +453,38 @@ parsed and displayed. | `upnp_renderer_port` | `7880` | MediaRenderer HTTP port | | `upnp_friendly_name` | `"Rockbox"` | Display name shown to control points | +### HTTP file cache + +Remote audio files are cached on disk automatically. Repeat plays are instant +with no network traffic. Files are downloaded in the background using parallel +range-request parts so playback is never delayed. + +```toml +# All fields are optional — defaults shown. +cache_enabled = true +cache_dir = "~/.config/rockbox.org/cache" +cache_max_size_mb = 512 # total cache budget in MB +cache_min_free_space_mb = 100 # always keep this much free on disk +cache_parallel_parts = 4 # concurrent HTTP Range parts per file + +# URLs containing any of these substrings bypass the cache. +# Prevents live radio / HLS streams from filling the cache. +cache_no_cache_patterns = ["icecast", ".m3u8", "live"] +``` + +**How it works:** + +| Step | What happens | +| ---- | ------------ | +| 1 | On open, the SHA-256 of the URL is checked against `cache_dir`. | +| 2 | **Hit**: the local file is opened instantly; all seeks are O(1) — no network. | +| 3 | **Miss**: the live HTTP stream opens normally (zero latency) AND a background thread splits the file into `cache_parallel_parts` ranges and fetches them concurrently via `pwrite` into a pre-allocated file. | +| 4 | When the download completes and is verified, the file is atomically renamed. The next open is a cache hit. | +| 5 | When the cache exceeds `cache_max_size_mb`, the least-recently-used files are evicted to make room. | + +URLs with no `Content-Length` (infinite/live streams) are **never** cached — +caching is skipped automatically before a download is even started. + --- ## 🚚 Installation diff --git a/crates/cache/Cargo.toml b/crates/cache/Cargo.toml new file mode 100644 index 00000000000..3f904718aaf --- /dev/null +++ b/crates/cache/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "rockbox-cache" +version = "0.1.0" +edition = "2021" + +[lib] +name = "rockbox_cache" +crate-type = ["rlib"] + +[dependencies] +once_cell = "1.17.1" +sha2 = "0.10" +libc = "0.2.168" +reqwest = { version = "0.12.5", features = ["blocking", "rustls-tls-native-roots"], default-features = false } +tracing = { workspace = true } diff --git a/crates/cache/src/lib.rs b/crates/cache/src/lib.rs new file mode 100644 index 00000000000..0e61dfc20f3 --- /dev/null +++ b/crates/cache/src/lib.rs @@ -0,0 +1,558 @@ +use once_cell::sync::Lazy; +use std::collections::HashSet; +use std::fs::{self, File, OpenOptions}; +use std::io::{self, BufWriter, Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use tracing::{debug, info, warn}; + +// ─── Config ────────────────────────────────────────────────────────────────── + +#[derive(Debug, Clone, Default)] +pub struct CacheConfig { + /// Whether HTTP file caching is enabled (default: true). + pub enabled: bool, + /// Directory where cached files are stored. + /// Default: ~/.config/rockbox.org/cache + pub dir: PathBuf, + /// Maximum total cache size in bytes (default: 512 MB). + pub max_size_bytes: u64, + /// Minimum free disk space that must remain before a new file is cached. + /// Default: 100 MB + pub min_free_space_bytes: u64, + /// Number of parallel HTTP range-request parts used for large files. + /// Set to 1 to disable parallel downloading. Default: 4. + pub parallel_parts: usize, + /// URL substrings whose presence causes caching to be skipped entirely. + /// Useful for live radio streams, HLS manifests, etc. + /// Example entries: `"icecast"`, `".m3u8"`, `"live"`, `"stream"`. + pub no_cache_patterns: Vec, +} + +impl CacheConfig { + pub fn with_defaults() -> Self { + let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); + Self { + enabled: true, + dir: PathBuf::from(format!("{}/.config/rockbox.org/cache", home)), + max_size_bytes: 512 * 1024 * 1024, + min_free_space_bytes: 100 * 1024 * 1024, + parallel_parts: 4, + no_cache_patterns: Vec::new(), + } + } +} + +// ─── Global state ──────────────────────────────────────────────────────────── + +struct CacheManager { + config: CacheConfig, + /// URLs whose background download is currently in-flight. + in_progress: HashSet, +} + +static CACHE: Lazy> = Lazy::new(|| { + Mutex::new(CacheManager { + config: CacheConfig::with_defaults(), + in_progress: HashSet::new(), + }) +}); + +static CLIENT: Lazy = Lazy::new(|| { + reqwest::blocking::Client::builder() + .use_rustls_tls() + .build() + .expect("failed to build cache HTTP client") +}); + +// ─── Public API ────────────────────────────────────────────────────────────── + +/// Replace the active cache configuration. Called once from `load_settings`. +pub fn configure(config: CacheConfig) { + let mut mgr = CACHE.lock().unwrap(); + mgr.config = config; + mgr.in_progress.clear(); +} + +/// Check whether `url` is already cached on disk. +/// +/// On a hit: touches the file (updates mtime for LRU eviction ordering) and +/// returns the local path. On a miss: returns `None`. +pub fn lookup(url: &str) -> Option { + let config = CACHE.lock().unwrap().config.clone(); + if !config.enabled { + return None; + } + let key = url_to_key(url); + let path = config.dir.join(format!("{key}.cache")); + if path.exists() { + touch_file(&path); + debug!("cache: HIT {}", url); + Some(path) + } else { + None + } +} + +/// Kick off a parallel background download of `url` if it is not already +/// cached or in-flight. +/// +/// Skips silently when: +/// - caching is disabled +/// - the URL matches a `no_cache_patterns` entry (e.g. live radio streams) +/// - the server reports no Content-Length (infinite/chunked streams) +/// - disk space is low +/// +/// Returns immediately; the caller's live HTTP stream continues uninterrupted. +pub fn start_background_fetch(url: &str) { + { + let mut mgr = CACHE.lock().unwrap(); + if !mgr.config.enabled { + return; + } + // Skip URLs that match a no-cache pattern (e.g. live streams, HLS). + if mgr + .config + .no_cache_patterns + .iter() + .any(|p| url.contains(p.as_str())) + { + debug!("cache: no-cache pattern match, skipping: {}", url); + return; + } + if mgr.in_progress.contains(url) { + return; + } + let key = url_to_key(url); + if mgr.config.dir.join(format!("{key}.cache")).exists() { + return; + } + let avail = available_disk_space(&mgr.config.dir).unwrap_or(0); + if avail < mgr.config.min_free_space_bytes { + debug!( + "cache: low disk space — skipping background fetch for {}", + url + ); + return; + } + mgr.in_progress.insert(url.to_string()); + } + + let url_owned = url.to_string(); + std::thread::Builder::new() + .name("cache-fetch".into()) + .spawn(move || { + let config = CACHE.lock().unwrap().config.clone(); + perform_download(&url_owned, &config); + CACHE.lock().unwrap().in_progress.remove(&url_owned); + }) + .ok(); +} + +// ─── Key derivation ────────────────────────────────────────────────────────── + +/// Convert a URL into a fixed-length, filesystem-safe cache key (SHA-256 hex). +pub fn url_to_key(url: &str) -> String { + use sha2::{Digest, Sha256}; + let hash = Sha256::digest(url.as_bytes()); + hash.iter().map(|b| format!("{:02x}", b)).collect() +} + +// ─── Disk helpers ──────────────────────────────────────────────────────────── + +#[cfg(unix)] +fn available_disk_space(path: &Path) -> Option { + use std::ffi::CString; + let p = CString::new(path.to_string_lossy().as_bytes()).ok()?; + let mut stat: libc::statvfs = unsafe { std::mem::zeroed() }; + if unsafe { libc::statvfs(p.as_ptr(), &mut stat) } != 0 { + return None; + } + Some(stat.f_bavail as u64 * stat.f_frsize as u64) +} + +#[cfg(not(unix))] +fn available_disk_space(_path: &Path) -> Option { + Some(u64::MAX) +} + +fn current_cache_size(dir: &Path) -> u64 { + let Ok(rd) = fs::read_dir(dir) else { + return 0; + }; + rd.flatten() + .filter_map(|e| { + let p = e.path(); + if p.extension().and_then(|x| x.to_str()) == Some("cache") { + e.metadata().ok().map(|m| m.len()) + } else { + None + } + }) + .sum() +} + +fn evict_lru_until(dir: &Path, needed: u64) { + let Ok(rd) = fs::read_dir(dir) else { + return; + }; + let mut files: Vec<(PathBuf, u64, std::time::SystemTime)> = rd + .flatten() + .filter_map(|e| { + let p = e.path(); + if p.extension().and_then(|x| x.to_str()) != Some("cache") { + return None; + } + let meta = e.metadata().ok()?; + let mtime = meta.modified().ok()?; + Some((p, meta.len(), mtime)) + }) + .collect(); + files.sort_by_key(|(_, _, mtime)| *mtime); // oldest first + + let mut freed = 0u64; + for (path, size, _) in files { + if freed >= needed { + break; + } + if fs::remove_file(&path).is_ok() { + freed += size; + debug!("cache: evicted {}", path.display()); + } + } +} + +#[cfg(unix)] +fn touch_file(path: &Path) { + use std::ffi::CString; + if let Ok(p) = CString::new(path.to_string_lossy().as_bytes()) { + unsafe { libc::utimes(p.as_ptr(), std::ptr::null()) }; + } +} + +#[cfg(not(unix))] +fn touch_file(_path: &Path) {} + +// ─── Parallel-write helper ─────────────────────────────────────────────────── + +/// Write `buf` starting at byte `offset` in `file` without advancing the file +/// cursor (uses `pwrite` on Unix, `seek_write` on Windows). Loops until all +/// bytes are written so callers never see short writes. +#[cfg(unix)] +fn write_at_offset(file: &File, buf: &[u8], mut offset: u64) -> io::Result<()> { + use std::os::unix::fs::FileExt; + let mut pos = 0; + while pos < buf.len() { + let n = file.write_at(&buf[pos..], offset)?; + if n == 0 { + return Err(io::Error::new( + io::ErrorKind::WriteZero, + "write_at returned 0", + )); + } + pos += n; + offset += n as u64; + } + Ok(()) +} + +#[cfg(windows)] +fn write_at_offset(file: &File, buf: &[u8], mut offset: u64) -> io::Result<()> { + use std::os::windows::fs::FileExt; + let mut pos = 0; + while pos < buf.len() { + let n = file.seek_write(&buf[pos..], offset)?; + if n == 0 { + return Err(io::Error::new( + io::ErrorKind::WriteZero, + "seek_write returned 0", + )); + } + pos += n; + offset += n as u64; + } + Ok(()) +} + +#[cfg(not(any(unix, windows)))] +fn write_at_offset(_file: &File, _buf: &[u8], _offset: u64) -> io::Result<()> { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "write_at_offset unavailable", + )) +} + +// ─── Download strategies ───────────────────────────────────────────────────── + +/// Files at or above this size use parallel range-request downloading. +const MIN_PARALLEL_SIZE: u64 = 2 * 1024 * 1024; // 2 MB + +/// Download `url` using `num_parts` simultaneous HTTP Range requests. +/// +/// Each part writes directly to its allocated region in a pre-allocated temp +/// file via `pwrite` / `seek_write`, so threads never contend on a lock. +/// +/// Returns `Ok(())` on success or `Err` if the server rejects range requests — +/// the caller falls back to `sequential_download` in that case. +fn parallel_download(url: &str, temp: &Path, total_size: u64, num_parts: usize) -> io::Result<()> { + // Pre-allocate the complete file so every thread can write at any offset. + { + let f = File::create(temp)?; + f.set_len(total_size)?; + } + + let chunk = (total_size + num_parts as u64 - 1) / num_parts as u64; // ⌈total/parts⌉ + let mut handles: Vec>> = Vec::with_capacity(num_parts); + + for i in 0..num_parts { + let start = i as u64 * chunk; + if start >= total_size { + break; + } + let end = (start + chunk - 1).min(total_size - 1); + let url = url.to_string(); + let temp = temp.to_path_buf(); + + let handle = std::thread::Builder::new() + .name(format!("cache-part-{i}")) + .spawn(move || -> io::Result<()> { + let range = format!("bytes={start}-{end}"); + let mut resp = CLIENT + .get(&url) + .header("Range", range) + .send() + .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + + if resp.status().as_u16() != 206 { + return Err(io::Error::new( + io::ErrorKind::Other, + format!("expected 206, got {}", resp.status()), + )); + } + + let file = OpenOptions::new().write(true).open(&temp)?; + let mut offset = start; + let mut buf = [0u8; 65536]; + + loop { + let n = resp + .read(&mut buf) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + if n == 0 { + break; + } + write_at_offset(&file, &buf[..n], offset)?; + offset += n as u64; + } + Ok(()) + }) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + + handles.push(handle); + } + + // Wait for all threads; surface the first error. + let mut first_err: Option = None; + for handle in handles { + match handle.join() { + Ok(Ok(())) => {} + Ok(Err(e)) if first_err.is_none() => first_err = Some(e), + Err(_) if first_err.is_none() => { + first_err = Some(io::Error::new( + io::ErrorKind::Other, + "download thread panicked", + )) + } + _ => {} + } + } + + if let Some(e) = first_err { + return Err(e); + } + Ok(()) +} + +/// Sequential (single-connection) download. Used for small files and as a +/// fallback when the server does not support range requests. +fn sequential_download(url: &str, temp: &Path) -> io::Result { + let mut resp = CLIENT + .get(url) + .send() + .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + + if !resp.status().is_success() { + return Err(io::Error::new( + io::ErrorKind::Other, + format!("HTTP {}", resp.status()), + )); + } + + let file = File::create(temp)?; + let mut writer = BufWriter::with_capacity(64 * 1024, file); + let mut buf = [0u8; 65536]; + let mut total = 0u64; + + loop { + let n = resp + .read(&mut buf) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + if n == 0 { + break; + } + writer.write_all(&buf[..n])?; + total += n as u64; + } + writer.flush()?; + Ok(total) +} + +// ─── Main download orchestrator ────────────────────────────────────────────── + +fn perform_download(url: &str, config: &CacheConfig) { + let key = url_to_key(url); + let temp_path = config.dir.join(format!("{key}.tmp")); + let final_path = config.dir.join(format!("{key}.cache")); + + if final_path.exists() { + return; // another thread finished first + } + + if let Err(e) = fs::create_dir_all(&config.dir) { + warn!("cache: cannot create cache dir: {}", e); + return; + } + + // HEAD request: get content-length and check range support. + // If the server returns no Content-Length the resource is likely a live / + // infinite stream — never attempt to cache it. + let (content_length, server_accepts_ranges) = match CLIENT.head(url).send() { + Ok(r) if r.status().is_success() => { + let cl = r.content_length(); + let ok = r + .headers() + .get("accept-ranges") + .and_then(|v| v.to_str().ok()) + .map(|v| v.trim() != "none") + .unwrap_or(true); // assume yes when header is absent + (cl, ok) + } + _ => (None, false), + }; + + let content_length = match content_length { + Some(cl) => cl, + None => { + // No Content-Length → infinite/chunked stream; never cache. + debug!( + "cache: no Content-Length for {} — skipping (live stream?)", + url + ); + return; + } + }; + + // Enforce max cache size, evicting the oldest files first if needed. + let current = current_cache_size(&config.dir); + if current + content_length > config.max_size_bytes { + let excess = (current + content_length).saturating_sub(config.max_size_bytes); + evict_lru_until(&config.dir, excess); + } + let avail = available_disk_space(&config.dir).unwrap_or(u64::MAX); + if avail < config.min_free_space_bytes.saturating_add(content_length) { + debug!( + "cache: not enough free space for {} ({} bytes needed)", + url, content_length + ); + return; + } + + // Strategy: parallel range-requests for large files, sequential otherwise. + let num_parts = config.parallel_parts.max(1); + let use_parallel = + server_accepts_ranges && num_parts > 1 && content_length >= MIN_PARALLEL_SIZE; + + let downloaded: u64 = if use_parallel { + debug!( + "cache: parallel download ({} parts) {} ({} bytes)", + num_parts, url, content_length + ); + match parallel_download(url, &temp_path, content_length, num_parts) { + Ok(()) => content_length, + Err(e) => { + debug!( + "cache: parallel failed for {} ({}), retrying sequential", + url, e + ); + let _ = fs::remove_file(&temp_path); + match sequential_download(url, &temp_path) { + Ok(n) => n, + Err(e) => { + debug!("cache: sequential also failed for {}: {}", url, e); + let _ = fs::remove_file(&temp_path); + return; + } + } + } + } + } else { + match sequential_download(url, &temp_path) { + Ok(n) => n, + Err(e) => { + debug!("cache: download failed for {}: {}", url, e); + let _ = fs::remove_file(&temp_path); + return; + } + } + }; + + // Verify the download is complete before promoting to the final path. + if downloaded != content_length { + debug!( + "cache: incomplete download for {} ({}/{} bytes) — discarding", + url, downloaded, content_length + ); + let _ = fs::remove_file(&temp_path); + return; + } + + match fs::rename(&temp_path, &final_path) { + Ok(()) => info!( + "cache: stored {} ({} bytes, {} parts)", + url, + downloaded, + if use_parallel { num_parts } else { 1 } + ), + Err(e) => { + warn!("cache: rename failed for {}: {}", url, e); + let _ = fs::remove_file(&temp_path); + } + } +} + +// ─── Tests ─────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn url_to_key_is_deterministic() { + let k1 = url_to_key("https://example.com/song.mp3"); + let k2 = url_to_key("https://example.com/song.mp3"); + assert_eq!(k1, k2); + assert_eq!(k1.len(), 64, "SHA-256 hex is always 64 chars"); + } + + #[test] + fn url_to_key_differs_for_distinct_urls() { + let k1 = url_to_key("https://example.com/a.mp3"); + let k2 = url_to_key("https://example.com/b.mp3"); + assert_ne!(k1, k2); + } + + #[test] + fn lookup_returns_none_for_uncached_url() { + assert!(lookup("https://example.com/definitely-not-cached-xyz.mp3").is_none()); + } +} diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index d3faf94845d..398dbc1efc9 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -21,6 +21,8 @@ rockbox-typesense = {path = "../typesense"} rockbox-fts5 = {path = "../fts5", optional = true} rockbox-cpal-sink = {path = "../cpal-sink", optional = true} rockbox-settings = {path = "../settings"} +rockbox-cache = {path = "../cache"} +rockbox-sys = {path = "../sys"} rockbox-rocksky = {path = "../rocksky"} tokio = {version = "1.36.0", features = ["full"]} dirs = "6.0.0" diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index b8176f3ab60..5eb967d70df 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -239,9 +239,65 @@ Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \ }); spawn_typesense_subprocess(); + spawn_precache_monitor(); return 0; } +fn spawn_precache_monitor() { + thread::Builder::new() + .name("precache-monitor".into()) + .spawn(|| { + // Give the firmware time to initialise before we start polling. + sleep(Duration::from_secs(10)); + let mut last_current_path: Option = None; + let mut last_precached: Option = None; + loop { + sleep(Duration::from_secs(3)); + let current = match rockbox_sys::playback::current_track() { + Some(t) => t, + None => { + last_current_path = None; + last_precached = None; + continue; + } + }; + // Reset per-track state whenever the track changes. + if last_current_path.as_deref() != Some(¤t.path) { + last_current_path = Some(current.path.clone()); + last_precached = None; + } + if current.length == 0 { + continue; + } + let progress = current.elapsed as f64 / current.length as f64; + if progress < 0.20 { + continue; + } + let next = match rockbox_sys::playback::next_track() { + Some(t) => t, + None => continue, + }; + if next.path.is_empty() { + continue; + } + if !next.path.starts_with("http://") && !next.path.starts_with("https://") { + continue; + } + if last_precached.as_deref() == Some(&next.path) { + continue; + } + tracing::info!( + "precache: {}% through current track — pre-caching next: {}", + (progress * 100.0) as u32, + next.path + ); + rockbox_cache::start_background_fetch(&next.path); + last_precached = Some(next.path); + } + }) + .ok(); +} + #[cfg(not(feature = "fts5"))] async fn run_indexing(path: String, update_library: bool) -> Result<(), Error> { info!("Setting up Typesense search engine..."); diff --git a/crates/netstream/Cargo.toml b/crates/netstream/Cargo.toml index 96012cb2a54..19789e34a56 100644 --- a/crates/netstream/Cargo.toml +++ b/crates/netstream/Cargo.toml @@ -12,6 +12,7 @@ reqwest = { version = "0.12.5", features = ["blocking", "rustls-tls-native-roots once_cell = "1.17.1" libc = "0.2.168" tracing = "0.1" +rockbox-cache = { path = "../cache" } [dev-dependencies] mockito = "1.7.2" diff --git a/crates/netstream/src/lib.rs b/crates/netstream/src/lib.rs index 4a040050ad6..5a06dabb42c 100644 --- a/crates/netstream/src/lib.rs +++ b/crates/netstream/src/lib.rs @@ -1,7 +1,8 @@ use once_cell::sync::Lazy; use std::collections::HashMap; use std::ffi::CStr; -use std::io::{self, Read}; +use std::fs::File; +use std::io::{self, BufReader, Read, Seek}; use std::os::raw::c_char; use std::sync::atomic::{AtomicI32, Ordering}; use std::sync::{Arc, Mutex}; @@ -10,13 +11,26 @@ use tracing::{debug, warn}; /// Sentinel handle ID returned on error. const INVALID_HANDLE: i32 = -1; -/// Per-stream state. +// ─── Stream source ─────────────────────────────────────────────────────────── + +/// Where bytes for a stream come from. +enum StreamSource { + /// Live HTTP response (standard path). + Http { + response: Option, + }, + /// Local file served from the on-disk cache (cache-hit path). + CachedFile { reader: BufReader }, +} + +// ─── Per-stream state ──────────────────────────────────────────────────────── + struct StreamState { url: String, pos: u64, content_length: Option, content_type: Option, - response: Option, + source: StreamSource, } impl StreamState { @@ -34,7 +48,8 @@ impl StreamState { } } - fn new(url: String) -> Option { + /// Open a new HTTP stream. + fn new_http(url: String) -> Option { let response = CLIENT.get(&url).send().ok()?; if !response.status().is_success() { return None; @@ -46,22 +61,55 @@ impl StreamState { pos: 0, content_length, content_type, - response: Some(response), + source: StreamSource::Http { + response: Some(response), + }, + }) + } + + /// Open a cached file as a stream. + fn new_from_cache(url: &str, path: &std::path::Path) -> Option { + let file = File::open(path).ok()?; + let size = file.metadata().ok()?.len(); + let reader = BufReader::new(file); + // Best-effort content-type from the URL's file extension (strip query string first). + let url_path = url.split('?').next().unwrap_or(url); + let ext = url_path + .rsplit('.') + .next() + .unwrap_or("") + .to_ascii_lowercase(); + let content_type = match ext.as_str() { + "mp3" => Some("audio/mpeg".to_string()), + "flac" => Some("audio/flac".to_string()), + "ogg" | "oga" => Some("audio/ogg".to_string()), + "m4a" | "aac" => Some("audio/aac".to_string()), + "wav" => Some("audio/wav".to_string()), + "opus" => Some("audio/opus".to_string()), + "wv" => Some("audio/x-wavpack".to_string()), + "ape" => Some("audio/x-ape".to_string()), + "mpc" => Some("audio/x-musepack".to_string()), + _ => None, + }; + Some(StreamState { + url: url.to_string(), + pos: 0, + content_length: Some(size), + content_type, + source: StreamSource::CachedFile { reader }, }) } fn skip_bytes(resp: &mut reqwest::blocking::Response, mut to_skip: u64) -> bool { let mut buf = [0u8; 8192]; - while to_skip > 0 { let chunk = usize::min(to_skip as usize, buf.len()); match resp.read(&mut buf[..chunk]) { Ok(0) => return false, - Ok(bytes_read) => to_skip -= bytes_read as u64, + Ok(n) => to_skip -= n as u64, Err(_) => return false, } } - true } @@ -69,7 +117,6 @@ impl StreamState { if self.content_length.is_some() { return; } - if let Some(cr) = resp.headers().get("content-range") { if let Ok(cr_str) = cr.to_str() { if let Some(total_str) = cr_str.split('/').last() { @@ -89,37 +136,47 @@ impl StreamState { start.trim().parse::().ok() } - /// Re-issue the request starting at `new_pos` using an HTTP Range header. - /// Falls back to reopening from byte 0 and discarding bytes if the server - /// ignores Range and responds with the full body. + /// Seek the stream to `new_pos`. /// - /// The existing response is only replaced on success. A failed seek leaves - /// the stream at its current position so reads can still continue. + /// For cached files: instant O(1) file seek. + /// For HTTP streams: inline byte-skip for small forward seeks, Range + /// request for anything larger or backward. fn seek_to(&mut self, new_pos: u64) -> bool { + // CachedFile: direct seek — no network round-trip ever needed. + if let StreamSource::CachedFile { reader } = &mut self.source { + use std::io::{Seek, SeekFrom}; + match reader.seek(SeekFrom::Start(new_pos)) { + Ok(_) => { + self.pos = new_pos; + return true; + } + Err(_) => return false, + } + } + + // HTTP path ────────────────────────────────────────────────────────── + // Small forward seek: skip bytes in the existing response body rather - // than issuing a new Range request. Avoids a full round-trip for the - // tiny metadata seeks that codecs commonly do (ID3 tags, MP4 atoms). + // than issuing a new Range request. const INLINE_SKIP_MAX: u64 = 128 * 1024; // 128 KB if new_pos > self.pos && new_pos - self.pos <= INLINE_SKIP_MAX { - if let Some(resp) = &mut self.response { + if let StreamSource::Http { + response: Some(resp), + } = &mut self.source + { let to_skip = new_pos - self.pos; if Self::skip_bytes(resp, to_skip) { self.pos = new_pos; return true; } - // Inline skip failed (connection dropped); fall through to Range request. + // Inline skip failed; fall through to a Range request. } } - // Use a bounded range when content_length is known so the request is - // always within [0, content_length). This prevents spurious 416 - // responses and tells the server exactly how many bytes we want. let range_header = match self.content_length { Some(cl) if cl > 0 => format!("bytes={}-{}", new_pos, cl - 1), _ => format!("bytes={}-", new_pos), }; - // Do NOT clear self.response before the request succeeds. If the new - // request fails the stream stays readable at the current position. let result = CLIENT.get(&self.url).header("Range", range_header).send(); match result { @@ -128,12 +185,12 @@ impl StreamState { if self.content_type.is_none() { self.content_type = Self::response_content_type(&resp); } - if Self::parse_content_range_start(&resp) != Some(new_pos) { return false; } - - self.response = Some(resp); + if let StreamSource::Http { response } = &mut self.source { + *response = Some(resp); + } self.pos = new_pos; true } @@ -144,12 +201,12 @@ impl StreamState { if self.content_type.is_none() { self.content_type = Self::response_content_type(&resp); } - if new_pos > 0 && !Self::skip_bytes(&mut resp, new_pos) { return false; } - - self.response = Some(resp); + if let StreamSource::Http { response } = &mut self.source { + *response = Some(resp); + } self.pos = new_pos; true } @@ -160,11 +217,10 @@ impl StreamState { fn read_as_file(reader: &mut R, buf: &mut [u8]) -> io::Result { let mut total = 0; - while total < buf.len() { match reader.read(&mut buf[total..]) { Ok(0) => break, - Ok(bytes_read) => total += bytes_read, + Ok(n) => total += n, Err(err) if err.kind() == io::ErrorKind::Interrupted => continue, Err(err) => { if total > 0 { @@ -174,7 +230,6 @@ fn read_as_file(reader: &mut R, buf: &mut [u8]) -> io::Result { } } } - Ok(total) } @@ -190,12 +245,17 @@ static CLIENT: Lazy = Lazy::new(|| { static NEXT_HANDLE: AtomicI32 = AtomicI32::new(0); -// ------------------------------------------------------------------ -// Public C ABI -// ------------------------------------------------------------------ +// ─── Public C ABI ──────────────────────────────────────────────────────────── /// Open a URL and return an integer handle, or -1 on failure. /// +/// For remote HTTP(S) URLs: +/// - If the URL is already in the local cache the stream is served from the +/// cached file (instant open, zero network traffic, full seek support). +/// - On a cache miss, the URL is opened via HTTP as normal **and** a +/// background thread is spawned to download and cache the full file in +/// parallel so subsequent opens are instant. +/// /// # Safety /// `url` must be a valid, NUL-terminated C string. #[no_mangle] @@ -206,14 +266,38 @@ pub unsafe extern "C" fn rb_net_open(url: *const c_char) -> i32 { let url_str = match CStr::from_ptr(url).to_str() { Ok(s) => { let s = s.trim(); - // Drop any URL fragment — the codec only needs the resource itself. - let s = s.split('#').next().unwrap_or(s); + let s = s.split('#').next().unwrap_or(s); // drop URL fragment s.to_owned() } Err(_) => return INVALID_HANDLE, }; - let state = match StreamState::new(url_str.clone()) { + // ── Cache integration ────────────────────────────────────────────────── + let is_http = url_str.starts_with("http://") || url_str.starts_with("https://"); + if is_http { + // Cache hit: serve directly from disk, no network needed. + if let Some(cached_path) = rockbox_cache::lookup(&url_str) { + if let Some(state) = StreamState::new_from_cache(&url_str, &cached_path) { + let handle = NEXT_HANDLE.fetch_add(1, Ordering::SeqCst); + STREAMS + .lock() + .unwrap() + .insert(handle, Arc::new(Mutex::new(state))); + debug!( + "[netstream] rb_net_open: cache HIT url={} -> handle={}", + url_str, handle + ); + return handle; + } + // File was deleted between lookup and open; fall through to HTTP. + } + // Cache miss: kick off a parallel background download while this + // stream continues reading from the live HTTP connection. + rockbox_cache::start_background_fetch(&url_str); + } + // ────────────────────────────────────────────────────────────────────── + + let state = match StreamState::new_http(url_str.clone()) { Some(s) => { debug!( "[netstream] rb_net_open: url={} content_length={:?} content_type={:?}", @@ -249,8 +333,6 @@ pub unsafe extern "C" fn rb_net_read(h: i32, dst: *mut libc::c_void, n: libc::si if dst.is_null() || n == 0 { return 0; } - // Acquire the global map lock only long enough to clone the per-handle Arc, - // then release it so other handles can proceed concurrently. let handle_arc = { let streams = STREAMS.lock().unwrap(); match streams.get(&h) { @@ -260,19 +342,42 @@ pub unsafe extern "C" fn rb_net_read(h: i32, dst: *mut libc::c_void, n: libc::si }; let mut state = handle_arc.lock().unwrap(); let pos_before = state.pos; - let resp = match &mut state.response { - Some(r) => r, - None => { - // No active response. If we know content_length and pos is at or - // beyond it, signal EOF (0) rather than an error (-1) so callers - // that iterate until EOF terminate cleanly. + let buf = std::slice::from_raw_parts_mut(dst as *mut u8, n); + + // ── Cached file: read directly from local disk ───────────────────────── + if let StreamSource::CachedFile { reader } = &mut state.source { + match read_as_file(reader, buf) { + Ok(bytes_read) => { + state.pos += bytes_read as u64; + tracing::trace!( + "[netstream] rb_net_read (cached): h={} n={} pos_before={} -> read={} pos_after={}", + h, n, pos_before, bytes_read, state.pos + ); + return bytes_read as i64; + } + Err(e) => { + warn!( + "[netstream] rb_net_read (cached): h={} n={} pos={} -> ERROR {:?}", + h, n, pos_before, e + ); + return -1; + } + } + } + + // ── HTTP stream ──────────────────────────────────────────────────────── + let resp = match &mut state.source { + StreamSource::Http { + response: Some(r), .. + } => r, + StreamSource::Http { response: None, .. } => { if state.content_length.map_or(false, |cl| state.pos >= cl) { - return 0; + return 0; // EOF } return -1; } + StreamSource::CachedFile { .. } => unreachable!(), }; - let buf = std::slice::from_raw_parts_mut(dst as *mut u8, n); match read_as_file(resp, buf) { Ok(bytes_read) => { state.pos += bytes_read as u64; @@ -305,8 +410,6 @@ pub extern "C" fn rb_net_lseek(h: i32, off: i64, whence: libc::c_int) -> i64 { const SEEK_CUR: libc::c_int = 1; const SEEK_END: libc::c_int = 2; - // Acquire the global map lock only long enough to clone the per-handle Arc, - // then release it so other handles can proceed concurrently. let handle_arc = { let streams = STREAMS.lock().unwrap(); match streams.get(&h) { @@ -352,13 +455,12 @@ pub extern "C" fn rb_net_lseek(h: i32, off: i64, whence: libc::c_int) -> i64 { }; // Guard 1: never seek gigabytes forward regardless of content_length. - // A seek >256 MB past current position is always a codec arithmetic bug - // (e.g. WAV trying to skip a 0xFFFFFFFF-byte data chunk, or MP4 with - // uint32_t underflow). Issue a Range request for such an offset and the - // server either returns 416 or streams from the beginning — either way - // skip_bytes would block for hours reading a live stream. - const MAX_SKIP: u64 = 256 * 1024 * 1024; // 256 MB - if new_pos > state.pos && new_pos - state.pos > MAX_SKIP { + // (Only applies to HTTP streams; cached files handle any seek trivially.) + const MAX_SKIP: u64 = 256 * 1024 * 1024; + if matches!(state.source, StreamSource::Http { .. }) + && new_pos > state.pos + && new_pos - state.pos > MAX_SKIP + { warn!( "[netstream] rb_net_lseek: h={} off={} whence={} huge skip ({} bytes) clamped", h, @@ -369,8 +471,7 @@ pub extern "C" fn rb_net_lseek(h: i32, off: i64, whence: libc::c_int) -> i64 { return -1; } - // Guard 2: never issue a Range request past EOF — the server would return - // 416 and leave response=None, permanently breaking the stream. + // Guard 2: never seek past EOF — clamp to content_length. if let Some(cl) = state.content_length { if new_pos >= cl { warn!( @@ -378,12 +479,17 @@ pub extern "C" fn rb_net_lseek(h: i32, off: i64, whence: libc::c_int) -> i64 { h, off, whence, new_pos, cl ); state.pos = cl; - state.response = None; // EOF: rb_net_read will return 0 + match &mut state.source { + StreamSource::Http { response } => *response = None, + StreamSource::CachedFile { reader } => { + let _ = reader.seek(std::io::SeekFrom::Start(cl)); + } + } return cl as i64; } } - // Fast-path: already there (no need to restart the request). + // Fast-path: already at the target position. if new_pos == state.pos { debug!( "[netstream] rb_net_lseek: h={} off={} whence={} -> already at pos={} (no-op)", @@ -465,9 +571,7 @@ pub extern "C" fn rb_net_close(h: i32) { STREAMS.lock().unwrap().remove(&h); } -// ------------------------------------------------------------------ -// Tests -// ------------------------------------------------------------------ +// ─── Tests ─────────────────────────────────────────────────────────────────── #[cfg(test)] mod tests { use super::*; @@ -475,7 +579,6 @@ mod tests { use std::ffi::CString; use std::io::Cursor; - /// Helper: build a NUL-terminated C URL string for a path on the mock server. fn c_url(server: &mockito::Server, path: &str) -> CString { CString::new(format!("{}{}", server.url(), path)).unwrap() } @@ -501,11 +604,8 @@ mod tests { } } - // ------------------------------------------------------------------ - // Open / close - // ------------------------------------------------------------------ + // ── Open / close ────────────────────────────────────────────────────── - /// Opening a valid URL returns a non-negative handle. #[test] fn test_open_and_close() { let mut server = mockito::Server::new(); @@ -521,34 +621,22 @@ mod tests { assert!(handle >= 0, "open should return a valid handle"); rb_net_close(handle); - // After close, rb_net_len should return -1 (unknown handle). - assert_eq!( - rb_net_len(handle), - -1, - "closed handle should return -1 from rb_net_len" - ); + assert_eq!(rb_net_len(handle), -1, "closed handle should return -1"); } - /// Passing a null pointer returns INVALID_HANDLE. #[test] fn test_open_null_url() { let handle = unsafe { rb_net_open(std::ptr::null()) }; assert_eq!(handle, INVALID_HANDLE); } - /// Connecting to a port where nothing is listening returns INVALID_HANDLE. #[test] fn test_open_unreachable_host() { - // Port 19998 is extremely unlikely to be in use. let url = CString::new("http://127.0.0.1:19998/file.mp3").unwrap(); let handle = unsafe { rb_net_open(url.as_ptr()) }; - assert_eq!( - handle, INVALID_HANDLE, - "unreachable server should return INVALID_HANDLE" - ); + assert_eq!(handle, INVALID_HANDLE); } - /// A 404 response causes rb_net_open to return INVALID_HANDLE. #[test] fn test_open_404() { let mut server = mockito::Server::new(); @@ -556,17 +644,11 @@ mod tests { let url = c_url(&server, "/missing.mp3"); let handle = unsafe { rb_net_open(url.as_ptr()) }; - assert_eq!( - handle, INVALID_HANDLE, - "404 response should return INVALID_HANDLE" - ); + assert_eq!(handle, INVALID_HANDLE); } - // ------------------------------------------------------------------ - // Content-Length / rb_net_len - // ------------------------------------------------------------------ + // ── Content-Length / rb_net_len ─────────────────────────────────────── - /// rb_net_len returns the Content-Length when the server provides it. #[test] fn test_known_content_length() { let mut server = mockito::Server::new(); @@ -580,9 +662,7 @@ mod tests { let url = c_url(&server, "/known.mp3"); let handle = unsafe { rb_net_open(url.as_ptr()) }; assert!(handle >= 0); - assert_eq!(rb_net_len(handle), 1234); - rb_net_close(handle); } @@ -604,18 +684,13 @@ mod tests { let n = unsafe { rb_net_content_type(handle, buf.as_mut_ptr(), buf.len()) }; assert_eq!(n, "audio/m4a".len() as i64); - let content_type = unsafe { CStr::from_ptr(buf.as_ptr()) }.to_str().unwrap(); - assert_eq!(content_type, "audio/m4a"); - + let ct = unsafe { CStr::from_ptr(buf.as_ptr()) }.to_str().unwrap(); + assert_eq!(ct, "audio/m4a"); rb_net_close(handle); } - /// rb_net_len returns the value from the Content-Length response header. #[test] fn test_unknown_content_length() { - // Note: mockito automatically sets a content-length header from the body - // length when serving responses, so we verify here that rb_net_len - // correctly reads whatever content-length the server sends. let body: &[u8] = b"some data"; let mut server = mockito::Server::new(); let _mock = server @@ -627,23 +702,12 @@ mod tests { let url = c_url(&server, "/stream.mp3"); let handle = unsafe { rb_net_open(url.as_ptr()) }; assert!(handle >= 0); - - // mockito sets content-length = body.len() when no explicit header is given. - let len = rb_net_len(handle); - assert_eq!( - len, - body.len() as i64, - "rb_net_len should reflect the server's content-length" - ); - + assert_eq!(rb_net_len(handle), body.len() as i64); rb_net_close(handle); } - // ------------------------------------------------------------------ - // Reading - // ------------------------------------------------------------------ + // ── Reading ─────────────────────────────────────────────────────────── - /// rb_net_read returns the expected bytes. #[test] fn test_read_bytes() { let body: &[u8] = b"Hello, Rockbox!"; @@ -662,7 +726,6 @@ mod tests { let n = unsafe { rb_net_read(handle, buf.as_mut_ptr() as *mut libc::c_void, buf.len()) }; assert_eq!(n, body.len() as i64); assert_eq!(&buf[..n as usize], body); - rb_net_close(handle); } @@ -670,14 +733,11 @@ mod tests { fn test_read_as_file_retries_partial_reads() { let mut reader = PartialReader::new(b"Hello, Rockbox!", 3); let mut buf = vec![0u8; 15]; - let n = read_as_file(&mut reader, &mut buf).unwrap(); - assert_eq!(n, 15); assert_eq!(&buf, b"Hello, Rockbox!"); } - /// rb_net_read returns 0 at EOF (after all bytes have been consumed). #[test] fn test_read_eof() { let body: &[u8] = b"tiny"; @@ -693,37 +753,29 @@ mod tests { assert!(handle >= 0); let mut buf = vec![0u8; 1024]; - // First read drains the body. let n1 = unsafe { rb_net_read(handle, buf.as_mut_ptr() as *mut libc::c_void, buf.len()) }; assert_eq!(n1, body.len() as i64); - // Second read should signal EOF. let n2 = unsafe { rb_net_read(handle, buf.as_mut_ptr() as *mut libc::c_void, buf.len()) }; assert_eq!(n2, 0, "second read should return 0 at EOF"); - rb_net_close(handle); } - /// rb_net_read on an unknown handle returns -1. #[test] fn test_read_invalid_handle() { let mut buf = vec![0u8; 16]; let result = unsafe { rb_net_read(i32::MAX, buf.as_mut_ptr() as *mut libc::c_void, buf.len()) }; - assert_eq!(result, -1, "read on unknown handle should return -1"); + assert_eq!(result, -1); } - // ------------------------------------------------------------------ - // Seeking - // ------------------------------------------------------------------ + // ── Seeking ─────────────────────────────────────────────────────────── - /// SEEK_SET re-issues a Range request and returns the new position. #[test] fn test_seek_set_range_request() { - let full_body: &[u8] = b"0123456789ABCDEF"; // 16 bytes + let full_body: &[u8] = b"0123456789ABCDEF"; let mut server = mockito::Server::new(); - // Initial GET (no Range header). let _initial = server .mock("GET", "/seekable.mp3") .match_header("range", Matcher::Missing) @@ -732,7 +784,6 @@ mod tests { .with_body(full_body) .create(); - // Range request from byte 8 — bounded range since content-length is known. let _range = server .mock("GET", "/seekable.mp3") .match_header("range", "bytes=8-15") @@ -747,21 +798,18 @@ mod tests { assert!(handle >= 0); let new_pos = rb_net_lseek(handle, 8, libc::SEEK_SET); - assert_eq!(new_pos, 8, "SEEK_SET(8) should return position 8"); + assert_eq!(new_pos, 8); - // Read the remaining 8 bytes and verify they match the tail of full_body. let mut buf = vec![0u8; 16]; let n = unsafe { rb_net_read(handle, buf.as_mut_ptr() as *mut libc::c_void, buf.len()) }; assert_eq!(n, 8); assert_eq!(&buf[..8], &full_body[8..]); - rb_net_close(handle); } - /// SEEK_CUR(0) is a no-op that queries the current position without a new request. #[test] fn test_seek_cur_no_op() { - let body: &[u8] = b"ABCDEFGHIJ"; // 10 bytes + let body: &[u8] = b"ABCDEFGHIJ"; let mut server = mockito::Server::new(); let _mock = server .mock("GET", "/cur.mp3") @@ -774,22 +822,18 @@ mod tests { let handle = unsafe { rb_net_open(url.as_ptr()) }; assert!(handle >= 0); - // Read 5 bytes → position advances to 5. let mut buf = vec![0u8; 5]; let n = unsafe { rb_net_read(handle, buf.as_mut_ptr() as *mut libc::c_void, 5) }; assert_eq!(n, 5); - // SEEK_CUR(0) should return current position without a new HTTP request. let pos = rb_net_lseek(handle, 0, libc::SEEK_CUR); - assert_eq!(pos, 5, "SEEK_CUR(0) should return current position"); - + assert_eq!(pos, 5); rb_net_close(handle); } - /// SEEK_END(-2) on a 10-byte file should yield position 8. #[test] fn test_seek_end() { - let full_body: &[u8] = b"XXXXXXXXXX"; // 10 bytes + let full_body: &[u8] = b"XXXXXXXXXX"; let mut server = mockito::Server::new(); let _initial = server @@ -813,32 +857,17 @@ mod tests { assert!(handle >= 0); let pos = rb_net_lseek(handle, -2, libc::SEEK_END); - assert_eq!( - pos, 8, - "SEEK_END(-2) on 10-byte file should give position 8" - ); - + assert_eq!(pos, 8); rb_net_close(handle); } - /// SEEK_END on a stream with unknown Content-Length returns -1. - /// This tests the graceful failure path for SEEK_END. #[test] fn test_seek_end_unknown_length() { - // We use a 416 mock to trigger the failure in seek_to; this also tests - // the seek failure path for SEEK_END when content-length becomes - // unavailable (e.g. after a failed Range response cleared the state). let full_body: &[u8] = b"data"; let mut server = mockito::Server::new(); - - // Initial GET: explicitly provide no content-length so SEEK_END has nothing. - // We do this by setting content-length to 0 on a 200 response without body, - // then testing SEEK_END with a negative offset. let _mock = server .mock("GET", "/nosize.mp3") .with_status(200) - // mockito sets content-length from body; use a large body to get a real length, - // then test that SEEK_END(-offset > length) correctly fails. .with_header("content-length", "4") .with_body(full_body) .create(); @@ -847,22 +876,15 @@ mod tests { let handle = unsafe { rb_net_open(url.as_ptr()) }; assert!(handle >= 0); - // Seeking past the beginning is invalid: SEEK_END with |offset| > length. let pos = rb_net_lseek(handle, -100, libc::SEEK_END); - assert_eq!( - pos, -1, - "SEEK_END with offset beyond file start should return -1" - ); - + assert_eq!(pos, -1); rb_net_close(handle); } - /// When the server returns 416 for a Range request, seek fails gracefully. #[test] fn test_seek_range_not_supported() { let mut server = mockito::Server::new(); - // Initial GET succeeds. let _initial = server .mock("GET", "/noseek.mp3") .match_header("range", Matcher::Missing) @@ -871,7 +893,6 @@ mod tests { .with_body(vec![0u8; 100]) .create(); - // Range request returns "416 Range Not Satisfiable". let _no_range = server .mock("GET", "/noseek.mp3") .match_header("range", Matcher::Any) @@ -883,16 +904,10 @@ mod tests { assert!(handle >= 0); let result = rb_net_lseek(handle, 50, libc::SEEK_SET); - assert_eq!( - result, -1, - "seek should fail gracefully when Range is not supported" - ); - + assert_eq!(result, -1); rb_net_close(handle); } - /// If the server ignores Range and returns 200 with the full body, seek - /// still succeeds by discarding bytes until the requested position. #[test] fn test_seek_falls_back_when_range_is_ignored() { let full_body: &[u8] = b"0123456789ABCDEF"; @@ -919,18 +934,15 @@ mod tests { assert!(handle >= 0); let new_pos = rb_net_lseek(handle, 8, libc::SEEK_SET); - assert_eq!(new_pos, 8, "seek should land at byte 8 even without 206"); + assert_eq!(new_pos, 8); let mut buf = vec![0u8; 16]; let n = unsafe { rb_net_read(handle, buf.as_mut_ptr() as *mut libc::c_void, buf.len()) }; assert_eq!(n, 8); assert_eq!(&buf[..8], &full_body[8..]); - rb_net_close(handle); } - /// A malformed 206 response that does not start at the requested offset - /// must fail instead of silently desynchronizing the stream position. #[test] fn test_seek_rejects_wrong_content_range_start() { let full_body: &[u8] = b"0123456789ABCDEF"; @@ -957,20 +969,15 @@ mod tests { assert!(handle >= 0); let result = rb_net_lseek(handle, 8, libc::SEEK_SET); - assert_eq!(result, -1, "mismatched Content-Range should fail"); - + assert_eq!(result, -1); rb_net_close(handle); } - /// Content-Range header from a 206 response populates content_length - /// when it was not provided in the initial response. - /// We verify here that after seeking, the total length is available. #[test] fn test_content_length_from_content_range() { - let full_body: &[u8] = b"0123456789"; // 10 bytes + let full_body: &[u8] = b"0123456789"; let mut server = mockito::Server::new(); - // Initial GET: content-length matches body (10). let _initial = server .mock("GET", "/range-len.mp3") .match_header("range", Matcher::Missing) @@ -979,7 +986,6 @@ mod tests { .with_body(full_body) .create(); - // Range request: 206 includes Content-Range which also reveals total size. let _range = server .mock("GET", "/range-len.mp3") .match_header("range", "bytes=5-9") @@ -991,34 +997,17 @@ mod tests { let url = c_url(&server, "/range-len.mp3"); let handle = unsafe { rb_net_open(url.as_ptr()) }; assert!(handle >= 0); + assert_eq!(rb_net_len(handle), 10); - // Total length is known from the initial response. - assert_eq!( - rb_net_len(handle), - 10, - "length should be known from initial response" - ); - - // Seeking causes a 206 response whose Content-Range also confirms the total length. let pos = rb_net_lseek(handle, 5, libc::SEEK_SET); - assert_eq!(pos, 5, "seek should succeed"); - - // Length is still correctly reported after seek. - assert_eq!( - rb_net_len(handle), - 10, - "length should remain correct after seek" - ); - + assert_eq!(pos, 5); + assert_eq!(rb_net_len(handle), 10); rb_net_close(handle); } - /// Seeking past content_length (e.g. from a uint32_t underflow in the MP4 - /// parser) is clamped to content_length. The stream is not permanently - /// broken: reads return 0 (EOF) rather than -1 (error). #[test] fn test_seek_past_eof_is_clamped() { - let full_body: &[u8] = b"0123456789"; // 10 bytes, content-length=10 + let full_body: &[u8] = b"0123456789"; let mut server = mockito::Server::new(); let _initial = server @@ -1033,16 +1022,12 @@ mod tests { let handle = unsafe { rb_net_open(url.as_ptr()) }; assert!(handle >= 0); - // Seek 4 GB past current position (simulates uint32_t underflow in C). let result = rb_net_lseek(handle, 4_294_967_295, libc::SEEK_CUR); - // Should clamp to content_length (10), not return -1. assert_eq!(result, 10, "seek past EOF should clamp to content_length"); - // Subsequent reads must return 0 (EOF), not -1 (broken stream). let mut buf = vec![0u8; 16]; let n = unsafe { rb_net_read(handle, buf.as_mut_ptr() as *mut libc::c_void, buf.len()) }; assert_eq!(n, 0, "read after clamped seek should return 0 (EOF)"); - rb_net_close(handle); } } diff --git a/crates/settings/Cargo.toml b/crates/settings/Cargo.toml index 7a59f4cd18e..d22bedf77ca 100644 --- a/crates/settings/Cargo.toml +++ b/crates/settings/Cargo.toml @@ -7,5 +7,6 @@ version = "0.1.0" anyhow = "1.0.91" rockbox-sys = {path = "../sys"} rockbox-upnp = {path = "../upnp"} +rockbox-cache = {path = "../cache"} toml = "0.8.19" tracing = { workspace = true } diff --git a/crates/settings/src/lib.rs b/crates/settings/src/lib.rs index c5b88164ebb..e473fdf78a0 100644 --- a/crates/settings/src/lib.rs +++ b/crates/settings/src/lib.rs @@ -1,4 +1,5 @@ use anyhow::Error; +use rockbox_cache::CacheConfig; use rockbox_sys::{self as rb, sound::pcm, types::user_settings::NewGlobalSettings}; pub fn load_settings(new_settings: Option) -> Result<(), Error> { @@ -158,6 +159,41 @@ pub fn load_settings(new_settings: Option) -> Result<(), Erro rb::sound::dsp::eq_enable(enabled); rb::sound::pcmbuf_set_low_latency(false); + // ── HTTP file cache ──────────────────────────────────────────────────── + { + let default_dir = format!("{}/.config/rockbox.org/cache", home); + let cache_enabled = settings.cache_enabled.unwrap_or(true); + let cache_dir = settings + .cache_dir + .as_deref() + .unwrap_or(&default_dir) + .replace("$HOME", &home); + let max_size_bytes = settings.cache_max_size_mb.unwrap_or(512) * 1024 * 1024; + let min_free_space_bytes = settings.cache_min_free_space_mb.unwrap_or(100) * 1024 * 1024; + let parallel_parts = settings.cache_parallel_parts.unwrap_or(4) as usize; + let no_cache_patterns = settings.cache_no_cache_patterns.clone().unwrap_or_default(); + + rockbox_cache::configure(CacheConfig { + enabled: cache_enabled, + dir: std::path::PathBuf::from(&cache_dir), + max_size_bytes, + min_free_space_bytes, + parallel_parts, + no_cache_patterns, + }); + tracing::info!( + "http cache: enabled={} dir={} max={}MB parts={} skip_patterns={}", + cache_enabled, + cache_dir, + settings.cache_max_size_mb.unwrap_or(512), + parallel_parts, + settings + .cache_no_cache_patterns + .as_ref() + .map_or(0, |v| v.len()), + ); + } + Ok(()) } diff --git a/crates/sys/src/types/user_settings.rs b/crates/sys/src/types/user_settings.rs index d6b9bd43597..209e7ef3933 100644 --- a/crates/sys/src/types/user_settings.rs +++ b/crates/sys/src/types/user_settings.rs @@ -743,6 +743,20 @@ pub struct NewGlobalSettings { pub subsonic_password: Option, /// Port for the Subsonic-compatible API server (default: 4533). pub subsonic_port: Option, + /// Enable HTTP file caching (default: true). + pub cache_enabled: Option, + /// Cache directory path (default: ~/.config/rockbox.org/cache). + pub cache_dir: Option, + /// Maximum total cache size in megabytes (default: 512). + pub cache_max_size_mb: Option, + /// Minimum free disk space in MB that must remain before caching (default: 100). + pub cache_min_free_space_mb: Option, + /// Number of parallel HTTP range-request parts for large file caching (default: 4). + /// Set to 1 to disable parallel downloading. + pub cache_parallel_parts: Option, + /// URL substrings that bypass the cache entirely (e.g. "icecast", ".m3u8"). + /// Useful to prevent live radio streams from being cached. + pub cache_no_cache_patterns: Option>, } impl From for NewGlobalSettings { @@ -810,6 +824,12 @@ impl From for NewGlobalSettings { subsonic_username: None, subsonic_password: None, subsonic_port: None, + cache_enabled: None, + cache_dir: None, + cache_max_size_mb: None, + cache_min_free_space_mb: None, + cache_parallel_parts: None, + cache_no_cache_patterns: None, } } } diff --git a/mintlify/configuration.mdx b/mintlify/configuration.mdx index 911c8535147..4087d716536 100644 --- a/mintlify/configuration.mdx +++ b/mintlify/configuration.mdx @@ -118,6 +118,59 @@ release_time = 300 attack_time = 5 ``` +## HTTP file cache + +Rockbox caches remote audio files on disk so repeat plays are instant and +require zero network traffic. Files are downloaded in the background in +**parallel range-request parts** so the cache is populated without any +interruption to the live stream. + +```toml +# All fields are optional — shown with their defaults. +cache_enabled = true +cache_dir = "~/.config/rockbox.org/cache" +cache_max_size_mb = 512 # total disk budget in MB +cache_min_free_space_mb = 100 # headroom to always preserve on disk +cache_parallel_parts = 4 # concurrent HTTP Range requests per file + +# Substrings that opt a URL out of caching entirely. +# Useful for live radio streams, HLS manifests, etc. +cache_no_cache_patterns = ["icecast", ".m3u8", "live", "stream"] +``` + +### How it works + +| Step | What happens | +| ---- | ------------ | +| 1 | `stream_open` checks the on-disk cache (SHA-256 of URL → `.cache` file). | +| 2 | **Cache hit**: the local file is opened instantly; seeks are O(1) file seeks — no network at all. | +| 3 | **Cache miss**: the live HTTP stream is opened normally (zero latency), AND a background thread downloads the full file in `cache_parallel_parts` simultaneous range requests. | +| 4 | Once the background download is complete (and verified), the file is atomically renamed to its final cache path. The next open is a hit. | +| 5 | When the cache exceeds `cache_max_size_mb`, the least-recently-used files are evicted to make room. | + +### Stream / radio protection + +The cache automatically skips URLs that have no `Content-Length` — infinite +streams (ICY radio, HLS) never produce one, so they are never queued for +caching. For URLs that *do* return a `Content-Length` but should still be +skipped (e.g. a CDN-delivered live stream), add a matching substring to +`cache_no_cache_patterns`. + +### Key derivation + +Each cached URL is stored as `SHA-256(url).cache` in `cache_dir`. The hash +is deterministic and URL-specific, so query-string variations produce distinct +cache entries. + +### Parallel download + +Files ≥ 2 MB are split into `cache_parallel_parts` equal byte ranges and +fetched concurrently. Each thread writes directly to its allocated region of a +pre-allocated file via `pwrite` (Unix) / `seek_write` (Windows), so threads +never contend. Set `cache_parallel_parts = 1` to fall back to single-connection +downloads. If any range request fails (e.g. the server returns 200 instead of +206), the whole file is re-fetched sequentially as a fallback. + ## Where settings come from There are three layers, in order of precedence: From f519a8b7e41d0aaba61eca91a0783206e4c308c2 Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Wed, 27 May 2026 19:06:28 +0300 Subject: [PATCH 02/12] fix: add missing cache_* fields to NewGlobalSettings initializer in rpc crate Co-Authored-By: Claude Sonnet 4.6 --- crates/rpc/src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/rpc/src/lib.rs b/crates/rpc/src/lib.rs index 156d6a3ab55..af7fa3f4d6f 100644 --- a/crates/rpc/src/lib.rs +++ b/crates/rpc/src/lib.rs @@ -983,6 +983,12 @@ pub mod api { subsonic_username: None, subsonic_password: None, subsonic_port: None, + cache_enabled: None, + cache_dir: None, + cache_max_size_mb: None, + cache_min_free_space_mb: None, + cache_parallel_parts: None, + cache_no_cache_patterns: None, } } } From 7b3b3593e51cef9d88dc38c8b095119fda0d024c Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Wed, 27 May 2026 19:11:22 +0300 Subject: [PATCH 03/12] fix: use ROCKBOX_CONFIG_DIR for cache path so Android gets the correct dir On Android, HOME is set to the app sandbox root (not a Unix home dir), so $HOME/.config/rockbox.org is wrong for any crate whose Lazy static fires before configure_environment runs. - configure_environment now sets ROCKBOX_CONFIG_DIR to $config_dir/.config/rockbox.org before any threads start - CacheConfig::with_defaults() prefers ROCKBOX_CONFIG_DIR, falling back to $HOME/.config/rockbox.org on desktop/WASM where the var is unset - load_settings default cache dir uses the same logic --- crates/cache/src/lib.rs | 7 +++++-- crates/expo/src/daemon.rs | 7 +++++++ crates/settings/src/lib.rs | 4 +++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/crates/cache/src/lib.rs b/crates/cache/src/lib.rs index 0e61dfc20f3..cb7d15f43d8 100644 --- a/crates/cache/src/lib.rs +++ b/crates/cache/src/lib.rs @@ -31,10 +31,13 @@ pub struct CacheConfig { impl CacheConfig { pub fn with_defaults() -> Self { - let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); + let config_base = std::env::var("ROCKBOX_CONFIG_DIR").unwrap_or_else(|_| { + let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); + format!("{}/.config/rockbox.org", home) + }); Self { enabled: true, - dir: PathBuf::from(format!("{}/.config/rockbox.org/cache", home)), + dir: PathBuf::from(format!("{}/cache", config_base)), max_size_bytes: 512 * 1024 * 1024, min_free_space_bytes: 100 * 1024 * 1024, parallel_parts: 4, diff --git a/crates/expo/src/daemon.rs b/crates/expo/src/daemon.rs index 6e02707f3ad..60b0a8c791b 100644 --- a/crates/expo/src/daemon.rs +++ b/crates/expo/src/daemon.rs @@ -154,6 +154,13 @@ fn configure_environment(config_dir: &str, music_dir: &str, device_name: &str) { // Safety: env::set_var is only safe before any other thread that reads // env exists. We're called from JNI before the engine pthread spawns. std::env::set_var("HOME", config_dir); + // Explicit config base so crates that construct paths like + // "$HOME/.config/rockbox.org/..." don't need to know that on Android + // HOME already IS the app sandbox root (not a Unix home dir). + std::env::set_var( + "ROCKBOX_CONFIG_DIR", + format!("{}/.config/rockbox.org", config_dir), + ); std::env::set_var("ROCKBOX_DEVICE_NAME", device_name); // Canonical music-dir env var read by crates/{settings,server,graphql,sys}. // ROCKBOX_MUSIC_DIR was a misnomer — nothing reads it. The browse diff --git a/crates/settings/src/lib.rs b/crates/settings/src/lib.rs index e473fdf78a0..08c2636875b 100644 --- a/crates/settings/src/lib.rs +++ b/crates/settings/src/lib.rs @@ -161,7 +161,9 @@ pub fn load_settings(new_settings: Option) -> Result<(), Erro // ── HTTP file cache ──────────────────────────────────────────────────── { - let default_dir = format!("{}/.config/rockbox.org/cache", home); + let config_base = std::env::var("ROCKBOX_CONFIG_DIR") + .unwrap_or_else(|_| format!("{}/.config/rockbox.org", home)); + let default_dir = format!("{}/cache", config_base); let cache_enabled = settings.cache_enabled.unwrap_or(true); let cache_dir = settings .cache_dir From 2ab86f6acf8e2edaac1f81c6262a7467a0642d3a Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Wed, 27 May 2026 19:21:32 +0300 Subject: [PATCH 04/12] chore: run cargo fmt on navidrome server mod --- crates/navidrome/src/server/mod.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/crates/navidrome/src/server/mod.rs b/crates/navidrome/src/server/mod.rs index 5968551b5a2..4069e07562f 100644 --- a/crates/navidrome/src/server/mod.rs +++ b/crates/navidrome/src/server/mod.rs @@ -7,10 +7,7 @@ use rockbox_library::create_connection_pool; use rockbox_playlists::PlaylistStore; use rockbox_settings::read_settings; use sqlx::{Pool, Sqlite}; -use std::sync::{ - atomic::{AtomicBool, Ordering}, - Arc, Mutex, OnceLock, -}; +use std::sync::{atomic::AtomicBool, Arc, Mutex, OnceLock}; // ── Now-playing shared state ────────────────────────────────────────────────── From 3dd1c273918b2be5aecafa503579dfca4ba2cbb9 Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Wed, 27 May 2026 19:37:17 +0300 Subject: [PATCH 05/12] fix: create cache dir eagerly in configure() instead of on first download --- crates/cache/src/lib.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/cache/src/lib.rs b/crates/cache/src/lib.rs index cb7d15f43d8..515f04b6693 100644 --- a/crates/cache/src/lib.rs +++ b/crates/cache/src/lib.rs @@ -72,6 +72,11 @@ static CLIENT: Lazy = Lazy::new(|| { /// Replace the active cache configuration. Called once from `load_settings`. pub fn configure(config: CacheConfig) { + if let Err(e) = fs::create_dir_all(&config.dir) { + warn!("http cache: could not create cache dir {:?}: {}", config.dir, e); + } else { + debug!("http cache: dir {:?} ready", config.dir); + } let mut mgr = CACHE.lock().unwrap(); mgr.config = config; mgr.in_progress.clear(); From 10842b2793572331cf106aaf847595ffeee46818 Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Wed, 27 May 2026 20:14:21 +0300 Subject: [PATCH 06/12] fix: create cache dir eagerly in with_defaults + verbose fetch logging - with_defaults() now calls create_dir_all so the dir exists from the first CACHE static access, regardless of whether configure() is called - start_background_fetch: all skip paths now log at info level so the reason is always visible in the default log filter - perform_download: no-Content-Length skip promoted from debug to info (catches transcoded Navidrome streams that never have a content-length) Co-Authored-By: Claude Sonnet 4.6 --- crates/cache/src/lib.rs | 42 +++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/crates/cache/src/lib.rs b/crates/cache/src/lib.rs index 515f04b6693..0f9d4e1bd5b 100644 --- a/crates/cache/src/lib.rs +++ b/crates/cache/src/lib.rs @@ -35,9 +35,13 @@ impl CacheConfig { let home = std::env::var("HOME").unwrap_or_else(|_| ".".to_string()); format!("{}/.config/rockbox.org", home) }); + let dir = PathBuf::from(format!("{}/cache", config_base)); + if let Err(e) = fs::create_dir_all(&dir) { + eprintln!("rockbox-cache: could not create default cache dir {:?}: {}", dir, e); + } Self { enabled: true, - dir: PathBuf::from(format!("{}/cache", config_base)), + dir, max_size_bytes: 512 * 1024 * 1024, min_free_space_bytes: 100 * 1024 * 1024, parallel_parts: 4, @@ -75,7 +79,7 @@ pub fn configure(config: CacheConfig) { if let Err(e) = fs::create_dir_all(&config.dir) { warn!("http cache: could not create cache dir {:?}: {}", config.dir, e); } else { - debug!("http cache: dir {:?} ready", config.dir); + info!("http cache: dir {:?} ready", config.dir); } let mut mgr = CACHE.lock().unwrap(); mgr.config = config; @@ -113,9 +117,11 @@ pub fn lookup(url: &str) -> Option { /// /// Returns immediately; the caller's live HTTP stream continues uninterrupted. pub fn start_background_fetch(url: &str) { + info!("cache: start_background_fetch called for {}", url); { let mut mgr = CACHE.lock().unwrap(); if !mgr.config.enabled { + info!("cache: caching disabled — skipping {}", url); return; } // Skip URLs that match a no-cache pattern (e.g. live streams, HLS). @@ -125,27 +131,30 @@ pub fn start_background_fetch(url: &str) { .iter() .any(|p| url.contains(p.as_str())) { - debug!("cache: no-cache pattern match, skipping: {}", url); + info!("cache: no-cache pattern match, skipping: {}", url); return; } if mgr.in_progress.contains(url) { + info!("cache: already in-flight, skipping: {}", url); return; } let key = url_to_key(url); if mgr.config.dir.join(format!("{key}.cache")).exists() { + info!("cache: already cached, skipping: {}", url); return; } - let avail = available_disk_space(&mgr.config.dir).unwrap_or(0); + let avail = available_disk_space(&mgr.config.dir).unwrap_or(u64::MAX); if avail < mgr.config.min_free_space_bytes { - debug!( - "cache: low disk space — skipping background fetch for {}", - url + info!( + "cache: low disk space ({} bytes free) — skipping background fetch for {}", + avail, url ); return; } mgr.in_progress.insert(url.to_string()); } + info!("cache: spawning download thread for {}", url); let url_owned = url.to_string(); std::thread::Builder::new() .name("cache-fetch".into()) @@ -171,7 +180,20 @@ pub fn url_to_key(url: &str) -> String { #[cfg(unix)] fn available_disk_space(path: &Path) -> Option { use std::ffi::CString; - let p = CString::new(path.to_string_lossy().as_bytes()).ok()?; + // Walk up to the nearest existing ancestor so this works even when the + // cache directory hasn't been created yet. + let mut candidate = path.to_path_buf(); + loop { + if candidate.exists() { + break; + } + match candidate.parent().map(|p| p.to_path_buf()) { + Some(p) => candidate = p, + None => return None, + } + } + let candidate = candidate.as_path(); + let p = CString::new(candidate.to_string_lossy().as_bytes()).ok()?; let mut stat: libc::statvfs = unsafe { std::mem::zeroed() }; if unsafe { libc::statvfs(p.as_ptr(), &mut stat) } != 0 { return None; @@ -452,8 +474,8 @@ fn perform_download(url: &str, config: &CacheConfig) { Some(cl) => cl, None => { // No Content-Length → infinite/chunked stream; never cache. - debug!( - "cache: no Content-Length for {} — skipping (live stream?)", + info!( + "cache: no Content-Length for {} — skipping (live stream or transcoded?)", url ); return; From 4bab65a0d3d550a8f868c9a61bcb2bc100782b8b Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Wed, 27 May 2026 20:22:08 +0300 Subject: [PATCH 07/12] fix: guarantee cache dir creation from parse_args entry point MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Create ~/.config/rockbox.org/cache in parse_args() immediately after tracing is initialized — this is the very first Rust code that runs, so the dir exists before load_settings, configure(), or any HTTP open. Co-Authored-By: Claude Sonnet 4.6 --- crates/cli/src/lib.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 5eb967d70df..416c4d7fea1 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -121,6 +121,16 @@ pub extern "C" fn parse_args(argc: usize, argv: *const *const u8) -> i32 { .finish(); let _ = tracing::subscriber::set_global_default(subscriber); + // Create the cache dir eagerly — before settings are loaded — so it always + // exists regardless of whether load_settings succeeds. + if let Ok(home) = env::var("HOME") { + let cache_dir = format!("{}/.config/rockbox.org/cache", home); + match fs::create_dir_all(&cache_dir) { + Ok(_) => info!("cache dir ready: {}", cache_dir), + Err(e) => warn!("could not create cache dir {}: {}", cache_dir, e), + } + } + let string_array = unsafe { std::slice::from_raw_parts(argv, argc) }; let args: Vec<&str> = string_array .iter() From a9844e0cbb9f2a8198a5cf1897b43d28b6ff51e0 Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Wed, 27 May 2026 21:26:26 +0300 Subject: [PATCH 08/12] fix: init tracing subscriber in start_server() for server-side log visibility librockbox_server.a and librockbox_cli.a each compile their own copy of tracing_core, giving them separate GLOBAL_INIT statics. parse_args() only sets the CLI copy's dispatcher, so all logs from server-compiled code (rockbox_cache::start_background_fetch, configure, etc.) were silently dropped. Initialize the server copy's dispatcher in start_server() so cache, netstream, and server logs become visible. Co-Authored-By: Claude Sonnet 4.6 --- crates/server/Cargo.toml | 1 + crates/server/src/lib.rs | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index d6c53467626..563f522ce2c 100644 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -50,6 +50,7 @@ tokio = {version = "1.36.0", features = ["full"]} url = "2.3.1" urlencoding = "2.1.3" tracing = { workspace = true } +tracing-subscriber = { workspace = true } [features] default = [] diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index 55b940fa0d7..d65e3f496ea 100644 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -1,5 +1,6 @@ use anyhow::Error; use tracing::{error, warn}; +use tracing_subscriber::EnvFilter; use lazy_static::lazy_static; use rockbox_graphql::{ @@ -68,6 +69,18 @@ pub extern "C" fn debugfn(args: *const c_char, value: c_int) { #[no_mangle] pub extern "C" fn start_server() { + // librockbox_server.a has its own copy of tracing_core (separate GLOBAL_INIT + // from librockbox_cli.a). Initialize this copy's subscriber so that logs + // from server-compiled code (rockbox_cache, netstream, etc.) are visible. + let subscriber = tracing_subscriber::fmt() + .with_writer(std::io::stderr) + .with_env_filter( + EnvFilter::try_from_default_env() + .unwrap_or_else(|_| EnvFilter::new("info")), + ) + .finish(); + let _ = tracing::subscriber::set_global_default(subscriber); + match rockbox_settings::load_settings(None) { Ok(_) => {} Err(e) => { From 24e5a43478d5e86a4356b89ea117d3f087e5dd3d Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Wed, 27 May 2026 21:51:41 +0300 Subject: [PATCH 09/12] fix: init tracing subscriber in start_server() for server-side log visibility librockbox_server.a and librockbox_cli.a each compile their own copy of tracing_core, giving them separate GLOBAL_INIT statics. parse_args() only sets the CLI copy's dispatcher, so all logs from server-compiled code (rockbox_cache::start_background_fetch, configure, etc.) were silently dropped. Initialize the server copy's dispatcher in start_server() so cache, netstream, and server logs become visible. --- Cargo.lock | 1 + crates/cache/src/lib.rs | 10 ++++++++-- crates/server/src/lib.rs | 3 +-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 95af9474d6c..77bf221f97c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9792,6 +9792,7 @@ dependencies = [ "sqlx", "tokio", "tracing", + "tracing-subscriber", "url", "urlencoding", ] diff --git a/crates/cache/src/lib.rs b/crates/cache/src/lib.rs index 0f9d4e1bd5b..834191f840b 100644 --- a/crates/cache/src/lib.rs +++ b/crates/cache/src/lib.rs @@ -37,7 +37,10 @@ impl CacheConfig { }); let dir = PathBuf::from(format!("{}/cache", config_base)); if let Err(e) = fs::create_dir_all(&dir) { - eprintln!("rockbox-cache: could not create default cache dir {:?}: {}", dir, e); + eprintln!( + "rockbox-cache: could not create default cache dir {:?}: {}", + dir, e + ); } Self { enabled: true, @@ -77,7 +80,10 @@ static CLIENT: Lazy = Lazy::new(|| { /// Replace the active cache configuration. Called once from `load_settings`. pub fn configure(config: CacheConfig) { if let Err(e) = fs::create_dir_all(&config.dir) { - warn!("http cache: could not create cache dir {:?}: {}", config.dir, e); + warn!( + "http cache: could not create cache dir {:?}: {}", + config.dir, e + ); } else { info!("http cache: dir {:?} ready", config.dir); } diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index d65e3f496ea..4de1cdb866c 100644 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -75,8 +75,7 @@ pub extern "C" fn start_server() { let subscriber = tracing_subscriber::fmt() .with_writer(std::io::stderr) .with_env_filter( - EnvFilter::try_from_default_env() - .unwrap_or_else(|_| EnvFilter::new("info")), + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")), ) .finish(); let _ = tracing::subscriber::set_global_default(subscriber); From 15a29b6faff1872509920524766b662bfc9a4be3 Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Wed, 27 May 2026 22:17:35 +0300 Subject: [PATCH 10/12] feat: add Content-Length to HEAD /rest/stream.view in navidrome server --- crates/navidrome/src/server/handlers.rs | 47 +++++++++++++++++++++++++ crates/navidrome/src/server/mod.rs | 8 +++++ 2 files changed, 55 insertions(+) diff --git a/crates/navidrome/src/server/handlers.rs b/crates/navidrome/src/server/handlers.rs index 889c0895bc5..dd7a063d15f 100644 --- a/crates/navidrome/src/server/handlers.rs +++ b/crates/navidrome/src/server/handlers.rs @@ -533,6 +533,53 @@ pub async fn get_song(state: web::Data, query: web::Query, + query: web::Query, +) -> HttpResponse { + let q = query.into_inner(); + let f = q.f.as_deref(); + if let Some(r) = auth_check( + &state, + q.u.as_deref(), + q.p.as_deref(), + q.t.as_deref(), + q.s.as_deref(), + f, + ) { + return r; + } + let id = match q.id.as_deref() { + Some(id) => id, + None => return response::respond_error(f, 10, "Required parameter is missing: id"), + }; + let track = match repo::track::find(state.pool.clone(), id).await { + Ok(Some(t)) => t, + Ok(None) => return response::respond_error(f, 70, "Song not found"), + Err(e) => { + tracing::error!("head_stream: {e}"); + return response::respond_error(f, 0, "database error"); + } + }; + let content_type = mime_for_path(&track.path); + let file_size = match std::fs::metadata(&track.path) { + Ok(m) => m.len(), + Err(e) => { + tracing::error!("head_stream stat {}: {e}", track.path); + return response::respond_error(f, 0, "could not read file"); + } + }; + HttpResponse::Ok() + .content_type(content_type) + .insert_header(("Accept-Ranges", "bytes")) + .insert_header(("Content-Length", file_size.to_string())) + .insert_header(( + "Content-Disposition", + format!("attachment; filename=\"{}\"", safe_filename(&track.path)), + )) + .finish() +} + pub async fn stream( state: web::Data, query: web::Query, diff --git a/crates/navidrome/src/server/mod.rs b/crates/navidrome/src/server/mod.rs index 4069e07562f..92af72bc152 100644 --- a/crates/navidrome/src/server/mod.rs +++ b/crates/navidrome/src/server/mod.rs @@ -261,6 +261,10 @@ pub async fn start() -> anyhow::Result<()> { web::post().to(handlers::get_starred2), ) // Playback + .route( + "/rest/stream{_:(\\.view)?}", + web::head().to(handlers::head_stream), + ) .route( "/rest/stream{_:(\\.view)?}", web::get().to(handlers::stream), @@ -269,6 +273,10 @@ pub async fn start() -> anyhow::Result<()> { "/rest/stream{_:(\\.view)?}", web::post().to(handlers::stream), ) + .route( + "/rest/download{_:(\\.view)?}", + web::head().to(handlers::head_stream), + ) .route( "/rest/download{_:(\\.view)?}", web::get().to(handlers::stream), From 5d22a869a7556d45a53410c4796e480d2ab89824 Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Wed, 27 May 2026 22:25:18 +0300 Subject: [PATCH 11/12] feat: fall back to GET Range:bytes=0-0 probe when HEAD returns no Content-Length Some servers omit Content-Length on HEAD but respond to a range probe with 206 + Content-Range: bytes 0-0/. Parse the total from the Content-Range header so these files are still cached instead of being skipped as live streams. Co-Authored-By: Claude Sonnet 4.6 --- crates/cache/src/lib.rs | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/crates/cache/src/lib.rs b/crates/cache/src/lib.rs index 834191f840b..330bfb15dad 100644 --- a/crates/cache/src/lib.rs +++ b/crates/cache/src/lib.rs @@ -460,9 +460,7 @@ fn perform_download(url: &str, config: &CacheConfig) { } // HEAD request: get content-length and check range support. - // If the server returns no Content-Length the resource is likely a live / - // infinite stream — never attempt to cache it. - let (content_length, server_accepts_ranges) = match CLIENT.head(url).send() { + let (head_cl, head_accepts_ranges) = match CLIENT.head(url).send() { Ok(r) if r.status().is_success() => { let cl = r.content_length(); let ok = r @@ -476,10 +474,31 @@ fn perform_download(url: &str, config: &CacheConfig) { _ => (None, false), }; + // If HEAD gave no Content-Length, probe with GET Range: bytes=0-0. + // Servers that support ranges but omit Content-Length on HEAD will + // respond 206 with Content-Range: bytes 0-0/, giving us the size. + let (content_length, server_accepts_ranges) = if let Some(cl) = head_cl { + (Some(cl), head_accepts_ranges) + } else { + match CLIENT.get(url).header("Range", "bytes=0-0").send() { + Ok(r) if r.status().as_u16() == 206 => { + let total = r + .headers() + .get("content-range") + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.rsplit('/').next()) + .and_then(|s| s.trim().parse::().ok()); + debug!("cache: range probe for {} → total {:?}", url, total); + (total, true) + } + _ => (None, false), + } + }; + let content_length = match content_length { Some(cl) => cl, None => { - // No Content-Length → infinite/chunked stream; never cache. + // No Content-Length from HEAD or range probe → live/infinite stream. info!( "cache: no Content-Length for {} — skipping (live stream or transcoded?)", url From 13859005ff8b3026e99fb5bc207fe4fd533cc4ab Mon Sep 17 00:00:00 2001 From: Tsiry Sandratraina Date: Wed, 27 May 2026 22:54:25 +0300 Subject: [PATCH 12/12] fix: promote cache file when server delivers more bytes than Content-Length Transcoding servers like Navidrome advertise one Content-Length on HEAD but stream a slightly larger payload (e.g. 11 527 315 vs 11 617 210). The strict downloaded != content_length check was discarding every completed download as "incomplete". Changed to downloaded < content_length so over-delivery is accepted and the .tmp is promoted to .cache. --- crates/cache/src/lib.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/cache/src/lib.rs b/crates/cache/src/lib.rs index 330bfb15dad..cc88ac86067 100644 --- a/crates/cache/src/lib.rs +++ b/crates/cache/src/lib.rs @@ -561,8 +561,10 @@ fn perform_download(url: &str, config: &CacheConfig) { } }; - // Verify the download is complete before promoting to the final path. - if downloaded != content_length { + // Verify the download is not short. Servers (especially transcoding ones like + // Navidrome) sometimes deliver more bytes than Content-Length advertised, so + // only discard if we received strictly fewer bytes than promised. + if downloaded < content_length { debug!( "cache: incomplete download for {} ({}/{} bytes) — discarding", url, downloaded, content_length