Bound media request queues, filmstrip cache keys, and export job sizes

.env.example

@@ -128,6 +128,8 @@ API_BIND=0.0.0.0:8080
 EXPORT_DIR=/exports                       # top-level (own volume), NOT under the /data:ro mount
 EXPORT_TTL_SECONDS=86400
 # EXPORT_CACHE_MAX_BYTES=21474836480       # 20 GiB budget for the export cache
+# EXPORT_MAX_CONCURRENT=2                  # export jobs allowed to be queued/running at once
+# EXPORT_MAX_RANGE_SECONDS=86400           # longest window one export may cover (1 day); raise for multi-day exports
 
 # --- Server-generated streams + caches (optional) ---
 # MOBILE_STREAM_ENABLED=true               # on-demand H.264 transcode mobile clients fall back to when a
@@ -141,6 +143,8 @@ EXPORT_TTL_SECONDS=86400
                                            # Media3 can't read), so this is a real re-encode: leave off and
                                            # Android just uses the H.264 sub (SD); on = HD at recorder CPU cost.
 # SEGMENT_LOW_CACHE_MAX_BYTES=2147483648   # 2 GiB budget for the low-res playback segment cache
+# FRAME_PROXY_MAX_CONCURRENCY=             # env-only; unset → one per CPU core, clamped 8..32. Caps how many
+                                           # live camera stills are fetched at once (the low-bandwidth tile walls)
 
 # --- Scrub-preview cache (optional; five of these are also editable in the console, which wins) ---
 # THUMB_PREGEN_ENABLED=false               # build previews in the background so the FIRST drag is instant too

docker-compose.yml

@@ -152,6 +152,8 @@ services:
       ARCHIVE_STORAGE_PATH: ${ARCHIVE_STORAGE_PATH:-/data/archive}
       EXPORT_DIR: ${EXPORT_DIR:-/exports}               # top-level, NOT under /data:ro (can't nest a mount there)
       EXPORT_TTL_SECONDS: ${EXPORT_TTL_SECONDS:-86400}
+      EXPORT_MAX_CONCURRENT: ${EXPORT_MAX_CONCURRENT:-}   # queued+running export jobs allowed at once (empty = 2)
+      EXPORT_MAX_RANGE_SECONDS: ${EXPORT_MAX_RANGE_SECONDS:-}  # longest window one export may cover (empty = 86400)
       SEED_ADMIN_USERNAME: ${SEED_ADMIN_USERNAME:-admin}
       SEED_ADMIN_PASSWORD: ${SEED_ADMIN_PASSWORD:-}     # seeded by default (setup-env.sh generates it); blank = browser create-admin wizard
       ONVIF_CONFIG_B64: ${ONVIF_CONFIG_B64:-}           # legacy fallback; per-camera ONVIF creds live in the DB now
@@ -207,6 +209,7 @@ services:
       MAIN_REPAIR_TRANSCODE_ENABLED: ${MAIN_REPAIR_TRANSCODE_ENABLED:-}
       SEGMENT_LOW_CACHE_MAX_BYTES: ${SEGMENT_LOW_CACHE_MAX_BYTES:-}
       EXPORT_CACHE_MAX_BYTES: ${EXPORT_CACHE_MAX_BYTES:-}
+      FRAME_PROXY_MAX_CONCURRENCY: ${FRAME_PROXY_MAX_CONCURRENCY:-}   # live-still fetches in flight at once (empty = one per core, 8..32)
       # Behind the bundled Caddy (or any reverse proxy), set TRUST_PROXY=1 so the
       # rate limiter keys on the client IP from X-Forwarded-For instead of the
       # proxy's own address (one bucket for ALL HTTPS users otherwise). Leave

docs-site/docs/configuration/environment-reference.md

@@ -115,8 +115,10 @@ value there, the database copy wins and the env value is just the default.
 One honest footnote: a few more `THUMB_*` names exist in the source
 (`THUMB_INTERVAL_SECS`, `THUMB_MAX_ATTEMPTS`, `THUMB_MAX_WIDTH`,
 `THUMB_MIN_WIDTH`, `THUMB_NEAR_BLACK_LUMA`, `THUMB_EXTRACT_TIMEOUT_SECS`) as
-fixed built-in constants (a 4-second preview grid, widths clamped 48-640, a
-12-second extract timeout, and the black-frame retry logic). They are *not*
+fixed built-in constants (a 4-second preview grid, widths clamped 48-640 and
+then snapped to the nearest of 80/160/320/480/640 so the preview cache holds a
+handful of sizes instead of hundreds, a 12-second extract timeout, and the
+black-frame retry logic). They are *not*
 read from the environment and the compose file deliberately does not forward
 them, because forwarding a name implies a tunability that does not exist.
 Setting them in `.env` does nothing, so they don't get rows here.
@@ -178,7 +180,9 @@ See [Hardware decode](/configuration/hardware-decode) for enabling this.
 |---|---|---|
 | `EXPORT_DIR` | `/exports` | its own volume, not under the read-only `/data` mount |
 | `EXPORT_TTL_SECONDS` | `86400` | how long a completed export survives before cleanup |
-| `EXPORT_CACHE_MAX_BYTES` | `21474836480` (20 GiB) | size budget for the on-disk export cache; oldest entries are dropped past it |
+| `EXPORT_CACHE_MAX_BYTES` | `21474836480` (20 GiB) | size budget for the on-disk export cache; oldest entries are dropped past it. In-flight jobs are never deleted, but the space they already occupy counts against this budget, so finished exports are cleared to make room for them |
+| `EXPORT_MAX_CONCURRENT` | `2` | how many export jobs may be queued or running at once. Each job runs one video encode per camera, so this is the main brake on export CPU. A request that arrives while the slots are full gets a "too many requests" answer and should be retried shortly |
+| `EXPORT_MAX_RANGE_SECONDS` | `86400` (1 day) | the longest time window a single export may cover, per camera. Beyond this the request is refused with a clear message rather than starting an encode that would run for hours. Raise it if you genuinely export multi-day ranges; the same limit applies to each clip in a batch export |
 
 ## Streams the server generates on demand
 
@@ -191,6 +195,7 @@ are sized for a phone on a slow link.
 | `MOBILE_STREAM_WIDTH` | `640` | transcode width in pixels, floored at 160 |
 | `MAIN_REPAIR_TRANSCODE_ENABLED` | `false` | opt-in, per-camera full-resolution H.265 to H.264 transcode of a main stream whose SDP has no `fmtp` attribute. Android's video player rejects such a main ("missing attribute fmtp", seen on some Uniview LPR cameras) and otherwise steps down to the H.264 sub in SD. Leave it off and those cameras play in SD on Android; turn it on to get HD, at the cost of recorder CPU while an Android viewer is watching that camera fullscreen. A cheaper copy-only repair does not work for this case, which is why it is a real re-encode and off by default. The cheapest fix of all, when the camera allows it, is to set the camera's main stream to H.264 in its own web UI |
 | `SEGMENT_LOW_CACHE_MAX_BYTES` | `2147483648` (2 GiB) | size budget for the cache of low-resolution playback segments |
+| `FRAME_PROXY_MAX_CONCURRENCY` | scales with cores (one per core, at least 8, at most 32) | how many live camera stills Crumb fetches at once. The low-bandwidth tile walls on the phone apps poll one still per tile per second, so this is what keeps a big wall from queueing. Past the limit a still request is answered with "busy, retry shortly" and the tile keeps its previous image until the next poll |
 
 ## Database backup
 

docs/DECISIONS.md

@@ -8,6 +8,107 @@ revisit.
 
 ---
 
+## 2026-09-07, Media routes: bounded waits answering 503, not unbounded queues
+
+**Context.** Every expensive media path (clip and low-bitrate transcodes, the
+filmstrip's single-frame extractions, and now the live-still proxy) gates its
+work behind a shared semaphore. Those acquires were plain
+`acquire_owned().await`, an unbounded queue: once the permits were gone, further
+requests parked indefinitely, holding a connection and never answering. The
+media router also carried neither a request timeout nor a rate limit, both of
+which the JSON routes have had since #17.
+
+**Decision.** Every media semaphore acquire goes through one helper
+(`services/api/src/media_limits.rs::acquire_bounded`) that gives up after a
+per-call-site budget and returns `503` plus `Retry-After`. The media router
+gains its own, much larger rate-limit bucket (burst 1200, 240/s sustained,
+roughly 6x the busiest real client) and a 180 s time-to-response bound.
+
+**Why a timeout is safe over streaming media.** `tower_http`'s `TimeoutLayer`
+bounds only the handler future; response bodies have a separate
+`ResponseBodyTimeoutLayer` that is deliberately not used. Segment downloads,
+export archives, and the open-ended live `stream.mp4` all return headers
+immediately and stream afterwards, so none of them can be cut. The one route
+that legitimately produces for minutes before answering, the on-demand
+DB-vs-disk size verification walk, is mounted outside the timeout for exactly
+that reason.
+
+**Rejected: making the clients wait longer.** Raising the permit counts or the
+wait budgets keeps the failure mode (a request that never answers) and only
+moves the threshold. The clients already treat a failed media fetch as "keep the
+placeholder, poll again", so "busy, retry shortly" is both truthful and the
+behaviour they already handle.
+
+**Rejected: a per-user or per-camera semaphore instead of a global one.** It
+would give fairer degradation, but it multiplies the tuning surface and the
+memory footprint for a system whose realistic worst case is a handful of
+operators. Revisit if a deployment reports one client starving others despite
+the bounded waits.
+
+**Trades knowingly accepted.** Under genuine saturation a scrub thumbnail or a
+wall tile now fails fast instead of eventually arriving; that is the intended
+swap. A camera whose still cannot be fetched is latched for 10 s so subsequent
+polls skip the cold-start retry ladder, which means a camera coming back can be
+up to 10 s late to serve its first still.
+
+**Revisit if:** operators report 503s on the media routes during normal use
+(the budgets or permit counts are too tight), or a deployment large enough to
+need per-principal fairness appears.
+
+---
+
+## 2026-09-07, Filmstrip widths snap to a fixed ladder; export requests are capped
+
+**Context.** The thumbnail width was clamped to 48..640 but otherwise free, and
+it is part of the on-disk cache key, so 593 distinct widths for one instant meant
+593 ffmpeg runs and 593 cache files for what is visually one frame. Separately,
+`POST /export` validated only `start < end`: nothing capped the window, the
+camera count, or duplicates in the camera list, and `filter_camera_ids` is a
+scope filter rather than a de-duplicator (for an admin it returns the list
+verbatim), so `[X, X, X, ...]` ran N sequential full-range encodes all writing
+the same output file with `-y`, counted as one job against
+`EXPORT_MAX_CONCURRENT`.
+
+**Decision.**
+
+- Widths clamp and then snap to `80/160/320/480/640` (nearest, ties down). The
+  buckets are chosen so every width the shipped clients ask for lands on itself:
+  160 (Android and desktop scrub lists), 320 (the Android playback wall), 480
+  (the iOS scrub still, the desktop preview frame, and the `THUMB_PREGEN_WIDTH`
+  default), so no existing or pre-generated cache entry is invalidated.
+- `POST /export` sorts and de-duplicates the camera list, caps it at 50 distinct
+  cameras (the same ceiling `/export/batch` puts on clips, since both fan out to
+  one sequential encode per unit of work), and caps the window with a new
+  `EXPORT_MAX_RANGE_SECONDS` (default 86400, a full day of one camera).
+- Each per-camera encode gets a wall-clock budget derived from the range
+  (4x realtime, floored at 10 minutes, capped at 24 hours) and `kill_on_drop`.
+  A child that wedges now fails the job with a stored error instead of leaving
+  it `Running` forever, which used to consume an `EXPORT_MAX_CONCURRENT` slot
+  permanently until the api was restarted.
+
+**Rejected: quantizing by rounding to a multiple (say 32 px).** It bounds the
+key space too, but it does not guarantee the clients' existing widths are
+fixed points, so the whole warm thumbnail cache (including anything
+pre-generated at 480) would be re-rendered at neighbouring keys on upgrade.
+
+**Rejected: de-duplicating inside `filter_camera_ids`.** That function is the
+RBAC scope filter used by several handlers, including ones that rely on
+comparing the filtered length to the input length to detect a partial-scope
+request. Making it also dedup would silently change those comparisons.
+
+**Rejected: a stall watchdog on ffmpeg progress instead of a wall-clock
+budget.** More precise (it would catch a wedged child in seconds rather than
+hours) and the progress parsing already exists, but it is a larger change to
+the export worker than the failure mode warrants right now.
+
+**Revisit if:** a deployment genuinely exports multi-day ranges and finds the
+default cap or the derived encode budget too tight (both are configurable; the
+budget is not), or if per-camera export throughput makes the 4x realtime factor
+the binding constraint, at which point the stall watchdog becomes the better
+mechanism.
+
+---
+
 ## 2026-08-10, Home Assistant `climate` (thermostat/HVAC setpoint) control is out of scope
 
 **Context.** #442 introduced value-setting HA controls. Light dimming

services/api/src/cameras.rs

@@ -43,6 +43,30 @@ use crate::{
     state::AppState,
 };
 
+// ─── live-still proxy tuning ──────────────────────────────────────────────────
+
+/// Attempts made against go2rtc for a still on the normal (camera believed
+/// healthy) path. go2rtc starts producers lazily, so the first fetch for a cold
+/// stream legitimately needs a couple of tries.
+const FRAME_MAX_ATTEMPTS: u32 = 4;
+
+/// Delay between those attempts.
+const FRAME_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(900);
+
+/// Per-attempt upstream timeout on the normal path.
+const FRAME_REQUEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
+
+/// Per-attempt upstream timeout on the known-bad fast path. Short: the point is
+/// to answer "not available" quickly and release the permit, not to wait out a
+/// camera that has already failed.
+const FRAME_FAST_FAIL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2);
+
+/// How long a camera stays latched as "still unavailable" after a failed fetch.
+/// Longer than the ~1 s poll interval of the low-bandwidth walls (so the latch
+/// actually covers their polls) but short enough that a camera coming back is
+/// served the normal way within a few seconds even if no poll succeeds first.
+const FRAME_UNAVAILABLE_TTL: std::time::Duration = std::time::Duration::from_secs(10);
+
 // ─── route registry ───────────────────────────────────────────────────────────
 
 /// Mount per-camera utility routes.
@@ -132,13 +156,31 @@ async fn list_visible_cameras(
 /// (`GET /media-token`) for per-camera media, so a login credential never lands
 /// in a snapshot URL.
 ///
+/// # Concurrency
+///
+/// The low-bandwidth walls on Android and iOS poll one still per tile roughly
+/// once a second, so this route is the highest-frequency media endpoint there
+/// is. Every fetch takes a permit from the frame-proxy semaphore
+/// (`FRAME_PROXY_MAX_CONCURRENCY`) with a bounded wait: a saturated proxy
+/// answers `503` + `Retry-After` and the tile shows its placeholder until the
+/// next poll, instead of queueing an unbounded pile of connection-holding
+/// requests.
+///
+/// A camera whose still could not be fetched is latched as unavailable for
+/// [`FRAME_UNAVAILABLE_TTL`], and while the latch holds the retry ladder
+/// below collapses to a single quick attempt. That is what keeps a wall pointed
+/// at a camera that is down cheap: the first poll pays the full cold-start
+/// ladder, every poll after it fails in about a second and releases its permit.
+/// A successful fetch clears the latch immediately.
+///
 /// # Errors
 ///
 /// * `401` / `403` — auth / scope failure.
 /// * `404` — camera UUID not found in the database.
 /// * `502` — go2rtc was unreachable, returned a non-2xx status, or the frame
 ///   body could not be read.  Detail is logged at `warn!`, not `error!`, so a
 ///   momentarily unavailable stream doesn't trip 5xx alerting.
+/// * `503`, the still proxy is saturated; retry after the `Retry-After` hint.
 async fn get_camera_frame(
     // Media-read: browsers fetch this still with a scoped `?token=` (see
     // MediaUrls in the clients / admin.html snapshot cache), so it opts into the
@@ -180,13 +222,35 @@ async fn get_camera_frame(
     // percent-encoding, so a plain format string is safe here.
     let upstream_url = format!("{api_base}/api/frame.jpeg?src={}", cam.go2rtc_name);
 
+    // Fast path when the camera is already known not to be producing stills: a
+    // disabled camera, a stream go2rtc has rejected, or one whose last fetch
+    // exhausted the ladder. Each of those means the cold-start retry ladder
+    // below has nothing to wait for, so it only burns a permit.
+    let known_bad = !cam.enabled
+        || state.stream_rejected(&cam.go2rtc_name)
+        || state.frame_recently_unavailable(camera_id);
+    let (max_attempts, request_timeout) = if known_bad {
+        (1, FRAME_FAST_FAIL_TIMEOUT)
+    } else {
+        (FRAME_MAX_ATTEMPTS, FRAME_REQUEST_TIMEOUT)
+    };
+
     let http_client = reqwest::Client::builder()
-        .timeout(std::time::Duration::from_secs(5))
+        .timeout(request_timeout)
         .build()
         .map_err(|e| {
             ApiError::Internal(anyhow::anyhow!("frame proxy: build reqwest client: {e}"))
         })?;
 
+    // Bound how many stills are in flight at once. Held across the fetch only;
+    // released on drop when this handler returns.
+    let _permit = crate::media_limits::acquire_bounded(
+        &state.frame_semaphore(),
+        crate::media_limits::FRAME_PROXY_WAIT,
+        "camera still proxy",
+    )
+    .await?;
+
     // P0-GO2RTC (lighter lockdown): Crumb's own go2rtc REST API now requires
     // Basic auth (`local_auth: true` — this call crosses the Docker bridge
     // network, which go2rtc does not treat as "localhost"). Only send Crumb's
@@ -203,12 +267,11 @@ async fn get_camera_frame(
     // returns 500 while the source connects, then succeeds once a keyframe lands.
     // Retry a few times with a short delay so a cold camera loads on first touch
     // instead of leaning on the client to retry (and to keep the error logs quiet).
-    const MAX_ATTEMPTS: u32 = 4;
-    const RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(900);
+    // `max_attempts` is 1 on the known-bad fast path above.
     let mut frame = None;
     let mut last_status: Option<reqwest::StatusCode> = None;
     let mut last_err: Option<String> = None;
-    for attempt in 1..=MAX_ATTEMPTS {
+    for attempt in 1..=max_attempts {
         let mut req = http_client.get(&upstream_url);
         if let Some((ref user, ref pass)) = go2rtc_auth {
             req = req.basic_auth(user, Some(pass));
@@ -236,16 +299,20 @@ async fn get_camera_frame(
             // operator needs when go2rtc is down or the API base is wrong.
             Err(e) => last_err = Some(format!("connect: {:#}", anyhow::Error::new(e))),
         }
-        if attempt < MAX_ATTEMPTS {
-            tokio::time::sleep(RETRY_DELAY).await;
+        if attempt < max_attempts {
+            tokio::time::sleep(FRAME_RETRY_DELAY).await;
         }
     }
-    let bytes = frame.ok_or_else(|| {
-        ApiError::BadGateway(format!(
-            "go2rtc frame ({upstream_url}) unavailable after {MAX_ATTEMPTS} tries (last status {last_status:?}{})",
+    let Some(bytes) = frame else {
+        // Latch the camera so the next poll takes the single-attempt path.
+        state.mark_frame_unavailable(camera_id, FRAME_UNAVAILABLE_TTL);
+        return Err(ApiError::BadGateway(format!(
+            "go2rtc frame ({upstream_url}) unavailable after {max_attempts} tries (last status {last_status:?}{})",
             last_err.map(|e| format!(", {e}")).unwrap_or_default(),
-        ))
-    })?;
+        )));
+    };
+    // Back in service: clear the latch so the full ladder is available again.
+    state.clear_frame_unavailable(camera_id);
 
     Ok((
         StatusCode::OK,