feat(ocr): serialize uploads via a persistent on-disk queue

Dockerfile

@@ -28,7 +28,7 @@ apt-get -y --no-install-recommends install \
 apt-get -y clean && \
 rm -rf /var/lib/apt/lists/* && \
 wget https://download.brother.com/welcome/dlf105200/brscan4-0.4.11-1.amd64.deb --progress=dot:giga -O /tmp/brscan4.deb && \
-wget https://download.brother.com/welcome/dlf006652/brscan-skey-0.3.2-0.amd64.deb --progress=dot:giga -O /tmp/brscan-skey.deb && \
+wget https://download.brother.com/welcome/dlf006652/brscan-skey-0.3.5-0.amd64.deb --progress=dot:giga -O /tmp/brscan-skey.deb && \
 dpkg -i --force-all /tmp/brscan4.deb && \
 dpkg -i --force-all /tmp/brscan-skey.deb && \
 rm -f /tmp/brscan4.deb /tmp/brscan-skey.deb

files/runScanner.sh

@@ -63,6 +63,23 @@ su - "$NAME" -c "/usr/bin/brsaneconfig4 -a name=$NAME model=$MODEL ip=$IPADDRESS
 su - "$NAME" -c "/usr/bin/brscan-skey"
 echo "-----"
 
+echo "starting OCR queue worker (supervised)..."
+OCR_QUEUE_DIR="${OCR_QUEUE_DIR:-/scans/.ocr_queue}"
+mkdir -p "$OCR_QUEUE_DIR/pending" "$OCR_QUEUE_DIR/in_progress" "$OCR_QUEUE_DIR/failed"
+chmod -R 777 "$OCR_QUEUE_DIR"
+# Supervisor: relaunch worker forever, with a small backoff to avoid a
+# crash-loop. Exported env vars (OCR_*, FTP_*, SSH_*, TELEGRAM_*,
+# REMOVE_ORIGINAL_AFTER_OCR) are inherited by the worker.
+(
+  while true; do
+    /opt/brother/scanner/brscan-skey/script/ocr_worker.sh
+    echo "[ocr_worker supervisor] worker exited with $?; restarting in 5s"
+    sleep 5
+  done
+) >>/var/log/scanner.log 2>&1 &
+echo "OCR queue worker pid: $!"
+echo "-----"
+
 echo "setting up webserver:"
 if [ "$WEBSERVER" == "true" ]; then
   echo "www-data ALL=($NAME) NOPASSWD:ALL" >>/etc/sudoers

script/ocr_enqueue.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+# Append a job to the OCR worker's pending queue.
+#
+# Usage: ocr_enqueue.sh <pdf_path> <date> <suffix>
+#   pdf_path - absolute path to the PDF to OCR (typically /scans/<date>.pdf)
+#   date     - the date stamp used elsewhere in the pipeline; becomes the
+#              filename for the OCR output (/scans/<date>-ocr.pdf)
+#   suffix   - "front", "rear", or similar; used only for log/notification text
+
+set -eu
+
+PDF="${1:-}"
+DATE="${2:-}"
+SUFFIX="${3:-}"
+
+if [ -z "$PDF" ] || [ -z "$DATE" ] || [ -z "$SUFFIX" ]; then
+  echo "ocr_enqueue.sh: usage: $0 <pdf_path> <date> <suffix>" >&2
+  exit 2
+fi
+
+QUEUE_ROOT="${OCR_QUEUE_DIR:-/scans/.ocr_queue}"
+PENDING_DIR="${QUEUE_ROOT}/pending"
+mkdir -p "$PENDING_DIR"
+
+# Write to a hidden temp file first then atomically rename into place so
+# the worker can never observe a partially written job. Filename starts
+# with a sortable timestamp so the worker drains in roughly FIFO order.
+ts=$(date +%s%N)
+basename="${ts}-${DATE}-${SUFFIX}.job"
+tmpfile="${PENDING_DIR}/.${basename}.tmp"
+final="${PENDING_DIR}/${basename}"
+
+{
+  echo "PDF=${PDF}"
+  echo "DATE=${DATE}"
+  echo "SUFFIX=${SUFFIX}"
+  echo "ATTEMPTS=0"
+} >"$tmpfile"
+
+mv "$tmpfile" "$final"
+echo "ocr_enqueue: queued ${basename}"

script/ocr_worker.sh

@@ -0,0 +1,156 @@
+#!/bin/bash
+# OCR queue worker.
+#
+# Serializes uploads to the OCR microservice so concurrent scans cannot
+# saturate the upload bandwidth or overwhelm the OCR backend. One worker
+# per container; coordination is done with atomic rename(2) ("mv") so the
+# queue works on filesystems that do not support inotify or flock
+# (e.g. SMB-mounted /scans).
+#
+# Queue layout (under $OCR_QUEUE_DIR, default /scans/.ocr_queue):
+#   pending/      newly enqueued jobs; oldest first by filename prefix
+#   in_progress/  the single job currently being processed
+#   failed/       jobs that exceeded $OCR_MAX_ATTEMPTS
+#
+# Job file (KEY=VALUE text):
+#   PDF=/scans/2026-05-17-17-22-59.pdf
+#   DATE=2026-05-17-17-22-59
+#   SUFFIX=front
+#   ATTEMPTS=0
+
+set -u
+
+QUEUE_ROOT="${OCR_QUEUE_DIR:-/scans/.ocr_queue}"
+PENDING_DIR="${QUEUE_ROOT}/pending"
+IN_PROGRESS_DIR="${QUEUE_ROOT}/in_progress"
+FAILED_DIR="${QUEUE_ROOT}/failed"
+OUTPUT_DIR="${OCR_OUTPUT_DIR:-/scans}"
+POLL_INTERVAL="${OCR_QUEUE_POLL_SECONDS:-5}"
+MAX_ATTEMPTS="${OCR_MAX_ATTEMPTS:-5}"
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)"
+
+mkdir -p "$PENDING_DIR" "$IN_PROGRESS_DIR" "$FAILED_DIR"
+
+log() {
+  echo "[ocr_worker $(date +%Y-%m-%dT%H:%M:%S)] $*"
+}
+
+read_field() {
+  local file="$1" key="$2"
+  grep "^${key}=" "$file" 2>/dev/null | tail -n1 | cut -d= -f2-
+}
+
+write_attempts() {
+  local file="$1" n="$2"
+  local tmp="${file}.rewrite.$$"
+  grep -v '^ATTEMPTS=' "$file" >"$tmp"
+  echo "ATTEMPTS=$n" >>"$tmp"
+  mv "$tmp" "$file"
+}
+
+# On startup, return half-processed jobs to the pending queue so a crash
+# or container restart does not strand them. ATTEMPTS in the file is
+# already incremented (see claim path) so we will not retry forever.
+recover() {
+  shopt -s nullglob
+  for f in "$IN_PROGRESS_DIR"/*.job; do
+    name=$(basename "$f")
+    if mv "$f" "$PENDING_DIR/$name" 2>/dev/null; then
+      log "recovered $name from in_progress"
+    fi
+  done
+  shopt -u nullglob
+}
+
+# Upload one PDF and run the post-OCR side effects. Returns 0 on success,
+# non-zero on any failure (network, missing file, missing config).
+process_job() {
+  local jobfile="$1"
+  local pdf date suffix
+  pdf=$(read_field "$jobfile" PDF)
+  date=$(read_field "$jobfile" DATE)
+  suffix=$(read_field "$jobfile" SUFFIX)
+
+  if [ -z "${OCR_SERVER:-}" ] || [ -z "${OCR_PORT:-}" ] || [ -z "${OCR_PATH:-}" ]; then
+    log "OCR_SERVER/OCR_PORT/OCR_PATH not set; cannot process"
+    return 1
+  fi
+  if [ -z "$pdf" ] || [ -z "$date" ]; then
+    log "malformed job $(basename "$jobfile"): PDF or DATE missing"
+    return 1
+  fi
+  if [ ! -f "$pdf" ]; then
+    log "input PDF missing: $pdf"
+    return 1
+  fi
+
+  local out="${OUTPUT_DIR}/${date}-ocr.pdf"
+  log "uploading $pdf to ${OCR_SERVER}:${OCR_PORT}/${OCR_PATH}"
+  if ! curl --fail --silent --show-error \
+        -F "userfile=@${pdf}" -H "Expect:" \
+        -o "$out" \
+        "${OCR_SERVER}:${OCR_PORT}/${OCR_PATH}"; then
+    log "OCR upload failed for $pdf"
+    rm -f "$out"
+    return 1
+  fi
+
+  log "OCR finished -> $out"
+
+  # Best-effort notifications; do not fail the job if any of these error.
+  "${SCRIPT_DIR}/trigger_inotify.sh" "${SSH_USER:-}" "${SSH_PASSWORD:-}" "${SSH_HOST:-}" "${SSH_PATH:-}" "${date}-ocr.pdf" || true
+  "${SCRIPT_DIR}/trigger_telegram.sh" "${date}-ocr.pdf (${suffix:-?}) OCR finished" || true
+  "${SCRIPT_DIR}/sendtoftps.sh" "${FTP_USER:-}" "${FTP_PASSWORD:-}" "${FTP_HOST:-}" "${FTP_PATH:-}" "$out" || true
+
+  if [ "${REMOVE_ORIGINAL_AFTER_OCR:-}" = "true" ] && [ -f "$out" ]; then
+    rm -f "$pdf"
+  fi
+  return 0
+}
+
+requeue_or_fail() {
+  local jobfile="$1" attempts="$2"
+  local name
+  name=$(basename "$jobfile")
+  if [ "$attempts" -ge "$MAX_ATTEMPTS" ]; then
+    mv "$jobfile" "$FAILED_DIR/$name" 2>/dev/null || rm -f "$jobfile"
+    log "$name exceeded $MAX_ATTEMPTS attempts; moved to failed/"
+    return
+  fi
+  local backoff=$(( attempts * 30 ))
+  log "$name attempt $attempts failed; retrying in ${backoff}s"
+  sleep "$backoff"
+  mv "$jobfile" "$PENDING_DIR/$name" 2>/dev/null || true
+}
+
+recover
+log "queue worker started (poll=${POLL_INTERVAL}s, max_attempts=${MAX_ATTEMPTS})"
+
+while true; do
+  candidate=$(ls -1 "$PENDING_DIR"/*.job 2>/dev/null | head -n1 || true)
+  if [ -z "$candidate" ]; then
+    sleep "$POLL_INTERVAL"
+    continue
+  fi
+
+  name=$(basename "$candidate")
+  claim="$IN_PROGRESS_DIR/$name"
+
+  # Atomic claim. If a competing worker (or recovery sweep) won, just retry.
+  if ! mv "$candidate" "$claim" 2>/dev/null; then
+    continue
+  fi
+
+  # Bump ATTEMPTS on disk *before* processing so a crash mid-upload is
+  # accounted for and we will not loop forever on a poison job.
+  current=$(read_field "$claim" ATTEMPTS)
+  [ -z "$current" ] && current=0
+  next=$(( current + 1 ))
+  write_attempts "$claim" "$next"
+
+  if process_job "$claim"; then
+    rm -f "$claim"
+  else
+    requeue_or_fail "$claim" "$next"
+  fi
+done

script/scanRear.sh

@@ -103,24 +103,8 @@ fi
     if [ -z "${OCR_SERVER}" ] || [ -z "${OCR_PORT}" ] || [ -z "${OCR_PATH}" ]; then
       echo "OCR environment variables not set, skipping OCR."
     else
-      echo "starting OCR for $date..."
-      (
-        curl -F "userfile=@${output_pdf_file}" -H "Expect:" -o "/scans/${date}-ocr.pdf" "${OCR_SERVER}":"${OCR_PORT}"/"${OCR_PATH}"
-        ${script_dir}/trigger_inotify.sh "${SSH_USER}" "${SSH_PASSWORD}" "${SSH_HOST}" "${SSH_PATH}" "${date}-ocr.pdf"
-        ${script_dir}/trigger_telegram.sh "${date}-ocr.pdf (rear) OCR finished"
-        ${script_dir}/sendtoftps.sh \
-          "${FTP_USER}" \
-          "${FTP_PASSWORD}" \
-          "${FTP_HOST}" \
-          "${FTP_PATH}" \
-          "/scans/${date}-ocr.pdf"
-
-          if [ "${REMOVE_ORIGINAL_AFTER_OCR}" == "true" ]; then
-			if [ -f "/scans/${date}-ocr.pdf" ]; then
-				rm ${output_pdf_file}
-			fi
-          fi
-      ) &
+      echo "queueing OCR for $date..."
+      ${script_dir}/ocr_enqueue.sh "${output_pdf_file}" "${date}" "rear"
     fi
   ) &
 ) &

script/scantofile-0.2.4-1.sh

@@ -78,24 +78,8 @@
       if [ -z "${OCR_SERVER}" ] || [ -z "${OCR_PORT}" ] || [ -z "${OCR_PATH}" ]; then
         echo "OCR environment variables not set, skipping OCR."
       else
-        echo "starting OCR for $date..."
-        (
-          curl -F "userfile=@${output_pdf_file}" -H "Expect:" -o "/scans/${date}-ocr.pdf" "${OCR_SERVER}":"${OCR_PORT}"/"${OCR_PATH}"
-          ${script_dir}/trigger_inotify.sh "${SSH_USER}" "${SSH_PASSWORD}" "${SSH_HOST}" "${SSH_PATH}" "${date}-ocr.pdf"
-          ${script_dir}/trigger_telegram.sh "${date}-ocr.pdf (front) OCR finished"
-          ${script_dir}/sendtoftps.sh \
-            "${FTP_USER}" \
-            "${FTP_PASSWORD}" \
-            "${FTP_HOST}" \
-            "${FTP_PATH}" \
-            "/scans/${date}-ocr.pdf"
-
-          if [ "${REMOVE_ORIGINAL_AFTER_OCR}" == "true" ]; then
-		    if [ -f "/scans/${date}-ocr.pdf" ]; then
-              rm ${output_pdf_file}
-			fi
-          fi
-        ) &
+        echo "queueing OCR for $date..."
+        ${script_dir}/ocr_enqueue.sh "${output_pdf_file}" "${date}" "front"
       fi
     ) &
   ) &

tests/helpers/common.bash

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# Shared bats test helpers.
+#
+# Each test should `load '../helpers/common'` in its setup() and then use:
+#   * REPO_ROOT       - absolute path to the repo
+#   * SCRIPT_DIR      - absolute path to the script/ directory
+#   * test_tmp        - a fresh per-test temp directory (auto-cleaned)
+#   * mock_command    - drop a fake executable on PATH for one command
+#   * mock_record     - drop a fake executable that records its argv
+#   * mock_calls      - read back what mock_record captured
+
+REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd -P)"
+SCRIPT_DIR="$REPO_ROOT/script"
+
+common_setup() {
+  test_tmp="$(mktemp -d)"
+  MOCK_BIN="$test_tmp/mock_bin"
+  MOCK_LOG_DIR="$test_tmp/mock_log"
+  mkdir -p "$MOCK_BIN" "$MOCK_LOG_DIR"
+  export PATH="$MOCK_BIN:$PATH"
+}
+
+common_teardown() {
+  if [ -n "${test_tmp:-}" ] && [ -d "$test_tmp" ]; then
+    rm -rf "$test_tmp"
+  fi
+}
+
+# mock_command <name> [exit_code] [stdout]
+mock_command() {
+  local name="$1"
+  local exit_code="${2:-0}"
+  local stdout="${3:-}"
+  cat >"$MOCK_BIN/$name" <<EOF
+#!/usr/bin/env bash
+printf '%s' "$stdout"
+exit $exit_code
+EOF
+  chmod +x "$MOCK_BIN/$name"
+}
+
+# mock_record <name> [exit_code]
+# Each invocation appends a line to $MOCK_LOG_DIR/<name>.log of the form
+#   <argc>\t<arg1>\t<arg2>...
+mock_record() {
+  local name="$1"
+  local exit_code="${2:-0}"
+  cat >"$MOCK_BIN/$name" <<EOF
+#!/usr/bin/env bash
+{
+  printf '%d' "\$#"
+  for a in "\$@"; do printf '\t%s' "\$a"; done
+  printf '\n'
+} >>"$MOCK_LOG_DIR/$name.log"
+exit $exit_code
+EOF
+  chmod +x "$MOCK_BIN/$name"
+}
+
+# mock_calls <name> - print the recorded invocations (one per line)
+mock_calls() {
+  local name="$1"
+  cat "$MOCK_LOG_DIR/$name.log" 2>/dev/null || true
+}
+
+# mock_call_count <name>
+mock_call_count() {
+  local name="$1"
+  if [ -f "$MOCK_LOG_DIR/$name.log" ]; then
+    wc -l <"$MOCK_LOG_DIR/$name.log" | tr -d ' '
+  else
+    echo 0
+  fi
+}