From c9a855369a97203e40de61c1e7528220ad1561a8 Mon Sep 17 00:00:00 2001 From: Pierre Date: Sun, 25 Oct 2026 12:19:52 +0100 Subject: [PATCH 01/31] Add provider-agnostic cloud storage and release packaging Implement Google Drive, S3-compatible, Dropbox, OneDrive and WebDAV storage providers with AES-GCM encrypted credentials, background upload/sync/restore/cleanup tasks, and safety checks that refuse destructive operations without a verified local copy. Add the /admin/storage WebUI and 'streamvault storage' CLI, integrate auto-upload after recording and remux, extend the worker pool, and add Docker/supervisord worker packaging plus Nginx X-Accel aliases for archives, legacy files, VOD cache, thumbnails and HLS. Update install/docs, harden secret handling, redact provider errors, and add a 93-test regression suite. --- .dockerignore | 35 + .env.example | 20 +- INSTALL.md | 157 +- README.md | 68 +- WEBUI/templates/admin.html | 1 + WEBUI/templates/storage.html | 521 ++++++ WEBUI/webui.py | 393 ++++- bin/streamvault | 60 +- core/database.py | 525 +++++- core/storage/__init__.py | 25 + core/storage/base.py | 375 +++++ core/storage/cli.py | 404 +++++ core/storage/crypto.py | 128 ++ core/storage/http.py | 179 ++ core/storage/providers/__init__.py | 1 + core/storage/providers/dropbox.py | 450 +++++ core/storage/providers/google_drive.py | 632 +++++++ core/storage/providers/onedrive.py | 424 +++++ core/storage/providers/s3.py | 324 ++++ core/storage/providers/webdav.py | 315 ++++ core/storage/registry.py | 49 + core/storage/service.py | 1518 +++++++++++++++++ docker-compose.yml | 39 +- docker-worker.conf | 63 + install.sh | 75 +- nginx.conf | 80 +- recorder/Streamrecorder.py | 64 +- recorder/worker.py | 86 +- requirements.txt | 5 +- tests/__init__.py | 0 tests/test_storage.py | 2082 ++++++++++++++++++++++++ 31 files changed, 8974 insertions(+), 124 deletions(-) create mode 100644 .dockerignore create mode 100644 WEBUI/templates/storage.html create mode 100644 core/storage/__init__.py create mode 100644 core/storage/base.py create mode 100644 core/storage/cli.py create mode 100644 core/storage/crypto.py create mode 100644 core/storage/http.py create mode 100644 core/storage/providers/__init__.py create mode 100644 core/storage/providers/dropbox.py create mode 100644 core/storage/providers/google_drive.py create mode 100644 core/storage/providers/onedrive.py create mode 100644 core/storage/providers/s3.py create mode 100644 core/storage/providers/webdav.py create mode 100644 core/storage/registry.py create mode 100644 core/storage/service.py create mode 100644 docker-worker.conf create mode 100644 tests/__init__.py create mode 100644 tests/test_storage.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..564a79f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,35 @@ +.git +.gitignore +.env +.env.* +!.env.example +.venv +venv +__pycache__ +*.py[cod] +*.db +*.db-wal +*.db-shm +*.sqlite +*.sqlite3 +data +files +.thumbs +.meta +.hls +.hls_live +.vodcache +.gunicorn +logs +*.log +*.mp4 +*.mkv +*.ts +*.m3u8 +*.part +*.tmp +docs/screenshots +node_modules +.pytest_cache +.coverage +htmlcov diff --git a/.env.example b/.env.example index 32af36a..475c9b8 100644 --- a/.env.example +++ b/.env.example @@ -6,14 +6,28 @@ # --- Core WebUI Server Settings --- VOD_HOST=0.0.0.0 VOD_PORT=8080 -VOD_PUBLIC_BASE=http://localhost:8080 -VOD_SECRET_KEY=change-this-to-a-very-secure-random-secret-key-32-chars-minimum +# Public base URL used for share links and OAuth callbacks. +# Leave empty to derive it from the incoming request (recommended behind a proxy). +# For OAuth providers set the exact public HTTPS origin, e.g. https://recorder.example.com +VOD_PUBLIC_BASE= +# Required. Generate with: python -c "import secrets; print(secrets.token_hex(32))" +VOD_SECRET_KEY= VOD_SECURE_COOKIES=false +# --- Cloud / External Storage (optional) --- +# Dedicated AES-GCM key for provider credentials. Falls back to VOD_SECRET_KEY. +# Generate with: python -c "import secrets; print(secrets.token_hex(32))" +# STORAGE_ENCRYPTION_KEY= +# Background worker threads for uploads/restores (default: 1) +# WORKER_STORAGE_THREADS=1 +# Auto-upload, local deletion and retention are configured in the WebUI +# (Systemsteuerung -> Cloud-Speicher) or via `streamvault storage settings`. + # --- Authentication & Initial Bootstrap --- # NOTE: Required in production. Cannot be 'admin'/'admin'. VOD_USER=vaultadmin -VOD_PASS=GenerateAStrongPasswordHere123! +# Required. Generate with: python -c "import secrets; print(secrets.token_urlsafe(24))" +VOD_PASS= VOD_ALLOW_ENV_LOGIN=false VOD_SESSION_MAX_AGE_SEC=0 VOD_SESSION_COOKIE_MAX_AGE_SEC=34560000 diff --git a/INSTALL.md b/INSTALL.md index 1286dab..88b512d 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -30,14 +30,15 @@ sudo ./install.sh - Installs all apt dependencies (`ffmpeg`, `python3-venv`, `nginx`, `sqlite3`). - Sets up system user `streamvault` and directories (`/opt/streamvault`, `/var/lib/streamvault`). - Generates cryptographically secure secrets in `/etc/streamvault/env`. -- Deploys and enables systemd daemons (`streamvault-webui.service`, `streamvault-recorder.service`). +- Deploys and enables systemd daemons (`streamvault-webui.service`, `streamvault-recorder.service`, `streamvault-worker.service`). - Configures Nginx with `X-Accel-Redirect` byte-range media offloading. - **Registers the global Linux management command** `/usr/local/bin/streamvault`: ```bash streamvault status # Check daemons and live health streamvault logs webui # View WebUI log stream -streamvault logs recorder# View Stream Recorder log stream +streamvault logs recorder # View Stream Recorder log stream +streamvault logs worker # View background task worker log stream streamvault restart # Restart application daemons streamvault update # Pull updates, upgrade dependencies, and reload streamvault check # Validate Python syntax & Nginx configurations @@ -65,7 +66,9 @@ nano .env ```bash docker compose up -d --build ``` -Access the application at `http://localhost:8090` (proxied by Nginx) or directly at `http://localhost:8080`. +Access the application at `http://localhost:8090` (proxied by Nginx). The +Compose stack runs WebUI, recorder and worker under supervisord; the `hls_live` +volume is shared between the app and Nginx containers. --- @@ -114,12 +117,22 @@ Ensure `/etc/streamvault/env` contains: ```ini VOD_HOST=127.0.0.1 VOD_PORT=8080 +# Leave empty to derive from the request, or set the public HTTPS origin. +VOD_PUBLIC_BASE= VOD_BASE_DIR=/var/lib/streamvault/data LIVE_HLS_DIR=/dev/shm/streamvault_hls VOD_USER=vaultadmin -VOD_PASS=YourSuperSecretPasswordHere987! -VOD_SECRET_KEY=generate_a_random_32_character_string_here +VOD_PASS= +VOD_SECRET_KEY= VOD_USE_X_ACCEL=true +# Optional dedicated key for cloud credentials (falls back to VOD_SECRET_KEY). +# STORAGE_ENCRYPTION_KEY= +``` + +Fill the two empty secret values with freshly generated ones: +```bash +python3 -c "import secrets; print('VOD_PASS=' + secrets.token_urlsafe(24))" +python3 -c "import secrets; print('VOD_SECRET_KEY=' + secrets.token_hex(32))" ``` --- @@ -176,11 +189,33 @@ RestartSec=10s WantedBy=multi-user.target ``` +### Background Task Worker (`/etc/systemd/system/streamvault-worker.service`) +```ini +[Unit] +Description=StreamVault - Background Task Worker +After=network.target streamvault-webui.service streamvault-recorder.service + +[Service] +Type=simple +User=streamvault +Group=streamvault +WorkingDirectory=/opt/streamvault/app +EnvironmentFile=/etc/streamvault/env +Environment="PYTHONPATH=/opt/streamvault/app" +ExecStart=/opt/streamvault/venv/bin/python3 recorder/worker.py +Restart=always +RestartSec=10s + +[Install] +WantedBy=multi-user.target +``` + Enable and start services: ```bash sudo systemctl daemon-reload sudo systemctl enable --now streamvault-webui sudo systemctl enable --now streamvault-recorder +sudo systemctl enable --now streamvault-worker ``` --- @@ -215,24 +250,82 @@ server { proxy_set_header X-Forwarded-Proto $scheme; } - # 2. X-Accel-Redirect Internal Media Offload + # 2. X-Accel-Redirect media offload. + # WebUI emits /_protected//; archives live directly below + # VOD_BASE_DIR (not below VOD_BASE_DIR/files). location /_protected/ { + internal; + alias /var/lib/streamvault/data/; + sendfile on; + sendfile_max_chunk 1m; + output_buffers 2 1m; + gzip off; + add_header Accept-Ranges bytes always; + add_header Access-Control-Allow-Origin "*" always; + } + + # Legacy archives below VOD_FILES_DIR (default VOD_BASE_DIR/files). + location /_protected_files/ { internal; alias /var/lib/streamvault/data/files/; - mp4; - mp4_buffer_size 1m; - mp4_max_buffer_size 10m; + sendfile on; + sendfile_max_chunk 1m; + output_buffers 2 1m; + gzip off; + add_header Accept-Ranges bytes always; + add_header Access-Control-Allow-Origin "*" always; } - # 3. Live HLS RAM-Disk Offload - location /_hls_live/ { + location /_protected_vodcache/ { + internal; + alias /var/lib/streamvault/data/.vodcache/; + sendfile on; + sendfile_max_chunk 1m; + output_buffers 2 1m; + gzip off; + } + + location /_protected_thumbs/ { + internal; + alias /var/lib/streamvault/data/.thumbs/; + expires 1d; + add_header Cache-Control "public"; + } + + location /_protected_hls/ { + internal; + alias /var/lib/streamvault/data/.hls/; + types { + application/vnd.apple.mpegurl m3u8; + video/mp2t ts; + video/mp4 m4s mp4; + } + default_type application/octet-stream; + add_header Cache-Control "no-cache"; + } + + # 3. Live HLS RAM-Disk offload + location /_protected_hls_live/ { + internal; alias /dev/shm/streamvault_hls/; + open_file_cache off; + types { + application/vnd.apple.mpegurl m3u8; + video/mp2t ts; + video/mp4 m4s mp4; + } + default_type application/octet-stream; add_header Cache-Control "no-cache, no-store, must-revalidate"; - add_header Access-Control-Allow-Origin *; + add_header Access-Control-Allow-Origin "*" always; } } ``` +Keep the aliases in sync with `VOD_BASE_DIR`, `VOD_FILES_DIR`, `VOD_THUMB_DIR`, +`VOD_HLS_DIR` and `LIVE_HLS_DIR` if you changed those environment variables. +The `/_protected_files/` alias is only needed for legacy archives stored below +`VOD_FILES_DIR`; new recordings normally live directly below `VOD_BASE_DIR`. + Enable site and restart Nginx: ```bash sudo ln -sf /etc/nginx/sites-available/streamvault.conf /etc/nginx/sites-enabled/ @@ -241,7 +334,40 @@ sudo nginx -t && sudo systemctl reload nginx --- -## 7. Local Developer Setup +## 7. Cloud / External Storage + +Cloud offloading is optional and requires the `streamvault-worker` daemon +(installed by `install.sh`). Configure it in **Systemsteuerung → Cloud-Speicher** +(`/admin/storage`) or via the console: + +```bash +streamvault storage types +streamvault storage add --type google_drive --name "Drive Archive" +streamvault storage add --type webdav --name "Nextcloud" \ + --set base_url=https://cloud.example.com/remote.php/dav/files/vault/ \ + --set username=vault --set password=app-password +streamvault storage test 1 +streamvault storage sync +streamvault storage objects +streamvault storage restore 42 +``` + +For Google Drive, Dropbox and OneDrive, register this redirect URI in the +provider console and set `VOD_PUBLIC_BASE` to the public HTTPS origin: + +``` +https:///admin/storage/oauth/callback +``` + +Credentials are encrypted with AES-GCM. Set a dedicated +`STORAGE_ENCRYPTION_KEY` (32+ random characters) or rely on the existing strong +`VOD_SECRET_KEY`. Keep the key stable; changing it makes stored credentials +unreadable. Safety defaults are automatic upload **off**, local deletion +**off**, and 14 days local retention. + +--- + +## 8. Local Developer Setup To run the application locally for development: @@ -257,7 +383,10 @@ pip install -r requirements.txt # Configure environment variables cp .env.example .env -# Edit .env with your desired VOD_USER and VOD_PASS credentials +# Edit .env and set VOD_USER, VOD_PASS, and VOD_SECRET_KEY. +# Generate them with: +python -c "import secrets; print('VOD_PASS=' + secrets.token_urlsafe(24))" +python -c "import secrets; print('VOD_SECRET_KEY=' + secrets.token_hex(32))" # Start WebUI server python WEBUI/webui.py --port 8080 diff --git a/README.md b/README.md index d5f5ffc..07f80fe 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,7 @@ This open-core edition is **100% sanitized**: free of private streamer configura - **Kernel-Level Media Offloading**: Integrates with Nginx via `X-Accel-Redirect` (HTTP 206 Partial Content / Range Requests). Python workers release memory immediately while the Linux kernel streams gigabyte-scale MP4 files directly to viewers. - **Live RAM-Disk Transmuxing**: Low-latency HLS restreaming (`/dev/shm`) allows authenticated viewers to watch private live streams without exposing individual client IPs. - **Lossless In-Browser Clip Cutter**: Web-based timeline selector exports shareable MP4 clips in seconds via keyframe-aligned FFmpeg slicing (`-c copy`). +- **Provider-Agnostic Cloud Archive**: Offload VODs to Google Drive, S3-compatible storage (AWS, R2, B2, Wasabi, MinIO, Hetzner), Dropbox, OneDrive or WebDAV. Configure and test providers in the WebUI (`Systemsteuerung → Cloud-Speicher`) or with `streamvault storage`; credentials are encrypted with AES-GCM. - **Granular RBAC Security**: Multi-tier permissions (`Admin`, `Viewer`, `Member`, `Live-only`) with channel-specific allowlists and brute-force protection. --- @@ -96,7 +97,7 @@ cd /tmp/streamvault && sudo ./install.sh The installer automatically: 1. Installs all system dependencies (`ffmpeg`, `python3-venv`, `nginx`, `sqlite3`). 2. Configures the isolated service user and directories (`/opt/streamvault`, `/var/lib/streamvault`). -3. Registers systemd services (`streamvault-webui.service`, `streamvault-recorder.service`). +3. Registers systemd services (`streamvault-webui.service`, `streamvault-recorder.service`, `streamvault-worker.service`). 4. Configures Nginx with `X-Accel-Redirect` kernel video offload. 5. **Registers the global Linux management command** `/usr/local/bin/streamvault`: @@ -125,6 +126,9 @@ cd streamvault cp .env.example .env # 3. Set your production secrets in .env +# Generate values with: +python -c "import secrets; print('VOD_PASS=' + secrets.token_urlsafe(24))" +python -c "import secrets; print('VOD_SECRET_KEY=' + secrets.token_hex(32))" nano .env # 4. Build and run @@ -133,10 +137,69 @@ docker compose up -d --build Access the service at `http://localhost:8090`. +The Compose stack runs the WebUI, recorder and background worker under +supervisord and shares the `hls_live` volume with the Nginx container. The +application directory is bind-mounted at `/app`; recordings therefore live in +the project directory. Set `VOD_PUBLIC_BASE` in `.env` to the public URL used +by browsers (for example `https://recorder.example.com`) so OAuth callbacks and +share links are generated correctly. + For detailed step-by-step production instructions (including bare-metal Ubuntu/Debian deployment, systemd daemons, and SSL setup), see [INSTALL.md](INSTALL.md). --- +## ☁️ Cloud / External Storage + +StreamVault can archive VOD files to external storage without changing playback: +the local archive stays authoritative, uploads run in the background worker, and +local files are only removed after a successful upload and an optional remote +size check. + +Supported providers: + +| Provider | Configuration | +| :--- | :--- | +| **Google Drive** | OAuth Client ID/Secret, then "Mit Google verbinden" in the WebUI | +| **S3-compatible** | Bucket, access key, secret, region, optional endpoint (AWS, R2, B2, Wasabi, MinIO, Hetzner) | +| **Dropbox** | App Key/Secret, then OAuth connection | +| **OneDrive / Microsoft 365** | Client ID/Secret + tenant, then OAuth connection | +| **WebDAV** | Base URL, username, app password (Nextcloud, ownCloud, Hetzner Storage Box) | + +Open **Systemsteuerung → Cloud-Speicher** (`/admin/storage`) to add providers, +test connections, choose the default target, and configure automation. The +same operations are available from the console: + +```bash +streamvault storage types +streamvault storage add --type google_drive --name "Drive Archive" +streamvault storage add --type s3 --name "R2" \ + --set bucket=streamvault --set access_key=... --set secret_key=... \ + --set endpoint_url=https://.r2.cloudflarestorage.com +streamvault storage test 1 +streamvault storage sync +streamvault storage upload /var/lib/streamvault/data/mychannel/video.mp4 +streamvault storage objects +streamvault storage restore 42 +streamvault storage settings --set auto_upload=true +``` + +OAuth providers require this callback URL to be registered with the provider: + +``` +https:///admin/storage/oauth/callback +``` + +Set `VOD_PUBLIC_BASE` to the exact public HTTPS origin if it differs from the +request host. Credentials are encrypted at rest with AES-GCM using +`STORAGE_ENCRYPTION_KEY` (or `VOD_SECRET_KEY` as a fallback). Keep that key +stable and backed up; changing it makes stored credentials unreadable. + +Safety defaults: automatic upload is **off**, deletion of local copies is +**off**, and the local retention period is **14 days**. Enable these settings +deliberately in the WebUI or with `streamvault storage settings`. + +--- + ## ⚙️ Configuration Reference Key variables configurable via `.env`: @@ -149,8 +212,11 @@ Key variables configurable via `.env`: | `VOD_PASS` | *required* | Initial administrator password (production). | | `VOD_SECRET_KEY` | *required* | Secret key for signed session cookies. | | `VOD_BASE_DIR` | `./data` | Root path for databases and video storage. | +| `VOD_PUBLIC_BASE` | *empty* | Public origin for share links and OAuth callbacks. Empty = derive from request. | | `LIVE_HLS_DIR` | `/dev/shm/...` | High-speed RAM-disk path for live streaming. | | `VOD_USE_X_ACCEL` | `true` | Enables Nginx `X-Accel-Redirect` byte-range streaming. | +| `STORAGE_ENCRYPTION_KEY` | *fallback* | Dedicated AES-GCM key for cloud credentials; falls back to `VOD_SECRET_KEY`. | +| `WORKER_STORAGE_THREADS` | `1` | Parallel upload/restore workers in the background task worker. | | `REDIS_URL` | *none* | Optional Redis connection string for caching. | --- diff --git a/WEBUI/templates/admin.html b/WEBUI/templates/admin.html index d04ac56..9f73000 100644 --- a/WEBUI/templates/admin.html +++ b/WEBUI/templates/admin.html @@ -258,6 +258,7 @@

Optionen & Systemsteuerung

Streamer verwalten Benutzer & Gruppen Traffic-Übersicht + Cloud-Speicher diff --git a/WEBUI/templates/storage.html b/WEBUI/templates/storage.html new file mode 100644 index 0000000..d6008ad --- /dev/null +++ b/WEBUI/templates/storage.html @@ -0,0 +1,521 @@ +{% extends 'base.html' %} + +{% block title %}Cloud-Speicher — StreamVault{% endblock %} +{% block page_title %}Cloud-Speicher{% endblock %} +{% block page_subtitle %}VOD-Archiv automatisch oder manuell in externe Speicher auslagern{% endblock %} + +{% block header_actions %} + + + Systemsteuerung + + +{% endblock %} + +{% block head %} + +{% endblock %} + +{% block content %} +
+ {% if msg %}
{{ msg }}
{% endif %} + {% if error %}
{{ error }}
{% endif %} + {% if not encryption_ready %} +
+ Verschlüsselung nicht bereit. + Setze VOD_SECRET_KEY (mindestens 16 Zeichen) oder + STORAGE_ENCRYPTION_KEY, bevor Zugangsdaten gespeichert werden. +
+ {% endif %} + +
+
+
Anbieter
+
{{ providers|length }}
+
{{ providers|selectattr('enabled')|list|length }} aktiv
+
+
+
Archiv-Objekte
+
{{ summary.total }}
+
{{ summary.statuses.get('uploaded', {}).get('count', 0) }} in der Cloud
+
+
+
Cloud-Volumen
+
{{ format_bytes(summary.bytes) }}
+
laut Objekt-Datenbank
+
+
+
Standard-Ziel
+
{{ default_provider.name if default_provider else '–' }}
+
{{ default_provider.type_label if default_provider else 'Kein Anbieter aktiv' }}
+
+
+ +
+
+
+

Speicher-Anbieter

+

Google Drive, S3-kompatible Dienste, Dropbox, OneDrive und WebDAV. Zugangsdaten werden AES-GCM-verschlüsselt gespeichert.

+
+ +
+ + {% if providers %} +
+ {% for p in providers %} +
+
+
+

{{ p.name }}

+
+ {{ p.type_label or p.provider_type }} + {% if p.id == settings.default_provider_id or (not settings.default_provider_id and default_provider and p.id == default_provider.id) %} + Standard + {% endif %} + {% if p.enabled %} + Aktiv + {% else %} + Deaktiviert + {% endif %} + {% if p.last_check_ok %} + Verbindung OK + {% elif p.last_check_ok is sameas false %} + Verbindung fehlerhaft + {% endif %} +
+
+
+ +
+ {% if p.config_error %} +
{{ p.config_error }}
+ {% else %} +
Basis: {{ p.config.get('base_path') or '–' }}
+ {% if p.config.get('folder_id') %}
Ordner-ID: {{ p.config.get('folder_id') }}
{% endif %} + {% if p.config.get('bucket') %}
Bucket: {{ p.config.get('bucket') }}
{% endif %} + {% if p.config.get('endpoint_url') %}
Endpunkt: {{ p.config.get('endpoint_url') }}
{% endif %} + {% if p.config.get('base_url') %}
URL: {{ p.config.get('base_url') }}
{% endif %} + {% if p.last_check_at %}
Letzter Test: {{ format_ts(p.last_check_at) }}
{% endif %} + {% if p.last_check_message %}
Status: {{ p.last_check_message }}
{% endif %} + {% if p.last_error %}
Fehler: {{ p.last_error }}
{% endif %} + {% endif %} +
+ +
+
+ + +
+ {% if p.oauth_supported %} + {{ p.oauth_label }} + {% endif %} + +
+ + +
+
+
+ {% endfor %} +
+ {% else %} +
+ Noch kein Cloud-Speicher eingerichtet. Lege einen Anbieter an oder nutze die Konsole: + streamvault storage add --type google_drive --name "Mein Drive" +
+ {% endif %} +
+ +
+

Automatisierung & Aufbewahrung

+

Standardmäßig wird nichts automatisch hochgeladen oder gelöscht. Aktiviere die Funktionen bewusst.

+
+ +
+
+ + +
+
+ + + 0 = sofort löschen, sobald die Cloud-Kopie verifiziert ist. +
+
+ + + 0 = kein automatischer Sync. Wird nur bei aktivem Auto-Upload genutzt. +
+
+ + +
+
+ + +
+
+ +
+ + + + + +
+ +
+ +
+
+
+ +
+

Manueller Sync

+

Der Recorder-Worker führt den Abgleich aus. Bereits hochgeladene unveränderte Dateien werden übersprungen.

+
+ +
+ + +
+
+ + +
+ + +
+
+ +
+

Archiv-Objekte

+

Status, Fortschritt und Aktionen der zuletzt synchronisierten Dateien. „Vergessen“ entfernt nur den Datenbankeintrag.

+ {% if objects %} +
+ + + + + + + + + + + + + + + {% for o in objects %} + + + + + + + + + + + {% endfor %} + +
IDStatusDateiAnbieterGrößeLokalAktualisiertAktionen
#{{ o.id }} + {{ o.status }} + {% if o.progress and o.status in ('uploading','pending') %}
{{ '%.0f'|format(o.progress) }} %
{% endif %} + 		+
{{ o.filename or o.local_path }}
+
{{ o.local_path }}
+
→ {{ o.remote_path }}
+ {% if o.last_error %}
{{ o.last_error }}
{% endif %} + 		{{ provider_names.get(o.provider_id, '–') }}{{ format_bytes(o.remote_size or o.local_size) }}{% if o.local_exists and not o.local_deleted_at %}Vorhanden{% else %}Nur Cloud{% endif %}{{ format_ts(o.updated_at) }} +
+ {% if o.local_exists and not o.local_deleted_at and o.status != 'uploading' %} +
+ + +
+ {% endif %} + {% if o.status in ('uploaded','restored') and o.local_exists and not o.local_deleted_at %} +
+ + +
+ {% endif %} + {% if o.status in ('uploaded','restored') and not o.local_exists and not o.remote_deleted_at %} +
+ + +
+ {% endif %} + {% if not o.remote_deleted_at and not o.local_deleted_at and o.local_exists and o.status in ('uploaded','restored') %} +
+ + +
+ {% endif %} +
+ + +
+
+
+
+ {% else %} +
Noch keine Archiv-Objekte. Starte einen Sync oder lade eine Aufnahme hoch.
+ {% endif %} +
+ +
+

OAuth-Hinweis

+

+ Für Google Drive, Dropbox und OneDrive muss diese Callback-URL beim Anbieter registriert sein: + {{ oauth_redirect_uri }}
+ Setze bei Bedarf VOD_PUBLIC_BASE auf deine öffentliche HTTPS-Adresse, damit die URL exakt passt. +

+
+
+ + + + + + +{% endblock %} diff --git a/WEBUI/webui.py b/WEBUI/webui.py index 771dbb1..4abcdc2 100644 --- a/WEBUI/webui.py +++ b/WEBUI/webui.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import bisect +import glob import json import logging import math @@ -103,19 +104,30 @@ def detect_x_accel(): if os.environ.get("VOD_USE_X_ACCEL", "false").lower() in ("true", "1", "yes"): return True - nginx_conf = "/etc/nginx/conf.d/streamrecorder.conf" - if os.path.isfile(nginx_conf): + candidates = [ + "/etc/nginx/conf.d/streamrecorder.conf", + "/etc/nginx/conf.d/default.conf", + "/etc/nginx/sites-enabled/streamvault.conf", + ] + try: + candidates.extend(glob.glob("/etc/nginx/sites-enabled/*")) + except OSError: + pass + for nginx_conf in dict.fromkeys(candidates): + if not os.path.isfile(nginx_conf): + continue try: with open(nginx_conf, "r", encoding="utf-8") as f: content = f.read() - if "/_protected/" in content and "proxy_pass http://127.0.0.1:8081" in content: - return True - except Exception: - pass + except OSError: + continue + if "/_protected/" in content and "proxy_pass" in content: + return True return False USE_X_ACCEL = detect_x_accel() X_ACCEL_PREFIX = os.environ.get("VOD_X_ACCEL_PREFIX", "/_protected") +FILES_ACCEL_PREFIX = os.environ.get("VOD_FILES_ACCEL_PREFIX", "/_protected_files") VOD_CACHE_ACCEL_PREFIX = os.environ.get("VOD_CACHE_ACCEL_PREFIX", "/_protected_vodcache") HLS_TIME = int(os.environ.get("VOD_HLS_TIME", "4")) HLS_GENERATE_TIMEOUT_SEC = int(os.environ.get("VOD_HLS_GENERATE_TIMEOUT_SEC", "7200")) @@ -271,14 +283,17 @@ def load_html_template(name, fallback=""): METRICS_TOKEN = os.environ.get("METRICS_TOKEN", "").strip() def load_secret_key(): - if config.SECRET_KEY != "change-me": - return config.SECRET_KEY + from core.storage.crypto import is_weak_secret + + configured = str(config.SECRET_KEY or "").strip() + if configured and not is_weak_secret(configured): + return configured key_path = os.path.join(config.META_DIR, "secret_key.txt") try: if os.path.isfile(key_path): with open(key_path, "r", encoding="utf-8") as f: key = f.read().strip() - if key: + if key and not is_weak_secret(key): try: os.chmod(key_path, 0o600) except OSError: @@ -289,8 +304,20 @@ def load_secret_key(): os.makedirs(config.META_DIR, exist_ok=True) key = secrets.token_hex(32) try: - with open(key_path, "w", encoding="utf-8") as f: - f.write(key) + fd = os.open( + key_path, + os.O_WRONLY | os.O_CREAT | os.O_TRUNC, + 0o600, + ) + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + f.write(key) + except Exception: + try: + os.close(fd) + except OSError: + pass + raise os.chmod(key_path, 0o600) except Exception: pass @@ -668,6 +695,7 @@ def authenticate_user(username, password): ADMIN_TEMPLATE = load_html_template("admin.html") ADMIN_TRAFFIC_TEMPLATE = load_html_template("admin_traffic.html") ADMIN_USERS_TEMPLATE = load_html_template("admin_users.html") +STORAGE_TEMPLATE = load_html_template("storage.html") MAIL_ADMIN_TEMPLATE = None STALK_ADMIN_TEMPLATE = None STALK_DETAIL_TEMPLATE = None @@ -1573,13 +1601,31 @@ def streamer_dir(slug): return p -def media_accel_path(streamer, filename, *, clip=False): +def _path_is_under(path, root): + path_value = str(path or "").strip() + root_value = str(root or "").strip() + if not path_value or not root_value: + return False + try: + path_real = os.path.realpath(path_value) + root_real = os.path.realpath(root_value) + common = os.path.commonpath([path_real, root_real]) + return os.path.normcase(common) == os.path.normcase(root_real) + except (OSError, ValueError): + return False + + +def media_accel_path(streamer, filename, *, clip=False, file_path=None): """X-Accel path using the on-disk archive slug (aliases map correctly).""" archive = archive_name_for(streamer) or normalize_channel_name(streamer) or str(streamer or "").strip() name = os.path.basename(str(filename or "")) if not archive or not name: return None + if file_path is None: + file_path = resolve_media_path(streamer, filename, "clip" if clip else "vod") prefix = X_ACCEL_PREFIX.rstrip("/") + if file_path and _path_is_under(file_path, getattr(config, "FILES_DIR", "")): + prefix = FILES_ACCEL_PREFIX.rstrip("/") if clip: return f"{prefix}/{urllib.parse.quote(archive)}/{CLIP_SUBDIR}/{urllib.parse.quote(name)}" return f"{prefix}/{urllib.parse.quote(archive)}/{urllib.parse.quote(name)}" @@ -7609,6 +7655,317 @@ def admin_traffic_view(): return "Fehler beim Laden der Traffic-Übersicht.", 500 +# ============ CLOUD STORAGE (Google Drive, S3, Dropbox, OneDrive, WebDAV) ============ + +def _storage_service(): + from core.storage import service as storage_service + + return storage_service + + +def _storage_error(exc): + from core.storage.base import mask_error + + return mask_error(exc) + + +def _storage_redirect(msg=None, error=None): + from core.storage.base import mask_text + + params = {} + if msg: + params["msg"] = mask_text(msg) + if error: + params["error"] = mask_text(error) + return redirect(url_for("admin_storage_view", **params)) + + +def _storage_redirect_uri(): + return absolute_public_url(url_for("storage_oauth_callback")) + + +@app.route("/admin/storage") +@admin_required +@limiter.limit("120 per hour") +def admin_storage_view(): + try: + svc = _storage_service() + context = svc.admin_context() + provider_names = {row["id"]: row.get("name") or "" for row in context["providers"]} + return render_template_string( + STORAGE_TEMPLATE, + msg=request.args.get("msg"), + error=request.args.get("error"), + provider_names=provider_names, + format_bytes=svc.format_bytes, + format_ts=svc.format_timestamp, + oauth_redirect_uri=_storage_redirect_uri(), + **context, + ) + except Exception as exc: + logging.error("Admin storage error: %s", _storage_error(exc)) + return "Fehler beim Laden der Cloud-Speicher-Verwaltung.", 500 + + +@app.route("/admin/storage/settings", methods=["POST"]) +@admin_required +@limiter.limit("60 per hour") +def admin_storage_settings(): + svc = _storage_service() + values = { + "auto_upload": request.form.get("auto_upload") == "1", + "delete_local_after_upload": request.form.get("delete_local_after_upload") == "1", + "verify_before_local_delete": request.form.get("verify_before_local_delete") == "1", + "include_chat_logs": request.form.get("include_chat_logs") == "1", + "include_thumbnails": request.form.get("include_thumbnails") == "1", + "local_retention_days": request.form.get("local_retention_days", "14"), + "auto_sync_interval_minutes": request.form.get("auto_sync_interval_minutes", "10"), + "cleanup_interval_minutes": request.form.get("cleanup_interval_minutes", "360"), + "max_sync_files": request.form.get("max_sync_files", "500"), + "default_provider_id": request.form.get("default_provider_id") or None, + } + try: + svc.save_settings(values) + return _storage_redirect(msg="Einstellungen gespeichert.") + except Exception as exc: + return _storage_redirect(error=_storage_error(exc)) + + +@app.route("/admin/storage/providers/add", methods=["POST"]) +@admin_required +@limiter.limit("30 per hour") +def admin_storage_provider_add(): + svc = _storage_service() + from core.storage.registry import get_provider_class + + try: + type_id = str(request.form.get("provider_type") or "").strip() + cls = get_provider_class(type_id) + values = {} + for field in cls.config_fields: + raw = request.form.get(f"cfg_{field.key}", "") + if field.secret and not str(raw or "").strip(): + continue + values[field.key] = raw + provider_id = svc.create_provider( + request.form.get("name", ""), + type_id, + values, + enabled=request.form.get("enabled") == "1", + ) + return _storage_redirect( + msg=f"Anbieter #{provider_id} angelegt. Bitte Verbindung testen." + ) + except Exception as exc: + return _storage_redirect(error=_storage_error(exc)) + + +@app.route("/admin/storage/providers//update", methods=["POST"]) +@admin_required +@limiter.limit("60 per hour") +def admin_storage_provider_update(provider_id): + svc = _storage_service() + from core.storage.registry import get_provider_class + + try: + row = svc.get_provider_record(provider_id) + if not row: + raise ValueError("Anbieter nicht gefunden.") + cls = get_provider_class(row["provider_type"]) + values = {} + for field in cls.config_fields: + values[field.key] = request.form.get(f"cfg_{field.key}", "") + svc.update_provider( + provider_id, + name=request.form.get("name", row["name"]), + values=values, + enabled=request.form.get("enabled") == "1", + ) + return _storage_redirect(msg=f"Anbieter #{provider_id} gespeichert.") + except Exception as exc: + return _storage_redirect(error=_storage_error(exc)) + + +@app.route("/admin/storage/providers//delete", methods=["POST"]) +@admin_required +@limiter.limit("20 per hour") +def admin_storage_provider_delete(provider_id): + svc = _storage_service() + try: + svc.delete_provider(provider_id) + return _storage_redirect( + msg=f"Anbieter #{provider_id} entfernt. Cloud-Dateien bleiben erhalten." + ) + except Exception as exc: + return _storage_redirect(error=_storage_error(exc)) + + +@app.route("/admin/storage/providers//test", methods=["POST"]) +@admin_required +@limiter.limit("20 per hour") +def admin_storage_provider_test(provider_id): + svc = _storage_service() + try: + result = svc.test_provider(provider_id) + if result.get("ok"): + return _storage_redirect(msg=result.get("message")) + return _storage_redirect(error=result.get("message")) + except Exception as exc: + return _storage_redirect(error=_storage_error(exc)) + + +@app.route("/admin/storage/providers//check", methods=["POST"]) +@admin_required +@limiter.limit("20 per hour") +def admin_storage_provider_check(provider_id): + svc = _storage_service() + try: + task_id = svc.enqueue_check(provider_id) + return _storage_redirect(msg=f"Verbindungstest eingeordnet (Task #{task_id}).") + except Exception as exc: + return _storage_redirect(error=_storage_error(exc)) + + +@app.route("/admin/storage/providers//oauth/start") +@admin_required +@limiter.limit("30 per hour") +def admin_storage_oauth_start(provider_id): + svc = _storage_service() + try: + return redirect(svc.oauth_authorize_url(provider_id, _storage_redirect_uri())) + except Exception as exc: + return _storage_redirect(error=_storage_error(exc)) + + +@app.route("/admin/storage/oauth/callback") +@admin_required +@limiter.limit("30 per hour") +def storage_oauth_callback(): + svc = _storage_service() + if request.args.get("error"): + return _storage_redirect(error=f"Anbieter meldete: {request.args.get('error')}") + try: + provider_id = svc.verify_oauth_state(request.args.get("state", "")) + code = str(request.args.get("code") or "").strip() + if not code: + raise ValueError("OAuth-Code fehlt.") + svc.oauth_complete(provider_id, code, _storage_redirect_uri()) + result = svc.test_provider(provider_id) + if result.get("ok"): + return _storage_redirect(msg=result.get("message")) + return _storage_redirect(error=result.get("message")) + except Exception as exc: + return _storage_redirect(error=_storage_error(exc)) + + +@app.route("/admin/storage/sync", methods=["POST"]) +@admin_required +@limiter.limit("20 per hour") +def admin_storage_sync(): + svc = _storage_service() + try: + provider_id = request.form.get("provider_id") or None + streamer = str(request.form.get("streamer") or "").strip() or None + task_id = svc.enqueue_sync(provider_id, streamer) + if task_id: + return _storage_redirect(msg=f"Cloud-Sync eingeordnet (Task #{task_id}).") + return _storage_redirect(msg="Es läuft bereits ein Cloud-Sync.") + except Exception as exc: + return _storage_redirect(error=_storage_error(exc)) + + +@app.route("/admin/storage/cleanup", methods=["POST"]) +@admin_required +@limiter.limit("20 per hour") +def admin_storage_cleanup(): + svc = _storage_service() + try: + provider_id = request.form.get("provider_id") or None + task_id = svc.enqueue_cleanup(provider_id) + if task_id: + return _storage_redirect(msg=f"Aufräumung eingeordnet (Task #{task_id}).") + return _storage_redirect(msg="Es läuft bereits eine Aufräumung.") + except Exception as exc: + return _storage_redirect(error=_storage_error(exc)) + + +@app.route("/admin/storage/objects//upload", methods=["POST"]) +@admin_required +@limiter.limit("60 per hour") +def admin_storage_object_upload(object_id): + svc = _storage_service() + try: + obj = database.get_storage_object(object_id) + if not obj: + raise ValueError("Archiv-Objekt nicht gefunden.") + if obj.get("local_deleted_at"): + raise ValueError("Die lokale Datei wurde bereits entfernt. Bitte zuerst wiederherstellen.") + result = svc.enqueue_upload( + obj["local_path"], obj["provider_id"], force=True + ) + if result.get("task_id"): + return _storage_redirect(msg=f"Upload eingeordnet (Task #{result['task_id']}).") + return _storage_redirect(msg="Es läuft bereits ein Upload für diese Datei.") + except Exception as exc: + return _storage_redirect(error=_storage_error(exc)) + + +@app.route("/admin/storage/objects//restore", methods=["POST"]) +@admin_required +@limiter.limit("30 per hour") +def admin_storage_object_restore(object_id): + svc = _storage_service() + try: + task_id = svc.enqueue_restore(object_id) + if task_id: + return _storage_redirect(msg=f"Wiederherstellung eingeordnet (Task #{task_id}).") + return _storage_redirect(msg="Es läuft bereits eine Wiederherstellung.") + except Exception as exc: + return _storage_redirect(error=_storage_error(exc)) + + +@app.route("/admin/storage/objects//delete-remote", methods=["POST"]) +@admin_required +@limiter.limit("30 per hour") +def admin_storage_object_delete_remote(object_id): + svc = _storage_service() + try: + task_id = svc.enqueue_delete(object_id) + if task_id: + return _storage_redirect(msg=f"Cloud-Löschung eingeordnet (Task #{task_id}).") + return _storage_redirect(msg="Es läuft bereits eine Cloud-Löschung.") + except Exception as exc: + return _storage_redirect(error=_storage_error(exc)) + + +@app.route("/admin/storage/objects//delete-local", methods=["POST"]) +@admin_required +@limiter.limit("30 per hour") +def admin_storage_object_delete_local(object_id): + svc = _storage_service() + try: + ok, message = svc.delete_local_copy(object_id) + if ok: + return _storage_redirect(msg=message) + return _storage_redirect(error=message) + except Exception as exc: + return _storage_redirect(error=_storage_error(exc)) + + +@app.route("/admin/storage/objects//forget", methods=["POST"]) +@admin_required +@limiter.limit("30 per hour") +def admin_storage_object_forget(object_id): + try: + if database.delete_storage_object(object_id): + return _storage_redirect( + msg=f"Objekt #{object_id} aus der Datenbank entfernt. Dateien bleiben erhalten." + ) + return _storage_redirect(error="Archiv-Objekt nicht gefunden.") + except Exception as exc: + return _storage_redirect(error=_storage_error(exc)) + + @app.route("/admin/unban/", methods=["POST"]) @admin_required def admin_unban(ip): @@ -9828,7 +10185,7 @@ def media(streamer, filename): pass file_path = resolve_media_path(streamer, filename, "vod") if file_path: - accel_path = media_accel_path(streamer, filename) if USE_X_ACCEL else None + accel_path = media_accel_path(streamer, filename, file_path=file_path) if USE_X_ACCEL else None return send_mp4_with_range(file_path, accel_path=accel_path) abort(404) @@ -9925,7 +10282,7 @@ def download(streamer, filename): abort_hidden_streamer(streamer) file_path = resolve_media_path(streamer, filename, "vod") if file_path: - accel_path = media_accel_path(streamer, filename) if USE_X_ACCEL else None + accel_path = media_accel_path(streamer, filename, file_path=file_path) if USE_X_ACCEL else None return send_mp4_with_range(file_path, accel_path=accel_path, attachment_name=filename) abort(404) @@ -9942,7 +10299,7 @@ def clip_media(streamer, filename): if not file_path: abort(404) if USE_X_ACCEL: - accel_path = media_accel_path(streamer, filename, clip=True) + accel_path = media_accel_path(streamer, filename, clip=True, file_path=file_path) return send_mp4_with_range(file_path, accel_path=accel_path) return send_mp4_with_range(file_path) @@ -9957,7 +10314,7 @@ def clip_download(streamer, filename): file_path = resolve_media_path(streamer, filename, "clip") if not file_path: abort(404) - accel_path = media_accel_path(streamer, filename, clip=True) if USE_X_ACCEL else None + accel_path = media_accel_path(streamer, filename, clip=True, file_path=file_path) if USE_X_ACCEL else None return send_mp4_with_range(file_path, accel_path=accel_path, attachment_name=filename) @@ -10559,7 +10916,7 @@ def public_media(token): abort(404) accel_path = None if USE_X_ACCEL: - accel_path = media_accel_path(streamer, filename, clip=(route == "clip")) + accel_path = media_accel_path(streamer, filename, clip=(route == "clip"), file_path=file_path) return send_mp4_with_range(file_path, accel_path=accel_path) @@ -10576,7 +10933,7 @@ def public_download(token): abort(404) accel_path = None if USE_X_ACCEL: - accel_path = media_accel_path(streamer, filename, clip=(route == "clip")) + accel_path = media_accel_path(streamer, filename, clip=(route == "clip"), file_path=file_path) return send_mp4_with_range(file_path, accel_path=accel_path, attachment_name=filename) diff --git a/bin/streamvault b/bin/streamvault index 008fe46..24af394 100644 --- a/bin/streamvault +++ b/bin/streamvault @@ -8,6 +8,7 @@ set -euo pipefail APP_DIR="${STREAMVAULT_DIR:-/opt/streamvault/app}" VENV_DIR="${STREAMVAULT_VENV:-/opt/streamvault/venv}" ENV_FILE="${STREAMVAULT_ENV:-/etc/streamvault/env}" +SERVICE_USER="${STREAMVAULT_USER:-streamvault}" # Visual helpers RED='\033[0;31m' @@ -33,17 +34,21 @@ usage() { echo -e "${BOLD}Usage:${NC} streamvault [COMMAND] [OPTIONS]" echo "" echo -e "${BOLD}Service Management:${NC}" - echo -e " ${GREEN}start${NC} Start WebUI and Recorder daemons" + echo -e " ${GREEN}start${NC} Start WebUI, Recorder and Worker daemons" echo -e " ${GREEN}stop${NC} Stop all running daemons" - echo -e " ${GREEN}restart${NC} Restart WebUI and Recorder daemons" + echo -e " ${GREEN}restart${NC} Restart WebUI, Recorder and Worker daemons" echo -e " ${GREEN}status${NC} Display health, system load, and systemd status" - echo -e " ${GREEN}logs${NC} [service] Follow live logs (webui, recorder, or all)" + echo -e " ${GREEN}logs${NC} [service] Follow live logs (webui, recorder, worker, or all)" echo "" echo -e "${BOLD}Operations:${NC}" echo -e " ${GREEN}update${NC} Pull latest Git commit, upgrade deps, and restart" echo -e " ${GREEN}config${NC} Inspect or edit active environment (/etc/streamvault/env)" echo -e " ${GREEN}check${NC} Validate Python syntax, Nginx, and database schemas" echo "" + echo -e "${BOLD}Cloud Storage:${NC}" + echo -e " ${GREEN}storage${NC} [args] Manage external storage (list, add, test, sync, ...)" + echo -e " Example: streamvault storage add --type google_drive --name 'Drive'" + echo "" exit 1 } @@ -54,31 +59,40 @@ require_root() { fi } +load_env() { + if [[ -f "${ENV_FILE}" ]]; then + set -a + # shellcheck disable=SC1090 + source "${ENV_FILE}" + set +a + fi +} + cmd_start() { require_root echo -e "${BLUE}[*] Starting StreamVault daemons...${NC}" - systemctl start streamvault-webui.service streamvault-recorder.service + systemctl start streamvault-webui.service streamvault-recorder.service streamvault-worker.service echo -e "${GREEN}[OK] Services started successfully.${NC}" } cmd_stop() { require_root echo -e "${BLUE}[*] Stopping StreamVault daemons...${NC}" - systemctl stop streamvault-recorder.service streamvault-webui.service + systemctl stop streamvault-worker.service streamvault-recorder.service streamvault-webui.service echo -e "${GREEN}[OK] Services stopped.${NC}" } cmd_restart() { require_root echo -e "${BLUE}[*] Restarting StreamVault daemons...${NC}" - systemctl restart streamvault-webui.service streamvault-recorder.service + systemctl restart streamvault-webui.service streamvault-recorder.service streamvault-worker.service echo -e "${GREEN}[OK] Restart complete.${NC}" } cmd_status() { header echo -e "${BOLD}Daemons Status:${NC}" - systemctl status streamvault-webui.service streamvault-recorder.service --no-pager -l || true + systemctl status streamvault-webui.service streamvault-recorder.service streamvault-worker.service --no-pager -l || true echo "" echo -e "${BOLD}Storage & RAM-Disk:${NC}" df -h /dev/shm | grep -v "udev" || true @@ -94,11 +108,14 @@ cmd_logs() { recorder) journalctl -u streamvault-recorder.service -f -n 50 ;; + worker) + journalctl -u streamvault-worker.service -f -n 50 + ;; all) - journalctl -u streamvault-webui.service -u streamvault-recorder.service -f -n 50 + journalctl -u streamvault-webui.service -u streamvault-recorder.service -u streamvault-worker.service -f -n 50 ;; *) - echo -e "${RED}[ERROR] Unknown service: $target (use webui, recorder, or all)${NC}" + echo -e "${RED}[ERROR] Unknown service: $target (use webui, recorder, worker, or all)${NC}" exit 1 ;; esac @@ -110,7 +127,7 @@ cmd_update() { cd "${APP_DIR}" git pull --ff-only "${VENV_DIR}/bin/pip" install --upgrade -r requirements.txt - systemctl restart streamvault-webui.service streamvault-recorder.service + systemctl restart streamvault-webui.service streamvault-recorder.service streamvault-worker.service echo -e "${GREEN}[OK] Update completed and daemons restarted.${NC}" } @@ -124,9 +141,10 @@ cmd_config() { } cmd_check() { + load_env echo -e "${BLUE}[*] Validating StreamVault integrity...${NC}" cd "${APP_DIR}" - "${VENV_DIR}/bin/python3" -m py_compile core/*.py WEBUI/webui.py recorder/*.py + "${VENV_DIR}/bin/python3" -m compileall -q core recorder WEBUI echo -e "${GREEN}[OK] Python syntax: valid.${NC}" if command -v nginx >/dev/null 2>&1; then nginx -t @@ -134,6 +152,22 @@ cmd_check() { fi } +cmd_storage() { + load_env + cd "${APP_DIR}" + # The WebUI/worker own the SQLite database. Running storage commands as + # root can leave root-owned WAL files that the service user cannot write. + if [[ $EUID -eq 0 ]] && id "${SERVICE_USER}" >/dev/null 2>&1; then + if command -v runuser >/dev/null 2>&1; then + runuser -u "${SERVICE_USER}" -- "${VENV_DIR}/bin/python3" -m core.storage.cli "$@" + else + sudo -u "${SERVICE_USER}" -- "${VENV_DIR}/bin/python3" -m core.storage.cli "$@" + fi + else + "${VENV_DIR}/bin/python3" -m core.storage.cli "$@" + fi +} + # Subcommand dispatch case "${1:-}" in start) @@ -160,6 +194,10 @@ case "${1:-}" in check) cmd_check ;; + storage) + shift + cmd_storage "$@" + ;; help|--help|-h|"") usage ;; diff --git a/core/database.py b/core/database.py index 3bc6837..77b9ee7 100644 --- a/core/database.py +++ b/core/database.py @@ -363,6 +363,73 @@ def init_db(): ) """ ) + # External storage providers. Credentials are AES-GCM encrypted by + # core.storage.crypto before they reach this table. + c.execute( + """ + CREATE TABLE IF NOT EXISTS storage_providers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + provider_type TEXT NOT NULL, + config TEXT NOT NULL DEFAULT '', + enabled INTEGER NOT NULL DEFAULT 1, + created_at REAL NOT NULL, + updated_at REAL NOT NULL, + last_check_at REAL, + last_check_ok INTEGER, + last_check_message TEXT, + last_error TEXT + ) + """ + ) + c.execute( + "CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_providers_name " + "ON storage_providers(name COLLATE NOCASE)" + ) + c.execute( + """ + CREATE TABLE IF NOT EXISTS storage_objects ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + provider_id INTEGER NOT NULL, + local_path TEXT NOT NULL, + remote_path TEXT NOT NULL, + streamer TEXT DEFAULT '', + filename TEXT DEFAULT '', + kind TEXT DEFAULT 'vod', + status TEXT NOT NULL DEFAULT 'pending', + local_size INTEGER DEFAULT 0, + local_mtime REAL DEFAULT 0, + remote_size INTEGER DEFAULT 0, + remote_modified TEXT DEFAULT '', + remote_checksum TEXT DEFAULT '', + progress REAL DEFAULT 0, + attempts INTEGER DEFAULT 0, + last_error TEXT DEFAULT '', + created_at REAL NOT NULL, + updated_at REAL NOT NULL, + uploaded_at REAL, + local_deleted_at REAL, + remote_deleted_at REAL, + FOREIGN KEY(provider_id) REFERENCES storage_providers(id) ON DELETE CASCADE + ) + """ + ) + c.execute( + "CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_objects_provider_local " + "ON storage_objects(provider_id, local_path)" + ) + c.execute( + "CREATE INDEX IF NOT EXISTS idx_storage_objects_status " + "ON storage_objects(status, updated_at)" + ) + c.execute( + "CREATE INDEX IF NOT EXISTS idx_storage_objects_media " + "ON storage_objects(streamer COLLATE NOCASE, filename COLLATE NOCASE)" + ) + c.execute( + "CREATE INDEX IF NOT EXISTS idx_storage_objects_provider " + "ON storage_objects(provider_id, status)" + ) # Performance & Concurrency Indexes (users, logs, channels, traffic) c.execute("CREATE INDEX IF NOT EXISTS idx_users_group_id ON users(group_id)") c.execute("CREATE INDEX IF NOT EXISTS idx_users_active ON users(active)") @@ -910,7 +977,7 @@ def set_channel_record(name, enabled: bool): conn.close() -def has_active_task(task_type, streamer=None, filename=None): +def has_active_task(task_type, streamer=None, filename=None, *, object_id=None, provider_id=None): conn = get_db() try: rows = conn.execute( @@ -918,7 +985,7 @@ def has_active_task(task_type, streamer=None, filename=None): SELECT payload FROM tasks WHERE type=? AND status IN ('pending', 'running') ORDER BY id DESC - LIMIT 200 + LIMIT 1000 """, (str(task_type or ""),), ).fetchall() @@ -926,6 +993,18 @@ def has_active_task(task_type, streamer=None, filename=None): conn.close() want_streamer = str(streamer or "").strip() want_filename = str(filename or "").strip() + want_object = None + if object_id is not None: + try: + want_object = int(object_id) + except (TypeError, ValueError): + want_object = None + want_provider = None + if provider_id is not None: + try: + want_provider = int(provider_id) + except (TypeError, ValueError): + want_provider = None for row in rows: try: payload = json.loads(row["payload"] or "{}") @@ -933,6 +1012,19 @@ def has_active_task(task_type, streamer=None, filename=None): payload = {} if not isinstance(payload, dict): payload = {} + if want_provider is not None: + try: + if int(payload.get("provider_id")) != want_provider: + continue + except (TypeError, ValueError): + continue + if want_object is not None: + try: + if int(payload.get("object_id")) == want_object: + return True + except (TypeError, ValueError): + pass + continue if want_streamer and str(payload.get("streamer") or "").strip() != want_streamer: continue if want_filename and str(payload.get("filename") or "").strip() != want_filename: @@ -2039,3 +2131,432 @@ def get_user_permissions(username: str) -> dict: "allowed_channels": [], "disabled_channels": [], } + + +# --------------------------------------------------------------------------- +# External storage providers and archive objects +# --------------------------------------------------------------------------- + +def _storage_provider_row(row): + if not row: + return None + data = dict(row) + data["enabled"] = bool(data.get("enabled")) + data["last_check_ok"] = ( + None if data.get("last_check_ok") is None else bool(data.get("last_check_ok")) + ) + return data + + +def list_storage_providers(): + conn = get_db() + try: + rows = conn.execute( + "SELECT * FROM storage_providers ORDER BY name COLLATE NOCASE ASC" + ).fetchall() + return [_storage_provider_row(row) for row in rows] + finally: + conn.close() + + +def get_storage_provider(provider_id): + try: + pid = int(provider_id) + except (TypeError, ValueError): + return None + conn = get_db() + try: + row = conn.execute( + "SELECT * FROM storage_providers WHERE id = ?", + (pid,), + ).fetchone() + return _storage_provider_row(row) + finally: + conn.close() + + +def create_storage_provider(name, provider_type, config, enabled=True): + now = time.time() + conn = get_db() + try: + cur = conn.execute( + """ + INSERT INTO storage_providers + (name, provider_type, config, enabled, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + """, + ( + str(name or "").strip(), + str(provider_type or "").strip(), + str(config or ""), + 1 if enabled else 0, + now, + now, + ), + ) + conn.commit() + return cur.lastrowid + finally: + conn.close() + + +def update_storage_provider( + provider_id, + *, + name=None, + provider_type=None, + config=None, + enabled=None, +): + try: + pid = int(provider_id) + except (TypeError, ValueError): + return False + fields = [] + values = [] + if name is not None: + fields.append("name = ?") + values.append(str(name or "").strip()) + if provider_type is not None: + fields.append("provider_type = ?") + values.append(str(provider_type or "").strip()) + if config is not None: + fields.append("config = ?") + values.append(str(config or "")) + if enabled is not None: + fields.append("enabled = ?") + values.append(1 if enabled else 0) + if not fields: + return False + fields.append("updated_at = ?") + values.append(time.time()) + values.append(pid) + conn = get_db() + try: + cur = conn.execute( + f"UPDATE storage_providers SET {', '.join(fields)} WHERE id = ?", + tuple(values), + ) + conn.commit() + return bool(cur.rowcount) + finally: + conn.close() + + +def delete_storage_provider(provider_id): + try: + pid = int(provider_id) + except (TypeError, ValueError): + return False + conn = get_db() + try: + conn.execute("BEGIN IMMEDIATE") + conn.execute("DELETE FROM storage_objects WHERE provider_id = ?", (pid,)) + cur = conn.execute("DELETE FROM storage_providers WHERE id = ?", (pid,)) + conn.commit() + return bool(cur.rowcount) + except Exception: + conn.rollback() + raise + finally: + conn.close() + + +def set_storage_provider_check(provider_id, ok, message=""): + try: + pid = int(provider_id) + except (TypeError, ValueError): + return False + conn = get_db() + try: + conn.execute( + """ + UPDATE storage_providers + SET last_check_at = ?, last_check_ok = ?, last_check_message = ?, updated_at = ? + WHERE id = ? + """, + (time.time(), 1 if ok else 0, str(message or "")[:500], time.time(), pid), + ) + conn.commit() + return True + finally: + conn.close() + + +def set_storage_provider_error(provider_id, error=""): + try: + pid = int(provider_id) + except (TypeError, ValueError): + return False + conn = get_db() + try: + conn.execute( + "UPDATE storage_providers SET last_error = ?, updated_at = ? WHERE id = ?", + (str(error or "")[:500], time.time(), pid), + ) + conn.commit() + return True + finally: + conn.close() + + +def _storage_object_row(row): + return dict(row) if row else None + + +def get_storage_object(object_id): + try: + oid = int(object_id) + except (TypeError, ValueError): + return None + conn = get_db() + try: + row = conn.execute( + "SELECT * FROM storage_objects WHERE id = ?", + (oid,), + ).fetchone() + return _storage_object_row(row) + finally: + conn.close() + + +def find_storage_object(provider_id, local_path): + try: + pid = int(provider_id) + except (TypeError, ValueError): + return None + conn = get_db() + try: + row = conn.execute( + "SELECT * FROM storage_objects WHERE provider_id = ? AND local_path = ?", + (pid, str(local_path or "")), + ).fetchone() + return _storage_object_row(row) + finally: + conn.close() + + +def upsert_storage_object( + provider_id, + local_path, + remote_path, + *, + streamer="", + filename="", + kind="vod", + local_size=0, + local_mtime=0, + status="pending", +): + try: + pid = int(provider_id) + except (TypeError, ValueError): + raise ValueError("Ungültige Provider-ID.") + now = time.time() + conn = get_db() + try: + conn.execute( + """ + INSERT INTO storage_objects + (provider_id, local_path, remote_path, streamer, filename, kind, + status, local_size, local_mtime, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(provider_id, local_path) DO UPDATE SET + remote_path = excluded.remote_path, + streamer = excluded.streamer, + filename = excluded.filename, + kind = excluded.kind, + updated_at = excluded.updated_at + """, + ( + pid, + str(local_path or ""), + str(remote_path or ""), + str(streamer or ""), + str(filename or ""), + str(kind or "vod"), + str(status or "pending"), + max(0, int(local_size or 0)), + float(local_mtime or 0), + now, + now, + ), + ) + conn.commit() + row = conn.execute( + "SELECT * FROM storage_objects WHERE provider_id = ? AND local_path = ?", + (pid, str(local_path or "")), + ).fetchone() + return _storage_object_row(row) + finally: + conn.close() + + +_STORAGE_OBJECT_WRITABLE = { + "remote_path", + "streamer", + "filename", + "kind", + "status", + "local_size", + "local_mtime", + "remote_size", + "remote_modified", + "remote_checksum", + "progress", + "attempts", + "last_error", + "uploaded_at", + "local_deleted_at", + "remote_deleted_at", +} + + +def update_storage_object(object_id, **fields): + try: + oid = int(object_id) + except (TypeError, ValueError): + return False + updates = {key: value for key, value in fields.items() if key in _STORAGE_OBJECT_WRITABLE} + if not updates: + return False + if "last_error" in updates: + updates["last_error"] = str(updates["last_error"] or "")[:500] + columns = ", ".join(f"{key} = ?" for key in updates) + values = list(updates.values()) + values.extend([time.time(), oid]) + conn = get_db() + try: + cur = conn.execute( + f"UPDATE storage_objects SET {columns}, updated_at = ? WHERE id = ?", + tuple(values), + ) + conn.commit() + return bool(cur.rowcount) + finally: + conn.close() + + +def list_storage_objects( + provider_id=None, + status=None, + streamer=None, + *, + limit=200, + offset=0, + uploaded_before=None, + local_deleted=None, + remote_deleted=False, +): + clauses = [] + values = [] + if provider_id is not None: + try: + clauses.append("provider_id = ?") + values.append(int(provider_id)) + except (TypeError, ValueError): + return [] + if status: + statuses = [status] if isinstance(status, str) else list(status) + statuses = [str(item) for item in statuses if item] + if statuses: + clauses.append("status IN (" + ",".join("?" for _ in statuses) + ")") + values.extend(statuses) + if streamer: + clauses.append("streamer = ? COLLATE NOCASE") + values.append(str(streamer)) + if uploaded_before is not None: + clauses.append("uploaded_at IS NOT NULL AND uploaded_at <= ?") + values.append(float(uploaded_before)) + if local_deleted is True: + clauses.append("local_deleted_at IS NOT NULL") + elif local_deleted is False: + clauses.append("local_deleted_at IS NULL") + if remote_deleted: + clauses.append("remote_deleted_at IS NOT NULL") + where = ("WHERE " + " AND ".join(clauses)) if clauses else "" + try: + limit_i = max(1, min(5000, int(limit))) + offset_i = max(0, int(offset)) + except (TypeError, ValueError): + limit_i, offset_i = 200, 0 + conn = get_db() + try: + rows = conn.execute( + f""" + SELECT * FROM storage_objects + {where} + ORDER BY updated_at DESC, id DESC + LIMIT ? OFFSET ? + """, + (*values, limit_i, offset_i), + ).fetchall() + return [_storage_object_row(row) for row in rows] + finally: + conn.close() + + +def storage_object_summary(provider_id=None): + conn = get_db() + try: + clauses = ["remote_deleted_at IS NULL", "status != 'remote_deleted'"] + params = [] + if provider_id is not None: + try: + clauses.append("provider_id = ?") + params.append(int(provider_id)) + except (TypeError, ValueError): + return {"total": 0, "bytes": 0, "statuses": {}} + where = "WHERE " + " AND ".join(clauses) + rows = conn.execute( + f""" + SELECT status, COUNT(*) AS count, COALESCE(SUM(remote_size), 0) AS bytes + FROM storage_objects {where} + GROUP BY status + """, + tuple(params), + ).fetchall() + total = 0 + total_bytes = 0 + statuses = {} + for row in rows: + count = int(row["count"] or 0) + total += count + total_bytes += int(row["bytes"] or 0) + statuses[str(row["status"] or "")] = { + "count": count, + "bytes": int(row["bytes"] or 0), + } + return {"total": total, "bytes": total_bytes, "statuses": statuses} + finally: + conn.close() + + +def delete_storage_object(object_id): + try: + oid = int(object_id) + except (TypeError, ValueError): + return False + conn = get_db() + try: + cur = conn.execute("DELETE FROM storage_objects WHERE id = ?", (oid,)) + conn.commit() + return bool(cur.rowcount) + finally: + conn.close() + + +def list_storage_objects_for_media(streamer, filename): + conn = get_db() + try: + rows = conn.execute( + """ + SELECT * FROM storage_objects + WHERE streamer = ? COLLATE NOCASE AND filename = ? COLLATE NOCASE + ORDER BY updated_at DESC + """, + (str(streamer or ""), str(filename or "")), + ).fetchall() + return [_storage_object_row(row) for row in rows] + finally: + conn.close() diff --git a/core/storage/__init__.py b/core/storage/__init__.py new file mode 100644 index 0000000..3adfad2 --- /dev/null +++ b/core/storage/__init__.py @@ -0,0 +1,25 @@ +"""External storage integration for StreamVault.""" + +from .base import ( + ProviderField, + RemoteObject, + StorageConfigError, + StorageError, + StorageProvider, +) +from .registry import ( + PROVIDER_CLASSES, + get_provider_class, + provider_catalog, +) + +__all__ = [ + "PROVIDER_CLASSES", + "ProviderField", + "RemoteObject", + "StorageConfigError", + "StorageError", + "StorageProvider", + "get_provider_class", + "provider_catalog", +] diff --git a/core/storage/base.py b/core/storage/base.py new file mode 100644 index 0000000..0b6cbac --- /dev/null +++ b/core/storage/base.py @@ -0,0 +1,375 @@ +"""Provider-agnostic external storage interface for StreamVault. + +The storage layer is intentionally independent from Flask and the recorder so +that the same code can be used by the WebUI, the background worker and the +console CLI. Providers only deal with bytes and remote paths; database state, +encryption and task scheduling live in :mod:`core.storage.service`. +""" + +from __future__ import annotations + +import mimetypes +import os +import posixpath +import re +from dataclasses import dataclass +from typing import Callable, Iterable, Iterator, Optional + + +class StorageError(RuntimeError): + """A provider/configuration error with optional retry semantics.""" + + def __init__( + self, + message: str, + *, + provider: str = "", + status_code: Optional[int] = None, + retryable: bool = False, + details: str = "", + ): + super().__init__(str(message or "storage error")) + self.provider = str(provider or "") + self.status_code = status_code + self.retryable = bool(retryable) + self.details = str(details or "") + + def user_message(self) -> str: + base = str(self) or "Speicherfehler" + if self.status_code: + base = f"{base} (HTTP {self.status_code})" + return base + + +class StorageConfigError(StorageError): + """Invalid or incomplete provider configuration.""" + + +@dataclass(frozen=True) +class ProviderField: + key: str + label: str + type: str = "text" + required: bool = False + secret: bool = False + placeholder: str = "" + help: str = "" + default: str = "" + choices: tuple[tuple[str, str], ...] = () + + def as_dict(self) -> dict: + return { + "key": self.key, + "label": self.label, + "type": self.type, + "required": self.required, + "secret": self.secret, + "placeholder": self.placeholder, + "help": self.help, + "default": self.default, + "choices": [{"value": value, "label": label} for value, label in self.choices], + } + + +@dataclass +class RemoteObject: + path: str + remote_id: str = "" + size: int = 0 + modified_at: str = "" + is_folder: bool = False + + +ProgressCallback = Callable[[int, int], None] + + +def normalize_remote_path(path: str) -> str: + """Normalise a POSIX-style remote path and reject traversal attempts.""" + raw = str(path or "").replace("\\", "/").strip() + if "\x00" in raw: + raise StorageConfigError("Ungültiger Pfad (Nullbyte).") + parts = [] + for part in raw.split("/"): + if part in ("", "."): + continue + if part == "..": + raise StorageConfigError("Ungültiger Pfad (Pfad-Traversal).") + if any(ord(ch) < 32 for ch in part): + raise StorageConfigError("Ungültiger Pfad (Steuerzeichen).") + parts.append(part) + if not parts: + raise StorageConfigError("Der Remote-Pfad darf nicht leer sein.") + return "/".join(parts) + + +def remote_parent(path: str) -> str: + path = normalize_remote_path(path) + parent = posixpath.dirname(path) + return parent + + +def remote_name(path: str) -> str: + return posixpath.basename(normalize_remote_path(path)) + + +def guess_mime_type(filename: str, default: str = "application/octet-stream") -> str: + return mimetypes.guess_type(str(filename or ""))[0] or default + + +def file_size(path: str) -> int: + try: + return int(os.path.getsize(path)) + except OSError as exc: + raise StorageError(f"Lokale Datei nicht lesbar: {exc}") from exc + + +def iter_file_chunks( + path: str, + *, + start: int = 0, + end: Optional[int] = None, + chunk_size: int = 1024 * 1024, +) -> Iterator[bytes]: + """Yield a byte range from a local file (end inclusive).""" + with open(path, "rb") as handle: + handle.seek(max(0, int(start))) + remaining = None if end is None else max(0, int(end) - int(start) + 1) + while True: + size = int(chunk_size) + if remaining is not None: + if remaining <= 0: + break + size = min(size, remaining) + data = handle.read(size) + if not data: + break + if remaining is not None: + remaining -= len(data) + yield data + + +def iter_response_range( + response, + start: int, + end: int, + *, + chunk_size: int = 256 * 1024, +) -> Iterator[bytes]: + """Yield exactly the requested inclusive byte range from an HTTP response. + + Some storage servers ignore ``Range`` and answer with ``200`` plus the whole + file. Skip the prefix and stop at ``end`` so callers never receive extra + bytes beyond the range they asked for. + """ + start = max(0, int(start)) + end = max(start, int(end)) + remaining = end - start + 1 + skip = start if getattr(response, "status_code", 200) == 200 else 0 + for chunk in response.iter_content(chunk_size=chunk_size): + if not chunk or remaining <= 0: + continue + if skip: + if len(chunk) <= skip: + skip -= len(chunk) + continue + chunk = chunk[skip:] + skip = 0 + if len(chunk) > remaining: + chunk = chunk[:remaining] + remaining -= len(chunk) + if chunk: + yield chunk + if remaining <= 0: + break + + +class StorageProvider: + """Base class implemented by every external storage backend.""" + + type_id = "" + label = "" + description = "" + config_fields: tuple[ProviderField, ...] = () + oauth_supported = False + oauth_label = "Konto verbinden" + docs_url = "" + + def __init__(self, config: dict): + self.config = dict(config or {}) + self._config_changed_callback = None + + def set_config_changed_callback(self, callback) -> None: + """Register a callback used to persist rotated OAuth tokens.""" + self._config_changed_callback = callback + + def _config_changed(self) -> None: + callback = self._config_changed_callback + if callback is None: + return + try: + callback(dict(self.config)) + except Exception: + # Token rotation must never make a request fail just because the + # optional persistence callback is unavailable. + pass + + # ------------------------------------------------------------------ + # Configuration helpers + # ------------------------------------------------------------------ + @classmethod + def fields(cls) -> tuple[ProviderField, ...]: + return cls.config_fields + + @classmethod + def secret_keys(cls) -> set[str]: + return {field.key for field in cls.config_fields if field.secret} + + @classmethod + def validate_config(cls, config: dict, *, require_secrets: bool = True) -> list[str]: + errors: list[str] = [] + values = config or {} + for field in cls.config_fields: + if not field.required: + continue + if field.secret and not require_secrets: + continue + value = values.get(field.key) + if value is None or str(value).strip() == "": + errors.append(f"{field.label} fehlt.") + return errors + + @classmethod + def default_config(cls) -> dict: + return {field.key: field.default for field in cls.config_fields if field.default} + + @classmethod + def public_config(cls, config: dict) -> dict: + """Return config values safe for the browser (secrets masked).""" + out = {} + for field in cls.config_fields: + value = (config or {}).get(field.key, "") + if field.secret: + out[field.key] = "••••••••" if str(value or "") else "" + out[f"{field.key}__set"] = bool(str(value or "")) + else: + out[field.key] = value if value is not None else "" + return out + + def masked_config(self) -> dict: + return self.public_config(self.config) + + # ------------------------------------------------------------------ + # Provider operations + # ------------------------------------------------------------------ + def test_connection(self) -> dict: + raise NotImplementedError + + def upload_file( + self, + local_path: str, + remote_path: str, + *, + progress: Optional[ProgressCallback] = None, + ) -> dict: + raise NotImplementedError + + def download_file( + self, + remote_path: str, + dest_path: str, + *, + progress: Optional[ProgressCallback] = None, + ) -> dict: + raise NotImplementedError + + def open_range(self, remote_path: str, start: int, end: int) -> Iterable[bytes]: + """Yield bytes for an inclusive range. Optional for restore-only use.""" + raise StorageError( + f"{self.label} unterstützt kein direktes Range-Streaming.", + provider=self.type_id, + ) + + def delete(self, remote_path: str) -> None: + raise NotImplementedError + + def list(self, prefix: str = "") -> list[RemoteObject]: + raise NotImplementedError + + def exists(self, remote_path: str, expected_size: int = 0) -> bool: + """Return whether a remote object exists (optionally with a size check). + + The default implementation lists the parent folder. Providers with a + cheaper metadata endpoint should override this method. + """ + wanted_path = normalize_remote_path(remote_path) + parent = remote_parent(wanted_path) + for item in self.list(parent): + if item.path.rstrip("/") != wanted_path: + continue + expected = int(expected_size or 0) + if expected <= 0: + return True + # Unknown remote size (0) must never satisfy a known expected size. + return int(item.size or 0) == expected + return False + + def quota(self) -> Optional[dict]: + return None + + # ------------------------------------------------------------------ + # Optional OAuth hooks + # ------------------------------------------------------------------ + @classmethod + def oauth_ready(cls, config: dict) -> bool: + return False + + @classmethod + def oauth_linked(cls, config: dict) -> bool: + """Whether an OAuth account/token has been stored for this provider.""" + values = config or {} + return bool( + str(values.get("refresh_token") or "").strip() + or str(values.get("access_token") or "").strip() + ) + + @classmethod + def oauth_authorize_url(cls, config: dict, redirect_uri: str, state: str) -> str: + raise StorageConfigError(f"{cls.label} unterstützt OAuth nicht.") + + @classmethod + def oauth_exchange_code(cls, config: dict, code: str, redirect_uri: str) -> dict: + raise StorageConfigError(f"{cls.label} unterstützt OAuth nicht.") + + +_SECRET_PATTERNS = ( + re.compile( + r"(?i)(?P(?P[\"']?)\b(?:access_token|refresh_token|" + r"id_token|token|client_secret|code|password|passwd|secret|api_key|" + r"apikey|authorization)\b(?P=qkey)(?P\s*[=:]\s*|\s+)" + r"(?:bearer\s+)?)" + r"(?:(?P[\"'])(?P.*?)(?P=qval)|(?P[^\s&,;<>]*))" + ), +) + + +def _mask_secret_match(match) -> str: + quote = match.group("qval") or "" + return f"{match.group('prefix')}{quote}***{quote}" + + +def mask_text(value) -> str: + """Redact credential-looking key/value pairs from arbitrary error text.""" + text = str(value or "").strip() + for pattern in _SECRET_PATTERNS: + text = pattern.sub(_mask_secret_match, text) + text = re.sub( + r"(?i)\b(bearer)\s+[A-Za-z0-9._~+/=-]+", + r"\1 ***", + text, + ) + return text[:500] + + +def mask_error(exc: Exception) -> str: + """Never leak credential-looking values into logs/UI.""" + return mask_text(exc) or "Storage-Fehler" diff --git a/core/storage/cli.py b/core/storage/cli.py new file mode 100644 index 0000000..94dbd23 --- /dev/null +++ b/core/storage/cli.py @@ -0,0 +1,404 @@ +"""Console interface for external storage. + +Usage: + python -m core.storage.cli list + python -m core.storage.cli add --type google_drive --name "Mein Drive" + python -m core.storage.cli test 1 + python -m core.storage.cli sync + python -m core.storage.cli upload /pfad/zum/video.mp4 +""" + +from __future__ import annotations + +import argparse +import getpass +import os +import sys +import time + +from core import database + +from .base import StorageConfigError, StorageError, mask_error +from . import service +from .crypto import encryption_ready +from .registry import get_provider_class, provider_catalog + + +def _print(message=""): + print(message) + + +def _parse_pairs(pairs): + values = {} + for pair in pairs or []: + if "=" not in str(pair): + raise StorageConfigError(f"Erwartet key=value, erhalten: {pair}") + key, value = str(pair).split("=", 1) + key = key.strip() + if not key: + raise StorageConfigError("Leerer Konfigurationsschlüssel.") + values[key] = value + return values + + +def _prompt_config(type_id: str, provided: dict) -> dict: + from .registry import get_provider_class + + cls = get_provider_class(type_id) + values = dict(provided) + interactive = sys.stdin.isatty() + for field in cls.config_fields: + if field.key in values: + continue + if not interactive: + if field.required and not field.secret: + raise StorageConfigError(f"{field.label} fehlt (nutze --set {field.key}=...).") + continue + suffix = f" [{field.default}]" if field.default else "" + if field.required: + suffix += " *" + prompt = f"{field.label}{suffix}: " + if field.secret: + value = getpass.getpass(prompt) + else: + value = input(prompt).strip() + if not value and field.default: + value = field.default + if value: + values[field.key] = value + return values + + +def _format_timestamp(value): + if not value: + return "-" + try: + return time.strftime("%d.%m.%Y %H:%M", time.localtime(float(value))) + except (TypeError, ValueError, OSError): + return "-" + + +def _cmd_list(args): + rows = service.list_providers() + if args.json: + import json + + _print(json.dumps(rows, ensure_ascii=False, indent=2)) + return 0 + if not rows: + _print("Keine Cloud-Speicher konfiguriert.") + _print("Anlegen: python -m core.storage.cli add --type google_drive --name 'Mein Drive'") + return 0 + settings = service.get_settings() + default_id = settings.get("default_provider_id") + _print(f"{'ID':<4} {'Name':<26} {'Typ':<14} {'Aktiv':<6} {'Standard':<9} {'Letzter Test':<17} Status") + for row in rows: + status = "OK" if row.get("last_check_ok") else ("Fehler" if row.get("last_check_ok") is False else "-") + _print( + f"{row['id']:<4} {str(row.get('name') or '')[:26]:<26} " + f"{str(row.get('type_label') or row.get('provider_type'))[:14]:<14} " + f"{'ja' if row.get('enabled') else 'nein':<6} " + f"{'ja' if row['id'] == default_id else '-':<9} " + f"{_format_timestamp(row.get('last_check_at')):<17} {status}" + ) + _print("") + _print(f"Verschlüsselung: {'bereit' if encryption_ready() else 'NICHT BEREIT (VOD_SECRET_KEY setzen)'}") + return 0 + + +def _cmd_types(args): + for entry in provider_catalog(): + _print(f"{entry['type_id']:<14} {entry['label']}") + _print(f" {entry['description']}") + if entry["docs_url"]: + _print(f" Doku: {entry['docs_url']}") + return 0 + + +def _cmd_add(args): + values = _parse_pairs(args.set) + values = _prompt_config(args.type, values) + provider_id = service.create_provider( + args.name, args.type, values, enabled=not args.disabled + ) + _print(f"[OK] Anbieter #{provider_id} '{args.name}' angelegt.") + if args.no_test: + return 0 + row = service.get_provider_record(provider_id) or {} + cls = get_provider_class(row.get("provider_type") or args.type) + if cls.oauth_supported and not cls.oauth_linked(row.get("config") or {}): + _print( + "[INFO] OAuth-Konto ist noch nicht verbunden. Der automatische " + "Verbindungstest wurde übersprungen; nutze zuerst 'oauth-url' / " + "'oauth-complete' (oder die WebUI)." + ) + return 0 + result = service.test_provider(provider_id) + _print(f"[{'OK' if result.get('ok') else 'FEHLER'}] {result.get('message')}") + return 0 if result.get("ok") else 2 + + +def _cmd_update(args): + values = _parse_pairs(args.set) + enabled = None + if args.enable: + enabled = True + if args.disable: + enabled = False + service.update_provider( + args.provider_id, + name=args.name, + provider_type=args.type, + values=values, + enabled=enabled, + ) + _print(f"[OK] Anbieter #{args.provider_id} aktualisiert.") + return 0 + + +def _cmd_remove(args): + row = service.get_provider_record(args.provider_id) + if not row: + raise StorageConfigError("Anbieter nicht gefunden.") + if not args.yes: + if not sys.stdin.isatty(): + raise StorageConfigError("Löschen ohne --yes nicht möglich.") + answer = input( + f"Anbieter '{row.get('name')}' wirklich aus der Datenbank entfernen? " + "Die Dateien im Cloud-Speicher bleiben erhalten. [j/N]: " + ).strip().lower() + if answer not in ("j", "ja", "y", "yes"): + _print("Abgebrochen.") + return 1 + service.delete_provider(args.provider_id) + _print(f"[OK] Anbieter #{args.provider_id} entfernt. Cloud-Dateien wurden nicht gelöscht.") + return 0 + + +def _cmd_test(args): + result = service.test_provider(args.provider_id) + _print(f"[{'OK' if result.get('ok') else 'FEHLER'}] {result.get('message')}") + quota = result.get("quota") or {} + if quota: + _print( + f"Speicher: {service.format_bytes(quota.get('usage'))} von " + f"{service.format_bytes(quota.get('limit'))} belegt." + ) + return 0 if result.get("ok") else 2 + + +def _cmd_settings(args): + if args.set: + values = _parse_pairs(args.set) + settings = service.save_settings(values) + _print("[OK] Einstellungen gespeichert.") + else: + settings = service.get_settings() + for key in sorted(settings): + _print(f"{key:<32} {settings[key]}") + return 0 + + +def _cmd_sync(args): + task_id = service.enqueue_sync(args.provider, args.streamer) + if task_id: + _print(f"[OK] Sync eingeordnet (Task #{task_id}). Der Recorder-Worker führt ihn aus.") + else: + _print("[INFO] Es läuft bereits ein Sync-Task.") + return 0 + + +def _cmd_upload(args): + path = str(args.path or "") + if path and not os.path.isabs(path): + # A path the user can see from the current shell takes precedence over + # archive-relative notation. If it does not exist here, keep the + # archive-relative interpretation used by `storage upload`. + candidate = os.path.abspath(path) + if os.path.isfile(candidate): + path = candidate + obj = service.enqueue_upload(path, args.provider, force=args.force) + if obj.get("task_id"): + _print( + f"[OK] Upload eingeordnet (Task #{obj['task_id']}, Objekt #{obj['id']}): " + f"{obj.get('local_path')} -> {obj.get('remote_path')}" + ) + else: + _print(f"[INFO] Kein neuer Upload nötig (Objekt #{obj['id']}, Status {obj.get('status')}).") + return 0 + + +def _cmd_objects(args): + rows = database.list_storage_objects( + provider_id=args.provider, + status=args.status, + limit=args.limit, + ) + if not rows: + _print("Keine Archiv-Objekte gefunden.") + return 0 + for row in rows: + local_state = "nur Cloud" if row.get("local_deleted_at") else "lokal" + _print( + f"#{row['id']:<5} {str(row.get('status') or ''):<14} " + f"{service.format_bytes(row.get('remote_size') or row.get('local_size')):<10} " + f"{local_state:<9} {row.get('local_path')}" + ) + _print(f" remote: {row.get('remote_path')}") + if row.get("last_error"): + _print(f" Fehler: {row['last_error']}") + return 0 + + +def _cmd_restore(args): + task_id = service.enqueue_restore(args.object_id) + if task_id: + _print(f"[OK] Wiederherstellung eingeordnet (Task #{task_id}).") + else: + _print("[INFO] Es läuft bereits eine Wiederherstellung für diese Datei.") + return 0 + + +def _cmd_delete_remote(args): + obj = database.get_storage_object(args.object_id) + if not obj: + raise StorageConfigError("Archiv-Objekt nicht gefunden.") + if obj.get("remote_deleted_at"): + _print("[INFO] Die Cloud-Kopie wurde bereits gelöscht.") + return 0 + reason = service.remote_delete_block_reason(obj) + if reason: + raise StorageConfigError(reason) + if not args.yes: + if not sys.stdin.isatty(): + raise StorageConfigError("Löschen ohne --yes nicht möglich.") + answer = input( + f"Cloud-Datei '{obj.get('remote_path')}' wirklich löschen? " + "Eine lokale Kopie bleibt erhalten. [j/N]: " + ).strip().lower() + if answer not in ("j", "ja", "y", "yes"): + _print("Abgebrochen.") + return 1 + task_id = service.enqueue_delete(args.object_id) + if task_id: + _print(f"[OK] Löschen in der Cloud eingeordnet (Task #{task_id}).") + else: + _print("[INFO] Es läuft bereits ein Lösch-Task für diese Datei.") + return 0 + + +def _cmd_oauth_url(args): + url = service.oauth_authorize_url(args.provider_id, args.redirect_uri) + _print(url) + return 0 + + +def _cmd_oauth_complete(args): + service.oauth_complete(args.provider_id, args.code, args.redirect_uri) + _print(f"[OK] OAuth-Konto für Anbieter #{args.provider_id} verbunden.") + result = service.test_provider(args.provider_id) + _print(f"[{'OK' if result.get('ok') else 'FEHLER'}] {result.get('message')}") + return 0 if result.get("ok") else 2 + + +def build_parser(): + parser = argparse.ArgumentParser( + prog="streamvault storage", + description="Cloud-Speicher für StreamVault verwalten.", + ) + sub = parser.add_subparsers(dest="command", required=True) + + p = sub.add_parser("list", help="Konfigurierte Anbieter anzeigen") + p.add_argument("--json", action="store_true") + p.set_defaults(func=_cmd_list) + + p = sub.add_parser("types", help="Unterstützte Anbieter anzeigen") + p.set_defaults(func=_cmd_types) + + p = sub.add_parser("add", help="Anbieter anlegen") + p.add_argument("--type", required=True, choices=[entry["type_id"] for entry in provider_catalog()]) + p.add_argument("--name", required=True) + p.add_argument("--set", action="append", default=[], metavar="KEY=VALUE") + p.add_argument("--disabled", action="store_true") + p.add_argument("--no-test", action="store_true") + p.set_defaults(func=_cmd_add) + + p = sub.add_parser("update", help="Anbieter bearbeiten") + p.add_argument("provider_id", type=int) + p.add_argument("--name") + p.add_argument("--type", choices=[entry["type_id"] for entry in provider_catalog()]) + p.add_argument("--set", action="append", default=[], metavar="KEY=VALUE") + p.add_argument("--enable", action="store_true") + p.add_argument("--disable", action="store_true") + p.set_defaults(func=_cmd_update) + + p = sub.add_parser("remove", help="Anbieter aus der Datenbank entfernen") + p.add_argument("provider_id", type=int) + p.add_argument("--yes", action="store_true") + p.set_defaults(func=_cmd_remove) + + p = sub.add_parser("test", help="Verbindung testen") + p.add_argument("provider_id", type=int) + p.set_defaults(func=_cmd_test) + + p = sub.add_parser("settings", help="Einstellungen anzeigen/ändern") + p.add_argument("--set", action="append", default=[], metavar="KEY=VALUE") + p.set_defaults(func=_cmd_settings) + + p = sub.add_parser("sync", help="Archiv mit Cloud abgleichen") + p.add_argument("--provider", type=int) + p.add_argument("--streamer") + p.set_defaults(func=_cmd_sync) + + p = sub.add_parser("upload", help="Einzelne Datei hochladen") + p.add_argument("path") + p.add_argument("--provider", type=int) + p.add_argument("--force", action="store_true") + p.set_defaults(func=_cmd_upload) + + p = sub.add_parser("objects", help="Archiv-Objekte anzeigen") + p.add_argument("--provider", type=int) + p.add_argument("--status") + p.add_argument("--limit", type=int, default=50) + p.set_defaults(func=_cmd_objects) + + p = sub.add_parser("restore", help="Datei aus der Cloud wiederherstellen") + p.add_argument("object_id", type=int) + p.set_defaults(func=_cmd_restore) + + p = sub.add_parser("delete-remote", help="Cloud-Datei löschen (lokal behalten)") + p.add_argument("object_id", type=int) + p.add_argument("--yes", action="store_true") + p.set_defaults(func=_cmd_delete_remote) + + p = sub.add_parser("oauth-url", help="OAuth-Autorisierungs-URL ausgeben") + p.add_argument("provider_id", type=int) + p.add_argument("--redirect-uri", required=True) + p.set_defaults(func=_cmd_oauth_url) + + p = sub.add_parser("oauth-complete", help="OAuth-Code einlösen") + p.add_argument("provider_id", type=int) + p.add_argument("--code", required=True) + p.add_argument("--redirect-uri", required=True) + p.set_defaults(func=_cmd_oauth_complete) + return parser + + +def main(argv=None): + parser = build_parser() + args = parser.parse_args(argv) + try: + return int(args.func(args) or 0) + except (StorageConfigError, StorageError) as exc: + _print(f"[FEHLER] {mask_error(exc)}") + return 2 + except KeyboardInterrupt: + _print("\nAbgebrochen.") + return 130 + except Exception as exc: + _print(f"[FEHLER] {mask_error(exc)}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/core/storage/crypto.py b/core/storage/crypto.py new file mode 100644 index 0000000..6f37043 --- /dev/null +++ b/core/storage/crypto.py @@ -0,0 +1,128 @@ +"""AES-GCM encryption for provider credentials at rest. + +The key is derived from ``STORAGE_ENCRYPTION_KEY`` when set, otherwise from +``VOD_SECRET_KEY``. Credentials are never written to logs or returned to the +browser in clear text. +""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import os + +from Crypto.Cipher import AES +from Crypto.Random import get_random_bytes + +from core import config +from .base import StorageConfigError + +_PREFIX = "sv1:" +_WEAK_KEYS = { + "", + "change-me", + "changeme", + "secret", + "streamvault", + # Public placeholders from .env.example / INSTALL.md must never become a + # production encryption key, even if an operator forgets to replace them. + "change-this-to-a-very-secure-random-secret-key-32-chars-minimum", + "generate_a_random_32_character_string_here", + "generate_another_random_32_character_string", +} + + +def is_weak_secret(value) -> bool: + """Return whether a session/encryption secret is empty or a known placeholder.""" + raw = str(value or "").strip() + return raw.lower() in _WEAK_KEYS or len(raw) < 16 + + +def _secret_from_file() -> str: + """Use the WebUI-generated secret file when no env secret is configured.""" + path = os.path.join(getattr(config, "META_DIR", ""), "secret_key.txt") + try: + with open(path, "r", encoding="utf-8") as handle: + return handle.read().strip() + except OSError: + return "" + + +def _secret() -> str: + # An explicitly configured dedicated key is authoritative. Falling back + # to another source would silently make stored credentials depend on a + # different key than the operator expects. + dedicated = os.environ.get("STORAGE_ENCRYPTION_KEY", "").strip() + if dedicated: + return dedicated + + raw = ( + os.environ.get("VOD_SECRET_KEY", "").strip() + or str(getattr(config, "SECRET_KEY", "") or "").strip() + ) + if is_weak_secret(raw): + raw = _secret_from_file() + return raw + + +def _key() -> bytes: + raw = _secret() + if is_weak_secret(raw): + raise StorageConfigError( + "Für die verschlüsselte Speicherung von Zugangsdaten muss " + "VOD_SECRET_KEY (oder STORAGE_ENCRYPTION_KEY) auf einen starken, " + "einmaligen Wert gesetzt sein (mindestens 16 Zeichen)." + ) + return hashlib.sha256(("streamvault-storage-v1:" + raw).encode("utf-8")).digest() + + +def encryption_ready() -> bool: + try: + _key() + return True + except StorageConfigError: + return False + + +def signing_secret() -> str: + """Raw secret used to sign short-lived OAuth state tokens.""" + raw = _secret() + if is_weak_secret(raw): + raise StorageConfigError( + "Für OAuth-Status-Token muss ein starker VOD_SECRET_KEY gesetzt sein." + ) + return raw + + +def encrypt_config(data: dict) -> str: + plaintext = json.dumps(data or {}, ensure_ascii=False, sort_keys=True).encode("utf-8") + nonce = get_random_bytes(12) + cipher = AES.new(_key(), AES.MODE_GCM, nonce=nonce) + ciphertext, tag = cipher.encrypt_and_digest(plaintext) + token = base64.urlsafe_b64encode(nonce + tag + ciphertext).decode("ascii") + return _PREFIX + token + + +def decrypt_config(token: str) -> dict: + raw = str(token or "").strip() + if not raw: + return {} + if not raw.startswith(_PREFIX): + raise StorageConfigError("Unbekanntes Format für gespeicherte Zugangsdaten.") + try: + payload = base64.urlsafe_b64decode(raw[len(_PREFIX):].encode("ascii")) + nonce, tag, ciphertext = payload[:12], payload[12:28], payload[28:] + cipher = AES.new(_key(), AES.MODE_GCM, nonce=nonce) + plaintext = cipher.decrypt_and_verify(ciphertext, tag) + data = json.loads(plaintext.decode("utf-8")) + except StorageConfigError: + raise + except Exception as exc: + raise StorageConfigError( + "Zugangsdaten konnten nicht entschlüsselt werden. " + "Wurde VOD_SECRET_KEY geändert?" + ) from exc + if not isinstance(data, dict): + raise StorageConfigError("Ungültige gespeicherte Zugangsdaten.") + return data diff --git a/core/storage/http.py b/core/storage/http.py new file mode 100644 index 0000000..c1a4e7d --- /dev/null +++ b/core/storage/http.py @@ -0,0 +1,179 @@ +"""Small, dependency-free HTTP helper shared by storage providers.""" + +from __future__ import annotations + +import json +import logging +import random +import re +import threading +import time +from typing import Any, Optional + +import requests + +from .base import StorageError + +logger = logging.getLogger("storage.http") + +DEFAULT_TIMEOUT = (10, 90) +RETRY_STATUS = {408, 425, 429, 500, 502, 503, 504} +_LOCAL = threading.local() + +_URL_RE = re.compile(r"https?://[^\s'\"<>]+", re.IGNORECASE) +_URL_FIELD_RE = re.compile(r"(?i)\burl:\s*[^\s,)]+") +_QUERY_RE = re.compile(r"\?[^\s'\"<>)]*") + + +def sanitize_error_text(value, limit: int = 500) -> str: + """Remove URLs (which may contain upload-session tokens) from errors.""" + text = str(value or "").strip() + text = _URL_RE.sub("[URL]", text) + text = _URL_FIELD_RE.sub("url: [URL]", text) + text = _QUERY_RE.sub("?***", text) + return text[:limit] + + +def session() -> requests.Session: + """One session per thread (requests sessions are not fully thread-safe).""" + current = getattr(_LOCAL, "session", None) + if current is None: + current = requests.Session() + current.headers.update({ + "User-Agent": "StreamVault-Storage/1.0", + # Providers explicitly set their own Content-Type where needed. + # A wildcard Accept avoids accidentally forcing a JSON response + # from download endpoints that inspect the header. + "Accept": "*/*", + }) + _LOCAL.session = current + return current + + +def _retry_after_seconds(response: requests.Response, attempt: int) -> float: + raw = (response.headers.get("Retry-After") or "").strip() + if raw: + try: + return min(60.0, max(1.0, float(raw))) + except ValueError: + pass + return min(30.0, (2 ** attempt) + random.uniform(0, 0.4)) + + +def error_text(response: requests.Response) -> str: + """Extract a short, non-secret error message from common API formats.""" + try: + body = response.json() + except Exception: + body = None + if isinstance(body, dict): + for path in ( + ("error", "message"), + ("error_description",), + ("error_summary",), + ("message",), + ("error",), + ): + value: Any = body + for key in path: + if not isinstance(value, dict) or key not in value: + value = None + break + value = value[key] + if value: + return sanitize_error_text(value) + text = (response.text or "").strip() + if "<" in text and ">" in text: + # Do not dump full XML/HTML error pages into logs/UI. + text = "Antwort des Anbieters konnte nicht gelesen werden." + return sanitize_error_text(text) + + +def request( + method: str, + url: str, + *, + provider: str = "", + timeout: tuple[float, float] = DEFAULT_TIMEOUT, + retries: int = 3, + retry_status: Optional[set[int]] = None, + stream: bool = False, + **kwargs, +) -> requests.Response: + """HTTP request with bounded retries for transient failures. + + The response is returned for all statuses; callers decide whether a status + is acceptable (uploads need to handle 308, for example). Network failures + and retryable statuses are retried. + """ + statuses = RETRY_STATUS if retry_status is None else retry_status + last_exc: Optional[Exception] = None + for attempt in range(max(1, int(retries))): + try: + response = session().request( + method, + url, + timeout=timeout, + stream=stream, + **kwargs, + ) + except requests.RequestException as exc: + last_exc = exc + if attempt + 1 >= retries: + break + time.sleep(min(20.0, (2 ** attempt) + random.uniform(0, 0.4))) + continue + + if response.status_code not in statuses or attempt + 1 >= retries: + return response + delay = _retry_after_seconds(response, attempt) + try: + response.close() + except Exception: + pass + time.sleep(delay) + + raise StorageError( + "Netzwerkfehler bei " + f"{provider or 'Storage'}: {sanitize_error_text(last_exc) or 'unbekannt'}", + provider=provider, + retryable=True, + ) + + +def raise_for_status( + response: requests.Response, + action: str, + *, + provider: str = "", + retryable_status: Optional[set[int]] = None, +) -> None: + if 200 <= response.status_code < 300: + return + retryable = response.status_code in (retryable_status or RETRY_STATUS) + detail = error_text(response) or response.reason or "unbekannter Fehler" + raise StorageError( + f"{action} fehlgeschlagen: {sanitize_error_text(detail)}", + provider=provider, + status_code=response.status_code, + retryable=retryable, + ) + + +def json_response(response: requests.Response, action: str, *, provider: str = "") -> dict: + raise_for_status(response, action, provider=provider) + try: + data = response.json() + except ValueError as exc: + raise StorageError( + f"{action}: ungültige JSON-Antwort.", + provider=provider, + status_code=response.status_code, + ) from exc + if not isinstance(data, dict): + raise StorageError(f"{action}: unerwartete Antwort.", provider=provider) + return data + + +def dumps(data: dict) -> str: + return json.dumps(data, ensure_ascii=False, separators=(",", ":")) diff --git a/core/storage/providers/__init__.py b/core/storage/providers/__init__.py new file mode 100644 index 0000000..94cb20d --- /dev/null +++ b/core/storage/providers/__init__.py @@ -0,0 +1 @@ +"""Built-in storage provider implementations.""" diff --git a/core/storage/providers/dropbox.py b/core/storage/providers/dropbox.py new file mode 100644 index 0000000..454c356 --- /dev/null +++ b/core/storage/providers/dropbox.py @@ -0,0 +1,450 @@ +"""Dropbox provider (OAuth 2.0 refresh tokens + upload sessions).""" + +from __future__ import annotations + +import json +import os +import posixpath +import time +import urllib.parse +from typing import Iterable, Optional + +from core.storage.base import ( + ProgressCallback, + ProviderField, + RemoteObject, + StorageConfigError, + StorageError, + StorageProvider, + file_size, + iter_response_range, + normalize_remote_path, +) +from core.storage import http + +API = "https://api.dropboxapi.com/2" +CONTENT = "https://content.dropboxapi.com/2" +AUTH_URL = "https://www.dropbox.com/oauth2/authorize" +TOKEN_URL = "https://api.dropboxapi.com/oauth2/token" +SESSION_CHUNK = 32 * 1024 * 1024 +SIMPLE_UPLOAD_MAX = 140 * 1024 * 1024 + + +class DropboxProvider(StorageProvider): + type_id = "dropbox" + label = "Dropbox" + description = "Dropbox mit OAuth 2.0, Refresh Token und großen Upload-Sessions." + oauth_supported = True + oauth_label = "Mit Dropbox verbinden" + docs_url = "https://www.dropbox.com/developers/documentation/http/documentation" + config_fields = ( + ProviderField("app_key", "App Key", placeholder="Dropbox App Console → App Key"), + ProviderField("app_secret", "App Secret", type="password", secret=True), + ProviderField("refresh_token", "Refresh Token", type="password", secret=True, + help="Wird beim Verbinden automatisch gespeichert."), + ProviderField("access_token", "Access Token (Alternative)", type="password", secret=True, + help="Nur nötig, wenn kein Refresh Token verwendet wird."), + ProviderField("base_path", "Unterordner", default="StreamVault", placeholder="StreamVault"), + ) + + def __init__(self, config: dict): + super().__init__(config) + self._access = "" + self._expires_at = 0.0 + + @classmethod + def validate_config(cls, config: dict, *, require_secrets: bool = True) -> list[str]: + cfg = config or {} + if str(cfg.get("access_token") or "").strip(): + return [] + errors = [] + for key, label in (("app_key", "App Key"), ("app_secret", "App Secret"), ("refresh_token", "Refresh Token")): + if require_secrets and not str(cfg.get(key) or "").strip(): + errors.append(f"{label} fehlt.") + return errors + + @classmethod + def oauth_ready(cls, config: dict) -> bool: + return bool(str(config.get("app_key") or "").strip() and str(config.get("app_secret") or "").strip()) + + @classmethod + def oauth_authorize_url(cls, config: dict, redirect_uri: str, state: str) -> str: + if not cls.oauth_ready(config): + raise StorageConfigError("Dropbox App Key und App Secret fehlen.") + params = { + "client_id": str(config.get("app_key") or "").strip(), + "response_type": "code", + "redirect_uri": redirect_uri, + "token_access_type": "offline", + "state": state, + } + return f"{AUTH_URL}?{urllib.parse.urlencode(params)}" + + @classmethod + def oauth_exchange_code(cls, config: dict, code: str, redirect_uri: str) -> dict: + response = http.request( + "POST", + TOKEN_URL, + data={ + "code": str(code or "").strip(), + "grant_type": "authorization_code", + "client_id": str(config.get("app_key") or "").strip(), + "client_secret": str(config.get("app_secret") or "").strip(), + "redirect_uri": redirect_uri, + }, + provider=cls.type_id, + ) + data = http.json_response(response, "Dropbox-Token-Austausch", provider=cls.type_id) + refresh = str(data.get("refresh_token") or "").strip() + if not refresh: + raise StorageError("Dropbox lieferte keinen Refresh Token.", provider=cls.type_id) + return {"refresh_token": refresh} + + def _token(self, force: bool = False) -> str: + access = str(self.config.get("access_token") or "").strip() + if access and not str(self.config.get("refresh_token") or "").strip(): + return access + now = time.time() + if not force and self._access and now < self._expires_at - 60: + return self._access + refresh = str(self.config.get("refresh_token") or "").strip() + key = str(self.config.get("app_key") or "").strip() + secret = str(self.config.get("app_secret") or "").strip() + if not refresh or not key or not secret: + raise StorageConfigError("Dropbox ist nicht vollständig konfiguriert.") + response = http.request( + "POST", + TOKEN_URL, + data={ + "grant_type": "refresh_token", + "refresh_token": refresh, + "client_id": key, + "client_secret": secret, + }, + provider=self.type_id, + ) + data = http.json_response(response, "Dropbox-Token-Erneuerung", provider=self.type_id) + self._access = str(data.get("access_token") or "").strip() + if not self._access: + raise StorageError("Dropbox lieferte kein Access Token.", provider=self.type_id) + try: + self._expires_at = now + int(data.get("expires_in") or 14400) + except (TypeError, ValueError): + self._expires_at = now + 14400 + return self._access + + def _headers(self, extra: Optional[dict] = None) -> dict: + headers = {"Authorization": f"Bearer {self._token()}"} + if extra: + headers.update(extra) + return headers + + def _rpc(self, endpoint: str, payload: dict, action: str) -> dict: + for attempt in range(2): + response = http.request( + "POST", + f"{API}{endpoint}", + headers=self._headers({"Content-Type": "application/json"}), + data=http.dumps(payload), + provider=self.type_id, + timeout=(10, 60), + ) + if response.status_code == 401 and attempt == 0: + try: + response.close() + except Exception: + pass + self._token(force=True) + continue + return http.json_response(response, action, provider=self.type_id) + raise StorageError(f"{action}: Authentifizierung fehlgeschlagen.", provider=self.type_id) + + def _content( + self, + endpoint: str, + arg: dict, + *, + data=None, + action: str, + stream=False, + headers=None, + ): + request_headers = self._headers( + { + # HTTP headers are latin-1/ASCII; non-ASCII filenames must be + # JSON-escaped here (Dropbox decodes the escapes server-side). + "Dropbox-API-Arg": json.dumps( + arg, ensure_ascii=True, separators=(",", ":") + ), + "Content-Type": "application/octet-stream", + } + ) + if headers: + request_headers.update(headers) + return http.request( + "POST", + f"{CONTENT}{endpoint}", + headers=request_headers, + data=data, + provider=self.type_id, + stream=stream, + timeout=(10, 600), + retries=1, + ) + + def _path(self, remote_path: str) -> str: + return "/" + normalize_remote_path(remote_path) + + def _ensure_folder(self, folder_path: str) -> None: + """Create a Dropbox folder and any missing ancestors. + + Dropbox ``/files/upload`` does not create parent folders. Checking the + immediate parent first keeps repeated uploads to an existing archive + cheap while still creating a complete base_path on first use. + """ + folder = str(folder_path or "").strip() + if folder in ("", "/"): + return + try: + data = self._rpc( + "/files/get_metadata", + {"path": folder}, + "Dropbox-Ordnerpruefung", + ) + except StorageError as exc: + if exc.status_code != 409: + raise + self._ensure_folder(posixpath.dirname(folder)) + try: + self._rpc( + "/files/create_folder_v2", + {"path": folder, "autorename": False}, + "Dropbox-Ordner anlegen", + ) + except StorageError as create_exc: + if create_exc.status_code != 409: + raise + # Another worker may have created the folder concurrently. + data = self._rpc( + "/files/get_metadata", + {"path": folder}, + "Dropbox-Ordnerpruefung", + ) + else: + return + if str(data.get(".tag") or "") != "folder": + raise StorageError( + f"Dropbox-Pfad ist kein Ordner: {folder}", + provider=self.type_id, + ) + + def _ensure_parent_folders(self, remote_path: str) -> None: + self._ensure_folder(posixpath.dirname(self._path(remote_path))) + + def test_connection(self) -> dict: + data = self._rpc("/users/get_current_account", {}, "Dropbox-Verbindungstest") + return { + "ok": True, + "account": (data.get("email") or data.get("display_name") or ""), + "quota": self.quota(), + } + + def quota(self) -> Optional[dict]: + try: + data = self._rpc("/users/get_space_usage", {}, "Dropbox-Quota") + return { + "usage": int(data.get("used") or 0), + "limit": int((data.get("allocation") or {}).get("allocated") or 0), + } + except Exception: + return None + + def upload_file( + self, + local_path: str, + remote_path: str, + *, + progress: Optional[ProgressCallback] = None, + ) -> dict: + size = file_size(local_path) + path = self._path(remote_path) + self._ensure_parent_folders(path) + commit = {"path": path, "mode": {".tag": "overwrite"}, "autorename": False, "mute": False} + if size <= SIMPLE_UPLOAD_MAX: + with open(local_path, "rb") as handle: + response = self._content( + "/files/upload", + {**commit, "strict_conflict": False}, + data=handle, + action="Dropbox-Upload", + ) + data = http.json_response(response, "Dropbox-Upload", provider=self.type_id) + if progress: + progress(size, size) + return {"remote_id": str(data.get("id") or path), "size": int(data.get("size") or size), + "checksum": str(data.get("content_hash") or ""), "modified_at": str(data.get("server_modified") or "")} + + with open(local_path, "rb") as handle: + first = handle.read(SESSION_CHUNK) + response = self._content( + "/files/upload_session/start", + {"close": False}, + data=first, + action="Dropbox-Upload-Session", + ) + data = http.json_response(response, "Dropbox-Upload-Session", provider=self.type_id) + session_id = str(data.get("session_id") or "") + if not session_id: + raise StorageError("Dropbox lieferte keine Upload-Session.", provider=self.type_id) + offset = len(first) + if progress: + progress(offset, size) + while offset < size: + remaining = size - offset + chunk = handle.read(min(SESSION_CHUNK, remaining)) + cursor = {"session_id": session_id, "offset": offset} + if offset + len(chunk) >= size: + response = self._content( + "/files/upload_session/finish", + {"cursor": cursor, "commit": commit}, + data=chunk, + action="Dropbox-Upload abschließen", + ) + data = http.json_response(response, "Dropbox-Upload abschließen", provider=self.type_id) + if progress: + progress(size, size) + return { + "remote_id": str(data.get("id") or path), + "size": int(data.get("size") or size), + "checksum": str(data.get("content_hash") or ""), + "modified_at": str(data.get("server_modified") or ""), + } + response = self._content( + "/files/upload_session/append_v2", + {"cursor": cursor, "close": False}, + data=chunk, + action="Dropbox-Upload fortsetzen", + ) + if response.status_code not in (200, 201): + http.raise_for_status(response, "Dropbox-Upload fortsetzen", provider=self.type_id) + offset += len(chunk) + if progress: + progress(offset, size) + raise StorageError("Dropbox-Upload endete unerwartet.", provider=self.type_id) + + def _download(self, remote_path: str, *, range_header: str = "", stream: bool = True): + return self._content( + "/files/download", + {"path": self._path(remote_path)}, + action="Dropbox-Download", + stream=stream, + headers={"Range": range_header} if range_header else None, + ) + + def download_file( + self, + remote_path: str, + dest_path: str, + *, + progress: Optional[ProgressCallback] = None, + ) -> dict: + response = self._download(remote_path) + http.raise_for_status(response, "Dropbox-Download", provider=self.type_id) + total = int(response.headers.get("Content-Length") or 0) + tmp_path = f"{dest_path}.part" + os.makedirs(os.path.dirname(os.path.abspath(dest_path)), exist_ok=True) + done = 0 + try: + with open(tmp_path, "wb") as handle: + for chunk in response.iter_content(chunk_size=1024 * 1024): + if not chunk: + continue + handle.write(chunk) + done += len(chunk) + if progress: + progress(done, total) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp_path, dest_path) + finally: + try: + response.close() + except Exception: + pass + if os.path.exists(tmp_path): + try: + os.remove(tmp_path) + except OSError: + pass + return {"remote_id": self._path(remote_path), "size": done} + + def open_range(self, remote_path: str, start: int, end: int) -> Iterable[bytes]: + start = max(0, int(start)) + end = max(start, int(end)) + response = self._download(remote_path, range_header=f"bytes={start}-{end}") + http.raise_for_status(response, "Dropbox-Range-Stream", provider=self.type_id) + try: + yield from iter_response_range(response, start, end) + finally: + try: + response.close() + except Exception: + pass + + def delete(self, remote_path: str) -> None: + try: + self._rpc("/files/delete_v2", {"path": self._path(remote_path)}, "Dropbox-Datei löschen") + except StorageError as exc: + if exc.status_code == 409: + return + raise + + def list(self, prefix: str = "") -> list[RemoteObject]: + path = "" + if str(prefix or "").strip(): + path = self._path(prefix) + data = self._rpc( + "/files/list_folder", + {"path": path, "recursive": True, "include_deleted": False, "limit": 2000}, + "Dropbox-Dateiliste", + ) + entries = list(data.get("entries") or []) + cursor = data.get("cursor") + while data.get("has_more") and cursor: + data = self._rpc( + "/files/list_folder/continue", + {"cursor": cursor}, + "Dropbox-Dateiliste", + ) + entries.extend(data.get("entries") or []) + cursor = data.get("cursor") + items = [] + for entry in entries: + if entry.get(".tag") != "file": + continue + display = str(entry.get("path_display") or entry.get("path_lower") or "").lstrip("/") + items.append( + RemoteObject( + path=display, + remote_id=str(entry.get("id") or display), + size=int(entry.get("size") or 0), + modified_at=str(entry.get("server_modified") or ""), + ) + ) + return items + + def exists(self, remote_path: str, expected_size: int = 0) -> bool: + try: + data = self._rpc( + "/files/get_metadata", + {"path": self._path(remote_path)}, + "Dropbox-Existenzprüfung", + ) + except StorageError as exc: + if exc.status_code == 409: + return False + raise + expected = int(expected_size or 0) + if expected <= 0: + return True + return int(data.get("size") or 0) == expected diff --git a/core/storage/providers/google_drive.py b/core/storage/providers/google_drive.py new file mode 100644 index 0000000..765dac4 --- /dev/null +++ b/core/storage/providers/google_drive.py @@ -0,0 +1,632 @@ +"""Google Drive provider (OAuth 2.0 + resumable uploads, no SDK required).""" + +from __future__ import annotations + +import json +import os +import time +import urllib.parse +from typing import Iterable, Optional + +from core.storage.base import ( + ProgressCallback, + ProviderField, + RemoteObject, + StorageConfigError, + StorageError, + StorageProvider, + file_size, + guess_mime_type, + iter_response_range, + normalize_remote_path, + remote_name, + remote_parent, +) +from core.storage import http + +DRIVE_API = "https://www.googleapis.com/drive/v3" +DRIVE_UPLOAD_API = "https://www.googleapis.com/upload/drive/v3" +AUTH_URL = "https://accounts.google.com/o/oauth2/v2/auth" +TOKEN_URL = "https://oauth2.googleapis.com/token" +DEFAULT_SCOPE = "https://www.googleapis.com/auth/drive.file" +FOLDER_MIME = "application/vnd.google-apps.folder" +UPLOAD_CHUNK = 16 * 1024 * 1024 + + +class GoogleDriveProvider(StorageProvider): + type_id = "google_drive" + label = "Google Drive" + description = "Google Drive / Workspace Drive mit OAuth 2.0 und resumierbaren Uploads." + oauth_supported = True + oauth_label = "Mit Google verbinden" + docs_url = "https://developers.google.com/drive/api/guides/about-sdk" + config_fields = ( + ProviderField( + "client_id", + "OAuth Client ID", + required=True, + placeholder="1234567890-xxxx.apps.googleusercontent.com", + help="Google Cloud Console → APIs & Dienste → Anmeldedaten → OAuth-Client (Webanwendung).", + ), + ProviderField( + "client_secret", + "OAuth Client Secret", + type="password", + required=True, + secret=True, + ), + ProviderField( + "refresh_token", + "Refresh Token", + type="password", + required=True, + secret=True, + help="Wird beim Verbinden automatisch gespeichert. Alternativ manuell einfügen.", + ), + ProviderField( + "folder_id", + "Zielordner-ID (optional)", + placeholder="1AbCdEfGh...", + help="Leer lassen, um automatisch einen Ordner 'StreamVault' anzulegen.", + ), + ProviderField( + "base_path", + "Unterordner", + default="StreamVault", + placeholder="StreamVault", + help="Pfad unterhalb der Zielordner-ID. Leer = direkt in der Zielordner-ID.", + ), + ProviderField( + "scope", + "OAuth Scope", + default=DEFAULT_SCOPE, + help="drive.file = nur von StreamVault angelegte Dateien (empfohlen). " + "Für bestehende Ordner ggf. https://www.googleapis.com/auth/drive.", + ), + ) + + def __init__(self, config: dict): + super().__init__(config) + self._access_token = "" + self._token_expires_at = 0.0 + # Only folder lookups are cached. File metadata (especially size) must + # be fetched fresh so exists()-verification never trusts stale values. + self._folder_cache: dict[tuple[str, str], dict] = {} + + # ------------------------------------------------------------------ + # OAuth + # ------------------------------------------------------------------ + @classmethod + def oauth_ready(cls, config: dict) -> bool: + return bool(str(config.get("client_id") or "").strip() and str(config.get("client_secret") or "").strip()) + + @classmethod + def oauth_authorize_url(cls, config: dict, redirect_uri: str, state: str) -> str: + if not cls.oauth_ready(config): + raise StorageConfigError("Google Client ID und Client Secret fehlen.") + scope = str(config.get("scope") or DEFAULT_SCOPE).strip() or DEFAULT_SCOPE + params = { + "client_id": str(config.get("client_id") or "").strip(), + "redirect_uri": redirect_uri, + "response_type": "code", + "scope": scope, + "access_type": "offline", + "prompt": "consent", + "include_granted_scopes": "true", + "state": state, + } + return f"{AUTH_URL}?{urllib.parse.urlencode(params)}" + + @classmethod + def oauth_exchange_code(cls, config: dict, code: str, redirect_uri: str) -> dict: + payload = { + "client_id": str(config.get("client_id") or "").strip(), + "client_secret": str(config.get("client_secret") or "").strip(), + "code": str(code or "").strip(), + "grant_type": "authorization_code", + "redirect_uri": redirect_uri, + } + response = http.request("POST", TOKEN_URL, data=payload, provider=cls.type_id) + data = http.json_response(response, "Google-Token-Austausch", provider=cls.type_id) + refresh = str(data.get("refresh_token") or "").strip() + if not refresh: + raise StorageError( + "Google hat keinen Refresh Token geliefert. Bitte Zugriff in den " + "Google-Kontoeinstellungen entfernen und erneut verbinden.", + provider=cls.type_id, + ) + return {"refresh_token": refresh, "scope": data.get("scope") or config.get("scope") or DEFAULT_SCOPE} + + # ------------------------------------------------------------------ + # Auth / HTTP + # ------------------------------------------------------------------ + def _token(self, force: bool = False) -> str: + now = time.time() + if not force and self._access_token and now < self._token_expires_at - 60: + return self._access_token + refresh = str(self.config.get("refresh_token") or "").strip() + client_id = str(self.config.get("client_id") or "").strip() + client_secret = str(self.config.get("client_secret") or "").strip() + if not refresh or not client_id or not client_secret: + raise StorageConfigError( + "Google Drive ist nicht vollständig konfiguriert (Client ID, Client Secret, Refresh Token)." + ) + response = http.request( + "POST", + TOKEN_URL, + data={ + "client_id": client_id, + "client_secret": client_secret, + "refresh_token": refresh, + "grant_type": "refresh_token", + }, + provider=self.type_id, + ) + data = http.json_response(response, "Google-Token-Erneuerung", provider=self.type_id) + token = str(data.get("access_token") or "").strip() + if not token: + raise StorageError("Google lieferte kein Access Token.", provider=self.type_id) + self._access_token = token + try: + self._token_expires_at = now + int(data.get("expires_in") or 3600) + except (TypeError, ValueError): + self._token_expires_at = now + 3600 + return token + + def _headers(self, **extra) -> dict: + headers = {"Authorization": f"Bearer {self._token()}"} + headers.update(extra) + return headers + + def _api(self, method: str, path: str, **kwargs): + url = path if path.startswith("http") else f"{DRIVE_API}{path}" + for attempt in range(2): + kwargs["headers"] = self._headers(**(kwargs.get("headers") or {})) + response = http.request(method, url, provider=self.type_id, **kwargs) + if response.status_code == 401 and attempt == 0: + try: + response.close() + except Exception: + pass + self._token(force=True) + continue + return response + return response + + @staticmethod + def _q(value: str) -> str: + return str(value or "").replace("\\", "\\\\").replace("'", "\\'") + + # ------------------------------------------------------------------ + # Folder handling + # ------------------------------------------------------------------ + def _find_child(self, parent_id: str, name: str, *, folder_only: bool = False) -> Optional[dict]: + cache_key = (parent_id, f"{'d' if folder_only else 'f'}:{name}") + if folder_only: + cached = self._folder_cache.get(cache_key) + if cached: + return dict(cached) + query = f"name = '{self._q(name)}' and '{self._q(parent_id)}' in parents and trashed = false" + if folder_only: + query += f" and mimeType = '{FOLDER_MIME}'" + else: + # A folder with the same name must never be mistaken for a file + # (Google Drive permits duplicate names of different types). + query += f" and mimeType != '{FOLDER_MIME}'" + response = self._api( + "GET", + "/files", + params={ + "q": query, + "fields": "files(id,name,mimeType,size,modifiedTime,md5Checksum)", + "pageSize": "10", + "supportsAllDrives": "true", + "includeItemsFromAllDrives": "true", + }, + ) + data = http.json_response(response, "Google-Dateisuche", provider=self.type_id) + files = data.get("files") or [] + if not files: + return None + item = files[0] + if folder_only and item.get("id"): + self._folder_cache[cache_key] = dict(item) + return item + + def _create_folder(self, parent_id: str, name: str) -> str: + response = self._api( + "POST", + "/files", + params={"fields": "id,name", "supportsAllDrives": "true"}, + json={"name": name, "mimeType": FOLDER_MIME, "parents": [parent_id]}, + ) + data = http.json_response(response, "Google-Ordner anlegen", provider=self.type_id) + folder_id = str(data.get("id") or "").strip() + if not folder_id: + raise StorageError("Google legte keinen Ordner an.", provider=self.type_id) + self._folder_cache[(parent_id, f"d:{name}")] = { + "id": folder_id, + "name": name, + "mimeType": FOLDER_MIME, + } + return folder_id + + def _root_folder_id(self) -> str: + configured = str(self.config.get("folder_id") or "").strip() + if configured: + return configured + # No explicit target folder: write below the Drive root. The optional + # base_path is created by _folder_for_parent() like any other segment. + return "root" + + def _folder_for_parent(self, remote_path: str, *, create: bool = True) -> Optional[str]: + parent = remote_parent(remote_path) + parts = parent.split("/") if parent else [] + current = self._root_folder_id() + for part in parts: + found = self._find_child(current, part, folder_only=True) + if found and found.get("id"): + current = str(found["id"]) + elif create: + current = self._create_folder(current, part) + else: + return None + return current + + # ------------------------------------------------------------------ + # Provider API + # ------------------------------------------------------------------ + def test_connection(self) -> dict: + response = self._api( + "GET", + "/about", + params={"fields": "user(displayName,emailAddress),storageQuota(limit,usage,usageInDrive)"}, + ) + data = http.json_response(response, "Google-Verbindungstest", provider=self.type_id) + quota = data.get("storageQuota") or {} + return { + "ok": True, + "account": (data.get("user") or {}).get("emailAddress") or (data.get("user") or {}).get("displayName") or "", + "quota": { + "limit": int(quota.get("limit") or 0), + "usage": int(quota.get("usage") or 0), + "usage_in_drive": int(quota.get("usageInDrive") or 0), + }, + } + + def quota(self) -> Optional[dict]: + response = self._api("GET", "/about", params={"fields": "storageQuota(limit,usage,usageInDrive)"}) + data = http.json_response(response, "Google-Quota", provider=self.type_id) + quota = data.get("storageQuota") or {} + return { + "limit": int(quota.get("limit") or 0), + "usage": int(quota.get("usage") or 0), + "usage_in_drive": int(quota.get("usageInDrive") or 0), + } + + def _upload_empty(self, name: str, parent_id: str, existing_id: str, mime: str) -> dict: + """Multipart upload for zero-byte files (resumable uploads need chunks).""" + boundary = "streamvault-empty-" + os.urandom(12).hex() + metadata = {"name": name} + if existing_id: + url = f"{DRIVE_UPLOAD_API}/files/{urllib.parse.quote(existing_id)}" + method = "PATCH" + else: + url = f"{DRIVE_UPLOAD_API}/files" + method = "POST" + metadata["parents"] = [parent_id] + body = ( + f"--{boundary}\r\n" + "Content-Type: application/json; charset=UTF-8\r\n\r\n" + f"{json.dumps(metadata)}\r\n" + f"--{boundary}\r\n" + f"Content-Type: {mime}\r\n\r\n" + f"\r\n--{boundary}--\r\n" + ).encode("utf-8") + response = None + for attempt in range(2): + response = http.request( + method, + url, + params={ + "uploadType": "multipart", + "supportsAllDrives": "true", + "fields": "id,name,size,md5Checksum,modifiedTime", + }, + headers={ + "Authorization": f"Bearer {self._token(force=attempt > 0)}", + "Content-Type": f"multipart/related; boundary={boundary}", + }, + data=body, + provider=self.type_id, + timeout=(10, 120), + ) + if response.status_code != 401 or attempt == 1: + break + try: + response.close() + except Exception: + pass + data = http.json_response(response, "Google-Upload (leer)", provider=self.type_id) + return { + "remote_id": str(data.get("id") or existing_id or ""), + "size": int(data.get("size") or 0), + "checksum": str(data.get("md5Checksum") or ""), + "modified_at": str(data.get("modifiedTime") or ""), + } + + def upload_file( + self, + local_path: str, + remote_path: str, + *, + progress: Optional[ProgressCallback] = None, + _auth_retried: bool = False, + ) -> dict: + remote_path = normalize_remote_path(remote_path) + size = file_size(local_path) + name = remote_name(remote_path) + parent_id = self._folder_for_parent(remote_path) + existing = self._find_child(parent_id, name) + existing_id = str((existing or {}).get("id") or "").strip() + mime = guess_mime_type(name, "video/mp4") + if size == 0: + result = self._upload_empty(name, parent_id, existing_id, mime) + if progress: + progress(0, 0) + return result + + params = { + "uploadType": "resumable", + "supportsAllDrives": "true", + "fields": "id,name,size,md5Checksum,modifiedTime", + } + if existing_id: + url = f"{DRIVE_UPLOAD_API}/files/{existing_id}" + method = "PATCH" + body = {"name": name} + else: + url = f"{DRIVE_UPLOAD_API}/files" + method = "POST" + body = {"name": name, "parents": [parent_id]} + response = None + for attempt in range(2): + headers = { + "Authorization": f"Bearer {self._token(force=attempt > 0)}", + "Content-Type": "application/json; charset=UTF-8", + "X-Upload-Content-Type": mime, + "X-Upload-Content-Length": str(size), + } + response = http.request( + method, + url, + params=params, + headers=headers, + data=json.dumps(body), + provider=self.type_id, + timeout=(10, 120), + ) + if response.status_code != 401 or attempt == 1: + break + try: + response.close() + except Exception: + pass + http.raise_for_status(response, "Google-Upload starten", provider=self.type_id) + location = str(response.headers.get("Location") or "").strip() + if not location: + raise StorageError("Google lieferte keine Upload-URL.", provider=self.type_id) + + offset = 0 + attempts = 0 + while offset < size: + end = min(offset + UPLOAD_CHUNK, size) - 1 + with open(local_path, "rb") as handle: + handle.seek(offset) + chunk = handle.read(end - offset + 1) + chunk_headers = { + "Content-Length": str(len(chunk)), + "Content-Range": f"bytes {offset}-{end}/{size}", + "Content-Type": mime, + } + try: + chunk_resp = http.request( + "PUT", + location, + headers=chunk_headers, + data=chunk, + provider=self.type_id, + timeout=(10, 300), + retries=2, + ) + except StorageError: + attempts += 1 + if attempts >= 4: + raise + time.sleep(min(15, 2 ** attempts)) + continue + if chunk_resp.status_code in (200, 201): + data = chunk_resp.json() if chunk_resp.content else {} + if progress: + progress(size, size) + return { + "remote_id": str((data or {}).get("id") or existing_id or ""), + "size": int((data or {}).get("size") or size), + "checksum": str((data or {}).get("md5Checksum") or ""), + "modified_at": str((data or {}).get("modifiedTime") or ""), + } + if chunk_resp.status_code == 308: + next_offset = offset + len(chunk) + range_header = str(chunk_resp.headers.get("Range") or "") + if range_header.startswith("bytes="): + try: + next_offset = int(range_header.split("-", 1)[1]) + 1 + except (IndexError, ValueError): + pass + offset = max(offset, min(size, next_offset)) + if progress: + progress(offset, size) + attempts = 0 + continue + if chunk_resp.status_code == 401: + if _auth_retried: + http.raise_for_status(chunk_resp, "Google-Upload", provider=self.type_id) + self._token(force=True) + return self.upload_file( + local_path, + remote_path, + progress=progress, + _auth_retried=True, + ) + if chunk_resp.status_code in http.RETRY_STATUS: + attempts += 1 + if attempts >= 4: + http.raise_for_status(chunk_resp, "Google-Upload", provider=self.type_id) + time.sleep(min(20, 2 ** attempts)) + continue + http.raise_for_status(chunk_resp, "Google-Upload", provider=self.type_id) + raise StorageError( + "Google-Upload endete ohne Abschlussantwort.", + provider=self.type_id, + retryable=True, + ) + + def _download_response(self, remote_path: str, *, range_header: str = "") -> "object": + remote_path = normalize_remote_path(remote_path) + parent_id = self._folder_for_parent(remote_path, create=False) + if not parent_id: + raise StorageError(f"Datei nicht auf Google Drive gefunden: {remote_path}", provider=self.type_id) + found = self._find_child(parent_id, remote_name(remote_path)) + if not found or not found.get("id"): + raise StorageError(f"Datei nicht auf Google Drive gefunden: {remote_path}", provider=self.type_id) + headers = self._headers() + if range_header: + headers["Range"] = range_header + return http.request( + "GET", + f"{DRIVE_API}/files/{urllib.parse.quote(str(found['id']))}", + params={"alt": "media", "supportsAllDrives": "true"}, + headers=headers, + provider=self.type_id, + stream=True, + timeout=(10, 300), + ), found + + def download_file( + self, + remote_path: str, + dest_path: str, + *, + progress: Optional[ProgressCallback] = None, + ) -> dict: + response, found = self._download_response(remote_path) + http.raise_for_status(response, "Google-Download", provider=self.type_id) + total = int(response.headers.get("Content-Length") or found.get("size") or 0) + tmp_path = f"{dest_path}.part" + os.makedirs(os.path.dirname(os.path.abspath(dest_path)), exist_ok=True) + done = 0 + try: + with open(tmp_path, "wb") as handle: + for chunk in response.iter_content(chunk_size=1024 * 1024): + if not chunk: + continue + handle.write(chunk) + done += len(chunk) + if progress: + progress(done, total) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp_path, dest_path) + finally: + try: + response.close() + except Exception: + pass + if os.path.exists(tmp_path): + try: + os.remove(tmp_path) + except OSError: + pass + return {"remote_id": str(found.get("id") or ""), "size": done} + + def open_range(self, remote_path: str, start: int, end: int) -> Iterable[bytes]: + start = max(0, int(start)) + end = max(start, int(end)) + response, _found = self._download_response(remote_path, range_header=f"bytes={start}-{end}") + http.raise_for_status(response, "Google-Range-Stream", provider=self.type_id) + try: + yield from iter_response_range(response, start, end) + finally: + try: + response.close() + except Exception: + pass + + def delete(self, remote_path: str) -> None: + remote_path = normalize_remote_path(remote_path) + parent_id = self._folder_for_parent(remote_path, create=False) + if not parent_id: + return + found = self._find_child(parent_id, remote_name(remote_path)) + if not found or not found.get("id"): + return + response = self._api( + "DELETE", + f"/files/{urllib.parse.quote(str(found['id']))}", + params={"supportsAllDrives": "true"}, + ) + if response.status_code == 404: + return + http.raise_for_status(response, "Google-Datei löschen", provider=self.type_id) + + def list(self, prefix: str = "") -> list[RemoteObject]: + prefix = str(prefix or "").strip().strip("/") + if prefix: + parent_id = self._folder_for_parent(f"{prefix}/x", create=False) + else: + base_path = str(self.config.get("base_path") or "").strip().strip("/") + parent_id = ( + self._folder_for_parent(f"{base_path}/x", create=False) + if base_path + else self._root_folder_id() + ) + if not parent_id: + return [] + params = { + "q": f"'{self._q(parent_id)}' in parents and trashed = false", + "fields": "nextPageToken,files(id,name,mimeType,size,modifiedTime)", + "pageSize": "1000", + "supportsAllDrives": "true", + "includeItemsFromAllDrives": "true", + } + items = [] + while True: + response = self._api("GET", "/files", params=params) + data = http.json_response(response, "Google-Dateiliste", provider=self.type_id) + for item in data.get("files") or []: + items.append( + RemoteObject( + path=(f"{prefix}/" if prefix else "") + str(item.get("name") or ""), + remote_id=str(item.get("id") or ""), + size=int(item.get("size") or 0), + modified_at=str(item.get("modifiedTime") or ""), + is_folder=item.get("mimeType") == FOLDER_MIME, + ) + ) + token = str(data.get("nextPageToken") or "") + if not token: + break + params["pageToken"] = token + return items + + def exists(self, remote_path: str, expected_size: int = 0) -> bool: + parent_id = self._folder_for_parent(remote_path, create=False) + if not parent_id: + return False + found = self._find_child(parent_id, remote_name(remote_path)) + if not found or not found.get("id"): + return False + expected = int(expected_size or 0) + if expected <= 0: + return True + return int(found.get("size") or 0) == expected diff --git a/core/storage/providers/onedrive.py b/core/storage/providers/onedrive.py new file mode 100644 index 0000000..cea5edc --- /dev/null +++ b/core/storage/providers/onedrive.py @@ -0,0 +1,424 @@ +"""Microsoft OneDrive provider via Microsoft Graph (OAuth 2.0).""" + +from __future__ import annotations + +import json +import os +import time +import urllib.parse +from typing import Iterable, Optional + +from core.storage.base import ( + ProgressCallback, + ProviderField, + RemoteObject, + StorageConfigError, + StorageError, + StorageProvider, + file_size, + guess_mime_type, + iter_response_range, + normalize_remote_path, + remote_name, + remote_parent, +) +from core.storage import http + +GRAPH = "https://graph.microsoft.com/v1.0" +AUTH_URL = "https://login.microsoftonline.com/{tenant}/oauth2/v2.0/authorize" +TOKEN_URL = "https://login.microsoftonline.com/{tenant}/oauth2/v2.0/token" +DEFAULT_SCOPE = "offline_access Files.ReadWrite" +SIMPLE_UPLOAD_MAX = 4 * 1024 * 1024 +SESSION_CHUNK = 10 * 1024 * 1024 # multiple of 320 KiB + + +class OneDriveProvider(StorageProvider): + type_id = "onedrive" + label = "Microsoft OneDrive" + description = "OneDrive / Microsoft 365 über Microsoft Graph mit OAuth 2.0." + oauth_supported = True + oauth_label = "Mit Microsoft verbinden" + docs_url = "https://learn.microsoft.com/graph/api/resources/driveitem" + config_fields = ( + ProviderField("client_id", "Application (Client) ID", required=True), + ProviderField("client_secret", "Client Secret", type="password", required=True, secret=True), + ProviderField("tenant_id", "Tenant ID", default="common", placeholder="common / contoso.onmicrosoft.com"), + ProviderField("refresh_token", "Refresh Token", type="password", secret=True, + help="Wird beim Verbinden automatisch gespeichert."), + ProviderField("scope", "OAuth Scope", default=DEFAULT_SCOPE), + ProviderField("base_path", "Unterordner", default="StreamVault", placeholder="StreamVault"), + ) + + def __init__(self, config: dict): + super().__init__(config) + self._access = "" + self._expires_at = 0.0 + + @classmethod + def oauth_ready(cls, config: dict) -> bool: + return bool(str(config.get("client_id") or "").strip() and str(config.get("client_secret") or "").strip()) + + @classmethod + def _tenant(cls, config: dict) -> str: + return str(config.get("tenant_id") or "common").strip() or "common" + + @classmethod + def oauth_authorize_url(cls, config: dict, redirect_uri: str, state: str) -> str: + if not cls.oauth_ready(config): + raise StorageConfigError("OneDrive Client ID und Client Secret fehlen.") + params = { + "client_id": str(config.get("client_id") or "").strip(), + "response_type": "code", + "redirect_uri": redirect_uri, + "response_mode": "query", + "scope": str(config.get("scope") or DEFAULT_SCOPE).strip() or DEFAULT_SCOPE, + "state": state, + } + return AUTH_URL.format(tenant=urllib.parse.quote(cls._tenant(config))) + "?" + urllib.parse.urlencode(params) + + @classmethod + def oauth_exchange_code(cls, config: dict, code: str, redirect_uri: str) -> dict: + response = http.request( + "POST", + TOKEN_URL.format(tenant=urllib.parse.quote(cls._tenant(config))), + data={ + "client_id": str(config.get("client_id") or "").strip(), + "client_secret": str(config.get("client_secret") or "").strip(), + "code": str(code or "").strip(), + "redirect_uri": redirect_uri, + "grant_type": "authorization_code", + "scope": str(config.get("scope") or DEFAULT_SCOPE).strip() or DEFAULT_SCOPE, + }, + provider=cls.type_id, + ) + data = http.json_response(response, "Microsoft-Token-Austausch", provider=cls.type_id) + refresh = str(data.get("refresh_token") or "").strip() + if not refresh: + raise StorageError("Microsoft lieferte keinen Refresh Token.", provider=cls.type_id) + return {"refresh_token": refresh} + + def _token(self, force: bool = False) -> str: + now = time.time() + if not force and self._access and now < self._expires_at - 60: + return self._access + refresh = str(self.config.get("refresh_token") or "").strip() + client_id = str(self.config.get("client_id") or "").strip() + client_secret = str(self.config.get("client_secret") or "").strip() + if not refresh or not client_id or not client_secret: + raise StorageConfigError("OneDrive ist nicht vollständig konfiguriert.") + response = http.request( + "POST", + TOKEN_URL.format(tenant=urllib.parse.quote(self._tenant(self.config))), + data={ + "client_id": client_id, + "client_secret": client_secret, + "refresh_token": refresh, + "grant_type": "refresh_token", + "scope": str(self.config.get("scope") or DEFAULT_SCOPE).strip() or DEFAULT_SCOPE, + }, + provider=self.type_id, + ) + data = http.json_response(response, "Microsoft-Token-Erneuerung", provider=self.type_id) + self._access = str(data.get("access_token") or "").strip() + if not self._access: + raise StorageError("Microsoft lieferte kein Access Token.", provider=self.type_id) + try: + self._expires_at = now + int(data.get("expires_in") or 3600) + except (TypeError, ValueError): + self._expires_at = now + 3600 + rotated = str(data.get("refresh_token") or "").strip() + if rotated and rotated != refresh: + # Microsoft may rotate refresh tokens. Persist the new value so a + # later worker process does not keep using an invalidated token. + self.config["refresh_token"] = rotated + self._config_changed() + return self._access + + def _headers(self, extra: Optional[dict] = None) -> dict: + headers = {"Authorization": f"Bearer {self._token()}"} + if extra: + headers.update(extra) + return headers + + def _path(self, remote_path: str) -> str: + path = normalize_remote_path(remote_path) + return "/".join(urllib.parse.quote(part, safe="") for part in path.split("/")) + + def _item_url(self, remote_path: str, suffix: str = "") -> str: + return f"{GRAPH}/me/drive/root:/{self._path(remote_path)}:{suffix}" + + def _api(self, method: str, url: str, **kwargs): + for attempt in range(2): + kwargs["headers"] = self._headers(**(kwargs.get("headers") or {})) + response = http.request(method, url, provider=self.type_id, **kwargs) + if response.status_code == 401 and attempt == 0: + try: + response.close() + except Exception: + pass + self._token(force=True) + continue + return response + return response + + def _ensure_dirs(self, remote_path: str) -> None: + parent = remote_parent(remote_path) + if not parent: + return + current = "" + for part in parent.split("/"): + # POST creates a child below the *existing* parent. The first + # segment therefore belongs to /root/children; later segments use + # the already-created folder path. + if current: + url = f"{GRAPH}/me/drive/root:/{self._path(current)}:/children" + else: + url = f"{GRAPH}/me/drive/root/children" + body = {"name": part, "folder": {}, "@microsoft.graph.conflictBehavior": "fail"} + response = self._api("POST", url, json=body, timeout=(10, 60)) + if response.status_code in (200, 201, 409): + current = f"{current}/{part}" if current else part + continue + http.raise_for_status(response, "OneDrive-Ordner anlegen", provider=self.type_id) + + def test_connection(self) -> dict: + response = self._api("GET", f"{GRAPH}/me/drive", params={"$select": "id,driveType,owner,quota"}) + data = http.json_response(response, "OneDrive-Verbindungstest", provider=self.type_id) + owner = data.get("owner") or {} + user = owner.get("user") or {} + quota = data.get("quota") or {} + return { + "ok": True, + "account": str(user.get("email") or user.get("displayName") or ""), + "quota": { + "limit": int(quota.get("total") or 0), + "usage": int(quota.get("used") or 0), + }, + } + + def quota(self) -> Optional[dict]: + try: + response = self._api("GET", f"{GRAPH}/me/drive", params={"$select": "quota"}) + data = http.json_response(response, "OneDrive-Quota", provider=self.type_id) + quota = data.get("quota") or {} + return {"limit": int(quota.get("total") or 0), "usage": int(quota.get("used") or 0)} + except Exception: + return None + + def upload_file( + self, + local_path: str, + remote_path: str, + *, + progress: Optional[ProgressCallback] = None, + ) -> dict: + size = file_size(local_path) + remote_path = normalize_remote_path(remote_path) + self._ensure_dirs(remote_path) + mime = guess_mime_type(remote_path, "video/mp4") + if size <= SIMPLE_UPLOAD_MAX: + with open(local_path, "rb") as handle: + payload = handle.read() + response = self._api( + "PUT", + self._item_url(remote_path, "/content"), + data=payload, + headers={"Content-Type": mime, "Content-Length": str(size)}, + timeout=(10, 300), + ) + data = http.json_response(response, "OneDrive-Upload", provider=self.type_id) + if progress: + progress(size, size) + return { + "remote_id": str(data.get("id") or remote_path), + "size": int(data.get("size") or size), + "checksum": "", + "modified_at": str(data.get("lastModifiedDateTime") or ""), + } + + response = self._api( + "POST", + self._item_url(remote_path, "/createUploadSession"), + json={"item": {"@microsoft.graph.conflictBehavior": "replace", "name": remote_name(remote_path)}}, + timeout=(10, 60), + ) + data = http.json_response(response, "OneDrive-Upload-Session", provider=self.type_id) + upload_url = str(data.get("uploadUrl") or "") + if not upload_url: + raise StorageError("OneDrive lieferte keine Upload-URL.", provider=self.type_id) + offset = 0 + attempts = 0 + with open(local_path, "rb") as handle: + while offset < size: + handle.seek(offset) + chunk = handle.read(min(SESSION_CHUNK, size - offset)) + end = offset + len(chunk) - 1 + try: + chunk_resp = http.request( + "PUT", + upload_url, + headers={ + "Content-Length": str(len(chunk)), + "Content-Range": f"bytes {offset}-{end}/{size}", + "Content-Type": mime, + }, + data=chunk, + provider=self.type_id, + timeout=(10, 300), + retries=2, + ) + except StorageError: + attempts += 1 + if attempts >= 4: + raise + time.sleep(min(15, 2 ** attempts)) + continue + if chunk_resp.status_code in (200, 201): + result = chunk_resp.json() if chunk_resp.content else {} + if progress: + progress(size, size) + return { + "remote_id": str((result or {}).get("id") or remote_path), + "size": int((result or {}).get("size") or size), + "checksum": "", + "modified_at": str((result or {}).get("lastModifiedDateTime") or ""), + } + if chunk_resp.status_code == 202: + offset = end + 1 + attempts = 0 + if progress: + progress(offset, size) + continue + if chunk_resp.status_code in http.RETRY_STATUS: + attempts += 1 + if attempts >= 4: + http.raise_for_status(chunk_resp, "OneDrive-Upload", provider=self.type_id) + time.sleep(min(20, 2 ** attempts)) + continue + http.raise_for_status(chunk_resp, "OneDrive-Upload", provider=self.type_id) + raise StorageError("OneDrive-Upload endete unerwartet.", provider=self.type_id) + + def _download_response(self, remote_path: str, *, range_header: str = ""): + headers = {"Range": range_header} if range_header else None + response = self._api( + "GET", + self._item_url(remote_path, "/content"), + headers=headers, + allow_redirects=False, + stream=True, + timeout=(10, 600), + ) + if response.status_code in (301, 302, 303, 307, 308): + location = response.headers.get("Location") or "" + response.close() + if not location: + raise StorageError("OneDrive-Download ohne Redirect-Ziel.", provider=self.type_id) + extra = {"Range": range_header} if range_header else None + response = http.request( + "GET", + location, + headers=extra, + stream=True, + provider=self.type_id, + timeout=(10, 600), + retries=2, + ) + return response + + def download_file( + self, + remote_path: str, + dest_path: str, + *, + progress: Optional[ProgressCallback] = None, + ) -> dict: + response = self._download_response(remote_path) + http.raise_for_status(response, "OneDrive-Download", provider=self.type_id) + total = int(response.headers.get("Content-Length") or 0) + tmp_path = f"{dest_path}.part" + os.makedirs(os.path.dirname(os.path.abspath(dest_path)), exist_ok=True) + done = 0 + try: + with open(tmp_path, "wb") as handle: + for chunk in response.iter_content(chunk_size=1024 * 1024): + if not chunk: + continue + handle.write(chunk) + done += len(chunk) + if progress: + progress(done, total) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp_path, dest_path) + finally: + try: + response.close() + except Exception: + pass + if os.path.exists(tmp_path): + try: + os.remove(tmp_path) + except OSError: + pass + return {"remote_id": remote_path, "size": done} + + def open_range(self, remote_path: str, start: int, end: int) -> Iterable[bytes]: + start = max(0, int(start)) + end = max(start, int(end)) + response = self._download_response(remote_path, range_header=f"bytes={start}-{end}") + http.raise_for_status(response, "OneDrive-Range-Stream", provider=self.type_id) + try: + yield from iter_response_range(response, start, end) + finally: + try: + response.close() + except Exception: + pass + + def delete(self, remote_path: str) -> None: + response = self._api("DELETE", self._item_url(remote_path), timeout=(10, 60)) + if response.status_code == 404: + return + http.raise_for_status(response, "OneDrive-Datei löschen", provider=self.type_id) + + def list(self, prefix: str = "") -> list[RemoteObject]: + prefix = str(prefix or "").strip().strip("/") + if prefix: + url = f"{GRAPH}/me/drive/root:/{self._path(prefix)}:/children" + else: + url = f"{GRAPH}/me/drive/root/children" + items = [] + params = {"$top": "1000"} + while url: + response = self._api("GET", url, params=params, timeout=(10, 60)) + data = http.json_response(response, "OneDrive-Dateiliste", provider=self.type_id) + for item in data.get("value") or []: + name = str(item.get("name") or "") + path = f"{prefix}/{name}" if prefix else name + items.append( + RemoteObject( + path=path, + remote_id=str(item.get("id") or path), + size=int(item.get("size") or 0), + modified_at=str(item.get("lastModifiedDateTime") or ""), + is_folder=bool(item.get("folder")), + ) + ) + url = str(data.get("@odata.nextLink") or "") + params = None + return items + + def exists(self, remote_path: str, expected_size: int = 0) -> bool: + response = self._api( + "GET", + self._item_url(remote_path), + params={"$select": "size"}, + timeout=(10, 30), + ) + if response.status_code == 404: + return False + data = http.json_response(response, "OneDrive-Existenzprüfung", provider=self.type_id) + expected = int(expected_size or 0) + if expected <= 0: + return True + return int(data.get("size") or 0) == expected diff --git a/core/storage/providers/s3.py b/core/storage/providers/s3.py new file mode 100644 index 0000000..0078f85 --- /dev/null +++ b/core/storage/providers/s3.py @@ -0,0 +1,324 @@ +"""S3-compatible object storage (AWS, R2, B2, Wasabi, MinIO, Hetzner, ...).""" + +from __future__ import annotations + +import os +import urllib.parse +from typing import Iterable, Optional + +from core.storage.base import ( + ProgressCallback, + ProviderField, + RemoteObject, + StorageConfigError, + StorageError, + StorageProvider, + file_size, + guess_mime_type, + normalize_remote_path, +) + +try: # optional dependency: the UI/CLI explains how to install it + import boto3 + from boto3.s3.transfer import TransferConfig + from botocore.config import Config as BotoConfig + from botocore.exceptions import BotoCoreError, ClientError +except Exception: # pragma: no cover - exercised only when boto3 is missing + boto3 = None + TransferConfig = None + BotoConfig = None + BotoCoreError = Exception + # An empty tuple keeps isinstance() checks valid when boto3 is absent. + ClientError = () + + +class S3Provider(StorageProvider): + type_id = "s3" + label = "S3-kompatibel" + description = "AWS S3, Cloudflare R2, Backblaze B2, Wasabi, MinIO, Hetzner oder jeder S3-kompatible Endpunkt." + docs_url = "https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html" + config_fields = ( + ProviderField("bucket", "Bucket", required=True, placeholder="streamvault-archive"), + ProviderField("access_key", "Access Key ID", required=True, secret=True), + ProviderField("secret_key", "Secret Access Key", type="password", required=True, secret=True), + ProviderField( + "region", + "Region", + placeholder="us-east-1 / eu-central-1 / auto", + help="Leer = us-east-1 für AWS. Cloudflare R2 verwendet 'auto'; " + "andere S3-Anbieter ihre jeweilige Region.", + ), + ProviderField( + "endpoint_url", + "Endpunkt-URL (optional)", + placeholder="https://.r2.cloudflarestorage.com", + help="Leer = AWS. Für R2, B2, Wasabi, MinIO etc. den S3-Endpunkt eintragen.", + ), + ProviderField( + "addressing_style", + "Adressierung", + type="select", + default="auto", + choices=(("auto", "Automatisch"), ("path", "Path-Style"), ("virtual", "Virtual-Hosted")), + ), + ProviderField("base_path", "Präfix", default="StreamVault", placeholder="StreamVault"), + ProviderField( + "server_side_encryption", + "Serverseitige Verschlüsselung", + type="select", + default="", + choices=(("", "Anbieter-Standard"), ("AES256", "S3 AES256"), ("aws:kms", "AWS KMS")), + ), + ) + + def __init__(self, config: dict): + super().__init__(config) + self._client_obj = None + + @classmethod + def validate_config(cls, config: dict, *, require_secrets: bool = True) -> list[str]: + errors = super().validate_config(config, require_secrets=require_secrets) + endpoint = str((config or {}).get("endpoint_url") or "").strip() + if endpoint: + parsed = urllib.parse.urlparse(endpoint) + if parsed.scheme not in ("http", "https") or not parsed.netloc: + errors.append("Endpunkt-URL muss eine gültige http(s)-URL sein.") + if parsed.username is not None or parsed.password is not None: + errors.append( + "Endpunkt-URL darf keine Zugangsdaten enthalten; " + "nutze Access Key und Secret Key." + ) + if parsed.query or parsed.fragment: + errors.append("Endpunkt-URL darf keine Query oder Fragment enthalten.") + bucket = str((config or {}).get("bucket") or "").strip() + if bucket and ("/" in bucket or " " in bucket): + errors.append("Bucket-Name enthält ungültige Zeichen.") + return errors + + def _client(self): + if boto3 is None: + raise StorageConfigError( + "S3-Unterstützung benötigt das Python-Paket 'boto3'. " + "Installation: pip install -r requirements.txt" + ) + if self._client_obj is not None: + return self._client_obj + cfg = self.config + bucket = str(cfg.get("bucket") or "").strip() + access = str(cfg.get("access_key") or "").strip() + secret = str(cfg.get("secret_key") or "").strip() + if not bucket or not access or not secret: + raise StorageConfigError("S3 ist nicht vollständig konfiguriert (Bucket, Access Key, Secret Key).") + addressing = str(cfg.get("addressing_style") or "auto").strip().lower() + if addressing not in ("auto", "path", "virtual"): + addressing = "auto" + boto_cfg = BotoConfig( + retries={"max_attempts": 4, "mode": "standard"}, + connect_timeout=10, + read_timeout=120, + signature_version="s3v4", + s3={"addressing_style": addressing}, + ) + kwargs = { + "aws_access_key_id": access, + "aws_secret_access_key": secret, + "region_name": str(cfg.get("region") or "").strip() or "us-east-1", + "config": boto_cfg, + } + endpoint = str(cfg.get("endpoint_url") or "").strip() + if endpoint: + kwargs["endpoint_url"] = endpoint + self._client_obj = boto3.client("s3", **kwargs) + return self._client_obj + + def _key(self, remote_path: str) -> str: + return normalize_remote_path(remote_path) + + def _extra_args(self) -> dict: + args = {} + encryption = str(self.config.get("server_side_encryption") or "").strip() + if encryption: + args["ServerSideEncryption"] = encryption + return args + + @staticmethod + def _error(action: str, exc: Exception) -> StorageError: + if isinstance(exc, ClientError): + error = (exc.response or {}).get("Error") or {} + code = str(error.get("Code") or "") + message = str(error.get("Message") or exc) + status = (exc.response or {}).get("ResponseMetadata", {}).get("HTTPStatusCode") + retryable = code in {"SlowDown", "RequestTimeout", "InternalError", "ServiceUnavailable"} or ( + isinstance(status, int) and status >= 500 + ) + return StorageError( + f"{action} fehlgeschlagen: {message}", + provider="s3", + status_code=status if isinstance(status, int) else None, + retryable=retryable, + details=code, + ) + return StorageError(f"{action} fehlgeschlagen: {exc}", provider="s3", retryable=True) + + def test_connection(self) -> dict: + try: + client = self._client() + bucket = str(self.config.get("bucket") or "").strip() + client.head_bucket(Bucket=bucket) + client.list_objects_v2(Bucket=bucket, Prefix=str(self.config.get("base_path") or "").strip("/"), MaxKeys=1) + return {"ok": True, "account": bucket, "quota": None} + except StorageError: + raise + except Exception as exc: + raise self._error("S3-Verbindungstest", exc) from exc + + def upload_file( + self, + local_path: str, + remote_path: str, + *, + progress: Optional[ProgressCallback] = None, + ) -> dict: + size = file_size(local_path) + key = self._key(remote_path) + callback = None + if progress: + def callback(bytes_amount): + progress(min(size, int(bytes_amount)), size) + + try: + client = self._client() + config = TransferConfig( + multipart_threshold=64 * 1024 * 1024, + multipart_chunksize=64 * 1024 * 1024, + max_concurrency=4, + use_threads=True, + ) + extra = {"ContentType": guess_mime_type(key, "video/mp4")} + extra.update(self._extra_args()) + client.upload_file( + local_path, + str(self.config.get("bucket") or "").strip(), + key, + ExtraArgs=extra, + Config=config, + Callback=callback, + ) + return {"remote_id": key, "size": size, "checksum": "", "modified_at": ""} + except StorageError: + raise + except Exception as exc: + raise self._error("S3-Upload", exc) from exc + + def download_file( + self, + remote_path: str, + dest_path: str, + *, + progress: Optional[ProgressCallback] = None, + ) -> dict: + key = self._key(remote_path) + tmp_path = f"{dest_path}.part" + os.makedirs(os.path.dirname(os.path.abspath(dest_path)), exist_ok=True) + callback = None + if progress: + def callback(bytes_amount): + progress(int(bytes_amount), 0) + try: + client = self._client() + client.download_file( + str(self.config.get("bucket") or "").strip(), + key, + tmp_path, + Callback=callback, + ) + os.replace(tmp_path, dest_path) + return {"remote_id": key, "size": os.path.getsize(dest_path)} + except StorageError: + raise + except Exception as exc: + raise self._error("S3-Download", exc) from exc + finally: + if os.path.exists(tmp_path): + try: + os.remove(tmp_path) + except OSError: + pass + + def open_range(self, remote_path: str, start: int, end: int) -> Iterable[bytes]: + key = self._key(remote_path) + try: + client = self._client() + response = client.get_object( + Bucket=str(self.config.get("bucket") or "").strip(), + Key=key, + Range=f"bytes={max(0, int(start))}-{max(0, int(end))}", + ) + except Exception as exc: + raise self._error("S3-Range-Stream", exc) from exc + body = response.get("Body") + if body is None: + raise StorageError("S3 lieferte keinen Stream.", provider="s3") + try: + while True: + chunk = body.read(256 * 1024) + if not chunk: + break + yield chunk + finally: + try: + body.close() + except Exception: + pass + + def delete(self, remote_path: str) -> None: + key = self._key(remote_path) + try: + client = self._client() + client.delete_object(Bucket=str(self.config.get("bucket") or "").strip(), Key=key) + except Exception as exc: + raise self._error("S3-Datei löschen", exc) from exc + + def list(self, prefix: str = "") -> list[RemoteObject]: + base = normalize_remote_path(prefix) if str(prefix or "").strip() else "" + try: + client = self._client() + paginator = client.get_paginator("list_objects_v2") + kwargs = {"Bucket": str(self.config.get("bucket") or "").strip()} + if base: + kwargs["Prefix"] = base.rstrip("/") + "/" + items = [] + for page in paginator.paginate(**kwargs): + for item in page.get("Contents") or []: + items.append( + RemoteObject( + path=str(item.get("Key") or ""), + remote_id=str(item.get("Key") or ""), + size=int(item.get("Size") or 0), + modified_at=str(item.get("LastModified") or ""), + ) + ) + return items + except Exception as exc: + raise self._error("S3-Dateiliste", exc) from exc + + def exists(self, remote_path: str, expected_size: int = 0) -> bool: + key = self._key(remote_path) + try: + client = self._client() + head = client.head_object( + Bucket=str(self.config.get("bucket") or "").strip(), + Key=key, + ) + except Exception as exc: + if isinstance(exc, ClientError): + error = (exc.response or {}).get("Error") or {} + code = str(error.get("Code") or "") + status = (exc.response or {}).get("ResponseMetadata", {}).get("HTTPStatusCode") + if code in {"404", "NoSuchKey", "NotFound"} or status == 404: + return False + raise self._error("S3-Existenzprüfung", exc) from exc + expected = int(expected_size or 0) + if expected <= 0: + return True + return int(head.get("ContentLength") or 0) == expected diff --git a/core/storage/providers/webdav.py b/core/storage/providers/webdav.py new file mode 100644 index 0000000..0914d88 --- /dev/null +++ b/core/storage/providers/webdav.py @@ -0,0 +1,315 @@ +"""Generic WebDAV provider (Nextcloud, ownCloud, Hetzner Storage Box, ...).""" + +from __future__ import annotations + +import os +import urllib.parse +import xml.etree.ElementTree as ET +from typing import Iterable, Optional + +from core.storage.base import ( + ProgressCallback, + ProviderField, + RemoteObject, + StorageConfigError, + StorageError, + StorageProvider, + file_size, + guess_mime_type, + iter_response_range, + normalize_remote_path, + remote_name, + remote_parent, +) +from core.storage import http + +DAV = "{DAV:}" + + +class WebDAVProvider(StorageProvider): + type_id = "webdav" + label = "WebDAV" + description = "Nextcloud, ownCloud, Hetzner Storage Box oder jeder WebDAV-Server mit App-Passwort." + docs_url = "https://datatracker.ietf.org/doc/html/rfc4918" + config_fields = ( + ProviderField( + "base_url", + "WebDAV-Basis-URL", + required=True, + placeholder="https://cloud.example.com/remote.php/dav/files/user/", + help="Muss auf einen beschreibbaren Ordner zeigen und mit / enden (wird automatisch ergänzt).", + ), + ProviderField("username", "Benutzername", required=True), + ProviderField("password", "Passwort / App-Passwort", type="password", required=True, secret=True), + ProviderField("base_path", "Unterordner", default="StreamVault", placeholder="StreamVault"), + ProviderField( + "verify_ssl", + "TLS-Zertifikat prüfen", + type="select", + default="1", + choices=(("1", "Ja (empfohlen)"), ("0", "Nein (nur bei Self-Signed)")), + ), + ) + + @classmethod + def validate_config(cls, config: dict, *, require_secrets: bool = True) -> list[str]: + errors = super().validate_config(config, require_secrets=require_secrets) + base = str((config or {}).get("base_url") or "").strip() + if base: + parsed = urllib.parse.urlparse(base) + if parsed.scheme not in ("http", "https") or not parsed.netloc: + errors.append("WebDAV-Basis-URL muss eine gültige http(s)-URL sein.") + if parsed.username is not None or parsed.password is not None: + errors.append( + "WebDAV-Basis-URL darf keine Zugangsdaten enthalten; " + "nutze die Felder Benutzername und Passwort." + ) + if parsed.query or parsed.fragment: + errors.append("WebDAV-Basis-URL darf keine Query oder Fragment enthalten.") + return errors + + def _base(self) -> str: + base = str(self.config.get("base_url") or "").strip() + if not base: + raise StorageConfigError("WebDAV-Basis-URL fehlt.") + return base.rstrip("/") + "/" + + def _auth(self): + user = str(self.config.get("username") or "") + password = str(self.config.get("password") or "") + return (user, password) if user else None + + def _verify(self) -> bool: + return str(self.config.get("verify_ssl") or "1").strip().lower() not in ("0", "false", "no", "off") + + def _url(self, remote_path: str) -> str: + path = normalize_remote_path(remote_path) + quoted = "/".join(urllib.parse.quote(part, safe="") for part in path.split("/")) + return self._base() + quoted + + def _request(self, method: str, url: str, **kwargs): + kwargs.setdefault("auth", self._auth()) + kwargs.setdefault("verify", self._verify()) + kwargs.setdefault("provider", self.type_id) + return http.request(method, url, **kwargs) + + def _ensure_dirs(self, remote_path: str) -> None: + parent = remote_parent(remote_path) + if not parent: + return + parts = parent.split("/") + current = "" + for part in parts: + current = f"{current}/{part}" if current else part + url = self._url(current) + response = self._request("MKCOL", url, timeout=(10, 30), retries=2) + if response.status_code in (200, 201, 204, 301, 302, 405): + continue + if response.status_code == 409: + # Parent was just created; retry once. + response = self._request("MKCOL", url, timeout=(10, 30), retries=1) + if response.status_code in (200, 201, 204, 405): + continue + http.raise_for_status(response, "WebDAV-Ordner anlegen", provider=self.type_id) + + def test_connection(self) -> dict: + response = self._request( + "PROPFIND", + self._base(), + headers={"Depth": "0", "Content-Type": "application/xml; charset=utf-8"}, + data='', + timeout=(10, 30), + ) + if response.status_code not in (200, 207): + http.raise_for_status(response, "WebDAV-Verbindungstest", provider=self.type_id) + return {"ok": True, "account": str(self.config.get("username") or ""), "quota": self.quota()} + + def upload_file( + self, + local_path: str, + remote_path: str, + *, + progress: Optional[ProgressCallback] = None, + ) -> dict: + remote_path = normalize_remote_path(remote_path) + size = file_size(local_path) + self._ensure_dirs(remote_path) + url = self._url(remote_path) + with open(local_path, "rb") as handle: + response = self._request( + "PUT", + url, + data=handle, + headers={ + "Content-Type": guess_mime_type(remote_path, "video/mp4"), + "Content-Length": str(size), + }, + timeout=(10, 600), + retries=1, + ) + http.raise_for_status(response, "WebDAV-Upload", provider=self.type_id) + if progress: + progress(size, size) + return {"remote_id": remote_path, "size": size, "checksum": "", "modified_at": ""} + + def download_file( + self, + remote_path: str, + dest_path: str, + *, + progress: Optional[ProgressCallback] = None, + ) -> dict: + remote_path = normalize_remote_path(remote_path) + response = self._request( + "GET", + self._url(remote_path), + stream=True, + timeout=(10, 600), + ) + http.raise_for_status(response, "WebDAV-Download", provider=self.type_id) + total = int(response.headers.get("Content-Length") or 0) + tmp_path = f"{dest_path}.part" + os.makedirs(os.path.dirname(os.path.abspath(dest_path)), exist_ok=True) + done = 0 + try: + with open(tmp_path, "wb") as handle: + for chunk in response.iter_content(chunk_size=1024 * 1024): + if not chunk: + continue + handle.write(chunk) + done += len(chunk) + if progress: + progress(done, total) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp_path, dest_path) + finally: + try: + response.close() + except Exception: + pass + if os.path.exists(tmp_path): + try: + os.remove(tmp_path) + except OSError: + pass + return {"remote_id": remote_path, "size": done} + + def open_range(self, remote_path: str, start: int, end: int) -> Iterable[bytes]: + remote_path = normalize_remote_path(remote_path) + response = self._request( + "GET", + self._url(remote_path), + headers={"Range": f"bytes={max(0, int(start))}-{max(0, int(end))}"}, + stream=True, + timeout=(10, 300), + ) + http.raise_for_status(response, "WebDAV-Range-Stream", provider=self.type_id) + try: + yield from iter_response_range(response, start, end) + finally: + try: + response.close() + except Exception: + pass + + def delete(self, remote_path: str) -> None: + response = self._request("DELETE", self._url(remote_path), timeout=(10, 60), retries=2) + if response.status_code == 404: + return + http.raise_for_status(response, "WebDAV-Datei löschen", provider=self.type_id) + + def list(self, prefix: str = "") -> list[RemoteObject]: + prefix = str(prefix or "").strip().strip("/") + url = self._url(prefix) if prefix else self._base() + response = self._request( + "PROPFIND", + url, + headers={"Depth": "1", "Content-Type": "application/xml; charset=utf-8"}, + data='', + timeout=(10, 60), + ) + if response.status_code not in (200, 207): + http.raise_for_status(response, "WebDAV-Dateiliste", provider=self.type_id) + try: + root = ET.fromstring(response.content) + except ET.ParseError as exc: + raise StorageError("WebDAV lieferte ungültiges XML.", provider=self.type_id) from exc + items = [] + base_path = urllib.parse.unquote( + urllib.parse.urlparse(self._base()).path.rstrip("/") + ) + for node in root.findall(f".//{DAV}response"): + href = node.findtext(f"{DAV}href") or "" + path = urllib.parse.unquote(urllib.parse.urlparse(href).path) + if path.startswith(base_path): + path = path[len(base_path):].lstrip("/") + if not path or path.rstrip("/") == prefix.rstrip("/"): + continue + prop = node.find(f"{DAV}propstat/{DAV}prop") + if prop is None: + continue + is_folder = prop.find(f"{DAV}resourcetype/{DAV}collection") is not None + size_text = prop.findtext(f"{DAV}getcontentlength") or "0" + try: + size = int(size_text) + except ValueError: + size = 0 + items.append( + RemoteObject( + path=path.rstrip("/"), + remote_id=path.rstrip("/"), + size=size, + modified_at=prop.findtext(f"{DAV}getlastmodified") or "", + is_folder=is_folder, + ) + ) + return items + + def exists(self, remote_path: str, expected_size: int = 0) -> bool: + response = self._request( + "HEAD", + self._url(remote_path), + timeout=(10, 30), + ) + if response.status_code == 404: + return False + if response.status_code in (405, 501): + return super().exists(remote_path, expected_size) + if response.status_code not in (200, 204): + http.raise_for_status(response, "WebDAV-Existenzprüfung", provider=self.type_id) + size_text = str(response.headers.get("Content-Length") or "0").strip() + try: + size = int(size_text) + except ValueError: + size = 0 + expected = int(expected_size or 0) + if expected <= 0: + return True + return size == expected + + def quota(self) -> Optional[dict]: + try: + response = self._request( + "PROPFIND", + self._base(), + headers={"Depth": "0", "Content-Type": "application/xml; charset=utf-8"}, + data=( + '' + '' + '' + ), + timeout=(10, 30), + ) + if response.status_code not in (200, 207): + return None + root = ET.fromstring(response.content) + available = root.findtext(f".//{DAV}quota-available-bytes") + used = root.findtext(f".//{DAV}quota-used-bytes") + if available is None and used is None: + return None + available_i = int(available) if available and available.isdigit() else 0 + used_i = int(used) if used and used.isdigit() else 0 + return {"limit": available_i + used_i, "usage": used_i} + except Exception: + return None diff --git a/core/storage/registry.py b/core/storage/registry.py new file mode 100644 index 0000000..e4d09ba --- /dev/null +++ b/core/storage/registry.py @@ -0,0 +1,49 @@ +"""Provider registry used by WebUI, worker and CLI.""" + +from __future__ import annotations + +from .base import StorageConfigError, StorageProvider +from .providers.dropbox import DropboxProvider +from .providers.google_drive import GoogleDriveProvider +from .providers.onedrive import OneDriveProvider +from .providers.s3 import S3Provider +from .providers.webdav import WebDAVProvider + +PROVIDER_CLASSES: dict[str, type[StorageProvider]] = { + cls.type_id: cls + for cls in ( + GoogleDriveProvider, + S3Provider, + DropboxProvider, + OneDriveProvider, + WebDAVProvider, + ) +} + + +def get_provider_class(type_id: str) -> type[StorageProvider]: + key = str(type_id or "").strip().lower() + cls = PROVIDER_CLASSES.get(key) + if cls is None: + raise StorageConfigError(f"Unbekannter Speicher-Anbieter: {type_id}") + return cls + + +def provider_catalog() -> list[dict]: + """Serialisable provider metadata for templates and CLI output.""" + catalog = [] + for type_id, cls in PROVIDER_CLASSES.items(): + catalog.append( + { + "type_id": type_id, + "label": cls.label, + "description": cls.description, + "oauth_supported": bool(cls.oauth_supported), + "oauth_label": cls.oauth_label, + "docs_url": cls.docs_url, + "fields": [field.as_dict() for field in cls.config_fields], + "secret_keys": sorted(cls.secret_keys()), + "default_config": cls.default_config(), + } + ) + return catalog diff --git a/core/storage/service.py b/core/storage/service.py new file mode 100644 index 0000000..675cdad --- /dev/null +++ b/core/storage/service.py @@ -0,0 +1,1518 @@ +"""Database-backed external storage orchestration. + +This module is the single integration point for the WebUI, worker and CLI. It +owns provider configuration (encrypted at rest), archive object bookkeeping, +remote path building, upload/restore/delete operations and the safe local +retention rules. +""" + +from __future__ import annotations + +import logging +import os +import shutil +import stat +import time +from typing import Optional + +from itsdangerous import BadSignature, SignatureExpired, URLSafeTimedSerializer + +from core import config, database +from core.mp4_utils import is_transient_media_filename + +from .base import ( + StorageConfigError, + StorageError, + mask_error, + mask_text, + normalize_remote_path, +) +from .crypto import ( + decrypt_config, + encrypt_config, + encryption_ready, + signing_secret, +) +from .registry import PROVIDER_CLASSES, get_provider_class, provider_catalog + +logger = logging.getLogger("storage.service") + +SETTINGS_KEY = "storage_settings" +SCHEDULE_KEY = "storage_schedule" +OAUTH_SALT = "streamvault-storage-oauth-v1" +OAUTH_MAX_AGE = 900 + +DEFAULT_SETTINGS = { + "auto_upload": False, + "delete_local_after_upload": False, + "local_retention_days": 14, + "verify_before_local_delete": True, + "default_provider_id": None, + "auto_sync_interval_minutes": 10, + "cleanup_interval_minutes": 360, + "max_sync_files": 500, + "include_chat_logs": False, + "include_thumbnails": False, +} + +VIDEO_EXTENSIONS = { + ".mp4", + ".mkv", + ".flv", + ".ts", + ".mov", + ".webm", + ".m4a", + ".mp3", + ".aac", + ".ogg", + ".wav", + ".flac", +} +SIDECAR_EXTENSIONS = {".txt", ".vtt", ".srt", ".json"} +THUMB_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp"} +SKIP_DIRS = { + ".hls", + ".hls_live", + ".meta", + ".ruff_cache", + "__pycache__", + "bin", + "content", + "core", + "docs", + "files", + "node_modules", + "recorder", + "scripts", + "static", + "tests", + "venv", + "WEBUI", +} +SKIP_SUFFIXES = (".part", ".tmp", ".temp", ".lock", ".db-wal", ".db-shm") +THUMB_DIR = ".thumbs" + + +def _safe_error(exc: Exception) -> str: + """Sanitised error text with the HTTP status kept for diagnostics.""" + message = mask_error(exc) + status = getattr(exc, "status_code", None) + if status and f"HTTP {status}" not in message: + message = f"{message} (HTTP {status})" + return message[:500] + + +def _as_bool(value, default=False): + if value is None: + return bool(default) + if isinstance(value, bool): + return value + return str(value).strip().lower() in ("1", "true", "yes", "on") + + +def _as_int(value, default, minimum=0, maximum=100000): + try: + number = int(value) + except (TypeError, ValueError): + return int(default) + return max(int(minimum), min(int(maximum), number)) + + +def _clean_provider_id(value): + try: + number = int(value) + except (TypeError, ValueError): + return None + return number if number > 0 else None + + +# --------------------------------------------------------------------------- +# Settings +# --------------------------------------------------------------------------- + +def get_settings() -> dict: + raw = database.get_system_state(SETTINGS_KEY, {}) + if not isinstance(raw, dict): + raw = {} + settings = dict(DEFAULT_SETTINGS) + settings["auto_upload"] = _as_bool(raw.get("auto_upload"), False) + settings["delete_local_after_upload"] = _as_bool( + raw.get("delete_local_after_upload"), False + ) + settings["verify_before_local_delete"] = _as_bool( + raw.get("verify_before_local_delete"), True + ) + settings["include_chat_logs"] = _as_bool(raw.get("include_chat_logs"), False) + settings["include_thumbnails"] = _as_bool(raw.get("include_thumbnails"), False) + settings["local_retention_days"] = _as_int( + raw.get("local_retention_days"), 14, 0, 3650 + ) + settings["auto_sync_interval_minutes"] = _as_int( + raw.get("auto_sync_interval_minutes"), 10, 0, 10080 + ) + settings["cleanup_interval_minutes"] = _as_int( + raw.get("cleanup_interval_minutes"), 360, 0, 10080 + ) + settings["max_sync_files"] = _as_int(raw.get("max_sync_files"), 500, 1, 5000) + default_id = _clean_provider_id(raw.get("default_provider_id")) + settings["default_provider_id"] = default_id + return settings + + +def save_settings(values: dict) -> dict: + settings = get_settings() + values = values or {} + for key in ( + "auto_upload", + "delete_local_after_upload", + "verify_before_local_delete", + "include_chat_logs", + "include_thumbnails", + ): + if key in values: + settings[key] = _as_bool(values.get(key)) + if "local_retention_days" in values: + settings["local_retention_days"] = _as_int( + values.get("local_retention_days"), settings["local_retention_days"], 0, 3650 + ) + if "auto_sync_interval_minutes" in values: + settings["auto_sync_interval_minutes"] = _as_int( + values.get("auto_sync_interval_minutes"), + settings["auto_sync_interval_minutes"], + 0, + 10080, + ) + if "cleanup_interval_minutes" in values: + settings["cleanup_interval_minutes"] = _as_int( + values.get("cleanup_interval_minutes"), + settings["cleanup_interval_minutes"], + 0, + 10080, + ) + if "max_sync_files" in values: + settings["max_sync_files"] = _as_int( + values.get("max_sync_files"), settings["max_sync_files"], 1, 5000 + ) + if "default_provider_id" in values: + provider_id = _clean_provider_id(values.get("default_provider_id")) + if provider_id is not None: + row = database.get_storage_provider(provider_id) + if not row: + raise StorageConfigError("Der ausgewählte Standard-Anbieter existiert nicht.") + if not row.get("enabled"): + raise StorageConfigError( + "Der ausgewählte Standard-Anbieter ist deaktiviert. " + "Aktiviere ihn zuerst oder wähle einen anderen Anbieter." + ) + settings["default_provider_id"] = provider_id + database.set_system_state(SETTINGS_KEY, settings) + return settings + + +# --------------------------------------------------------------------------- +# Provider configuration +# --------------------------------------------------------------------------- + +def _decrypt_provider_row(row: dict) -> dict: + row = dict(row or {}) + encrypted = row.get("config") + row["config_error"] = "" + try: + row["config"] = decrypt_config(encrypted) + except StorageConfigError as exc: + row["config"] = {} + row["config_error"] = str(exc) + return row + + +def get_provider_record(provider_id, *, include_config=True) -> Optional[dict]: + row = database.get_storage_provider(provider_id) + if not row: + return None + row = _decrypt_provider_row(row) + if include_config: + return row + return _public_provider(row) + + +def _public_provider(row: dict) -> dict: + row = dict(row or {}) + cls = PROVIDER_CLASSES.get(str(row.get("provider_type") or "").strip().lower()) + cfg = row.get("config") if isinstance(row.get("config"), dict) else {} + row["type_label"] = cls.label if cls else str(row.get("provider_type") or "") + row["oauth_supported"] = bool(getattr(cls, "oauth_supported", False)) + row["oauth_label"] = getattr(cls, "oauth_label", "Konto verbinden") + row["docs_url"] = getattr(cls, "docs_url", "") + row["config"] = cls.public_config(cfg) if cls else {} + row["config_set"] = bool(cfg) + row["config_error"] = mask_text(row.get("config_error")) + row["last_error"] = mask_text(row.get("last_error")) + row["last_check_message"] = mask_text(row.get("last_check_message")) + return row + + +def list_providers(*, include_config=False) -> list[dict]: + out = [] + for row in database.list_storage_providers(): + row = _decrypt_provider_row(row) + out.append(row if include_config else _public_provider(row)) + return out + + +def _apply_defaults(cls, values: dict) -> dict: + """Fill defaults only for fields the caller did not provide. + + An explicitly submitted empty value is meaningful (for example + ``base_path`` means "write directly into the target folder"), so it must + not be silently replaced by a default. + """ + merged = dict(values or {}) + for field in cls.config_fields: + if not field.default: + continue + if field.key not in merged or merged[field.key] is None: + merged[field.key] = field.default + return merged + + +def _validate_provider_paths(cls, values: dict) -> dict: + """Normalise and validate path-like provider settings at save time.""" + merged = dict(values or {}) + if "base_path" in {field.key for field in cls.config_fields}: + base = str(merged.get("base_path") or "").strip() + if base: + try: + merged["base_path"] = normalize_remote_path(base) + except StorageConfigError as exc: + raise StorageConfigError(f"Ungültiger Unterordner: {exc}") from exc + else: + merged["base_path"] = "" + return merged + + +def _config_errors(cls, values: dict) -> list[str]: + errors = cls.validate_config(values, require_secrets=True) + if cls.oauth_supported: + # OAuth providers are allowed to be saved before the account is linked. + oauth_secret_keys = {"refresh_token", "access_token"} + labels = { + f"{field.label} fehlt." + for field in cls.config_fields + if field.key in oauth_secret_keys + } + errors = [error for error in errors if error not in labels] + return errors + + +def _clean_name(name: str) -> str: + value = str(name or "").strip() + if not value: + raise StorageConfigError("Bitte einen Namen für den Speicher-Anbieter angeben.") + if len(value) > 80: + raise StorageConfigError("Der Anbietername darf höchstens 80 Zeichen lang sein.") + return value + + +def _ensure_unique_name(name: str, *, exclude_id=None): + wanted = name.lower() + for row in database.list_storage_providers(): + if row["id"] == exclude_id: + continue + if str(row.get("name") or "").strip().lower() == wanted: + raise StorageConfigError(f"Ein Anbieter mit dem Namen '{name}' existiert bereits.") + + +def create_provider(name: str, provider_type: str, values: dict, *, enabled=True) -> int: + if not encryption_ready(): + raise StorageConfigError( + "VOD_SECRET_KEY muss auf einen starken Wert gesetzt sein, bevor " + "Zugangsdaten gespeichert werden können." + ) + name = _clean_name(name) + _ensure_unique_name(name) + cls = get_provider_class(provider_type) + values = _validate_provider_paths(cls, _apply_defaults(cls, values)) + errors = _config_errors(cls, values) + if errors: + raise StorageConfigError(" ".join(errors)) + provider_id = database.create_storage_provider( + name, cls.type_id, encrypt_config(values), enabled=enabled + ) + settings = get_settings() + if enabled and not settings.get("default_provider_id"): + save_settings({"default_provider_id": provider_id}) + return provider_id + + +def update_provider( + provider_id: int, + *, + name: Optional[str] = None, + provider_type: Optional[str] = None, + values: Optional[dict] = None, + enabled: Optional[bool] = None, +) -> dict: + existing = get_provider_record(provider_id) + if not existing: + raise StorageConfigError("Speicher-Anbieter nicht gefunden.") + new_type = str(provider_type or existing["provider_type"]).strip().lower() + cls = get_provider_class(new_type) + type_changed = new_type != str(existing["provider_type"]).strip().lower() + if type_changed and database.list_storage_objects( + provider_id=provider_id, limit=1 + ): + raise StorageConfigError( + "Der Anbieter-Typ kann nicht geändert werden, solange Archiv-Objekte " + "zugeordnet sind. Lege stattdessen einen neuen Anbieter an." + ) + current = {} if type_changed else dict(existing.get("config") or {}) + if type_changed: + current = cls.default_config() + values = values or {} + valid_keys = {field.key for field in cls.config_fields} + for key, value in values.items(): + if key not in valid_keys: + continue + field = next(item for item in cls.config_fields if item.key == key) + text = "" if value is None else str(value) + if field.secret and text in ("", "••••••••", "********"): + continue + current[key] = text + current = _validate_provider_paths(cls, _apply_defaults(cls, current)) + errors = _config_errors(cls, current) + if errors: + raise StorageConfigError(" ".join(errors)) + clean_name = _clean_name(name if name is not None else existing["name"]) + _ensure_unique_name(clean_name, exclude_id=int(provider_id)) + database.update_storage_provider( + provider_id, + name=clean_name, + provider_type=cls.type_id, + config=encrypt_config(current), + enabled=enabled if enabled is not None else existing["enabled"], + ) + updated = get_provider_record(provider_id) + settings = get_settings() + if not updated.get("enabled") and settings.get("default_provider_id") == int(provider_id): + fallback = next( + (row["id"] for row in database.list_storage_providers() if row.get("enabled")), + None, + ) + save_settings({"default_provider_id": fallback}) + elif updated.get("enabled") and not settings.get("default_provider_id"): + save_settings({"default_provider_id": provider_id}) + return updated + + +def delete_provider(provider_id: int) -> bool: + existing = get_provider_record(provider_id) + if not existing: + return False + removed = database.delete_storage_provider(provider_id) + settings = get_settings() + if settings.get("default_provider_id") == int(provider_id): + fallback = next( + (row["id"] for row in database.list_storage_providers() if row.get("enabled")), + None, + ) + save_settings({"default_provider_id": fallback}) + return removed + + +def resolve_provider(provider_id=None) -> dict: + if provider_id is not None: + row = get_provider_record(provider_id) + if not row: + raise StorageConfigError("Speicher-Anbieter nicht gefunden.") + if not row.get("enabled"): + raise StorageConfigError("Der Speicher-Anbieter ist deaktiviert.") + if row.get("config_error"): + raise StorageConfigError(row["config_error"]) + return row + settings = get_settings() + row = None + default_id = settings.get("default_provider_id") + if default_id: + row = database.get_storage_provider(default_id) + if row and not row.get("enabled"): + row = None + if row is None: + row = next( + (item for item in database.list_storage_providers() if item.get("enabled")), + None, + ) + if row is None: + raise StorageConfigError("Kein aktiver Cloud-Speicher konfiguriert.") + row = _decrypt_provider_row(row) + if row.get("config_error"): + raise StorageConfigError(row["config_error"]) + return row + + +def _persist_provider_config(provider_id, updated: dict) -> None: + """Merge a provider's rotated credentials back into encrypted storage.""" + if not provider_id: + return + current = get_provider_record(provider_id) + if not current: + return + merged = dict(current.get("config") or {}) + merged.update(updated or {}) + database.update_storage_provider( + int(provider_id), config=encrypt_config(merged) + ) + + +def build_provider(row: dict): + cls = get_provider_class(row.get("provider_type")) + provider = cls(row.get("config") or {}) + provider_id = row.get("id") + if provider_id is not None: + provider.set_config_changed_callback( + lambda updated: _persist_provider_config(provider_id, updated) + ) + return provider + + +def test_provider(provider_id: int) -> dict: + row = get_provider_record(provider_id) + if not row: + raise StorageConfigError("Speicher-Anbieter nicht gefunden.") + if row.get("config_error"): + message = row["config_error"] + database.set_storage_provider_check(provider_id, False, message) + return {"ok": False, "message": message} + try: + provider = build_provider(row) + result = provider.test_connection() or {} + account = str(result.get("account") or "") + message = f"Verbindung erfolgreich{f' ({account})' if account else ''}." + database.set_storage_provider_check(provider_id, True, message) + database.set_storage_provider_error(provider_id, "") + return { + "ok": True, + "message": message, + "account": account, + "quota": result.get("quota"), + } + except StorageError as exc: + message = _safe_error(exc) + database.set_storage_provider_check(provider_id, False, message) + return {"ok": False, "message": message} + except Exception as exc: + message = f"Verbindung fehlgeschlagen: {_safe_error(exc)}" + database.set_storage_provider_check(provider_id, False, message) + return {"ok": False, "message": message} + + +# --------------------------------------------------------------------------- +# OAuth helpers +# --------------------------------------------------------------------------- + +def _oauth_serializer() -> URLSafeTimedSerializer: + return URLSafeTimedSerializer(signing_secret(), salt=OAUTH_SALT) + + +def make_oauth_state(provider_id: int) -> str: + return _oauth_serializer().dumps({"pid": int(provider_id)}) + + +def verify_oauth_state(state: str) -> int: + try: + payload = _oauth_serializer().loads(str(state or ""), max_age=OAUTH_MAX_AGE) + except SignatureExpired as exc: + raise StorageConfigError("Der OAuth-Link ist abgelaufen. Bitte erneut starten.") from exc + except (BadSignature, TypeError, ValueError) as exc: + raise StorageConfigError("Ungültiger OAuth-Status.") from exc + if not isinstance(payload, dict): + raise StorageConfigError("Ungültiger OAuth-Status.") + provider_id = _clean_provider_id(payload.get("pid")) + if not provider_id: + raise StorageConfigError("Ungültiger OAuth-Status.") + return provider_id + + +def oauth_authorize_url(provider_id: int, redirect_uri: str) -> str: + row = get_provider_record(provider_id) + if not row: + raise StorageConfigError("Speicher-Anbieter nicht gefunden.") + cls = get_provider_class(row["provider_type"]) + if not cls.oauth_supported: + raise StorageConfigError(f"{cls.label} unterstützt OAuth nicht.") + if not cls.oauth_ready(row.get("config") or {}): + raise StorageConfigError( + "Bitte zuerst Client-ID und Client-Secret speichern, dann verbinden." + ) + state = make_oauth_state(provider_id) + return cls.oauth_authorize_url(row.get("config") or {}, redirect_uri, state) + + +def oauth_complete(provider_id: int, code: str, redirect_uri: str) -> dict: + row = get_provider_record(provider_id) + if not row: + raise StorageConfigError("Speicher-Anbieter nicht gefunden.") + cls = get_provider_class(row["provider_type"]) + if not cls.oauth_supported: + raise StorageConfigError(f"{cls.label} unterstützt OAuth nicht.") + values = dict(row.get("config") or {}) + received = cls.oauth_exchange_code(values, code, redirect_uri) + if not isinstance(received, dict): + raise StorageError("Der Anbieter lieferte keine gültigen OAuth-Daten.") + values.update({key: value for key, value in received.items() if value}) + database.update_storage_provider( + provider_id, config=encrypt_config(values) + ) + return get_provider_record(provider_id) or {} + + +# --------------------------------------------------------------------------- +# Local archive paths +# --------------------------------------------------------------------------- + +def _base_dir() -> str: + return os.path.realpath(config.BASE_DIR) + + +def local_abs_path(local_path: str) -> str: + base = _base_dir() + raw = str(local_path or "").strip() + if not raw: + raise StorageConfigError("Leerer lokaler Pfad.") + if os.path.isabs(raw): + full = os.path.realpath(raw) + else: + full = os.path.realpath(os.path.join(base, raw.replace("/", os.sep))) + try: + common = os.path.commonpath([base, full]) + except ValueError as exc: + raise StorageConfigError("Ungültiger lokaler Pfad.") from exc + if common != base: + raise StorageConfigError("Der lokale Pfad liegt außerhalb des Archivs.") + return full + + +def relative_local_path(local_path: str) -> str: + base = _base_dir() + full = local_abs_path(local_path) + rel = os.path.relpath(full, base).replace(os.sep, "/") + if rel in (".", "") or rel.startswith("../"): + raise StorageConfigError("Ungültiger lokaler Pfad.") + return rel + + +def classify_local_path(relative_path: str) -> tuple[str, str, str]: + rel = str(relative_path or "").strip().strip("/") + parts = [part for part in rel.split("/") if part] + if not parts: + return "", "", "other" + filename = parts[-1] + if parts[0] == THUMB_DIR: + if len(parts) < 3: + return "", filename, "other" + return parts[1], filename, "thumb" + if len(parts) < 2: + return "", filename, "other" + streamer = parts[0] + middle = parts[1:-1] + if "clips" in middle: + return streamer, filename, "clip" + if "chat_logs" in middle: + return streamer, filename, "chat" + return streamer, filename, "vod" + + +def _file_allowed(rel: str, settings: dict) -> bool: + parts = rel.split("/") + if len(parts) < 2: + return False + name = parts[-1] + if name.startswith(".") or name.lower().endswith(SKIP_SUFFIXES): + return False + if is_transient_media_filename(name): + return False + if any(part in SKIP_DIRS for part in parts[:-1]): + return False + if any(part.startswith(".") and part != THUMB_DIR for part in parts[:-1]): + return False + streamer, filename, kind = classify_local_path(rel) + if not streamer or not filename: + return False + ext = os.path.splitext(filename)[1].lower() + if kind == "chat": + return bool(settings.get("include_chat_logs")) and ext == ".jsonl" + if kind == "thumb": + return bool(settings.get("include_thumbnails")) and ext in THUMB_EXTENSIONS + if kind in ("vod", "clip"): + if ext in VIDEO_EXTENSIONS: + return True + if kind == "vod" and ext in SIDECAR_EXTENSIONS: + return True + return False + + +def _streamer_name(value: str) -> str: + """Validate a streamer directory name without allowing drive/path tricks.""" + name = str(value or "").strip() + if ( + not name + or name in (".", "..") + or "/" in name + or "\\" in name + or ":" in name + or os.path.isabs(name) + or os.path.splitdrive(name)[0] + or any(ord(ch) < 32 for ch in name) + ): + raise StorageConfigError("Ungültiger Streamer-Name.") + return name + + +def collect_media_files( + streamer: Optional[str] = None, + *, + settings: Optional[dict] = None, + limit: Optional[int] = None, +) -> list[str]: + settings = settings or get_settings() + base = _base_dir() + root = base + if streamer: + name = _streamer_name(streamer) + root = os.path.join(base, name) + try: + root_real = os.path.realpath(root) + if os.path.commonpath([base, root_real]) != base: + return [] + except (OSError, ValueError): + return [] + if os.path.islink(root) or not os.path.isdir(root): + return [] + allow_thumbs = bool(settings.get("include_thumbnails")) + found = [] + for dirpath, dirnames, filenames in os.walk( + root, onerror=lambda _exc: None, followlinks=False + ): + dirnames[:] = [ + item + for item in dirnames + if ( + (item == THUMB_DIR and allow_thumbs) + or (item not in SKIP_DIRS and not item.startswith(".")) + ) + and not os.path.islink(os.path.join(dirpath, item)) + ] + for filename in filenames: + full = os.path.join(dirpath, filename) + # Never archive symlinks: they can point outside the archive and + # their target may disappear independently of the link. + if os.path.islink(full): + continue + try: + info = os.stat(full) + if not stat.S_ISREG(info.st_mode): + continue + rel = os.path.relpath(full, base).replace(os.sep, "/") + if not _file_allowed(rel, settings): + continue + # Resolve once so a file that is swapped for a symlink while + # walking cannot escape the archive. + local_abs_path(rel) + except (OSError, ValueError, StorageConfigError): + continue + found.append((info.st_mtime, rel)) + found.sort(key=lambda item: (item[0], item[1])) + files = [rel for _mtime, rel in found] + if limit is not None: + files = files[: max(0, int(limit))] + return files + + +def remote_path_for(provider_row: dict, relative_path: str) -> str: + rel = str(relative_path or "").strip().strip("/") + if not rel: + raise StorageConfigError("Leerer Remote-Pfad.") + cfg = provider_row.get("config") or {} + base_path = str(cfg.get("base_path") or "").strip().strip("/") + joined = f"{base_path}/{rel}" if base_path else rel + return normalize_remote_path(joined) + + +# --------------------------------------------------------------------------- +# Object bookkeeping +# --------------------------------------------------------------------------- + +def prepare_object( + local_path: str, + provider_row: dict, + *, + settings: Optional[dict] = None, +) -> dict: + rel = relative_local_path(local_path) + full = local_abs_path(rel) + if not os.path.isfile(full): + raise StorageConfigError(f"Lokale Datei nicht gefunden: {rel}") + settings = settings or get_settings() + if not _file_allowed(rel, settings): + raise StorageConfigError( + f"Die Datei ist kein zulässiges Archiv-Objekt: {rel}" + ) + streamer, filename, kind = classify_local_path(rel) + remote_path = remote_path_for(provider_row, rel) + info = os.stat(full) + return database.upsert_storage_object( + provider_row["id"], + rel, + remote_path, + streamer=streamer, + filename=filename, + kind=kind, + local_size=info.st_size, + local_mtime=info.st_mtime, + status="pending", + ) + + +def _object_is_current(obj: dict, stat) -> bool: + """True when the stored object already matches the local file exactly.""" + return bool( + obj + and obj.get("status") in ("uploaded", "restored") + and not obj.get("local_deleted_at") + and not obj.get("remote_deleted_at") + and int(obj.get("local_size") or 0) == int(stat.st_size) + and abs(float(obj.get("local_mtime") or 0) - float(stat.st_mtime)) < 1.0 + ) + + +def local_file_is_current(obj: dict) -> bool: + if not obj: + return False + try: + full = local_abs_path(obj.get("local_path")) + stat = os.stat(full) + except (StorageConfigError, OSError): + return False + return _object_is_current(obj, stat) + + +def enqueue_upload( + local_path: str, + provider_id=None, + *, + force: bool = False, + settings: Optional[dict] = None, +) -> dict: + settings = settings or get_settings() + provider_row = resolve_provider(provider_id) + rel = relative_local_path(local_path) + previous = database.find_storage_object(provider_row["id"], rel) + obj = prepare_object(local_path, provider_row, settings=settings) + if previous and str(previous.get("remote_path") or "") != str(obj.get("remote_path") or ""): + # base_path changed: the old remote copy is not at the new location. + database.update_storage_object( + obj["id"], + status="pending", + progress=0, + last_error="", + remote_deleted_at=None, + ) + obj = database.get_storage_object(obj["id"]) or obj + full = local_abs_path(obj["local_path"]) + info = os.stat(full) + if _object_is_current(obj, info) and not force: + return obj + if database.has_active_task( + "storage_upload", + obj.get("streamer"), + obj.get("filename"), + object_id=obj.get("id"), + ): + return obj + database.update_storage_object( + obj["id"], + status="pending", + local_size=info.st_size, + local_mtime=info.st_mtime, + local_deleted_at=None, + remote_deleted_at=None, + progress=0, + last_error="", + ) + task_id = database.add_task( + "storage_upload", + { + "object_id": obj["id"], + "provider_id": provider_row["id"], + "streamer": obj.get("streamer") or "", + "filename": obj.get("filename") or "", + }, + ) + obj = database.get_storage_object(obj["id"]) or obj + obj["task_id"] = task_id + return obj + + +def enqueue_auto_upload( + local_path: str, + *, + settings: Optional[dict] = None, +) -> Optional[dict]: + """Queue an upload only when automatic cloud upload is enabled.""" + settings = settings or get_settings() + if not settings.get("auto_upload"): + return None + if not local_path or not os.path.isfile(local_path): + return None + return enqueue_upload(local_path, settings=settings) + + +def enqueue_sync(provider_id=None, streamer: Optional[str] = None) -> Optional[int]: + provider_row = resolve_provider(provider_id) + if database.has_active_task( + "storage_sync", + streamer or None, + None, + provider_id=provider_row["id"], + ): + return None + return database.add_task( + "storage_sync", + { + "provider_id": provider_row["id"], + "streamer": streamer or "", + }, + ) + + +def enqueue_cleanup(provider_id=None) -> Optional[int]: + if database.has_active_task("storage_cleanup", None, None): + return None + payload = {} + if provider_id is not None: + payload["provider_id"] = int(provider_id) + return database.add_task("storage_cleanup", payload) + + +def _local_copy_is_current(obj: dict, full_path: str) -> bool: + """Check that a local file still matches the version that was uploaded. + + Older rows may not have local metadata yet; in that case the caller falls + back to the previous existence/size checks instead of blocking forever. + """ + expected_mtime = float(obj.get("local_mtime") or 0) + if not expected_mtime: + return True + try: + info = os.stat(full_path) + except OSError: + return False + return ( + info.st_size == int(obj.get("local_size") or 0) + and abs(info.st_mtime - expected_mtime) < 1.0 + ) + + +def remote_delete_block_reason(obj: dict) -> str: + """Return why deleting the remote copy would risk the only remaining copy.""" + if not obj: + return "Archiv-Objekt nicht gefunden." + if obj.get("remote_deleted_at"): + return "Die Cloud-Kopie wurde bereits gelöscht." + if obj.get("local_deleted_at"): + return ( + "Die lokale Kopie wurde bereits entfernt. Stelle sie zuerst aus der " + "Cloud wieder her, bevor du die Cloud-Kopie löschst." + ) + if obj.get("status") not in ("uploaded", "restored"): + return "Die Datei wurde noch nicht erfolgreich in die Cloud hochgeladen." + if database.has_active_task("storage_upload", object_id=obj.get("id")): + return "Für dieses Objekt läuft gerade ein Upload." + try: + full = local_abs_path(obj.get("local_path")) + except StorageConfigError: + return "Der lokale Pfad ist ungültig; die Cloud-Kopie bleibt geschützt." + if not os.path.isfile(full): + return ( + "Die lokale Kopie fehlt. Die Cloud-Kopie wird nicht gelöscht, damit " + "kein Datenverlust entsteht." + ) + if not _local_copy_is_current(obj, full): + return ( + "Die lokale Kopie wurde nach dem Upload geändert. Lade sie erneut " + "hoch, bevor du die Cloud-Kopie löschst." + ) + remote_size = int(obj.get("remote_size") or 0) + if remote_size > 0: + try: + local_size = os.path.getsize(full) + except OSError: + return "Die lokale Kopie ist nicht lesbar; die Cloud-Kopie bleibt erhalten." + if local_size != remote_size: + return ( + "Die lokale Kopie stimmt nicht mit der Cloud-Kopie überein. " + "Die Cloud-Kopie bleibt erhalten." + ) + return "" + + +def enqueue_delete(object_id: int) -> int: + obj = database.get_storage_object(object_id) + if not obj: + raise StorageConfigError("Archiv-Objekt nicht gefunden.") + if obj.get("remote_deleted_at"): + return 0 + reason = remote_delete_block_reason(obj) + if reason: + raise StorageConfigError(reason) + if database.has_active_task("storage_delete", object_id=obj["id"]): + return 0 + return database.add_task( + "storage_delete", + { + "object_id": obj["id"], + "provider_id": obj["provider_id"], + "streamer": obj.get("streamer") or "", + "filename": obj.get("filename") or "", + }, + ) + + +def enqueue_restore(object_id: int) -> int: + obj = database.get_storage_object(object_id) + if not obj: + raise StorageConfigError("Archiv-Objekt nicht gefunden.") + if database.has_active_task("storage_restore", object_id=obj["id"]): + return 0 + return database.add_task( + "storage_restore", + { + "object_id": obj["id"], + "provider_id": obj["provider_id"], + "streamer": obj.get("streamer") or "", + "filename": obj.get("filename") or "", + }, + ) + + +def enqueue_check(provider_id: int) -> int: + return database.add_task("storage_check", {"provider_id": int(provider_id)}) + + +# --------------------------------------------------------------------------- +# Remote verification and local retention +# --------------------------------------------------------------------------- + +def remote_exists(provider, remote_path: str, expected_size: int = 0) -> bool: + try: + return bool(provider.exists(remote_path, expected_size)) + except Exception as exc: + logger.warning( + "Remote-Verifikation fehlgeschlagen für %s: %s", + remote_path, + _safe_error(exc), + ) + return False + + +def delete_local_copy(object_id: int, *, verify: Optional[bool] = None) -> tuple[bool, str]: + obj = database.get_storage_object(object_id) + if not obj: + return False, "Archiv-Objekt nicht gefunden." + if obj.get("local_deleted_at"): + return False, "Lokale Kopie wurde bereits entfernt." + if obj.get("status") not in ("uploaded", "restored"): + return False, "Die Datei wurde noch nicht erfolgreich hochgeladen." + full = local_abs_path(obj["local_path"]) + if not os.path.isfile(full): + database.update_storage_object( + object_id, + local_deleted_at=time.time(), + last_error="Lokale Datei existiert nicht mehr.", + ) + return False, "Lokale Datei existiert nicht mehr." + if not _local_copy_is_current(obj, full): + message = "Die lokale Datei wurde nach dem Upload geändert und bleibt erhalten." + database.update_storage_object(object_id, last_error=message) + return False, message + if obj.get("kind") == "vod": + try: + from core import vod_watch_slots + + if vod_watch_slots.is_vod_watched(obj.get("streamer"), obj.get("filename")): + return False, "Das VOD wird gerade angesehen und bleibt lokal." + except Exception: + return False, "Wiedergabestatus konnte nicht geprüft werden." + settings = get_settings() + should_verify = ( + settings.get("verify_before_local_delete", True) + if verify is None + else bool(verify) + ) + if should_verify: + try: + provider_row = resolve_provider(obj["provider_id"]) + provider = build_provider(provider_row) + except StorageError as exc: + message = f"Cloud-Anbieter nicht verfügbar: {_safe_error(exc)}" + database.update_storage_object(object_id, last_error=message) + return False, message + if not remote_exists(provider, obj["remote_path"], int(obj.get("remote_size") or 0)): + message = "Cloud-Kopie konnte nicht verifiziert werden; lokale Datei bleibt erhalten." + database.update_storage_object(object_id, last_error=message) + return False, message + try: + os.remove(full) + except OSError as exc: + message = f"Lokale Datei konnte nicht gelöscht werden: {exc}" + database.update_storage_object(object_id, last_error=message) + return False, message + database.update_storage_object( + object_id, + local_deleted_at=time.time(), + last_error="", + ) + return True, "Lokale Kopie entfernt." + + +def cleanup_local_copies(provider_id=None, *, limit: int = 100) -> dict: + settings = get_settings() + if not settings.get("delete_local_after_upload"): + return {"checked": 0, "deleted": 0, "kept": 0, "disabled": True} + retention = int(settings.get("local_retention_days") or 0) + cutoff = time.time() - (retention * 86400) + rows = database.list_storage_objects( + provider_id=provider_id, + status="uploaded", + uploaded_before=cutoff, + local_deleted=False, + limit=limit, + ) + deleted = 0 + kept = 0 + reasons = [] + for obj in rows: + try: + ok, message = delete_local_copy(obj["id"]) + except Exception as exc: + ok, message = False, _safe_error(exc) + if ok: + deleted += 1 + else: + kept += 1 + if message: + reasons.append(f"{obj.get('filename')}: {message}") + return { + "checked": len(rows), + "deleted": deleted, + "kept": kept, + "reasons": reasons[:20], + } + + +# --------------------------------------------------------------------------- +# Worker task implementations +# --------------------------------------------------------------------------- + +def _progress_callback(object_id: int, total: int): + state = {"last": 0.0, "last_pct": -1.0} + + def callback(done: int, size: int): + now = time.time() + if now - state["last"] < 2.0: + return + state["last"] = now + denominator = int(size or total or 0) + pct = (float(done) / denominator * 100.0) if denominator else 0.0 + pct = max(0.0, min(100.0, pct)) + if abs(pct - state["last_pct"]) < 1.0: + return + state["last_pct"] = pct + try: + database.update_storage_object(object_id, progress=round(pct, 1)) + except Exception: + pass + + return callback + + +def run_upload(payload: dict) -> dict: + object_id = _clean_provider_id(payload.get("object_id")) + if not object_id: + raise StorageConfigError("storage_upload ohne object_id.") + obj = database.get_storage_object(object_id) + if not obj: + raise StorageConfigError("Archiv-Objekt nicht gefunden.") + provider_row = resolve_provider(obj["provider_id"]) + full = local_abs_path(obj["local_path"]) + if not os.path.isfile(full): + database.update_storage_object( + object_id, status="failed", last_error="Lokale Datei fehlt." + ) + raise StorageConfigError("Lokale Datei fehlt.") + stat = os.stat(full) + if _object_is_current(obj, stat) and not payload.get("force"): + return {"ok": True, "skipped": True} + database.update_storage_object( + object_id, + status="uploading", + attempts=int(obj.get("attempts") or 0) + 1, + progress=0, + last_error="", + local_size=stat.st_size, + local_mtime=stat.st_mtime, + local_deleted_at=None, + remote_deleted_at=None, + ) + provider = build_provider(provider_row) + try: + result = provider.upload_file( + full, + obj["remote_path"], + progress=_progress_callback(object_id, stat.st_size), + ) or {} + except Exception as exc: + message = _safe_error(exc) + database.update_storage_object( + object_id, + status="failed", + last_error=message, + ) + database.set_storage_provider_error(provider_row["id"], message) + raise + try: + remote_size = int(result.get("size") if result.get("size") is not None else stat.st_size) + except (TypeError, ValueError): + remote_size = stat.st_size + if remote_size != stat.st_size: + message = ( + f"Upload unvollständig: Anbieter meldet {remote_size} Bytes, " + f"lokal sind {stat.st_size} Bytes vorhanden." + ) + database.update_storage_object( + object_id, status="failed", last_error=message + ) + database.set_storage_provider_error(provider_row["id"], message) + raise StorageError(message, provider=provider_row.get("provider_type")) + database.update_storage_object( + object_id, + status="uploaded", + remote_size=remote_size, + remote_modified=str(result.get("modified_at") or ""), + remote_checksum=str(result.get("checksum") or ""), + uploaded_at=time.time(), + progress=100, + last_error="", + local_deleted_at=None, + remote_deleted_at=None, + ) + database.set_storage_provider_error(provider_row["id"], "") + settings = get_settings() + if settings.get("delete_local_after_upload") and int( + settings.get("local_retention_days") or 0 + ) <= 0: + try: + delete_local_copy(object_id) + except Exception as exc: + logger.warning( + "Lokale Löschung nach Upload fehlgeschlagen: %s", _safe_error(exc) + ) + return {"ok": True, "remote_path": obj["remote_path"], "size": stat.st_size} + + +def run_sync(payload: dict) -> dict: + provider_id = _clean_provider_id(payload.get("provider_id")) + streamer = str(payload.get("streamer") or "").strip() or None + provider_row = resolve_provider(provider_id) + settings = get_settings() + max_files = int(settings.get("max_sync_files") or 500) + files = collect_media_files(streamer, settings=settings) + queued = 0 + skipped = 0 + candidates = 0 + errors = [] + for rel in files: + try: + existing = database.find_storage_object(provider_row["id"], rel) + if existing and local_file_is_current(existing): + # A changed base_path moves the remote target. Do not skip a + # current local file when its stored remote path is stale. + if str(existing.get("remote_path") or "") == remote_path_for(provider_row, rel): + skipped += 1 + continue + if candidates >= max_files: + break + candidates += 1 + obj = enqueue_upload(rel, provider_row["id"], settings=settings) + if obj.get("task_id"): + queued += 1 + else: + skipped += 1 + except Exception as exc: + errors.append(f"{rel}: {_safe_error(exc)}") + schedule = database.get_system_state(SCHEDULE_KEY, {}) or {} + if not isinstance(schedule, dict): + schedule = {} + schedule.update( + { + "last_sync_at": time.time(), + "provider_id": provider_row["id"], + "queued": queued, + "skipped": skipped, + "files": len(files), + "errors": errors[:20], + } + ) + database.set_system_state(SCHEDULE_KEY, schedule) + if settings.get("delete_local_after_upload"): + try: + cleanup_local_copies(provider_row["id"]) + schedule["last_cleanup_at"] = time.time() + database.set_system_state(SCHEDULE_KEY, schedule) + except Exception as exc: + logger.warning("Lokale Aufräumung nach Sync fehlgeschlagen: %s", _safe_error(exc)) + return { + "ok": True, + "provider_id": provider_row["id"], + "files": len(files), + "candidates": candidates, + "queued": queued, + "skipped": skipped, + "errors": errors[:20], + } + + +def run_delete(payload: dict) -> dict: + object_id = _clean_provider_id(payload.get("object_id")) + if not object_id: + raise StorageConfigError("storage_delete ohne object_id.") + obj = database.get_storage_object(object_id) + if not obj: + raise StorageConfigError("Archiv-Objekt nicht gefunden.") + if obj.get("remote_deleted_at"): + return {"ok": True, "already_deleted": True} + reason = remote_delete_block_reason(obj) + if reason: + database.update_storage_object(object_id, last_error=reason) + raise StorageConfigError(reason) + try: + provider_row = resolve_provider(obj["provider_id"]) + provider = build_provider(provider_row) + provider.delete(obj["remote_path"]) + except Exception as exc: + message = _safe_error(exc) + database.update_storage_object(object_id, last_error=message) + database.set_storage_provider_error(obj["provider_id"], message) + raise + database.update_storage_object( + object_id, + status="remote_deleted", + remote_deleted_at=time.time(), + last_error="", + ) + database.set_storage_provider_error(obj["provider_id"], "") + return {"ok": True, "remote_path": obj["remote_path"]} + + +def run_restore(payload: dict) -> dict: + object_id = _clean_provider_id(payload.get("object_id")) + if not object_id: + raise StorageConfigError("storage_restore ohne object_id.") + obj = database.get_storage_object(object_id) + if not obj: + raise StorageConfigError("Archiv-Objekt nicht gefunden.") + if obj.get("remote_deleted_at"): + message = "Die Cloud-Kopie wurde bereits gelöscht." + database.update_storage_object(object_id, last_error=message) + raise StorageConfigError(message) + try: + provider_row = resolve_provider(obj["provider_id"]) + full = local_abs_path(obj["local_path"]) + expected = int(obj.get("remote_size") or 0) + if os.path.isfile(full): + size = os.path.getsize(full) + if expected > 0 and size == expected: + database.update_storage_object( + object_id, + status="restored", + local_deleted_at=None, + local_size=size, + local_mtime=os.path.getmtime(full), + last_error="", + ) + return {"ok": True, "already_local": True, "size": size} + # If the remote size is unknown we cannot prove that an existing + # local file is complete. Download it again instead of trusting it. + parent = os.path.dirname(full) + os.makedirs(parent, exist_ok=True) + if expected > 0: + free = shutil.disk_usage(parent).free + if free < expected + (64 * 1024 * 1024): + raise StorageConfigError( + "Nicht genügend lokaler Speicherplatz für die Wiederherstellung." + ) + provider = build_provider(provider_row) + temp_path = full + ".restore" + try: + try: + os.remove(temp_path) + except OSError: + pass + provider.download_file( + obj["remote_path"], + temp_path, + progress=_progress_callback(object_id, expected), + ) + if not os.path.isfile(temp_path): + raise StorageError("Die Cloud-Datei wurde nicht lokal geschrieben.") + size = os.path.getsize(temp_path) + if expected > 0 and size != expected: + raise StorageError( + f"Wiederherstellung unvollständig ({size} von {expected} Bytes)." + ) + os.replace(temp_path, full) + finally: + if os.path.exists(temp_path): + try: + os.remove(temp_path) + except OSError: + pass + database.update_storage_object( + object_id, + status="restored", + local_deleted_at=None, + local_size=size, + local_mtime=os.path.getmtime(full), + remote_size=size if expected <= 0 else expected, + progress=100, + last_error="", + ) + database.set_storage_provider_error(obj["provider_id"], "") + return {"ok": True, "size": size} + except Exception as exc: + message = _safe_error(exc) + database.update_storage_object(object_id, last_error=message) + try: + database.set_storage_provider_error(obj["provider_id"], message) + except Exception: + pass + raise + + +def run_check(payload: dict) -> dict: + provider_id = _clean_provider_id(payload.get("provider_id")) + if not provider_id: + raise StorageConfigError("storage_check ohne provider_id.") + result = test_provider(provider_id) + if not result.get("ok"): + raise StorageError(result.get("message") or "Verbindungstest fehlgeschlagen.") + return result + + +def run_cleanup(payload: dict) -> dict: + provider_id = _clean_provider_id(payload.get("provider_id")) + return cleanup_local_copies(provider_id) + + +# --------------------------------------------------------------------------- +# Periodic scheduling and admin context +# --------------------------------------------------------------------------- + +def tick() -> list[str]: + """Called from the worker maintenance loop; enqueues due background work.""" + actions = [] + settings = get_settings() + providers = [row for row in database.list_storage_providers() if row.get("enabled")] + if not providers: + return actions + now = time.time() + schedule = database.get_system_state(SCHEDULE_KEY, {}) or {} + if not isinstance(schedule, dict): + schedule = {} + + if settings.get("auto_upload"): + interval = int(settings.get("auto_sync_interval_minutes") or 0) + if interval > 0: + last = float(schedule.get("last_sync_at") or 0) + if now - last >= interval * 60: + try: + task_id = enqueue_sync() + if task_id: + # Record the attempt immediately so a failing sync is + # retried on the configured interval, not every loop. + schedule["last_sync_at"] = now + database.set_system_state(SCHEDULE_KEY, schedule) + actions.append(f"storage_sync:{task_id}") + except Exception as exc: + logger.warning("Automatischer Speicher-Sync fehlgeschlagen: %s", _safe_error(exc)) + + if settings.get("delete_local_after_upload"): + interval = int(settings.get("cleanup_interval_minutes") or 0) + last_cleanup = float(schedule.get("last_cleanup_at") or 0) + if interval > 0 and now - last_cleanup >= interval * 60: + try: + task_id = enqueue_cleanup() + if task_id: + schedule["last_cleanup_at"] = now + database.set_system_state(SCHEDULE_KEY, schedule) + actions.append(f"storage_cleanup:{task_id}") + except Exception as exc: + logger.warning("Automatische Speicher-Aufräumung fehlgeschlagen: %s", _safe_error(exc)) + return actions + + +def summary(provider_id=None) -> dict: + return database.storage_object_summary(provider_id) + + +def admin_context() -> dict: + settings = get_settings() + providers = list_providers() + default_id = settings.get("default_provider_id") + default_provider = next( + ( + row + for row in providers + if row["id"] == default_id and row.get("enabled") + ), + None, + ) + if default_provider is None: + default_provider = next((row for row in providers if row.get("enabled")), None) + objects = database.list_storage_objects(limit=150) + for obj in objects: + try: + obj["local_exists"] = os.path.isfile(local_abs_path(obj.get("local_path"))) + except (StorageConfigError, OSError): + obj["local_exists"] = False + return { + "providers": providers, + "catalog": provider_catalog(), + "settings": settings, + "summary": summary(), + "objects": objects, + "default_provider": default_provider, + "encryption_ready": encryption_ready(), + "schedule": database.get_system_state(SCHEDULE_KEY, {}) or {}, + } + + +def format_timestamp(value) -> str: + if not value: + return "–" + try: + return time.strftime("%d.%m.%Y %H:%M", time.localtime(float(value))) + except (TypeError, ValueError, OSError): + return "–" + + +def format_bytes(value) -> str: + try: + number = float(value or 0) + except (TypeError, ValueError): + number = 0.0 + for unit in ("B", "KB", "MB", "GB", "TB", "PB"): + if abs(number) < 1024.0 or unit == "PB": + return f"{number:.1f} {unit}" if unit != "B" else f"{int(number)} B" + number /= 1024.0 + return f"{number:.1f} PB" diff --git a/docker-compose.yml b/docker-compose.yml index e64e67e..b1a992c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,16 +5,26 @@ services: context: . dockerfile_inline: | FROM python:3.11-slim - RUN apt-get update && apt-get install -y ffmpeg procps supervisor fonts-dejavu && rm -rf /var/lib/apt/lists/* - RUN pip install flask werkzeug flask-limiter yt-dlp streamlink + ENV PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PYTHONDONTWRITEBYTECODE=1 + RUN apt-get update \ + && apt-get install -y --no-install-recommends ffmpeg procps supervisor fonts-dejavu-core ca-certificates \ + && rm -rf /var/lib/apt/lists/* + COPY requirements.txt /tmp/requirements.txt + RUN pip install --no-cache-dir -r /tmp/requirements.txt WORKDIR /app CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] restart: unless-stopped + stop_grace_period: 30s volumes: - # Mount application directory into container + # Application code and archive media (bind mount). - ./:/app - - ./docker-worker.conf:/etc/supervisor/conf.d/supervisord.conf + # Keep the live HLS segments shared with the Nginx container. + - hls_live:/app/.hls_live + - ./docker-worker.conf:/etc/supervisor/conf.d/supervisord.conf:ro environment: + - PYTHONPATH=/app - VOD_HOST=0.0.0.0 - VOD_PORT=8080 - VOD_BASE_DIR=/app @@ -22,18 +32,25 @@ services: - VOD_USER=${VOD_USER:?set VOD_USER in .env} - VOD_PASS=${VOD_PASS:?set VOD_PASS in .env} - VOD_SECRET_KEY=${VOD_SECRET_KEY:?set VOD_SECRET_KEY in .env} + # Optional dedicated key for storage credentials. Falls back to VOD_SECRET_KEY. + - STORAGE_ENCRYPTION_KEY=${STORAGE_ENCRYPTION_KEY:-} - VOD_SECURE_COOKIES=${VOD_SECURE_COOKIES:-false} - VOD_SHARE_TTL_SEC=${VOD_SHARE_TTL_SEC:-604800} - VOD_USE_X_ACCEL=true - LIVE_HLS_DIR=/app/.hls_live - # Oeffentliche Basis-URL fuer Share-Links, z.B. https://recorder.example.com + # Public base URL for share links and OAuth callbacks, e.g. https://recorder.example.com - VOD_PUBLIC_BASE=${VOD_PUBLIC_BASE:-} - # Speicherwarnung + # Background task pool sizing. + - WORKER_THUMB_THREADS=${WORKER_THUMB_THREADS:-2} + - WORKER_REMUX_THREADS=${WORKER_REMUX_THREADS:-1} + - WORKER_TRANSCRIBE_THREADS=${WORKER_TRANSCRIBE_THREADS:-1} + - WORKER_STORAGE_THREADS=${WORKER_STORAGE_THREADS:-1} + # Storage warning thresholds. - STORAGE_ALERT_THRESHOLD_PCT=90 - STORAGE_ALERT_THRESHOLD_FREE_GB=20 - STORAGE_ALERT_CHECK_INTERVAL_SEC=60 - STORAGE_ALERT_COOLDOWN_SEC=21600 - - STORAGE_ALERT_WEBHOOK_URL= + - STORAGE_ALERT_WEBHOOK_URL=${STORAGE_ALERT_WEBHOOK_URL:-} nginx: image: nginx:alpine @@ -42,7 +59,11 @@ services: ports: - "8090:80" volumes: - - ./nginx.conf:/etc/nginx/conf.d/default.conf - - ./:/app + - ./nginx.conf:/etc/nginx/conf.d/default.conf:ro + - ./:/app:ro + - hls_live:/app/.hls_live:ro depends_on: - app + +volumes: + hls_live: diff --git a/docker-worker.conf b/docker-worker.conf new file mode 100644 index 0000000..208d84f --- /dev/null +++ b/docker-worker.conf @@ -0,0 +1,63 @@ +; StreamVault container process supervisor +; Runs the WebUI, the recorder daemon and the background task worker in one +; container while keeping their lifecycles independent. + +[supervisord] +nodaemon=true +user=root +logfile=/dev/null +logfile_maxbytes=0 +pidfile=/tmp/supervisord.pid +childlogdir=/tmp + +[unix_http_server] +file=/tmp/supervisor.sock +chmod=0700 + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///tmp/supervisor.sock + +[program:webui] +directory=/app +command=/usr/local/bin/gunicorn --workers 2 --worker-class gthread --threads 4 --bind 0.0.0.0:8080 --timeout 120 --access-logfile - --error-logfile - WEBUI.webui:app +environment=PYTHONPATH="/app",PYTHONUNBUFFERED="1" +autostart=true +autorestart=true +startsecs=5 +stopasgroup=true +killasgroup=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +[program:recorder] +directory=/app +command=/usr/local/bin/python3 recorder/Streamrecorder.py +environment=PYTHONPATH="/app",PYTHONUNBUFFERED="1" +autostart=true +autorestart=true +startsecs=5 +stopasgroup=true +killasgroup=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 + +[program:worker] +directory=/app +command=/usr/local/bin/python3 recorder/worker.py +environment=PYTHONPATH="/app",PYTHONUNBUFFERED="1" +autostart=true +autorestart=true +startsecs=5 +stopasgroup=true +killasgroup=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 diff --git a/install.sh b/install.sh index 3c4cc73..1b6ef95 100644 --- a/install.sh +++ b/install.sh @@ -65,7 +65,11 @@ echo -e "${BLUE}[3/8] Creating directory structure...${NC}" mkdir -p "${INSTALL_DIR}" "${APP_DIR}" "${DATA_DIR}" "${LOG_DIR}" "${CONFIG_DIR}" "${RAMDISK_DIR}" chown -R "${SERVICE_USER}:${SERVICE_USER}" "${INSTALL_DIR}" "${DATA_DIR}" "${LOG_DIR}" "${RAMDISK_DIR}" chmod 755 "${INSTALL_DIR}" "${RAMDISK_DIR}" -chmod 750 "${DATA_DIR}" "${LOG_DIR}" +# Nginx (www-data) must be able to traverse and read the archive when it +# serves X-Accel-Redirect responses. Access is still gated by the internal +# Nginx location and the application's authentication checks. +chmod 755 "${DATA_DIR}" +chmod 750 "${LOG_DIR}" # 4. Deploying Source Code echo -e "${BLUE}[4/8] Deploying application codebase...${NC}" @@ -101,7 +105,8 @@ if [[ ! -f "${ENV_FILE}" ]]; then # StreamVault Production Environment VOD_HOST=127.0.0.1 VOD_PORT=8080 -VOD_PUBLIC_BASE=http://localhost:8080 +# Leave empty to derive from the request, or set the public HTTPS origin. +VOD_PUBLIC_BASE= VOD_BASE_DIR=${DATA_DIR} LIVE_HLS_DIR=${RAMDISK_DIR} VOD_USER=vaultadmin @@ -161,8 +166,28 @@ RestartSec=10s WantedBy=multi-user.target EOF +cat > /etc/systemd/system/streamvault-worker.service </; archives live directly + # below VOD_BASE_DIR (/app in the container). location /_protected/ { internal; - alias /app/data/; - - # Optimierungen für große Dateien + alias /app/; + sendfile on; - sendfile_max_chunk 1m; # Verhindert, dass ein Download den Server blockiert - output_buffers 2 1m; # Größerer Puffer (2MB) für flüssiges Playback - gzip off; # Videos nicht komprimieren (spart CPU) + sendfile_max_chunk 1m; + output_buffers 2 1m; + gzip off; + add_header Access-Control-Allow-Origin "*" always; + add_header Access-Control-Expose-Headers "Content-Length, Content-Range, Accept-Ranges, Content-Type" always; + add_header Accept-Ranges bytes always; + } + + # Legacy archives stored below VOD_FILES_DIR (default /app/files). + location /_protected_files/ { + internal; + alias /app/files/; + + sendfile on; + sendfile_max_chunk 1m; + output_buffers 2 1m; + gzip off; add_header Access-Control-Allow-Origin "*" always; add_header Access-Control-Expose-Headers "Content-Length, Content-Range, Accept-Ranges, Content-Type" always; add_header Accept-Ranges bytes always; - - # Optional: Bandbreite begrenzen, falls das Netzwerk der Flaschenhals ist - # limit_rate_after 50m; # Volle Geschwindigkeit für die ersten 50MB - # limit_rate 10m; # Danach auf 10MB/s begrenzen } location /_protected_vodcache/ { internal; - alias /app/data/.vodcache/; + alias /app/.vodcache/; sendfile on; sendfile_max_chunk 1m; output_buffers 2 1m; @@ -76,18 +83,16 @@ server { add_header Accept-Ranges bytes always; } - # X-Accel-Redirect für Thumbnails location /_protected_thumbs/ { internal; - alias /app/data/.thumbs/; - expires 1d; # Browser-Caching aktivieren + alias /app/.thumbs/; + expires 1d; add_header Cache-Control "public"; } - - # X-Accel-Redirect für HLS + location /_protected_hls/ { internal; - alias /app/data/.hls/; + alias /app/.hls/; types { application/vnd.apple.mpegurl m3u8; video/mp2t ts; @@ -99,11 +104,10 @@ server { add_header Access-Control-Expose-Headers "Content-Length, Content-Range, Accept-Ranges, Content-Type" always; add_header Accept-Ranges bytes always; } - - # X-Accel-Redirect für Live HLS + location /_protected_hls_live/ { internal; - alias /dev/shm/streamrecorder_hls/; + alias /app/.hls_live/; open_file_cache off; types { application/vnd.apple.mpegurl m3u8; diff --git a/recorder/Streamrecorder.py b/recorder/Streamrecorder.py index 0733e0d..8f29c29 100644 --- a/recorder/Streamrecorder.py +++ b/recorder/Streamrecorder.py @@ -668,6 +668,35 @@ def enqueue_thumbnail_task(media_path, channel, discord_notify_ready=False): return False +def maybe_enqueue_storage_upload(media_path): + """Queue a cloud upload after recording finalisation when enabled.""" + try: + from core.storage import service as storage_service + + settings = storage_service.get_settings() + if not settings.get("auto_upload"): + return False + if not media_path or not os.path.isfile(media_path): + return False + obj = storage_service.enqueue_upload(media_path, settings=settings) + if obj.get("task_id"): + logger.info( + "Cloud-Upload eingereiht: %s -> %s (Task %s)", + obj.get("local_path"), + obj.get("remote_path"), + obj.get("task_id"), + ) + return True + return False + except Exception as exc: + from core.storage.base import mask_error + + logger.warning( + "Cloud-Upload konnte nicht eingeordnet werden: %s", mask_error(exc) + ) + return False + + def notify_discord_streamer_online(archive_channel, source_channel, recording_metadata, conf, recording_started_ts): _ = (archive_channel, source_channel, recording_metadata, conf, recording_started_ts) @@ -3359,22 +3388,29 @@ def record_stream(source_channel, chat_text_file, recordings_dir): archive_channel, discord_notify_ready=False, ) + # Never offload a file that failed integrity validation. if not test_video_file(finalized_path): logger.warning("Video-Datei scheint beschädigt zu sein") - elif (not short_recording) and AUTO_TRANSCRIBE and ( - finalized_path.lower().endswith(".mp4") or os.path.exists(output_file) - ): - media_for_tx = output_file if os.path.exists(output_file) else finalized_path - if media_for_tx.lower().endswith(".mp4"): - transcript_path = os.path.splitext(output_file)[0] + ".transcript.txt" - try: - database.add_task( - 'transcribe', - {'media': output_file, 'transcript': transcript_path}, - ) - logger.info(f"Automatische Transkription eingeordnet: {output_file}") - except Exception as exc: - logger.error(f"Konnte automatische Transkription nicht einreihen: {exc}") + else: + # An asynchronous remux may replace the file later; in + # that case the periodic storage sync uploads the final + # version instead of this intermediate file. + if not (AUTO_REMUX and not SYNC_REMUX and not short_recording): + maybe_enqueue_storage_upload(finalized_path) + if (not short_recording) and AUTO_TRANSCRIBE and ( + finalized_path.lower().endswith(".mp4") or os.path.exists(output_file) + ): + media_for_tx = output_file if os.path.exists(output_file) else finalized_path + if media_for_tx.lower().endswith(".mp4"): + transcript_path = os.path.splitext(output_file)[0] + ".transcript.txt" + try: + database.add_task( + 'transcribe', + {'media': output_file, 'transcript': transcript_path}, + ) + logger.info(f"Automatische Transkription eingeordnet: {output_file}") + except Exception as exc: + logger.error(f"Konnte automatische Transkription nicht einreihen: {exc}") cleanup_duplicate_recordings( recordings_dir, output_file, diff --git a/recorder/worker.py b/recorder/worker.py index 595adff..963ca6d 100644 --- a/recorder/worker.py +++ b/recorder/worker.py @@ -65,14 +65,24 @@ def __init__(self, message, delay_sec=60): # WORKER_THUMB_THREADS — IO pool for thumbnail (+ delete/empty); default 2 # WORKER_REMUX_THREADS — CPU/media pool for remux_vod / remux_all; default 1 # WORKER_TRANSCRIBE_THREADS — ASR pool for Whisper; default 1 (keep low if RAM-heavy) +# WORKER_STORAGE_THREADS — network pool for cloud uploads/downloads; default 1 WORKER_THUMB_THREADS = max(1, int(os.environ.get("WORKER_THUMB_THREADS", "2"))) WORKER_REMUX_THREADS = max(1, int(os.environ.get("WORKER_REMUX_THREADS", "1"))) WORKER_TRANSCRIBE_THREADS = max(1, int(os.environ.get("WORKER_TRANSCRIBE_THREADS", "1"))) +WORKER_STORAGE_THREADS = max(1, int(os.environ.get("WORKER_STORAGE_THREADS", "1"))) IO_TASK_TYPES = frozenset({ "thumbnail", "delete_folder", "empty_folder", }) REMUX_TASK_TYPES = frozenset({"remux_vod", "remux_all"}) ASR_TASK_TYPES = frozenset({"transcribe"}) +STORAGE_TASK_TYPES = frozenset({ + "storage_upload", + "storage_sync", + "storage_delete", + "storage_restore", + "storage_check", + "storage_cleanup", +}) TASK_MAX_ATTEMPTS = { "thumbnail": int(os.environ.get("TASK_MAX_ATTEMPTS_THUMBNAIL", "3")), "delete_folder": int(os.environ.get("TASK_MAX_ATTEMPTS_DELETE_FOLDER", "2")), @@ -80,15 +90,22 @@ def __init__(self, message, delay_sec=60): "transcribe": int(os.environ.get("TASK_MAX_ATTEMPTS_TRANSCRIBE", "1")), "remux_all": int(os.environ.get("TASK_MAX_ATTEMPTS_REMUX_ALL", "1")), "remux_vod": int(os.environ.get("TASK_MAX_ATTEMPTS_REMUX_VOD", "2")), + "storage_upload": int(os.environ.get("TASK_MAX_ATTEMPTS_STORAGE_UPLOAD", "3")), + "storage_sync": int(os.environ.get("TASK_MAX_ATTEMPTS_STORAGE_SYNC", "1")), + "storage_delete": int(os.environ.get("TASK_MAX_ATTEMPTS_STORAGE_DELETE", "3")), + "storage_restore": int(os.environ.get("TASK_MAX_ATTEMPTS_STORAGE_RESTORE", "3")), + "storage_check": int(os.environ.get("TASK_MAX_ATTEMPTS_STORAGE_CHECK", "1")), + "storage_cleanup": int(os.environ.get("TASK_MAX_ATTEMPTS_STORAGE_CLEANUP", "1")), } REMUX_FILE_TIMEOUT_SEC = int(os.environ.get("REMUX_FILE_TIMEOUT_SEC", "7200")) -_INFLIGHT = {"io": 0, "remux": 0, "asr": 0} +_INFLIGHT = {"io": 0, "remux": 0, "asr": 0, "storage": 0} _INFLIGHT_LOCK = threading.Lock() _POOL_LIMITS = { "io": WORKER_THUMB_THREADS, "remux": WORKER_REMUX_THREADS, "asr": WORKER_TRANSCRIBE_THREADS, + "storage": WORKER_STORAGE_THREADS, } _LAST_STORAGE_CHECK = 0.0 _LAST_TASK_CLEANUP = 0.0 @@ -1045,6 +1062,26 @@ def _safe_vod_task_path(streamer, filename): return path +def _maybe_enqueue_storage_upload(path): + """Queue the final remuxed archive when automatic cloud upload is enabled.""" + try: + from core.storage import service as storage_service + + obj = storage_service.enqueue_auto_upload(path) + if obj and obj.get("task_id"): + logging.info( + "Cloud-Upload nach Remux eingeordnet: %s (Task %s)", + path, + obj.get("task_id"), + ) + return True + except Exception as exc: + from core.storage.base import mask_error + + logging.warning("Cloud-Upload nach Remux fehlgeschlagen: %s", mask_error(exc)) + return False + + def process_remux_vod(payload): path = _safe_vod_task_path(payload.get("streamer"), payload.get("filename")) if not path: @@ -1086,6 +1123,7 @@ def process_remux_vod(payload): redis_client.invalidate_dashboard_cache() except Exception: pass + _maybe_enqueue_storage_upload(final_path) return True @@ -1162,6 +1200,22 @@ def process_task(task_type, payload): if not ok: raise RuntimeError("remux_vod failed") return + if task_type in STORAGE_TASK_TYPES: + from core.storage import service as storage_service + + if task_type == 'storage_upload': + storage_service.run_upload(payload) + elif task_type == 'storage_sync': + storage_service.run_sync(payload) + elif task_type == 'storage_delete': + storage_service.run_delete(payload) + elif task_type == 'storage_restore': + storage_service.run_restore(payload) + elif task_type == 'storage_check': + storage_service.run_check(payload) + elif task_type == 'storage_cleanup': + storage_service.run_cleanup(payload) + return raise RuntimeError(f"unknown task type: {task_type}") @@ -1170,6 +1224,8 @@ def _pool_name_for_task(task_type): return "remux" if task_type in ASR_TASK_TYPES: return "asr" + if task_type in STORAGE_TASK_TYPES: + return "storage" # thumbnail, delete/empty, and unknown → IO pool return "io" @@ -1184,9 +1240,18 @@ def _types_with_capacity(): allowed.extend(REMUX_TASK_TYPES) if _INFLIGHT["asr"] < _POOL_LIMITS["asr"]: allowed.extend(ASR_TASK_TYPES) + if _INFLIGHT["storage"] < _POOL_LIMITS["storage"]: + allowed.extend(STORAGE_TASK_TYPES) return allowed +def storage_tick(): + """Enqueue due cloud sync/cleanup tasks from the maintenance loop.""" + from core.storage import service as storage_service + + return storage_service.tick() + + def _execute_claimed_task(task): """Run one claimed task in a pool thread; each DB op uses its own get_db().""" task_id = task["id"] @@ -1208,10 +1273,16 @@ def _execute_claimed_task(task): status, ) except Exception as e: - logging.error(f"Task {task_id} failed: {e}") + if task_type in STORAGE_TASK_TYPES: + from core.storage.base import mask_error + + error_message = mask_error(e) + else: + error_message = str(e) + logging.error(f"Task {task_id} failed: {error_message}") max_attempts = TASK_MAX_ATTEMPTS.get(task_type, 1) status = database.fail_task( - task_id, str(e), max_attempts=max_attempts, worker_id=WORKER_ID + task_id, error_message, max_attempts=max_attempts, worker_id=WORKER_ID ) if status == "pending": logging.info( @@ -1228,16 +1299,18 @@ def _execute_claimed_task(task): def main(): init_db() logging.info( - "Worker gestartet: %s (thumb=%s remux=%s asr=%s)", + "Worker gestartet: %s (thumb=%s remux=%s asr=%s storage=%s)", WORKER_ID, WORKER_THUMB_THREADS, WORKER_REMUX_THREADS, WORKER_TRANSCRIBE_THREADS, + WORKER_STORAGE_THREADS, ) with ThreadPoolExecutor(max_workers=WORKER_THUMB_THREADS, thread_name_prefix="worker-io") as io_pool, \ ThreadPoolExecutor(max_workers=WORKER_REMUX_THREADS, thread_name_prefix="worker-remux") as remux_pool, \ - ThreadPoolExecutor(max_workers=WORKER_TRANSCRIBE_THREADS, thread_name_prefix="worker-asr") as asr_pool: - pools = {"io": io_pool, "remux": remux_pool, "asr": asr_pool} + ThreadPoolExecutor(max_workers=WORKER_TRANSCRIBE_THREADS, thread_name_prefix="worker-asr") as asr_pool, \ + ThreadPoolExecutor(max_workers=WORKER_STORAGE_THREADS, thread_name_prefix="worker-storage") as storage_pool: + pools = {"io": io_pool, "remux": remux_pool, "asr": asr_pool, "storage": storage_pool} while True: try: def _safe_run(func): @@ -1250,6 +1323,7 @@ def _safe_run(func): _safe_run(cleanup_old_tasks) _safe_run(cleanup_stale_thumbnail_locks) _safe_run(backfill_missing_thumbnails) + _safe_run(storage_tick) allowed = _types_with_capacity() if not allowed: diff --git a/requirements.txt b/requirements.txt index 1126799..d6133f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,6 @@ redis>=5.0.0 limits>=3.10.0 markupsafe>=2.1.0 gunicorn>=21.2.0; sys_platform != "win32" -playwright>=1.40.0 pycryptodome>=3.20.0 -lxml>=5.0.0 -typing-extensions>=4.8.0 +numpy>=1.26.0,<3.0.0 +boto3>=1.34.0 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_storage.py b/tests/test_storage.py new file mode 100644 index 0000000..7a2c6e4 --- /dev/null +++ b/tests/test_storage.py @@ -0,0 +1,2082 @@ +"""Storage-layer regression tests. + +These tests deliberately use only the standard library test runner so they can +run in a clean deployment checkout: + + python -m unittest discover -s tests -v + +The environment is configured before importing ``core.config`` because that +module intentionally refuses to start without credentials and a strong secret. +""" + +from __future__ import annotations + +import base64 +import io +import json +import os +import stat +import tempfile +import time +import unittest +from contextlib import redirect_stdout +from pathlib import Path +from unittest import mock + +_INITIAL_BASE = tempfile.mkdtemp(prefix="streamvault-tests-") +os.environ["VOD_USER"] = "vaultadmin" +os.environ["VOD_PASS"] = "VaultTestPass123!" +os.environ["VOD_SECRET_KEY"] = "test-session-secret-key-0123456789" +os.environ["STORAGE_ENCRYPTION_KEY"] = "test-storage-encryption-key-0123456789" +os.environ["VOD_BASE_DIR"] = _INITIAL_BASE +os.environ["VOD_ALLOW_ENV_LOGIN"] = "1" +os.environ["REDIS_HOST"] = "127.0.0.1" +os.environ["REDIS_PORT"] = "1" +os.environ.pop("REDIS_URL", None) +os.environ.pop("VOD_RATELIMIT_STORAGE_URI", None) + +from core import config, database # noqa: E402 +from core.storage import cli, http, service # noqa: E402 +from core.storage.base import ( # noqa: E402 + StorageConfigError, + StorageError, + guess_mime_type, + iter_file_chunks, + mask_error, + mask_text, + normalize_remote_path, + remote_name, + remote_parent, +) +from core.storage.crypto import ( # noqa: E402 + decrypt_config, + encrypt_config, + encryption_ready, + is_weak_secret, +) +from core.storage.providers import dropbox as dropbox_module # noqa: E402 +from core.storage.providers import google_drive as google_module # noqa: E402 +from core.storage.providers import onedrive as onedrive_module # noqa: E402 +from core.storage.providers import s3 as s3_module # noqa: E402 +from core.storage.providers.dropbox import DropboxProvider # noqa: E402 +from core.storage.providers.google_drive import GoogleDriveProvider # noqa: E402 +from core.storage.providers.onedrive import OneDriveProvider # noqa: E402 +from core.storage.providers.s3 import S3Provider # noqa: E402 +from core.storage.providers.webdav import WebDAVProvider # noqa: E402 +from core.storage.registry import provider_catalog # noqa: E402 + + +class StorageTestBase(unittest.TestCase): + """Isolated archive root and database for every test.""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory(prefix="streamvault-storage-test-") + self.base = Path(self._tmp.name) + os.environ["VOD_BASE_DIR"] = str(self.base) + self._patches = [ + mock.patch.object(config, "BASE_DIR", str(self.base)), + mock.patch.object(config, "DB_FILE", str(self.base / "streamrecorder.db")), + mock.patch.object(config, "FILES_DIR", str(self.base / "files")), + mock.patch.object(config, "HLS_ROOT", str(self.base / ".hls")), + mock.patch.object(config, "THUMB_ROOT", str(self.base / ".thumbs")), + mock.patch.object(config, "META_DIR", str(self.base / ".meta")), + ] + for patcher in self._patches: + patcher.start() + self.addCleanup(patcher.stop) + self.addCleanup(self._tmp.cleanup) + os.makedirs(config.META_DIR, exist_ok=True) + os.makedirs(config.FILES_DIR, exist_ok=True) + database.init_db() + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + def _provider(self, name="Test WebDAV", *, base_path="StreamVault", password="dav-secret-password"): + return service.create_provider( + name, + "webdav", + { + "base_url": "https://dav.example.com/remote.php/dav/files/vault/", + "username": "vault", + "password": password, + "base_path": base_path, + "verify_ssl": "1", + }, + ) + + def _write(self, relative_path: str, data: bytes = b"data", mtime: float | None = None) -> Path: + path = self.base / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + if mtime is not None: + os.utime(path, (mtime, mtime)) + return path + + def _complete_tasks(self): + conn = database.get_db() + try: + conn.execute( + "UPDATE tasks SET status='completed' WHERE status IN ('pending', 'running')" + ) + conn.commit() + finally: + conn.close() + + def _uploaded_object( + self, + relative_path: str, + data: bytes = b"payload", + *, + provider_name: str = "Test WebDAV", + ): + pid = self._provider(provider_name) + path = self._write(relative_path, data) + obj = service.enqueue_upload(relative_path) + self._complete_tasks() + database.update_storage_object( + obj["id"], + status="uploaded", + remote_size=len(data), + uploaded_at=time.time(), + ) + return pid, path, database.get_storage_object(obj["id"]) + + +class PathAndCryptoTests(StorageTestBase): + def test_normalize_remote_path(self): + self.assertEqual(normalize_remote_path(" /foo//bar/./baz "), "foo/bar/baz") + self.assertEqual(normalize_remote_path(r"foo\bar\baz"), "foo/bar/baz") + self.assertEqual(remote_parent("foo/bar/baz.mp4"), "foo/bar") + self.assertEqual(remote_name("foo/bar/baz.mp4"), "baz.mp4") + for value in ("", " ", "/", "foo/../bar", "..", "foo/\x00bar", "foo/\x01bar"): + with self.subTest(value=value): + with self.assertRaises(StorageConfigError): + normalize_remote_path(value) + + def test_iter_file_chunks_inclusive_range(self): + path = self._write("chunks.bin", b"0123456789") + chunks = list(iter_file_chunks(str(path), start=2, end=5, chunk_size=2)) + self.assertEqual(chunks, [b"23", b"45"]) + + def test_local_path_traversal_is_rejected(self): + local = self._write("streamer/file.mp4") + self.assertEqual( + os.path.normcase(service.local_abs_path("streamer/file.mp4")), + os.path.normcase(str(local.resolve())), + ) + outside = self.base.parent / "outside.mp4" + for value in ("../outside.mp4", str(outside), ""): + with self.subTest(value=value): + with self.assertRaises(StorageConfigError): + service.local_abs_path(value) + + def test_classify_local_path(self): + self.assertEqual(service.classify_local_path("streamer/file.mp4"), ("streamer", "file.mp4", "vod")) + self.assertEqual( + service.classify_local_path("streamer/clips/clip.mp4"), + ("streamer", "clip.mp4", "clip"), + ) + self.assertEqual( + service.classify_local_path("streamer/chat_logs/chat.jsonl"), + ("streamer", "chat.jsonl", "chat"), + ) + self.assertEqual( + service.classify_local_path(".thumbs/streamer/thumb.jpg"), + ("streamer", "thumb.jpg", "thumb"), + ) + self.assertEqual(service.classify_local_path("loose-file.mp4"), ("", "loose-file.mp4", "other")) + + def test_encrypt_decrypt_roundtrip(self): + payload = {"client_secret": "super-secret-value", "refresh_token": "refresh-abc"} + token = encrypt_config(payload) + self.assertTrue(token.startswith("sv1:")) + self.assertNotIn("super-secret-value", token) + self.assertEqual(decrypt_config(token), payload) + + def test_wrong_key_and_tampered_token_are_rejected(self): + token = encrypt_config({"password": "hunter2"}) + with mock.patch.dict( + os.environ, {"STORAGE_ENCRYPTION_KEY": "a-different-strong-key-0123456789"} + ): + with self.assertRaises(StorageConfigError): + decrypt_config(token) + + raw = base64.urlsafe_b64decode(token[len("sv1:") :]) + tampered = raw[:-1] + bytes([raw[-1] ^ 0x01]) + bad_token = "sv1:" + base64.urlsafe_b64encode(tampered).decode("ascii") + with self.assertRaises(StorageConfigError): + decrypt_config(bad_token) + + def test_weak_secret_blocks_encryption(self): + with mock.patch.dict( + os.environ, {"STORAGE_ENCRYPTION_KEY": "", "VOD_SECRET_KEY": ""} + ), mock.patch.object(config, "SECRET_KEY", "change-me"): + self.assertFalse(encryption_ready()) + with self.assertRaises(StorageConfigError): + encrypt_config({"password": "x"}) + + def test_mask_text_redacts_credentials(self): + text = mask_text( + "failed access_token=supersecret&password=hunter2 " + "Authorization: Bearer abc.def.ghi client_secret: shhh" + ) + for secret in ("supersecret", "hunter2", "abc.def.ghi", "shhh"): + self.assertNotIn(secret, text) + self.assertIn("***", text) + self.assertEqual(mask_error(ValueError("token=abc")), "token=***") + self.assertLessEqual(len(mask_text("x" * 2000)), 500) + + def test_mask_text_redacts_quoted_json_credentials(self): + text = mask_text( + '{"access_token": "supersecret", "password": "hello world", ' + '"nested": {"refresh_token": "refresh-value"}}' + ) + for secret in ("supersecret", "hello world", "refresh-value"): + self.assertNotIn(secret, text) + self.assertIn('"access_token": "***"', text) + self.assertIn('"password": "***"', text) + self.assertIn('"refresh_token": "***"', text) + + def test_webui_secret_key_rejects_public_placeholder(self): + from WEBUI import webui + + placeholder = ( + "change-this-to-a-very-secure-random-secret-key-32-chars-minimum" + ) + key_path = Path(config.META_DIR) / "secret_key.txt" + key_path.parent.mkdir(parents=True, exist_ok=True) + key_path.write_text("existing-strong-secret-0123456789", encoding="utf-8") + with mock.patch.object(config, "SECRET_KEY", placeholder): + self.assertEqual( + webui.load_secret_key(), "existing-strong-secret-0123456789" + ) + + key_path.unlink() + with mock.patch.object(config, "SECRET_KEY", placeholder): + generated = webui.load_secret_key() + self.assertNotEqual(generated, placeholder) + self.assertFalse(is_weak_secret(generated)) + self.assertEqual(key_path.read_text(encoding="utf-8").strip(), generated) + if os.name != "nt": + self.assertEqual(stat.S_IMODE(key_path.stat().st_mode), 0o600) + + +class ProviderConfigTests(StorageTestBase): + def test_create_provider_encrypts_and_masks_secrets(self): + pid = self._provider() + raw = database.get_storage_provider(pid) + self.assertTrue(raw["config"].startswith("sv1:")) + self.assertNotIn("dav-secret-password", raw["config"]) + + public = service.list_providers()[0] + self.assertEqual(public["config"]["password"], "••••••••") + self.assertTrue(public["config"]["password__set"]) + self.assertNotIn("dav-secret-password", json.dumps(public)) + self.assertEqual(service.get_settings()["default_provider_id"], pid) + + record = service.get_provider_record(pid) + self.assertEqual(record["config"]["password"], "dav-secret-password") + + def test_update_preserves_masked_secret(self): + pid = self._provider() + service.update_provider( + pid, values={"password": "••••••••", "base_path": "Archive"} + ) + record = service.get_provider_record(pid) + self.assertEqual(record["config"]["password"], "dav-secret-password") + self.assertEqual(record["config"]["base_path"], "Archive") + + service.update_provider(pid, values={"password": "new-secret-value"}) + self.assertEqual( + service.get_provider_record(pid)["config"]["password"], "new-secret-value" + ) + + def test_duplicate_and_invalid_provider_values_are_rejected(self): + self._provider() + with self.assertRaises(StorageConfigError): + self._provider("test webdav") + with self.assertRaises(StorageConfigError): + service.create_provider("", "webdav", {}) + with self.assertRaises(StorageConfigError): + service.create_provider("Unknown", "not-a-provider", {}) + with self.assertRaises(StorageConfigError): + service.create_provider( + "Bad URL", + "webdav", + { + "base_url": "ftp://example.com/", + "username": "u", + "password": "p", + }, + ) + + def test_provider_urls_reject_embedded_credentials_query_and_fragment(self): + bad_webdav = ( + "https://user:secret@dav.example.com/root/", + "https://dav.example.com/root/?token=secret", + "https://dav.example.com/root/#fragment", + ) + for index, base_url in enumerate(bad_webdav): + with self.subTest(base_url=base_url), self.assertRaises( + StorageConfigError + ): + service.create_provider( + f"Bad WebDAV {index}", + "webdav", + { + "base_url": base_url, + "username": "vault", + "password": "secret", + }, + ) + + bad_s3 = ( + "https://user:secret@s3.example.com", + "https://s3.example.com/?token=secret", + "https://s3.example.com/#fragment", + ) + for index, endpoint_url in enumerate(bad_s3): + with self.subTest(endpoint_url=endpoint_url), self.assertRaises( + StorageConfigError + ): + service.create_provider( + f"Bad S3 {index}", + "s3", + { + "bucket": "archive", + "access_key": "key", + "secret_key": "secret", + "endpoint_url": endpoint_url, + }, + ) + self.assertEqual(service.list_providers(), []) + + def test_google_oauth_provider_can_be_saved_before_link(self): + pid = service.create_provider( + "Google", + "google_drive", + {"client_id": "client-id", "client_secret": "client-secret"}, + ) + record = service.get_provider_record(pid) + self.assertEqual(record["config"]["client_id"], "client-id") + self.assertEqual(record["config"]["base_path"], "StreamVault") + self.assertFalse(str(record["config"].get("refresh_token") or "").strip()) + + def test_provider_type_change_blocked_while_objects_exist(self): + pid = self._provider() + self._write("streamer/file.mp4") + service.prepare_object("streamer/file.mp4", service.resolve_provider(pid)) + with self.assertRaises(StorageConfigError): + service.update_provider(pid, provider_type="s3") + + def test_delete_provider_removes_objects_and_selects_fallback(self): + first = self._provider("First") + second = self._provider("Second") + self._write("streamer/file.mp4") + obj = service.prepare_object("streamer/file.mp4", service.resolve_provider(first)) + self.assertTrue(service.delete_provider(first)) + self.assertIsNone(database.get_storage_object(obj["id"])) + self.assertEqual(service.get_settings()["default_provider_id"], second) + + def test_settings_defaults_and_clamping(self): + settings = service.get_settings() + self.assertFalse(settings["auto_upload"]) + self.assertFalse(settings["delete_local_after_upload"]) + self.assertTrue(settings["verify_before_local_delete"]) + self.assertEqual(settings["local_retention_days"], 14) + self.assertEqual(settings["max_sync_files"], 500) + self.assertFalse(settings["include_chat_logs"]) + self.assertFalse(settings["include_thumbnails"]) + + saved = service.save_settings( + { + "auto_upload": "1", + "delete_local_after_upload": "yes", + "local_retention_days": "-5", + "max_sync_files": "99999", + "auto_sync_interval_minutes": "0", + } + ) + self.assertTrue(saved["auto_upload"]) + self.assertTrue(saved["delete_local_after_upload"]) + self.assertEqual(saved["local_retention_days"], 0) + self.assertEqual(saved["max_sync_files"], 5000) + self.assertEqual(saved["auto_sync_interval_minutes"], 0) + with self.assertRaises(StorageConfigError): + service.save_settings({"default_provider_id": "999999"}) + + +class MediaCollectionTests(StorageTestBase): + def test_collection_respects_flags_and_skips(self): + t = 1_700_000_000 + self._write("streamer/main.mp4", b"main", t) + self._write("streamer/main.mp4.txt", b"sidecar", t + 1) + self._write("streamer/clips/clip.mp4", b"clip", t + 2) + self._write("streamer/chat_logs/chat.jsonl", b"{}", t + 3) + self._write(".thumbs/streamer/thumb.jpg", b"jpg", t + 4) + self._write("streamer/main.mp4.part", b"part", t + 5) + self._write("streamer/main.remux.mp4", b"remux", t + 6) + self._write("streamer/notes.pdf", b"pdf", t + 7) + self._write("core/code.mp4", b"code", t + 8) + self._write("streamer/.hidden/secret.mp4", b"hidden", t + 9) + self._write("streamer/thumb.png", b"png", t + 10) + + files = service.collect_media_files() + self.assertEqual( + files, + ["streamer/main.mp4", "streamer/main.mp4.txt", "streamer/clips/clip.mp4"], + ) + self.assertEqual(service.collect_media_files(limit=1), ["streamer/main.mp4"]) + self.assertEqual(service.collect_media_files("missing"), []) + with self.assertRaises(StorageConfigError): + service.collect_media_files("../escape") + + settings = dict(service.DEFAULT_SETTINGS) + settings["include_chat_logs"] = True + settings["include_thumbnails"] = True + with_optional = service.collect_media_files(settings=settings) + self.assertIn("streamer/chat_logs/chat.jsonl", with_optional) + self.assertIn(".thumbs/streamer/thumb.jpg", with_optional) + self.assertNotIn("streamer/main.mp4.part", with_optional) + self.assertNotIn("streamer/thumb.png", with_optional) + + def test_remote_path_for_uses_base_path_and_rejects_traversal(self): + self.assertEqual( + service.remote_path_for({"config": {"base_path": "StreamVault"}}, "streamer/file.mp4"), + "StreamVault/streamer/file.mp4", + ) + self.assertEqual( + service.remote_path_for({"config": {"base_path": "/Archive/"}}, "streamer/file.mp4"), + "Archive/streamer/file.mp4", + ) + self.assertEqual( + service.remote_path_for({"config": {}}, "streamer/file.mp4"), + "streamer/file.mp4", + ) + with self.assertRaises(StorageConfigError): + service.remote_path_for({"config": {"base_path": "../escape"}}, "streamer/file.mp4") + + def test_prepare_object_classifies_and_builds_remote_path(self): + pid = self._provider() + self._write("streamer/clips/clip.mp4", b"clip") + obj = service.prepare_object("streamer/clips/clip.mp4", service.resolve_provider(pid)) + self.assertEqual(obj["streamer"], "streamer") + self.assertEqual(obj["filename"], "clip.mp4") + self.assertEqual(obj["kind"], "clip") + self.assertEqual(obj["remote_path"], "StreamVault/streamer/clips/clip.mp4") + self.assertEqual(obj["status"], "pending") + + +class UploadBookkeepingTests(StorageTestBase): + def test_enqueue_upload_change_detection(self): + self._provider() + self._write("streamer/file.mp4", b"one") + first = service.enqueue_upload("streamer/file.mp4") + self.assertTrue(first.get("task_id")) + self.assertEqual(first["status"], "pending") + self._complete_tasks() + database.update_storage_object( + first["id"], status="uploaded", remote_size=3, uploaded_at=time.time() + ) + + second = service.enqueue_upload("streamer/file.mp4") + self.assertFalse(second.get("task_id")) + self.assertEqual(second["status"], "uploaded") + + database.update_storage_object(first["id"], local_deleted_at=time.time()) + self._write("streamer/file.mp4", b"one-two", mtime=time.time() + 5) + third = service.enqueue_upload("streamer/file.mp4") + self.assertTrue(third.get("task_id")) + self.assertEqual(third["status"], "pending") + self.assertIsNone(third["local_deleted_at"]) + + def test_enqueue_upload_base_path_change_requeues(self): + pid = self._provider() + self._write("streamer/file.mp4", b"payload") + first = service.enqueue_upload("streamer/file.mp4") + self._complete_tasks() + database.update_storage_object( + first["id"], + status="uploaded", + remote_size=7, + uploaded_at=time.time(), + local_deleted_at=time.time(), + ) + + service.update_provider(pid, values={"base_path": "Archive"}) + second = service.enqueue_upload("streamer/file.mp4") + self.assertTrue(second.get("task_id")) + self.assertEqual(second["status"], "pending") + self.assertEqual(second["remote_path"], "Archive/streamer/file.mp4") + self.assertIsNone(second["local_deleted_at"]) + + def test_enqueue_upload_does_not_duplicate_active_task(self): + self._provider() + self._write("streamer/file.mp4", b"payload") + first = service.enqueue_upload("streamer/file.mp4") + self.assertTrue(first.get("task_id")) + second = service.enqueue_upload("streamer/file.mp4") + self.assertFalse(second.get("task_id")) + conn = database.get_db() + try: + count = conn.execute( + "SELECT COUNT(*) AS n FROM tasks WHERE type='storage_upload'" + ).fetchone()["n"] + finally: + conn.close() + self.assertEqual(count, 1) + + def test_sync_fairness_filters_current_files_before_limit(self): + pid = self._provider() + start = 1_700_000_000 + for index in range(4): + self._write(f"streamer/v{index}.mp4", f"data-{index}".encode(), start + index) + + for index in (0, 1): + rel = f"streamer/v{index}.mp4" + obj = service.enqueue_upload(rel) + self._complete_tasks() + database.update_storage_object( + obj["id"], + status="uploaded", + remote_size=len(f"data-{index}"), + uploaded_at=time.time(), + ) + + service.save_settings({"max_sync_files": 2}) + result = service.run_sync({"provider_id": pid}) + self.assertEqual(result["files"], 4) + self.assertEqual(result["skipped"], 2) + self.assertEqual(result["candidates"], 2) + self.assertEqual(result["queued"], 2) + + objects = { + row["local_path"]: row + for row in database.list_storage_objects(provider_id=pid, limit=100) + } + self.assertEqual(objects["streamer/v0.mp4"]["status"], "uploaded") + self.assertEqual(objects["streamer/v1.mp4"]["status"], "uploaded") + self.assertEqual(objects["streamer/v2.mp4"]["status"], "pending") + self.assertEqual(objects["streamer/v3.mp4"]["status"], "pending") + + def test_run_sync_requeues_current_file_after_base_path_change(self): + pid = self._provider() + path = self._write("streamer/file.mp4", b"payload") + obj = database.upsert_storage_object( + pid, + "streamer/file.mp4", + "StreamVault/streamer/file.mp4", + streamer="streamer", + filename="file.mp4", + kind="vod", + local_size=len(b"payload"), + local_mtime=os.path.getmtime(path), + status="uploaded", + ) + database.update_storage_object( + obj["id"], remote_size=len(b"payload"), uploaded_at=time.time() + ) + service.update_provider(pid, values={"base_path": "Archive"}) + + result = service.run_sync({"provider_id": pid}) + + self.assertEqual(result["files"], 1) + self.assertEqual(result["skipped"], 0) + self.assertEqual(result["candidates"], 1) + self.assertEqual(result["queued"], 1) + stored = database.get_storage_object(obj["id"]) + self.assertEqual(stored["remote_path"], "Archive/streamer/file.mp4") + self.assertEqual(stored["status"], "pending") + self.assertTrue( + database.has_active_task("storage_upload", object_id=obj["id"]) + ) + + def test_run_sync_preserves_existing_schedule_fields(self): + pid = self._provider() + self._write("streamer/file.mp4", b"payload") + database.set_system_state( + service.SCHEDULE_KEY, + {"last_cleanup_at": 111.0, "custom": "keep", "last_sync_at": 222.0}, + ) + + result = service.run_sync({"provider_id": pid}) + + self.assertEqual(result["queued"], 1) + schedule = database.get_system_state(service.SCHEDULE_KEY, {}) + self.assertEqual(schedule["last_cleanup_at"], 111.0) + self.assertEqual(schedule["custom"], "keep") + self.assertGreater(schedule["last_sync_at"], 222.0) + self.assertEqual(schedule["queued"], 1) + + def test_run_sync_records_inline_cleanup_without_losing_schedule(self): + pid = self._provider() + self._write("streamer/file.mp4", b"payload") + service.save_settings( + {"delete_local_after_upload": True, "local_retention_days": 0} + ) + database.set_system_state( + service.SCHEDULE_KEY, {"last_cleanup_at": 5.0, "custom": "keep"} + ) + + with mock.patch.object( + service, + "cleanup_local_copies", + return_value={"checked": 0, "deleted": 0, "kept": 0}, + ) as cleanup: + service.run_sync({"provider_id": pid}) + + cleanup.assert_called_once_with(pid) + schedule = database.get_system_state(service.SCHEDULE_KEY, {}) + self.assertEqual(schedule["custom"], "keep") + self.assertGreater(schedule["last_cleanup_at"], 5.0) + + def test_upsert_preserves_local_metadata_and_status(self): + pid = self._provider() + database.upsert_storage_object( + pid, + "streamer/file.mp4", + "old/streamer/file.mp4", + local_size=10, + local_mtime=123.0, + status="uploaded", + ) + updated = database.upsert_storage_object( + pid, + "streamer/file.mp4", + "new/streamer/file.mp4", + local_size=0, + local_mtime=0, + status="pending", + ) + self.assertEqual(updated["local_size"], 10) + self.assertEqual(updated["local_mtime"], 123.0) + self.assertEqual(updated["status"], "uploaded") + self.assertEqual(updated["remote_path"], "new/streamer/file.mp4") + + def test_local_deleted_filter_and_task_object_filter(self): + pid = self._provider() + first = database.upsert_storage_object(pid, "a.mp4", "a.mp4", status="uploaded") + second = database.upsert_storage_object(pid, "b.mp4", "b.mp4", status="uploaded") + database.update_storage_object(second["id"], local_deleted_at=time.time()) + + self.assertEqual(len(database.list_storage_objects(provider_id=pid)), 2) + self.assertEqual( + [row["id"] for row in database.list_storage_objects(provider_id=pid, local_deleted=False)], + [first["id"]], + ) + self.assertEqual( + [row["id"] for row in database.list_storage_objects(provider_id=pid, local_deleted=True)], + [second["id"]], + ) + + task_id = database.add_task( + "storage_upload", + {"object_id": first["id"], "streamer": "streamer", "filename": "a.mp4"}, + ) + self.assertTrue(task_id) + self.assertTrue(database.has_active_task("storage_upload", object_id=first["id"])) + self.assertFalse(database.has_active_task("storage_upload", object_id=second["id"])) + self.assertTrue( + database.has_active_task("storage_upload", "streamer", "a.mp4") + ) + + def test_sync_dedupe_is_provider_scoped(self): + first = self._provider("First") + second = self._provider("Second") + first_task = service.enqueue_sync(first, "streamer") + second_task = service.enqueue_sync(second, "streamer") + self.assertTrue(first_task) + self.assertTrue(second_task) + self.assertIsNone(service.enqueue_sync(first, "streamer")) + self.assertTrue( + database.has_active_task( + "storage_sync", "streamer", None, provider_id=first + ) + ) + self.assertTrue( + database.has_active_task( + "storage_sync", "streamer", None, provider_id=second + ) + ) + + def test_auto_upload_helper_respects_setting(self): + self._provider() + path = self._write("streamer/auto.mp4", b"auto") + self.assertIsNone(service.enqueue_auto_upload(str(path))) + service.save_settings({"auto_upload": True}) + obj = service.enqueue_auto_upload(str(path)) + self.assertIsNotNone(obj) + self.assertTrue(obj.get("task_id")) + self.assertEqual(obj["status"], "pending") + + +class WorkerStorageHookTests(StorageTestBase): + def _worker(self): + import sys + + module = sys.modules.get("recorder.worker") + if module is not None: + return module + with mock.patch.dict(sys.modules, {"numpy": mock.Mock()}): + from recorder import worker + + return worker + + def test_successful_remux_enqueues_auto_upload(self): + worker = self._worker() + pid = self._provider() + service.save_settings({"auto_upload": True}) + path = self._write("streamer/file.mp4", b"payload") + + with mock.patch.object( + worker, "_safe_vod_task_path", return_value=str(path) + ), mock.patch.object( + worker, "_is_current_recording", return_value=False + ), mock.patch.object( + worker, "_is_recent", return_value=False + ), mock.patch.object( + worker, "_remux_file", return_value=(True, "") + ), mock.patch.object( + worker, "enqueue_thumbnail_if_missing" + ): + result = worker.process_remux_vod( + {"streamer": "streamer", "filename": "file.mp4"} + ) + + self.assertTrue(result) + obj = database.find_storage_object(pid, "streamer/file.mp4") + self.assertIsNotNone(obj) + self.assertEqual(obj["status"], "pending") + self.assertTrue( + database.has_active_task("storage_upload", object_id=obj["id"]) + ) + + +class RetentionTests(StorageTestBase): + class _FakeProvider: + def __init__(self, exists: bool = True): + self.exists_result = exists + self.exists_calls = 0 + self.deleted = [] + + def exists(self, remote_path, expected_size=0): + self.exists_calls += 1 + return self.exists_result + + def delete(self, remote_path): + self.deleted.append(remote_path) + + def test_delete_local_copy_requires_uploaded_status(self): + self._provider() + self._write("streamer/file.mp4", b"payload") + obj = service.enqueue_upload("streamer/file.mp4") + ok, message = service.delete_local_copy(obj["id"]) + self.assertFalse(ok) + self.assertIn("hochgeladen", message) + self.assertTrue((self.base / "streamer/file.mp4").exists()) + self.assertFalse(service.delete_local_copy(999999)[0]) + + def test_delete_local_copy_keeps_file_when_remote_unverified(self): + _, path, obj = self._uploaded_object("streamer/file.mp4") + fake = self._FakeProvider(exists=False) + with mock.patch.object(service, "build_provider", return_value=fake): + ok, message = service.delete_local_copy(obj["id"]) + self.assertFalse(ok) + self.assertIn("verifiziert", message) + self.assertTrue(path.exists()) + self.assertEqual(fake.exists_calls, 1) + self.assertIn("verifiziert", database.get_storage_object(obj["id"])["last_error"]) + + with mock.patch.object(service, "build_provider", return_value=fake): + ok, _ = service.delete_local_copy(obj["id"], verify=False) + self.assertTrue(ok) + self.assertFalse(path.exists()) + self.assertIsNotNone(database.get_storage_object(obj["id"])["local_deleted_at"]) + + def test_delete_local_copy_refuses_changed_local_file(self): + _, path, obj = self._uploaded_object("streamer/file.mp4") + path.write_bytes(b"changed-payload") + fake = self._FakeProvider(exists=True) + with mock.patch.object(service, "build_provider", return_value=fake): + ok, message = service.delete_local_copy(obj["id"]) + self.assertFalse(ok) + self.assertIn("geändert", message) + self.assertTrue(path.exists()) + stored = database.get_storage_object(obj["id"]) + self.assertIsNone(stored["local_deleted_at"]) + self.assertIn("geändert", stored["last_error"]) + + def test_delete_local_copy_respects_active_watcher(self): + _, path, obj = self._uploaded_object("streamer/file.mp4") + fake = self._FakeProvider(exists=True) + with mock.patch( + "core.vod_watch_slots.is_vod_watched", return_value=True + ), mock.patch.object(service, "build_provider", return_value=fake): + ok, message = service.delete_local_copy(obj["id"]) + self.assertFalse(ok) + self.assertIn("angesehen", message) + self.assertTrue(path.exists()) + + def test_cleanup_disabled_by_default(self): + self._provider() + result = service.cleanup_local_copies() + self.assertTrue(result["disabled"]) + self.assertEqual(result["deleted"], 0) + + def test_cleanup_deletes_after_retention_when_enabled(self): + _, path, obj = self._uploaded_object("streamer/file.mp4") + service.save_settings( + {"delete_local_after_upload": True, "local_retention_days": 0} + ) + fake = self._FakeProvider(exists=True) + with mock.patch.object(service, "build_provider", return_value=fake): + result = service.cleanup_local_copies(limit=10) + self.assertEqual(result["checked"], 1) + self.assertEqual(result["deleted"], 1) + self.assertFalse(path.exists()) + self.assertIsNotNone(database.get_storage_object(obj["id"])["local_deleted_at"]) + + def test_cleanup_continues_after_per_object_failure(self): + _, _, first = self._uploaded_object("streamer/first.mp4", b"first") + _, _, second = self._uploaded_object( + "streamer/second.mp4", b"second", provider_name="Second WebDAV" + ) + service.save_settings( + {"delete_local_after_upload": True, "local_retention_days": 0} + ) + seen = [] + + def fake_delete(object_id): + seen.append(object_id) + if object_id == first["id"]: + raise RuntimeError("provider exploded") + return True, "entfernt" + + with mock.patch.object( + service, "delete_local_copy", side_effect=fake_delete + ): + result = service.cleanup_local_copies(limit=10) + + self.assertEqual(result["checked"], 2) + self.assertEqual(result["deleted"], 1) + self.assertEqual(result["kept"], 1) + self.assertEqual(sorted(seen), sorted([first["id"], second["id"]])) + self.assertTrue( + any( + "first.mp4" in reason and "provider exploded" in reason + for reason in result["reasons"] + ) + ) + + +class ProviderBehaviourTests(StorageTestBase): + def test_catalog_contains_all_supported_providers(self): + catalog = provider_catalog() + self.assertEqual( + {entry["type_id"] for entry in catalog}, + {"google_drive", "s3", "dropbox", "onedrive", "webdav"}, + ) + for entry in catalog: + self.assertTrue(entry["label"]) + self.assertTrue(entry["fields"]) + + def test_s3_missing_dependency_is_explained(self): + provider = S3Provider( + {"bucket": "archive", "access_key": "key", "secret_key": "secret"} + ) + with mock.patch.object(s3_module, "boto3", None): + with self.assertRaises(StorageConfigError) as ctx: + provider._client() + self.assertIn("boto3", str(ctx.exception)) + + def test_google_drive_root_and_folder_resolution(self): + self.assertEqual(GoogleDriveProvider({})._root_folder_id(), "root") + self.assertEqual( + GoogleDriveProvider({"folder_id": "folder-123"})._root_folder_id(), + "folder-123", + ) + + created = [] + provider = GoogleDriveProvider({}) + provider._find_child = lambda parent, name, folder_only=False: None + provider._create_folder = lambda parent, name: ( + created.append((parent, name)) or f"id-{name}" + ) + parent = provider._folder_for_parent("StreamVault/streamer/file.mp4") + self.assertEqual(parent, "id-streamer") + self.assertEqual( + created, [("root", "StreamVault"), ("id-StreamVault", "streamer")] + ) + + def test_google_drive_zero_byte_upload_uses_multipart(self): + empty = self._write("empty.mp4", b"") + provider = GoogleDriveProvider( + {"client_id": "id", "client_secret": "secret", "refresh_token": "refresh"} + ) + provider._token = lambda force=False: "token" + provider._folder_for_parent = lambda remote_path, create=True: "parent-id" + provider._find_child = lambda parent, name, folder_only=False: None + + calls = [] + + class Response: + status_code = 200 + content = b"{}" + + def fake_request(method, url, **kwargs): + calls.append((method, url, kwargs)) + return Response() + + with mock.patch.object(google_module.http, "request", fake_request), mock.patch.object( + google_module.http, + "json_response", + lambda response, action, provider=None: { + "id": "file-1", + "size": "0", + "md5Checksum": "", + "modifiedTime": "", + }, + ): + result = provider.upload_file(str(empty), "StreamVault/streamer/empty.mp4") + + self.assertEqual(result["remote_id"], "file-1") + self.assertEqual(result["size"], 0) + self.assertEqual(len(calls), 1) + method, url, kwargs = calls[0] + self.assertEqual(method, "POST") + self.assertTrue(url.endswith("/files")) + self.assertEqual(kwargs["params"]["uploadType"], "multipart") + self.assertTrue(kwargs["headers"]["Content-Type"].startswith("multipart/related")) + + def test_google_drive_upload_401_retry_is_bounded(self): + class Unauthorized: + status_code = 401 + reason = "Unauthorized" + text = "denied" + content = b"" + headers = {} + + def json(self): + raise ValueError("not json") + + def close(self): + pass + + empty = self._write("empty.mp4", b"") + provider = GoogleDriveProvider({}) + token_calls = [] + provider._token = lambda force=False: token_calls.append(bool(force)) or "token" + provider._folder_for_parent = lambda remote_path, create=True: "parent-id" + provider._find_child = lambda parent, name, folder_only=False: None + requests = [] + + def fake_request(method, url, **kwargs): + requests.append((method, url)) + return Unauthorized() + + with mock.patch.object(google_module.http, "request", fake_request): + with self.assertRaises(StorageError): + provider.upload_file(str(empty), "StreamVault/streamer/empty.mp4") + + self.assertEqual(len(requests), 2) + self.assertEqual(token_calls, [False, True]) + + def test_google_drive_chunk_401_retry_is_bounded(self): + class Initiated: + status_code = 200 + reason = "OK" + text = "" + content = b"" + headers = {"Location": "https://upload.example/session"} + + def json(self): + return {} + + def close(self): + pass + + class Unauthorized: + status_code = 401 + reason = "Unauthorized" + text = "denied" + content = b"" + headers = {} + + def json(self): + raise ValueError("not json") + + def close(self): + pass + + local = self._write("clip.mp4", b"x") + provider = GoogleDriveProvider({}) + token_calls = [] + provider._token = lambda force=False: token_calls.append(bool(force)) or "token" + provider._folder_for_parent = lambda remote_path, create=True: "parent-id" + provider._find_child = lambda parent, name, folder_only=False: None + requests = [] + responses = [Initiated(), Unauthorized(), Initiated(), Unauthorized()] + + def fake_request(method, url, **kwargs): + requests.append((method, url)) + return responses.pop(0) + + with mock.patch.object(google_module.http, "request", fake_request): + with self.assertRaises(StorageError): + provider.upload_file(str(local), "StreamVault/streamer/clip.mp4") + + self.assertEqual(len(requests), 4) + # One forced refresh is enough; the repeated 401 must not recurse again. + self.assertEqual(token_calls, [False, True, False]) + + def test_google_drive_list_pagination(self): + provider = GoogleDriveProvider({"base_path": "StreamVault"}) + provider._folder_for_parent = lambda remote_path, create=True: "parent-id" + pages = [ + {"files": [{"id": "1", "name": "a.mp4", "size": "1"}], "nextPageToken": "next"}, + {"files": [{"id": "2", "name": "b.mp4", "size": "2"}]}, + ] + seen = [] + + def fake_api(method, path, **kwargs): + seen.append(dict(kwargs.get("params") or {})) + response = mock.Mock() + response.page = pages[len(seen) - 1] + return response + + with mock.patch.object(provider, "_api", fake_api), mock.patch.object( + google_module.http, + "json_response", + lambda response, action, provider=None: response.page, + ): + items = provider.list("StreamVault/streamer") + self.assertEqual([item.remote_id for item in items], ["1", "2"]) + self.assertEqual(seen[1]["pageToken"], "next") + + def test_google_drive_caches_folders_but_not_file_metadata(self): + provider = GoogleDriveProvider({}) + calls = [] + payloads = [ + { + "files": [ + { + "id": "folder-1", + "name": "StreamVault", + "mimeType": google_module.FOLDER_MIME, + } + ] + }, + {"files": [{"id": "file-1", "name": "a.mp4", "size": "5"}]}, + {"files": [{"id": "file-1", "name": "a.mp4", "size": "6"}]}, + ] + + def fake_api(method, path, **kwargs): + calls.append(kwargs.get("params") or {}) + response = mock.Mock() + response.payload = payloads[len(calls) - 1] + return response + + with mock.patch.object(provider, "_api", fake_api), mock.patch.object( + google_module.http, + "json_response", + lambda response, action, provider=None: response.payload, + ): + first_folder = provider._find_child( + "root", "StreamVault", folder_only=True + ) + cached_folder = provider._find_child( + "root", "StreamVault", folder_only=True + ) + first_file = provider._find_child("root", "a.mp4") + refreshed_file = provider._find_child("root", "a.mp4") + + self.assertEqual(first_folder["id"], "folder-1") + self.assertEqual(cached_folder["id"], "folder-1") + self.assertEqual(len(calls), 3) + self.assertIn( + f"mimeType = '{google_module.FOLDER_MIME}'", calls[0]["q"] + ) + self.assertIn( + f"mimeType != '{google_module.FOLDER_MIME}'", calls[1]["q"] + ) + self.assertIn( + f"mimeType != '{google_module.FOLDER_MIME}'", calls[2]["q"] + ) + self.assertEqual(first_file["size"], "5") + self.assertEqual(refreshed_file["size"], "6") + self.assertEqual( + provider._folder_cache[("root", "d:StreamVault")]["id"], "folder-1" + ) + self.assertNotIn(("root", "f:a.mp4"), provider._folder_cache) + + def test_onedrive_folder_creation_uses_existing_parent_urls(self): + provider = OneDriveProvider( + { + "client_id": "id", + "client_secret": "secret", + "refresh_token": "refresh", + } + ) + calls = [] + statuses = [201, 409, 201] + + class Response: + def __init__(self, status_code): + self.status_code = status_code + + def fake_api(method, url, **kwargs): + calls.append((method, url, kwargs.get("json") or {})) + return Response(statuses[len(calls) - 1]) + + with mock.patch.object(provider, "_api", fake_api): + provider._ensure_dirs("StreamVault/streamer/clips/file.mp4") + + self.assertEqual( + [url for _, url, _ in calls], + [ + f"{onedrive_module.GRAPH}/me/drive/root/children", + f"{onedrive_module.GRAPH}/me/drive/root:/StreamVault:/children", + f"{onedrive_module.GRAPH}/me/drive/root:/StreamVault/streamer:/children", + ], + ) + self.assertEqual( + [body["name"] for _, _, body in calls], + ["StreamVault", "streamer", "clips"], + ) + self.assertEqual( + [body["@microsoft.graph.conflictBehavior"] for _, _, body in calls], + ["fail", "fail", "fail"], + ) + + def test_onedrive_list_follows_next_link(self): + provider = OneDriveProvider( + { + "client_id": "id", + "client_secret": "secret", + "refresh_token": "refresh", + } + ) + pages = [ + { + "value": [{"id": "1", "name": "a.mp4", "size": 1}], + "@odata.nextLink": "https://graph.example/next", + }, + {"value": [{"id": "2", "name": "b.mp4", "size": 2}]}, + ] + seen = [] + + def fake_api(method, url, **kwargs): + seen.append((url, kwargs.get("params"))) + response = mock.Mock() + response.page = pages[len(seen) - 1] + return response + + with mock.patch.object(provider, "_api", fake_api), mock.patch.object( + onedrive_module.http, + "json_response", + lambda response, action, provider=None: response.page, + ): + items = provider.list("StreamVault/streamer") + self.assertEqual([item.remote_id for item in items], ["1", "2"]) + self.assertEqual(seen[1][0], "https://graph.example/next") + self.assertIsNone(seen[1][1]) + + def test_dropbox_upload_creates_missing_parent_folders(self): + provider = DropboxProvider({"access_token": "token"}) + existing = {"/StreamVault"} + created = [] + + def fake_rpc(endpoint, payload, action): + path = payload.get("path") + if endpoint == "/files/get_metadata": + if path in existing: + return {".tag": "folder"} + raise StorageError("not found", status_code=409) + if endpoint == "/files/create_folder_v2": + existing.add(path) + created.append(path) + return {".tag": "folder"} + raise AssertionError(f"unexpected endpoint: {endpoint}") + + provider._rpc = fake_rpc + provider._ensure_parent_folders("StreamVault/streamer/clips/file.mp4") + self.assertEqual( + created, + ["/StreamVault/streamer", "/StreamVault/streamer/clips"], + ) + + def test_dropbox_upload_ensures_parent_folder_before_transfer(self): + path = self._write("dropbox.mp4", b"data") + provider = DropboxProvider({"access_token": "token"}) + provider._ensure_parent_folders = mock.Mock() + + class Response: + status_code = 200 + + with mock.patch.object(provider, "_content", return_value=Response()), mock.patch.object( + dropbox_module.http, + "json_response", + return_value={"id": "file-1", "size": 4}, + ): + result = provider.upload_file( + str(path), "StreamVault/streamer/dropbox.mp4" + ) + provider._ensure_parent_folders.assert_called_once_with( + "/StreamVault/streamer/dropbox.mp4" + ) + self.assertEqual(result["size"], 4) + + def test_dropbox_range_falls_back_when_server_ignores_range(self): + provider = DropboxProvider({"access_token": "token"}) + + class Response: + status_code = 200 + + def iter_content(self, chunk_size): + yield b"01234" + yield b"56789" + + def close(self): + pass + + provider._download = lambda remote_path, range_header="": Response() + self.assertEqual( + b"".join(provider.open_range("StreamVault/streamer/file.mp4", 3, 7)), + b"34567", + ) + + def test_webdav_url_quotes_path_segments(self): + provider = WebDAVProvider( + { + "base_url": "https://dav.example.com/root/", + "username": "u", + "password": "p", + } + ) + self.assertEqual( + provider._url("folder/a b#c.mp4"), + "https://dav.example.com/root/folder/a%20b%23c.mp4", + ) + + +class ErrorHandlingTests(StorageTestBase): + def test_test_provider_sanitizes_remote_errors(self): + pid = self._provider() + + class BrokenProvider: + def test_connection(self): + raise StorageError( + "authentication failed access_token=supersecret password=hunter2", + status_code=401, + ) + + with mock.patch.object(service, "build_provider", return_value=BrokenProvider()): + result = service.test_provider(pid) + self.assertFalse(result["ok"]) + self.assertIn("HTTP 401", result["message"]) + self.assertNotIn("supersecret", result["message"]) + self.assertNotIn("hunter2", result["message"]) + row = database.get_storage_provider(pid) + self.assertNotIn("supersecret", row["last_check_message"]) + self.assertEqual(row["last_check_ok"], False) + + def test_run_upload_marks_object_uploaded(self): + self._provider() + self._write("streamer/empty.mp4", b"") + obj = service.enqueue_upload("streamer/empty.mp4") + self._complete_tasks() + + class FakeUploadProvider: + def upload_file(self, local_path, remote_path, *, progress=None): + if progress: + progress(0, 0) + return {"size": 0, "modified_at": "now"} + + with mock.patch.object(service, "build_provider", return_value=FakeUploadProvider()): + result = service.run_upload({"object_id": obj["id"]}) + self.assertTrue(result["ok"]) + stored = database.get_storage_object(obj["id"]) + self.assertEqual(stored["status"], "uploaded") + self.assertEqual(stored["remote_size"], 0) + self.assertEqual(stored["progress"], 100) + + def test_run_upload_sanitizes_failure_and_persists_status(self): + self._provider() + self._write("streamer/file.mp4", b"payload") + obj = service.enqueue_upload("streamer/file.mp4") + self._complete_tasks() + + class BrokenUploadProvider: + def upload_file(self, local_path, remote_path, *, progress=None): + raise StorageError( + "upload failed access_token=supersecret", status_code=500 + ) + + with mock.patch.object(service, "build_provider", return_value=BrokenUploadProvider()): + with self.assertRaises(StorageError): + service.run_upload({"object_id": obj["id"]}) + stored = database.get_storage_object(obj["id"]) + self.assertEqual(stored["status"], "failed") + self.assertIn("HTTP 500", stored["last_error"]) + self.assertNotIn("supersecret", stored["last_error"]) + self.assertNotIn("supersecret", database.get_storage_provider(stored["provider_id"])["last_error"]) + + +class ProviderDefaultsTests(StorageTestBase): + def test_explicit_empty_base_path_is_not_replaced_by_default(self): + pid = service.create_provider( + "Empty Root", + "webdav", + { + "base_url": "https://dav.example.com/root/", + "username": "vault", + "password": "secret", + "base_path": "", + }, + ) + record = service.get_provider_record(pid) + self.assertEqual(record["config"]["base_path"], "") + self.assertEqual( + service.remote_path_for(record, "streamer/file.mp4"), + "streamer/file.mp4", + ) + + service.update_provider(pid, values={"base_path": "Archive"}) + self.assertEqual( + service.get_provider_record(pid)["config"]["base_path"], "Archive" + ) + service.update_provider(pid, values={"base_path": " "}) + self.assertEqual(service.get_provider_record(pid)["config"]["base_path"], "") + + def test_invalid_base_path_is_rejected_at_save_time(self): + pid = self._provider() + with self.assertRaises(StorageConfigError): + service.update_provider(pid, values={"base_path": "../escape"}) + self.assertEqual( + service.get_provider_record(pid)["config"]["base_path"], "StreamVault" + ) + with self.assertRaises(StorageConfigError): + service.create_provider( + "Traversal", + "webdav", + { + "base_url": "https://dav.example.com/root/", + "username": "vault", + "password": "secret", + "base_path": "foo/../bar", + }, + ) + + +class PrepareObjectHardeningTests(StorageTestBase): + def test_prepare_object_rejects_non_archive_paths(self): + pid = self._provider() + provider = service.resolve_provider(pid) + rejected = ( + ".meta/secret.mp4", + "core/code.mp4", + "bin/tool.mp4", + "streamer/main.mp4.part", + "streamer/main.tmp", + "streamer/chat_logs/chat.jsonl", + ".thumbs/streamer/thumb.jpg", + "streamer/.hidden/secret.mp4", + "loose.mp4", + ) + for rel in rejected: + self._write(rel) + with self.subTest(path=rel), self.assertRaises(StorageConfigError): + service.prepare_object(rel, provider) + + settings = dict(service.DEFAULT_SETTINGS) + settings["include_chat_logs"] = True + settings["include_thumbnails"] = True + self.assertEqual( + service.prepare_object( + "streamer/chat_logs/chat.jsonl", provider, settings=settings + )["kind"], + "chat", + ) + self.assertEqual( + service.prepare_object( + ".thumbs/streamer/thumb.jpg", provider, settings=settings + )["kind"], + "thumb", + ) + + def test_collect_media_files_rejects_unsafe_streamer_names(self): + for value in ("../escape", "foo/bar", "foo\\bar", "C:foo", ".", "..", "bad\x01name"): + with self.subTest(value=value), self.assertRaises(StorageConfigError): + service.collect_media_files(value) + + def test_collect_media_files_skips_symlinks_and_vanishing_files(self): + self._write("streamer/real.mp4", b"real") + with tempfile.TemporaryDirectory(prefix="streamvault-outside-") as outside_dir: + outside = Path(outside_dir) / "outside.mp4" + outside.write_bytes(b"outside") + link = self.base / "streamer" / "link.mp4" + try: + os.symlink(outside, link) + except (OSError, NotImplementedError): + self.skipTest("Symlinks are not available on this platform") + self.assertNotIn("streamer/link.mp4", service.collect_media_files()) + + self._write("streamer/vanish.mp4", b"vanish") + real_stat = os.stat + + def flaky_stat(path, *args, **kwargs): + if os.path.normcase(str(path)).endswith(os.path.normcase("vanish.mp4")): + raise FileNotFoundError(str(path)) + return real_stat(path, *args, **kwargs) + + with mock.patch.object(service.os, "stat", side_effect=flaky_stat): + files = service.collect_media_files() + self.assertIn("streamer/real.mp4", files) + self.assertNotIn("streamer/vanish.mp4", files) + + +class RemoteDeleteSafetyTests(StorageTestBase): + class _FakeDeleteProvider: + def __init__(self): + self.deleted = [] + + def delete(self, remote_path): + self.deleted.append(remote_path) + + def test_enqueue_delete_blocks_unsafe_objects_and_dedupes(self): + _, path, obj = self._uploaded_object("streamer/file.mp4", b"payload") + database.add_task( + "storage_upload", + { + "object_id": obj["id"], + "streamer": "streamer", + "filename": "file.mp4", + }, + ) + with self.assertRaises(StorageConfigError) as ctx: + service.enqueue_delete(obj["id"]) + self.assertIn("Upload", str(ctx.exception)) + self._complete_tasks() + + database.update_storage_object(obj["id"], local_deleted_at=time.time()) + with self.assertRaises(StorageConfigError) as ctx: + service.enqueue_delete(obj["id"]) + self.assertIn("lokale Kopie", str(ctx.exception)) + database.update_storage_object(obj["id"], local_deleted_at=None) + + path.unlink() + with self.assertRaises(StorageConfigError) as ctx: + service.enqueue_delete(obj["id"]) + self.assertIn("fehlt", str(ctx.exception)) + + path.write_bytes(b"different-size") + with self.assertRaises(StorageConfigError) as ctx: + service.enqueue_delete(obj["id"]) + self.assertIn("geändert", str(ctx.exception)) + + # Simulate a fresh upload of the original content before deleting. + path.write_bytes(b"payload") + database.update_storage_object( + obj["id"], + local_size=len(b"payload"), + local_mtime=os.path.getmtime(path), + ) + first = service.enqueue_delete(obj["id"]) + self.assertTrue(first) + self.assertEqual(service.enqueue_delete(obj["id"]), 0) + + def test_run_delete_refuses_missing_local_and_records_error(self): + _, path, obj = self._uploaded_object("streamer/file.mp4", b"payload") + path.unlink() + fake = self._FakeDeleteProvider() + with mock.patch.object(service, "build_provider", return_value=fake): + with self.assertRaises(StorageConfigError) as ctx: + service.run_delete({"object_id": obj["id"]}) + self.assertIn("fehlt", str(ctx.exception)) + self.assertEqual(fake.deleted, []) + stored = database.get_storage_object(obj["id"]) + self.assertEqual(stored["status"], "uploaded") + self.assertIn("fehlt", stored["last_error"]) + self.assertIsNone(stored["remote_deleted_at"]) + + def test_run_delete_marks_remote_deleted_and_requeues_upload(self): + _, _, obj = self._uploaded_object("streamer/file.mp4", b"payload") + fake = self._FakeDeleteProvider() + with mock.patch.object(service, "build_provider", return_value=fake): + result = service.run_delete({"object_id": obj["id"]}) + self.assertTrue(result["ok"]) + self.assertEqual(fake.deleted, ["StreamVault/streamer/file.mp4"]) + stored = database.get_storage_object(obj["id"]) + self.assertEqual(stored["status"], "remote_deleted") + self.assertIsNotNone(stored["remote_deleted_at"]) + + again = service.run_delete({"object_id": obj["id"]}) + self.assertTrue(again.get("already_deleted")) + self.assertEqual(fake.deleted, ["StreamVault/streamer/file.mp4"]) + + requeued = service.enqueue_upload("streamer/file.mp4") + self.assertTrue(requeued.get("task_id")) + self.assertEqual(requeued["status"], "pending") + self.assertIsNone(requeued["remote_deleted_at"]) + + def test_run_upload_clears_remote_deleted_marker(self): + _, _, obj = self._uploaded_object("streamer/file.mp4", b"payload") + database.update_storage_object( + obj["id"], status="remote_deleted", remote_deleted_at=time.time() + ) + observed = {} + + class InspectingProvider: + def upload_file(self, local_path, remote_path, *, progress=None): + row = database.get_storage_object(obj["id"]) + observed["status"] = row["status"] + observed["remote_deleted_at"] = row["remote_deleted_at"] + return {"size": len(b"payload")} + + with mock.patch.object( + service, "build_provider", return_value=InspectingProvider() + ): + result = service.run_upload({"object_id": obj["id"]}) + self.assertTrue(result["ok"]) + self.assertEqual(observed["status"], "uploading") + self.assertIsNone(observed["remote_deleted_at"]) + stored = database.get_storage_object(obj["id"]) + self.assertEqual(stored["status"], "uploaded") + self.assertIsNone(stored["remote_deleted_at"]) + + def test_run_upload_rejects_provider_size_mismatch(self): + _, _, obj = self._uploaded_object("streamer/file.mp4", b"payload") + + class ShortUploadProvider: + def upload_file(self, local_path, remote_path, *, progress=None): + return {"size": 0} + + with mock.patch.object( + service, "build_provider", return_value=ShortUploadProvider() + ): + with self.assertRaises(StorageError) as ctx: + service.run_upload({"object_id": obj["id"], "force": True}) + self.assertIn("unvollständig", str(ctx.exception)) + stored = database.get_storage_object(obj["id"]) + self.assertEqual(stored["status"], "failed") + self.assertIn("unvollständig", stored["last_error"]) + provider = database.get_storage_provider(stored["provider_id"]) + self.assertIn("unvollständig", provider["last_error"]) + + +class RestoreSafetyTests(StorageTestBase): + class _FakeDownloadProvider: + def __init__(self, data=b"", *, error=None, write_partial=False): + self.data = data + self.error = error + self.write_partial = write_partial + self.calls = 0 + self.remote_path = "" + + def download_file(self, remote_path, local_path, *, progress=None): + self.calls += 1 + self.remote_path = remote_path + if self.write_partial: + Path(local_path).write_bytes(b"partial") + if self.error: + raise self.error + Path(local_path).write_bytes(self.data) + if progress: + progress(len(self.data), len(self.data)) + return {"size": len(self.data)} + + def _restore_object( + self, + data=b"payload", + *, + remote_size=None, + local=b"old-local", + remote_deleted_at=None, + ): + pid = self._provider() + path = self._write("streamer/file.mp4", local) + obj = database.upsert_storage_object( + pid, + "streamer/file.mp4", + "StreamVault/streamer/file.mp4", + streamer="streamer", + filename="file.mp4", + kind="vod", + local_size=len(local), + local_mtime=os.path.getmtime(path), + status="uploaded", + ) + fields = { + "remote_size": len(data) if remote_size is None else remote_size, + "uploaded_at": time.time(), + } + if remote_deleted_at is not None: + fields["remote_deleted_at"] = remote_deleted_at + database.update_storage_object(obj["id"], **fields) + return path, database.get_storage_object(obj["id"]) + + def test_restore_unknown_remote_size_redownloads_existing_file(self): + path, obj = self._restore_object( + b"new-remote-data", remote_size=0, local=b"old-local" + ) + fake = self._FakeDownloadProvider(b"new-remote-data") + with mock.patch.object(service, "build_provider", return_value=fake): + result = service.run_restore({"object_id": obj["id"]}) + self.assertTrue(result["ok"]) + self.assertEqual(fake.calls, 1) + self.assertEqual(path.read_bytes(), b"new-remote-data") + stored = database.get_storage_object(obj["id"]) + self.assertEqual(stored["status"], "restored") + self.assertEqual(stored["remote_size"], len(b"new-remote-data")) + self.assertIsNone(stored["local_deleted_at"]) + + def test_restore_trusts_matching_local_copy_without_download(self): + path, obj = self._restore_object(b"payload", remote_size=7, local=b"payload") + fake = self._FakeDownloadProvider(b"payload") + with mock.patch.object(service, "build_provider", return_value=fake): + result = service.run_restore({"object_id": obj["id"]}) + self.assertTrue(result.get("already_local")) + self.assertEqual(fake.calls, 0) + self.assertEqual(path.read_bytes(), b"payload") + + def test_restore_failure_preserves_local_copy_and_cleans_temp(self): + path, obj = self._restore_object(b"new-data", remote_size=8, local=b"old-local") + fake = self._FakeDownloadProvider( + error=StorageError("download failed token=supersecret", status_code=500), + write_partial=True, + ) + with mock.patch.object(service, "build_provider", return_value=fake): + with self.assertRaises(StorageError): + service.run_restore({"object_id": obj["id"]}) + self.assertEqual(path.read_bytes(), b"old-local") + self.assertFalse(Path(str(path) + ".restore").exists()) + stored = database.get_storage_object(obj["id"]) + self.assertEqual(stored["status"], "uploaded") + self.assertIn("HTTP 500", stored["last_error"]) + self.assertNotIn("supersecret", stored["last_error"]) + + def test_restore_size_mismatch_does_not_replace_local_copy(self): + path, obj = self._restore_object( + b"1234567890", remote_size=10, local=b"old-local" + ) + fake = self._FakeDownloadProvider(b"short") + with mock.patch.object(service, "build_provider", return_value=fake): + with self.assertRaises(StorageError) as ctx: + service.run_restore({"object_id": obj["id"]}) + self.assertIn("unvollständig", str(ctx.exception)) + self.assertEqual(path.read_bytes(), b"old-local") + self.assertFalse(Path(str(path) + ".restore").exists()) + + def test_restore_refuses_remote_deleted_object(self): + _, obj = self._restore_object(remote_deleted_at=time.time()) + with mock.patch.object(service, "build_provider") as build_provider: + with self.assertRaises(StorageConfigError) as ctx: + service.run_restore({"object_id": obj["id"]}) + self.assertIn("gelöscht", str(ctx.exception)) + build_provider.assert_not_called() + self.assertIn( + "gelöscht", database.get_storage_object(obj["id"])["last_error"] + ) + + def test_enqueue_restore_dedupes_by_object_id(self): + _, obj = self._restore_object() + first = service.enqueue_restore(obj["id"]) + self.assertTrue(first) + self.assertEqual(service.enqueue_restore(obj["id"]), 0) + + +class BookkeepingHardeningTests(StorageTestBase): + def test_summary_excludes_remote_deleted_objects(self): + pid = self._provider() + first = database.upsert_storage_object(pid, "a.mp4", "a.mp4", status="uploaded") + database.update_storage_object(first["id"], remote_size=10) + second = database.upsert_storage_object( + pid, "b.mp4", "b.mp4", status="remote_deleted" + ) + database.update_storage_object( + second["id"], remote_size=99, remote_deleted_at=time.time() + ) + + result = database.storage_object_summary(pid) + self.assertEqual(result["total"], 1) + self.assertEqual(result["bytes"], 10) + self.assertIn("uploaded", result["statuses"]) + self.assertNotIn("remote_deleted", result["statuses"]) + self.assertEqual( + database.storage_object_summary("not-a-provider"), + {"total": 0, "bytes": 0, "statuses": {}}, + ) + + +class CryptoAndHttpHardeningTests(StorageTestBase): + def test_public_placeholder_dedicated_key_is_rejected(self): + placeholder = ( + "change-this-to-a-very-secure-random-secret-key-32-chars-minimum" + ) + with mock.patch.dict( + os.environ, + { + "STORAGE_ENCRYPTION_KEY": placeholder, + "VOD_SECRET_KEY": "strong-fallback-secret-0123456789", + }, + ): + self.assertFalse(encryption_ready()) + with self.assertRaises(StorageConfigError): + encrypt_config({"password": "x"}) + + def test_weak_env_secret_uses_strong_secret_file(self): + secret_file = self.base / ".meta" / "secret_key.txt" + secret_file.parent.mkdir(parents=True, exist_ok=True) + secret_file.write_text( + "strong-generated-secret-0123456789", encoding="utf-8" + ) + with mock.patch.dict( + os.environ, {"STORAGE_ENCRYPTION_KEY": "", "VOD_SECRET_KEY": "change-me"} + ), mock.patch.object(config, "SECRET_KEY", "change-me"): + self.assertTrue(encryption_ready()) + token = encrypt_config({"password": "x"}) + self.assertEqual(decrypt_config(token), {"password": "x"}) + + def test_http_errors_strip_urls_and_upload_tokens(self): + text = http.sanitize_error_text( + "POST https://content.dropboxapi.com/2/files/upload?token=supersecret failed" + ) + self.assertIn("[URL]", text) + self.assertNotIn("dropboxapi", text) + self.assertNotIn("supersecret", text) + + response = mock.Mock() + response.status_code = 500 + response.reason = "Server Error" + response.json.side_effect = ValueError("no json") + response.text = "secret=supersecret" + self.assertEqual( + http.error_text(response), + "Antwort des Anbieters konnte nicht gelesen werden.", + ) + + response.json.side_effect = None + response.json.return_value = { + "error": {"message": "failed https://api.example/upload?token=abc"} + } + message = http.error_text(response) + self.assertIn("[URL]", message) + self.assertNotIn("api.example", message) + self.assertNotIn("token=abc", message) + + def test_http_session_uses_wildcard_accept(self): + previous = getattr(http._LOCAL, "session", None) + http._LOCAL.session = None + self.addCleanup(setattr, http._LOCAL, "session", previous) + with mock.patch.object(http.requests, "Session") as session_cls: + fake = mock.Mock() + fake.headers = {} + session_cls.return_value = fake + current = http.session() + self.assertIs(current, fake) + self.assertEqual(fake.headers["Accept"], "*/*") + + def test_dropbox_api_arg_header_is_ascii_safe(self): + provider = DropboxProvider({"access_token": "token"}) + captured = {} + + def fake_request(method, url, **kwargs): + captured.update(kwargs) + return mock.Mock() + + with mock.patch.object(dropbox_module.http, "request", fake_request): + provider._content( + "/files/upload", + {"path": "/StreamVault/ü.mp4"}, + data=b"x", + action="Dropbox-Test", + ) + raw = captured["headers"]["Dropbox-API-Arg"] + self.assertTrue(raw.isascii()) + self.assertIn("\\u00fc", raw) + + +class ProviderExistsTests(StorageTestBase): + def test_webdav_exists_uses_head_and_strict_size(self): + provider = WebDAVProvider( + { + "base_url": "https://dav.example.com/root/", + "username": "u", + "password": "p", + } + ) + response = mock.Mock(status_code=200, headers={"Content-Length": "5"}) + with mock.patch.object(provider, "_request", return_value=response): + self.assertTrue(provider.exists("a/b.mp4", 5)) + self.assertFalse(provider.exists("a/b.mp4", 6)) + response.headers = {"Content-Length": "0"} + with mock.patch.object(provider, "_request", return_value=response): + self.assertFalse(provider.exists("a/b.mp4", 5)) + not_found = mock.Mock(status_code=404, headers={}) + with mock.patch.object(provider, "_request", return_value=not_found): + self.assertFalse(provider.exists("a/b.mp4", 5)) + + def test_google_drive_exists_checks_size_and_missing_folder(self): + provider = GoogleDriveProvider({"base_path": "StreamVault"}) + provider._folder_for_parent = lambda remote_path, create=True: "parent" + provider._find_child = lambda parent, name, folder_only=False: { + "id": "file", + "size": "5", + } + self.assertTrue(provider.exists("StreamVault/a.mp4", 5)) + self.assertFalse(provider.exists("StreamVault/a.mp4", 6)) + + provider._find_child = lambda parent, name, folder_only=False: { + "id": "file", + "size": "0", + } + self.assertFalse(provider.exists("StreamVault/a.mp4", 5)) + + provider._folder_for_parent = lambda remote_path, create=True: None + self.assertFalse(provider.exists("StreamVault/a.mp4", 5)) + + def test_onedrive_exists_checks_size_and_not_found(self): + provider = OneDriveProvider( + {"client_id": "id", "client_secret": "secret", "refresh_token": "refresh"} + ) + response = mock.Mock(status_code=200) + with mock.patch.object(provider, "_api", return_value=response), mock.patch.object( + onedrive_module.http, "json_response", return_value={"size": 5} + ): + self.assertTrue(provider.exists("StreamVault/a.mp4", 5)) + self.assertFalse(provider.exists("StreamVault/a.mp4", 6)) + + not_found = mock.Mock(status_code=404) + with mock.patch.object(provider, "_api", return_value=not_found): + self.assertFalse(provider.exists("StreamVault/a.mp4", 5)) + + def test_dropbox_exists_checks_size_and_conflict(self): + provider = DropboxProvider({"access_token": "token"}) + with mock.patch.object(provider, "_rpc", return_value={"size": 5}): + self.assertTrue(provider.exists("StreamVault/a.mp4", 5)) + self.assertFalse(provider.exists("StreamVault/a.mp4", 6)) + + with mock.patch.object( + provider, + "_rpc", + side_effect=StorageError("not found", status_code=409), + ): + self.assertFalse(provider.exists("StreamVault/a.mp4", 5)) + + def test_s3_exists_uses_head_content_length(self): + provider = S3Provider( + {"bucket": "archive", "access_key": "key", "secret_key": "secret"} + ) + client = mock.Mock() + client.head_object.return_value = {"ContentLength": 5} + with mock.patch.object(provider, "_client", return_value=client): + self.assertTrue(provider.exists("StreamVault/a.mp4", 5)) + self.assertFalse(provider.exists("StreamVault/a.mp4", 6)) + + +class CliSafetyTests(StorageTestBase): + def test_upload_resolves_cwd_relative_path(self): + pid = self._provider() + self._write("streamer/file.mp4", b"payload") + previous = os.getcwd() + os.chdir(self.base) + self.addCleanup(os.chdir, previous) + output = io.StringIO() + with redirect_stdout(output): + code = cli.main(["upload", "streamer/file.mp4"]) + self.assertEqual(code, 0) + self.assertIn("Upload eingeordnet", output.getvalue()) + self.assertIsNotNone(database.find_storage_object(pid, "streamer/file.mp4")) + + def test_delete_remote_prechecks_before_confirmation(self): + _, path, obj = self._uploaded_object("streamer/file.mp4", b"payload") + path.unlink() + output = io.StringIO() + with redirect_stdout(output), mock.patch( + "builtins.input", side_effect=AssertionError("prompted") + ): + code = cli.main(["delete-remote", str(obj["id"])]) + self.assertEqual(code, 2) + self.assertIn("lokale Kopie fehlt", output.getvalue()) + self.assertIn("Datenverlust", output.getvalue()) + + def test_cli_masks_provider_errors(self): + pid = self._provider() + output = io.StringIO() + with mock.patch.object( + cli.service, + "test_provider", + side_effect=StorageError("access_token=supersecret", status_code=401), + ), redirect_stdout(output): + code = cli.main(["test", str(pid)]) + self.assertEqual(code, 2) + self.assertNotIn("supersecret", output.getvalue()) + self.assertIn("token=***", output.getvalue()) + + def test_add_skips_auto_test_for_unlinked_oauth_provider(self): + output = io.StringIO() + with mock.patch.object( + cli.sys, "stdin", mock.Mock(isatty=lambda: False) + ), mock.patch.object( + cli.service, "test_provider" + ) as test_provider, redirect_stdout(output): + code = cli.main( + [ + "add", + "--type", + "google_drive", + "--name", + "Unlinked Google", + "--set", + "client_id=client-id", + "--set", + "client_secret=client-secret", + ] + ) + self.assertEqual(code, 0) + test_provider.assert_not_called() + self.assertIn("übersprungen", output.getvalue()) + self.assertEqual(service.list_providers()[0]["name"], "Unlinked Google") + + +class StoragePageTests(StorageTestBase): + def _client(self): + from WEBUI import webui + + webui.ensure_admin_user() + webui.app.config.update(TESTING=True) + client = webui.app.test_client() + with client.session_transaction() as session: + now = time.time() + session["logged_in"] = True + session["user_role"] = "admin" + session["username"] = config.VOD_USER + session["login_time"] = now + session["last_active"] = now + session["_csrf_token"] = "test-csrf-token" + return client + + def test_cloud_only_object_offers_restore_not_remote_delete(self): + _, path, obj = self._uploaded_object("streamer/file.mp4", b"payload") + path.unlink() + database.update_storage_object(obj["id"], local_deleted_at=time.time()) + + response = self._client().get("/admin/storage") + self.assertEqual(response.status_code, 200) + body = response.get_data(as_text=True) + self.assertIn("Nur Cloud", body) + self.assertIn("Wiederherstellen", body) + self.assertNotIn("Lokal löschen", body) + self.assertNotIn("Cloud löschen", body) + + def test_storage_page_escapes_provider_supplied_values(self): + from WEBUI import webui + + webui.ensure_admin_user() + pid = service.create_provider( + "", + "webdav", + { + "base_url": "https://dav.example.com/root/", + "username": "vault", + "password": "secret", + "base_path": "", + }, + ) + database.set_storage_provider_check( + pid, False, "" + ) + + webui.app.config.update(TESTING=True) + client = webui.app.test_client() + with client.session_transaction() as session: + now = time.time() + session["logged_in"] = True + session["user_role"] = "admin" + session["username"] = config.VOD_USER + session["login_time"] = now + session["last_active"] = now + session["_csrf_token"] = "test-csrf-token" + + response = client.get("/admin/storage") + self.assertEqual(response.status_code, 200) + body = response.get_data(as_text=True) + self.assertNotIn("", body) + self.assertNotIn("", body) + self.assertNotIn("", body) + self.assertIn("<script>alert(1)</script>", body) + self.assertIn("\\u003cscript\\u003ealert(1)\\u003c/script\\u003e", body) + self.assertIn(f'data-update-url="/admin/storage/providers/{pid}/update"', body) + + def test_media_accel_path_uses_legacy_files_alias(self): + from WEBUI import webui + + base_file = self._write("legacy/file.mp4", b"base") + files_file = self._write("files/legacy/file.mp4", b"files") + self.assertEqual( + webui.media_accel_path( + "legacy", "file.mp4", file_path=str(base_file) + ), + f"{webui.X_ACCEL_PREFIX}/legacy/file.mp4", + ) + self.assertEqual( + webui.media_accel_path( + "legacy", "file.mp4", file_path=str(files_file) + ), + f"{webui.FILES_ACCEL_PREFIX}/legacy/file.mp4", + ) + + clip_file = self._write("files/legacy/clips/clip.mp4", b"clip") + self.assertEqual( + webui.media_accel_path( + "legacy", "clip.mp4", clip=True, file_path=str(clip_file) + ), + f"{webui.FILES_ACCEL_PREFIX}/legacy/clips/clip.mp4", + ) + + def test_unchecked_enable_creates_disabled_provider(self): + from WEBUI import webui + + webui.ensure_admin_user() + client = self._client() + response = client.post( + "/admin/storage/providers/add", + data={ + "csrf_token": "test-csrf-token", + "provider_type": "webdav", + "name": "Disabled WebDAV", + "cfg_base_url": "https://dav.example.com/root/", + "cfg_username": "vault", + "cfg_password": "secret", + "cfg_base_path": "Archive", + "cfg_verify_ssl": "1", + }, + follow_redirects=False, + ) + self.assertEqual(response.status_code, 302) + providers = [ + row for row in service.list_providers() if row["name"] == "Disabled WebDAV" + ] + self.assertEqual(len(providers), 1) + self.assertFalse(providers[0]["enabled"]) + + def test_storage_admin_error_log_omits_traceback_and_secrets(self): + from WEBUI import webui + + client = self._client() + with mock.patch.object( + service, + "admin_context", + side_effect=StorageError("access_token=supersecret", status_code=500), + ), mock.patch.object(webui.logging, "error") as log_error: + response = client.get("/admin/storage") + + self.assertEqual(response.status_code, 500) + self.assertEqual(log_error.call_count, 1) + self.assertNotIn("exc_info", log_error.call_args.kwargs) + self.assertNotIn("supersecret", str(log_error.call_args)) + + +if __name__ == "__main__": + unittest.main() From efbf754d61468b3f4c0179946c3eed40bd06704d Mon Sep 17 00:00:00 2001 From: Pierre Date: Sat, 31 Oct 2026 23:41:53 +0000 Subject: [PATCH 02/31] test(storage): disable import-time badge sync to fix flaky teardown WEBUI.webui starts a network-bound "twitch-badge-sync" daemon thread on import. The thread writes into whichever BASE_DIR is active and races TemporaryDirectory cleanup, causing intermittent "OSError: [Errno 39] Directory not empty" teardown errors (6/25 runs with test_storage.py alone, 0/25 with the sync disabled). Only the test environment changes (TWITCH_BADGE_AUTO_SYNC=0); no assertion is touched. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tzan3xULuUxadM3Azn5fzJ --- tests/test_storage.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_storage.py b/tests/test_storage.py index 7a2c6e4..458921b 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -34,6 +34,10 @@ os.environ["REDIS_PORT"] = "1" os.environ.pop("REDIS_URL", None) os.environ.pop("VOD_RATELIMIT_STORAGE_URI", None) +# WEBUI.webui starts a network-bound badge sync daemon thread on import. It +# writes into whichever BASE_DIR is active and races TemporaryDirectory cleanup +# (intermittent 'Directory not empty' teardown errors), so keep it off in tests. +os.environ["TWITCH_BADGE_AUTO_SYNC"] = "0" from core import config, database # noqa: E402 from core.storage import cli, http, service # noqa: E402 From 3faa05325187e013ff4ce3585d23c48539143a40 Mon Sep 17 00:00:00 2001 From: Pierre Date: Sat, 31 Oct 2026 23:41:53 +0000 Subject: [PATCH 03/31] fix(core,recorder): stop-before-unlock, bounded waits, retry budget Recorder: - stop_channel_and_release_lock(): release the channel flock only after the child process has exited, so a new recorder cannot start writing the same channel while the old FFmpeg is still running. Removes the duplicated reap/restart block in the main loop. - wait_for_process_exit(): replace unbounded process.wait() on FFmpeg with a hard deadline (FFMPEG_FINALIZE_TIMEOUT, default 30 s), then kill and reap. - disk_blocks_recording(): resolve not-yet-existing target paths to the nearest existing parent; log probe errors instead of swallowing them. - archive_is_shared(): skip duplicate cleanup when several sources feed one archive (fail closed). - Live-only monitor: honour shutdown_requested, use interruptible_sleep and define CHECK_INTERVAL (previously undefined -> NameError). Core: - claim_next_task(max_attempts_by_type=...): stale running tasks that exhausted their retry budget are marked 'failed' instead of being requeued forever. - Sentry: send_default_pii off and traces sampled at 0.0 by default (SENTRY_SEND_DEFAULT_PII, SENTRY_TRACES_SAMPLE_RATE), documented in .env.example. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tzan3xULuUxadM3Azn5fzJ --- .env.example | 7 ++ core/config.py | 11 ++- core/database.py | 45 +++++++++-- recorder/Streamrecorder.py | 152 ++++++++++++++++++++++++++----------- 4 files changed, 161 insertions(+), 54 deletions(-) diff --git a/.env.example b/.env.example index 475c9b8..997af42 100644 --- a/.env.example +++ b/.env.example @@ -32,6 +32,13 @@ VOD_ALLOW_ENV_LOGIN=false VOD_SESSION_MAX_AGE_SEC=0 VOD_SESSION_COOKIE_MAX_AGE_SEC=34560000 +# --- Error Monitoring (optional) --- +# SENTRY_DSN= +# PII is opt-in; keep disabled unless your privacy policy allows it. +# SENTRY_SEND_DEFAULT_PII=false +# Performance tracing is sampled and disabled by default (0.0 - 1.0). +# SENTRY_TRACES_SAMPLE_RATE=0.0 + # --- Storage Paths --- # Base directory for recordings, metadata, and database # Default: ./data (or the app directory) diff --git a/core/config.py b/core/config.py index beea6a0..f8f4d3e 100644 --- a/core/config.py +++ b/core/config.py @@ -165,6 +165,13 @@ def streamlink_twitch_auth_args(token): DISCORD_ERROR_WEBHOOK_URL = os.environ.get("DISCORD_ERROR_WEBHOOK_URL", "").strip() SENTRY_DSN = os.environ.get("SENTRY_DSN", "").strip() +SENTRY_SEND_DEFAULT_PII = _env_flag("SENTRY_SEND_DEFAULT_PII", False) +try: + SENTRY_TRACES_SAMPLE_RATE = max( + 0.0, min(1.0, float(os.environ.get("SENTRY_TRACES_SAMPLE_RATE", "0") or 0)) + ) +except (TypeError, ValueError): + SENTRY_TRACES_SAMPLE_RATE = 0.0 if SENTRY_DSN: try: import sentry_sdk @@ -174,8 +181,8 @@ def streamlink_twitch_auth_args(token): sentry_sdk.init( dsn=SENTRY_DSN, integrations=[FlaskIntegration(), ThreadingIntegration(propagate_hub=True)], - traces_sample_rate=1.0, - send_default_pii=True, + traces_sample_rate=SENTRY_TRACES_SAMPLE_RATE, + send_default_pii=SENTRY_SEND_DEFAULT_PII, ) except ImportError: pass diff --git a/core/database.py b/core/database.py index 77b9ee7..104db95 100644 --- a/core/database.py +++ b/core/database.py @@ -691,11 +691,13 @@ def list_tasks(limit=30): @db_retry(max_retries=5, delay=0.1, backoff=2.0) -def claim_next_task(worker_id, stale_after=7200, allowed_types=None): +def claim_next_task(worker_id, stale_after=7200, allowed_types=None, max_attempts_by_type=None): """Claim the oldest pending task, optionally restricted to allowed_types. When allowed_types is set (non-empty sequence), only those task types are - considered — used by the worker to avoid claiming work for saturated pools. + considered - used by the worker to avoid claiming work for saturated pools. + When max_attempts_by_type is provided, stale tasks that exhausted their + retry budget are failed instead of being requeued forever. """ conn = get_db() now = time.time() @@ -708,14 +710,43 @@ def claim_next_task(worker_id, stale_after=7200, allowed_types=None): return None try: conn.execute("BEGIN IMMEDIATE") - conn.execute( + stale_rows = conn.execute( """ - UPDATE tasks - SET status='pending', worker_id=NULL, locked_at=NULL, updated_at=?, error='Worker timeout, neu eingereiht.' + SELECT id, type, attempts + FROM tasks WHERE status='running' AND locked_at IS NOT NULL AND locked_at < ? """, - (now, cutoff), - ) + (cutoff,), + ).fetchall() + for stale in stale_rows: + attempts = int(stale["attempts"] or 0) + if max_attempts_by_type is None: + exhausted = False + else: + max_attempts = max( + 1, int(max_attempts_by_type.get(stale["type"], 1) or 1) + ) + exhausted = attempts >= max_attempts + if exhausted: + conn.execute( + """ + UPDATE tasks + SET status='failed', worker_id=NULL, locked_at=NULL, updated_at=?, + error='Worker timeout; max attempts reached.' + WHERE id=? AND status='running' + """, + (now, stale["id"]), + ) + else: + conn.execute( + """ + UPDATE tasks + SET status='pending', worker_id=NULL, locked_at=NULL, updated_at=?, + error='Worker timeout, neu eingereiht.' + WHERE id=? AND status='running' + """, + (now, stale["id"]), + ) if type_filter: placeholders = ",".join("?" for _ in type_filter) row = conn.execute( diff --git a/recorder/Streamrecorder.py b/recorder/Streamrecorder.py index 8f29c29..cd69e84 100644 --- a/recorder/Streamrecorder.py +++ b/recorder/Streamrecorder.py @@ -60,6 +60,8 @@ os.makedirs(RECORDINGS_BASE_DIR, exist_ok=True) RETRY_DELAY = 30 HEARTBEAT_INTERVAL = 60 +# Poll interval for live-only monitors (no recording, notifications only). +CHECK_INTERVAL = int(os.environ.get('STREAM_RECORDER_CHECK_INTERVAL', '30')) TEMP_OUTPUT_SUFFIX = ".part" MIN_RECORD_FILE_SIZE = 1024 SALVAGE_MIN_BYTES = int(os.environ.get('STREAM_RECORDER_SALVAGE_MIN_BYTES', str(5 * 1024 * 1024))) @@ -105,6 +107,8 @@ YTDLP_STREAM_TIMEOUT = int(os.environ.get('STREAM_RECORDER_YTDLP_STREAM_TIMEOUT', '30')) YTDLP_CHAT_IDLE_TIMEOUT = int(os.environ.get('STREAM_RECORDER_YTDLP_CHAT_IDLE_TIMEOUT', '90')) FFMPEG_OUTPUT_TIMEOUT = int(os.environ.get('STREAM_RECORDER_FFMPEG_OUTPUT_TIMEOUT', '90')) +# Bound the final wait after stdout EOF; a wedged FFmpeg must not hang the channel forever. +FFMPEG_FINALIZE_TIMEOUT = int(os.environ.get('STREAM_RECORDER_FFMPEG_FINALIZE_TIMEOUT', '30')) # Cheap live-poll while the channel is actually offline. Do not exponential-backoff # this to minutes — that was the missing VOD-start gap (up to 10 min). STREAM_OFFLINE_RETRY_BASE = int(os.environ.get('STREAM_RECORDER_OFFLINE_RETRY_BASE', '5')) @@ -416,8 +420,21 @@ def disk_blocks_recording(path): """True when free space is critically low — pause starting new recordings.""" if _storage_pause_flag_active(): return True + target = path or RECORDINGS_BASE_DIR try: - usage = shutil.disk_usage(path or RECORDINGS_BASE_DIR) + check_path = os.path.abspath(str(target)) + while check_path and not os.path.exists(check_path): + parent = os.path.dirname(check_path) + if parent == check_path: + break + check_path = parent + if not check_path or not os.path.exists(check_path): + check_path = ( + RECORDINGS_BASE_DIR + if os.path.exists(RECORDINGS_BASE_DIR) + else os.path.dirname(os.path.abspath(RECORDINGS_BASE_DIR)) + ) + usage = shutil.disk_usage(check_path) if usage.total <= 0: return False used_pct = (float(usage.used) / float(usage.total)) * 100.0 @@ -427,7 +444,10 @@ def disk_blocks_recording(path): if free_gb <= STORAGE_PAUSE_FREE_GB: return True return False - except Exception: + except Exception as exc: + logger.warning( + "Fehler bei der Speicherplatzprüfung für %s: %s", target, exc + ) return False @@ -1257,12 +1277,39 @@ def recording_platform_tag(filename): return None +def archive_is_shared(archive_channel): + """True when more than one configured source writes into this archive. + + Duplicate cleanup must fail closed here: a same-platform file may belong to + a sibling source that shares the archive directory. + """ + archive = str(archive_channel or "").strip() + if not archive: + return True + try: + channels = database.channels_for_archive(archive) + except Exception as exc: + logger.warning( + "Archiv-Zuordnung für %s nicht prüfbar - Duplikat-Bereinigung übersprungen: %s", + archive, + exc, + ) + return True + return len(channels) > 1 + + def cleanup_duplicate_recordings(recordings_dir, recording_file, source_channel, archive_channel=None): try: if not os.path.exists(recording_file) or os.path.getsize(recording_file) < MIN_RECORD_FILE_SIZE: return recording_name = os.path.basename(recording_file) archive = archive_channel or source_channel + if archive_is_shared(archive): + logger.info( + "Duplikat-Bereinigung übersprungen: Archiv %s wird von mehreren Quellen genutzt.", + archive, + ) + return recording_ts = parse_recording_timestamp(recording_name, source_channel, archive_channel=archive) if recording_ts is None: return @@ -1344,6 +1391,38 @@ def stop_process(process): except Exception as exc: logger.warning("Unexpected error stopping process: %s", exc) +def wait_for_process_exit(process, timeout=FFMPEG_FINALIZE_TIMEOUT): + """Wait for a child with a hard deadline; kill and reap if it is wedged. + + Returns True when the process exited before the deadline. + """ + if process is None: + return True + try: + process.wait(timeout=max(1, int(timeout or 1))) + return True + except subprocess.TimeoutExpired: + logger.warning( + "Prozess %s reagiert nicht innerhalb von %ss - erzwinge Beendigung.", + getattr(process, "pid", "?"), + timeout, + ) + try: + process.kill() + except OSError as exc: + # ProcessLookupError: child exited between the timeout and kill(). + logger.warning( + "Kill fuer Prozess %s fehlgeschlagen: %s", getattr(process, "pid", "?"), exc + ) + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + logger.error("Prozess %s konnte nicht beendet werden.", getattr(process, "pid", "?")) + return False + except Exception as exc: + logger.warning("Fehler beim Warten auf Prozess %s: %s", getattr(process, "pid", "?"), exc) + return False + def cleanup_live_hls(except_channel=None): if not os.path.isdir(LIVE_HLS_ROOT): return @@ -3329,14 +3408,9 @@ def record_stream(source_channel, chat_text_file, recordings_dir): logger.error("FFmpeg-Prozess stdout ist None. Kann keine Ausgabe lesen.") if hang_detected and process.poll() is None: - try: - process.wait(timeout=10) - except subprocess.TimeoutExpired: - logger.warning("FFmpeg reagiert nicht, erzwinge Beendigung...") - process.kill() - process.wait() + wait_for_process_exit(process, timeout=10) else: - process.wait() + wait_for_process_exit(process, timeout=FFMPEG_FINALIZE_TIMEOUT) stop_process(source_process) if process.returncode == 0: @@ -3663,6 +3737,24 @@ def stop_channel_process(proc, channel, timeout=5): logger.error("Channel-Prozess %s konnte nicht gestoppt werden: %s", channel, exc) +def stop_channel_and_release_lock(recordings_dir, channel, proc, timeout=5): + """Stop the channel worker *before* releasing its flock. + + Releasing first lets a new recorder process acquire the same channel while + the old FFmpeg child is still writing, producing duplicate recordings. + """ + try: + stop_channel_process(proc, channel, timeout=timeout) + finally: + try: + release_held_channel_lock(recordings_dir, channel) + except Exception as exc: + # Must not mask an exception raised by stop_channel_process(). + logger.warning( + "Channel-Lock fuer %s konnte nicht freigegeben werden: %s", channel, exc + ) + + def request_shutdown(signum=None, frame=None): global shutdown_requested shutdown_requested = True @@ -3705,7 +3797,7 @@ def monitor_stream_live_only(source_channel, archive_channel, conf=None): is_live_now = False logger.info(f"Live-Only Monitor aktiv für {source_channel} (Discord-Alert + Live-Stream, keine Aufnahme).") - while True: + while not shutdown_requested: try: current_conf = get_source_config(source_channel) if not current_conf: @@ -3754,12 +3846,12 @@ def monitor_stream_live_only(source_channel, archive_channel, conf=None): except Exception: pass - time.sleep(CHECK_INTERVAL) + interruptible_sleep(CHECK_INTERVAL) except (KeyboardInterrupt, SystemExit): break except Exception as exc: logger.error(f"Fehler im Live-Only Monitor für {source_channel}: {exc}") - time.sleep(CHECK_INTERVAL) + interruptible_sleep(CHECK_INTERVAL) def run_channel(source_channel): """Startet Aufnahme und Chat-Prozess für einen Channel.""" @@ -3867,28 +3959,7 @@ def run_channel(source_channel): old_mode = channel_modes.get(channel) if proc.is_alive() and old_mode == mode: continue - try: - release_held_channel_lock(recordings_dir, channel) - except Exception: - pass - if proc.is_alive(): - stop_channel_process(proc, channel) - channel_procs.pop(channel, None) - channel_modes.pop(channel, None) - - # Reap / restart existing process before lock checks so our own - # held flock does not permanently block restart. - if channel in channel_procs: - proc = channel_procs[channel] - old_mode = channel_modes.get(channel) - if proc.is_alive() and old_mode == mode: - continue - try: - release_held_channel_lock(recordings_dir, channel) - except Exception: - pass - if proc.is_alive(): - stop_channel_process(proc, channel) + stop_channel_and_release_lock(recordings_dir, channel, proc) channel_procs.pop(channel, None) channel_modes.pop(channel, None) @@ -3932,11 +4003,7 @@ def run_channel(source_channel): conf = get_source_config(channel) archive_channel = conf.get("archive_name") or channel recordings_dir = os.path.join(RECORDINGS_BASE_DIR, archive_channel) - try: - release_held_channel_lock(recordings_dir, channel) - except Exception: - pass - stop_channel_process(proc, channel) + stop_channel_and_release_lock(recordings_dir, channel, proc) logger.info(f"Channel-Prozess beendet: {channel}") time.sleep(5) except KeyboardInterrupt: @@ -3946,12 +4013,7 @@ def run_channel(source_channel): conf = get_source_config(channel) or {} archive_channel = conf.get("archive_name") or channel recordings_dir = os.path.join(RECORDINGS_BASE_DIR, archive_channel) - try: - release_held_channel_lock(recordings_dir, channel) - except Exception: - pass - if proc and proc.is_alive(): - proc.terminate() + stop_channel_and_release_lock(recordings_dir, channel, proc) for proc in channel_procs.values(): if proc and proc.is_alive(): From 86e1ff249615273f4b5f8b3c18b0c1bb52289e51 Mon Sep 17 00:00:00 2001 From: Pierre Date: Sat, 31 Oct 2026 23:41:54 +0000 Subject: [PATCH 04/31] fix(worker): guard folder cleanup against live writes, drop fuser -k -9 - delete_folder / empty_folder re-check at execution time and defer (TaskDeferred, 120 s) while a recording (.part, recording lock with a live PID, fresh current_recording pointer) or a remux lock is active; the WebUI check only ran at enqueue time. - empty_folder never removes *.part files, even if the check raced. - Remove "fuser -k -9" on archive folders: it could SIGKILL the recorder/FFmpeg writing into them. - _submit_claimed_task(): release the in-flight slot and defer the task when pool dispatch fails, so capacity does not leak. - Pass TASK_MAX_ATTEMPTS to claim_next_task(). - Transcription: log and fail fast when run_transcribe.py is missing. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tzan3xULuUxadM3Azn5fzJ --- recorder/worker.py | 159 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 137 insertions(+), 22 deletions(-) diff --git a/recorder/worker.py b/recorder/worker.py index 963ca6d..6baf460 100644 --- a/recorder/worker.py +++ b/recorder/worker.py @@ -60,6 +60,8 @@ def __init__(self, message, delay_sec=60): THUMB_LOCK_CLEANUP_INTERVAL_SEC = int(os.environ.get("THUMB_LOCK_CLEANUP_INTERVAL_SEC", "1800")) THUMB_BACKFILL_INTERVAL_SEC = int(os.environ.get("THUMB_BACKFILL_INTERVAL_SEC", "300")) THUMB_BACKFILL_MAX_PER_RUN = int(os.environ.get("THUMB_BACKFILL_MAX_PER_RUN", "20")) +# A current-recording pointer is considered live only while its media file is fresh. +ACTIVE_RECORDING_FRESH_SEC = int(os.environ.get("WORKER_ACTIVE_RECORDING_FRESH_SEC", "60")) WORKER_ID = f"{os.uname().nodename}:{os.getpid()}" if hasattr(os, "uname") else f"local:{os.getpid()}" # Parallel pools (env): # WORKER_THUMB_THREADS — IO pool for thumbnail (+ delete/empty); default 2 @@ -676,21 +678,15 @@ def process_delete_folder(payload): if not streamer: streamer = os.path.basename(os.path.realpath(folder_path)) + if _folder_has_active_recording(folder_path) or _folder_has_active_remux(folder_path): + raise TaskDeferred( + f"Ordner wird noch beschrieben: {folder_path}", delay_sec=120 + ) + logging.info("Löschauftrag für: %s", folder_path) # Wartezeit, damit der Recorder-Prozess Zeit hat, sauber zu stoppen time.sleep(8) - # Prozesse killen, die den Ordner blockieren könnten - if os.path.exists(folder_path): - try: - subprocess.run( - ["fuser", "-k", "-9", folder_path], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - except Exception: - pass - def _rm_tree(path): if not path or not os.path.exists(path): return True @@ -721,14 +717,6 @@ def _rm_tree(path): logging.warning("Löschversuch %s fehlgeschlagen: %s", i + 1, exc) ok = False time.sleep(2) - try: - subprocess.run( - ["fuser", "-k", "-9", folder_path], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - except Exception: - pass continue logging.info("Ordner erfolgreich gelöscht: %s", folder_path) ok = True @@ -793,6 +781,98 @@ def _is_direct_child(parent, path): return False +def _pid_is_alive(pid): + """Best-effort PID liveness check that is safe on Windows test hosts.""" + try: + pid = int(pid) + except (TypeError, ValueError): + return False + if pid <= 0: + return False + if os.name == "nt": + # os.kill(pid, 0) can terminate the target on Windows; fail closed. + return True + try: + os.kill(pid, 0) + return True + except OSError: + return False + + +def _folder_has_active_recording(folder_path): + """Execution-time guard before destructive folder cleanup. + + The WebUI check only runs when the task is enqueued; a recording can start + while the task waits in the queue. + """ + try: + names = os.listdir(folder_path) + except OSError: + return False + now = time.time() + for name in names: + if not name.endswith(".part"): + continue + try: + if now - os.path.getmtime(os.path.join(folder_path, name)) <= ACTIVE_RECORDING_FRESH_SEC: + return True + except OSError: + continue + for name in names: + if name != ".recording.lock" and not ( + name.startswith(".recording.") and name.endswith(".lock") + ): + continue + try: + with open(os.path.join(folder_path, name), "r", encoding="utf-8") as handle: + content = handle.read().strip() + except OSError: + continue + if _pid_is_alive(content): + return True + for name in names: + if name != ".current_recording.json" and not ( + name.startswith(".current_recording.") and name.endswith(".json") + ): + continue + try: + with open(os.path.join(folder_path, name), "r", encoding="utf-8") as handle: + data = json.load(handle) + except (OSError, ValueError): + continue + filename = os.path.basename(str((data or {}).get("filename") or "")) + if not filename: + continue + for candidate in ( + os.path.join(folder_path, filename), + os.path.join(folder_path, f"{filename}.part"), + ): + try: + if time.time() - os.path.getmtime(candidate) <= ACTIVE_RECORDING_FRESH_SEC: + return True + except OSError: + continue + return False + + +def _folder_has_active_remux(folder_path): + """True while a remux lock in this archive is still fresh.""" + try: + names = os.listdir(folder_path) + except OSError: + return False + now = time.time() + for name in names: + if not name.endswith(".remux.lock"): + continue + try: + if now - os.path.getmtime(os.path.join(folder_path, name)) <= REMUX_FILE_TIMEOUT_SEC + 300: + return True + except OSError: + continue + return False + + def process_empty_folder(payload): folder_path = payload.get("path") if not folder_path or not os.path.isdir(folder_path): @@ -800,6 +880,10 @@ def process_empty_folder(payload): if not _is_direct_child(config.BASE_DIR, folder_path): logging.error("Unsicherer empty_folder Pfad abgelehnt: %s", folder_path) return False + if _folder_has_active_recording(folder_path) or _folder_has_active_remux(folder_path): + raise TaskDeferred( + f"Ordner wird noch beschrieben: {folder_path}", delay_sec=120 + ) logging.info("Leere Ordnerinhalt: %s", folder_path) deleted = 0 @@ -825,6 +909,9 @@ def process_empty_folder(payload): ) def _keep_runtime_file(name): + # Preserve in-progress FFmpeg output even if the active check raced. + if name.endswith(".part"): + return True if name in keep_names: return True for prefix, suffix in keep_prefixes: @@ -885,6 +972,9 @@ def process_transcribe(payload): try: # Run the helper script so transcription runs in a separate process run_py = os.path.join(config.APP_DIR, 'run_transcribe.py') + if not os.path.isfile(run_py): + logging.error("Transcribe-Helfer fehlt: %s", run_py) + return False cmd = [sys.executable, run_py, media] if transcript: cmd.append(transcript) @@ -1252,6 +1342,31 @@ def storage_tick(): return storage_service.tick() +def _submit_claimed_task(pools, pool_name, task): + """Submit a claimed task, releasing its capacity slot if dispatch fails.""" + with _INFLIGHT_LOCK: + _INFLIGHT[pool_name] += 1 + try: + pools[pool_name].submit(_execute_claimed_task, task) + except Exception: + with _INFLIGHT_LOCK: + _INFLIGHT[pool_name] = max(0, _INFLIGHT[pool_name] - 1) + try: + database.defer_task( + task["id"], + "Worker dispatch failed", + delay_sec=30, + worker_id=WORKER_ID, + ) + except Exception as exc: + logging.error( + "Konnte Task %s nach Dispatch-Fehler nicht zurückstellen: %s", + task.get("id"), + exc, + ) + raise + + def _execute_claimed_task(task): """Run one claimed task in a pool thread; each DB op uses its own get_db().""" task_id = task["id"] @@ -1351,18 +1466,18 @@ def _safe_run(func): WORKER_ID, stale_after=TASK_STALE_SEC, allowed_types=["thumbnail"], + max_attempts_by_type=TASK_MAX_ATTEMPTS, ) if not task: task = database.claim_next_task( WORKER_ID, stale_after=TASK_STALE_SEC, allowed_types=allowed, + max_attempts_by_type=TASK_MAX_ATTEMPTS, ) if task: pool_name = _pool_name_for_task(task["type"]) - with _INFLIGHT_LOCK: - _INFLIGHT[pool_name] += 1 - pools[pool_name].submit(_execute_claimed_task, task) + _submit_claimed_task(pools, pool_name, task) else: time.sleep(2) except Exception as e: From fdf5fa0b5b2bea665d30b23e37af84f761238649 Mon Sep 17 00:00:00 2001 From: Pierre Date: Sat, 31 Oct 2026 23:41:54 +0000 Subject: [PATCH 05/31] feat(webui): health parser module, import-only guard, transcribe helper - core/health.py: dependency-free parse_supervisor_status(); /healthz now reports missing supervisor programs and no longer expects the removed "livebot" program (optional if present). - WEBUI/webui.py: STREAMVAULT_IMPORT_ONLY=1 skips import-time side effects (admin bootstrap, badge sync thread); a failing admin bootstrap is now logged instead of silently ignored. - run_transcribe.py: standalone Whisper runner used by the worker. It was referenced by worker.process_transcribe() but missing from the tree, so ASR tasks could not run. Heavy ML imports stay out of the long-lived worker process. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tzan3xULuUxadM3Azn5fzJ --- WEBUI/webui.py | 52 ++++++++++++++++++++++++++--------------------- core/health.py | 30 +++++++++++++++++++++++++++ run_transcribe.py | 34 +++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 23 deletions(-) create mode 100644 core/health.py create mode 100644 run_transcribe.py diff --git a/WEBUI/webui.py b/WEBUI/webui.py index 4abcdc2..afdfea2 100644 --- a/WEBUI/webui.py +++ b/WEBUI/webui.py @@ -62,6 +62,7 @@ from core import config from core import database from core import twitch_api +from core.health import parse_supervisor_status from core import twitch_eventsub from core import twitch_templates from core import user_traffic @@ -1955,7 +1956,7 @@ def collect_health_status(): status_ok = False checks[name] = {"ok": False, "error": str(exc)} - expected = {"webui", "recorder", "worker", "livebot"} + expected = {"webui", "recorder", "worker"} services = {name: {"state": "unknown"} for name in sorted(expected)} try: result = subprocess.run( @@ -1967,18 +1968,15 @@ def collect_health_status(): check=False, ) if result.returncode == 0: - for line in result.stdout.splitlines(): - parts = line.split() - if len(parts) < 2: - continue - program = parts[0].split(":", 1)[-1] - if program in services: - state = parts[1] - services[program] = {"state": state, "ok": state == "RUNNING"} - if state != "RUNNING": - status_ok = False - checks["services"] = {"ok": all(s.get("ok") for s in services.values()), "programs": services} - if not checks["services"]["ok"]: + services, missing, services_ok = parse_supervisor_status( + result.stdout, expected + ) + checks["services"] = {"ok": services_ok, "programs": services} + if missing: + checks["services"]["error"] = ( + "missing supervisor programs: " + ", ".join(missing) + ) + if not services_ok: status_ok = False else: checks["services"] = {"ok": None, "error": result.stderr.strip()[:300], "programs": services} @@ -12852,17 +12850,25 @@ def delete_folder_route(streamer): msg = "Ordner nicht gefunden oder ungültig." return redirect(url_for('index', channels_msg=msg)) -# Ensure baseline users exist (runs on import for Gunicorn) -try: - ensure_admin_user() -except Exception: - pass +# Ensure baseline users exist (runs on import for Gunicorn). The standalone +# transcription helper imports this module in a child process and must not +# mutate users or start background sync threads. +_IMPORT_ONLY = os.environ.get("STREAMVAULT_IMPORT_ONLY", "").lower() in ( + "1", + "true", + "yes", +) +if not _IMPORT_ONLY: + try: + ensure_admin_user() + except Exception: + logging.exception("Bootstrap: Admin-Benutzer konnte nicht sichergestellt werden") -# Keep a full local Twitch badge library (~a few MB) for 1:1 chat icons. -try: - maybe_start_twitch_badge_library_sync() -except Exception: - logging.exception("Could not start Twitch badge library sync") + # Keep a full local Twitch badge library (~a few MB) for 1:1 chat icons. + try: + maybe_start_twitch_badge_library_sync() + except Exception: + logging.exception("Could not start Twitch badge library sync") diff --git a/core/health.py b/core/health.py new file mode 100644 index 0000000..e908ce4 --- /dev/null +++ b/core/health.py @@ -0,0 +1,30 @@ +"""Dependency-free helpers for the WebUI health endpoint.""" + +from __future__ import annotations + + +def parse_supervisor_status(stdout, expected): + """Parse ``supervisorctl status`` output for the programs we supervise. + + Returns ``(services, missing, ok)``. Programs that are not configured in + supervisor are reported as missing only when they are in ``expected``; + an optional ``livebot`` program is included when present. + """ + expected_names = {str(name) for name in expected if name} + services = {name: {"state": "unknown"} for name in sorted(expected_names)} + seen = set() + for line in str(stdout or "").splitlines(): + parts = line.split() + if len(parts) < 2: + continue + program = parts[0].split(":", 1)[-1] + if program not in expected_names and program != "livebot": + continue + state = parts[1] + services[program] = {"state": state, "ok": state == "RUNNING"} + seen.add(program) + missing = sorted(expected_names - seen) + ok = not missing and all( + item.get("ok") is not False for item in services.values() + ) + return services, missing, ok diff --git a/run_transcribe.py b/run_transcribe.py new file mode 100644 index 0000000..303af85 --- /dev/null +++ b/run_transcribe.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Standalone transcription runner used by the background worker. + +Running transcription in a child process keeps the Whisper model out of the +long-lived worker. The implementation stays in ``WEBUI.webui``; importing it in +this helper sets ``STREAMVAULT_IMPORT_ONLY=1`` so import-time bootstrap side +effects (admin creation, badge sync) are skipped. +""" +from __future__ import annotations + +import os +import sys + + +def main(argv=None) -> int: + args = list(sys.argv[1:] if argv is None else argv) + if len(args) not in (1, 2): + print("Usage: run_transcribe.py [transcript]", file=sys.stderr) + return 2 + + root = os.path.dirname(os.path.abspath(__file__)) + if root not in sys.path: + sys.path.insert(0, root) + os.environ["STREAMVAULT_IMPORT_ONLY"] = "1" + + from WEBUI.webui import transcribe_media_file + + ok, message = transcribe_media_file(args[0], args[1] if len(args) == 2 else None) + print(message or ("Transkript erstellt" if ok else "Transkription fehlgeschlagen")) + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) From 568444347021f7e92e2561506d89fb50579dc95a Mon Sep 17 00:00:00 2001 From: Pierre Date: Sat, 31 Oct 2026 23:41:55 +0000 Subject: [PATCH 06/31] test(recorder): add reliability regression suite (33 tests) Covers lock ordering (stop before unlock, unlock on stop failure), bounded FFmpeg waits, disk-space guard on missing paths, shared-archive duplicate protection, worker folder-safety guards, dispatch capacity release, stale-task retry budget, supervisor status parsing, the run_transcribe helper / import-only guard, live-only monitor shutdown and network probe timeouts. fcntl is shimmed on Windows; the suite runs unchanged on Linux. Full suite: 126 tests (93 storage + 33 reliability), Linux 0 skipped. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tzan3xULuUxadM3Azn5fzJ --- tests/test_recorder_reliability.py | 504 +++++++++++++++++++++++++++++ 1 file changed, 504 insertions(+) create mode 100644 tests/test_recorder_reliability.py diff --git a/tests/test_recorder_reliability.py b/tests/test_recorder_reliability.py new file mode 100644 index 0000000..4cb283e --- /dev/null +++ b/tests/test_recorder_reliability.py @@ -0,0 +1,504 @@ +"""Reliability regressions for recorder, worker, and task-dispatch fixes. + +Runs with the standard library runner: + + python -m unittest discover -s tests -v + +The recorder is Linux-oriented (``fcntl``) and the worker imports numpy at +module level. Both are shimmed here so the tests also run on a Windows audit +host without changing production code. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import tempfile +import time +import types +import unittest +from pathlib import Path +from unittest import mock + +_INITIAL_BASE = tempfile.mkdtemp(prefix="streamvault-reliability-tests-") +os.environ["VOD_USER"] = "vaultadmin" +os.environ["VOD_PASS"] = "VaultTestPass123!" +os.environ["VOD_SECRET_KEY"] = "test-session-secret-key-0123456789" +os.environ["STORAGE_ENCRYPTION_KEY"] = "test-storage-encryption-key-0123456789" +os.environ["VOD_BASE_DIR"] = _INITIAL_BASE +os.environ["STREAMRECORDER_LOG_DIR"] = _INITIAL_BASE +os.environ["VOD_ALLOW_ENV_LOGIN"] = "1" +os.environ["REDIS_HOST"] = "127.0.0.1" +os.environ["REDIS_PORT"] = "1" +os.environ.pop("REDIS_URL", None) +os.environ.pop("VOD_RATELIMIT_STORAGE_URI", None) +# WEBUI.webui starts a network-bound badge sync daemon thread on import. It +# writes into whichever BASE_DIR is active and races TemporaryDirectory cleanup +# (intermittent 'Directory not empty' teardown errors), so keep it off in tests. +os.environ["TWITCH_BADGE_AUTO_SYNC"] = "0" + +if os.name == "nt" and "fcntl" not in sys.modules: + _fcntl = types.ModuleType("fcntl") + _fcntl.LOCK_EX = 2 + _fcntl.LOCK_NB = 4 + _fcntl.LOCK_UN = 8 + _fcntl.flock = lambda *args, **kwargs: None + sys.modules["fcntl"] = _fcntl + +try: + import numpy # noqa: F401 +except ImportError: + _numpy = types.ModuleType("numpy") + _numpy.ndarray = object + sys.modules["numpy"] = _numpy + +import WEBUI # noqa: F401,E402 (parent package for the helper import test) +import run_transcribe # noqa: E402 +from core import config, database # noqa: E402 +from core.health import parse_supervisor_status # noqa: E402 +from recorder import Streamrecorder as recorder # noqa: E402 +from recorder import worker # noqa: E402 + + +class IsolatedBaseTest(unittest.TestCase): + """Give every test its own base directory and patched config paths.""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory(prefix="streamvault-reliability-") + self.base = Path(self._tmp.name) + self.addCleanup(self._tmp.cleanup) + self._previous_base_env = os.environ.get("VOD_BASE_DIR") + os.environ["VOD_BASE_DIR"] = str(self.base) + self.addCleanup(self._restore_base_env) + self._patches = [ + mock.patch.object(config, "BASE_DIR", str(self.base)), + mock.patch.object(config, "DB_FILE", str(self.base / "streamrecorder.db")), + mock.patch.object(config, "FILES_DIR", str(self.base / "files")), + mock.patch.object(config, "HLS_ROOT", str(self.base / ".hls")), + mock.patch.object(config, "THUMB_ROOT", str(self.base / ".thumbs")), + mock.patch.object(config, "META_DIR", str(self.base / ".meta")), + mock.patch.object(config, "APP_DIR", str(self.base)), + ] + for patcher in self._patches: + patcher.start() + self.addCleanup(patcher.stop) + os.makedirs(config.META_DIR, exist_ok=True) + os.makedirs(config.FILES_DIR, exist_ok=True) + database.init_db() + + def _restore_base_env(self): + if self._previous_base_env is None: + os.environ.pop("VOD_BASE_DIR", None) + else: + os.environ["VOD_BASE_DIR"] = self._previous_base_env + + +class LiveOnlyMonitorTests(IsolatedBaseTest): + def test_check_interval_constant_is_defined(self): + self.assertIsInstance(recorder.CHECK_INTERVAL, int) + self.assertGreaterEqual(recorder.CHECK_INTERVAL, 1) + + def test_monitor_completes_iteration_without_name_error(self): + conf = {"record": False, "platform": "twitch"} + calls = {"count": 0} + + def fake_config(_channel): + calls["count"] += 1 + return conf if calls["count"] == 1 else None + + with mock.patch.object(recorder, "get_source_config", side_effect=fake_config), \ + mock.patch.object(recorder, "cheap_live_presence", return_value={"live": False}), \ + mock.patch.object(recorder, "CHECK_INTERVAL", 0), \ + mock.patch.object(recorder, "shutdown_requested", False): + recorder.monitor_stream_live_only("audit_channel", "audit_channel", conf) + + self.assertGreaterEqual(calls["count"], 2) + + def test_monitor_exits_immediately_during_shutdown(self): + with mock.patch.object(recorder, "shutdown_requested", True), \ + mock.patch.object(recorder, "get_source_config") as get_conf: + recorder.monitor_stream_live_only("audit_channel", "audit_channel", {"record": False}) + get_conf.assert_not_called() + + +class DuplicateCleanupTests(IsolatedBaseTest): + def _recordings(self, directory: Path): + first = directory / "shared_tw_2026-10-25_12-00-00.mp4" + second = directory / "shared_tw_2026-10-25_12-00-05.mp4" + for path in (first, second): + path.write_bytes(b"\0" * 8192) + os.utime(path, (time.time(), time.time())) + return first, second + + def test_shared_archive_is_not_pruned(self): + directory = self.base / "shared" + directory.mkdir() + first, second = self._recordings(directory) + with mock.patch.object( + recorder.database, + "channels_for_archive", + return_value=[{"name": "source_a"}, {"name": "source_b"}], + ): + recorder.cleanup_duplicate_recordings( + str(directory), str(first), "source_a", archive_channel="shared" + ) + self.assertTrue(first.exists()) + self.assertTrue(second.exists()) + + def test_single_source_archive_still_prunes(self): + directory = self.base / "single" + directory.mkdir() + first, second = self._recordings(directory) + with mock.patch.object( + recorder.database, + "channels_for_archive", + return_value=[{"name": "source_a"}], + ): + recorder.cleanup_duplicate_recordings( + str(directory), str(first), "source_a", archive_channel="shared" + ) + self.assertEqual(sum(path.exists() for path in (first, second)), 1) + + def test_lookup_failure_fails_closed(self): + directory = self.base / "unknown" + directory.mkdir() + first, second = self._recordings(directory) + with mock.patch.object( + recorder.database, + "channels_for_archive", + side_effect=RuntimeError("database unavailable"), + ): + recorder.cleanup_duplicate_recordings( + str(directory), str(first), "source_a", archive_channel="shared" + ) + self.assertTrue(first.exists()) + self.assertTrue(second.exists()) + + +class ChannelLockOrderingTests(unittest.TestCase): + def test_stop_happens_before_release(self): + events = [] + with mock.patch.object( + recorder, "stop_channel_process", side_effect=lambda *a, **k: events.append("stop") + ), mock.patch.object( + recorder, + "release_held_channel_lock", + side_effect=lambda *a, **k: events.append("release"), + ): + recorder.stop_channel_and_release_lock("/archive", "channel", object()) + self.assertEqual(events, ["stop", "release"]) + + def test_lock_is_released_when_stop_raises(self): + events = [] + with mock.patch.object( + recorder, + "stop_channel_process", + side_effect=RuntimeError("terminate failed"), + ), mock.patch.object( + recorder, + "release_held_channel_lock", + side_effect=lambda *a, **k: events.append("release"), + ): + with self.assertRaises(RuntimeError): + recorder.stop_channel_and_release_lock("/archive", "channel", object()) + self.assertEqual(events, ["release"]) + + +class BoundedProcessWaitTests(unittest.TestCase): + def test_normal_exit_does_not_kill(self): + class Process: + pid = 1 + + def __init__(self): + self.killed = False + + def wait(self, timeout=None): + return 0 + + def kill(self): + self.killed = True + + process = Process() + self.assertTrue(recorder.wait_for_process_exit(process, timeout=1)) + self.assertFalse(process.killed) + + def test_timeout_kills_and_reaps(self): + class Process: + pid = 2 + + def __init__(self): + self.killed = False + self.waits = 0 + + def wait(self, timeout=None): + self.waits += 1 + if self.waits == 1: + raise subprocess.TimeoutExpired(cmd="ffmpeg", timeout=timeout) + return 1 + + def kill(self): + self.killed = True + + process = Process() + self.assertFalse(recorder.wait_for_process_exit(process, timeout=1)) + self.assertTrue(process.killed) + self.assertEqual(process.waits, 2) + + +class WorkerFolderSafetyTests(IsolatedBaseTest): + def _folder(self): + directory = self.base / "channel" + directory.mkdir() + return directory + + def test_empty_folder_defers_while_part_file_exists(self): + directory = self._folder() + part = directory / "recording.mp4.part" + part.write_bytes(b"in progress") + with self.assertRaises(worker.TaskDeferred): + worker.process_empty_folder({"path": str(directory)}) + self.assertTrue(part.exists()) + + def test_empty_folder_preserves_part_if_guard_races(self): + directory = self._folder() + part = directory / "recording.mp4.part" + part.write_bytes(b"in progress") + other = directory / "old.mp4" + other.write_bytes(b"old") + with mock.patch.object(worker, "_folder_has_active_recording", return_value=False): + self.assertTrue(worker.process_empty_folder({"path": str(directory)})) + self.assertTrue(part.exists()) + self.assertFalse(other.exists()) + + def test_active_lock_file_blocks_cleanup(self): + directory = self._folder() + (directory / ".recording.channel.lock").write_text( + str(os.getpid()), encoding="utf-8" + ) + self.assertTrue(worker._folder_has_active_recording(str(directory))) + + def test_stale_part_file_does_not_block_forever(self): + directory = self._folder() + part = directory / "recording.mp4.part" + part.write_bytes(b"stale") + old = time.time() - (worker.ACTIVE_RECORDING_FRESH_SEC + 30) + os.utime(part, (old, old)) + self.assertFalse(worker._folder_has_active_recording(str(directory))) + + def test_transcribe_missing_helper_fails_clearly(self): + media = self.base / "media.mp4" + media.write_bytes(b"video") + missing_app = self.base / "missing-app" + with mock.patch.object(worker.config, "APP_DIR", str(missing_app)): + self.assertFalse(worker.process_transcribe({"media": str(media)})) + + def test_delete_folder_defers_while_recording_active(self): + directory = self._folder() + (directory / "recording.mp4.part").write_bytes(b"in progress") + with mock.patch.object(worker.subprocess, "run") as run: + with self.assertRaises(worker.TaskDeferred): + worker.process_delete_folder( + {"path": str(directory), "streamer": "channel"} + ) + run.assert_not_called() + + def test_delete_folder_never_invokes_fuser(self): + directory = self._folder() + (directory / "old.mp4").write_bytes(b"old") + with mock.patch.object(worker.time, "sleep", return_value=None), \ + mock.patch.object(worker.subprocess, "run") as run: + self.assertTrue( + worker.process_delete_folder( + {"path": str(directory), "streamer": "channel"} + ) + ) + run.assert_not_called() + self.assertFalse(directory.exists()) + + +class WorkerDispatchTests(unittest.TestCase): + def test_dispatch_failure_releases_capacity_and_requeues(self): + class ClosedPool: + def submit(self, *args, **kwargs): + raise RuntimeError("executor is shut down") + + with mock.patch.dict(worker._INFLIGHT, {"io": 0}, clear=False): + with mock.patch.object(worker.database, "defer_task") as defer: + with self.assertRaises(RuntimeError): + worker._submit_claimed_task( + {"io": ClosedPool()}, + "io", + {"id": 17, "type": "thumbnail"}, + ) + self.assertEqual(worker._INFLIGHT["io"], 0) + defer.assert_called_once() + + +class TaskRequeueTests(IsolatedBaseTest): + def test_stale_task_over_attempt_limit_is_failed(self): + task_id = database.add_task("thumbnail", {"input_path": "ignored"}) + claimed = database.claim_next_task( + "worker-a", + stale_after=60, + allowed_types=["thumbnail"], + max_attempts_by_type={"thumbnail": 3}, + ) + self.assertIsNotNone(claimed) + conn = database.get_db() + try: + conn.execute( + "UPDATE tasks SET attempts=3, locked_at=? WHERE id=?", + (time.time() - 3600, task_id), + ) + conn.commit() + finally: + conn.close() + + result = database.claim_next_task( + "worker-b", + stale_after=60, + allowed_types=["thumbnail"], + max_attempts_by_type={"thumbnail": 3}, + ) + self.assertIsNone(result) + + conn = database.get_db() + try: + row = conn.execute( + "SELECT status, error FROM tasks WHERE id=?", (task_id,) + ).fetchone() + finally: + conn.close() + self.assertEqual(row["status"], "failed") + self.assertIn("max attempts", row["error"]) + + +class RunTranscribeTests(unittest.TestCase): + def test_usage_error(self): + self.assertEqual(run_transcribe.main([]), 2) + + def test_delegates_to_existing_webui_implementation(self): + fake_module = types.ModuleType("WEBUI.webui") + fake_module.transcribe_media_file = mock.Mock(return_value=(True, "fertig")) + with mock.patch.dict(sys.modules, {"WEBUI.webui": fake_module}): + self.assertEqual( + run_transcribe.main(["media.mp4", "transcript.txt"]), + 0, + ) + fake_module.transcribe_media_file.assert_called_once_with( + "media.mp4", "transcript.txt" + ) + self.assertEqual(os.environ.get("STREAMVAULT_IMPORT_ONLY"), "1") + + +class HealthStatusTests(unittest.TestCase): + EXPECTED = {"webui", "recorder", "worker"} + + def test_all_expected_programs_running(self): + output = ( + "webui:webui RUNNING pid 1, uptime 1:00:00\n" + "recorder:recorder RUNNING pid 2, uptime 1:00:00\n" + "worker:worker RUNNING pid 3, uptime 1:00:00\n" + ) + services, missing, ok = parse_supervisor_status(output, self.EXPECTED) + self.assertTrue(ok) + self.assertEqual(missing, []) + self.assertEqual(set(services), self.EXPECTED) + + def test_missing_worker_is_degraded(self): + output = ( + "webui:webui RUNNING pid 1, uptime 1:00:00\n" + "recorder:recorder RUNNING pid 2, uptime 1:00:00\n" + ) + services, missing, ok = parse_supervisor_status(output, self.EXPECTED) + self.assertFalse(ok) + self.assertEqual(missing, ["worker"]) + + def test_optional_livebot_is_included_when_present(self): + output = ( + "webui:webui RUNNING pid 1, uptime 1:00:00\n" + "recorder:recorder RUNNING pid 2, uptime 1:00:00\n" + "worker:worker RUNNING pid 3, uptime 1:00:00\n" + "livebot:livebot RUNNING pid 4, uptime 1:00:00\n" + ) + services, missing, ok = parse_supervisor_status(output, self.EXPECTED) + self.assertTrue(ok) + self.assertEqual(missing, []) + self.assertIn("livebot", services) + + def test_stopped_program_is_degraded(self): + output = ( + "webui:webui RUNNING pid 1, uptime 1:00:00\n" + "recorder:recorder RUNNING pid 2, uptime 1:00:00\n" + "worker:worker STOPPED Not started\n" + ) + _services, missing, ok = parse_supervisor_status(output, self.EXPECTED) + self.assertFalse(ok) + self.assertEqual(missing, []) + + +class DiskSpaceGuardTests(IsolatedBaseTest): + def test_disk_blocks_recording_on_nonexistent_dir_when_disk_full(self): + from collections import namedtuple + Usage = namedtuple("Usage", ["total", "used", "free"]) + fake_usage = Usage(total=100 * (1024 ** 3), used=98 * (1024 ** 3), free=2 * (1024 ** 3)) + nonexistent = self.base / "does_not_exist_yet" / "streamer" + with mock.patch("shutil.disk_usage", return_value=fake_usage): + self.assertTrue(recorder.disk_blocks_recording(str(nonexistent))) + + def test_disk_allows_recording_on_nonexistent_dir_when_space_plentiful(self): + from collections import namedtuple + Usage = namedtuple("Usage", ["total", "used", "free"]) + fake_usage = Usage(total=100 * (1024 ** 3), used=20 * (1024 ** 3), free=80 * (1024 ** 3)) + nonexistent = self.base / "does_not_exist_yet" / "streamer" + with mock.patch("shutil.disk_usage", return_value=fake_usage): + self.assertFalse(recorder.disk_blocks_recording(str(nonexistent))) + + def test_disk_pause_flag_blocks_immediately(self): + with mock.patch.object(recorder, "_storage_pause_flag_active", return_value=True): + self.assertTrue(recorder.disk_blocks_recording(str(self.base))) + + +class NetworkProbeTimeoutTests(unittest.TestCase): + def test_streamlink_probe_timeout_returns_error_dict(self): + with mock.patch.object(recorder, "find_streamlink", return_value="streamlink"), \ + mock.patch("subprocess.run", side_effect=subprocess.TimeoutExpired(cmd="streamlink", timeout=10)): + res = recorder.probe_streamlink_playable("testchannel", {"platform": "twitch"}) + self.assertFalse(res["ok"]) + self.assertIn("Timeout", res["error"]) + + def test_ytdlp_probe_timeout_returns_error_dict(self): + with mock.patch("subprocess.run", side_effect=subprocess.TimeoutExpired(cmd="yt-dlp", timeout=10)): + res = recorder.probe_ytdlp_playable("testchannel", {"platform": "youtube"}) + self.assertFalse(res["ok"]) + self.assertIn("Timeout", res["error"]) + + def test_gql_timeout_returns_error_dict(self): + import urllib.error + with mock.patch("urllib.request.urlopen", side_effect=urllib.error.URLError("timed out")): + res = recorder.get_twitch_stream_status("testchannel") + self.assertFalse(res["ok"]) + self.assertIn("timed out", res["error"]) + + +class ShutdownControlTests(unittest.TestCase): + def test_interruptible_sleep_exits_early_on_shutdown(self): + start = time.time() + with mock.patch.object(recorder, "shutdown_requested", True): + recorder.interruptible_sleep(10) + elapsed = time.time() - start + self.assertLess(elapsed, 1.0) + + def test_request_shutdown_sets_flag(self): + original = recorder.shutdown_requested + try: + recorder.shutdown_requested = False + recorder.request_shutdown() + self.assertTrue(recorder.shutdown_requested) + finally: + recorder.shutdown_requested = original + + +if __name__ == "__main__": + unittest.main() From f331430c00532044b64ca899dba8a2948b2351cf Mon Sep 17 00:00:00 2001 From: Pierre Date: Sat, 31 Oct 2026 23:41:55 +0000 Subject: [PATCH 07/31] docs(deploy): document VOD_SECURE_COOKIES=false for HTTP-only Nginx install.sh and INSTALL.md configure Nginx on plain HTTP port 80. Secure-only session cookies are never sent back over HTTP, which makes the login redirect loop. Write VOD_SECURE_COOKIES=false into the generated env file and document switching it to true after enabling HTTPS. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tzan3xULuUxadM3Azn5fzJ --- INSTALL.md | 3 +++ install.sh | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/INSTALL.md b/INSTALL.md index 88b512d..5800585 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -124,6 +124,9 @@ LIVE_HLS_DIR=/dev/shm/streamvault_hls VOD_USER=vaultadmin VOD_PASS= VOD_SECRET_KEY= +# The bundled Nginx site listens on HTTP port 80. Keep secure-only cookies +# disabled until HTTPS is enabled, otherwise login redirects loop. +VOD_SECURE_COOKIES=false VOD_USE_X_ACCEL=true # Optional dedicated key for cloud credentials (falls back to VOD_SECRET_KEY). # STORAGE_ENCRYPTION_KEY= diff --git a/install.sh b/install.sh index 1b6ef95..4fe2343 100644 --- a/install.sh +++ b/install.sh @@ -112,6 +112,10 @@ LIVE_HLS_DIR=${RAMDISK_DIR} VOD_USER=vaultadmin VOD_PASS=${INITIAL_PASS} VOD_SECRET_KEY=${SECRET_KEY} +# Nginx in this installer terminates plain HTTP on port 80. Secure-only +# cookies would make login fail until TLS is configured; set this to true +# after enabling HTTPS. +VOD_SECURE_COOKIES=false VOD_USE_X_ACCEL=true EOF chown root:"${SERVICE_USER}" "${ENV_FILE}" From 233087183a4ae40bb1580523f27770c03fae412d Mon Sep 17 00:00:00 2001 From: Pierre Date: Sat, 31 Oct 2026 23:41:56 +0000 Subject: [PATCH 08/31] feat(scripts): add VPS deployment and Linux validation script scripts/streamvault_vps.sh (validate | deploy | rollback) checks what the Windows audit host cannot exercise: - runtime: venv Python >= 3.12, compile gate, import smoke test with STREAMVAULT_IMPORT_ONLY, optional unit tests as the service user (asserts the Linux-only symlink test runs instead of skipping) - filesystem: flock semantics, network-FS detection for SQLite WAL, archive symlinks (worker cleanup is fail-closed), www-data traversal along resolved paths, service UMask, disk-pause thresholds - supervisor: supervisorctl status parsed via core.health (same code as /healthz), socket access for the service user, recorder stopwaitsecs vs. FFmpeg finalize; systemd fallback with PrivateDevices and restart-loop checks - Nginx X-Accel: internal flag and alias target of every /_protected* location, disable_symlinks, cookie/TLS consistency, direct-access 404, plus an end-to-end round trip on a throwaway Nginx instance (production config is not modified) - HLS RAM disk: LIVE_HLS_DIR set, tmpfs, ownership, read/write permissions, free space, tmpfiles.d persistence (--fix) deploy snapshots the app, refuses to cut active recordings without --force, gates on compile + tests before restarting services and rolls back automatically on failure. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tzan3xULuUxadM3Azn5fzJ --- scripts/streamvault_vps.sh | 908 +++++++++++++++++++++++++++++++++++++ 1 file changed, 908 insertions(+) create mode 100755 scripts/streamvault_vps.sh diff --git a/scripts/streamvault_vps.sh b/scripts/streamvault_vps.sh new file mode 100755 index 0000000..6d218b9 --- /dev/null +++ b/scripts/streamvault_vps.sh @@ -0,0 +1,908 @@ +#!/usr/bin/env bash +# ============================================================================== +# StreamVault - VPS Deployment & Validation (Debian/Ubuntu, Venv-Python >= 3.12) +# +# streamvault_vps.sh validate [--run-tests] [--no-e2e] [--fix] [--strict] +# streamvault_vps.sh deploy (--source DIR | --ref GITREF) [--force] +# [--skip-tests] [--prune] +# streamvault_vps.sh rollback +# +# Validates the Linux-only parts that the Windows audit host cannot exercise: +# * Python runtime, compile gate, import smoke test, unit tests on Linux +# * Symlink handling (archive symlinks, Nginx disable_symlinks, www-data +# traversal along the resolved path, Linux-only symlink unit test) +# * flock semantics + filesystem type for SQLite WAL +# * Process supervisor (supervisorctl status via core.health, or systemd) +# * Nginx X-Accel-Redirect (internal locations, alias targets, isolated +# end-to-end test on a throwaway Nginx instance - production config is +# never modified) +# * HLS RAM disk (tmpfs, ownership, permissions, reboot persistence, +# systemd sandboxing that would hide /dev/shm from Nginx) +# +# Paths default to the layout created by install.sh and can be overridden via +# SV_INSTALL_DIR, SV_APP_DIR, SV_VENV_DIR, SV_ENV_FILE, SV_SERVICE_USER, +# SV_NGINX_USER, SV_BACKUP_DIR. +# +# Exit codes: 0 = no FAIL (and no WARN with --strict), 1 = FAIL, 2 = usage. +# ============================================================================== +set -Eeuo pipefail + +INSTALL_DIR="${SV_INSTALL_DIR:-/opt/streamvault}" +APP_DIR="${SV_APP_DIR:-${INSTALL_DIR}/app}" +VENV_DIR="${SV_VENV_DIR:-${INSTALL_DIR}/venv}" +ENV_FILE="${SV_ENV_FILE:-/etc/streamvault/env}" +SERVICE_USER="${SV_SERVICE_USER:-streamvault}" +NGINX_USER="${SV_NGINX_USER:-www-data}" +BACKUP_DIR="${SV_BACKUP_DIR:-/var/backups/streamvault}" +PY="${VENV_DIR}/bin/python3" +SERVICES=(webui recorder worker) +MIN_PY_MINOR=12 # WEBUI/webui.py uses PEP 701 f-strings (Python >= 3.12) + +RUN_TESTS=0; E2E=1; FIX=0; STRICT=0; FORCE=0; SKIP_TESTS=0; PRUNE=0 +SOURCE_DIR=""; GIT_REF="" + +if [[ -t 1 ]]; then + C_RED=$'\033[0;31m'; C_GRN=$'\033[0;32m'; C_YLW=$'\033[1;33m' + C_BLU=$'\033[0;34m'; C_BLD=$'\033[1m'; C_OFF=$'\033[0m' +else + C_RED=""; C_GRN=""; C_YLW=""; C_BLU=""; C_BLD=""; C_OFF="" +fi + +N_PASS=0; N_WARN=0; N_FAIL=0 +pass() { N_PASS=$((N_PASS + 1)); printf ' %s[PASS]%s %s\n' "$C_GRN" "$C_OFF" "$*"; } +warn() { N_WARN=$((N_WARN + 1)); printf ' %s[WARN]%s %s\n' "$C_YLW" "$C_OFF" "$*"; } +fail() { N_FAIL=$((N_FAIL + 1)); printf ' %s[FAIL]%s %s\n' "$C_RED" "$C_OFF" "$*"; } +info() { printf ' [INFO] %s\n' "$*"; } +section() { printf '\n%s== %s ==%s\n' "$C_BLU$C_BLD" "$*" "$C_OFF"; } +die() { printf '%s[ERROR]%s %s\n' "$C_RED" "$C_OFF" "$*" >&2; exit 1; } + +# Scratch state, cleaned on every exit path. +TMP_ROOT="" +E2E_DIR="" +PROBE="" +PROBE_DIRS=() +BG_PIDS=() +cleanup() { + local pid dir + for pid in "${BG_PIDS[@]:-}"; do + [[ -n "$pid" ]] && kill "$pid" 2>/dev/null || true + done + for dir in "${PROBE_DIRS[@]:-}"; do + [[ -n "$dir" && -d "$dir" && "$dir" == *"/.sv_validate_"* ]] && rm -rf -- "$dir" + done + [[ -n "$TMP_ROOT" && -d "$TMP_ROOT" ]] && rm -rf -- "$TMP_ROOT" + [[ -n "$E2E_DIR" && -d "$E2E_DIR" ]] && rm -rf -- "$E2E_DIR" + return 0 +} +trap cleanup EXIT + +as_service() { runuser -u "$SERVICE_USER" -- "$@"; } +as_nginx() { runuser -u "$NGINX_USER" -- "$@"; } +have() { command -v "$1" >/dev/null 2>&1; } + +# ------------------------------------------------------------------------------ +# Environment file: parsed (never sourced) with systemd EnvironmentFile rules. +# ------------------------------------------------------------------------------ +env_get() { + local key="$1" default="${2:-}" line value + if [[ -r "$ENV_FILE" ]]; then + line="$(grep -E "^[[:space:]]*(export[[:space:]]+)?${key}=" "$ENV_FILE" | tail -n 1 || true)" + if [[ -n "$line" ]]; then + value="${line#*=}" + value="${value%$'\r'}" + if [[ "$value" =~ ^\"(.*)\"$ || "$value" =~ ^\'(.*)\'$ ]]; then + value="${BASH_REMATCH[1]}" + fi + printf '%s' "$value" + return 0 + fi + fi + printf '%s' "$default" +} +env_flag() { + case "$(env_get "$1" "${2:-false}" | tr '[:upper:]' '[:lower:]')" in + 1|true|yes|on) return 0 ;; + *) return 1 ;; + esac +} + +# Run Python from the app venv as the service user with the production +# environment loaded the same way systemd/supervisor would. +PY_PRELUDE='import os, sys +path = os.environ.get("SV_ENV_FILE") +try: + with open(path, encoding="utf-8") as fh: + for raw in fh: + line = raw.strip() + if not line or line.startswith("#") or "=" not in line: + continue + if line.startswith("export "): + line = line[len("export "):] + key, value = line.split("=", 1) + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"\x27": + value = value[1:-1] + os.environ.setdefault(key.strip(), value) +except OSError as exc: + print(f"env file not readable as service user: {exc}", file=sys.stderr) +' +py_app() { + local code="$1"; shift + as_service timeout "${PY_TIMEOUT:-90}" env -C "$APP_DIR" \ + PYTHONPATH="${APP_DIR}:${APP_DIR}/WEBUI" PYTHONDONTWRITEBYTECODE=1 \ + SV_ENV_FILE="$ENV_FILE" "$PY" -c "${PY_PRELUDE}${code}" "$@" +} + +load_paths() { + BASE_DIR="$(env_get VOD_BASE_DIR "$APP_DIR")" + FILES_DIR="$(env_get VOD_FILES_DIR "${BASE_DIR}/files")" + HLS_ROOT="$(env_get VOD_HLS_DIR "${BASE_DIR}/.hls")" + THUMB_ROOT="$(env_get VOD_THUMB_DIR "${BASE_DIR}/.thumbs")" + VODCACHE_DIR="${BASE_DIR}/.vodcache" + LIVE_HLS_DIR="$(env_get LIVE_HLS_DIR "")" + PAUSE_PCT="$(env_get STREAM_RECORDER_STORAGE_PAUSE_PCT "$(env_get STORAGE_ALERT_THRESHOLD_PCT 90)")" + PAUSE_FREE_GB="$(env_get STREAM_RECORDER_STORAGE_PAUSE_FREE_GB "$(env_get STORAGE_ALERT_THRESHOLD_FREE_GB 20)")" +} + +# Supervisor flavour: "supervisor", "systemd" or "none". +detect_supervisor() { + # supervisorctl exits 3 when some programs are not RUNNING - still a supervisor host. + local rc=0 + if have supervisorctl; then + supervisorctl status >/dev/null 2>&1 || rc=$? + if [[ $rc -eq 0 || $rc -eq 3 ]]; then echo supervisor; return; fi + fi + if have systemctl && systemctl list-unit-files 'streamvault-*.service' --no-legend 2>/dev/null | grep -q streamvault; then + echo systemd; return + fi + echo none +} + +make_probe_dir() { + # $1 = parent directory. Creates it as the service user (inherits its umask) + # and returns the path in $PROBE. Must not run in a subshell: the cleanup + # trap needs the registration in PROBE_DIRS. + PROBE="${1%/}/.sv_validate_$$" + as_service mkdir -p "$PROBE" || return 1 + PROBE_DIRS+=("$PROBE") +} + +# ============================================================================== +# Validation checks +# ============================================================================== +check_system() { + section "System & Python-Laufzeit" + if [[ -r /etc/os-release ]]; then + # shellcheck disable=SC1091 + . /etc/os-release + case "${ID:-}" in + debian|ubuntu) pass "OS: ${PRETTY_NAME:-$ID}" ;; + *) warn "OS ${PRETTY_NAME:-unbekannt} ist kein getestetes Ziel (Debian/Ubuntu)" ;; + esac + fi + local cmd + for cmd in python3 ffmpeg ffprobe nginx flock findmnt runuser curl; do + if have "$cmd"; then pass "Befehl vorhanden: $cmd"; else fail "Befehl fehlt: $cmd"; fi + done + for cmd in sqlite3 rsync namei; do + have "$cmd" || warn "Optionaler Befehl fehlt: $cmd (Diagnose/deploy)" + done + have fuser && info "fuser ist installiert - der Worker nutzt es bewusst nicht mehr (kein 'fuser -k -9')." + + id "$SERVICE_USER" >/dev/null 2>&1 && pass "Service-User '$SERVICE_USER' existiert" \ + || fail "Service-User '$SERVICE_USER' fehlt" + id "$NGINX_USER" >/dev/null 2>&1 && pass "Nginx-User '$NGINX_USER' existiert" \ + || fail "Nginx-User '$NGINX_USER' fehlt (SV_NGINX_USER setzen)" + + if [[ ! -x "$PY" ]]; then + fail "Venv-Python fehlt: $PY" + return + fi + local ver + ver="$("$PY" -c 'import sys; print("%d.%d.%d" % sys.version_info[:3])')" + if "$PY" -c "import sys; sys.exit(0 if sys.version_info >= (3, ${MIN_PY_MINOR}) else 1)"; then + pass "Venv-Python $ver (>= 3.${MIN_PY_MINOR})" + else + fail "Venv-Python $ver < 3.${MIN_PY_MINOR}: WEBUI/webui.py nutzt PEP-701-f-Strings und laesst sich nicht kompilieren (Ubuntu 24.04+/Debian 13 oder Python 3.12 via deadsnakes/pyenv verwenden)" + fi + + # Compile gate without writing __pycache__ into the deployed tree. + local out + if out="$(as_service "$PY" - "$APP_DIR" 2>&1 <<'PYEOF' +import os, sys +root = sys.argv[1] +bad, count = [], 0 +for r, dirs, files in os.walk(root): + dirs[:] = [d for d in dirs if d not in (".git", ".venv", "venv", "__pycache__", "node_modules")] + for f in files: + if f.endswith(".py"): + p = os.path.join(r, f) + count += 1 + try: + with open(p, "rb") as fh: + compile(fh.read(), p, "exec") + except SyntaxError as exc: + bad.append(f"{os.path.relpath(p, root)}:{exc.lineno}: {exc.msg}") +print(count) +for b in bad: + print(b) +sys.exit(1 if bad else 0) +PYEOF +)"; then + pass "Syntax-Gate: $(head -n1 <<<"$out") Python-Dateien kompilieren" + else + fail "Syntax-Gate fehlgeschlagen:"; tail -n +2 <<<"$out" | sed 's/^/ /' + fi + + local tool + for tool in gunicorn streamlink yt-dlp; do + if [[ -x "${VENV_DIR}/bin/${tool}" ]]; then + pass "Venv-CLI vorhanden: ${tool}" + else + fail "Venv-CLI fehlt: ${VENV_DIR}/bin/${tool} (find_ytdlp/find_streamlink suchen neben sys.executable)" + fi + done + + # Import smoke test: STREAMVAULT_IMPORT_ONLY must keep webui side-effect free. + if out="$(py_app ' +os.environ["STREAMVAULT_IMPORT_ONLY"] = "1" +import threading +import core.health, core.database +from recorder import worker +from WEBUI import webui +bad = [t.name for t in threading.enumerate() if t.name == "twitch-badge-sync"] +print("badge-sync-thread" if bad else "no-side-effect-threads") +' 2>&1)"; then + if grep -q "no-side-effect-threads" <<<"$out"; then + pass "Import-Smoke-Test (core, worker, WEBUI.webui mit STREAMVAULT_IMPORT_ONLY=1) ohne Badge-Sync-Thread" + else + fail "WEBUI.webui startet trotz STREAMVAULT_IMPORT_ONLY den Badge-Sync-Thread" + fi + else + fail "Import-Smoke-Test fehlgeschlagen:"; tail -n 8 <<<"$out" | sed 's/^/ /' + fi +} + +check_tests() { + section "Unit-Tests auf Linux" + if [[ ! -d "${APP_DIR}/tests" ]]; then + warn "Kein tests/-Verzeichnis im Deployment (${APP_DIR}/tests) - Test-Gate uebersprungen" + return + fi + TMP_ROOT="${TMP_ROOT:-$(mktemp -d)}" + chmod 755 "$TMP_ROOT" + local log="${TMP_ROOT}/unittest.log" rc=0 + as_service env -C "$APP_DIR" HOME="$TMP_ROOT" PYTHONPATH="$APP_DIR" PYTHONDONTWRITEBYTECODE=1 \ + TWITCH_BADGE_AUTO_SYNC=0 "$PY" -m unittest discover -s tests -p 'test_*.py' -v >"$log" 2>&1 || rc=$? + local ran skipped + ran="$(grep -Eo '^Ran [0-9]+ tests' "$log" | grep -Eo '[0-9]+' || echo 0)" + skipped="$(grep -Eo 'skipped=[0-9]+' "$log" | grep -Eo '[0-9]+' || echo 0)" + if [[ $rc -eq 0 ]]; then + pass "unittest: ${ran} Tests OK (skipped=${skipped})" + else + fail "unittest fehlgeschlagen (rc=$rc, ${ran} Tests):" + { grep -E '^(ERROR|FAIL):' "$log" || tail -n 15 "$log"; } | head -n 15 | sed 's/^/ /' + fi + if grep -E 'test_collect_media_files_skips_symlinks_and_vanishing_files .* \.\.\. ok' "$log" >/dev/null; then + pass "Linux-Symlink-Test (auf Windows uebersprungen) laeuft und besteht" + elif grep -E 'test_collect_media_files_skips_symlinks_and_vanishing_files .* skipped' "$log" >/dev/null; then + fail "Symlink-Test wurde auch auf Linux uebersprungen" + else + warn "Symlink-Test nicht im Testlauf gefunden" + fi +} + +check_filesystem() { + section "Dateisystem, Symlinks & Locking" + if [[ ! -d "$BASE_DIR" ]]; then + fail "VOD_BASE_DIR existiert nicht: $BASE_DIR" + return + fi + local real fstype + real="$(readlink -f "$BASE_DIR")" + if [[ -L "$BASE_DIR" ]]; then + info "VOD_BASE_DIR ist ein Symlink: $BASE_DIR -> $real (realpath-Pruefungen nutzen das Ziel)" + fi + fstype="$(findmnt -n -o FSTYPE -T "$real" 2>/dev/null | tail -n 1)"; fstype="${fstype:-unknown}" + case "$fstype" in + nfs*|cifs|smb*|fuse.sshfs|fuse.rclone|9p) + fail "VOD_BASE_DIR liegt auf '$fstype': SQLite-WAL und flock sind dort nicht zuverlaessig" ;; + unknown) warn "Dateisystemtyp von $real nicht ermittelbar" ;; + *) pass "VOD_BASE_DIR auf lokalem Dateisystem ($fstype): $real" ;; + esac + + # flock: exclusive lock must block a second non-blocking locker. + local probe lockf holder + if make_probe_dir "$BASE_DIR"; then + probe="$PROBE"; lockf="${probe}/channel.lock" + as_service touch "$lockf" + as_service flock -x "$lockf" sleep 4 & + holder=$!; BG_PIDS+=("$holder") + sleep 1 + if as_service flock -n -x "$lockf" true; then + fail "flock ist auf $fstype wirkungslos - doppelte Recorder pro Kanal moeglich" + else + pass "flock-Semantik funktioniert (zweiter Locker wird blockiert)" + fi + wait "$holder" 2>/dev/null || true + else + fail "Service-User kann in $BASE_DIR nicht schreiben" + fi + + # Archive-level symlinks: worker cleanup is fail-closed for them. + local link target n_links=0 + while IFS= read -r -d '' link; do + n_links=$((n_links + 1)) + target="$(readlink -f "$link" || true)" + if [[ -z "$target" || ! -e "$target" ]]; then + fail "Verwaister Archiv-Symlink: $link -> $(readlink "$link")" + continue + fi + warn "Archiv-Symlink $link -> $target: delete_folder/empty_folder werden verweigert (_is_direct_child via realpath, fail-closed)" + if as_nginx test -r "$target" -a -x "$target"; then + pass "Nginx-User kann Symlink-Ziel lesen: $target" + else + fail "Nginx-User kann Symlink-Ziel nicht lesen: $target"; namei -l "$target" 2>/dev/null | sed 's/^/ /' || true + fi + done < <(find "$BASE_DIR" -mindepth 1 -maxdepth 1 -type l -print0 2>/dev/null) + [[ $n_links -eq 0 ]] && pass "Keine Archiv-Symlinks direkt unter VOD_BASE_DIR" + local file_links + file_links="$(find "$real" -mindepth 2 -maxdepth 2 -type l 2>/dev/null | wc -l)" + [[ "$file_links" -gt 0 ]] && info "${file_links} Datei-Symlinks in Archiven (Cloud-Upload ueberspringt Symlinks)" + SYMLINKS_PRESENT=$(( n_links + file_links )) + + # Nginx must traverse every directory on the resolved path. + local d + for d in "$real" "$HLS_ROOT" "$THUMB_ROOT"; do + [[ -d "$d" ]] || { info "Verzeichnis existiert (noch) nicht: $d"; continue; } + if as_nginx test -x "$d" -a -r "$d"; then + pass "Nginx-User kann traversieren/lesen: $d" + else + fail "Nginx-User kann $d nicht traversieren:"; namei -l "$(readlink -f "$d")" 2>/dev/null | sed 's/^/ /' || true + fi + done + + # Files created by the services must be readable for X-Accel. + local umask_val unit + if [[ "$(detect_supervisor)" == systemd ]]; then + for unit in recorder worker webui; do + umask_val="$(systemctl show -p UMask --value "streamvault-${unit}.service" 2>/dev/null || true)" + case "$umask_val" in + 0022|0002|"") pass "streamvault-${unit}: UMask=${umask_val:-default}" ;; + *) fail "streamvault-${unit}: UMask=${umask_val} - neue Aufnahmen sind fuer ${NGINX_USER} unlesbar" ;; + esac + done + fi + local sample + sample="$(find "$real" -mindepth 2 -maxdepth 2 -type f \( -name '*.mp4' -o -name '*.mkv' -o -name '*.ts' \) -print -quit 2>/dev/null || true)" + if [[ -n "$sample" ]]; then + if as_nginx head -c 1 "$sample" >/dev/null 2>&1; then + pass "Bestehende Aufnahme fuer ${NGINX_USER} lesbar: ${sample#"$real"/}" + else + fail "Bestehende Aufnahme fuer ${NGINX_USER} NICHT lesbar: $sample" + fi + fi + + # Disk thresholds used by disk_blocks_recording(). + local used_pct free_gb + used_pct="$(df -P "$real" | awk 'NR==2 {gsub("%","",$5); print $5}')" + free_gb="$(df -P -k "$real" | awk 'NR==2 {printf "%.1f", $4/1048576}')" + if awk -v u="$used_pct" -v f="$free_gb" -v p="$PAUSE_PCT" -v g="$PAUSE_FREE_GB" 'BEGIN {exit !(u >= p || f <= g)}'; then + fail "Speicher: ${used_pct}% belegt, ${free_gb} GB frei - Recorder pausiert neue Aufnahmen (Schwellen ${PAUSE_PCT}% / ${PAUSE_FREE_GB} GB)" + else + pass "Speicher: ${used_pct}% belegt, ${free_gb} GB frei (Pause ab ${PAUSE_PCT}% oder <= ${PAUSE_FREE_GB} GB)" + fi +} + +check_supervisor() { + section "Prozess-Supervisor" + local flavour; flavour="$(detect_supervisor)" + case "$flavour" in + supervisor) + local out rc=0 parsed + out="$(supervisorctl status 2>&1)" || rc=$? + printf '%s\n' "$out" | sed 's/^/ /' + parsed="$(py_app ' +import json +from core.health import parse_supervisor_status +services, missing, ok = parse_supervisor_status(sys.stdin.read(), {"webui", "recorder", "worker"}) +print(json.dumps({"services": services, "missing": missing, "ok": ok})) +' <<<"$out" 2>/dev/null || true)" + if [[ -z "$parsed" ]]; then + fail "core.health.parse_supervisor_status nicht ausfuehrbar" + else + local name state + for name in "${SERVICES[@]}"; do + state="$("$PY" -c 'import json,sys; d=json.loads(sys.argv[1]); print(d["services"].get(sys.argv[2],{}).get("state","unknown"))' "$parsed" "$name")" + if [[ "$state" == RUNNING ]]; then pass "supervisor: $name RUNNING"; else fail "supervisor: $name $state"; fi + done + fi + # /healthz calls supervisorctl as the WebUI process user. + if as_service supervisorctl status >/dev/null 2>&1 || [[ $? -eq 3 ]]; then + pass "supervisorctl als '$SERVICE_USER' nutzbar (/healthz sieht Dienststatus)" + else + warn "supervisorctl als '$SERVICE_USER' nicht nutzbar (Socket-Rechte) - /healthz meldet services.ok=null, falls die WebUI nicht als root laeuft" + fi + # Supervisor default stopwaitsecs=10 kills the recorder before the 30 s FFmpeg finalize. + local conf sw + conf="$(grep -lE '^\[program:recorder\]' /etc/supervisor/conf.d/*.conf /etc/supervisor/supervisord.conf /etc/supervisord.conf 2>/dev/null | head -n1 || true)" + if [[ -n "$conf" ]]; then + sw="$(awk '/^\[program:recorder\]/{f=1;next} /^\[/{f=0} f && /^stopwaitsecs/{gsub(/[^0-9]/,"",$0); print}' "$conf")" + if [[ -z "$sw" || "$sw" -lt 45 ]]; then + warn "[program:recorder] stopwaitsecs=${sw:-10 (Default)} < 45: SIGKILL vor FFMPEG_FINALIZE_TIMEOUT (30 s) -> abgeschnittene MP4s ($conf)" + else + pass "[program:recorder] stopwaitsecs=$sw" + fi + fi + ;; + systemd) + local unit state enabled restarts tstop priv + for unit in "${SERVICES[@]}"; do + state="$(systemctl is-active "streamvault-${unit}.service" 2>/dev/null || true)" + enabled="$(systemctl is-enabled "streamvault-${unit}.service" 2>/dev/null || true)" + restarts="$(systemctl show -p NRestarts --value "streamvault-${unit}.service" 2>/dev/null || echo 0)" + if [[ "$state" == active ]]; then pass "systemd: streamvault-${unit} active ($enabled)"; else fail "systemd: streamvault-${unit} $state"; fi + [[ "${restarts:-0}" -gt 5 ]] && warn "streamvault-${unit}: ${restarts} Neustarts - moegliche Crash-Schleife (journalctl -u streamvault-${unit})" + priv="$(systemctl show -p PrivateDevices --value "streamvault-${unit}.service" 2>/dev/null || true)" + [[ "$priv" == yes ]] && fail "streamvault-${unit}: PrivateDevices=yes isoliert /dev/shm - Nginx sieht keine Live-HLS-Segmente" + done + tstop="$(systemctl show -p TimeoutStopUSec --value streamvault-recorder.service 2>/dev/null || true)" + info "streamvault-recorder TimeoutStopSec=${tstop:-?} (muss > FFMPEG_FINALIZE_TIMEOUT=30 s sein)" + warn "WebUI-/healthz fragt nur 'supervisorctl status' ab - auf systemd-Hosts ist checks.services.ok=null (Dienstausfall wird dort nicht gemeldet)" + ;; + *) + fail "Weder supervisorctl noch streamvault-*.service gefunden" + ;; + esac +} + +# Extract "internal alias" pairs for a location from `nginx -T`. +nginx_location() { + # $1 = dump file, $2 = location prefix -> prints " " + awk -v loc="$2" ' + $1 == "location" && ($2 == loc || ($2 == "^~" && $3 == loc)) { inblk = 1; depth = 0; internal = 0; alias = ""; found = 1 } + inblk { + n = gsub(/\{/, "{"); m = gsub(/\}/, "}"); depth += n - m + if ($1 == "internal;") internal = 1 + if ($1 == "alias") { alias = $2; sub(/;$/, "", alias) } + if (depth <= 0 && (n > 0 || m > 0)) { print internal, alias; inblk = 0; exit } + } + END { if (!found) print "missing" } + ' "$1" +} + +check_nginx() { + section "Nginx & X-Accel-Redirect" + if ! have nginx; then fail "nginx nicht installiert"; return; fi + if nginx -t >/dev/null 2>&1; then pass "nginx -t"; else fail "nginx -t:"; { nginx -t 2>&1 || true; } | sed 's/^/ /'; return; fi + TMP_ROOT="${TMP_ROOT:-$(mktemp -d)}" + local dump="${TMP_ROOT}/nginx.dump" + nginx -T >"$dump" 2>/dev/null + + if env_flag VOD_USE_X_ACCEL true; then pass "VOD_USE_X_ACCEL=true"; else warn "VOD_USE_X_ACCEL ist aus - Flask streamt Medien selbst"; fi + + local -a prefixes=( + "$(env_get VOD_X_ACCEL_PREFIX /_protected)|$BASE_DIR" + "$(env_get VOD_FILES_ACCEL_PREFIX /_protected_files)|$FILES_DIR" + "$(env_get VOD_CACHE_ACCEL_PREFIX /_protected_vodcache)|$VODCACHE_DIR" + "$(env_get VOD_THUMB_ACCEL_PREFIX /_protected_thumbs)|$THUMB_ROOT" + "$(env_get VOD_HLS_ACCEL_PREFIX /_protected_hls)|$HLS_ROOT" + "$(env_get LIVE_HLS_ACCEL_PREFIX /_protected_hls_live)|${LIVE_HLS_DIR:-/dev/shm/streamvault_hls}" + ) + local entry prefix expected res internal alias + for entry in "${prefixes[@]}"; do + prefix="${entry%%|*}/"; prefix="${prefix//\/\//\/}"; expected="${entry#*|}" + res="$(nginx_location "$dump" "$prefix")" + if [[ "$res" == missing ]]; then fail "location $prefix fehlt"; continue; fi + internal="${res%% *}"; alias="${res#* }" + [[ "$internal" == 1 ]] && pass "location $prefix ist internal" \ + || fail "location $prefix ist NICHT internal - Archiv waere ohne Login abrufbar" + if [[ -z "$alias" ]]; then fail "location $prefix ohne alias"; continue; fi + [[ "$alias" == */ ]] || fail "alias '$alias' in $prefix ohne abschliessenden '/' (Pfad-Verkettung bricht)" + if [[ "$(readlink -m "${alias%/}")" == "$(readlink -m "${expected%/}")" ]]; then + pass "alias $prefix -> $alias" + else + fail "alias $prefix -> $alias, erwartet ${expected%/}/ (aus $ENV_FILE)" + fi + done + + local ds + ds="$({ grep -E '^[[:space:]]*disable_symlinks[[:space:]]' "$dump" || true; } | awk '{print $2}' | tr -d ';' | sort -u | tr '\n' ' ')" + if [[ -n "$ds" && "$ds" != "off " ]]; then + if [[ "${SYMLINKS_PRESENT:-0}" -gt 0 ]]; then + fail "disable_symlinks ${ds}aktiv, aber Archive enthalten Symlinks -> X-Accel liefert 403/404" + else + warn "disable_symlinks ${ds}aktiv - spaetere Archiv-Symlinks (z. B. Block-Storage) wuerden blockiert" + fi + else + pass "disable_symlinks off (Nginx folgt Archiv-Symlinks)" + fi + + local has_tls=0 + grep -Eq '^[[:space:]]*listen[[:space:]].*(443|ssl)' "$dump" && has_tls=1 + if env_flag VOD_SECURE_COOKIES false; then + [[ $has_tls -eq 1 ]] && pass "VOD_SECURE_COOKIES=true mit TLS-Listener" \ + || fail "VOD_SECURE_COOKIES=true ohne TLS-Listener - Login-Redirect-Schleife auf Port 80" + else + [[ $has_tls -eq 1 ]] && warn "TLS aktiv, aber VOD_SECURE_COOKIES=false - nach HTTPS-Umstellung auf true setzen" \ + || pass "VOD_SECURE_COOKIES=false passend zu HTTP-only" + fi + + # Live checks against the production vhost (read-only requests). + local probe code + if make_probe_dir "$BASE_DIR"; then + probe="$PROBE" + as_service bash -c "printf 'svprobe' > '${probe}/probe.mp4'" + code="$(curl -s -o /dev/null -w '%{http_code}' -H 'Host: localhost' "http://127.0.0.1/_protected/$(basename "$probe")/probe.mp4" || true)"; code="${code:-000}" + case "$code" in + 404) pass "Direkter Zugriff auf /_protected/... -> 404 (internal greift)" ;; + 200) fail "Direkter Zugriff auf /_protected/... -> 200: Archiv oeffentlich!" ;; + 000) warn "Nginx auf 127.0.0.1:80 nicht erreichbar" ;; + *) warn "Direkter Zugriff auf /_protected/... -> HTTP $code (erwartet 404)" ;; + esac + if as_nginx head -c 1 "${probe}/probe.mp4" >/dev/null 2>&1; then + pass "Neu angelegte Datei des Service-Users ist fuer ${NGINX_USER} lesbar" + else + fail "Neu angelegte Datei des Service-Users ist fuer ${NGINX_USER} NICHT lesbar (umask/ACL)" + fi + fi + code="$(curl -s -o "${TMP_ROOT}/healthz.json" -w '%{http_code}' -H 'Host: localhost' http://127.0.0.1/healthz || true)"; code="${code:-000}" + case "$code" in + 200) pass "GET /healthz ueber Nginx -> 200 $(head -c 160 "${TMP_ROOT}/healthz.json")" ;; + 503) fail "GET /healthz -> 503 degraded $(head -c 160 "${TMP_ROOT}/healthz.json")" ;; + *) fail "GET /healthz ueber Nginx -> HTTP $code (WebUI/Proxy pruefen)" ;; + esac + + if [[ $E2E -eq 1 ]]; then xaccel_e2e; fi + return 0 +} + +# Isolated X-Accel round trip: throwaway Nginx (same worker user, same alias +# targets, same disable_symlinks) in front of a stub backend that only emits +# X-Accel-Redirect. Production config and services are not touched. +xaccel_e2e() { + local probe live_probe port_ng port_be dir conf ds + make_probe_dir "$BASE_DIR" || { warn "E2E: kein Probe-Verzeichnis"; return 0; } + probe="$PROBE" + as_service bash -c "head -c 65536 /dev/urandom > '${probe}/e2e.mp4'" + if [[ -n "$LIVE_HLS_DIR" && -d "$LIVE_HLS_DIR" ]]; then + live_probe="" + make_probe_dir "$LIVE_HLS_DIR" && live_probe="$PROBE" + [[ -n "$live_probe" ]] && as_service bash -c "printf '#EXTM3U\n' > '${live_probe}/index.m3u8'" + fi + dir="$(mktemp -d)"; chmod 755 "$dir"; E2E_DIR="$dir" + read -r port_ng port_be < <("$PY" -c 'import socket +s=[socket.socket() for _ in range(2)] +[x.bind(("127.0.0.1",0)) for x in s] +print(*[x.getsockname()[1] for x in s])') + ds="$(grep -hE '^[[:space:]]*disable_symlinks[[:space:]]' "${TMP_ROOT}/nginx.dump" | head -n1 || true)" + local name live_name="" live_loc="" + name="$(basename "$probe")" + if [[ -n "${live_probe:-}" ]]; then + live_name="$(basename "$live_probe")" + live_loc="location /_protected_hls_live/ { + internal; alias ${LIVE_HLS_DIR%/}/; + types { application/vnd.apple.mpegurl m3u8; video/mp2t ts; } + add_header Cache-Control \"no-cache, no-store, must-revalidate\" always; + }" + fi + conf="${dir}/nginx.conf" + cat >"$conf" <"${dir}/backend.log" 2>&1 & + BG_PIDS+=("$!") + nginx -c "$conf" -p "$dir" & + BG_PIDS+=("$!") + sleep 1.5 + + local code size headers + code="$(curl -s -o "${dir}/out.bin" -w '%{http_code} %{size_download}' "http://127.0.0.1:${port_ng}/accel/_protected/${name}/e2e.mp4" || true)"; code="${code:-000 0}" + size="${code#* }"; code="${code%% *}" + if [[ "$code" == 200 && "$size" == 65536 ]] && cmp -s "${dir}/out.bin" "${probe}/e2e.mp4"; then + pass "X-Accel E2E: /_protected/ liefert Datei als ${NGINX_USER} bytegenau aus (alias ${BASE_DIR%/}/)" + else + fail "X-Accel E2E: /_protected/ -> HTTP ${code}, ${size} Bytes"; tail -n 5 "${dir}/error.log" 2>/dev/null | sed 's/^/ /' || true + fi + code="$(curl -s -o /dev/null -w '%{http_code}' -r 100-199 "http://127.0.0.1:${port_ng}/accel/_protected/${name}/e2e.mp4" || true)"; code="${code:-000}" + [[ "$code" == 206 ]] && pass "X-Accel E2E: Range-Request -> 206 (Seeking im Player)" \ + || fail "X-Accel E2E: Range-Request -> HTTP $code (erwartet 206)" + if [[ -n "${live_probe:-}" ]]; then + headers="$(curl -s -D - -o /dev/null "http://127.0.0.1:${port_ng}/accel/_protected_hls_live/${live_name}/index.m3u8" || true)" + if grep -q '^HTTP/1.1 200' <<<"$headers" && grep -qi 'content-type: application/vnd.apple.mpegurl' <<<"$headers"; then + pass "X-Accel E2E: Live-HLS aus RAM-Disk (m3u8, korrekter MIME-Typ)" + else + fail "X-Accel E2E: Live-HLS aus ${LIVE_HLS_DIR} nicht auslieferbar:"; head -n 3 <<<"$headers" | sed 's/^/ /' + fi + fi + rm -rf -- "$dir" +} + +check_hls_ramdisk() { + section "HLS RAM-Disk" + if [[ -z "$LIVE_HLS_DIR" ]]; then + fail "LIVE_HLS_DIR fehlt in $ENV_FILE - WebUI (/dev/shm/streamvault_hls) und Recorder (/dev/shm/streamrecorder_hls) haben unterschiedliche Defaults" + return + fi + pass "LIVE_HLS_DIR=$LIVE_HLS_DIR" + if [[ ! -d "$LIVE_HLS_DIR" ]]; then + if [[ $FIX -eq 1 ]]; then + install -d -m 0755 -o "$SERVICE_USER" -g "$SERVICE_USER" "$LIVE_HLS_DIR" && pass "LIVE_HLS_DIR angelegt (--fix)" + else + fail "LIVE_HLS_DIR existiert nicht (nach Reboot geloescht?) - mit --fix anlegen" + return + fi + fi + local fstype owner avail_mb + fstype="$(findmnt -n -o FSTYPE -T "$LIVE_HLS_DIR" 2>/dev/null | tail -n 1)"; fstype="${fstype:-unknown}" + [[ "$fstype" == tmpfs ]] && pass "LIVE_HLS_DIR liegt auf tmpfs" \ + || warn "LIVE_HLS_DIR liegt auf '$fstype' statt tmpfs - Segmente erzeugen Disk-I/O" + owner="$(stat -c %U "$LIVE_HLS_DIR")" + [[ "$owner" == "$SERVICE_USER" ]] && pass "Owner: $owner" || warn "Owner von LIVE_HLS_DIR ist $owner (erwartet $SERVICE_USER)" + local probe + if make_probe_dir "$LIVE_HLS_DIR"; then + probe="$PROBE" + as_service bash -c "printf 'x' > '${probe}/seg0.ts'" && pass "Service-User kann Segmente schreiben" + if as_nginx head -c 1 "${probe}/seg0.ts" >/dev/null 2>&1; then + pass "${NGINX_USER} kann Segmente lesen" + else + fail "${NGINX_USER} kann Segmente in $LIVE_HLS_DIR nicht lesen"; namei -l "${probe}/seg0.ts" 2>/dev/null | sed 's/^/ /' || true + fi + else + fail "Service-User kann nicht in $LIVE_HLS_DIR schreiben" + fi + avail_mb="$(df -P -k "$LIVE_HLS_DIR" | awk 'NR==2 {printf "%d", $4/1024}')" + if [[ "$avail_mb" -lt 256 ]]; then fail "RAM-Disk: nur ${avail_mb} MB frei" + elif [[ "$avail_mb" -lt 1024 ]]; then warn "RAM-Disk: ${avail_mb} MB frei (< 1 GB, mehrere Live-Kanaele knapp)" + else pass "RAM-Disk: ${avail_mb} MB frei"; fi + + if grep -rqsF "$LIVE_HLS_DIR" /etc/tmpfiles.d /usr/lib/tmpfiles.d 2>/dev/null; then + pass "tmpfiles.d-Eintrag vorhanden (Verzeichnis ueberlebt Reboot)" + elif [[ $FIX -eq 1 ]]; then + printf 'd %s 0755 %s %s -\n' "$LIVE_HLS_DIR" "$SERVICE_USER" "$SERVICE_USER" > /etc/tmpfiles.d/streamvault.conf + systemd-tmpfiles --create /etc/tmpfiles.d/streamvault.conf && pass "tmpfiles.d-Eintrag angelegt (--fix)" + else + warn "Kein tmpfiles.d-Eintrag: /dev/shm wird beim Reboot geleert (mit --fix anlegen)" + fi + local stale + stale="$(find "$LIVE_HLS_DIR" -mindepth 2 -maxdepth 2 -name '*.m3u8' -mmin +10 2>/dev/null | wc -l)" + [[ "$stale" -gt 0 ]] && info "${stale} Playlists aelter als 10 min in der RAM-Disk (cleanup_live_hls raeumt beim naechsten Lauf)" + return 0 +} + +check_config() { + section "Konfiguration & Sicherheit" + if [[ ! -f "$ENV_FILE" ]]; then fail "$ENV_FILE fehlt"; return; fi + local mode owner group + mode="$(stat -c %a "$ENV_FILE")"; owner="$(stat -c %U "$ENV_FILE")"; group="$(stat -c %G "$ENV_FILE")" + if [[ "${mode: -1}" != 0 ]]; then fail "$ENV_FILE ist world-readable ($mode)" + elif [[ "$mode" == 640 && "$owner" == root && "$group" == "$SERVICE_USER" ]]; then pass "$ENV_FILE 640 root:$SERVICE_USER" + else warn "$ENV_FILE: $mode $owner:$group (erwartet 640 root:$SERVICE_USER)"; fi + [[ -n "$(env_get VOD_PASS)" ]] && pass "VOD_PASS gesetzt" || fail "VOD_PASS leer" + local sk; sk="$(env_get VOD_SECRET_KEY)" + [[ ${#sk} -ge 32 ]] && pass "VOD_SECRET_KEY >= 32 Zeichen" || fail "VOD_SECRET_KEY fehlt/zu kurz" + if env_flag SENTRY_SEND_DEFAULT_PII false; then warn "SENTRY_SEND_DEFAULT_PII=true - PII geht an Sentry"; else pass "Sentry-PII deaktiviert"; fi + return 0 +} + +do_validate() { + [[ $EUID -eq 0 ]] || die "validate muss als root laufen (runuser/nginx -T)." + load_paths + printf '%sStreamVault VPS-Validierung%s app=%s venv=%s env=%s\n' "$C_BLD" "$C_OFF" "$APP_DIR" "$VENV_DIR" "$ENV_FILE" + check_system + [[ $RUN_TESTS -eq 1 ]] && check_tests + check_config + check_filesystem + check_supervisor + check_hls_ramdisk + check_nginx + section "Ergebnis" + printf ' PASS=%d WARN=%d FAIL=%d\n' "$N_PASS" "$N_WARN" "$N_FAIL" + if [[ $N_FAIL -gt 0 ]]; then printf ' %sNO-GO%s\n' "$C_RED$C_BLD" "$C_OFF"; return 1; fi + if [[ $STRICT -eq 1 && $N_WARN -gt 0 ]]; then printf ' %sNO-GO (--strict)%s\n' "$C_YLW$C_BLD" "$C_OFF"; return 1; fi + printf ' %sGO%s\n' "$C_GRN$C_BLD" "$C_OFF" + return 0 +} + +# ============================================================================== +# Deployment +# ============================================================================== +restart_services() { + case "$(detect_supervisor)" in + supervisor) supervisorctl restart worker webui recorder ;; + systemd) systemctl restart streamvault-worker.service streamvault-webui.service streamvault-recorder.service ;; + *) die "Kein Supervisor erkannt - Dienste nicht neu gestartet." ;; + esac +} + +services_healthy() { + local unit + case "$(detect_supervisor)" in + supervisor) + for unit in "${SERVICES[@]}"; do + supervisorctl status "$unit" 2>/dev/null | grep -q RUNNING || return 1 + done ;; + systemd) + for unit in "${SERVICES[@]}"; do + systemctl is-active --quiet "streamvault-${unit}.service" || return 1 + done ;; + *) return 1 ;; + esac +} + +gate_release() { + # Compile + tests with the venv interpreter as the service user. + N_FAIL=0 + check_system + [[ $SKIP_TESTS -eq 1 ]] || check_tests + [[ $N_FAIL -eq 0 ]] +} + +restore_snapshot() { + local state="${BACKUP_DIR}/last_deploy.env" mode prev archive tmp + [[ -r "$state" ]] || die "Kein Deploy-Zustand in $state." + mode="$(grep '^MODE=' "$state" | cut -d= -f2-)" + prev="$(grep '^PREV_REV=' "$state" | cut -d= -f2-)" + archive="$(grep '^SNAPSHOT=' "$state" | cut -d= -f2-)" + if [[ "$mode" == git && -n "$prev" ]]; then + as_service git -C "$APP_DIR" checkout --quiet --detach "$prev" + printf 'Rollback: git checkout %s\n' "$prev" + else + [[ -r "$archive" ]] || die "Snapshot fehlt: $archive" + tmp="$(mktemp -d)" + tar -xzf "$archive" -C "$tmp" + rsync -a --delete --exclude '.git/' "$tmp"/ "$APP_DIR"/ + chown -R "$SERVICE_USER:$SERVICE_USER" "$APP_DIR" + rm -rf -- "$tmp" + printf 'Rollback: %s wiederhergestellt\n' "$archive" + fi +} + +do_deploy() { + [[ $EUID -eq 0 ]] || die "deploy muss als root laufen." + [[ -n "$SOURCE_DIR" || -n "$GIT_REF" ]] || die "deploy braucht --source DIR oder --ref GITREF." + [[ -n "$SOURCE_DIR" && -n "$GIT_REF" ]] && die "--source und --ref schliessen sich aus." + [[ -d "$APP_DIR" && -x "$PY" ]] || die "Keine Installation unter $APP_DIR / $VENV_DIR - zuerst install.sh ausfuehren." + have rsync || die "rsync fehlt (apt-get install rsync)." + load_paths + + # 1. Never cut running recordings without explicit consent. + local active + active="$(find "$(readlink -f "$BASE_DIR")" -mindepth 2 -maxdepth 2 -name '*.part' -mmin -2 2>/dev/null || true)" + if [[ -n "$active" && $FORCE -eq 0 ]]; then + printf 'Aktive Aufnahmen (frische .part-Dateien):\n%s\n' "$active" | sed 's/^/ /' + die "Recorder-Neustart wuerde diese Aufnahmen splitten. Spaeter erneut versuchen oder --force." + fi + + # 2. Snapshot current code. + install -d -m 0750 "$BACKUP_DIR" + local ts mode prev snapshot + ts="$(date +%Y%m%d-%H%M%S)" + snapshot="${BACKUP_DIR}/app-${ts}.tgz" + tar -czf "$snapshot" -C "$APP_DIR" --exclude='./.git' --exclude='__pycache__' . + if [[ -d "${APP_DIR}/.git" ]]; then + mode=git; prev="$(as_service git -C "$APP_DIR" rev-parse HEAD)" + else + mode=copy; prev="" + fi + [[ -n "$SOURCE_DIR" ]] && mode=copy + printf 'MODE=%s\nPREV_REV=%s\nSNAPSHOT=%s\nTS=%s\n' "$mode" "$prev" "$snapshot" "$ts" > "${BACKUP_DIR}/last_deploy.env" + find "$BACKUP_DIR" -maxdepth 1 -name 'app-*.tgz' -printf '%T@ %p\n' | sort -rn | tail -n +6 | cut -d' ' -f2- | xargs -r rm -f -- + printf 'Snapshot: %s\n' "$snapshot" + + # 3. Update code. + if [[ -n "$GIT_REF" ]]; then + [[ -d "${APP_DIR}/.git" ]] || die "$APP_DIR ist kein git-Checkout - --source verwenden." + if [[ -n "$(as_service git -C "$APP_DIR" status --porcelain --untracked-files=no)" ]]; then + die "Lokale Aenderungen in $APP_DIR - Deploy abgebrochen." + fi + as_service git -C "$APP_DIR" fetch --quiet --tags origin + as_service git -C "$APP_DIR" checkout --quiet --detach "$GIT_REF" + printf 'Code: git %s -> %s\n' "${prev:0:12}" "$(as_service git -C "$APP_DIR" rev-parse --short=12 HEAD)" + else + [[ -f "${SOURCE_DIR}/WEBUI/webui.py" ]] || die "$SOURCE_DIR ist kein StreamVault-Quellbaum." + local -a rs=(-a --exclude '.git/' --exclude '.venv/' --exclude '__pycache__/' --exclude '*.pyc' --exclude '.env') + if [[ $PRUNE -eq 1 ]]; then + case "$(readlink -f "$BASE_DIR")/" in + "$(readlink -f "$APP_DIR")"/*) die "--prune verweigert: VOD_BASE_DIR liegt innerhalb von APP_DIR." ;; + esac + rs+=(--delete) + fi + rsync "${rs[@]}" "${SOURCE_DIR%/}"/ "$APP_DIR"/ + chown -R "$SERVICE_USER:$SERVICE_USER" "$APP_DIR" + printf 'Code: rsync aus %s%s\n' "$SOURCE_DIR" "$([[ $PRUNE -eq 1 ]] && echo ' (--prune)')" + fi + + # 4. Dependencies. + as_service "${VENV_DIR}/bin/pip" install -q -r "${APP_DIR}/requirements.txt" + + # 5. Release gate before touching running services. + if ! gate_release; then + printf '%sRelease-Gate fehlgeschlagen - Rollback.%s\n' "$C_RED" "$C_OFF" + restore_snapshot + exit 1 + fi + + # 6. Restart and verify. + restart_services + sleep 10 + if ! services_healthy; then + printf '%sDienste nach Neustart nicht gesund - Rollback.%s\n' "$C_RED" "$C_OFF" + restore_snapshot + restart_services + exit 1 + fi + N_PASS=0; N_WARN=0; N_FAIL=0 + RUN_TESTS=0 + do_validate +} + +do_rollback() { + [[ $EUID -eq 0 ]] || die "rollback muss als root laufen." + restore_snapshot + restart_services + sleep 10 + services_healthy && printf 'Rollback abgeschlossen, Dienste laufen.\n' || die "Dienste nach Rollback nicht gesund." +} + +usage() { sed -n '2,27p' "$0" | sed 's/^# \{0,1\}//'; exit 2; } + +CMD="${1:-}"; [[ -n "$CMD" ]] || usage; shift +while [[ $# -gt 0 ]]; do + case "$1" in + --run-tests) RUN_TESTS=1 ;; + --no-e2e) E2E=0 ;; + --fix) FIX=1 ;; + --strict) STRICT=1 ;; + --force) FORCE=1 ;; + --skip-tests) SKIP_TESTS=1 ;; + --prune) PRUNE=1 ;; + --source) SOURCE_DIR="${2:?--source braucht ein Verzeichnis}"; shift ;; + --ref) GIT_REF="${2:?--ref braucht einen git-Ref}"; shift ;; + -h|--help) usage ;; + *) printf 'Unbekannte Option: %s\n' "$1" >&2; usage ;; + esac + shift +done + +case "$CMD" in + validate) do_validate ;; + deploy) do_deploy ;; + rollback) do_rollback ;; + *) usage ;; +esac From d2e6e38b578feec9c47f5bdad4ea5f02816a6642 Mon Sep 17 00:00:00 2001 From: Pierre Date: Sat, 31 Oct 2026 23:41:56 +0000 Subject: [PATCH 09/31] docs: add CLAUDE.md engine guide Architecture map, essential commands and binding invariants (no silent failures, immutable tests, stop-before-unlock, bounded waits, SQLite WAL rules, worker file safety, ASR process isolation, storage hygiene). Records Python 3.12+ as the runtime floor, actual test counts (93 + 33) and the real disk-pause defaults (90 % / 20 GB). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tzan3xULuUxadM3Azn5fzJ --- CLAUDE.md | 167 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..e6815f6 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,167 @@ +# CLAUDE.md — StreamVault (nachtsilo-hub) Engine Guide + +## Project Overview +StreamVault is an automated 24/7 multi-platform live stream recording, HLS live restreaming, VOD transcoding, and cloud archival system. +- **Languages & Frameworks:** Python 3.12+ (`WEBUI/webui.py` uses PEP 701 f-strings) (Flask, SQLite WAL, Streamlink, yt-dlp, FFmpeg, Whisper ASR, Boto3, PyCryptodome). +- **Core Services:** + - `Streamrecorder.py`: Daemon managing per-channel recording processes (`flock`-isolated). + - `worker.py`: Multi-threaded background job runner (IO, Remux, ASR, Cloud Storage). + - `WEBUI/webui.py`: Flask dashboard, X-Accel-Redirect streaming, user traffic monitoring, supervisor health checks. + - `run_transcribe.py`: Isolated subprocess runner for Whisper ASR (`STREAMVAULT_IMPORT_ONLY=1`). + - `core/storage/`: Provider-agnostic cloud storage service (AES-GCM encryption; S3, WebDAV, Google Drive, OneDrive, Dropbox). + - `bin/streamvault`: Linux CLI for service orchestration (`start`, `stop`, `restart`, `status`, `logs`, `storage`). + +--- + +## Essential Commands + +### Environment Setup +```bash +# Linux +python3 -m venv .venv +source .venv/bin/activate +pip install --upgrade pip +pip install -r requirements.txt + +# Windows (Audit Host) +.\.venv\Scripts\activate +pip install -r requirements.txt +``` + +### Running Tests +The test suite uses Python's standard `unittest` discovery: +```bash +# Run entire test suite (all 126 tests) +python -m unittest discover -s tests -p "test_*.py" -v + +# Run specific test modules +python -m unittest tests/test_recorder_reliability.py -v +python -m unittest tests/test_storage.py -v + +# Run single test class or test method +python -m unittest tests.test_recorder_reliability.DiskSpaceGuardTests -v +python -m unittest tests.test_recorder_reliability.DiskSpaceGuardTests.test_disk_blocks_recording_on_nonexistent_dir_when_disk_full -v +``` + +### Syntax & Compilation Verification +Always verify Python syntax before committing: +```bash +# Single file +python -m py_compile recorder/Streamrecorder.py + +# Complete repository +python -c "import os, py_compile; [py_compile.compile(os.path.join(r, f), doraise=True) for r, _, fs in os.walk('.') if not any(x in r for x in ('.venv', '__pycache__', '.git')) for f in fs if f.endswith('.py')]" +``` + +### Service Orchestration +```bash +# Linux CLI +streamvault status +streamvault start +streamvault stop +streamvault restart [webui|recorder|worker] +streamvault logs [webui|recorder|worker] -f +streamvault storage sync + +# Direct CLI / Manual Invocation +python recorder/Streamrecorder.py +python recorder/worker.py +python -m WEBUI.webui +``` + +--- + +## Architectural Map & Directory Structure + +``` +nachtsilo-hub/ +├── bin/ +│ └── streamvault # Global bash CLI wrapper (/usr/local/bin/streamvault) +├── core/ +│ ├── config.py # Env parsing, default directories (BASE_DIR, HLS_ROOT, etc.) +│ ├── database.py # SQLite WAL access, schema init, @db_retry, task queues +│ ├── health.py # Lightweight supervisorctl status parser (no heavy imports) +│ ├── redis_client.py # Optional high-speed task queue & pub/sub fallback +│ ├── mp4_utils.py # Fast moov atom shifting & faststart verification +│ └── storage/ # Cloud storage subsystem +│ ├── base.py # Abstract BaseStorageProvider, mask_error, path normalization +│ ├── crypto.py # AES-GCM credential encryption (STORAGE_ENCRYPTION_KEY) +│ ├── http.py # Resilient HTTP session, backoff, sanitized error logs +│ ├── registry.py # Provider registration & runtime dispatch +│ ├── service.py # Scheduled background sync, retention pruning, task handlers +│ └── providers/ # S3, WebDAV, Google Drive, OneDrive, Dropbox implementations +├── recorder/ +│ ├── Streamrecorder.py # Multi-channel recording engine, flock locking, FFmpeg pipelines +│ ├── worker.py # Background task consumer (pools: io, remux, asr, storage) +│ ├── clip_engine.py # Chat-overlay and highlight clipping +│ └── chat_events.py # Chat event ingestion & IRC listeners +├── WEBUI/ +│ ├── webui.py # Flask application, auth, routes, X-Accel-Redirect streaming +│ ├── templates/ # Jinja2 HTML templates +│ └── static/ # CSS, JS, alert audio, icons +├── run_transcribe.py # Subprocess wrapper for Whisper ASR to keep worker memory clean +├── scripts/ +│ └── streamvault_vps.sh # VPS deploy/rollback + Linux validation (supervisor, X-Accel, HLS RAM disk) +└── tests/ + ├── test_recorder_reliability.py # 33 tests: process reaping, locking, folder safety, supervisor parsing + └── test_storage.py # 93 tests: crypto, providers, retention, error masking +``` + +--- + +## Critical Rules & Architectural Invariants + +### 1. Zero Silent Failures (Anti-Bypass Policy) +- **STRICTLY FORBIDDEN:** `except Exception: pass`, `except: pass`, or returning empty values without logging. +- Catch specific error classes (e.g., `sqlite3.OperationalError`, `subprocess.TimeoutExpired`, `OSError`). +- Always log caught unexpected exceptions using `logger.error(...)` or `logger.warning(...)`. + +### 2. Immutable Tests Rule +- **NEVER** weaken, comment out, or delete assertions in `tests/` to make a test pass. +- Fix bugs in the application code, not by relaxing tests. +- When adding new features, add corresponding tests in `tests/test_*.py`. + +### 3. Process Management & Lock Hierarchy +- **File Lock Ordering:** When stopping a channel process, **always stop the child process before releasing the channel `flock`**: + ```python + # CORRECT: + stop_channel_and_release_lock(recordings_dir, channel, proc) + # WRONG: + release_held_channel_lock(recordings_dir, channel) + stop_channel_process(proc, channel) # Races with next recorder process! + ``` +- **Bounded Waiting:** Never call indefinite `process.wait()`. Always use `wait_for_process_exit(proc, timeout=...)` with a hard deadline (default 30s) and fallback to `kill()`. +- **No `fuser`:** Never execute `fuser -k -9` or kill parent process groups. + +### 4. Database Concurrency & SQLite WAL +- SQLite is configured with WAL mode (`PRAGMA journal_mode=WAL`). +- All write operations that may experience busy/locked errors must use the `@db_retry(max_retries=5, delay=0.1, backoff=2.0)` decorator and `BEGIN IMMEDIATE`. +- In multi-threaded environments (`worker.py`), each thread **must** obtain its own connection via `get_db()`. Never share SQLite connection handles across threads. +- Stale running tasks are claimed using `claim_next_task()` and must respect `max_attempts_by_type` to avoid endless retry loops on permanently crashing jobs. + +### 5. Worker & File Safety +- **Directory Protection:** Before deleting or emptying an archive folder, always check: + - `_folder_has_active_recording(folder_path)` + - `_folder_has_active_remux(folder_path)` + If active, raise `TaskDeferred(delay_sec=120)`. +- **In-Progress Files:** Files ending in `.part` or `.remux.lock` that have a fresh modification timestamp (< 10 min) must **never** be deleted or pruned. +- **Disk Space Protection:** `disk_blocks_recording(path)` walks up to the nearest existing parent directory if `path` does not exist yet. New recordings are paused if disk usage reaches `STORAGE_PAUSE_PCT` (default 90%, env `STREAM_RECORDER_STORAGE_PAUSE_PCT`) or free space falls to `STORAGE_PAUSE_FREE_GB` (default 20 GB, env `STREAM_RECORDER_STORAGE_PAUSE_FREE_GB`). + +### 6. Memory & Process Isolation (Whisper ASR) +- Never load heavy ML frameworks (Whisper, PyTorch, CTranslate2) directly into the main `worker.py` process. +- All transcription tasks must execute via `run_transcribe.py` in a separate child process. +- `run_transcribe.py` sets `STREAMVAULT_IMPORT_ONLY=1` to prevent import-time side effects (admin creation, background threads) when loading `WEBUI.webui`. + +### 7. Security & Cloud Storage Hygiene +- **Credential Storage:** Provider credentials and OAuth refresh tokens must be encrypted with AES-GCM via `core.storage.crypto.encrypt_secret()`. +- **Log Sanitization:** All cloud errors must be masked via `mask_error(exc)` before writing to database or logs to strip URLs, auth tokens, and S3 credentials. +- **Path Traversal Protection:** All client-supplied filesystem paths must be validated with `_is_direct_child(config.BASE_DIR, path)` or `classify_local_path()`. +- **Cookies:** `VOD_SECURE_COOKIES` defaults to `false` in `.env.example` to prevent redirect loops when Nginx listens on plain HTTP port 80. Set to `true` only when HTTPS is configured. +- **Sentry / Telemetry:** PII sending is disabled by default (`SENTRY_SEND_DEFAULT_PII=false`). + +--- + +## Common Handoff & Troubleshooting Notes +- On Windows hosts, `fcntl` is mocked automatically in test suites via shims in `test_recorder_reliability.py`. In production, the system targets Debian/Ubuntu Linux. +- If Redis is unavailable, task queuing automatically falls back to SQLite `tasks` table with zero configuration needed. +- If Supervisor reports degraded status, check `parse_supervisor_status()` in `core/health.py`. Standard monitored services are `webui`, `recorder`, `worker`. From 6343f0fec9efc25fb5b04be520d4050af0c10dbe Mon Sep 17 00:00:00 2001 From: Pierre Date: Sat, 31 Oct 2026 23:48:44 +0000 Subject: [PATCH 10/31] fix(webui): restore Python 3.10/3.11 compatibility mobile_app_head_tags() nested an f-string containing "\n" inside an f-string expression. That is only valid from Python 3.12 (PEP 701), so WEBUI/webui.py did not compile on the distro Python of Ubuntu 22.04 (3.10) and Debian 12 (3.11) that install.sh targets - the WebUI would not start. The Tizen TV tags are now built in a variable first; the rendered HTML is unchanged (verified with is_tizen_tv True/False). - scripts/streamvault_vps.sh: Python floor back to 3.10 - CLAUDE.md: Python 3.10+ and a rule against PEP 701 syntax, which the 3.12 Windows audit host accepts silently Verified: all 45 files compile and 126/126 tests pass on Python 3.10.12. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Tzan3xULuUxadM3Azn5fzJ --- CLAUDE.md | 7 ++++++- WEBUI/webui.py | 12 +++++++++++- scripts/streamvault_vps.sh | 6 +++--- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index e6815f6..35c6e1c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,7 +2,7 @@ ## Project Overview StreamVault is an automated 24/7 multi-platform live stream recording, HLS live restreaming, VOD transcoding, and cloud archival system. -- **Languages & Frameworks:** Python 3.12+ (`WEBUI/webui.py` uses PEP 701 f-strings) (Flask, SQLite WAL, Streamlink, yt-dlp, FFmpeg, Whisper ASR, Boto3, PyCryptodome). +- **Languages & Frameworks:** Python 3.10+ (Ubuntu 22.04 / Debian 12 system Python) (Flask, SQLite WAL, Streamlink, yt-dlp, FFmpeg, Whisper ASR, Boto3, PyCryptodome). - **Core Services:** - `Streamrecorder.py`: Daemon managing per-channel recording processes (`flock`-isolated). - `worker.py`: Multi-threaded background job runner (IO, Remux, ASR, Cloud Storage). @@ -116,6 +116,11 @@ nachtsilo-hub/ - Catch specific error classes (e.g., `sqlite3.OperationalError`, `subprocess.TimeoutExpired`, `OSError`). - Always log caught unexpected exceptions using `logger.error(...)` or `logger.warning(...)`. +### 1a. Python 3.10 Compatibility +- Production runs on the distro Python (3.10 on Ubuntu 22.04, 3.11 on Debian 12). +- **No PEP 701 f-strings:** never put backslashes (`\n`), comments or the enclosing quote character inside an f-string `{...}` expression. Build the value in a variable first. Python 3.12+ accepts this silently, so the Windows audit host (3.12) will not catch it; `scripts/streamvault_vps.sh validate` compiles with the target interpreter. +- No 3.11+ stdlib APIs (`tomllib`, `datetime.UTC`, `typing.Self`, `ExceptionGroup`, `except*`). + ### 2. Immutable Tests Rule - **NEVER** weaken, comment out, or delete assertions in `tests/` to make a test pass. - Fix bugs in the application code, not by relaxing tests. diff --git a/WEBUI/webui.py b/WEBUI/webui.py index afdfea2..1060e5c 100644 --- a/WEBUI/webui.py +++ b/WEBUI/webui.py @@ -10459,6 +10459,16 @@ def _pwa_icon_entries(): def mobile_app_head_tags(): sw_url = url_for("service_worker_js") + # Built outside the f-string: a backslash inside an f-string expression is + # only valid from Python 3.12 (PEP 701); Debian 12 / Ubuntu 22.04 ship 3.11/3.10. + tizen_tags = "" + if is_tizen_tv(): + tizen_css = url_for("static", filename="css/tizen_tv.css", v=ASSET_VERSION) + tizen_js = url_for("static", filename="tizen_tv_nav.js", v=ASSET_VERSION) + tizen_tags = ( + f'\n' + f' ' + ) return f"""{PWA_HEAD_MARKER} @@ -10471,7 +10481,7 @@ def mobile_app_head_tags(): - {f'\n ' if is_tizen_tv() else ''} + {tizen_tags} - {% if session.username and not is_tv and not is_mobile %} + {% if session.username and not is_mobile %} - + diff --git a/WEBUI/templates/twitch_watch_mobile.html b/WEBUI/templates/twitch_watch_mobile.html index 6418545..2c88fbe 100644 --- a/WEBUI/templates/twitch_watch_mobile.html +++ b/WEBUI/templates/twitch_watch_mobile.html @@ -59,7 +59,6 @@ webkit-playsinline disablePictureInPicture preload="auto" - x-webkit-airplay="allow" > {% endif %} -
@@ -172,6 +157,6 @@

{{ display_name }}

{{ title or display_name }}

- + diff --git a/WEBUI/templates/twitch_watch_proxy.js.html b/WEBUI/templates/twitch_watch_proxy.js.html index ba5d9c2..d58e7f0 100644 --- a/WEBUI/templates/twitch_watch_proxy.js.html +++ b/WEBUI/templates/twitch_watch_proxy.js.html @@ -55,12 +55,6 @@ }); window.StreamVaultMediaSession && window.StreamVaultMediaSession.attach(video, {live: true}); - window.StreamVaultCast && window.StreamVaultCast.bindVideo(video, { - url: playlistUrl || ('/live/' + encodeURIComponent(channelName) + '/stream.m3u8'), - live: true, - contentType: 'application/vnd.apple.mpegurl', - ensureUrl: {{ (cast_ensure_url or '')|tojson }} - }); let mpegtsPlayer = null; let hls = null; @@ -296,9 +290,9 @@ try { // 1. Primary: mpegts.js Ultra-Low-Latency Byte-Stream const mseSupported = window.mpegts && mpegts.getFeatureList && mpegts.getFeatureList().mseLivePlayback; - const castWantsNative = window.StreamVaultCast && StreamVaultCast.preferNativeHls(video); + const nativeHlsPreferred = window.StreamVaultHls && StreamVaultHls.preferNativeHls(video); - if (mseSupported && !castWantsNative) { + if (mseSupported && !nativeHlsPreferred) { dbg('info', 'init_mpegts_player', { channel: channelName }); const tsUrl = '/live/' + encodeURIComponent(channelName) + '.ts'; mpegtsPlayer = mpegts.createPlayer({ @@ -343,7 +337,7 @@ return; } - // 2. Fallback: Low-Latency HLS (iOS / Safari / native cast) + // 2. Fallback: Low-Latency HLS (iOS / Safari) initLowLatencyHlsPlayer(video, '/live/' + encodeURIComponent(channelName) + '/stream.m3u8'); } catch (e) { diff --git a/WEBUI/templates/vod.html b/WEBUI/templates/vod.html index 25f52f4..03a98b5 100644 --- a/WEBUI/templates/vod.html +++ b/WEBUI/templates/vod.html @@ -351,7 +351,7 @@

Clip erstellen

{% endif %} {% if hls_ready %}{% endif %} - + @@ -605,12 +605,7 @@

Clip erstellen

localStorage.removeItem(key); }); - window.StreamVaultCast?.bindVideo(vPlayer, { - url: directMediaUrl, - live: false, - contentType: 'video/mp4' - }); - if (useVodHls && window.StreamVaultCast && StreamVaultCast.preferNativeHls(vPlayer)) { + if (useVodHls && window.StreamVaultHls && StreamVaultHls.preferNativeHls(vPlayer)) { vPlayer.src = vodHlsUrl; } else if (useVodHls && typeof Hls !== 'undefined' && Hls.isSupported()) { const hlsStartPosition = Number.isFinite(requestedStart) && requestedStart > 1 diff --git a/WEBUI/webui.py b/WEBUI/webui.py index ab1fcfa..85496ab 100644 --- a/WEBUI/webui.py +++ b/WEBUI/webui.py @@ -52,7 +52,6 @@ url_for, ) from flask_limiter import Limiter -from itsdangerous import BadSignature, SignatureExpired, URLSafeTimedSerializer from markupsafe import Markup, escape from werkzeug.security import check_password_hash, generate_password_hash from werkzeug.utils import safe_join, secure_filename @@ -172,11 +171,6 @@ def _env_int(name, default): # Browsers cap cookie lifetime near 400 days; SESSION_REFRESH_EACH_REQUEST extends it on use. SESSION_MAX_AGE_SEC = max(0, _env_int("VOD_SESSION_MAX_AGE_SEC", 0)) SESSION_COOKIE_MAX_AGE_SEC = max(86400, _env_int("VOD_SESSION_COOKIE_MAX_AGE_SEC", 400 * 86400)) -CAST_TOKEN_TTL_SEC = int(os.environ.get("VOD_CAST_TTL_SEC", str(12 * 3600))) -# OBS browser sources stay open for days; keep that bearer link shorter than a session cookie. -# OBS browser sources stay open indefinitely in streaming setups; default to 10 years (or set OBS_CAST_TTL_SEC=0 for infinite) -OBS_CAST_TTL_SEC = int(os.environ.get("OBS_CAST_TTL_SEC", str(3650 * 24 * 3600))) -CAST_TOKEN_SALT = "streamvault-cast-v1" _MEDIA_HTTP_METHODS = ("GET", "HEAD", "OPTIONS") LOGIN_BAN_TTL_SEC = int(os.environ.get("VOD_LOGIN_BAN_TTL_SEC", "1800")) # OBS browser sources historically pass overlay tokens in the query string. @@ -856,67 +850,8 @@ def wrapper(*args, **kwargs): return wrapper -def _cast_serializer(): - return URLSafeTimedSerializer(str(app.secret_key), salt=CAST_TOKEN_SALT) - - -_CAST_TOKEN_CACHE = {} - -def issue_cast_token(kind, streamer, name="", *, obs=False): - cache_key = f"{kind}:{streamer}:{name}:{obs}" - import time - now = time.time() - if cache_key in _CAST_TOKEN_CACHE: - token, ts = _CAST_TOKEN_CACHE[cache_key] - if now - ts < 86400: - return token - payload = {"k": str(kind or ""), "s": str(streamer or ""), "n": str(name or "")} - if obs: - payload["o"] = 1 - token = _cast_serializer().dumps(payload) - _CAST_TOKEN_CACHE[cache_key] = (token, now) - return token - - -def verify_cast_token(token, kind, streamer, name=""): - token = (token or "").strip() - if not token: - return False - try: - loaded = _cast_serializer().loads(token, max_age=None, return_timestamp=True) - except (BadSignature, SignatureExpired, TypeError, ValueError): - return False - if not isinstance(loaded, tuple) or len(loaded) != 2: - return False - data, issued_at = loaded - if not isinstance(data, dict): - return False - try: - issued_ts = issued_at.timestamp() if isinstance(issued_at, datetime) else float(issued_at) - age = time.time() - issued_ts - except (TypeError, ValueError, OSError, AttributeError): - return False - limit = OBS_CAST_TTL_SEC if data.get("o") else CAST_TOKEN_TTL_SEC - if int(limit) > 0 and age > int(limit): - return False - return ( - str(data.get("k") or "") == str(kind or "") - and str(data.get("s") or "") == str(streamer or "") - and str(data.get("n") or "") == str(name or "") - ) - - -def url_with_cast(endpoint, kind, streamer, name="", **kwargs): - params = dict(kwargs) - params["streamer"] = streamer - if name: - params.setdefault("filename", name) - params["cast"] = issue_cast_token(kind, streamer, name) - return url_for(endpoint, **params) - - def absolute_public_url(path): - """Absolute https URL for TVs. Chromecast/AirPlay do not resolve app-relative paths reliably.""" + """Absolute URL for links that leave the browser session (share links, OAuth callbacks).""" path = str(path or "") if path.startswith("http://") or path.startswith("https://"): return path @@ -937,26 +872,13 @@ def absolute_public_url(path): return f"{proto}://{host}{path}" -def cast_media_url(endpoint, **kwargs): - return absolute_public_url(url_for(endpoint, **kwargs)) - - def hls_playlist_response(body, *, live=False): - """Playlist MIME without charset — Chromecast rejects mpegurl; charset=utf-8.""" + """Playlist MIME without charset; some HLS players reject mpegurl; charset=utf-8.""" resp = Response(body, 200) resp.headers["Content-Type"] = "application/vnd.apple.mpegurl" return apply_hls_delivery_headers(resp, live=live) -def apply_cast_cors(resp): - resp.headers["Access-Control-Allow-Origin"] = "*" - resp.headers["Access-Control-Allow-Methods"] = "GET, HEAD, OPTIONS" - resp.headers["Access-Control-Allow-Headers"] = "Range, Origin, Accept, Content-Type" - resp.headers["Access-Control-Expose-Headers"] = "Content-Length, Content-Range, Accept-Ranges, Content-Type" - resp.headers["Access-Control-Max-Age"] = "86400" - return resp - - def hls_segment_mimetype(filename: str) -> str: """MIME for HLS media segments (Python's mimetypes maps .ts wrongly).""" name = str(filename or "").lower() @@ -1009,8 +931,7 @@ def rewrite_hls_playlist_for_client(lines, segment_url_for): def apply_hls_delivery_headers(resp, *, live: bool = False, immutable: bool = False): - """CORS + byte-range + cache headers for HLS playlist/segment responses.""" - apply_cast_cors(resp) + """Byte-range + cache headers for HLS playlist/segment responses.""" resp.headers["Accept-Ranges"] = "bytes" if live: resp.headers["Cache-Control"] = "no-cache, no-store, must-revalidate" @@ -1025,40 +946,24 @@ def apply_hls_delivery_headers(resp, *, live: bool = False, immutable: bool = Fa return resp -def _cast_query_kwargs(): - token = (request.args.get("cast") or "").strip() - if token: - return {"cast": token} - return {} - - -def wants_apple_airplay_player(): +def is_ios_client(): ua = (request.headers.get("User-Agent") or "").lower() return "iphone" in ua or "ipad" in ua or "ipod" in ua -def login_or_cast_required(kind, name_arg="filename"): - """Allow session cookies or a scoped ?cast= token (AirPlay/Chromecast have no cookies).""" - def decorator(func): - @wraps(func) - def wrapper(*args, **kwargs): - if request.method == "OPTIONS": - return apply_cast_cors(Response(status=204)) - streamer = str(kwargs.get("streamer") or "") - if kind == "live" or name_arg is None: - name = "" - else: - name = str(kwargs.get(name_arg) or "") - token = (request.args.get("cast") or "").strip() - if token and verify_cast_token(token, kind, streamer, name): - return func(*args, **kwargs) - if session.get("logged_in") and not session_is_expired(): - return func(*args, **kwargs) - if session.get("logged_in"): - session.clear() - return apply_cast_cors(Response("Unauthorized", 401, mimetype="text/plain")) - return wrapper - return decorator +def media_login_required(func): + """Session-only auth for media endpoints: 401 instead of a login redirect, + because players fetch these URLs directly and cannot follow an HTML login.""" + @wraps(func) + def wrapper(*args, **kwargs): + if request.method == "OPTIONS": + return Response(status=204) + if session.get("logged_in") and not session_is_expired(): + return func(*args, **kwargs) + if session.get("logged_in"): + session.clear() + return Response("Unauthorized", 401, mimetype="text/plain") + return wrapper def admin_required(func): @@ -3531,7 +3436,7 @@ def generate(): "X-Accel-Buffering": "no", } resp = Response(generate(), status=206, headers=headers, mimetype="video/mp4") - return apply_cast_cors(resp) + return resp def remux_media_file(file_path, reencode_audio=False): @@ -4056,13 +3961,7 @@ def current_username(): def vod_slot_holder(): - name = str(session.get("username") or "").strip() - if name: - return name - token = (request.args.get("cast") or "").strip() - if token: - return f"cast:{token[:32]}" - return "" + return str(session.get("username") or "").strip() def vod_slot_exempt(): @@ -9773,13 +9672,7 @@ def vod_view(streamer, filename): except OSError: hls_ready = False else: - hls_playlist_url = url_with_cast( - "hls_playlist", - "hls", - streamer, - filename, - v=hls_version, - ) + hls_playlist_url = url_for("hls_playlist", streamer=streamer, filename=filename, v=hls_version) transcript_text = load_transcript(file_path) if file_path else None transcript_status = None if file_path: @@ -9814,7 +9707,7 @@ def vod_view(streamer, filename): MOBILE_PLAYER_TEMPLATE, streamer=streamer, filename=filename, - media_url=url_with_cast("media", "vod", streamer, filename), + media_url=url_for("media", streamer=streamer, filename=filename), chat_url=url_for("chat_vod", streamer=streamer, filename=filename) if chat_available else None, back_url=url_for("streamer_view", streamer=streamer), is_live=False, @@ -9840,7 +9733,7 @@ def vod_view(streamer, filename): clip_error=clip_error, share_link=share_link, action_msg=action_msg, - media_url=url_with_cast("media", "vod", streamer, filename), + media_url=url_for("media", streamer=streamer, filename=filename), hls_playlist_url=hls_playlist_url, hls_ready=False, download_url=( @@ -10031,7 +9924,7 @@ def clip_view(streamer, filename): MOBILE_PLAYER_TEMPLATE, streamer=streamer, filename=filename, - media_url=url_with_cast("clip_media", "clip", streamer, filename), + media_url=url_for("clip_media", streamer=streamer, filename=filename), chat_url=url_for("chat_vod", streamer=streamer, filename=filename), # Clips might not have synced chat easily, but we leave it back_url=url_for("streamer_view", streamer=streamer), is_live=False, @@ -10056,7 +9949,7 @@ def clip_view(streamer, filename): clip_error=None, action_msg=None, share_link=share_link, - media_url=url_with_cast("clip_media", "clip", streamer, filename), + media_url=url_for("clip_media", streamer=streamer, filename=filename), hls_playlist_url=None, hls_ready=False, download_url=( @@ -10127,13 +10020,10 @@ def thumb_view(streamer, filename): @app.route("/media//", methods=_MEDIA_HTTP_METHODS) -@login_or_cast_required("vod") +@media_login_required def media(streamer, filename): abort_vod_denied() - token = (request.args.get("cast") or "").strip() - cast_ok = bool(token and verify_cast_token(token, "vod", streamer, filename)) - if not cast_ok: - abort_hidden_streamer(streamer) + abort_hidden_streamer(streamer) busy = enforce_vod_watch_slot(streamer, filename) if busy: logging.info( @@ -10253,13 +10143,10 @@ def download(streamer, filename): @app.route("/clip/media//", methods=_MEDIA_HTTP_METHODS) -@login_or_cast_required("clip") +@media_login_required def clip_media(streamer, filename): abort_vod_denied() - token = (request.args.get("cast") or "").strip() - cast_ok = bool(token and verify_cast_token(token, "clip", streamer, filename)) - if not cast_ok: - abort_hidden_streamer(streamer) + abort_hidden_streamer(streamer) file_path = resolve_media_path(streamer, filename, "clip") if not file_path: abort(404) @@ -10654,34 +10541,18 @@ def set_security_headers(response): "frame-src 'self' https://player.twitch.tv https://www.twitch.tv; " "child-src 'self' blob: https://player.twitch.tv https://www.twitch.tv; " "worker-src 'self' blob:; " - "script-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://www.gstatic.com; " + "script-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; " "style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; " "font-src 'self' https://fonts.gstatic.com data:; " - "img-src 'self' data: blob: https://static-cdn.jtvnw.net https://badges.twitch.tv https://cdn.discordapp.com https://www.gstatic.com; " + "img-src 'self' data: blob: https://static-cdn.jtvnw.net https://badges.twitch.tv https://cdn.discordapp.com; " "media-src 'self' blob:; " - "connect-src 'self' blob: https://www.gstatic.com https://*.googleapis.com https://clients3.google.com;" + "connect-src 'self' blob: https://*.googleapis.com https://clients3.google.com;" ) header = "Content-Security-Policy" if os.environ.get("VOD_CSP_ENFORCE", "false").lower() in ("true", "1", "yes") else "Content-Security-Policy-Report-Only" response.headers.setdefault(header, csp) return response -@app.after_request -def add_cast_cors_headers(response): - path = request.path or "" - if ( - path.startswith("/media/") - or path.startswith("/clip/media/") - or path.startswith("/public/media/") - or path.startswith("/hls/playlist/") - or path.startswith("/hls/segment/") - or path.startswith("/live/playlist/") - or path.startswith("/live/segment/") - ): - apply_cast_cors(response) - return response - - @app.route("/share/create//", methods=["POST"]) @admin_required def share_create(streamer, filename): @@ -11131,12 +11002,9 @@ def _live_chat_log_path(streamer): @app.route("/chat/live/") -@login_or_cast_required("live", name_arg=None) +@media_login_required def chat_live(streamer): - token = (request.args.get("cast") or "").strip() - cast_ok = bool(token and verify_cast_token(token, "live", streamer, "")) - if not cast_ok: - abort_hidden_streamer(streamer) + abort_hidden_streamer(streamer) # Prefer the live chat jsonl (wall-clock `t`) so channel-point / mod events # from the IRC chat process show up immediately in Live/Watch views. log_path = _live_chat_log_path(streamer) @@ -11196,7 +11064,7 @@ def editor_view(streamer, filename): CLIP_EDITOR_TEMPLATE, streamer=streamer, filename=filename, - media_url=url_with_cast("media", "vod", streamer, filename), + media_url=url_for("media", streamer=streamer, filename=filename), back_url=url_for("vod_view", streamer=streamer, filename=filename), clip_ok=request.args.get("clip_ok"), clip_error=request.args.get("clip_error"), @@ -11211,7 +11079,7 @@ def editor_live_view(streamer): CLIP_EDITOR_TEMPLATE, streamer=streamer, filename=f"LIVE {streamer}", - media_url=url_with_cast("live_playlist", "live", streamer), + media_url=url_for("live_playlist", streamer=streamer), back_url=url_for("live_view", streamer=streamer), clip_ok=request.args.get("clip_ok"), clip_error=request.args.get("clip_error"), @@ -11592,7 +11460,7 @@ def hls_view(streamer, filename): abort(404) _, playlist, _ = hls_paths(streamer, filename) ready = os.path.isfile(playlist) - playlist_url = url_with_cast("hls_playlist", "hls", streamer, filename) if ready else "" + playlist_url = url_for("hls_playlist", streamer=streamer, filename=filename) if ready else "" return render_template_string( HLS_TEMPLATE, streamer=streamer, @@ -11614,13 +11482,10 @@ def hls_generate(streamer, filename): @app.route("/hls/playlist//", methods=_MEDIA_HTTP_METHODS) -@login_or_cast_required("hls") +@media_login_required def hls_playlist(streamer, filename): abort_vod_denied() - token = (request.args.get("cast") or "").strip() - cast_ok = bool(token and verify_cast_token(token, "hls", streamer, filename)) - if not cast_ok: - abort_hidden_streamer(streamer) + abort_hidden_streamer(streamer) busy = enforce_vod_watch_slot(streamer, filename) if busy: return busy @@ -11636,32 +11501,18 @@ def hls_playlist(streamer, filename): except OSError: abort(404) - cast_kwargs = _cast_query_kwargs() - def segment_url_for(seg): - params = dict( - streamer=streamer, - filename=filename, - segment=seg, - v=hls_version, - **cast_kwargs, - ) - if cast_kwargs: - return cast_media_url("hls_segment", **params) - return url_for("hls_segment", **params) + return url_for("hls_segment", streamer=streamer, filename=filename, segment=seg, v=hls_version) output = rewrite_hls_playlist_for_client(lines, segment_url_for) return hls_playlist_response("".join(output), live=False) @app.route("/hls/segment///", methods=_MEDIA_HTTP_METHODS) -@login_or_cast_required("hls") +@media_login_required def hls_segment(streamer, filename, segment): abort_vod_denied() - token = (request.args.get("cast") or "").strip() - cast_ok = bool(token and verify_cast_token(token, "hls", streamer, filename)) - if not cast_ok: - abort_hidden_streamer(streamer) + abort_hidden_streamer(streamer) busy = enforce_vod_watch_slot(streamer, filename) if busy: return busy @@ -11767,31 +11618,6 @@ def api_live_client_debug(): return jsonify({"ok": True}) -@app.route("/api/live/cast-url/") -@login_required -def api_live_cast_url(streamer): - """Start live HLS if needed and return a cookie-free playlist URL for AirPlay/Chromecast.""" - channel = normalize_channel_name(streamer) or twitch_api.normalize_login(streamer) - if not channel: - abort(404) - abort_hidden_streamer(channel) - try: - add_live_channel(channel) - except Exception: - app.logger.exception("Could not enable live proxy for cast %s", channel) - return jsonify({"ok": False, "ready": False, "url": ""}), 500 - touch_live_heartbeat(channel) - ready = live_playlist_fresh(channel) - return jsonify({ - "ok": True, - "ready": ready, - "channel": channel, - "url": url_with_cast("live_playlist", "live", channel), - "contentType": "application/vnd.apple.mpegurl", - "live": True, - }) - - @app.route("/api/live/ready/") @login_required def api_live_ready(streamer): @@ -11821,7 +11647,7 @@ def api_live_ready(streamer): "title": (info.get("title") if isinstance(info, dict) else "") or "", "game_name": (info.get("game_name") if isinstance(info, dict) else "") or "", "viewer_count": (info.get("viewer_count") if isinstance(info, dict) else 0) or 0, - "playlist_url": url_with_cast("live_playlist", "live", channel) if ready else "", + "playlist_url": url_for("live_playlist", streamer=channel) if ready else "", }) @@ -11876,7 +11702,7 @@ def twitch_watch(channel): use_hls_proxy = True elif proxy_arg in ("0", "false", "no", "off") or embed_arg in ("1", "true", "yes", "on"): use_hls_proxy = False - elif is_mobile() or wants_apple_airplay_player(): + elif is_mobile() or is_ios_client(): use_hls_proxy = True elif config.LIVE_TWITCH_OAUTH_ENABLED: use_hls_proxy = True @@ -11926,10 +11752,9 @@ def twitch_watch(channel): "use_hls_proxy": use_hls_proxy, "live_debug": live_debug, "proxy_key": proxy_key, - "hls_playlist_url": url_with_cast("live_playlist", "live", proxy_key) if use_hls_proxy else "", + "hls_playlist_url": url_for("live_playlist", streamer=proxy_key) if use_hls_proxy else "", "hls_ready_url": url_for("api_live_ready", streamer=proxy_key) if use_hls_proxy else "", "hls_client_debug_url": url_for("api_live_client_debug"), - "cast_ensure_url": url_for("api_live_cast_url", streamer=login), "watch_embed_url": watch_embed_url, "watch_proxy_url": watch_proxy_url, "live_oauth_enabled": bool(config.LIVE_TWITCH_OAUTH_ENABLED), @@ -12032,7 +11857,7 @@ def live_view(streamer): app.logger.exception("Could not enable live channel for studio %s", channel) touch_live_heartbeat(channel) ready = live_playlist_exists(channel) - playlist_url = url_with_cast("live_playlist", "live", channel) + playlist_url = url_for("live_playlist", streamer=channel) if is_mobile(): return render_template_string( @@ -12066,25 +11891,19 @@ def live_view(streamer): @app.route("/live/obs/") -@login_or_cast_required("live", name_arg=None) +@login_required def live_obs(streamer): """Fullscreen picture and sound for an OBS browser source.""" channel = normalize_channel_name(streamer) or "" if not channel or channel != str(streamer or "").strip().lower(): abort(404) - token = (request.args.get("cast") or "").strip() - cast_ok = bool(token and verify_cast_token(token, "live", channel, "")) - if not cast_ok: - abort_hidden_streamer(channel) + abort_hidden_streamer(channel) try: add_live_channel(channel) except Exception: app.logger.exception("Could not enable live proxy for obs %s", channel) touch_live_heartbeat(channel) - if cast_ok: - playlist_url = url_for("live_playlist", streamer=channel, cast=token) - else: - playlist_url = url_with_cast("live_playlist", "live", channel) + playlist_url = url_for("live_playlist", streamer=channel) resp = make_response(render_template_string( LIVE_OBS_TEMPLATE, playlist_url=playlist_url, @@ -12094,20 +11913,16 @@ def live_obs(streamer): return resp @app.route("/chat/obs/") -@login_or_cast_required("live", name_arg=None) +@login_required def chat_obs(streamer): """Fullscreen transparent chat overlay for an OBS browser source.""" channel = normalize_channel_name(streamer) or "" if not channel or channel != str(streamer or "").strip().lower(): abort(404) - token = (request.args.get("cast") or "").strip() - cast_ok = bool(token and verify_cast_token(token, "live", channel, "")) - if not cast_ok: - abort_hidden_streamer(channel) + abort_hidden_streamer(channel) resp = make_response(render_template_string( CHAT_OBS_TEMPLATE, streamer=channel, - cast_token=token if cast_ok else "", )) resp.headers["Cache-Control"] = "no-store, no-cache, must-revalidate" resp.headers["Pragma"] = "no-cache" @@ -12116,12 +11931,9 @@ def chat_obs(streamer): @app.route("/live/playlist/", methods=_MEDIA_HTTP_METHODS) -@login_or_cast_required("live", name_arg=None) +@media_login_required def live_playlist(streamer): - token = (request.args.get("cast") or "").strip() - cast_ok = bool(token and verify_cast_token(token, "live", streamer, "")) - if not cast_ok: - abort_hidden_streamer(streamer) + abort_hidden_streamer(streamer) touch_live_heartbeat(streamer) if not live_playlist_fresh(streamer): abort(404) @@ -12136,11 +11948,7 @@ def live_playlist(streamer): except Exception: abort(404) - cast_kwargs = _cast_query_kwargs() - def segment_url_for(seg): - if cast_kwargs: - return cast_media_url("live_segment", streamer=streamer, segment=seg, **cast_kwargs) return url_for("live_segment", streamer=streamer, segment=seg) output = rewrite_hls_playlist_for_client(lines, segment_url_for) @@ -12148,12 +11956,9 @@ def segment_url_for(seg): @app.route("/live/segment//", methods=_MEDIA_HTTP_METHODS) -@login_or_cast_required("live", name_arg=None) +@media_login_required def live_segment(streamer, segment): - token = (request.args.get("cast") or "").strip() - cast_ok = bool(token and verify_cast_token(token, "live", streamer, "")) - if not cast_ok: - abort_hidden_streamer(streamer) + abort_hidden_streamer(streamer) touch_live_heartbeat(streamer) if os.path.basename(segment) != segment: abort(404) diff --git a/tests/test_recorder_reliability.py b/tests/test_recorder_reliability.py index 2f7eed0..323910b 100644 --- a/tests/test_recorder_reliability.py +++ b/tests/test_recorder_reliability.py @@ -601,6 +601,38 @@ def test_maintenance_has_no_default_bypass_user(self): self.assertEqual(site_maintenance._default_state()["bypass_users"], []) +class CastingRemovedTests(unittest.TestCase): + """Media is session-only: no ?cast= bearer tokens, no wildcard CORS.""" + + @classmethod + def setUpClass(cls): + from WEBUI import webui + webui.app.config.update(TESTING=True) + cls.client = webui.app.test_client() + + def test_cast_token_does_not_unlock_media(self): + for path in ( + "/media/alice/video.mp4?cast=forged", + "/hls/playlist/alice/video.mp4?cast=forged", + "/live/playlist/alice?cast=forged", + "/chat/live/alice?cast=forged", + ): + with self.subTest(path=path): + response = self.client.get(path) + self.assertEqual(response.status_code, 401) + self.assertNotIn("Access-Control-Allow-Origin", response.headers) + + def test_cast_url_endpoint_is_gone(self): + self.assertEqual(self.client.get("/api/live/cast-url/alice").status_code, 404) + + def test_obs_pages_require_login(self): + for path in ("/live/obs/alice", "/chat/obs/alice"): + with self.subTest(path=path): + response = self.client.get(path + "?cast=forged") + self.assertEqual(response.status_code, 302) + self.assertIn("/login", response.headers["Location"]) + + class DiskSpaceGuardTests(IsolatedBaseTest): def test_disk_blocks_recording_on_nonexistent_dir_when_disk_full(self): from collections import namedtuple From c3cc1759428f087edebc2edea825ae817fffb318 Mon Sep 17 00:00:00 2001 From: Pierre Date: Tue, 3 Nov 2026 17:26:18 +0100 Subject: [PATCH 17/31] fix(webui): stop client IP spoofing via X-Forwarded-For get_client_ip() took the left-most X-Forwarded-For hop, which the client controls: nginx's $proxy_add_x_forwarded_for only appends. Rotating that header bypassed Flask-Limiter, the login attempt counter and IP bans, and let an attacker get a victim's IP banned. Behind a trusted proxy the app now uses X-Real-IP (set by both shipped nginx configs) or the right-most X-Forwarded-For hop, validated as an IP. VOD_TRUSTED_PROXIES accepts CIDR networks; docker-compose trusts the private ranges of its network, since the app port is not published and all users otherwise shared the nginx container's IP for bans. Co-Authored-By: Claude Opus 5.5 --- .env.example | 4 ++ WEBUI/webui.py | 67 ++++++++++++++++++++++-------- docker-compose.yml | 3 ++ tests/test_recorder_reliability.py | 35 ++++++++++++++++ 4 files changed, 91 insertions(+), 18 deletions(-) diff --git a/.env.example b/.env.example index 997af42..07057b6 100644 --- a/.env.example +++ b/.env.example @@ -6,6 +6,10 @@ # --- Core WebUI Server Settings --- VOD_HOST=0.0.0.0 VOD_PORT=8080 +# Reverse proxies whose X-Real-IP / X-Forwarded-For headers are trusted +# (comma-separated IPs or CIDR networks). Rate limits and login bans use the +# client IP these proxies report; never list addresses clients can reach directly. +VOD_TRUSTED_PROXIES=127.0.0.1,::1 # Public base URL used for share links and OAuth callbacks. # Leave empty to derive it from the incoming request (recommended behind a proxy). # For OAuth providers set the exact public HTTPS origin, e.g. https://recorder.example.com diff --git a/WEBUI/webui.py b/WEBUI/webui.py index 85496ab..1902d43 100644 --- a/WEBUI/webui.py +++ b/WEBUI/webui.py @@ -2,6 +2,7 @@ import bisect import glob +import ipaddress import json import logging import math @@ -201,11 +202,21 @@ def _resolve_ratelimit_storage_uri(raw: str) -> str: os.environ.get("VOD_RATELIMIT_STORAGE_URI", "memory://") ) # Default credentials removed for security - only environment variables are allowed -TRUSTED_PROXIES = { - ip.strip() - for ip in os.environ.get("VOD_TRUSTED_PROXIES", "127.0.0.1,::1").split(",") - if ip.strip() -} +def _parse_trusted_proxies(raw): + """Comma-separated IPs or CIDR networks, e.g. ``127.0.0.1,::1,172.16.0.0/12``.""" + networks = [] + for item in str(raw or "").split(","): + item = item.strip() + if not item: + continue + try: + networks.append(ipaddress.ip_network(item, strict=False)) + except ValueError: + logging.error("Ignoring invalid VOD_TRUSTED_PROXIES entry: %r", item) + return tuple(networks) + + +TRUSTED_PROXY_NETWORKS = _parse_trusted_proxies(os.environ.get("VOD_TRUSTED_PROXIES", "127.0.0.1,::1")) CHAT_INDEX_MIN_SIZE = int(os.environ.get("VOD_CHAT_INDEX_MIN_SIZE", "5000000")) # 5 MB CHAT_INDEX_STEP = int(os.environ.get("VOD_CHAT_INDEX_STEP", "200")) MAX_CHAT_SEARCH_RESULTS = int(os.environ.get("VOD_CHAT_SEARCH_LIMIT", "500")) @@ -409,21 +420,41 @@ def get_sys_stats(): disk = storage_info() return {"cpu": cpu_usage, "ram": ram_pct, "disk": int(disk['pct']) if disk else 0} +def _is_trusted_proxy(addr): + try: + ip = ipaddress.ip_address(str(addr or "")) + except ValueError: + return False + return any(ip in net for net in TRUSTED_PROXY_NETWORKS) + + +def _valid_ip(value): + try: + return str(ipaddress.ip_address(str(value or "").strip())) + except ValueError: + return "" + + def get_client_ip(): - """Extract client IP with secure proxy handling.""" + """Client IP for rate limits and bans. + + Proxy headers are honoured only when the direct peer is a trusted proxy, and + only the values that proxy itself wrote: X-Real-IP, else the right-most + X-Forwarded-For hop. The left-most hop is client-controlled and must never be + used, otherwise rotating it bypasses login rate limits. + """ remote = request.remote_addr - - # Only trust X-Forwarded-For from trusted proxies - if remote in TRUSTED_PROXIES: - xff = request.headers.get("X-Forwarded-For") - if xff: - # Take first IP in chain (client IP) and validate - client_ip = xff.split(",")[0].strip() - # Basic validation to prevent header injection - if client_ip and len(client_ip) < 45: # Max IPv6 length - return client_ip - - return request.remote_addr + if not _is_trusted_proxy(remote): + return remote + real_ip = _valid_ip(request.headers.get("X-Real-IP")) + if real_ip: + return real_ip + hops = [h.strip() for h in (request.headers.get("X-Forwarded-For") or "").split(",") if h.strip()] + if hops: + last_hop = _valid_ip(hops[-1]) + if last_hop: + return last_hop + return remote def check_rate_limit(ip): """Return True if IP is under the failed-login threshold (shared via SQLite).""" diff --git a/docker-compose.yml b/docker-compose.yml index b1e75c7..c19cd83 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,6 +29,9 @@ services: - PYTHONPATH=/app - VOD_HOST=0.0.0.0 - VOD_PORT=8080 + # The app port is not published; only the nginx container on the compose + # network reaches it, so trust that network's private ranges for X-Real-IP. + - VOD_TRUSTED_PROXIES=127.0.0.1,::1,172.16.0.0/12,192.168.0.0/16,10.0.0.0/8 - VOD_BASE_DIR=/app - RECORDINGS_BASE_DIR=/app - VOD_USER=${VOD_USER:?set VOD_USER in .env} diff --git a/tests/test_recorder_reliability.py b/tests/test_recorder_reliability.py index 323910b..44c020b 100644 --- a/tests/test_recorder_reliability.py +++ b/tests/test_recorder_reliability.py @@ -601,6 +601,41 @@ def test_maintenance_has_no_default_bypass_user(self): self.assertEqual(site_maintenance._default_state()["bypass_users"], []) +class ClientIpTests(unittest.TestCase): + """Rate limits and bans key on get_client_ip(); it must not be spoofable.""" + + @classmethod + def setUpClass(cls): + from WEBUI import webui + cls.webui = webui + + def _ip(self, remote, headers=None): + with self.webui.app.test_request_context( + "/", headers=headers or {}, environ_base={"REMOTE_ADDR": remote} + ): + return self.webui.get_client_ip() + + def test_untrusted_peer_ignores_proxy_headers(self): + ip = self._ip("203.0.113.9", {"X-Forwarded-For": "1.2.3.4", "X-Real-IP": "1.2.3.4"}) + self.assertEqual(ip, "203.0.113.9") + + def test_trusted_proxy_uses_x_real_ip(self): + ip = self._ip("127.0.0.1", {"X-Real-IP": "198.51.100.7", "X-Forwarded-For": "6.6.6.6, 198.51.100.7"}) + self.assertEqual(ip, "198.51.100.7") + + def test_spoofed_left_most_forwarded_hop_is_ignored(self): + ip = self._ip("127.0.0.1", {"X-Forwarded-For": "6.6.6.6, 198.51.100.7"}) + self.assertEqual(ip, "198.51.100.7") + + def test_garbage_headers_fall_back_to_peer(self): + ip = self._ip("127.0.0.1", {"X-Real-IP": "not-an-ip", "X-Forwarded-For": "also bad"}) + self.assertEqual(ip, "127.0.0.1") + + def test_trusted_proxies_accept_cidr(self): + nets = self.webui._parse_trusted_proxies("10.0.0.0/8, ::1, bogus") + self.assertEqual([str(n) for n in nets], ["10.0.0.0/8", "::1/128"]) + + class CastingRemovedTests(unittest.TestCase): """Media is session-only: no ?cast= bearer tokens, no wildcard CORS.""" From d07a19e3df50d0be58c4554dc0df5574befb0f2c Mon Sep 17 00:00:00 2001 From: Pierre Date: Tue, 3 Nov 2026 17:27:26 +0100 Subject: [PATCH 18/31] fix(content): add frontmatter to the privacy policy core.tube_terms requires a frontmatter block with privacyVersion in content/datenschutz.md. The shipped file had none, so /datenschutz and /agb (which embeds the privacy version) both answered HTTP 500 on every fresh install. Drop the leading H1 as well; the renderer only handles H2/H3 and takes the title from the frontmatter. Co-Authored-By: Claude Opus 5.5 --- content/datenschutz.md | 6 +++++- tests/test_recorder_reliability.py | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/content/datenschutz.md b/content/datenschutz.md index c5bece2..0a05fbc 100644 --- a/content/datenschutz.md +++ b/content/datenschutz.md @@ -1,4 +1,8 @@ -# Datenschutzerklärung +--- +privacyVersion: 2026-11-03-v1 +effectiveDate: 2026-11-03 +title: Datenschutzerklärung – StreamVault +--- ## 1. Erhebung und Speicherung von Daten Bei der Nutzung dieses Systems werden Zugriffsdaten (IP-Adresse, Zeitpunkt des Abrufs, aufgerufene Seiten) zur Gewährleistung der Systemsicherheit und des störungsfreien Betriebs in lokalen Protokolldateien erfasst. diff --git a/tests/test_recorder_reliability.py b/tests/test_recorder_reliability.py index 44c020b..409ec72 100644 --- a/tests/test_recorder_reliability.py +++ b/tests/test_recorder_reliability.py @@ -601,6 +601,28 @@ def test_maintenance_has_no_default_bypass_user(self): self.assertEqual(site_maintenance._default_state()["bypass_users"], []) +class LegalPagesTests(unittest.TestCase): + """The shipped content/*.md must parse; /agb and /datenschutz returned 500 before.""" + + def test_shipped_documents_parse(self): + from core import tube_terms + self.assertTrue(tube_terms.load_document(force=True)["terms_version"]) + self.assertTrue(tube_terms.load_privacy_document(force=True)["privacy_version"]) + self.assertIn("privacy_url", tube_terms.terms_web_context(site_url="")) + + def test_legal_pages_render(self): + from WEBUI import webui + webui.app.config.update(TESTING=True) + client = webui.app.test_client() + with client.session_transaction() as sess: + now = time.time() + sess.update(logged_in=True, user_role="admin", username=config.VOD_USER, + login_time=now, last_active=now, _csrf_token="t") + for path in ("/agb", "/datenschutz"): + with self.subTest(path=path): + self.assertEqual(client.get(path).status_code, 200) + + class ClientIpTests(unittest.TestCase): """Rate limits and bans key on get_client_ip(); it must not be spoofable.""" From d8f636787b47683dd65a168d217d754ee529f597 Mon Sep 17 00:00:00 2001 From: Pierre Date: Tue, 3 Nov 2026 17:28:05 +0100 Subject: [PATCH 19/31] fix(worker): fail closed when checking for an active recording _is_current_recording() returned False on any error, and skipped half-written .current_recording*.json pointers. Its callers then treated a recording that was still being written as finished and remuxed it. os.kill() raising PermissionError for a recorder owned by another user was also read as "process gone". Unreadable pointers, PermissionError and directory errors now count as active and are logged. Only a removed pointer or a dead PID clears it. Co-Authored-By: Claude Opus 5.5 --- recorder/worker.py | 31 +++++++++++++++++----- tests/test_recorder_reliability.py | 41 ++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 7 deletions(-) diff --git a/recorder/worker.py b/recorder/worker.py index 6baf460..b68088d 100644 --- a/recorder/worker.py +++ b/recorder/worker.py @@ -1003,7 +1003,11 @@ def _is_recent(path, seconds=120): return False def _is_current_recording(channel_dir, filename): - """True if any current-recording pointer claims filename, or .part + active lock.""" + """True if any current-recording pointer claims filename, or .part + active lock. + + Fails closed: callers skip remux/cleanup on True, so an unreadable pointer or + an unexpected error must count as "still recording", never as "finished". + """ filename = os.path.basename(str(filename or "")) if not filename or not channel_dir or not os.path.isdir(channel_dir): return False @@ -1016,10 +1020,17 @@ def _is_current_recording(channel_dir, filename): try: with open(cur_path, "r", encoding="utf-8") as f: data = json.load(f) - if isinstance(data, dict) and data.get("filename") == filename: - return True - except Exception: - continue + except FileNotFoundError: + continue # pointer removed between listdir and open + except (OSError, ValueError) as exc: + # Half-written pointer while a recording starts or rotates. + logging.warning( + "Unreadable recording pointer %s, treating %s as active: %s", + cur_path, filename, exc, + ) + return True + if isinstance(data, dict) and data.get("filename") == filename: + return True part_path = os.path.join(channel_dir, f"{filename}.part") if not os.path.exists(part_path): @@ -1038,11 +1049,17 @@ def _is_current_recording(channel_dir, filename): pid = int(content) os.kill(pid, 0) return True + except PermissionError: + return True # process exists but belongs to another user except (OSError, ValueError, TypeError): continue return False - except Exception: - return False + except OSError as exc: + logging.error( + "Could not check active recording for %s in %s, treating as active: %s", + filename, channel_dir, exc, + ) + return True def _is_remuxable_vod_filename(filename): name = os.path.basename(str(filename or "")) diff --git a/tests/test_recorder_reliability.py b/tests/test_recorder_reliability.py index 409ec72..24998b0 100644 --- a/tests/test_recorder_reliability.py +++ b/tests/test_recorder_reliability.py @@ -601,6 +601,47 @@ def test_maintenance_has_no_default_bypass_user(self): self.assertEqual(site_maintenance._default_state()["bypass_users"], []) +class CurrentRecordingFailClosedTests(IsolatedBaseTest): + """_is_current_recording must never report a live file as finished.""" + + def _channel(self): + channel = self.base / "alice" + channel.mkdir() + return channel + + def test_pointer_claims_file(self): + channel = self._channel() + (channel / ".current_recording.json").write_text('{"filename": "a.mp4"}', encoding="utf-8") + self.assertTrue(worker._is_current_recording(str(channel), "a.mp4")) + self.assertFalse(worker._is_current_recording(str(channel), "b.mp4")) + + def test_half_written_pointer_counts_as_active(self): + channel = self._channel() + (channel / ".current_recording.json").write_text('{"filename": "a.mp', encoding="utf-8") + with self.assertLogs(level="WARNING"): + self.assertTrue(worker._is_current_recording(str(channel), "a.mp4")) + + def test_lock_owned_by_other_user_counts_as_active(self): + channel = self._channel() + (channel / "a.mp4.part").write_bytes(b"x") + (channel / ".recording.lock").write_text("4242", encoding="utf-8") + with mock.patch.object(worker.os, "kill", side_effect=PermissionError): + self.assertTrue(worker._is_current_recording(str(channel), "a.mp4")) + + def test_dead_lock_pid_is_not_active(self): + channel = self._channel() + (channel / "a.mp4.part").write_bytes(b"x") + (channel / ".recording.lock").write_text("4242", encoding="utf-8") + with mock.patch.object(worker.os, "kill", side_effect=ProcessLookupError): + self.assertFalse(worker._is_current_recording(str(channel), "a.mp4")) + + def test_listing_error_counts_as_active(self): + channel = self._channel() + with mock.patch.object(worker.os, "listdir", side_effect=OSError("EIO")), \ + self.assertLogs(level="ERROR"): + self.assertTrue(worker._is_current_recording(str(channel), "a.mp4")) + + class LegalPagesTests(unittest.TestCase): """The shipped content/*.md must parse; /agb and /datenschutz returned 500 before.""" From f04f5bfd1384c9be4236668fe650fd03d226dc6c Mon Sep 17 00:00:00 2001 From: Pierre Date: Tue, 3 Nov 2026 17:28:49 +0100 Subject: [PATCH 20/31] fix(webui): keep corrupt JSON state instead of overwriting it load_json() returned the default on any exception. Favorites, history, profiles, shares and user settings are read-modify-written, so a truncated file silently came back as {} or [] and the next save replaced the user's data for good. A missing file still yields the default. A file that does not parse is moved to .corrupt- and logged before the default is returned; other read errors are raised instead of being mistaken for "empty". Co-Authored-By: Claude Opus 5.5 --- WEBUI/webui.py | 18 +++++++++++++++- tests/test_recorder_reliability.py | 33 ++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/WEBUI/webui.py b/WEBUI/webui.py index 1902d43..42b8177 100644 --- a/WEBUI/webui.py +++ b/WEBUI/webui.py @@ -1047,10 +1047,26 @@ def ensure_dir(path): def load_json(path, default): + """Read a JSON state file; a missing file yields ``default``. + + A corrupt file is moved aside as ``.corrupt-`` before returning + ``default``. Callers save the value back later, so returning the default while + leaving the file in place would silently overwrite the only copy of the data. + Read errors other than "missing" are raised for the same reason. + """ try: with open(path, "r", encoding="utf-8") as f: return json.load(f) - except Exception: + except FileNotFoundError: + return default + except ValueError as exc: # json.JSONDecodeError, UnicodeDecodeError + backup = f"{path}.corrupt-{int(time.time())}" + try: + os.replace(path, backup) + except OSError as move_exc: + logging.error("Corrupt JSON %s (%s); could not move it aside: %s", path, exc, move_exc) + raise + logging.error("Corrupt JSON %s (%s); moved to %s", path, exc, backup) return default diff --git a/tests/test_recorder_reliability.py b/tests/test_recorder_reliability.py index 24998b0..7c47f3e 100644 --- a/tests/test_recorder_reliability.py +++ b/tests/test_recorder_reliability.py @@ -642,6 +642,39 @@ def test_listing_error_counts_as_active(self): self.assertTrue(worker._is_current_recording(str(channel), "a.mp4")) +class LoadJsonTests(IsolatedBaseTest): + """A corrupt state file must survive instead of being overwritten by the default.""" + + @classmethod + def setUpClass(cls): + from WEBUI import webui + cls.webui = webui + + def test_missing_file_returns_default(self): + self.assertEqual(self.webui.load_json(str(self.base / "nope.json"), {"d": 1}), {"d": 1}) + + def test_valid_file_is_loaded(self): + path = self.base / "ok.json" + path.write_text('{"a": 1}', encoding="utf-8") + self.assertEqual(self.webui.load_json(str(path), {}), {"a": 1}) + + def test_corrupt_file_is_moved_aside_not_lost(self): + path = self.base / "favorites.json" + path.write_text('["half', encoding="utf-8") + with self.assertLogs(level="ERROR"): + self.assertEqual(self.webui.load_json(str(path), []), []) + self.assertFalse(path.exists()) + backups = list(self.base.glob("favorites.json.corrupt-*")) + self.assertEqual(len(backups), 1) + self.assertEqual(backups[0].read_text(encoding="utf-8"), '["half') + + def test_unreadable_file_raises(self): + path = self.base / "dir.json" + path.mkdir() + with self.assertRaises(OSError): + self.webui.load_json(str(path), {}) + + class LegalPagesTests(unittest.TestCase): """The shipped content/*.md must parse; /agb and /datenschutz returned 500 before.""" From 06e873dea9953dabaedb27fc5e02db5065a05298 Mon Sep 17 00:00:00 2001 From: Pierre Date: Tue, 3 Nov 2026 17:30:28 +0100 Subject: [PATCH 21/31] fix(worker): never delete code or dot-directories as archives Folder deletion only checked that the target is a direct child of BASE_DIR. BASE_DIR defaults to the application directory when ./data is missing (and was /app in Docker), so deleting a streamer named "core", "WEBUI" or ".meta" removed application code or the database. config.is_reserved_archive_name() now rejects dot-names and the app's top-level directories; delete_folder and empty_folder tasks and the delete route refuse them and log the attempt. Co-Authored-By: Claude Opus 5.5 --- WEBUI/webui.py | 3 +++ core/config.py | 18 ++++++++++++++++ recorder/worker.py | 6 ++++++ tests/test_recorder_reliability.py | 34 ++++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+) diff --git a/WEBUI/webui.py b/WEBUI/webui.py index 42b8177..340d568 100644 --- a/WEBUI/webui.py +++ b/WEBUI/webui.py @@ -12600,6 +12600,9 @@ def empty_folder_route(streamer): @admin_required def delete_folder_route(streamer): archive = archive_name_for(streamer) or normalize_channel_name(streamer) or streamer + if config.is_reserved_archive_name(archive): + logging.error("Refusing to delete reserved directory %r (requested for %r)", archive, streamer) + abort(400) # Sicherheitscheck: Pfad muss im Basis-Verzeichnis liegen folder_path = safe_join(config.BASE_DIR, archive) # Strenge Prüfung: Muss ein Verzeichnis sein und direkt unter BASE_DIR liegen diff --git a/core/config.py b/core/config.py index f8f4d3e..615f067 100644 --- a/core/config.py +++ b/core/config.py @@ -104,6 +104,24 @@ def streamlink_twitch_auth_args(token): _default_base = os.path.join(APP_DIR, "data") if os.path.isdir(os.path.join(APP_DIR, "data")) else APP_DIR BASE_DIR = os.environ.get("VOD_BASE_DIR", _default_base) + +# Archive folders are direct children of BASE_DIR. BASE_DIR may be the +# application directory itself, so these names must never be treated (and +# deleted) as a streamer archive. +_RESERVED_ARCHIVE_NAMES = frozenset({ + "__pycache__", "bin", "content", "core", "data", "docs", "files", + "recorder", "scripts", "tests", "webui", +}) + + +def is_reserved_archive_name(name): + """True for names that are not a deletable archive folder below BASE_DIR.""" + name = str(name or "").strip() + if not name or name.startswith("."): + return True + return name.lower() in _RESERVED_ARCHIVE_NAMES + + DB_FILE = os.path.join(BASE_DIR, "streamrecorder.db") FILES_DIR = os.environ.get("VOD_FILES_DIR", os.path.join(BASE_DIR, "files")) diff --git a/recorder/worker.py b/recorder/worker.py index b68088d..9d7fc35 100644 --- a/recorder/worker.py +++ b/recorder/worker.py @@ -674,6 +674,9 @@ def process_delete_folder(payload): if not _is_direct_child(config.BASE_DIR, folder_path): logging.error("Unsicherer delete_folder Pfad abgelehnt: %s", folder_path) return False + if config.is_reserved_archive_name(os.path.basename(os.path.normpath(folder_path))): + logging.error("delete_folder rejected for reserved directory: %s", folder_path) + return False if not streamer: streamer = os.path.basename(os.path.realpath(folder_path)) @@ -880,6 +883,9 @@ def process_empty_folder(payload): if not _is_direct_child(config.BASE_DIR, folder_path): logging.error("Unsicherer empty_folder Pfad abgelehnt: %s", folder_path) return False + if config.is_reserved_archive_name(os.path.basename(os.path.normpath(folder_path))): + logging.error("empty_folder rejected for reserved directory: %s", folder_path) + return False if _folder_has_active_recording(folder_path) or _folder_has_active_remux(folder_path): raise TaskDeferred( f"Ordner wird noch beschrieben: {folder_path}", delay_sec=120 diff --git a/tests/test_recorder_reliability.py b/tests/test_recorder_reliability.py index 7c47f3e..08212cc 100644 --- a/tests/test_recorder_reliability.py +++ b/tests/test_recorder_reliability.py @@ -642,6 +642,40 @@ def test_listing_error_counts_as_active(self): self.assertTrue(worker._is_current_recording(str(channel), "a.mp4")) +class ReservedArchiveDirTests(IsolatedBaseTest): + """BASE_DIR may be the app directory: code and dot-dirs are never archives.""" + + def test_reserved_names(self): + for name in ("core", "WEBUI", "recorder", "data", "files", ".meta", ".hls", "", ".."): + with self.subTest(name=name): + self.assertTrue(config.is_reserved_archive_name(name)) + for name in ("alice", "core_gaming", "webui2"): + with self.subTest(name=name): + self.assertFalse(config.is_reserved_archive_name(name)) + + def test_worker_refuses_to_delete_code_directory(self): + code_dir = self.base / "core" + code_dir.mkdir() + (code_dir / "config.py").write_text("x", encoding="utf-8") + with self.assertLogs(level="ERROR"): + self.assertFalse(worker.process_delete_folder({"path": str(code_dir), "streamer": "core"})) + with self.assertLogs(level="ERROR"): + self.assertFalse(worker.process_empty_folder({"path": str(code_dir)})) + self.assertTrue((code_dir / "config.py").exists()) + + def test_delete_route_rejects_reserved_name(self): + from WEBUI import webui + webui.app.config.update(TESTING=True) + client = webui.app.test_client() + with client.session_transaction() as sess: + now = time.time() + sess.update(logged_in=True, user_role="admin", username=config.VOD_USER, + login_time=now, last_active=now, _csrf_token="t") + with self.assertLogs(level="ERROR"): + response = client.post("/delete_folder/core", data={"csrf_token": "t"}) + self.assertEqual(response.status_code, 400) + + class LoadJsonTests(IsolatedBaseTest): """A corrupt state file must survive instead of being overwritten by the default.""" From 4c6bf324d1c64b79c4dc159f250ae7b5bd5bf034 Mon Sep 17 00:00:00 2001 From: Pierre Date: Tue, 3 Nov 2026 17:30:28 +0100 Subject: [PATCH 22/31] fix(docker): keep data out of the code tree, drop wildcard media CORS docker-compose ran with VOD_BASE_DIR=/app, so recordings, the SQLite database and caches were mixed into the bind-mounted source checkout. Data now lives in ./data (/app/data); the nginx X-Accel aliases follow. The protected nginx locations, the installer's live-HLS block and the INSTALL.md samples still sent Access-Control-Allow-Origin: * for casting, which no longer exists. Media is same-origin only. Existing Docker installs: move archives, streamrecorder.db and the dot-directories from the project root into ./data before upgrading. Co-Authored-By: Claude Opus 5.5 --- INSTALL.md | 3 --- README.md | 4 ++-- docker-compose.yml | 7 ++++--- install.sh | 1 - nginx.conf | 24 +++++++----------------- 5 files changed, 13 insertions(+), 26 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 5800585..9b366e6 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -264,7 +264,6 @@ server { output_buffers 2 1m; gzip off; add_header Accept-Ranges bytes always; - add_header Access-Control-Allow-Origin "*" always; } # Legacy archives below VOD_FILES_DIR (default VOD_BASE_DIR/files). @@ -276,7 +275,6 @@ server { output_buffers 2 1m; gzip off; add_header Accept-Ranges bytes always; - add_header Access-Control-Allow-Origin "*" always; } location /_protected_vodcache/ { @@ -319,7 +317,6 @@ server { } default_type application/octet-stream; add_header Cache-Control "no-cache, no-store, must-revalidate"; - add_header Access-Control-Allow-Origin "*" always; } } ``` diff --git a/README.md b/README.md index 07f80fe..6adbaef 100644 --- a/README.md +++ b/README.md @@ -139,8 +139,8 @@ Access the service at `http://localhost:8090`. The Compose stack runs the WebUI, recorder and background worker under supervisord and shares the `hls_live` volume with the Nginx container. The -application directory is bind-mounted at `/app`; recordings therefore live in -the project directory. Set `VOD_PUBLIC_BASE` in `.env` to the public URL used +application directory is bind-mounted at `/app`; recordings, the database and +caches live in `./data` (`VOD_BASE_DIR=/app/data`). Set `VOD_PUBLIC_BASE` in `.env` to the public URL used by browsers (for example `https://recorder.example.com`) so OAuth callbacks and share links are generated correctly. diff --git a/docker-compose.yml b/docker-compose.yml index c19cd83..098e419 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,7 +20,7 @@ services: # Docker SIGKILLs the container before FFmpeg finishes the MP4. stop_grace_period: 75s volumes: - # Application code and archive media (bind mount). + # Application code; archive media and the database live in ./data. - ./:/app # Keep the live HLS segments shared with the Nginx container. - hls_live:/app/.hls_live @@ -32,8 +32,9 @@ services: # The app port is not published; only the nginx container on the compose # network reaches it, so trust that network's private ranges for X-Real-IP. - VOD_TRUSTED_PROXIES=127.0.0.1,::1,172.16.0.0/12,192.168.0.0/16,10.0.0.0/8 - - VOD_BASE_DIR=/app - - RECORDINGS_BASE_DIR=/app + # Archives, database and caches live in ./data, never in the code tree. + - VOD_BASE_DIR=/app/data + - RECORDINGS_BASE_DIR=/app/data - VOD_USER=${VOD_USER:?set VOD_USER in .env} - VOD_PASS=${VOD_PASS:?set VOD_PASS in .env} - VOD_SECRET_KEY=${VOD_SECRET_KEY:?set VOD_SECRET_KEY in .env} diff --git a/install.sh b/install.sh index 4fe2343..60096b5 100644 --- a/install.sh +++ b/install.sh @@ -274,7 +274,6 @@ server { } default_type application/octet-stream; add_header Cache-Control "no-cache, no-store, must-revalidate"; - add_header Access-Control-Allow-Origin *; } } EOF diff --git a/nginx.conf b/nginx.conf index a3f717a..0e14df3 100644 --- a/nginx.conf +++ b/nginx.conf @@ -43,56 +43,50 @@ server { # --- X-Accel-Redirect media offload --- # WebUI paths are /_protected//; archives live directly - # below VOD_BASE_DIR (/app in the container). + # below VOD_BASE_DIR (/app/data in the container). location /_protected/ { internal; - alias /app/; + alias /app/data/; sendfile on; sendfile_max_chunk 1m; output_buffers 2 1m; gzip off; - add_header Access-Control-Allow-Origin "*" always; - add_header Access-Control-Expose-Headers "Content-Length, Content-Range, Accept-Ranges, Content-Type" always; add_header Accept-Ranges bytes always; } - # Legacy archives stored below VOD_FILES_DIR (default /app/files). + # Legacy archives stored below VOD_FILES_DIR (default /app/data/files). location /_protected_files/ { internal; - alias /app/files/; + alias /app/data/files/; sendfile on; sendfile_max_chunk 1m; output_buffers 2 1m; gzip off; - add_header Access-Control-Allow-Origin "*" always; - add_header Access-Control-Expose-Headers "Content-Length, Content-Range, Accept-Ranges, Content-Type" always; add_header Accept-Ranges bytes always; } location /_protected_vodcache/ { internal; - alias /app/.vodcache/; + alias /app/data/.vodcache/; sendfile on; sendfile_max_chunk 1m; output_buffers 2 1m; gzip off; - add_header Access-Control-Allow-Origin "*" always; - add_header Access-Control-Expose-Headers "Content-Length, Content-Range, Accept-Ranges, Content-Type" always; add_header Accept-Ranges bytes always; } location /_protected_thumbs/ { internal; - alias /app/.thumbs/; + alias /app/data/.thumbs/; expires 1d; add_header Cache-Control "public"; } location /_protected_hls/ { internal; - alias /app/.hls/; + alias /app/data/.hls/; types { application/vnd.apple.mpegurl m3u8; video/mp2t ts; @@ -100,8 +94,6 @@ server { } default_type application/octet-stream; add_header Cache-Control "no-cache"; - add_header Access-Control-Allow-Origin "*" always; - add_header Access-Control-Expose-Headers "Content-Length, Content-Range, Accept-Ranges, Content-Type" always; add_header Accept-Ranges bytes always; } @@ -116,8 +108,6 @@ server { } default_type application/octet-stream; add_header Cache-Control "no-cache, no-store, must-revalidate" always; - add_header Access-Control-Allow-Origin "*" always; - add_header Access-Control-Expose-Headers "Content-Length, Content-Range, Accept-Ranges, Content-Type" always; add_header Accept-Ranges bytes always; } From 8d4f9240727455da790d9a3b207a90ff1995b4db Mon Sep 17 00:00:00 2001 From: Pierre Date: Tue, 3 Nov 2026 17:33:16 +0100 Subject: [PATCH 23/31] fix: drop built-in third-party proxies and host-specific /root paths - Streamlink: the opt-in playlist proxy defaulted to a hardcoded list of third-party ad-free proxy hosts and /etc/streamrecorder/plugins. No proxy is built in anymore; TWITCH_PROXY_PLAYLIST must be set by the operator, otherwise nothing is added and a warning is logged. Remove the shadowed duplicate streamlink_twitch_auth_args(). - Backups: the list/download/delete routes defaulted to /root/streamrecorder_backups and "create" ran a script from a fixed /root path that is not part of this repo. BACKUP_DIR now defaults to /backups and the script comes from VOD_BACKUP_SCRIPT. Backup deletion errors are logged instead of swallowed. - Remove the unused DISCORD_LOGS_DIR default (/root/Discord Bots/logs). - tests: pin tube_terms paths in LegalPagesTests, which failed when an isolated test imported the module while APP_DIR was patched. Co-Authored-By: Claude Opus 5.5 --- .env.example | 12 ++++++++ WEBUI/webui.py | 27 ++++++++++------ recorder/Streamrecorder.py | 27 ++++++++-------- tests/test_recorder_reliability.py | 49 ++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 22 deletions(-) diff --git a/.env.example b/.env.example index 07057b6..7cc1fee 100644 --- a/.env.example +++ b/.env.example @@ -65,10 +65,22 @@ VOD_USE_X_ACCEL=true # LIVE_TWITCH_OAUTH= # LIVE_TWITCH_ANONYMOUS=true +# Optional playlist proxy. Off by default and no proxy is built in: name one +# you operate or trust yourself. Check the platform's terms before enabling. +# STREAM_RECORDER_USE_PROXY=false +# TWITCH_PROXY_PLAYLIST=https://proxy.example.com +# STREAMLINK_PLUGIN_DIR= + # Optional Twitch API Client (for metadata auto-sync and EventSub) # TWITCH_CLIENT_ID=your_twitch_client_id_here # TWITCH_CLIENT_SECRET=your_twitch_client_secret_here +# --- System Backups (Admin -> Backups) --- +# Directory listed for streamrecorder_*.tar.gz files (default: /backups) +# BACKUP_DIR=/var/backups/streamvault +# Script run by "Backup erstellen"; it must write streamrecorder_*.tar.gz into BACKUP_DIR +# VOD_BACKUP_SCRIPT=/usr/local/bin/streamvault-backup.sh + # --- Performance & Caching --- # Optional Redis cache for high-concurrency deployments # REDIS_URL=redis://127.0.0.1:6379/0 diff --git a/WEBUI/webui.py b/WEBUI/webui.py index 340d568..0048b91 100644 --- a/WEBUI/webui.py +++ b/WEBUI/webui.py @@ -156,7 +156,6 @@ def detect_x_accel(): TWITCH_APP_ACCESS_TOKEN = ( os.environ.get("TWITCH_APP_ACCESS_TOKEN", "").strip() ) -DISCORD_LOGS_DIR = os.environ.get("VOD_DISCORD_LOGS_DIR", "/root/Discord Bots/logs") MAIL_ADMIN_ENABLED = os.environ.get("VOD_MAIL_ADMIN_ENABLED", "false").lower() in ("true", "1", "yes") STALK_ENABLED = False @@ -2361,9 +2360,14 @@ def list_all_streamer_folders(): return rows +def system_backups_dir(): + """Where streamrecorder_*.tar.gz backups are listed; a dot-dir so it is never an archive.""" + return os.environ.get("BACKUP_DIR") or os.path.join(config.META_DIR, "backups") + + def list_system_backups(): """Lists local system backups.""" - backups_dir = os.environ.get("BACKUP_DIR") or "/root/streamrecorder_backups" + backups_dir = system_backups_dir() items = {} if os.path.isdir(backups_dir): @@ -12868,7 +12872,7 @@ def admin_backup_download(filename): clean = secure_filename(filename) if not clean.startswith("streamrecorder_") or not clean.endswith(".tar.gz"): abort(400) - backups_dir = os.environ.get("BACKUP_DIR") or "/root/streamrecorder_backups" + backups_dir = system_backups_dir() local_path = os.path.join(backups_dir, clean) if os.path.isfile(local_path): return send_from_directory(backups_dir, clean, as_attachment=True) @@ -12878,9 +12882,13 @@ def admin_backup_download(filename): @app.route("/admin/backups/create", methods=["POST"]) @admin_required def admin_backup_create(): - script_path = "/root/Streamrecorder/scripts/backup_streamrecorder.sh" - if not os.path.isfile(script_path): - return redirect(url_for("admin_view", tab="backups", error="Backup-Skript nicht gefunden.")) + # Operator-provided script that writes streamrecorder_*.tar.gz into BACKUP_DIR. + script_path = os.environ.get("VOD_BACKUP_SCRIPT", "").strip() + if not script_path or not os.path.isfile(script_path): + return redirect(url_for( + "admin_view", tab="backups", + error="Kein Backup-Skript konfiguriert (VOD_BACKUP_SCRIPT).", + )) try: res = subprocess.run(["bash", script_path], capture_output=True, text=True, timeout=120) if res.returncode == 0: @@ -12900,15 +12908,14 @@ def admin_backup_delete(filename): clean = secure_filename(filename) if not clean.startswith("streamrecorder_") or not clean.endswith(".tar.gz"): abort(400) - backups_dir = os.environ.get("BACKUP_DIR") or "/root/streamrecorder_backups" - local_path = os.path.join(backups_dir, clean) + local_path = os.path.join(system_backups_dir(), clean) deleted = False if os.path.isfile(local_path): try: os.remove(local_path) deleted = True - except Exception: - pass + except OSError as exc: + logging.error("Could not delete backup %s: %s", local_path, exc) if deleted: bust_media_cache() return redirect(url_for("admin_view", tab="backups", msg=f"Backup {clean} erfolgreich gelöscht.")) diff --git a/recorder/Streamrecorder.py b/recorder/Streamrecorder.py index 6e6b3dd..8e90997 100644 --- a/recorder/Streamrecorder.py +++ b/recorder/Streamrecorder.py @@ -1454,19 +1454,22 @@ def get_vaapi_device(): return path return None -def streamlink_twitch_auth_args(): - """Bann-Schutz: Keine Übermittlung persönlicher OAuth-Tokens an Twitch.""" - return [] - - def streamlink_proxy_args(): - plugin_dir = os.environ.get("STREAMLINK_PLUGIN_DIR", "/etc/streamrecorder/plugins") - proxy_list = os.environ.get( - "TWITCH_PROXY_PLAYLIST", - "https://eu.luminous.dev,https://eu2.luminous.dev,https://as.luminous.dev,https://lb-eu.cdn-perfprod.com,https://lb-eu2.cdn-perfprod.com", - ) - return [ - "--plugin-dirs", plugin_dir, + """Opt-in playlist proxy (STREAM_RECORDER_USE_PROXY=true). + + No third-party proxy is built in: the operator must name one in + TWITCH_PROXY_PLAYLIST (and a plugin directory in STREAMLINK_PLUGIN_DIR if the + proxy needs a patched Twitch plugin). Without a proxy list nothing is added. + """ + proxy_list = os.environ.get("TWITCH_PROXY_PLAYLIST", "").strip() + if not proxy_list: + logger.warning("STREAM_RECORDER_USE_PROXY is set but TWITCH_PROXY_PLAYLIST is empty; recording without proxy") + return [] + args = [] + plugin_dir = os.environ.get("STREAMLINK_PLUGIN_DIR", "").strip() + if plugin_dir: + args += ["--plugin-dirs", plugin_dir] + return args + [ "--twitch-proxy-playlist", proxy_list, "--twitch-proxy-playlist-fallback", "--twitch-disable-ads", diff --git a/tests/test_recorder_reliability.py b/tests/test_recorder_reliability.py index 08212cc..97a1d45 100644 --- a/tests/test_recorder_reliability.py +++ b/tests/test_recorder_reliability.py @@ -18,6 +18,7 @@ import time import types import unittest +import urllib.parse from pathlib import Path from unittest import mock @@ -676,6 +677,42 @@ def test_delete_route_rejects_reserved_name(self): self.assertEqual(response.status_code, 400) +class DeploymentNeutralDefaultsTests(IsolatedBaseTest): + """No built-in third-party proxies or host-specific /root paths.""" + + def test_proxy_args_empty_without_configured_proxy(self): + with mock.patch.dict(os.environ, {"TWITCH_PROXY_PLAYLIST": "", "STREAMLINK_PLUGIN_DIR": ""}), \ + self.assertLogs(level="WARNING"): + self.assertEqual(recorder.streamlink_proxy_args(), []) + + def test_proxy_args_use_only_configured_proxy(self): + with mock.patch.dict(os.environ, {"TWITCH_PROXY_PLAYLIST": "https://proxy.example.com", + "STREAMLINK_PLUGIN_DIR": ""}): + args = recorder.streamlink_proxy_args() + self.assertIn("https://proxy.example.com", args) + self.assertNotIn("--plugin-dirs", args) + + def test_backups_default_to_meta_dir(self): + from WEBUI import webui + with mock.patch.dict(os.environ, {"BACKUP_DIR": ""}): + self.assertEqual(webui.system_backups_dir(), os.path.join(config.META_DIR, "backups")) + + def test_backup_create_without_script_reports_missing_config(self): + from WEBUI import webui + webui.app.config.update(TESTING=True) + client = webui.app.test_client() + with client.session_transaction() as sess: + now = time.time() + sess.update(logged_in=True, user_role="admin", username=config.VOD_USER, + login_time=now, last_active=now, _csrf_token="t") + with mock.patch.dict(os.environ, {"VOD_BACKUP_SCRIPT": ""}), \ + mock.patch.object(webui.subprocess, "run") as run: + response = client.post("/admin/backups/create", data={"csrf_token": "t"}) + run.assert_not_called() + self.assertEqual(response.status_code, 302) + self.assertIn("VOD_BACKUP_SCRIPT", urllib.parse.unquote(response.headers["Location"])) + + class LoadJsonTests(IsolatedBaseTest): """A corrupt state file must survive instead of being overwritten by the default.""" @@ -712,6 +749,18 @@ def test_unreadable_file_raises(self): class LegalPagesTests(unittest.TestCase): """The shipped content/*.md must parse; /agb and /datenschutz returned 500 before.""" + def setUp(self): + # tube_terms resolves its paths from config.APP_DIR at import time; another + # test may have imported it while APP_DIR pointed at a temp dir. + from core import tube_terms + content = Path(__file__).resolve().parents[1] / "content" + for name, path in (("AGB_PATH", content / "agb.md"), ("PRIVACY_PATH", content / "datenschutz.md")): + patcher = mock.patch.object(tube_terms, name, str(path)) + patcher.start() + self.addCleanup(patcher.stop) + tube_terms._cached_document.cache_clear() + tube_terms._cached_privacy.cache_clear() + def test_shipped_documents_parse(self): from core import tube_terms self.assertTrue(tube_terms.load_document(force=True)["terms_version"]) From 123e777eb8ae8abd5fb3007aaa4d2e51213a2694 Mon Sep 17 00:00:00 2001 From: Pierre Date: Tue, 3 Nov 2026 17:33:16 +0100 Subject: [PATCH 24/31] fix(webui): remove community-instance branding from login and titles The login page still pointed new users to a Discord invite (discord.gg/streamvault) that this project does not own, called itself "Cosmic Login" with an "N" logo and an "Enter Control" button, and said "Nur fuer unsere Community". Use neutral self-hosted wording, the "SV" mark used elsewhere, and plain titles for the dashboard and app shell. Co-Authored-By: Claude Opus 5.5 --- WEBUI/templates/base.html | 2 +- WEBUI/templates/index.html | 6 +++--- WEBUI/templates/login.html | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/WEBUI/templates/base.html b/WEBUI/templates/base.html index 8254ab1..ad96c2b 100644 --- a/WEBUI/templates/base.html +++ b/WEBUI/templates/base.html @@ -21,7 +21,7 @@

{% block page_title %}StreamVault{% endblock %}

- {% if not is_mobile %}

{% block page_subtitle %}Cosmic Streaming & Media Vault{% endblock %}

{% endif %} + {% if not is_mobile %}

{% block page_subtitle %}Stream- und VOD-Archiv{% endblock %}

{% endif %}
{% if not is_mobile %} diff --git a/WEBUI/templates/index.html b/WEBUI/templates/index.html index c5c35de..ca5d262 100644 --- a/WEBUI/templates/index.html +++ b/WEBUI/templates/index.html @@ -1,6 +1,6 @@ {% extends 'base.html' %} -{% block title %}StreamVault — Cosmic Stream & Media Vault{% endblock %} +{% block title %}Dashboard — StreamVault{% endblock %} {% block page_title %}{% if not is_mobile %}StreamVault{% endif %}{% endblock %} {% block page_subtitle %}{% endblock %} @@ -14,7 +14,7 @@
- + {% if hero_item and hero_item.is_live %}
{% if hero_item.thumbnail_url %} @@ -157,7 +157,7 @@

{{ vod.title or vod.name }}

{% if not is_mobile %} -
-

STREAMVAULT

+

StreamVault

diff --git a/WEBUI/webui.py b/WEBUI/webui.py index 0048b91..e42136a 100644 --- a/WEBUI/webui.py +++ b/WEBUI/webui.py @@ -2,6 +2,7 @@ import bisect import glob +import hashlib import ipaddress import json import logging @@ -245,7 +246,30 @@ def _parse_trusted_proxies(raw): {"key": "ocean", "name": "Ocean Blue"}, ] VALID_THEME_KEYS = {theme["key"] for theme in THEME_OPTIONS} -ASSET_VERSION = "20261024-instant-launch" +def _compute_asset_version(static_dir): + """Cache-busting token for ?v= on static URLs, derived from the shipped files. + + Any edited, added or removed file under WEBUI/static changes the token, so + browsers fetch new CSS/JS after an update without a manual version bump. + """ + digest = hashlib.sha1() + for root, dirs, files in os.walk(static_dir): + dirs.sort() + for name in sorted(files): + path = os.path.join(root, name) + try: + st = os.stat(path) + except OSError as exc: + logging.warning("Asset version: cannot stat %s: %s", path, exc) + continue + rel = os.path.relpath(path, static_dir).replace(os.sep, "/") + digest.update(f"{rel}:{st.st_size}:{st.st_mtime_ns};".encode("utf-8")) + return digest.hexdigest()[:12] + + +ASSET_VERSION = os.environ.get("VOD_ASSET_VERSION", "").strip() or _compute_asset_version( + os.path.join(os.path.dirname(os.path.abspath(__file__)), "static") +) UI_FONTS_HREF = ( "https://fonts.googleapis.com/css2?" "family=IBM+Plex+Mono:wght@400;500;600&" diff --git a/tests/test_recorder_reliability.py b/tests/test_recorder_reliability.py index 97a1d45..ddf86d3 100644 --- a/tests/test_recorder_reliability.py +++ b/tests/test_recorder_reliability.py @@ -713,6 +713,21 @@ def test_backup_create_without_script_reports_missing_config(self): self.assertIn("VOD_BACKUP_SCRIPT", urllib.parse.unquote(response.headers["Location"])) +class AssetVersionTests(IsolatedBaseTest): + """?v= must change whenever a static file changes, without a manual bump.""" + + def test_version_follows_static_files(self): + from WEBUI import webui + static = self.base / "static" + (static / "css").mkdir(parents=True) + style = static / "css" / "app.css" + style.write_text("h1{font-size:24px}", encoding="utf-8") + first = webui._compute_asset_version(str(static)) + self.assertEqual(first, webui._compute_asset_version(str(static))) + style.write_text("h1{font-size:28px;}", encoding="utf-8") + self.assertNotEqual(first, webui._compute_asset_version(str(static))) + + class LoadJsonTests(IsolatedBaseTest): """A corrupt state file must survive instead of being overwritten by the default.""" @@ -746,10 +761,11 @@ def test_unreadable_file_raises(self): self.webui.load_json(str(path), {}) -class LegalPagesTests(unittest.TestCase): +class LegalPagesTests(IsolatedBaseTest): """The shipped content/*.md must parse; /agb and /datenschutz returned 500 before.""" def setUp(self): + super().setUp() # own database, so the admin session below resolves # tube_terms resolves its paths from config.APP_DIR at import time; another # test may have imported it while APP_DIR pointed at a temp dir. from core import tube_terms @@ -769,6 +785,7 @@ def test_shipped_documents_parse(self): def test_legal_pages_render(self): from WEBUI import webui + webui.ensure_admin_user() webui.app.config.update(TESTING=True) client = webui.app.test_client() with client.session_transaction() as sess: From d6fe73659e0e06176aa5f89993f9a0858e2c4ab2 Mon Sep 17 00:00:00 2001 From: Pierre Date: Tue, 3 Nov 2026 18:05:40 +0100 Subject: [PATCH 26/31] fix(webui): stop mobile layouts from cutting off content A container-overflow scan at 375px found content clipped by the app shell's overflow:hidden on four pages: - header action buttons did not wrap (admin pages) - user-management tabs and table were cut off instead of scrolling - fixed min-widths wider than a phone column: minmax(280px, 1fr) grids and inline min-width: 260px fields (storage, channels, search, files) Header actions wrap, RBAC tabs and table scroll horizontally, and the fixed minimums are capped with min(..., 100%). Scan after the fix: no overflow on 11 pages at 375, 768 and 1280px. Co-Authored-By: Claude Opus 5.5 --- WEBUI/static/css/tube_design_system.css | 10 +++++++--- WEBUI/static/style.css | 8 ++++---- WEBUI/templates/admin_users.html | 4 +++- WEBUI/templates/channels.html | 2 +- WEBUI/templates/files.html | 4 ++-- WEBUI/templates/search.html | 4 ++-- WEBUI/templates/storage.html | 2 +- 7 files changed, 20 insertions(+), 14 deletions(-) diff --git a/WEBUI/static/css/tube_design_system.css b/WEBUI/static/css/tube_design_system.css index 5cd9388..208c487 100644 --- a/WEBUI/static/css/tube_design_system.css +++ b/WEBUI/static/css/tube_design_system.css @@ -731,7 +731,7 @@ body.ns-platform .streamer-hero { /* 16:9 Video Cards Elevation */ body.ns-platform .video-grid { display: grid !important; - grid-template-columns: repeat(auto-fill, minmax(280px, 1fr)) !important; + grid-template-columns: repeat(auto-fill, minmax(min(280px, 100%), 1fr)) !important; gap: 20px 16px !important; } @@ -3235,7 +3235,7 @@ select:focus { /* VOD Discovery Grid */ .nebula-vod-grid { display: grid; - grid-template-columns: repeat(auto-fill, minmax(280px, 1fr)); + grid-template-columns: repeat(auto-fill, minmax(min(280px, 100%), 1fr)); gap: 20px; margin-bottom: 32px; } @@ -3641,7 +3641,11 @@ video, #vodPlayer, .player-wrapper video, .twitch-embed-wrapper iframe { } .ns-header-actions { - flex: 0 0 auto !important; + flex: 1 1 100% !important; + display: flex !important; + flex-wrap: wrap !important; + gap: 8px !important; + min-width: 0 !important; } /* 5. Mobile Global Navigation Bar (Pixel-Perfect, Zero-Overlap) */ diff --git a/WEBUI/static/style.css b/WEBUI/static/style.css index b8748bb..13b7b56 100644 --- a/WEBUI/static/style.css +++ b/WEBUI/static/style.css @@ -1567,7 +1567,7 @@ body.page-index .stat-card:hover::before{ opacity: 1; } body.page-index .streamer-grid{ display: grid; - grid-template-columns: repeat(auto-fill, minmax(280px, 1fr)); + grid-template-columns: repeat(auto-fill, minmax(min(280px, 100%), 1fr)); gap: 24px; } @@ -1864,7 +1864,7 @@ body.page-streamer *{ box-sizing: border-box; } body.page-streamer .video-grid{ display: grid; - grid-template-columns: repeat(auto-fill, minmax(280px, 1fr)); + grid-template-columns: repeat(auto-fill, minmax(min(280px, 100%), 1fr)); gap: 20px; } body.page-streamer .video-card{ @@ -2441,7 +2441,7 @@ body.page-files *{ box-sizing: border-box; } body.page-files .btn-primary:hover::after{ left: 100%; } - body.page-files .files-grid{ display: grid; grid-template-columns: repeat(auto-fill, minmax(280px, 1fr)); gap: 20px; } + body.page-files .files-grid{ display: grid; grid-template-columns: repeat(auto-fill, minmax(min(280px, 100%), 1fr)); gap: 20px; } body.page-files .file-card{ background: var(--glass); @@ -5180,7 +5180,7 @@ body.page-streamer .tab.active { } body.page-streamer .video-grid { - grid-template-columns: repeat(auto-fill, minmax(280px, 1fr)) !important; + grid-template-columns: repeat(auto-fill, minmax(min(280px, 100%), 1fr)) !important; gap: 16px !important; } diff --git a/WEBUI/templates/admin_users.html b/WEBUI/templates/admin_users.html index 8b4cf22..766df23 100644 --- a/WEBUI/templates/admin_users.html +++ b/WEBUI/templates/admin_users.html @@ -23,17 +23,19 @@