diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4cffe17..70a4adbc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + submodules: true - uses: actions/setup-java@v4 with: diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 096f464f..9e3ff140 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -26,6 +26,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + submodules: true - uses: actions/setup-java@v4 with: diff --git a/.gitignore b/.gitignore index 36cf66ae..fc7b9016 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ .gradle/ build/ -codecov* \ No newline at end of file +codecov* +example_old/ +mcp-config.json \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..f80adeb3 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "debridav-frontend"] + path = debridav-frontend + url = https://github.com/skjaere/debridav-frontend.git diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..b5e6de70 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,279 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Build and Test Commands + +```bash +# Build +./gradlew build # Full build with tests +./gradlew bootJar # Build Spring Boot JAR +./gradlew bootRun # Run application directly +./gradlew jibDockerBuild # Build Docker image locally + +# Test +./gradlew test # Run all tests +./gradlew test --tests "io.skjaere.debridav.test.SomeTest" # Run single test class +./gradlew test --tests "*SomeTest.testMethod" # Run single test method + +# Other +./gradlew compileKotlin # Compile only (no tests) +``` + +## Technology Stack + +- **Kotlin 2.3.0** with **Java 25** (virtual threads via Loom) +- **Spring Boot 4.0.0** with Spring Data JPA +- **PostgreSQL** with Flyway migrations +- **Ktor 3.3.3** for HTTP client operations +- **Milton 4.0.4** for WebDAV protocol +- **Kotlin Coroutines** with custom `Dispatchers.LOOM` for virtual thread integration + +## Architecture Overview + +DebriDAV creates a WebDAV-mountable virtual filesystem backed by debrid service providers. It emulates the qBittorrent and SABnzbd APIs for integration with Sonarr/Radarr. + +### Core Modules + +| Package | Purpose | +|---------|---------| +| `debrid/client/` | Provider implementations (RealDebrid, Premiumize, Easynews, TorBox) with abstract `DebridClient` | +| `fs/` | Virtual filesystem layer - `DatabaseFileService` manages file hierarchy using PostgreSQL LTree | +| `resource/` | WebDAV resource factory connecting Milton to the virtual FS | +| `torrent/` | qBittorrent API emulation (`QBittorrentEmulationController`) | +| `nntp/` | Usenet/NZB support with streaming RAR parsing and Yenc decompression | +| `archive/` | RAR file parsing (`Rar4Parser`) for metadata extraction | +| `cache/` | Byte-range caching for metadata extraction (`FileChunkCachingService`) | +| `arrs/` | Sonarr/Radarr integration services | + +### Data Flow + +1. Sonarr/Radarr send requests to qBittorrent-emulated API +2. Torrent/NZB content checked against debrid provider caches +3. Cached content registered in PostgreSQL as virtual files +4. WebDAV server exposes virtual filesystem for media server mounting +5. On file access, content streamed from debrid provider with chunk caching + +### Database + +- PostgreSQL required (uses LTree extension for hierarchical paths) +- Entities: `Torrent`, `UsenetEntity`, `DebridFileContents`, `FileChunk` +- Migrations in `src/main/resources/db/migration/` (V1-V11) + +## Code Patterns + +**Configuration**: Use `@ConfigurationProperties` classes in `DebridavConfiguration.kt`. Properties defined in `application.properties`. + +**Async operations**: Use Kotlin coroutines with `Dispatchers.LOOM` for blocking I/O: +```kotlin +withContext(Dispatchers.LOOM) { + // blocking operation +} +``` + +**Transactions**: Use `TransactionTemplate` for explicit transaction boundaries in services. + +**Testing**: Integration tests use TestContainers (PostgreSQL), MockServer for HTTP, MockK for mocking. Tests in `src/test/kotlin/io/skjaere/debridav/test/`. + +## Real-Debrid Integration + +### Key Files + +| File | Purpose | +|------|---------| +| `debrid/client/realdebrid/RealDebridClient.kt` | Main client — cache checking, torrent management, link unrestriction | +| `debrid/client/realdebrid/support/RealDebridTorrentService.kt` | Torrent sync and DB persistence | +| `debrid/client/realdebrid/support/RealDebridDownloadService.kt` | Download sync and DB persistence | +| `debrid/client/realdebrid/RealDebridConfigurationProperties.kt` | Configuration properties (`real-debrid.*`) | +| `debrid/client/realdebrid/RealDebridConfiguration.kt` | Spring bean config including Resilience4j rate limiter | +| `debrid/client/realdebrid/RealDebridActuatorEndpoint.kt` | Actuator endpoint for toggling torrent import at runtime | + +### Class Hierarchy + +`RealDebridClient` extends `DebridCachedTorrentClient` and `DebridCachedContentClient`, implements `StreamableLinkPreparable` (delegated to `DefaultStreamableLinkPreparer`) and `ConfigurationTester`. + +### API Endpoints Used + +All calls go to `https://api.real-debrid.com/rest/1.0` (configurable). Authentication is via Bearer token (`real-debrid.api-key`). + +| Endpoint | Method | Purpose | +|----------|--------|---------| +| `/torrents/addMagnet` | POST | Submit magnet link (form-encoded `magnet=...`) | +| `/torrents/` | GET | List user's torrents (paginated, 100/page) | +| `/torrents/info/{id}` | GET | Get torrent info with files and links | +| `/torrents/selectFiles/{id}` | POST | Select files from torrent (form-encoded `files=1,2,3`) | +| `/torrents/delete/{id}` | DELETE | Remove torrent from account | +| `/unrestrict/link` | POST | Convert RD share link → direct download URL | +| `/downloads` | GET | List user's downloads (paginated, 100/page) | +| `/downloads/delete/{id}` | DELETE | Remove download from account | +| `/user` | GET | Validate API key (used by `ConfigurationTester`) | + +### End-to-End Flow + +**Phase 1 — Torrent Addition:** +1. `getCachedFiles(magnet)` checks DB for existing torrent by info hash +2. If not found: `POST /torrents/addMagnet` → `GET /torrents/info/{id}` → save to `RealDebridTorrentEntity` + +**Phase 2 — File Selection:** +1. `getIdsToSelect()` filters for video files (`.mp4`, `.mkv`, `.avi`, `.ts`) +2. `POST /torrents/selectFiles/{id}` with selected file IDs +3. `GET /torrents/info/{id}` to retrieve links for selected files +4. If no links available (not cached): DELETE torrent, return empty list + +**Phase 3 — Link Unrestriction:** +1. For each file link, check DB for existing `RealDebridDownloadEntity` +2. If not found: `POST /unrestrict/link` → returns direct download URL, saved to DB +3. Returns `List` with path, download URL, MIME type, and params (`torrentId`, `linkId`) + +**Phase 4 — Streaming:** +1. `getStreamableLink()` looks up download by hash + filename + size in DB +2. `isLinkAlive()` — HEAD request to download URL (rate-limited, cached 5 min) +3. If alive: return URL. If dead: delete and fetch fresh link via unrestrict +4. `DefaultStreamableLinkPreparer` builds Ktor HTTP GET with byte-range headers for seeking support + +### Rate Limiting + +Resilience4j `RateLimiter`: **249 requests per 1 minute** (just under RD's ~250/min limit), 5-second timeout per acquisition. + +### Scheduled Sync + +`syncTorrentsTask()` runs on a configurable schedule (`real-debrid.sync-poll-rate`, default `PT24H`): +- Clears and re-fetches all `RealDebridTorrentEntity` records (paginated `/torrents/`) +- Clears and re-fetches all `RealDebridDownloadEntity` records (paginated `/downloads`) +- Can be toggled at runtime via the actuator endpoint + +### Database Entities + +**`RealDebridTorrentEntity`**: `torrentId` (indexed), `name`, `hash` (indexed), `links` (ElementCollection), `files` (one-to-many `TorrentsInfoFile`) + +**`RealDebridDownloadEntity`**: `downloadId` (indexed), `filename`, `mimeType`, `fileSize`, `link` (RD share link), `host`, `download` (actual URL), `chunks`, `streamable` + +**Key query**: `getDownloadByHashAndFilenameAndSize()` — native SQL joining downloads → torrent links → torrents to find a download by torrent hash + filename + file size. + +### Error Handling + +- `isCached()` always returns `true` (RD doesn't expose a cache-check API; availability is determined during file selection) +- HTTP 4xx → `DebridClientError`, 5xx → `DebridProviderError` +- `addMagnet` failures return `FailedAddMagnetResponse` with reason (not thrown) +- `unrestrict` failures logged as warnings, return `ErrorUnrestrictLinkResponse` +- Configurable retries in `DebridCachedContentService` (default 1, 200ms delay) + +## NNTP/Usenet Integration + +### External Artifacts + +| Artifact | Version | Purpose | +|----------|---------|---------| +| `com.github.skjaere:nzb-streamer` | 0.7.0 | NZB parsing, NNTP article fetching, Yenc decompression, RAR/7zip archive parsing, and file streaming | +| `com.github.skjaere:mock-nntp-server` | 0.2.0 | Test-only mock NNTP server | + +**nzb-streamer** internally depends on **ktor-nntp-client** (a Ktor-based NNTP protocol client) for connecting to Usenet servers, fetching articles by message-ID, and managing connection pools with TLS support and server priority failover. + +### Key Files + +| File | Purpose | +|------|---------| +| `usenet/NzbStreamerConfiguration.kt` | NNTP server pool config, creates `NzbStreamer` bean | +| `usenet/NzbImportService.kt` | Orchestrates NZB import: parse → extract metadata → register in filesystem | +| `usenet/NzbHealthCheckService.kt` | Scheduled verification that NZB segments still exist on Usenet | +| `usenet/sabnzbd/SabnzbdApiController.kt` | SABnzbd API emulation endpoints | +| `usenet/sabnzbd/SabNzbdService.kt` | NZB handling and SABnzbd response building | +| `usenet/pgmq/PgmqSpringConfiguration.kt` | PostgreSQL message queue setup (3 queues) | +| `usenet/pgmq/PgmqConsumer.kt` | Generic message consumer loop | +| `usenet/pgmq/NzbHealthCheckHandler.kt` | Processes health check messages | +| `usenet/pgmq/NzbHealthRepairHandler.kt` | Blocklists failed NZBs in Sonarr/Radarr | +| `usenet/nzb/NzbDocumentEntity.kt` | JPA entity storing parsed NZB metadata as JSONB | +| `usenet/UsenetDownload.kt` | JPA entity tracking download status | +| `usenet/queue/NzbImportRecord.kt` | JPA entity tracking import queue status | +| `resource/NzbFileResource.kt` | WebDAV resource for streaming NZB files via Milton | + +### Configuration Properties + +**`nntp.*`** (all conditional on `nntp.enabled=true`): +- `enabled` — enable/disable NNTP support +- `concurrency` (default 4) — concurrent NNTP streaming threads +- `forwardThresholdBytes` (default 102400) — byte threshold for forward seeking +- `healthCheckInterval` (default 7 days) — how often to reverify NZB segments +- `healthCheckPollRate` (default 5 min) — poll rate for health check scheduling +- `pools` — list of NNTP server pools, each with: `host`, `port`, `username`, `password`, `useTls`, `maxConnections`, `priority` + +**`pgmq.*`**: +- `importConcurrency` (default 2) — workers processing NZB imports +- `importVisibilityTimeout` (default 10 min) — message lock duration +- `importPollInterval` (default 2 sec) — queue poll rate +- `healthCheckConcurrency` (default 1), `healthRepairConcurrency` (default 2) + +### End-to-End Flow + +**Phase 1 — NZB Upload (SABnzbd API emulation):** +1. Sonarr/Radarr POST NZB file to `/api?mode=addfile` +2. `SabNzbdService` creates `UsenetDownload` (QUEUED) and `NzbImportRecord` (QUEUED) +3. Sends `NzbImportMessage` (NZB bytes as Base64) to PGMQ `nzb_import` queue +4. Returns immediately to caller + +**Phase 2 — Async Import (PGMQ consumer):** +1. `PgmqConsumer` picks up message from `nzb_import` queue +2. `NzbImportService.executeImport()`: + - Decodes NZB bytes, calls `nzbStreamer.prepare(nzbBytes)` + - nzb-streamer parses NZB XML, fetches initial articles from NNTP servers via ktor-nntp-client + - Yenc-decodes article bodies, parses RAR/7zip archive headers to extract file metadata + - Returns `PrepareResult`: `Success`, `MissingArticles`, `Failure`, or `UnsupportedArchive` +3. On success: `nzbStreamer.resolveStreamableFiles(metadata)` → list of files with volume/offset info +4. Creates `NzbDocumentEntity` (files + streamableFiles stored as JSONB), `NzbContents` per file, and `RemotelyCachedEntity` entries in the virtual filesystem +5. Updates `UsenetDownload.status` → COMPLETED + +**Phase 3 — File Streaming (WebDAV access):** +1. Media server accesses file via WebDAV +2. `StreamableResourceFactory` creates `NzbFileResource` from `NzbContents` entity +3. `NzbFileResource.sendContent()` calls `nzbStreamer.streamFile(nzbDocument, streamableFile, range)` +4. nzb-streamer fetches NNTP articles on-demand, Yenc-decodes, reconstructs archive data, and streams the extracted file content via a `ByteReadChannel` +5. Supports byte-range requests for seeking/scrubbing + +**Phase 4 — Health Check & Repair:** +1. `NzbHealthCheckService` runs on schedule, finds NZB documents not verified within `healthCheckInterval` +2. Sends `NzbHealthCheckMessage` to PGMQ `nzb_health_check` queue +3. `NzbHealthCheckHandler` calls `nzbStreamer.verifySegments()` to check article availability +4. If articles missing: sends `NzbHealthRepairMessage` to `nzb_health_repair` queue +5. `NzbHealthRepairHandler` blocklists the download in Sonarr/Radarr and triggers a new search + +### Message Queue Architecture (PGMQ) + +Three PostgreSQL-backed queues (installed via Flyway migration `V12__install_pgmq.sql`): + +| Queue | Message Type | Handler | Concurrency | +|-------|-------------|---------|-------------| +| `nzb_import` | `NzbImportMessage` | `NzbImportService` | 2 workers | +| `nzb_health_check` | `NzbHealthCheckMessage` | `NzbHealthCheckHandler` | 1 worker | +| `nzb_health_repair` | `NzbHealthRepairMessage` | `NzbHealthRepairHandler` | 2 workers | + +### Supported Archive Types + +nzb-streamer handles: `RAW`, `RAR`, `SEVEN_ZIP`, `RAR_IN_SEVEN_ZIP`, `RAR_IN_RAR`, `SEVEN_ZIP_IN_RAR`, `SEVEN_ZIP_IN_SEVEN_ZIP` (nested archives). + +### Database Entities + +**`NzbDocumentEntity`** (table `nzb_document`): `files` (JSONB — Yenc headers + segment article IDs), `streamableFiles` (JSONB — file paths with volume/offset positions), `archiveType`, `lastVerified`, `name`, `category` + +**`UsenetDownload`**: `status` (QUEUED → DOWNLOADING → COMPLETED/FAILED/ARTICLES_MISSING), `name`, `hash` (MD5 of NZB), `size`, `category`, references `NzbDocumentEntity` + +**`NzbImportRecord`** (table `nzb_import`): tracks import queue status with `status`, `archiveType`, `errorMessage`, `files` (JSONB), timestamps + +**`NzbContents`** (extends `DebridFileContents`): `originalPath`, `size`, `mimeType`, references `NzbDocumentEntity` + +### SABnzbd API Emulation + +Endpoints at `/api` (emulating SABnzbd v4.4.0): +- `mode=addfile` — multipart NZB upload +- `mode=queue` — returns queue status +- `mode=history` — completed/failed downloads from DB +- `mode=get_config` — categories and configuration +- `mode=version` — returns "4.4.0" +- `mode=fullstatus` — static status with configured paths + +## Configuration + +Key properties in `application.properties`: +- `debridav.debrid-clients` - Enabled providers (real-debrid, premiumize, easynews, torbox) +- `debridav.root-path` - WebDAV root path +- `debridav.download-path` - Download directory path +- Provider-specific API keys and settings diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index c5a35fcf..00000000 --- a/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM openjdk:21-jdk-slim - -WORKDIR /app - -RUN apt-get update - -RUN mkdir app -COPY build/libs/debridav-0.1.0.jar app/app.jar -EXPOSE 8080 - -CMD ["java", "-jar", "app/app.jar"] diff --git a/README.md b/README.md index 35b8dd97..1a2a7692 100644 --- a/README.md +++ b/README.md @@ -16,11 +16,26 @@ protocol so that they can be mounted. ## Features - ☁️ **Stream from debrid providers** — Real Debrid, Premiumize, TorBox, and Easynews, with Plex/Jellyfin. -- 📡 **Stream from usenet via NNTP** *(coming in 0.12.0)* — Import NZBs and stream directly from your usenet provider, no intermediate download required. +- 📡 **Stream from usenet via NNTP** — Import NZBs and stream directly from your usenet provider with a pool of NNTP connections, no intermediate download required. - 🔀 **Multiple providers with fallback** — Enable multiple debrid providers concurrently with defined priorities. If content is not cached in the primary provider, DebriDav falls back to the next. - 🔗 **Arr integration** — Emulates the qBittorrent and SABnzbd APIs for seamless integration with Sonarr and Radarr. - 📁 **Virtual file management** — Sort content as you would regular files. Create directories, rename files, and move them around — no regular expressions needed. Files are exposed via WebDAV. -- 🩺 **Health checking and repair** — Automatically detect unhealthy NZB imports and trigger re-searches via Sonarr/Radarr. +- 🩺 **Health checking and repair** — Automatically detect unhealthy torrents and NZBs and trigger re-searches / blocklists via Sonarr/Radarr. +- 🖥️ **Built-in dashboard UI** — React frontend bundled into the backend, with a file browser, config editor, queue/history views, and live log tailing. +- 🔧 **Runtime configuration** — Most settings are editable at runtime via the UI / config API and persist to Postgres; no pod restart required. +- 🔐 **JWT authentication** — Opt-in auth layer protecting the API, qBittorrent/SABnzbd emulation, actuator, and temporary stream tokens. +- 📊 **Prometheus metrics + Grafana dashboards** — First-class observability for streams, health checks, PGMQ queues, and the NNTP connection pool. + +## Migrating from 0.11 to 1.0 + +1.0 contains a few breaking changes. If you are upgrading from 0.11.x, apply these in order: + +- **WebDAV moved under `/webdav/`**. Update every rclone mount, media-server, and WebDAV client URL from `http://host:8080/` to `http://host:8080/webdav/`. +- **Removed config keys** (silently ignored if left set): `DEBRIDAV_ROOTPATH`, `DEBRIDAV_ENABLEFILEIMPORTONSTARTUP`, and the legacy chunk-cache knobs (`DEBRIDAV_CHUNKCACHINGGRACEPERIOD`, `DEBRIDAV_CHUNKCACHINGSIZETHRESHOLD`, `DEBRIDAV_CACHEMAXSIZE`). +- **Database migrations apply automatically** via Flyway on first start; no manual action needed. +- **Any `config_override` rows** you created via the config API under one of the now-removed keys are orphaned and can be deleted from the UI / config API. + +The NNTP / Usenet streaming and Health-check + repair pipelines are new features in 1.0 — see their respective sections below — and don't require migrating existing config. ## How does it work? @@ -29,9 +44,10 @@ as download clients in the arrs. Once a magnet/nzb is sent to DebriDav it will check if it is cached in any of the available debrid providers and create file representations for the streamable files hosted at debrid providers. -Note that DebriDav does not read the torrents added to your Real Debrid account, or your Premiumize cloud storage. -Content you wish to be accessible through DebriDav must be added with the qBittorrent API. An feature to import -these files to DebriDav may be added in the future. +Note that content you wish to be accessible through DebriDav must be added with the qBittorrent API. +DebriDav does have an opt-in Real-Debrid sync (`REAL-DEBRID_SYNCENABLED`, default on) that periodically +pulls your account's existing torrents + downloads so they can be re-used on restart; Premiumize cloud +storage is not browsed. ## Which debrid services are supported? @@ -72,48 +88,73 @@ This feature includes: ### NNTP configuration -| Environment variable | Description | Default | -|--------------------------------------|-------------------------------------------------------------------------------------------------|-----------| -| NNTP_ENABLED | Enable NNTP/usenet support | `false` | -| NNTP_HOST | NNTP server hostname | | -| NNTP_PORT | NNTP server port | `563` | -| NNTP_USERNAME | NNTP server username | | -| NNTP_PASSWORD | NNTP server password | | -| NNTP_USETLS | Use TLS for NNTP connections | `true` | -| NNTP_CONCURRENCY | Number of concurrent article downloads per stream | `4` | -| NNTP_MAXCONNECTIONS | Maximum number of NNTP connections in the pool | `8` | +NNTP is enabled implicitly whenever at least one pool has a host. Configure each pool by its index +(`0`, `1`, …); lower-priority pools act as fill/fallback. + +| Environment variable | Description | Default | +|--------------------------------------|-------------------------------------------------------------------------------------------------|---------------------| +| NNTP_POOLS_0_HOST | NNTP server hostname for pool 0 (required to enable NNTP) | | +| NNTP_POOLS_0_PORT | NNTP server port | `563` | +| NNTP_POOLS_0_USERNAME | NNTP server username | | +| NNTP_POOLS_0_PASSWORD | NNTP server password | | +| NNTP_POOLS_0_USETLS | Use TLS for NNTP connections | `true` | +| NNTP_POOLS_0_MAXCONNECTIONS | Maximum connections in this pool | `8` | +| NNTP_POOLS_0_PRIORITY | Pool priority; pool with the lowest number is preferred | `0` | +| NNTP_CONCURRENCY | Number of concurrent article downloads per stream (shared across pools) | `4` | | NNTP_READAHEADSEGMENTS | Number of segments to read ahead during streaming | same as concurrency | -| NNTP_HEALTHCHECKINTERVAL | How often to health-check imported NZBs (ISO-8601 duration) | `P7D` | -| NNTP_HEALTHCHECKPOLLRATE | How often to poll the health check queue (ISO-8601 duration) | `PT5M` | -## Monitoring +Pools can also be managed live from the UI (Configuration → NNTP → Server Pools). -There is a docker compose file in /example/observability which includes some useful services for monitoring the DebriDav -and associated services. See [OBSERVABILITY.md](example/monitoring/MONITORING.md) +## Health checking & repair -## How do I use it? +Debrid links expire; hosters drop files; NZB segments age off news servers. DebriDav periodically +re-verifies every torrent's debrid links and every NZB's article availability in the background, +then routes anything that fails into a repair pipeline: blocklist the broken release in Sonarr / +Radarr, trigger a re-search, and delete the virtual file if no replacement is found. -### Elfhosted +Each check + repair runs as a PostgreSQL-backed queue (PGMQ), so retries and dead-lettering are +durable across restarts. Results show up on the Health page of the UI (Queue / History tabs) +and the Grafana dashboard. -Like the concept of streaming your Premiumize / EasyNews content, but don't want the hassle of configuring and -self-hosting? +| Environment variable | Description | Default | +|--------------------------------------|-------------------------------------------------------------------------------------------------|-----------| +| HEALTH-CHECK_REPAIR-ENABLED | Enable automatic repair (blocklist + re-search via Sonarr/Radarr) of unhealthy items. | `true` | +| HEALTH-CHECK_NZB-INTERVAL | How often to reverify a given NZB's segments (ISO-8601 duration). | `P7D` | +| HEALTH-CHECK_NZB-POLL-RATE | How often to scan for NZBs needing a check. | `PT5M` | +| HEALTH-CHECK_TORRENT-INTERVAL | How often to reverify a given torrent's debrid links. | `P1D` | +| HEALTH-CHECK_TORRENT-POLL-RATE | How often to scan for torrents needing a check. | `PT5M` | + +Arr integration is required for repair: without a Sonarr or Radarr client wired up for the +matching category, unhealthy items are deleted from the virtual filesystem rather than +re-sourced. Leave `HEALTH-CHECK_REPAIR-ENABLED=false` if you want checks-only observability +without any automated deletion. + +## Rclone VFS cache invalidation + +If you mount DebriDav via rclone and set rclone's `--dir-cache-time` to something comfortably long +(say, 120s) for snappy directory listings, DebriDav can push cache invalidations directly to +rclone whenever files are created, moved, or deleted — so changes show up in the mount +immediately instead of waiting for the dir-cache to expire. + +| NAME | Explanation | Default | +|-----------------------------------------|-----------------------------------------------------------------------------------------------|---------| +| DEBRIDAV_RCLONECACHEINVALIDATIONENABLED | Enable pushing VFS cache invalidations to rclone. | `false` | +| DEBRIDAV_RCLONE_RC-URL | Rclone remote-control endpoint (e.g. `http://rclone:5572`). Blank disables the integration. | | +| DEBRIDAV_RCLONE_RC-USER | Basic-auth user for rclone RC, if configured. | | +| DEBRIDAV_RCLONE_RC-PASSWORD | Basic-auth password for rclone RC, if configured. | | + +Rclone needs `--rc --rc-addr :5572 --rc-user ... --rc-pass ...` (or equivalent) on its command +line for the RC server to be reachable. The docker-compose example in `example/` already wires +this up. -[ElfHosted](https://elfhosted.com) is a geeky, [open-source](https://docs.elfhosted.com/open-source/) PaaS, which -provides all the "plumbing" (_hosting, security, updates, etc_) for your self-hosted apps. ElfHosted provide entire -hosted streaming "bundles", so all you have to do is plug in your EasyNews / Premiumize credentials, fire up Radarr / -Sonarr, and start streaming! +## Monitoring -ElfHosted offer pre-configured bundles (*with a $1 7-day trial*) for Streaming from Premiumize -with [Plex](https://store.elfhosted.com/product/hobbit-plex-premiumize-aars/), [Emby](https://store.elfhosted.com/product/hobbit-emby-premiumize-aars/), -or [Jellyfin](https://store.elfhosted.com/product/hobbit-jellyfin-premiumize-aars/), as well as from EasyNews -with [Plex](https://store.elfhosted.com/product/hobbit-plex-easynews-aars/), [Emby](https://store.elfhosted.com/product/hobbit-emby-easynews-aars/), -or [Jellyfin](https://store.elfhosted.com/product/hobbit-jellyfin-easynews-aars/), and also -offers [DebriDav "unbundled"](https://store.elfhosted.com/product/debridav/) to augment their existing, debrid-connected -stacks. +A `docker-compose.monitoring.yml` override in [`example/`](example/) layers Prometheus, Grafana, and +supporting exporters on top of the base stack. When the Grafana base URL is configured, the Dashboard +tab of the UI embeds every Grafana dashboard under the `debridav` folder. See +[example/README.md](example/README.md) for details. -> [!IMPORTANT] -> A portion of your ElfHosted DebriDav subscription supports further development of DebriDav, under -> the ["Elf-illiate" program](https://store.elfhosted.com/affiliate/) +## How do I use it? ### Requirements @@ -122,7 +163,7 @@ To build the project you will need a java 21 JDK. ### Running with Docker compose ( recommended ) -See [QUICKSTART](example/QUICKSTART.md) +See [example/README.md](example/README.md). ### Running the jar @@ -131,63 +172,104 @@ Alternatively `./gradlew bootRun` can be used. ### Running with docker -`docker run ghcr.io/skjaere/debridav:v0` +`docker run ghcr.io/skjaere/debridav:v1` ### Build docker image To build the docker image run `./gradlew jibDockerBuild` You will want to use rclone to mount DebriDav to a directory which can be shared among docker containers. -[docker-compose.yaml](example/docker-compose.yaml) in examples/ can be used as a starting point. +[docker-compose.yml](example/docker-compose.yml) in `example/` can be used as a starting point. + +### Accessing the UI + +Once DebriDav is running, point a browser at its HTTP port (default `8080`) to reach the dashboard — +`http://:8080/`. The UI covers live streams, imports, health checks, repair history, the file +browser, runtime config editor, and a log tailer. + +If `DEBRIDAV_AUTH_ENABLED=true`, you'll be prompted to log in with `DEBRIDAV_WEBDAV-USERNAME` / +`DEBRIDAV_WEBDAV-PASSWORD` (the same credentials also guard the WebDAV endpoint). With auth off +(the default), the UI is unauthenticated — only expose it to trusted networks. + +WebDAV itself is served under `/webdav/` (e.g. `http://:8080/webdav/`), separate from the +UI. Point your rclone mount or media server at that path. ## Configuration -The following values can be defined as environment variables. - -| NAME | Explanation | Default | -|------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------| -| DEBRIDAV_ROOTPATH | The root path of DebriDav. DebriDav will store configuration data, databases, files under this directory. When running as docker this directory refers to the path within the docker container. | ./debridav-files | -| DEBRIDAV_DOWNLOADPATH | The path under `DEBRIDAV_ROOTPATH` where downloaded files will be placed. | /downloads | -| DEBRIDAV_DEBRIDCLIENTS | A comma separated list of enabled debrid providers. Allowed values are `real_debrid`, `premiumize`, `easynews` and `torbox`. Note that the order determines the priority in which they are used. | | -| DEBRIDAV_DB_HOST | The host of the PostgresSQL database server | localhost | -| DEBRIDAV_DB_PORT | The port of the PostgresSQL database server | 5432 | -| DEBRIDAV_DB_DATABASENAME | The name of the database to use within the PostgresSQL server | debridav | -| DEBRIDAV_DB_USERNAME | The username to use when connecting the PostgresSQL server | debridav | -| DEBRIDAV_DB_PASSWORD | The password to use when connecting the PostgresSQL server | debridav | -| DEBRIDAV_ENABLEFILEIMPORTONSTARTUP | Enables importing content from the filesystem to the database. | debridav | -| DEBRIDAV_DEFAULTCATEGORIES | A comma separated list of categories to create on startup | | -| DEBRIDAV_LOCALENTITYMAXSIZEMB | The maximum allowed size in MB for locally stored files. Useful to prevent accidentally large files in the database. Set to 0 for no limit | 50 | -| DEBRIDAV_CHUNKCACHINGGRACEPERIOD | The amount of time to keep chunks in the cache as a duration string ( 2m, 4h, 2d etc) | 4h | -| DEBRIDAV_CHUNKCACHINGSIZETHRESHOLD | The maximum chunk size to cache in bytes. | 5120000 ( 5Mb ) | -| DEBRIDAV_CACHEMAXSIZE | The maximum size of the cache in gigabytes. | 2 | -| PREMIUMIZE_APIKEY | The api key for Premiumize | | -| REAL-DEBRID_APIKEY | The api key for Real Debrid | | -| REAL-DEBRID_SYNCENABLED | If set to true, DebriDav will periodically poll Real-Debrid's API for torrents and downloads for re-use | true | -| REAL-DEBRID_SYNCPOLLRATE | The rate at which DebriDav will sync downloads and torrents ( if enabled by DEBRID_SYNCENABLED ) as a [ISO8601 time string](https://en.wikipedia.org/wiki/ISO_8601#Durations). | PT4H ( 4 hours ) | -| EASYNEWS_USERNAME | The Easynews username | | -| EASYNEWS_PASSWORD | The Easynews password | | -| EASYNEWS_ENABLEDFORTORRENTS | If set to true, DebriDav will search for releases in Easynews matching the torrent name of torrents added via the qBittorrent API | true | -| EASYNEWS_RATELIMITWINDOWDURATION | The size of the time window to use for rate limiting. | 15 seconds | -| EASYNEWS_ALLOWEDREQUESTSINWINDOW | The number of requests allowed in the time window. eg: EASYNEWS_RATELIMITWINDOWDURATION=10s and EASYNEWS_ALLOWEDREQUESTSINWINDOW=3 will allow 3 requests per 10 seconds before forcing subsequent requests to wait. | 10 | -| EASYNEWS_CONNECTTIMEOUT | The amount of time in milliseconds to wait while establishing a connection to Easynews' servers. | 20000 | -| EASYNEWS_SOCKETTIMEOUT | The amount of time in milliseconds to wait between receiving bytes from Easynews' servers. | 5000 | -| TORBOX_APIKEY | The api key for TorBox | | -| SONARR_INTEGRATIONENABLED | Enable integration of Sonarr. | true | -| SONARR_HOST | The host of Sonarr | sonarr-debridav | -| SONARR_PORT | The port of Sonarr | 8989 | -| SONARR_API_KEY | The API key for Sonarr | | -| SONARR_CATEGORY | The qBittorrent cateogy Sonarr uses | tv-sonarr | -| RADARR_INTEGRATIONENABLED | Enable integration of Radarr. See description of SONARR_INTEGRATION_ENABLED | true | -| RADARR_HOST | The host of Radarr | radarr-debridav | -| RADARR_PORT | The port of Radarr | 7878 | -| RADARR_API_KEY | The API key for Radarr | | -| RADARR_CATEGORY | The qBittorrent cateogy Radarr uses | radarr | - -## Developing - -A docker compose file is provided in the dev directory, with Prowlarr and rclone defined. You can add a qBittorrent -download client in prowlarr and point it to the ip obtained by running `ip addr show docker0` in order to reach your -locally running DebriDav server. +Most settings are also editable at runtime from the UI's Configuration page; these env vars +bootstrap the defaults on first start. + +### Core + +| NAME | Explanation | Default | +|------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------| +| DEBRIDAV_DOWNLOADPATH | Path reported to Sonarr/Radarr as the "download complete" directory. | /downloads | +| DEBRIDAV_MOUNTPATH | Path reported to Sonarr/Radarr where the WebDAV mount is visible to them. | /data | +| DEBRIDAV_DEBRIDCLIENTS | Comma-separated list of enabled debrid providers. Allowed values: `real_debrid`, `premiumize`, `easynews`, `torbox`. Order determines fallback priority. | | +| DEBRIDAV_DEFAULTCATEGORIES | Comma-separated list of qBittorrent categories to create on startup. | | +| DEBRIDAV_LOCALENTITYMAXSIZEMB | Maximum size in MB for locally-stored (non-debrid) files. Prevents accidentally-large BLOBs in the database. `0` = unlimited. | 50 | + +### Database + +| NAME | Explanation | Default | +|--------------------------|-------------------------------------------------|-----------| +| DEBRIDAV_DB_HOST | Postgres host | localhost | +| DEBRIDAV_DB_PORT | Postgres port | 5432 | +| DEBRIDAV_DB_DATABASENAME | Database name | debridav | +| DEBRIDAV_DB_USERNAME | Database username | debridav | +| DEBRIDAV_DB_PASSWORD | Database password | debridav | + +### Authentication + +| NAME | Explanation | Default | +|----------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|---------| +| DEBRIDAV_AUTH_ENABLED | Protect the UI + API with JWT login. | `false` | +| DEBRIDAV_AUTH_JWT-SECRET | Signing key for JWTs. Must be ≥ 32 bytes. If blank, a random key is generated per process (tokens won't survive restarts). `openssl rand -base64 48`. | | +| DEBRIDAV_AUTH_TOKEN-EXPIRATION-HOURS | UI session token lifetime. | 24 | +| DEBRIDAV_AUTH_PROTECT-QBITTORRENT-API | Require auth on the qBittorrent emulation endpoints (turn off for Sonarr/Radarr local-network use). | `false` | +| DEBRIDAV_AUTH_PROTECT-SABNZBD-API | Require auth on the SABnzbd emulation endpoints. | `false` | +| DEBRIDAV_AUTH_PROTECT-ACTUATOR | Require auth on `/actuator/*`. | `false` | +| DEBRIDAV_WEBDAV-USERNAME | Basic-auth username for WebDAV clients (rclone, media servers). WebDAV auth is enabled if both username and password are set. | | +| DEBRIDAV_WEBDAV-PASSWORD | Basic-auth password for WebDAV clients. | | + +### Debrid providers + +| NAME | Explanation | Default | +|------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------| +| PREMIUMIZE_APIKEY | Premiumize API key. | | +| REAL-DEBRID_APIKEY | Real-Debrid API key. | | +| REAL-DEBRID_SYNCENABLED | Periodically pull existing torrents + downloads from RD for re-use. | `true` | +| REAL-DEBRID_SYNCPOLLRATE | RD sync poll rate ([ISO-8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations)). | PT24H | +| TORBOX_APIKEY | TorBox API key. | | +| EASYNEWS_USERNAME | Easynews username. | | +| EASYNEWS_PASSWORD | Easynews password. | | +| EASYNEWS_ENABLEDFORTORRENTS | Search Easynews for releases matching torrents added via the qBittorrent API. | `true` | +| EASYNEWS_RATELIMITWINDOWDURATION | Rate-limit time window. | 15s | +| EASYNEWS_ALLOWEDREQUESTSINWINDOW | Requests allowed per window. | 10 | +| EASYNEWS_CONNECTTIMEOUT | Easynews connect timeout (ms). | 20000 | +| EASYNEWS_SOCKETTIMEOUT | Easynews socket read timeout (ms). | 5000 | + +### Sonarr / Radarr + +| NAME | Explanation | Default | +|---------------------------|-------------------------------------------------------|-----------| +| SONARR_INTEGRATIONENABLED | Enable Sonarr integration (blocklist + re-search). | `false` | +| SONARR_HOST | Sonarr host. | localhost | +| SONARR_PORT | Sonarr port. | 8989 | +| SONARR_APIKEY | Sonarr API key. | | +| SONARR_CATEGORY | qBittorrent category mapped to Sonarr. | tv-sonarr | +| RADARR_INTEGRATIONENABLED | Enable Radarr integration. | `false` | +| RADARR_HOST | Radarr host. | localhost | +| RADARR_PORT | Radarr port. | 7878 | +| RADARR_APIKEY | Radarr API key. | | +| RADARR_CATEGORY | qBittorrent category mapped to Radarr. | radarr | + +### UI + +| NAME | Explanation | Default | +|-------------------------------|-----------------------------------------------------------------------------------------------------------------------|---------| +| DEBRIDAV_UI_GRAFANA_BASEURL | Base URL of a reachable Grafana. When set, the Dashboard tab embeds every dashboard under Grafana's `debridav` folder. | | +| DEBRIDAV_UI_GRAFANA_APIKEY | Optional Grafana API key if `/api/search` requires auth. Not needed for anonymous-viewer setups. | | ## Disclaimer diff --git a/build.gradle.kts b/build.gradle.kts index e37963d0..d79d165e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,3 +1,4 @@ +import com.github.gradle.node.npm.task.NpmTask import com.google.cloud.tools.jib.gradle.JibTask import dev.detekt.gradle.Detekt import dev.detekt.gradle.DetektCreateBaselineTask @@ -20,6 +21,7 @@ plugins { id("org.springframework.boot") version "4.0.3" id("com.google.cloud.tools.jib") version "3.5.3" id("io.github.simonhauck.release") version "1.5.1" + id("com.github.node-gradle.node") version "7.0.2" } application { @@ -61,6 +63,7 @@ tasks.jacocoTestReport { dependencies { implementation(platform(org.springframework.boot.gradle.plugin.SpringBootPlugin.BOM_COORDINATES)) + implementation(platform("org.springframework.cloud:spring-cloud-dependencies:${libs.versions.spring.cloud.get()}")) implementation(libs.spring.boot.starter.webmvc) implementation(libs.jackson.module.kotlin) @@ -90,6 +93,7 @@ dependencies { implementation(libs.resilience4j.kotlin) implementation(libs.resilience4j.ratelimiter) implementation(libs.resilience4j.retry) + implementation(libs.resilience4j.micrometer) implementation(libs.logstash.logback.encoder) implementation(libs.ktor.client.apache5) implementation(libs.ktor.client.java) @@ -97,6 +101,11 @@ dependencies { implementation(libs.sentry.spring.boot) implementation(libs.sentry.logback) implementation(libs.pgmq.kotlin.jvm) + implementation(libs.spring.cloud.context) + implementation(libs.spring.boot.starter.security) + implementation(libs.jjwt.api) + runtimeOnly(libs.jjwt.impl) + runtimeOnly(libs.jjwt.jackson) implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310") implementation("com.fasterxml.jackson.module:jackson-module-kotlin") @@ -114,6 +123,7 @@ dependencies { testImplementation(libs.sardine) testImplementation(libs.ktor.client.mock) testImplementation(libs.mock.nntp.server) + testImplementation(libs.spring.boot.starter.security.test) } java { @@ -154,10 +164,89 @@ configurations { } } +// Emit META-INF/build-info.properties so the running JVM can report its +// own version via Spring's BuildProperties bean. +springBoot { + buildInfo() +} + tasks.withType().configureEach { notCompatibleWithConfigurationCache("because https://github.com/GoogleContainerTools/jib/issues/3132") } +// --- Frontend build --- +// Builds the React frontend (debridav-frontend submodule) and bundles its +// static output into the Spring Boot JAR under /static/, so the backend +// serves the UI at /. Pin is tracked as a git submodule — run +// `git submodule update --init` on a fresh clone (or +// `actions/checkout@v4` with `submodules: true` in CI). Skipped if the +// submodule isn't checked out or -PskipFrontend=true is passed. + +val frontendDir = file("debridav-frontend") +val frontendStaticOutput = layout.buildDirectory.dir("generated/frontend/static") +val skipFrontend = providers.gradleProperty("skipFrontend").map { it == "true" }.orElse(false) +val hasFrontend = frontendDir.resolve("package.json").exists() + +node { + version.set("22.12.0") + download.set(true) + workDir.set(layout.buildDirectory.dir("nodejs")) + npmWorkDir.set(layout.buildDirectory.dir("npm")) + nodeProjectDir.set(frontendDir) +} + +// Cache-incompat reason shared across the frontend pipeline below. The +// node-gradle plugin (npmInstall, NpmTask) captures Project references at +// execution time, and our own onlyIf closures here also reference script +// state (`skipFrontend`, `hasFrontend`), which isn't serializable. Rather +// than fight it, mark each task as incompatible — matches the approach +// already taken for Jib higher up in this file. +val frontendCcReason = + "captures script/project references (node-gradle plugin + onlyIf closures)" + +tasks.npmInstall { + notCompatibleWithConfigurationCache(frontendCcReason) + onlyIf { !skipFrontend.get() && hasFrontend } +} + +val frontendBuild by tasks.registering(NpmTask::class) { + notCompatibleWithConfigurationCache(frontendCcReason) + description = "Build frontend static assets" + group = "frontend" + onlyIf { !skipFrontend.get() && hasFrontend } + dependsOn(tasks.npmInstall) + args.set(listOf("run", "build")) + inputs.dir(frontendDir.resolve("src")).optional() + inputs.dir(frontendDir.resolve("public")).optional() + inputs.files( + frontendDir.resolve("package.json"), + frontendDir.resolve("vite.config.ts"), + frontendDir.resolve("tsconfig.json"), + frontendDir.resolve("tsconfig.app.json"), + frontendDir.resolve("tsconfig.node.json"), + frontendDir.resolve("index.html"), + ).optional() + outputs.dir(frontendDir.resolve("dist")) +} + +val copyFrontend by tasks.registering(Copy::class) { + notCompatibleWithConfigurationCache(frontendCcReason) + description = "Copy built frontend into resources" + group = "frontend" + onlyIf { !skipFrontend.get() && hasFrontend } + dependsOn(frontendBuild) + from(frontendDir.resolve("dist")) + into(frontendStaticOutput) +} + +sourceSets.main { + resources.srcDir(layout.buildDirectory.dir("generated/frontend")) +} + +tasks.processResources { + dependsOn(copyFrontend) +} + jib { from { platforms { diff --git a/debridav-frontend b/debridav-frontend new file mode 160000 index 00000000..66375d1c --- /dev/null +++ b/debridav-frontend @@ -0,0 +1 @@ +Subproject commit 66375d1c58bf8bb02e9e19ae9a601885433effa1 diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml deleted file mode 100644 index 304c7e25..00000000 --- a/dev/docker-compose.yml +++ /dev/null @@ -1,92 +0,0 @@ -services: - rclone-dev: - container_name: rclone-dev - image: rclone/rclone:latest - restart: unless-stopped - environment: - TZ: Europe/Berlin - PUID: 1000 - PGID: 1000 - volumes: - - /home/william/debridavlocal/mounted:/data:rshared - - ./rclone.conf:/config/rclone/rclone.conf - cap_add: - - SYS_ADMIN - security_opt: - - apparmor:unconfined - devices: - - /dev/fuse:/dev/fuse:rwm - command: "mount debridav: /data --allow-other --allow-non-empty --dir-cache-time 0s --vfs-cache-mode off" - ports: - - "5572:5572" - networks: - - mediaserver - sonarr-debrid: - image: lscr.io/linuxserver/sonarr:latest - container_name: sonarr-debrid-dev - environment: - - PUID=1000 - - PGID=1000 - - TZ=Europe/Berlin - volumes: - - ./sonarr-config:/config - - /home/william/debridavlocal/mounted:/data:rshared - ports: - - "8990:8989" - depends_on: - - rclone-dev - restart: unless-stopped - networks: - - mediaserver - radarr-debrid-dev: - image: lscr.io/linuxserver/radarr:latest - container_name: radarr-debrid-dev - environment: - - PUID=1000 - - PGID=1000 - - TZ=Europe/Berlin - volumes: - - ./radarr-config:/config - - /home/william/debridav:/data:rshared - ports: - - 7878:7878 - depends_on: - - rclone-dev - restart: unless-stopped - networks: - - mediaserver - prowlarr-debrid-dev: - image: lscr.io/linuxserver/prowlarr:latest - container_name: prowlarr-debrid-dev - environment: - - PUID=1000 - - PGID=1000 - - TZ=Europe/Berlin - volumes: - - /home/william/debridavlocal/mounted:/data:rshared - - ./prowlarr-config:/config - ports: - - "9696:9696" - restart: unless-stopped - networks: - - mediaserver - radarr: - image: lscr.io/linuxserver/radarr:latest - container_name: radarr-debridav - environment: - - PUID=1000 - - PGID=1000 - - TZ=Europe/Berlin - volumes: - - ./radarr-config:/config - - /home/william/debridavlocal/mounted:/data:rshared - ports: - - "7878:7878" - depends_on: - - rclone-dev - restart: unless-stopped - networks: - - mediaserver -networks: - mediaserver: - diff --git a/dev/rclone.conf b/dev/rclone.conf deleted file mode 100644 index 300802a1..00000000 --- a/dev/rclone.conf +++ /dev/null @@ -1,5 +0,0 @@ -[debridav] -type = webdav -url = http://172.17.0.1:8080/ -vendor = other -pacer_min_sleep = 0 diff --git a/example/.env b/example/.env deleted file mode 100644 index dc1c67c9..00000000 --- a/example/.env +++ /dev/null @@ -1,53 +0,0 @@ -# -------------- Paths -------------- - -# DebriDav's internal storage ( NOT the path where DebriDav's WebDAV server should be mounted ) -# This only affects the internals of the DebriDav container, and should not need to be changed. -DEBRIDAV_ROOT_PATH=/debridav - -# Where DEBRIDAV_ROOT_PATH will be mapped to on the host FS -DEBRIDAV_ROOT_HOST_FS=/debridav-root - -# Where downloads will be placed, relative to DEBRIDAV_ROOT_PATH -DEBRIDAV_DOWNLOAD_PATH=/downloads - -# Where DebriDav's WebDAV server will be mounted inside the other containers ( Radarr, Sonarr etc). -# The Media Root of the arrs must be set under this directory -# Downloads will be visible to the other containers in /data/downloads with this configuration -DEBRIDAV_MOUNT_PATH_CONTAINERS=/data - -# Where DebriDavs WebDAV server will be mounted on the host FS -DEBRIDAV_MOUNT_PATH_HOST_FS=./debridav-mounted - -# -------------- Arrs -------------- - -SONARR_INTEGRATION_ENABLED=false -SONARR_HOST=sonarr-debridav -SONARR_PORT=8989 -SONARR_API_KEY= -RADARR_INTEGRATION_ENABLED=false -RADARR_HOST=radarr-debridav -RADARR_PORT=7878 -RADARR_API_KEY= - -# -------------- Debrid providers -------------- - -# Comma separated list of debrid providers. Allowed values are: premiumize, real_debrid, easynews, torbox -DEBRIDAV_DEBRID_CLIENTS= - -PREMIUMIZE_API_KEY= -REAL_DEBRID_API_KEY= -EASYNEWS_USERNAME= -EASYNEWS_PASSWORD= -TORBOX_API_KEY= - -# -------------- DATABASE -------------- -DEBRIDAV_DB_HOST=postgres-debridav -DEBRIDAV_DB_PORT=5432 -DEBRIDAV_DB_DATABASE_NAME=debridav -DEBRIDAV_DB_USERNAME=debridav -DEBRIDAV_DB_PASSWORD=debridav - -# -------------- MISC -------------- - -# Which port to expose DebriDav on on the host network -DEBRIDAV_PORT=8888 diff --git a/example/.env.example b/example/.env.example new file mode 100644 index 00000000..2b545a10 --- /dev/null +++ b/example/.env.example @@ -0,0 +1,85 @@ +# Most debridav settings — providers, *arr integration, NNTP pools, +# retry timings — are editable at runtime from the UI's Configuration +# pages. This file only needs the bootstrap essentials. + +# --- Required --- + +# Host:container UID/GID for file ownership (defaults to 1000) +PUID=1000 +PGID=1000 +TZ=Etc/UTC + +# Host directory where rclone will mount the WebDAV filesystem. +# Your media server (Jellyfin/Plex) reads from this path. +# Must already exist and be writable by PUID/PGID. +# +# Defaults to $HOME/debridav. On Ubuntu 23.10+ the AppArmor profile +# for fusermount3 only allows FUSE mounts under user home directories +# out of the box, so keep this path under $HOME unless you've loosened +# that profile on the host. +#RCLONE_MOUNT_PATH=/home/you/debridav + +# Database password (picked once, kept stable; stored in the pgdata volume) +POSTGRES_PASSWORD=changeme + +# WebDAV basic auth (rclone + anything mounting the WebDAV needs these) +DEBRIDAV_WEBDAV_USERNAME=debridav +DEBRIDAV_WEBDAV_PASSWORD=changeme + +# --- Optional pre-fill --- +# You can leave everything below blank and set it from the UI after first login. +# Anything set here becomes the default before a UI override is saved. + +# Comma-separated providers to enable on boot (e.g. real_debrid,torbox) +DEBRIDAV_DEBRID_CLIENTS= + +# Provider API keys (UI-editable) +REAL_DEBRID_API_KEY= +PREMIUMIZE_API_KEY= +TORBOX_API_KEY= +EASYNEWS_USERNAME= +EASYNEWS_PASSWORD= + +# Usenet (UI-editable). NNTP is enabled implicitly when at least one +# pool is configured — either via NNTP_HOST + credentials here, or by +# adding a pool from the UI's Configuration → NNTP → Server Pools tab. +NNTP_HOST= +NNTP_PORT=563 +NNTP_USERNAME= +NNTP_PASSWORD= +NNTP_USE_TLS=true + +# *arr integration (UI-editable) +SONARR_INTEGRATION_ENABLED=false +SONARR_HOST=sonarr +SONARR_PORT=8989 +SONARR_API_KEY= + +RADARR_INTEGRATION_ENABLED=false +RADARR_HOST=radarr +RADARR_PORT=7878 +RADARR_API_KEY= + +# --- Port overrides (only if you have conflicts) --- +DEBRIDAV_PORT=8080 +RCLONE_METRICS_PORT=9002 +RCLONE_RC_PORT=5572 +# rclone RC credentials — shared between rclone (which authenticates +# incoming requests) and debridav (which sends /vfs/refresh). Picked +# once, kept stable. +RCLONE_RC_USER=debridav +RCLONE_RC_PASSWORD=debridav +# *arrs stack (only used with docker-compose.arrs.yml) +SONARR_PORT_HOST=8989 +RADARR_PORT_HOST=7878 +PROWLARR_PORT_HOST=9696 +# Monitoring stack (only used with docker-compose.monitoring.yml) +PROMETHEUS_PORT=9090 +GRAFANA_PORT=3000 +GRAFANA_ADMIN_USER=admin +GRAFANA_ADMIN_PASSWORD=admin + +# URL the browser uses to reach Grafana for iframed dashboards. +# Defaults to localhost:3000 — if you access debridav from another +# machine, change this to e.g. http://your-server:3000 +UI_GRAFANA_BASEURL=http://localhost:3000 diff --git a/example/QUICKSTART.md b/example/QUICKSTART.md deleted file mode 100644 index d7df9e31..00000000 --- a/example/QUICKSTART.md +++ /dev/null @@ -1,183 +0,0 @@ -# Quickstart for docker compose - -This guide will help you get up and running with DebriDav and the *arr ecosystem. - -> [!WARNING] -> This guide is intended as a reference for how to set up DebriDav in a home environment, and is not suitable for -> deployment to a remote server. -> If you intend to deploy it on a remote server you should be comfortable with configuring firewalls and/or -> authentication proxies to prevent public access to DebriDav or any of the other services. - -## Requirements - -Docker, docker compose, and a basic understanding of how the *arr ecosystem works. - -## Configuring - -Open the .env file for editing. -Typically you need to change two values: - -- Set `DEBRIDAV_DEBRID-CLIENTS` to a comma separated list of debrid providers you would like to use. eg. - `premiumize,real_debrid`, or `premiumize`. If you add multiple providers they will be preferred in the order - specified. if `premiumize,real_debrid` is used, Real Debrid will only be used for torrents not cached at Premiumize. -- If using Premiumize, set the `PREMIUMIZE_API-KEY` property to your Premiumize api key, obtained by clicking the "Show - API Key" button at `https://www.premiumize.me/account` -- If using Real Debrid, set the `REAL-DEBRID_API-KEY` property to your real debrid API key, obtained at - `https://real-debrid.com/apitoken` -- If using EasyNews set `EASYNEWS_USERNAME` and `EASYNEWS_PASSWORD` to your EasyNews username and password respectively. -- Save when done. - -### Addtional configuration options - -In addition the the configuration options described in [README](../README.md#configuration), the following configuration -variables may be set for -docker compose: - -| NAME | Explanation | Default | -|--------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------| -| DEBRIDAV_MOUNT_PATH_LOCAL_FS | The path where DebriDav will be mounted on your host filesystem. | ./debridav | -| DEBRIDAV_MOUNT_PATH_CONTAINERS | The path where DebriDav will be mounted inside the docker containers. If kept at it's default values, downloads will be visible to the arrs in /data/downloads | /data | -| DEBRIDAV_ROOT_HOST_FS | The path on the host filesystem DebriDav will use for storage | ./debridav-storage | - -## Start the services - -Run `docker compose up --detach`, and verify that all services started successfully by running `docker container ls`. -If the DebriDav container failed to start examine the logs by running `docker logs `, where container id -is obtained from the output of `docker container ls` -Depending on your environment you may need to open the following ports in your firewall: - -- Radarr: 7878 -- Sonarr: 8989 -- Prowlarr: 9696 -- JellyFin: 8096 -- DebriDav: 8888 - -Once up and running, you will see some new directories appear. Each arr-service should have it's own directory where -configuration and databases are stored, and additionally you should see a `debridav` and a `debridav-files` directory. -The `debridav` directory is where rclone has mounted the debridav WebDav server to. You can open media files for playing -from this directory. The debridav-files directory is the internal storage of DebriDav. You should not need to do -anything there. You can change the name and location of these directories in `docker-compose.yaml` and/or `.env`. - -If you get a permissions error from rclone and have AppArmor running, you may need to disable it. - -## Configure Prowlarr - -Navigate to http://localhost:9696. You should be greeted with a welcome screen and asked to configure authentication. - -### Add an indexer - -Once authentication is configured, navigate to the Indexers section, and use the form to add an indexer. -Hint: The more popular well-known indexers will have better cache hit rates. - -### Add the download client - -Next, navigate to Settings -> Download Clients, and click the plus card. Under the torrents section, select qBittorrent. -Optionally change the name, and set the host to `debridav`, and leave the port at `8080`. Remove any values from the -username and password fields, and check the configuration by clicking the "Test" button. If you see a green tick, you're -all set and can save. - -Optionally add a usenet download client if you wish to use a usenet indexer for Easynews. Follow the same steps as above -to add SABnzbd as a download client. Set the host to `debridav`, and port to `8080`. SABnzbd requires that clients use -either username and password, or an API-key. DebriDav does not, so just fill in any non-null value ( eg. "a"/"a" ) in -the usernmame and password fields. - -All downloads will initially appear in debridav/downloads. Downloads added by Sonarr and Radarr will get moved to their -respective locations configured further down, while downloads added by Prowlarr stay in debridav/downloads. - -If the requested magnet is not available in any configured debrid services, adding the magnet will fail indicating -that the torrent is not cached. - -## Configure Sonarr/Radarr - -The steps for both of these services are exactly the same so they must be repeated for each of them. -Navigate to http://localhost:7878 and http://localhost:8989. Once again you will be asked to configure authentication. - -## Configure library - -From the directory containing `docker-compose.yaml`, create three new directories: - -- ./debridav/downloads -- ./debridav/tv -- ./debridav/movies - -Then, in both Sonarr and Radarr, navigate to Settings-> Media Management add click "Add Root Folder" - -- For Radarr, select /data/movies -- For Sonarr, select /data/tv - -> [!WARNING] -> Do not set the root folders outside of the DebriDav mount root ( /data in this case ). -> Doing so will cause Sonarr/Radarr to download the entire file. - -## Add download client - -Once done follow the same steps as for Prowlarr to add the download client in both Sonarr and Radarr. - -## Set up Prowlarr integrations - -In order for Radarr and Sonarr to be able to search for content, we need to set up the Prowlarr integration so that the -indexers we configured in Prowlarr can be used by Sonarr and Radarr. -Navigate to http://localhost:9696 and click on Settings -> Apps, and click the '+' card to add Sonarr and Radarr. - -For Sonarr: - -- Set Prowlarr Server to http://prowlarr-debridav:9696 -- Set Sonarr Server to http://sonarr-debridav:8989 -- Set API Key to the key obtained from http://localhost:8989/settings/general -- Click the test button to test the configuration, and save it if valid. - -For Radarr: - -- Set Prowlarr Server to http://prowlarr-debridav:9696 -- Set Radarr Server to http://radarr-debridav:7878 -- Set API Key to the key obtained from http://localhost:7878/settings/general -- Click the test button to test the configuration, and save it if valid. - -Once done, you should see the indexers you created in Prowlarr under Settings -> Indexers in both Sonarr and Radarr - -## Configure Arr integration with DebriDav - -> [!IMPORTANT] -> At the time of implementing this feature I was unaware of of a setting in Radarr/Sonarr that enables them to try a -> different release when receiving an error from the download client. Enabling `Redownload Failed` and -> `Redownload Failed from Interactive Search` at `/settings/downloadclients` achieves the same result as enabling the -> integration. Thus, it is recommended to do this rather than use the integration as DebriDav will return a 422 error -> response for un-cached torrents when the integration is disabled. - -DebriDav features an integration with the Arr-APIs in order to make the Arrs try a different release when a torrent is -not cached during an automatic search. The downside is that interactive searches will no longer feature instant feedback -on whether an item is cached or not. If you prefer using interactive search for manually selecting a release, it is -recommended to disable the integration. If you prefer using automatic search it is recommended to enable it. - -This feature only applies to torrents, as sabNZBD supports the concept of failed downloads whereas qBittorrent does -not. - -To enable the Sonarr API-integration, set `SONARR_INTEGRATION_ENABLED=true` in your `.env` file. - -To enable the Radarr API-integration, set `RADARR_INTEGRATION_ENABLED=true` in your `.env` file. - -### Get the API-keys - -The arrs will generate API-keys on their first run, so they will need to be started before we can get their API-keys. -Navigate to `/settings/general` in Sonarr/Radarr to get the keys, and apply them to `RADARR_API_KEY` and -`SONARR_API_KEY` -in your `.env` file respectively. - -Then restart the stack by running `docker compose stop && docker compose start` - -## Jellyfin - -Navigate to http://localhost:8096 and follow the set up wizard. The content will appear under /data. I recommend that -you add /data/tv as a tv library and /data/movies as a movie library. -As of right now, automatic adding of new files to libraries in Jellyfin is not working, so you may need to trigger -a scan manually if you've added new content. This may be fixed in a future release. - -And that's it! You should now be able to search for and download content with Prowlarr, Radarr and Sonarr. -Your content will be visible in the /debridav directory. - -## Monitoring - -This example comes with a preconfigured Grafana dashboard and Dozzle to allow for easier debugging. If you wish to -enable these additional services there is a docker compose file under example/monitoring. -See [MONITORING.md](monitoring/MONITORING.md) - diff --git a/example/README.md b/example/README.md new file mode 100644 index 00000000..d510650b --- /dev/null +++ b/example/README.md @@ -0,0 +1,165 @@ +# DebriDAV — Docker Compose example + +Three stacks in one directory, stackable via Compose overrides: + +- **Minimal** (`docker-compose.yml`) — debridav backend + Postgres + rclone mount. Enough to serve the WebDAV, run the UI, and have your media server read from a mounted directory. +- **Arrs** (`docker-compose.arrs.yml`) — adds Sonarr, Radarr, and Prowlarr co-located on the same network, with `/data` pointed at the same rclone mount. +- **Monitoring** (`docker-compose.monitoring.yml`) — adds Prometheus, Grafana (with pre-provisioned debridav dashboards), Postgres exporter, and cAdvisor. + +Arrs and monitoring files are *overrides*: you run them alongside the base file, not instead of it. They stack — combine any or all. + +## Quick start + +```bash +cp .env.example .env +# Edit .env: set POSTGRES_PASSWORD, DEBRIDAV_WEBDAV_USERNAME/PASSWORD, +# and RCLONE_MOUNT_PATH. Everything else can be configured from the UI. + +docker compose up -d +``` + +Wait ~30s for the backend to migrate the database, then: + +- **UI** → http://localhost:8080/ (bundled with the backend JAR) +- **WebDAV** → http://localhost:8080/webdav/ (basic auth: `DEBRIDAV_WEBDAV_USERNAME` / `..._PASSWORD`) +- **Mounted filesystem** → the path you set as `RCLONE_MOUNT_PATH` on the host + +Open the UI, head to the **Configuration** pages, and enable the debrid providers you use (add API keys, add an NNTP pool if you use Usenet, wire up Sonarr/Radarr). The settings persist in the database — no restart needed. + +Point Jellyfin/Plex at `RCLONE_MOUNT_PATH` for media. Point your *arrs at the debridav backend (qBittorrent-compatible API on `:8080`, SABnzbd-compatible API on `:8080/api`). + +## With *arrs (Sonarr / Radarr / Prowlarr) + +```bash +docker compose -f docker-compose.yml -f docker-compose.arrs.yml up -d +``` + +- **Sonarr** → http://localhost:8989 +- **Radarr** → http://localhost:7878 +- **Prowlarr** → http://localhost:9696 + +All three read their media from the same rclone mount as debridav (`/home/debridav/data` inside the containers). Configs live in named Docker volumes (`sonarr-config`, `radarr-config`, `prowlarr-config`). + +On first run, inside each *arr UI: + +- Add debridav as the download client — qBittorrent host `http://debridav:8080`, SABnzbd host `http://debridav:8080/api` for Usenet. +- Set the root/library folder to `/home/debridav/data/tv` (Sonarr) or `/home/debridav/data/movies` (Radarr). debridav creates these on demand. +- In Prowlarr, connect to Sonarr/Radarr via `http://sonarr:8989` / `http://radarr:7878`. + +If you also want debridav to push cleanup actions back to the *arrs (blocklist + research on failed downloads), set `SONARR_INTEGRATION_ENABLED=true` and `SONARR_API_KEY=…` in `.env` (same for Radarr). + +## With monitoring + +```bash +docker compose -f docker-compose.yml -f docker-compose.monitoring.yml up -d +``` + +- **Grafana** → http://localhost:3000 (default admin/admin; override in `.env`) +- **Prometheus** → http://localhost:9090 + +The monitoring stack also includes **scraparr**, a Prometheus exporter for Sonarr/Radarr. It's only useful when the arrs override is also running and `SONARR_API_KEY` / `RADARR_API_KEY` are set in `.env` — otherwise the Sonarr & Radarr dashboard renders empty. + +## Everything together + +```bash +docker compose \ + -f docker-compose.yml \ + -f docker-compose.arrs.yml \ + -f docker-compose.monitoring.yml \ + up -d +``` + +A tip: alias long invocations. E.g. in your shell: + +```bash +alias dc-full="docker compose -f docker-compose.yml -f docker-compose.arrs.yml -f docker-compose.monitoring.yml" +dc-full up -d +dc-full logs -f debridav +``` + +## Host FUSE prerequisites + +The `rclone` container mounts the WebDAV filesystem via FUSE. Two host-side requirements: + +- `/dev/fuse` accessible (default on most distros). +- `user_allow_other` in `/etc/fuse.conf` — required because the mount uses `--allow-other` so other containers (Jellyfin/Plex, the *arrs) can read it: + ```bash + grep -q '^user_allow_other' /etc/fuse.conf || echo 'user_allow_other' | sudo tee -a /etc/fuse.conf + ``` + +**Ubuntu 23.10+ note.** The default AppArmor profile for `fusermount3` only allows FUSE mounts under user home directories. `RCLONE_MOUNT_PATH` therefore defaults to `$HOME/debridav`. If you need the mount elsewhere (e.g. `/srv/debridav` or a separate disk), loosen the profile on the host: + +```bash +sudo ln -s /etc/apparmor.d/fusermount3 /etc/apparmor.d/disable/ +sudo apparmor_parser -R /etc/apparmor.d/fusermount3 +``` + +Or edit `/etc/apparmor.d/fusermount3` and change the `-> @{HOME}/**/` rule to `-> /**/`, then `sudo apparmor_parser -r /etc/apparmor.d/fusermount3`. + +## Rclone cache invalidation (optional) + +Rclone's directory cache can lag behind real filesystem state — newly imported files won't show up to the *arrs until the cache expires (see `--dir-cache-time` in the rclone command). To close this gap, debridav can call rclone's RC `/vfs/refresh` endpoint whenever a file is created, moved, or deleted. + +The compose stack wires this up by default but leaves it **off** in debridav: + +1. Rclone is already started with `--rc --rc-addr :5572 --rc-user/--rc-pass` (auth from `RCLONE_RC_USER` / `RCLONE_RC_PASSWORD` in `.env`). +2. Debridav already receives the URL and credentials via `DEBRIDAV_RCLONE_RC-URL` / `_RC-USER` / `_RC-PASSWORD`. +3. To activate: open the UI's **Configuration → Core** page, flip **Rclone Cache Invalidation** to on, save. No restart needed. + +Events are coalesced over a 500 ms window — bursts (e.g. an NZB import touching many files) produce one HTTP call per affected directory, not N calls per file. + +## Security scope + +This compose stack assumes you're running it on a private network. debridav's +built-in JWT auth protects the UI and its API-key endpoints, but everything +else — Grafana (anonymous Viewer), Prometheus, cAdvisor, rclone's RC port, +and the *arrs' web UIs — is published with its defaults. + +If you plan to expose any of this to the internet, put the whole stack behind +your own reverse proxy (Traefik, Caddy) and IdP (Authelia, Authentik, etc.). +TLS, forward-auth, rate limiting, and fine-grained access control are out of +scope for this example. + +## Config at boot vs. in the UI + +Only these need to be set in `.env` — they're the bootstrap essentials, required before the UI comes up: + +| Variable | What it is | +|---|---| +| `POSTGRES_PASSWORD` | Picked once, kept stable. Stored in the `debridav-pgdata` volume. | +| `DEBRIDAV_WEBDAV_USERNAME` / `_PASSWORD` | Basic auth for the WebDAV endpoint. rclone and anyone mounting the filesystem needs these. | +| `RCLONE_MOUNT_PATH` | Host path where rclone mounts the WebDAV filesystem. Must exist and be writable by `PUID:PGID`. | + +Everything else — which debrid providers are enabled, provider API keys, NNTP pools, *arr integration, cache sizes, retry timings — is editable from the UI's **Configuration** pages at runtime. Changes persist in the database and take effect without a restart. + +If you want to pre-seed any of that (e.g. to not have to click through the UI on a fresh deploy) the `.env.example` file has optional variables for all of them. + +## Volumes + +- `debridav-data` — backend's metadata filesystem (lightweight) +- `debridav-pgdata` — Postgres data dir (the important one; back this up) +- `prometheus-data`, `grafana-data` — only exist with the monitoring override + +All are named Docker volumes; inspect with `docker volume ls | grep debridav`. To wipe everything: `docker compose down -v`. + +## Updating + +```bash +docker compose pull debridav +docker compose up -d debridav +``` + +Flyway migrations run on every backend startup. + +## Mounting from outside the compose stack + +If you want to run rclone on the host OS (not in a container) — e.g. to mount debridav on a NAS or under systemd — here's an equivalent `rclone.conf` entry: + +```ini +[debridav] +type = webdav +url = http://:8080/webdav/ +vendor = other +user = +pass = +``` diff --git a/example/docker-compose.arrs.yml b/example/docker-compose.arrs.yml new file mode 100644 index 00000000..786eafa8 --- /dev/null +++ b/example/docker-compose.arrs.yml @@ -0,0 +1,56 @@ +services: + sonarr: + image: lscr.io/linuxserver/sonarr:latest + container_name: debridav-sonarr + restart: unless-stopped + environment: + PUID: ${PUID:-1000} + PGID: ${PGID:-1000} + TZ: ${TZ:-Etc/UTC} + volumes: + - sonarr-config:/config + - ${RCLONE_MOUNT_PATH:-$HOME/debridav}:/home/debridav/data:rshared + ports: + - "${SONARR_PORT_HOST:-8989}:8989" + depends_on: + - rclone + networks: + - debridav-network + + radarr: + image: lscr.io/linuxserver/radarr:latest + container_name: debridav-radarr + restart: unless-stopped + environment: + PUID: ${PUID:-1000} + PGID: ${PGID:-1000} + TZ: ${TZ:-Etc/UTC} + volumes: + - radarr-config:/config + - ${RCLONE_MOUNT_PATH:-$HOME/debridav}:/home/debridav/data:rshared + ports: + - "${RADARR_PORT_HOST:-7878}:7878" + depends_on: + - rclone + networks: + - debridav-network + + prowlarr: + image: lscr.io/linuxserver/prowlarr:latest + container_name: debridav-prowlarr + restart: unless-stopped + environment: + PUID: ${PUID:-1000} + PGID: ${PGID:-1000} + TZ: ${TZ:-Etc/UTC} + volumes: + - prowlarr-config:/config + ports: + - "${PROWLARR_PORT_HOST:-9696}:9696" + networks: + - debridav-network + +volumes: + sonarr-config: + radarr-config: + prowlarr-config: diff --git a/example/docker-compose.monitoring.yml b/example/docker-compose.monitoring.yml new file mode 100644 index 00000000..e6395d03 --- /dev/null +++ b/example/docker-compose.monitoring.yml @@ -0,0 +1,95 @@ +services: + # Extend the base debridav service with Grafana config so the UI's + # Dashboard tab can embed dashboards from this stack. The dashboard + # list is discovered dynamically from Grafana's `debridav` folder — + # no manual per-dashboard wiring needed. + debridav: + environment: + DEBRIDAV_UI_GRAFANA_BASEURL: ${UI_GRAFANA_BASEURL:-http://localhost:3000} + # Optional: only needed if Grafana requires auth for /api/search + #DEBRIDAV_UI_GRAFANA_APIKEY: ${UI_GRAFANA_APIKEY} + + prometheus: + image: prom/prometheus:v2.54.1 + container_name: debridav-prometheus + restart: unless-stopped + volumes: + - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus-data:/prometheus + ports: + - "${PROMETHEUS_PORT:-9090}:9090" + networks: + - debridav-network + + grafana: + image: grafana/grafana:11.2.0 + container_name: debridav-grafana + restart: unless-stopped + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin} + # Allow the debridav UI to embed dashboards via iframe + GF_SECURITY_ALLOW_EMBEDDING: "true" + # Anonymous read-only access so embedded iframes don't hit a login wall + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer + volumes: + - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - ./monitoring/grafana/defaults.ini:/etc/grafana/defaults.ini:ro + - grafana-data:/var/lib/grafana + ports: + - "${GRAFANA_PORT:-3000}:3000" + networks: + - debridav-network + + postgres-exporter: + image: quay.io/prometheuscommunity/postgres-exporter:v0.15.0 + container_name: debridav-postgres-exporter + restart: unless-stopped + environment: + DATA_SOURCE_URI: postgres:5432/debridav?sslmode=disable + DATA_SOURCE_USER: debridav + DATA_SOURCE_PASS: ${POSTGRES_PASSWORD} + PG_EXPORTER_EXTEND_QUERY_PATH: /custom-queries/queries.yaml + volumes: + - ./monitoring/pg-exporter:/custom-queries:ro + depends_on: + postgres: + condition: service_healthy + networks: + - debridav-network + + # Scrapes Sonarr/Radarr APIs and re-exposes as Prometheus metrics. + # Only useful when the arrs compose override is also running and + # SONARR_API_KEY / RADARR_API_KEY are set in .env. + scraparr: + image: ghcr.io/thecfu/scraparr:3.0.3 + container_name: debridav-scraparr + restart: unless-stopped + environment: + SONARR_URL: ${SONARR_URL_INTERNAL:-http://sonarr:8989} + SONARR_API_KEY: ${SONARR_API_KEY:-} + RADARR_URL: ${RADARR_URL_INTERNAL:-http://radarr:7878} + RADARR_API_KEY: ${RADARR_API_KEY:-} + networks: + - debridav-network + + cadvisor: + image: gcr.io/cadvisor/cadvisor:v0.52.1 + container_name: debridav-cadvisor + restart: unless-stopped + privileged: true + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker:/var/lib/docker:ro + - /dev/disk:/dev/disk:ro + devices: + - /dev/kmsg + networks: + - debridav-network + +volumes: + prometheus-data: + grafana-data: diff --git a/example/docker-compose.yaml b/example/docker-compose.yaml deleted file mode 100644 index dd03aaaa..00000000 --- a/example/docker-compose.yaml +++ /dev/null @@ -1,177 +0,0 @@ -services: - debridav: - image: ghcr.io/skjaere/debridav:v0.11.0 - container_name: debridav - restart: unless-stopped - environment: - - PUID=1000 - - PGID=1000 - - TZ=Europe/Berlin - - DEBRIDAV_ROOTPATH=${DEBRIDAV_ROOT_PATH} - - DEBRIDAV_DOWNLOADPATH=${DEBRIDAV_DOWNLOAD_PATH} - - DEBRIDAV_MOUNTPATH=${DEBRIDAV_MOUNT_PATH_CONTAINERS} - - DEBRIDAV_DEBRIDCLIENTS=${DEBRIDAV_DEBRID_CLIENTS} - - SPRING_DATASOURCE_URL=jdbc:postgresql://${DEBRIDAV_DB_HOST}:${DEBRIDAV_DB_PORT}/${DEBRIDAV_DB_DATABASE_NAME}?user=${DEBRIDAV_DB_USERNAME}&password=${DEBRIDAV_DB_PASSWORD} - - PREMIUMIZE_APIKEY=${PREMIUMIZE_API_KEY} - - REALDEBRID_APIKEY=${REAL_DEBRID_API_KEY} - - SONARR_INTEGRATIONENABLED=${SONARR_INTEGRATION_ENABLED} - - SONARR_HOST=${SONARR_HOST} - - SONARR_PORT=${SONARR_PORT} - - SONARR_APIKEY=${SONARR_API_KEY} - - RADARR_INTEGRATIONENABLED=${RADARR_INTEGRATION_ENABLED} - - RADARR_HOST=${RADARR_HOST} - - RADARR_PORT=${RADARR_PORT} - - RADARR_APIKEY=${RADARR_API_KEY} - - EASYNEWS_USERNAME=${EASYNEWS_USERNAME} - - EASYNEWS_PASSWORD=${EASYNEWS_PASSWORD} - - TORBOX_APIKEY=${TORBOX_API_KEY} - ports: - - 8081:8080 - - "8000:8000" - networks: - - debridav-network - volumes: - - ${DEBRIDAV_ROOT_HOST_FS}:${DEBRIDAV_ROOT_PATH} - healthcheck: - test: [ "CMD-SHELL", "bash -c 'exec 3<>/dev/tcp/localhost/8080 && echo -e \"GET /actuator/health/readiness HTTP/1.0\\r\\nHost: localhost\\r\\n\\r\\n\" >&3 && grep -q \"200\" <&3'" ] - interval: 2s - start_period: 2s - retries: 1000 - depends_on: - postgres-debridav: - condition: service_healthy - rclone: - container_name: rclone - image: rclone/rclone:latest - restart: unless-stopped - environment: - TZ: Europe/Berlin - PUID: 1000 - PGID: 1000 - volumes: - - ${DEBRIDAV_MOUNT_PATH_HOST_FS}:${DEBRIDAV_MOUNT_PATH_CONTAINERS}:rshared - - ./rclone.conf:/config/rclone/rclone.conf - cap_add: - - SYS_ADMIN - security_opt: - - apparmor:unconfined - devices: - - /dev/fuse:/dev/fuse:rwm - command: "mount debridav: $DEBRIDAV_MOUNT_PATH_CONTAINERS - --allow-other - --allow-non-empty - --vfs-cache-mode off - --rc-enable-metrics - --metrics-addr :9002 - --low-level-retries=1 - --dir-cache-time=1s - -vv - " - ports: - - "5572:5572" - - "9002:9002" - depends_on: - debridav: - condition: service_healthy - networks: - - debridav-network - sonarr: - image: lscr.io/linuxserver/sonarr:latest - container_name: sonarr-debridav - environment: - - PUID=1000 - - PGID=1000 - - TZ=Europe/Berlin - volumes: - - ./sonarr-config:/config - - ${DEBRIDAV_MOUNT_PATH_HOST_FS}:${DEBRIDAV_MOUNT_PATH_CONTAINERS}:rshared - ports: - - 8989:8989 - depends_on: - - rclone - restart: unless-stopped - labels: - filebeat_enabled: - networks: - - debridav-network - radarr: - image: lscr.io/linuxserver/radarr:latest - container_name: radarr-debridav - environment: - - PUID=1000 - - PGID=1000 - - TZ=Europe/Berlin - volumes: - - ./radarr-config:/config - - ${DEBRIDAV_MOUNT_PATH_HOST_FS}:${DEBRIDAV_MOUNT_PATH_CONTAINERS}:rshared - ports: - - "7878:7878" - depends_on: - - rclone - restart: unless-stopped - networks: - - debridav-network - jellyfin: - image: jellyfin/jellyfin:latest - container_name: jellyfin-debridav - environment: - - PUID=1000 - - PGID=1000 - - TZ=Europe/Berlin - - JELLYFIN_FFmpeg__probesize=30M - - JELLYFIN_FFmpeg__analyzeduration=10M - #- JELLYFIN_PublishedServerUrl=192.168.0.5 #optional - ports: - - "8096:8096" - - "8920:8920" #optional - - "7359:7359/udp" #optional - #- 1900:1900/udp #optional - restart: unless-stopped - depends_on: - - rclone - volumes: - - ./jellyfin-config:/config - - ${DEBRIDAV_MOUNT_PATH_HOST_FS}:${DEBRIDAV_MOUNT_PATH_CONTAINERS}:rshared - networks: - - debridav-network - prowlarr-debridav: - image: lscr.io/linuxserver/prowlarr:latest - container_name: prowlarr-debridav - environment: - - PUID=1000 - - PGID=1000 - - TZ=Europe/Berlin - volumes: - - ./prowlarr-config:/config - ports: - - "9696:9696" - restart: unless-stopped - depends_on: - - rclone - networks: - - debridav-network - postgres-debridav: - image: postgres:17 - container_name: postgres-debridav - environment: - - POSTGRES_PASSWORD=debridav - - POSTGRES_USER=debridav - - PGDATA=/var/lib/postgresql/data/pgdata - - PGUSER=debridav - volumes: - - ./pgdata:/var/lib/postgresql/data - ports: - - "5432:5432" - healthcheck: - test: [ "CMD-SHELL", "pg_isready", "-d", "debridav" ] - interval: 1s - timeout: 60s - retries: 10 - start_period: 2s - networks: - - debridav-network - restart: unless-stopped -networks: - debridav-network: - name: debridav - diff --git a/example/docker-compose.yml b/example/docker-compose.yml new file mode 100644 index 00000000..14f00f50 --- /dev/null +++ b/example/docker-compose.yml @@ -0,0 +1,149 @@ +services: + debridav: + image: ghcr.io/skjaere/debridav:latest + container_name: debridav + restart: unless-stopped + environment: + PUID: ${PUID:-1000} + PGID: ${PGID:-1000} + TZ: ${TZ:-Etc/UTC} + DEBRIDAV_ROOTPATH: /data/debridav + DEBRIDAV_DOWNLOADPATH: /downloads + DEBRIDAV_MOUNTPATH: /home/debridav/data + DEBRIDAV_DEBRIDCLIENTS: ${DEBRIDAV_DEBRID_CLIENTS} + DEBRIDAV_WEBDAV_USERNAME: ${DEBRIDAV_WEBDAV_USERNAME} + DEBRIDAV_WEBDAV_PASSWORD: ${DEBRIDAV_WEBDAV_PASSWORD} + SPRING_DATASOURCE_URL: jdbc:postgresql://postgres:5432/debridav?user=debridav&password=${POSTGRES_PASSWORD} + # Debrid providers (leave blank for unused ones) + REALDEBRID_APIKEY: ${REAL_DEBRID_API_KEY:-} + PREMIUMIZE_APIKEY: ${PREMIUMIZE_API_KEY:-} + TORBOX_APIKEY: ${TORBOX_API_KEY:-} + EASYNEWS_USERNAME: ${EASYNEWS_USERNAME:-} + EASYNEWS_PASSWORD: ${EASYNEWS_PASSWORD:-} + # NNTP (enabled implicitly when at least one pool is configured) + NNTP_POOLS_0_HOST: ${NNTP_HOST:-} + NNTP_POOLS_0_PORT: ${NNTP_PORT:-563} + NNTP_POOLS_0_USERNAME: ${NNTP_USERNAME:-} + NNTP_POOLS_0_PASSWORD: ${NNTP_PASSWORD:-} + NNTP_POOLS_0_USE_TLS: ${NNTP_USE_TLS:-true} + # *arr integration (optional) + SONARR_INTEGRATIONENABLED: ${SONARR_INTEGRATION_ENABLED:-false} + SONARR_HOST: ${SONARR_HOST:-} + SONARR_PORT: ${SONARR_PORT:-8989} + SONARR_APIKEY: ${SONARR_API_KEY:-} + RADARR_INTEGRATIONENABLED: ${RADARR_INTEGRATION_ENABLED:-false} + RADARR_HOST: ${RADARR_HOST:-} + RADARR_PORT: ${RADARR_PORT:-7878} + RADARR_APIKEY: ${RADARR_API_KEY:-} + # rclone RC (how debridav reaches rclone to invalidate its VFS cache) + DEBRIDAV_RCLONE_RC-URL: http://rclone:5572 + DEBRIDAV_RCLONE_RC-USER: ${RCLONE_RC_USER:-debridav} + DEBRIDAV_RCLONE_RC-PASSWORD: ${RCLONE_RC_PASSWORD:-debridav} + volumes: + - debridav-data:/data/debridav + ports: + - "${DEBRIDAV_PORT:-8080}:8080" + # Image is trimmed JRE — no curl/wget. Use bash's /dev/tcp to probe + # the readiness actuator directly. + healthcheck: + test: + - CMD-SHELL + - | + bash -c ' + exec 3<>/dev/tcp/localhost/8080 && + printf "GET /actuator/health/readiness HTTP/1.0\r\nHost: localhost\r\n\r\n" >&3 && + grep -q "200 " <&3 + ' + interval: 5s + timeout: 3s + start_period: 10s + retries: 60 + depends_on: + postgres: + condition: service_healthy + networks: + - debridav-network + + postgres: + image: postgres:17 + container_name: debridav-postgres + restart: unless-stopped + environment: + POSTGRES_USER: debridav + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: debridav + PGDATA: /var/lib/postgresql/data/pgdata + volumes: + - debridav-pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U debridav -d debridav"] + interval: 2s + timeout: 5s + retries: 30 + networks: + - debridav-network + + rclone: + image: rclone/rclone:latest + container_name: debridav-rclone + restart: unless-stopped + environment: + TZ: ${TZ:-Etc/UTC} + PUID: ${PUID:-1000} + PGID: ${PGID:-1000} + WEBDAV_USER: ${DEBRIDAV_WEBDAV_USERNAME} + WEBDAV_PASS_PLAIN: ${DEBRIDAV_WEBDAV_PASSWORD} + RCLONE_RC_USER_VAL: ${RCLONE_RC_USER:-debridav} + RCLONE_RC_PASS_VAL: ${RCLONE_RC_PASSWORD:-debridav} + volumes: + - ${RCLONE_MOUNT_PATH:-$HOME/debridav}:/home/debridav/data:rshared + cap_add: + - SYS_ADMIN + security_opt: + - apparmor:unconfined + devices: + - /dev/fuse:/dev/fuse:rwm + # Writes an rclone.conf at startup with the WebDAV password obscured, + # then mounts. Keeping the config in a file (vs. env vars) matches the + # pattern most rclone users are familiar with. + entrypoint: /bin/sh + command: + - -c + - | + OBSCURED=$$(rclone obscure "$$WEBDAV_PASS_PLAIN") + mkdir -p /config/rclone + cat > /config/rclone/rclone.conf < { val registration = FilterRegistrationBean(SpringMiltonFilter()) registration.setName("MiltonFilter") - registration.addUrlPatterns("/*") - registration.addInitParameter("milton.exclude.paths", "/files,/api,/version,/sabnzbd,/actuator") + registration.addUrlPatterns("/webdav/*") registration.addInitParameter( "resource.factory.class", "io.skjaere.debrid.resource.StreamableResourceFactory" ) @@ -44,9 +45,7 @@ class DebridavConfiguration { @Bean fun httpClient(debridavConfigurationProperties: DebridavConfigurationProperties): HttpClient = HttpClient(CIO) { - install(HttpTimeout) { - connectTimeoutMillis = debridavConfigurationProperties.connectTimeoutMilliseconds - } + install(HttpTimeout) install(ContentNegotiation) { json( Json { @@ -56,6 +55,15 @@ class DebridavConfiguration { } ) } + install( + createClientPlugin("DynamicConnectTimeout") { + onRequest { request, _ -> + request.timeout { + connectTimeoutMillis = debridavConfigurationProperties.connectTimeoutMilliseconds + } + } + } + ) } @Bean diff --git a/src/main/kotlin/io/skjaere/debridav/FrontendRoutingConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/FrontendRoutingConfiguration.kt new file mode 100644 index 00000000..15f91689 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/FrontendRoutingConfiguration.kt @@ -0,0 +1,31 @@ +package io.skjaere.debridav + +import org.springframework.context.annotation.Configuration +import org.springframework.web.servlet.config.annotation.ViewControllerRegistry +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer + +/** + * Forwards the SPA's client-side routes to `/index.html` so React Router can + * handle them on page load / hard refresh. Without this, deep links like + * `/config/health-check` return 404 because Spring only serves `/index.html` + * at the root and no controller matches the path. + * + * Keep in sync with the frontend router in `debridav-frontend/src/router/index.tsx`. + */ +@Configuration +class FrontendRoutingConfiguration : WebMvcConfigurer { + override fun addViewControllers(registry: ViewControllerRegistry) { + val topLevelRoutes = listOf( + "/login", + "/files", + "/usenet", + "/torrents", + "/health", + "/logs" + ) + topLevelRoutes.forEach { path -> + registry.addViewController(path).setViewName("forward:/index.html") + } + registry.addViewController("/config/{*rest}").setViewName("forward:/index.html") + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/MicrometerBridgeConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/MicrometerBridgeConfiguration.kt new file mode 100644 index 00000000..60cfdc17 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/MicrometerBridgeConfiguration.kt @@ -0,0 +1,31 @@ +package io.skjaere.debridav + +import io.micrometer.core.instrument.MeterRegistry +import io.micrometer.core.instrument.Metrics +import jakarta.annotation.PostConstruct +import jakarta.annotation.PreDestroy +import org.springframework.context.annotation.Configuration + +/** + * Wires Micrometer's static [Metrics.globalRegistry] composite to the + * Spring-managed [MeterRegistry] bean so meters registered by libraries + * that reach for `Metrics.globalRegistry` directly (e.g. `nzb-streamer`) + * end up on the same registry as Spring's own. That way + * `meterRegistry.find(...)` can locate every meter in the app, regardless + * of where it was registered, and `/actuator/prometheus` shows a single + * consistent view. + */ +@Configuration +class MicrometerBridgeConfiguration( + private val meterRegistry: MeterRegistry, +) { + @PostConstruct + fun addToGlobalRegistry() { + Metrics.addRegistry(meterRegistry) + } + + @PreDestroy + fun removeFromGlobalRegistry() { + Metrics.removeRegistry(meterRegistry) + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/Resilience4jConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/Resilience4jConfiguration.kt index feba9a50..36f2a7d1 100644 --- a/src/main/kotlin/io/skjaere/debridav/Resilience4jConfiguration.kt +++ b/src/main/kotlin/io/skjaere/debridav/Resilience4jConfiguration.kt @@ -1,15 +1,26 @@ package io.skjaere.debridav +import io.github.resilience4j.micrometer.tagged.TaggedRateLimiterMetrics +import io.github.resilience4j.micrometer.tagged.TaggedRetryMetrics import io.github.resilience4j.ratelimiter.RateLimiterRegistry import io.github.resilience4j.retry.RetryRegistry +import io.micrometer.core.instrument.MeterRegistry import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration @Configuration class Resilience4jConfiguration { @Bean - fun rateLimiterRegistry(): RateLimiterRegistry = RateLimiterRegistry.ofDefaults() + fun rateLimiterRegistry(meterRegistry: MeterRegistry): RateLimiterRegistry { + val registry = RateLimiterRegistry.ofDefaults() + TaggedRateLimiterMetrics.ofRateLimiterRegistry(registry).bindTo(meterRegistry) + return registry + } @Bean - fun retryRegistry(): RetryRegistry = RetryRegistry.ofDefaults() + fun retryRegistry(meterRegistry: MeterRegistry): RetryRegistry { + val registry = RetryRegistry.ofDefaults() + TaggedRetryMetrics.ofRetryRegistry(registry).bindTo(meterRegistry) + return registry + } } diff --git a/src/main/kotlin/io/skjaere/debridav/arrs/ArrConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/arrs/ArrConfiguration.kt index 8f42df81..12320818 100644 --- a/src/main/kotlin/io/skjaere/debridav/arrs/ArrConfiguration.kt +++ b/src/main/kotlin/io/skjaere/debridav/arrs/ArrConfiguration.kt @@ -1,12 +1,12 @@ package io.skjaere.debridav.arrs interface ArrConfiguration { - val host: String - val port: Int - val apiBasePath: String - val apiKey: String - val category: String - val integrationEnabled: Boolean + var host: String + var port: Int + var apiBasePath: String + var apiKey: String + var category: String + var integrationEnabled: Boolean fun getApiBaseUrl(): String = "http://$host:$port$apiBasePath" } diff --git a/src/main/kotlin/io/skjaere/debridav/arrs/ArrService.kt b/src/main/kotlin/io/skjaere/debridav/arrs/ArrService.kt index 88002299..7a7406ba 100644 --- a/src/main/kotlin/io/skjaere/debridav/arrs/ArrService.kt +++ b/src/main/kotlin/io/skjaere/debridav/arrs/ArrService.kt @@ -13,11 +13,11 @@ class ArrService( fun getClientForCategory(category: String): ArrClient? = arrClients.firstOrNull { it.getCategory() == category } - suspend fun deleteFileAndSearch(itemName: String, category: String) { + suspend fun deleteFileAndSearch(itemName: String, category: String): Boolean { logger.info("Deleting file and triggering search for {} in Arrs", itemName) - getClientForCategory(category)?.let { client -> + return getClientForCategory(category)?.let { client -> client.deleteFileAndSearch(itemName) - } + } ?: false } suspend fun blocklist(downloadId: String, category: String) { diff --git a/src/main/kotlin/io/skjaere/debridav/arrs/RadarrConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/arrs/RadarrConfigurationProperties.kt index 1fd1426a..3fbe5cb8 100644 --- a/src/main/kotlin/io/skjaere/debridav/arrs/RadarrConfigurationProperties.kt +++ b/src/main/kotlin/io/skjaere/debridav/arrs/RadarrConfigurationProperties.kt @@ -1,14 +1,21 @@ package io.skjaere.debridav.arrs +import io.skjaere.debridav.config.ConfigProperty import org.springframework.boot.context.properties.ConfigurationProperties @ConfigurationProperties(prefix = "radarr") -data class RadarrConfigurationProperties( - override val integrationEnabled: Boolean, - override val host: String, - override val port: Int = 7878, - override val apiBasePath: String = "/api/v3", - override val apiKey: String, - override val category: String, - ): ArrConfiguration - +class RadarrConfigurationProperties : ArrConfiguration { + @ConfigProperty(name = "Integration Enabled", description = "Enable Radarr integration") + override var integrationEnabled: Boolean = false + @ConfigProperty(name = "Host", description = "Radarr host") + override var host: String = "" + @ConfigProperty(name = "Port", description = "Radarr port") + @Suppress("MagicNumber") + override var port: Int = 7878 + @ConfigProperty(name = "API Base Path", description = "Radarr API base path", advanced = true) + override var apiBasePath: String = "/api/v3" + @ConfigProperty(name = "API Key", description = "Radarr API key", sensitive = true) + override var apiKey: String = "" + @ConfigProperty(name = "Category", description = "Radarr category") + override var category: String = "" +} diff --git a/src/main/kotlin/io/skjaere/debridav/arrs/SonarrConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/arrs/SonarrConfigurationProperties.kt index 4836adc4..a3c1a4dc 100644 --- a/src/main/kotlin/io/skjaere/debridav/arrs/SonarrConfigurationProperties.kt +++ b/src/main/kotlin/io/skjaere/debridav/arrs/SonarrConfigurationProperties.kt @@ -1,13 +1,21 @@ package io.skjaere.debridav.arrs +import io.skjaere.debridav.config.ConfigProperty import org.springframework.boot.context.properties.ConfigurationProperties @ConfigurationProperties(prefix = "sonarr") -data class SonarrConfigurationProperties( - override val integrationEnabled: Boolean, - override val host: String, - override val port: Int = 8989, - override val apiBasePath: String = "/api/v3", - override val apiKey: String, - override val category: String, -): ArrConfiguration +class SonarrConfigurationProperties : ArrConfiguration { + @ConfigProperty(name = "Integration Enabled", description = "Enable Sonarr integration") + override var integrationEnabled: Boolean = false + @ConfigProperty(name = "Host", description = "Sonarr host") + override var host: String = "" + @ConfigProperty(name = "Port", description = "Sonarr port") + @Suppress("MagicNumber") + override var port: Int = 8989 + @ConfigProperty(name = "API Base Path", description = "Sonarr API base path", advanced = true) + override var apiBasePath: String = "/api/v3" + @ConfigProperty(name = "API Key", description = "Sonarr API key", sensitive = true) + override var apiKey: String = "" + @ConfigProperty(name = "Category", description = "Sonarr category") + override var category: String = "" +} diff --git a/src/main/kotlin/io/skjaere/debridav/arrs/client/ArrClient.kt b/src/main/kotlin/io/skjaere/debridav/arrs/client/ArrClient.kt index 3699f311..131a87fd 100644 --- a/src/main/kotlin/io/skjaere/debridav/arrs/client/ArrClient.kt +++ b/src/main/kotlin/io/skjaere/debridav/arrs/client/ArrClient.kt @@ -3,5 +3,5 @@ package io.skjaere.debridav.arrs.client interface ArrClient : BaseArrClient { suspend fun getItemIdFromName(name: String): Long? fun getCategory(): String - suspend fun deleteFileAndSearch(name: String) + suspend fun deleteFileAndSearch(name: String): Boolean } diff --git a/src/main/kotlin/io/skjaere/debridav/arrs/client/RadarrApiClient.kt b/src/main/kotlin/io/skjaere/debridav/arrs/client/RadarrApiClient.kt index a021ad54..46b4ef53 100644 --- a/src/main/kotlin/io/skjaere/debridav/arrs/client/RadarrApiClient.kt +++ b/src/main/kotlin/io/skjaere/debridav/arrs/client/RadarrApiClient.kt @@ -4,6 +4,7 @@ import io.ktor.client.HttpClient import io.ktor.client.call.body import io.ktor.client.request.accept import io.ktor.client.request.delete +import io.ktor.client.request.get import io.ktor.client.request.header import io.ktor.client.request.post import io.ktor.client.request.setBody @@ -13,9 +14,12 @@ import io.ktor.http.contentType import io.ktor.http.isSuccess import io.skjaere.debridav.arrs.RadarrConfigurationProperties import io.skjaere.debridav.arrs.client.models.radarr.RadarrParseResponse +import io.skjaere.debridav.config.ConfigurationTester +import io.skjaere.debridav.config.TestResult import org.slf4j.LoggerFactory import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.stereotype.Component +import kotlin.reflect.KClass @Component @ConditionalOnExpression("\${radarr.integration-enabled:true}") @@ -23,7 +27,7 @@ class RadarrApiClient( private val httpClient: HttpClient, private val radarrConfigurationProperties: RadarrConfigurationProperties ) : BaseArrClient by DefaultBaseArrClient(httpClient, radarrConfigurationProperties), - ArrClient { + ArrClient, ConfigurationTester { private val logger = LoggerFactory.getLogger(RadarrApiClient::class.java) override suspend fun getItemIdFromName(name: String): Long { @@ -32,10 +36,15 @@ class RadarrApiClient( override fun getCategory(): String = radarrConfigurationProperties.category - override suspend fun deleteFileAndSearch(name: String) { + override suspend fun deleteFileAndSearch(name: String): Boolean { val parseResponse = parse(name).body() val movie = parseResponse.movie + if (movie.id == 0L) { + logger.warn("No movie found for '{}' in Radarr", name) + return false + } + if (movie.movieFileId > 0) { logger.info("Deleting movie file {} for '{}'", movie.movieFileId, name) val deleteResponse = httpClient.delete( @@ -70,5 +79,30 @@ class RadarrApiClient( searchResponse.bodyAsText() ) } + return true + } + + override val configurationClass: KClass<*> = RadarrConfigurationProperties::class + override val label: String = "Radarr" + + @Suppress("TooGenericExceptionCaught") + override suspend fun test(overrides: Map): TestResult = try { + val host = overrides["radarr.host"] ?: radarrConfigurationProperties.host + val port = overrides["radarr.port"]?.toIntOrNull() ?: radarrConfigurationProperties.port + val apiBasePath = overrides["radarr.api-base-path"] ?: radarrConfigurationProperties.apiBasePath + val apiKey = overrides["radarr.api-key"] ?: radarrConfigurationProperties.apiKey + val baseUrl = "http://$host:$port$apiBasePath" + + val response = httpClient.get("$baseUrl/system/status") { + accept(ContentType.Application.Json) + header("X-Api-Key", apiKey) + } + if (response.status.isSuccess()) { + TestResult(success = true, message = "Connected successfully") + } else { + TestResult(success = false, message = "HTTP ${response.status.value}: ${response.bodyAsText()}") + } + } catch (e: Exception) { + TestResult(success = false, message = e.message ?: "Unknown error") } } diff --git a/src/main/kotlin/io/skjaere/debridav/arrs/client/SonarrApiClient.kt b/src/main/kotlin/io/skjaere/debridav/arrs/client/SonarrApiClient.kt index cc87d861..e56aba8b 100644 --- a/src/main/kotlin/io/skjaere/debridav/arrs/client/SonarrApiClient.kt +++ b/src/main/kotlin/io/skjaere/debridav/arrs/client/SonarrApiClient.kt @@ -4,6 +4,7 @@ import io.ktor.client.HttpClient import io.ktor.client.call.body import io.ktor.client.request.accept import io.ktor.client.request.delete +import io.ktor.client.request.get import io.ktor.client.request.header import io.ktor.client.request.post import io.ktor.client.request.setBody @@ -13,9 +14,12 @@ import io.ktor.http.contentType import io.ktor.http.isSuccess import io.skjaere.debridav.arrs.SonarrConfigurationProperties import io.skjaere.debridav.arrs.client.models.sonarr.SonarrParseResponse +import io.skjaere.debridav.config.ConfigurationTester +import io.skjaere.debridav.config.TestResult import org.slf4j.LoggerFactory import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.stereotype.Component +import kotlin.reflect.KClass @Component @ConditionalOnExpression($$"${sonarr.integration-enabled:true}") @@ -23,7 +27,7 @@ class SonarrApiClient( private val httpClient: HttpClient, private val sonarrConfigurationProperties: SonarrConfigurationProperties ) : BaseArrClient by DefaultBaseArrClient(httpClient, sonarrConfigurationProperties), - ArrClient { + ArrClient, ConfigurationTester { private val logger = LoggerFactory.getLogger(SonarrApiClient::class.java) override suspend fun getItemIdFromName(name: String): Long? { @@ -32,12 +36,12 @@ class SonarrApiClient( override fun getCategory(): String = sonarrConfigurationProperties.category - override suspend fun deleteFileAndSearch(name: String) { + override suspend fun deleteFileAndSearch(name: String): Boolean { val parseResponse = parse(name).body() val episodes = parseResponse.episodes if (episodes.isEmpty()) { logger.warn("No episodes found for '{}' in Sonarr", name) - return + return false } episodes.filter { it.episodeFileId > 0 }.forEach { episode -> @@ -75,5 +79,30 @@ class SonarrApiClient( searchResponse.bodyAsText() ) } + return true + } + + override val configurationClass: KClass<*> = SonarrConfigurationProperties::class + override val label: String = "Sonarr" + + @Suppress("TooGenericExceptionCaught") + override suspend fun test(overrides: Map): TestResult = try { + val host = overrides["sonarr.host"] ?: sonarrConfigurationProperties.host + val port = overrides["sonarr.port"]?.toIntOrNull() ?: sonarrConfigurationProperties.port + val apiBasePath = overrides["sonarr.api-base-path"] ?: sonarrConfigurationProperties.apiBasePath + val apiKey = overrides["sonarr.api-key"] ?: sonarrConfigurationProperties.apiKey + val baseUrl = "http://$host:$port$apiBasePath" + + val response = httpClient.get("$baseUrl/system/status") { + accept(ContentType.Application.Json) + header("X-Api-Key", apiKey) + } + if (response.status.isSuccess()) { + TestResult(success = true, message = "Connected successfully") + } else { + TestResult(success = false, message = "HTTP ${response.status.value}: ${response.bodyAsText()}") + } + } catch (e: Exception) { + TestResult(success = false, message = e.message ?: "Unknown error") } } diff --git a/src/main/kotlin/io/skjaere/debridav/category/Category.kt b/src/main/kotlin/io/skjaere/debridav/category/Category.kt index 7371eb8c..4a1f69c9 100644 --- a/src/main/kotlin/io/skjaere/debridav/category/Category.kt +++ b/src/main/kotlin/io/skjaere/debridav/category/Category.kt @@ -21,4 +21,12 @@ open class Category() { this.name = name this.downloadPath = downloadPath } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is Category) return false + return name != null && name == other.name + } + + override fun hashCode(): Int = name?.hashCode() ?: 0 } diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigApiExceptionHandler.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigApiExceptionHandler.kt new file mode 100644 index 00000000..112333cf --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigApiExceptionHandler.kt @@ -0,0 +1,22 @@ +package io.skjaere.debridav.config + +import org.springframework.http.HttpStatus +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.ExceptionHandler +import org.springframework.web.bind.annotation.RestControllerAdvice + +@RestControllerAdvice(assignableTypes = [ConfigOverrideController::class]) +class ConfigApiExceptionHandler { + + @ExceptionHandler(KeyNotWhitelistedException::class) + fun handleNotWhitelisted(ex: KeyNotWhitelistedException): ResponseEntity = + ResponseEntity.status(HttpStatus.BAD_REQUEST) + .body(ErrorResponse(ex.message ?: "Key not whitelisted")) + + @ExceptionHandler(OverrideNotFoundException::class) + fun handleNotFound(ex: OverrideNotFoundException): ResponseEntity = + ResponseEntity.status(HttpStatus.NOT_FOUND) + .body(ErrorResponse(ex.message ?: "Override not found")) +} + +data class ErrorResponse(val error: String) diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigOverride.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverride.kt new file mode 100644 index 00000000..712c4cc6 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverride.kt @@ -0,0 +1,32 @@ +package io.skjaere.debridav.config + +import jakarta.persistence.Column +import jakarta.persistence.Entity +import jakarta.persistence.GeneratedValue +import jakarta.persistence.GenerationType +import jakarta.persistence.Id +import jakarta.persistence.Table +import java.time.Instant + +@Entity +@Table(name = "config_override") +open class ConfigOverride { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + open var id: Long? = null + + @Column(name = "prop_key", nullable = false, unique = true) + open var propKey: String = "" + + @Column(name = "prop_value", columnDefinition = "TEXT") + open var propValue: String? = null + + @Column(name = "sensitive") + open var sensitive: Boolean = false + + @Column(name = "created_at") + open var createdAt: Instant? = null + + @Column(name = "updated_at") + open var updatedAt: Instant? = null +} diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideController.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideController.kt new file mode 100644 index 00000000..a607e2cd --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideController.kt @@ -0,0 +1,96 @@ +package io.skjaere.debridav.config + +import kotlinx.coroutines.runBlocking +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.DeleteMapping +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.PutMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@RestController +@RequestMapping("/api/v1/config") +class ConfigOverrideController( + private val service: ConfigOverrideService, + private val registry: ConfigPropertyRegistry, + private val nntpPoolTester: NntpPoolTester +) { + @GetMapping + fun listAll(): ResponseEntity> = + ResponseEntity.ok(service.listAll()) + + @GetMapping("/{key}") + fun get(@PathVariable key: String): ResponseEntity = + ResponseEntity.ok(service.getEffective(key)) + + @PutMapping("/{key}") + fun upsert( + @PathVariable key: String, + @RequestBody body: UpsertRequest + ): ResponseEntity = + ResponseEntity.ok(service.upsert(key, body.value)) + + @DeleteMapping("/{key}") + fun delete(@PathVariable key: String): ResponseEntity = + ResponseEntity.ok(service.delete(key)) + + @GetMapping("/testable") + fun listTestable(): ResponseEntity> = + ResponseEntity.ok(registry.getTestablePrefixes()) + + @GetMapping("/nntp-pools") + fun getNntpPools(): ResponseEntity> = + ResponseEntity.ok(service.getNntpPools()) + + @PutMapping("/nntp-pools") + fun saveNntpPools(@RequestBody pools: List): ResponseEntity> { + service.saveNntpPools(pools) + return ResponseEntity.ok(service.getNntpPools()) + } + + @PostMapping("/nntp-pools/test") + fun testNntpPool(@RequestBody pool: NntpPoolDto): ResponseEntity { + val start = System.currentTimeMillis() + val result = runBlocking { nntpPoolTester.test(pool) } + val durationMs = System.currentTimeMillis() - start + + return ResponseEntity.ok( + ConfigTestResultDto( + prefix = "nntp", + label = "NNTP Pool", + success = result.success, + message = result.message, + durationMs = durationMs + ) + ) + } + + @PostMapping("/test/{prefix}") + fun test( + @PathVariable prefix: String, + @RequestBody(required = false) body: TestRequest? + ): ResponseEntity { + val tester = registry.getTester(prefix) + ?: return ResponseEntity.notFound().build() + + val start = System.currentTimeMillis() + val result = runBlocking { tester.test(body?.overrides ?: emptyMap()) } + val durationMs = System.currentTimeMillis() - start + + return ResponseEntity.ok( + ConfigTestResultDto( + prefix = prefix, + label = tester.label, + success = result.success, + message = result.message, + durationMs = durationMs + ) + ) + } +} + +data class UpsertRequest(val value: String? = null) +data class TestRequest(val overrides: Map = emptyMap()) diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideDto.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideDto.kt new file mode 100644 index 00000000..d3b55771 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideDto.kt @@ -0,0 +1,14 @@ +package io.skjaere.debridav.config + +data class ConfigOverrideDto( + val key: String, + val name: String?, + val effectiveValue: String?, + val defaultValue: String?, + val hasOverride: Boolean, + val sensitive: Boolean, + val group: String, + val description: String, + val type: String, + val advanced: Boolean +) diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideRepository.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideRepository.kt new file mode 100644 index 00000000..81a14ce1 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideRepository.kt @@ -0,0 +1,13 @@ +package io.skjaere.debridav.config + +import org.springframework.data.jpa.repository.JpaRepository +import org.springframework.transaction.annotation.Transactional + +interface ConfigOverrideRepository : JpaRepository { + fun findByPropKey(key: String): ConfigOverride? + fun findAllByPropKeyIn(keys: Collection): List + fun findAllByPropKeyStartingWith(prefix: String): List + + @Transactional + fun deleteAllByPropKeyStartingWith(prefix: String) +} diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideService.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideService.kt new file mode 100644 index 00000000..20cdad9a --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideService.kt @@ -0,0 +1,256 @@ +package io.skjaere.debridav.config + +import io.skjaere.debridav.usenet.NntpConfigurationProperties +import io.skjaere.debridav.usenet.NntpPoolProperties +import io.skjaere.nzbstreamer.NzbStreamer +import io.skjaere.nzbstreamer.config.NntpConfig +import jakarta.transaction.Transactional +import org.slf4j.LoggerFactory +import org.springframework.cloud.context.refresh.ContextRefresher +import org.springframework.core.env.ConfigurableEnvironment +import org.springframework.core.env.EnumerablePropertySource +import org.springframework.stereotype.Service +import java.time.Instant + +@Service +class ConfigOverrideService( + private val repository: ConfigOverrideRepository, + private val environment: ConfigurableEnvironment, + private val registry: ConfigPropertyRegistry, + private val nntpConfig: NntpConfigurationProperties, + private val contextRefresher: ContextRefresher, + private val dbPropertySourceInitializer: DatabasePropertySourceInitializer, + private val nzbStreamer: NzbStreamer? = null +) { + private val logger = LoggerFactory.getLogger(ConfigOverrideService::class.java) + companion object { + private const val MASKED = "***" + private const val POOL_PREFIX = "nntp.pools[" + private const val POOLS_MANAGED_KEY = "nntp.pools._managed" + } + + fun listAll(): List { + val overrides = repository.findAllByPropKeyIn(registry.properties.keys) + .associateBy { it.propKey } + + return registry.properties.map { (key, meta) -> + val override = overrides[key] + val defaultValue = getDefaultValue(key) + val effectiveValue = override?.propValue ?: defaultValue + + ConfigOverrideDto( + key = key, + name = meta.name, + effectiveValue = if (meta.sensitive) effectiveValue?.let { MASKED } else effectiveValue, + defaultValue = if (meta.sensitive) defaultValue?.let { MASKED } else defaultValue, + hasOverride = override != null, + sensitive = meta.sensitive, + group = meta.group, + description = meta.description, + type = meta.type, + advanced = meta.advanced + ) + } + } + + fun getEffective(key: String): ConfigOverrideDto { + val meta = registry.getMeta(key) + ?: throw KeyNotWhitelistedException(key) + + val override = repository.findByPropKey(key) + val defaultValue = getDefaultValue(key) + val effectiveValue = override?.propValue ?: defaultValue + + return ConfigOverrideDto( + key = key, + name = meta.name, + effectiveValue = if (meta.sensitive) effectiveValue?.let { MASKED } else effectiveValue, + defaultValue = if (meta.sensitive) defaultValue?.let { MASKED } else defaultValue, + hasOverride = override != null, + sensitive = meta.sensitive, + group = meta.group, + description = meta.description, + type = meta.type, + advanced = meta.advanced + ) + } + + fun upsert(key: String, value: String?): ConfigOverrideDto { + val meta = registry.getMeta(key) + ?: throw KeyNotWhitelistedException(key) + + val now = Instant.now() + val entity = repository.findByPropKey(key) ?: ConfigOverride().apply { + propKey = key + createdAt = now + } + entity.propValue = value + entity.sensitive = meta.sensitive + entity.updatedAt = now + repository.save(entity) + + refreshEnvironment() + + return getEffective(key) + } + + fun delete(key: String): ConfigOverrideDto { + if (!registry.isWhitelisted(key)) { + throw KeyNotWhitelistedException(key) + } + + val override = repository.findByPropKey(key) + ?: throw OverrideNotFoundException(key) + repository.delete(override) + + refreshEnvironment() + + return getEffective(key) + } + + private fun refreshEnvironment() { + val propertySource = dbPropertySourceInitializer.getOrCreatePropertySource() + val overrides = repository.findAll().associate { it.propKey to (it.propValue ?: "") } + propertySource.replaceAll(overrides) + contextRefresher.refreshEnvironment() + logger.info("Refreshed environment with {} database override(s)", overrides.size) + } + + @Suppress("ReturnCount") + private fun getDefaultValue(key: String): String? { + for (source in environment.propertySources) { + if (source.name == DatabasePropertySource.NAME) continue + if (source is EnumerablePropertySource<*>) { + val value = source.getProperty(key) + if (value != null) return value.toString() + } else { + val value = source.getProperty(key) + if (value != null) return value.toString() + } + } + return null + } + + fun getNntpPools(): List { + val overrides = repository.findAllByPropKeyStartingWith(POOL_PREFIX) + if (overrides.isNotEmpty()) { + return parsePoolOverrides(overrides).sortedBy { it.priority } + } + // No pool rows in the DB. Distinguish "user has explicitly emptied the + // list" from "first boot, never configured via UI": a managed-marker + // row is written by saveNntpPools on every save, so its presence means + // we should honor the empty state and not fall back to env defaults. + val managed = repository.findByPropKey(POOLS_MANAGED_KEY) != null + return if (managed) { + emptyList() + } else { + nntpConfig.pools.map { it.toDto() }.sortedBy { it.priority } + } + } + + @Transactional + fun saveNntpPools(pools: List) { + repository.deleteAllByPropKeyStartingWith(POOL_PREFIX) + val now = Instant.now() + if (repository.findByPropKey(POOLS_MANAGED_KEY) == null) { + repository.save(ConfigOverride().apply { + propKey = POOLS_MANAGED_KEY + propValue = "true" + sensitive = false + createdAt = now + updatedAt = now + }) + } + repository.flush() + pools.forEachIndexed { i, pool -> + val entries = mapOf( + "nntp.pools[$i].host" to pool.host, + "nntp.pools[$i].port" to pool.port.toString(), + "nntp.pools[$i].username" to pool.username, + "nntp.pools[$i].password" to pool.password, + "nntp.pools[$i].use-tls" to pool.useTls.toString(), + "nntp.pools[$i].max-connections" to pool.maxConnections.toString(), + "nntp.pools[$i].priority" to pool.priority.toString() + ) + for ((key, value) in entries) { + val entity = ConfigOverride().apply { + propKey = key + propValue = value + sensitive = key.endsWith(".password") + createdAt = now + updatedAt = now + } + repository.save(entity) + } + } + syncRunningPools(pools) + } + + fun syncRunningNntpPools() { + syncRunningPools(getNntpPools()) + } + + private fun syncRunningPools(saved: List) { + if (nzbStreamer == null) return + val savedConfigs = saved.map { it.toNntpConfig() }.toSet() + val runningConfigs = nzbStreamer.getPoolConfigs().toSet() + + val toRemove = runningConfigs - savedConfigs + val toAdd = savedConfigs - runningConfigs + + toRemove.forEach { nzbStreamer.removePool(it) } + toAdd.forEach { nzbStreamer.addPool(it) } + + if (toRemove.isNotEmpty() || toAdd.isNotEmpty()) { + logger.info("Synced NNTP pools: removed={}, added={}", toRemove.size, toAdd.size) + } + } + + private fun NntpPoolDto.toNntpConfig() = NntpConfig( + host = host, + port = port, + username = username, + password = password, + useTls = useTls, + maxConnections = maxConnections, + priority = priority + ) + + private fun parsePoolOverrides(overrides: List): List { + val poolMap = mutableMapOf>() + val regex = Regex("""nntp\.pools\[(\d+)]\.(.+)""") + for (ov in overrides) { + val match = regex.matchEntire(ov.propKey) ?: continue + val index = match.groupValues[1].toInt() + val field = match.groupValues[2] + poolMap.getOrPut(index) { mutableMapOf() }[field] = ov.propValue ?: "" + } + return poolMap.toSortedMap().map { (_, fields) -> + NntpPoolDto( + host = fields["host"] ?: "", + port = fields["port"]?.toIntOrNull() ?: 563, + username = fields["username"] ?: "", + password = fields["password"] ?: "", + useTls = fields["use-tls"]?.toBooleanStrictOrNull() ?: true, + maxConnections = fields["max-connections"]?.toIntOrNull() ?: 8, + priority = fields["priority"]?.toIntOrNull() ?: 0 + ) + } + } + + private fun NntpPoolProperties.toDto() = NntpPoolDto( + host = host, + port = port, + username = username, + password = password, + useTls = useTls, + maxConnections = maxConnections, + priority = priority + ) +} + +class KeyNotWhitelistedException(val key: String) : + RuntimeException("Property key '$key' is not whitelisted for override") + +class OverrideNotFoundException(val key: String) : + RuntimeException("No override found for key '$key'") diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigProperty.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigProperty.kt new file mode 100644 index 00000000..7bb38ccf --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigProperty.kt @@ -0,0 +1,11 @@ +package io.skjaere.debridav.config + +@Target(AnnotationTarget.VALUE_PARAMETER, AnnotationTarget.PROPERTY, AnnotationTarget.FIELD) +@Retention(AnnotationRetention.RUNTIME) +annotation class ConfigProperty( + val name: String, + val description: String = "", + val sensitive: Boolean = false, + val group: String = "", + val advanced: Boolean = false +) diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigPropertyRegistry.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigPropertyRegistry.kt new file mode 100644 index 00000000..5d0f5354 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigPropertyRegistry.kt @@ -0,0 +1,117 @@ +package io.skjaere.debridav.config + +import jakarta.annotation.PostConstruct +import org.slf4j.LoggerFactory +import org.springframework.boot.context.properties.ConfigurationProperties +import org.springframework.context.ApplicationContext +import org.springframework.stereotype.Component +import java.time.Duration +import kotlin.reflect.full.findAnnotation +import kotlin.reflect.full.memberProperties + +data class ConfigPropertyMeta( + val name: String, + val description: String, + val sensitive: Boolean = false, + val group: String, + val type: String = "STRING", + val advanced: Boolean = false +) + +@Component +class ConfigPropertyRegistry( + private val applicationContext: ApplicationContext +) { + private val logger = LoggerFactory.getLogger(ConfigPropertyRegistry::class.java) + private val _properties = mutableMapOf() + private val _testers = mutableMapOf() + + val properties: Map get() = _properties + + @Suppress("LoopWithTooManyJumpStatements") + @PostConstruct + fun init() { + val beanNames = applicationContext.getBeanNamesForAnnotation(ConfigurationProperties::class.java) + for (beanName in beanNames) { + val beanType = applicationContext.getType(beanName) ?: continue + val prefix = beanType.getAnnotation(ConfigurationProperties::class.java)?.prefix + ?: continue + + for (prop in beanType.kotlin.memberProperties) { + val annotation = prop.findAnnotation() ?: continue + + val kebabName = camelToKebab(prop.name) + val key = "$prefix.$kebabName" + val group = annotation.group.ifEmpty { deriveGroup(prefix) } + val type = when (prop.returnType.classifier) { + Boolean::class -> "BOOLEAN" + Int::class -> "INT" + Long::class -> "LONG" + Duration::class -> "DURATION" + List::class -> "STRING_LIST" + else -> "STRING" + } + + _properties[key] = ConfigPropertyMeta( + name = annotation.name, + description = annotation.description, + sensitive = annotation.sensitive, + group = group, + type = type, + advanced = annotation.advanced + ) + } + } + discoverTesters() + } + + private fun discoverTesters() { + val testers = applicationContext.getBeansOfType(ConfigurationTester::class.java).values + for (tester in testers) { + val prefix = tester.configurationClass.java + .getAnnotation(ConfigurationProperties::class.java) + ?.prefix + if (prefix != null) { + _testers[prefix] = tester + logger.info("Registered configuration tester '{}' for prefix '{}'", tester.label, prefix) + } else { + logger.warn( + "ConfigurationTester '{}' targets {} which has no @ConfigurationProperties annotation", + tester.label, tester.configurationClass + ) + } + } + } + + fun getTester(prefix: String): ConfigurationTester? = _testers[prefix] + + fun getTestablePrefixes(): List = _testers.map { (prefix, tester) -> + TestablePrefixDto(prefix = prefix, label = tester.label) + } + + fun isWhitelisted(key: String): Boolean = _properties.containsKey(key) + + fun getMeta(key: String): ConfigPropertyMeta? = _properties[key] + + companion object { + private val PROVIDER_PREFIXES = setOf("premiumize", "real-debrid", "torbox", "easynews") + private val ARR_PREFIXES = setOf("sonarr", "radarr") + + fun camelToKebab(name: String): String = buildString { + for ((i, ch) in name.withIndex()) { + if (ch.isUpperCase()) { + if (i > 0) append('-') + append(ch.lowercaseChar()) + } else { + append(ch) + } + } + } + + fun deriveGroup(prefix: String): String = when (prefix) { + in PROVIDER_PREFIXES -> "providers" + in ARR_PREFIXES -> "arrs" + else -> prefix + } + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigTestResultDto.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigTestResultDto.kt new file mode 100644 index 00000000..a4525690 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigTestResultDto.kt @@ -0,0 +1,14 @@ +package io.skjaere.debridav.config + +data class ConfigTestResultDto( + val prefix: String, + val label: String, + val success: Boolean, + val message: String, + val durationMs: Long +) + +data class TestablePrefixDto( + val prefix: String, + val label: String +) diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigurationTester.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigurationTester.kt new file mode 100644 index 00000000..cbae29a7 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigurationTester.kt @@ -0,0 +1,14 @@ +package io.skjaere.debridav.config + +import kotlin.reflect.KClass + +interface ConfigurationTester { + val configurationClass: KClass<*> + val label: String + suspend fun test(overrides: Map = emptyMap()): TestResult +} + +data class TestResult( + val success: Boolean, + val message: String +) diff --git a/src/main/kotlin/io/skjaere/debridav/config/DatabasePropertySource.kt b/src/main/kotlin/io/skjaere/debridav/config/DatabasePropertySource.kt new file mode 100644 index 00000000..15648f06 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/DatabasePropertySource.kt @@ -0,0 +1,18 @@ +package io.skjaere.debridav.config + +import org.springframework.core.env.MapPropertySource +import java.util.concurrent.ConcurrentHashMap + +class DatabasePropertySource( + private val map: ConcurrentHashMap = ConcurrentHashMap() +) : MapPropertySource(NAME, map) { + + fun replaceAll(overrides: Map) { + map.clear() + map.putAll(overrides) + } + + companion object { + const val NAME = "databaseOverrides" + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/config/DatabasePropertySourceInitializer.kt b/src/main/kotlin/io/skjaere/debridav/config/DatabasePropertySourceInitializer.kt new file mode 100644 index 00000000..58fcfc6a --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/DatabasePropertySourceInitializer.kt @@ -0,0 +1,44 @@ +package io.skjaere.debridav.config + +import org.slf4j.LoggerFactory +import org.springframework.boot.context.event.ApplicationReadyEvent +import org.springframework.cloud.context.refresh.ContextRefresher +import org.springframework.context.ApplicationListener +import org.springframework.context.annotation.Lazy +import org.springframework.core.env.ConfigurableEnvironment +import org.springframework.stereotype.Component + +@Component +class DatabasePropertySourceInitializer( + private val environment: ConfigurableEnvironment, + private val repository: ConfigOverrideRepository, + private val contextRefresher: ContextRefresher, + @Lazy private val configOverrideService: ConfigOverrideService +) : ApplicationListener { + + private val logger = LoggerFactory.getLogger(DatabasePropertySourceInitializer::class.java) + + override fun onApplicationEvent(event: ApplicationReadyEvent) { + val propertySource = getOrCreatePropertySource() + val overrides = repository.findAll().associate { it.propKey to (it.propValue ?: "") } + if (overrides.isNotEmpty()) { + propertySource.replaceAll(overrides) + contextRefresher.refreshEnvironment() + logger.info("Loaded {} database config override(s) and refreshed environment", overrides.size) + configOverrideService.syncRunningNntpPools() + } else { + logger.info("No database config overrides found") + } + } + + fun getOrCreatePropertySource(): DatabasePropertySource { + val sources = environment.propertySources + val existing = sources.get(DatabasePropertySource.NAME) + if (existing != null) { + return existing as DatabasePropertySource + } + val propertySource = DatabasePropertySource() + sources.addFirst(propertySource) + return propertySource + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/config/NntpPoolDto.kt b/src/main/kotlin/io/skjaere/debridav/config/NntpPoolDto.kt new file mode 100644 index 00000000..05383fb6 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/NntpPoolDto.kt @@ -0,0 +1,11 @@ +package io.skjaere.debridav.config + +data class NntpPoolDto( + val host: String = "", + val port: Int = 563, + val username: String = "", + val password: String = "", + val useTls: Boolean = true, + val maxConnections: Int = 8, + val priority: Int = 0 +) diff --git a/src/main/kotlin/io/skjaere/debridav/config/NntpPoolTester.kt b/src/main/kotlin/io/skjaere/debridav/config/NntpPoolTester.kt new file mode 100644 index 00000000..0eb53987 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/NntpPoolTester.kt @@ -0,0 +1,36 @@ +package io.skjaere.debridav.config + +import io.ktor.network.selector.SelectorManager +import io.skjaere.nntp.NntpAuthenticationException +import io.skjaere.nntp.NntpClient +import io.skjaere.nntp.NntpException +import kotlinx.coroutines.Dispatchers +import org.springframework.stereotype.Service + +@Service +class NntpPoolTester { + @Suppress("TooGenericExceptionCaught", "ReturnCount") + suspend fun test(pool: NntpPoolDto): TestResult { + val selectorManager = SelectorManager(Dispatchers.IO) + try { + val client = NntpClient.connect( + host = pool.host, + port = pool.port, + selectorManager = selectorManager, + useTls = pool.useTls, + username = pool.username, + password = pool.password + ) + client.use { it.quit() } + return TestResult(success = true, message = "Connected successfully") + } catch (e: NntpAuthenticationException) { + return TestResult(success = false, message = "Authentication failed: ${e.message}") + } catch (e: NntpException) { + return TestResult(success = false, message = e.message ?: "NNTP error") + } catch (e: Exception) { + return TestResult(success = false, message = e.message ?: "Connection failed") + } finally { + selectorManager.close() + } + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/config/auth/AuthConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/config/auth/AuthConfigurationProperties.kt new file mode 100644 index 00000000..98a0a358 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/auth/AuthConfigurationProperties.kt @@ -0,0 +1,14 @@ +package io.skjaere.debridav.config.auth + +import org.springframework.boot.context.properties.ConfigurationProperties + +@ConfigurationProperties(prefix = "debridav.auth") +class AuthConfigurationProperties { + var enabled: Boolean = false + var jwtSecret: String = "" + @Suppress("MagicNumber") + var tokenExpirationHours: Long = 24 + var protectQbittorrentApi: Boolean = false + var protectSabnzbdApi: Boolean = false + var protectActuator: Boolean = false +} diff --git a/src/main/kotlin/io/skjaere/debridav/config/auth/AuthController.kt b/src/main/kotlin/io/skjaere/debridav/config/auth/AuthController.kt new file mode 100644 index 00000000..d1bc4fd7 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/auth/AuthController.kt @@ -0,0 +1,40 @@ +package io.skjaere.debridav.config.auth + +import io.skjaere.debridav.configuration.DebridavConfigurationProperties +import org.springframework.http.HttpStatus +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@RestController +@RequestMapping("/api/v1/auth") +class AuthController( + private val jwtService: JwtService, + private val debridavConfig: DebridavConfigurationProperties +) { + @Suppress("ReturnCount") + @PostMapping("/login") + fun login(@RequestBody request: LoginRequest): ResponseEntity { + val expectedUsername = debridavConfig.webdavUsername + val expectedPassword = debridavConfig.webdavPassword + + if (expectedUsername.isNullOrBlank() || expectedPassword.isNullOrBlank()) { + return ResponseEntity.status(HttpStatus.UNAUTHORIZED) + .body(ErrorBody("No credentials configured")) + } + + if (request.username != expectedUsername || request.password != expectedPassword) { + return ResponseEntity.status(HttpStatus.UNAUTHORIZED) + .body(ErrorBody("Invalid credentials")) + } + + val token = jwtService.generateToken(request.username) + return ResponseEntity.ok(LoginResponse(token)) + } +} + +data class LoginRequest(val username: String, val password: String) +data class LoginResponse(val token: String) +data class ErrorBody(val error: String) diff --git a/src/main/kotlin/io/skjaere/debridav/config/auth/JwtAuthenticationFilter.kt b/src/main/kotlin/io/skjaere/debridav/config/auth/JwtAuthenticationFilter.kt new file mode 100644 index 00000000..1e6bc6e6 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/auth/JwtAuthenticationFilter.kt @@ -0,0 +1,33 @@ +package io.skjaere.debridav.config.auth + +import jakarta.servlet.FilterChain +import jakarta.servlet.http.HttpServletRequest +import jakarta.servlet.http.HttpServletResponse +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken +import org.springframework.security.core.context.SecurityContextHolder +import org.springframework.stereotype.Component +import org.springframework.web.filter.OncePerRequestFilter + +@Component +class JwtAuthenticationFilter( + private val jwtService: JwtService +) : OncePerRequestFilter() { + + override fun doFilterInternal( + request: HttpServletRequest, + response: HttpServletResponse, + filterChain: FilterChain + ) { + val authHeader = request.getHeader("Authorization") + if (authHeader != null && authHeader.startsWith("Bearer ")) { + @Suppress("MagicNumber") + val token = authHeader.substring(7) + val username = jwtService.validateTokenAndGetUsername(token) + if (username != null && SecurityContextHolder.getContext().authentication == null) { + val auth = UsernamePasswordAuthenticationToken(username, null, emptyList()) + SecurityContextHolder.getContext().authentication = auth + } + } + filterChain.doFilter(request, response) + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/config/auth/JwtService.kt b/src/main/kotlin/io/skjaere/debridav/config/auth/JwtService.kt new file mode 100644 index 00000000..04282838 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/auth/JwtService.kt @@ -0,0 +1,99 @@ +package io.skjaere.debridav.config.auth + +import io.jsonwebtoken.JwtException +import io.jsonwebtoken.Jwts +import io.jsonwebtoken.security.Keys +import jakarta.annotation.PostConstruct +import org.slf4j.LoggerFactory +import org.springframework.stereotype.Service +import java.util.Date +import javax.crypto.SecretKey + +private const val MIN_HS256_KEY_BYTES = 32 + +@Service +class JwtService( + private val authConfig: AuthConfigurationProperties +) { + private val logger = LoggerFactory.getLogger(JwtService::class.java) + + @PostConstruct + fun validateSecret() { + val secret = authConfig.jwtSecret + if (secret.isNotBlank() && secret.toByteArray().size < MIN_HS256_KEY_BYTES) { + error( + "DEBRIDAV_AUTH_JWT_SECRET must be at least $MIN_HS256_KEY_BYTES bytes for HS256 " + + "(got ${secret.toByteArray().size}). Generate one with: openssl rand -base64 48" + ) + } + } + + private val key: SecretKey by lazy { + if (authConfig.jwtSecret.isBlank()) { + logger.warn( + "DEBRIDAV_AUTH_JWT-SECRET is not set; generating a random key for this session. " + + "Tokens will be invalidated on restart — set a stable secret if that matters." + ) + Jwts.SIG.HS256.key().build() + } else { + Keys.hmacShaKeyFor(authConfig.jwtSecret.toByteArray()) + } + } + + @Suppress("MagicNumber") + fun generateToken(username: String): String { + val now = Date() + val expiration = Date(now.time + authConfig.tokenExpirationHours * 3600 * 1000) + + return Jwts.builder() + .subject(username) + .issuedAt(now) + .expiration(expiration) + .signWith(key) + .compact() + } + + fun validateTokenAndGetUsername(token: String): String? = try { + Jwts.parser() + .verifyWith(key) + .build() + .parseSignedClaims(token) + .payload + .subject + } catch (_: JwtException) { + null + } catch (_: IllegalArgumentException) { + null + } + + @Suppress("MagicNumber") + fun generateStreamToken(path: String): String { + val now = Date() + val expiration = Date(now.time + STREAM_TOKEN_EXPIRY_SECONDS * 1000) + + return Jwts.builder() + .subject(path) + .claim("type", "stream") + .issuedAt(now) + .expiration(expiration) + .signWith(key) + .compact() + } + + fun validateStreamToken(token: String): String? = try { + val claims = Jwts.parser() + .verifyWith(key) + .build() + .parseSignedClaims(token) + .payload + if (claims["type"] == "stream") claims.subject else null + } catch (_: JwtException) { + null + } catch (_: IllegalArgumentException) { + null + } + + companion object { + const val STREAM_TOKEN_EXPIRY_SECONDS = 86400L // 24 hours + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/config/auth/SecurityConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/config/auth/SecurityConfiguration.kt new file mode 100644 index 00000000..86d99d78 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/auth/SecurityConfiguration.kt @@ -0,0 +1,94 @@ +package io.skjaere.debridav.config.auth + +import jakarta.servlet.http.HttpServletRequest +import jakarta.servlet.http.HttpServletResponse +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.core.annotation.Order +import org.springframework.http.HttpStatus +import org.springframework.security.config.annotation.web.builders.HttpSecurity +import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity +import org.springframework.security.config.http.SessionCreationPolicy +import org.springframework.security.core.AuthenticationException +import org.springframework.security.web.AuthenticationEntryPoint +import org.springframework.security.web.SecurityFilterChain +import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter +import org.springframework.security.web.firewall.HttpFirewall +import org.springframework.security.web.firewall.StrictHttpFirewall + +@Configuration +@EnableWebSecurity +class SecurityConfiguration( + private val jwtAuthenticationFilter: JwtAuthenticationFilter, + private val authConfig: AuthConfigurationProperties +) { + @Bean + @Order(1) + fun apiSecurityFilterChain(http: HttpSecurity): SecurityFilterChain { + http + .securityMatcher("/api/**", "/actuator/**") + .csrf { it.disable() } + .sessionManagement { it.sessionCreationPolicy(SessionCreationPolicy.STATELESS) } + .exceptionHandling { it.authenticationEntryPoint(unauthorizedEntryPoint()) } + .addFilterBefore(jwtAuthenticationFilter, UsernamePasswordAuthenticationFilter::class.java) + .authorizeHttpRequests { auth -> + // Auth and stream endpoints are always public + auth.requestMatchers("/api/v1/auth/**").permitAll() + auth.requestMatchers("/api/v1/stream/**").permitAll() + + // Config API is protected when auth is enabled + if (authConfig.enabled) { + auth.requestMatchers("/api/v1/config/**").authenticated() + auth.requestMatchers("/api/v1/queue/**").authenticated() + } else { + auth.requestMatchers("/api/v1/config/**").permitAll() + auth.requestMatchers("/api/v1/queue/**").permitAll() + } + + // Conditionally protect qBittorrent API + if (authConfig.protectQbittorrentApi) { + auth.requestMatchers("/api/v2/**").authenticated() + } + + // Conditionally protect SABnzbd API + if (authConfig.protectSabnzbdApi) { + auth.requestMatchers("/api").authenticated() + } + + // Conditionally protect actuator + if (authConfig.protectActuator) { + auth.requestMatchers("/actuator/**").authenticated() + } + + // All other API/actuator paths are public + auth.anyRequest().permitAll() + } + + return http.build() + } + + @Bean + @Order(2) + fun webDavSecurityFilterChain(http: HttpSecurity): SecurityFilterChain { + http + .csrf { it.disable() } + .authorizeHttpRequests { it.anyRequest().permitAll() } + + return http.build() + } + + @Bean + fun httpFirewall(): HttpFirewall { + val firewall = StrictHttpFirewall() + // Allow WebDAV methods (PROPFIND, MKCOL, COPY, MOVE, LOCK, UNLOCK, PROPPATCH) + firewall.setUnsafeAllowAnyHttpMethod(true) + return firewall + } + + private fun unauthorizedEntryPoint() = AuthenticationEntryPoint { + _: HttpServletRequest, response: HttpServletResponse, _: AuthenticationException -> + response.status = HttpStatus.UNAUTHORIZED.value() + response.contentType = "application/json" + response.writer.write("""{"error":"Unauthorized"}""") + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/configuration/DbConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/configuration/DbConfigurationProperties.kt new file mode 100644 index 00000000..d0f66fd0 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/configuration/DbConfigurationProperties.kt @@ -0,0 +1,11 @@ +package io.skjaere.debridav.configuration + +import org.springframework.boot.context.properties.ConfigurationProperties + +@ConfigurationProperties(prefix = "debridav.db") +class DbConfigurationProperties { + var host: String = "localhost" + @Suppress("MagicNumber") + var port: Int = 5432 + var databaseName: String = "debridav" +} diff --git a/src/main/kotlin/io/skjaere/debridav/configuration/DebridavConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/configuration/DebridavConfigurationProperties.kt index e8993330..2ce1fa9d 100644 --- a/src/main/kotlin/io/skjaere/debridav/configuration/DebridavConfigurationProperties.kt +++ b/src/main/kotlin/io/skjaere/debridav/configuration/DebridavConfigurationProperties.kt @@ -1,36 +1,80 @@ package io.skjaere.debridav.configuration +import io.skjaere.debridav.config.ConfigProperty import io.skjaere.debridav.debrid.DebridProvider import org.springframework.boot.context.properties.ConfigurationProperties import java.time.Duration @ConfigurationProperties(prefix = "debridav") -data class DebridavConfigurationProperties( - val rootPath: String, - val downloadPath: String, - val mountPath: String, - var debridClients: List, - val waitAfterMissing: Duration, - val waitAfterProviderError: Duration, - val waitAfterNetworkError: Duration, - val waitAfterClientError: Duration, - val retriesOnProviderError: Long, - val delayBetweenRetries: Duration, - val connectTimeoutMilliseconds: Long, - val readTimeoutMilliseconds: Long, - val shouldDeleteNonWorkingFiles: Boolean, - val torrentLifetime: Duration, - val enableFileImportOnStartup: Boolean, - val defaultCategories: List, - val localEntityMaxSizeMb: Int, - val webdavUsername: String? = null, - val webdavPassword: String? = null, -) { - fun isWebdavAuthEnabled(): Boolean = !webdavUsername.isNullOrBlank() && !webdavPassword.isNullOrBlank() +class DebridavConfigurationProperties { + @ConfigProperty(name = "Download Path", description = "Download path") + lateinit var downloadPath: String + + @ConfigProperty(name = "Mount Path", description = "Mount path") + lateinit var mountPath: String + + @ConfigProperty(name = "Debrid Clients", description = "Enabled debrid providers (comma-separated)") + var debridClients: List = emptyList() + + @ConfigProperty(name = "Wait After Missing", description = "Wait duration after missing file", advanced = true) + var waitAfterMissing: Duration = Duration.ZERO + + @ConfigProperty( + name = "Wait After Provider Error", + description = "Wait duration after provider error", + advanced = true + ) + var waitAfterProviderError: Duration = Duration.ZERO + + @ConfigProperty( + name = "Wait After Network Error", + description = "Wait duration after network error", + advanced = true + ) + var waitAfterNetworkError: Duration = Duration.ZERO + + @ConfigProperty(name = "Wait After Client Error", description = "Wait duration after client error", advanced = true) + var waitAfterClientError: Duration = Duration.ZERO + + @ConfigProperty( + name = "Retries on Provider Error", + description = "Number of retries on provider error", + advanced = true + ) + var retriesOnProviderError: Long = 0 + + @ConfigProperty(name = "Delay Between Retries", description = "Delay between retries", advanced = true) + var delayBetweenRetries: Duration = Duration.ZERO + + @ConfigProperty(name = "Connect Timeout", description = "HTTP connect timeout in ms", advanced = true) + var connectTimeoutMilliseconds: Long = 0 - init { - require(debridClients.isNotEmpty()) { - "No debrid providers defined" - } - } + @ConfigProperty(name = "Read Timeout", description = "HTTP read timeout in ms", advanced = true) + var readTimeoutMilliseconds: Long = 0 + + @ConfigProperty(name = "Delete Non-Working Files", description = "Delete non-working files", advanced = true) + var shouldDeleteNonWorkingFiles: Boolean = false + + @ConfigProperty(name = "Torrent Lifetime", description = "Torrent lifetime duration", advanced = true) + var torrentLifetime: Duration = Duration.ZERO + + @ConfigProperty(name = "Default Categories", description = "Default categories (comma-separated)", advanced = true) + var defaultCategories: List = emptyList() + + @ConfigProperty(name = "Max Local Entity Size (MB)", description = "Max local entity size in MB", advanced = true) + var localEntityMaxSizeMb: Int = 0 + + @ConfigProperty(name = "WebDAV Username", description = "WebDAV username", group = "webdav") + var webdavUsername: String? = null + + @ConfigProperty(name = "WebDAV Password", description = "WebDAV password", sensitive = true, group = "webdav") + var webdavPassword: String? = null + + @ConfigProperty( + name = "Rclone Cache Invalidation", + description = "Notify rclone to refresh its directory cache on file create / move / delete" + ) + var rcloneCacheInvalidationEnabled: Boolean = false + + fun isWebdavAuthEnabled(): Boolean = !webdavUsername.isNullOrBlank() && !webdavPassword.isNullOrBlank() } diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/DebridCachedContentService.kt b/src/main/kotlin/io/skjaere/debridav/debrid/DebridCachedContentService.kt index 4a740900..9def7bcf 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/DebridCachedContentService.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/DebridCachedContentService.kt @@ -259,7 +259,11 @@ class DebridCachedContentService( .retry(debridavConfigurationProperties.retriesOnProviderError) { e -> (e.isRetryable()).also { if (it) delay(debridavConfigurationProperties.delayBetweenRetries.toMillis()) } }.catch { e -> - logger.error("error getting cached files from ${debridClient.getProvider()} : ${e.cause} : ${e.message}") + logger.error( + "error getting cached files from {} ({}: {})", + debridClient.getProvider(), e.javaClass.simpleName, e.message, + e, + ) when (e) { is DebridProviderError -> emit(ProviderErrorGetCachedFilesResponse(debridClient.getProvider())) is DebridClientError -> emit(ClientErrorGetCachedFilesResponse(debridClient.getProvider())) diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/DebridLinkService.kt b/src/main/kotlin/io/skjaere/debridav/debrid/DebridLinkService.kt index f3f41a7a..f18e4c29 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/DebridLinkService.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/DebridLinkService.kt @@ -151,7 +151,7 @@ class DebridLinkService( } } - private suspend fun getFlowOfDebridLinks(debridFileContents: DebridFileContents): Flow = flow { + suspend fun getFlowOfDebridLinks(debridFileContents: DebridFileContents): Flow = flow { debridavConfigurationProperties.debridClients .map { debridClients.getClient(it) } .map { debridClient -> diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/LibraryMetricsService.kt b/src/main/kotlin/io/skjaere/debridav/debrid/LibraryMetricsService.kt index 67f1c8e0..057fe3ed 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/LibraryMetricsService.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/LibraryMetricsService.kt @@ -1,8 +1,9 @@ package io.skjaere.debridav.debrid -import io.prometheus.metrics.core.metrics.Gauge -import io.prometheus.metrics.model.registry.PrometheusRegistry +import io.micrometer.core.instrument.MeterRegistry +import io.micrometer.core.instrument.MultiGauge +import io.micrometer.core.instrument.Tags import io.skjaere.debridav.repository.DebridFileContentsRepository import io.skjaere.debridav.repository.LibraryStats import org.springframework.scheduling.annotation.Scheduled @@ -11,43 +12,51 @@ import org.springframework.stereotype.Component @Component class LibraryMetricsService( private val debridFileContentsRepository: DebridFileContentsRepository, - prometheusRegistry: PrometheusRegistry + meterRegistry: MeterRegistry, ) { - private val cachedStatusGauge = Gauge.builder() - .name("debridav.library.metrics") - .help("Metrics for library files") - .labelNames("provider", "type") - .register(prometheusRegistry) + private val cachedStatusGauge = MultiGauge + .builder("debridav.library.metrics") + .description("Metrics for library files") + .register(meterRegistry) - private val librarySizeGauge = Gauge.builder() - .name("debridav.library.size") - .labelNames("source") - .help("Metrics for library files") - .register(prometheusRegistry) + private val librarySizeGauge = MultiGauge + .builder("debridav.library.size") + .description("Metrics for library files") + .register(meterRegistry) @Scheduled(fixedRate = 60000) fun recordLibraryMetrics() { val numberOfTorrentEntities = debridFileContentsRepository.numberOfRemotelyCachedTorrentEntities() - librarySizeGauge - .labelValues("torrent") - .set(numberOfTorrentEntities.toDouble()) - val numberOfUsenetEntities = debridFileContentsRepository.numberOfRemotelyCachedUsenetEntities() - librarySizeGauge - .labelValues("usenet") - .set(numberOfUsenetEntities.toDouble()) - debridFileContentsRepository.getLibraryMetricsTorrents() - .toLibraryTorrentStats(numberOfTorrentEntities) - .forEach { - cachedStatusGauge - .labelValues(it.provider, it.type) - .set(it.count.toDouble()) - } + librarySizeGauge.register( + listOf( + MultiGauge.Row.of(Tags.of("source", "torrent"), numberOfTorrentEntities.toDouble()), + MultiGauge.Row.of(Tags.of("source", "usenet"), numberOfUsenetEntities.toDouble()), + ), + true, + ) + + val torrentStats = debridFileContentsRepository.getLibraryMetricsTorrents() + .toLibraryStats(numberOfTorrentEntities) + .map { "torrent" to it } + val usenetStats = debridFileContentsRepository.getLibraryMetricsUsenet() + .toLibraryStats(numberOfUsenetEntities) + .map { "usenet" to it } + + cachedStatusGauge.register( + (torrentStats + usenetStats).map { (source, stat) -> + MultiGauge.Row.of( + Tags.of("source", source, "provider", stat.provider, "type", stat.type), + stat.count.toDouble(), + ) + }, + true, + ) } - fun List>.toLibraryTorrentStats(numberOfTotalEntities: Long): List { + fun List>.toLibraryStats(numberOfTotalEntities: Long): List { return this.map { LibraryStats( (it["provider"] as String).replace("\"", ""), diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/easynews/EasynewsClient.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/easynews/EasynewsClient.kt index d5b60539..16b0dbc3 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/easynews/EasynewsClient.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/easynews/EasynewsClient.kt @@ -22,6 +22,8 @@ import io.ktor.http.ContentType import io.ktor.http.HttpHeaders import io.ktor.http.HttpHeaders.Authorization import io.ktor.http.isSuccess +import io.skjaere.debridav.config.ConfigurationTester +import io.skjaere.debridav.config.TestResult import io.milton.http.Range import io.skjaere.debridav.debrid.CachedContentKey import io.skjaere.debridav.debrid.DebridProvider @@ -36,13 +38,13 @@ import kotlinx.coroutines.flow.flowOf import kotlinx.serialization.json.Json import org.slf4j.Logger import org.slf4j.LoggerFactory -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.stereotype.Component import java.net.URLEncoder import java.nio.charset.StandardCharsets import java.time.Duration import java.time.Instant import java.util.* +import kotlin.reflect.KClass const val TIMEOUT_MS = 5_000L const val RETRIES = 3 @@ -55,14 +57,13 @@ private const val RATE_LIMITER_TIMEOUT = 5L @Component @Suppress("UnusedPrivateProperty", "TooManyFunctions") -@ConditionalOnExpression("#{'\${debridav.debrid-clients}'.contains('easynews')}") class EasynewsClient( override val httpClient: HttpClient, private val easynewsConfiguration: EasynewsConfigurationProperties, private val easynewsReleaseNameMatchingService: EasynewsReleaseNameMatchingService, rateLimiterRegistry: RateLimiterRegistry, retryRegistry: RetryRegistry -) : DebridCachedContentClient { +) : DebridCachedContentClient, ConfigurationTester { private val jsonParser = Json { ignoreUnknownKeys = true } private val logger = LoggerFactory.getLogger(EasynewsClient::class.java) private val auth = getBasicAuth() @@ -387,4 +388,35 @@ class EasynewsClient( override fun logger(): Logger { return logger } + + override val configurationClass: KClass<*> = EasynewsConfigurationProperties::class + override val label: String = "Easynews" + + @Suppress("TooGenericExceptionCaught") + override suspend fun test(overrides: Map): TestResult = try { + val apiBaseUrl = overrides["easynews.api-base-url"] ?: easynewsConfiguration.apiBaseUrl + val username = overrides["easynews.username"] ?: easynewsConfiguration.username + val password = overrides["easynews.password"] ?: easynewsConfiguration.password + + val credentials = "$username:$password" + val testAuth = "Basic ${Base64.getEncoder().encodeToString(credentials.toByteArray())}" + + val response = httpClient.get("$apiBaseUrl/2.0/search/solr-search/") { + url { + parameters.append("gps", "test") + parameters.append("pby", "1") + } + headers { + append(Authorization, testAuth) + accept(ContentType.Application.Json) + } + } + if (response.status.isSuccess()) { + TestResult(success = true, message = "Connected successfully") + } else { + TestResult(success = false, message = "HTTP ${response.status.value}") + } + } catch (e: Exception) { + TestResult(success = false, message = e.message ?: "Unknown error") + } } diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/easynews/EasynewsConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/easynews/EasynewsConfigurationProperties.kt index ac6bafef..84865800 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/easynews/EasynewsConfigurationProperties.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/easynews/EasynewsConfigurationProperties.kt @@ -1,17 +1,25 @@ package io.skjaere.debridav.debrid.client.easynews +import io.skjaere.debridav.config.ConfigProperty import org.springframework.boot.context.properties.ConfigurationProperties import java.time.Duration @ConfigurationProperties(prefix = "easynews") -data class EasynewsConfigurationProperties( - val apiBaseUrl: String, - val username: String, - val password: String, - val enabledForTorrents: Boolean, - val rateLimitWindowDuration: Duration, - val allowedRequestsInWindow: Int, - val connectTimeout: Int, - val socketTimeout: Int -) - +class EasynewsConfigurationProperties { + @ConfigProperty(name = "API Base URL", description = "Easynews API base URL", advanced = true) + lateinit var apiBaseUrl: String + @ConfigProperty(name = "Username", description = "Easynews username") + lateinit var username: String + @ConfigProperty(name = "Password", description = "Easynews password", sensitive = true) + lateinit var password: String + @ConfigProperty(name = "Enabled for Torrents", description = "Enable Easynews for torrents") + var enabledForTorrents: Boolean = false + @ConfigProperty(name = "Rate Limit Window", description = "Easynews rate limit window") + var rateLimitWindowDuration: Duration = Duration.ZERO + @ConfigProperty(name = "Allowed Requests in Window", description = "Easynews allowed requests per window") + var allowedRequestsInWindow: Int = 0 + @ConfigProperty(name = "Connect Timeout", description = "Easynews connect timeout") + var connectTimeout: Int = 0 + @ConfigProperty(name = "Socket Timeout", description = "Easynews socket timeout") + var socketTimeout: Int = 0 +} diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/PremiumizeClient.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/PremiumizeClient.kt index 0e61f94f..55c838d9 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/PremiumizeClient.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/PremiumizeClient.kt @@ -3,11 +3,16 @@ package io.skjaere.debridav.debrid.client.premiumize import io.github.resilience4j.ratelimiter.RateLimiter import io.ktor.client.HttpClient import io.ktor.client.call.body +import io.ktor.client.request.accept import io.ktor.client.request.get import io.ktor.client.request.post +import io.ktor.http.ContentType import io.ktor.http.HttpHeaders import io.ktor.http.HttpStatusCode import io.ktor.http.headers +import io.ktor.http.isSuccess +import io.skjaere.debridav.config.ConfigurationTester +import io.skjaere.debridav.config.TestResult import io.skjaere.debridav.configuration.DebridavConfigurationProperties import io.skjaere.debridav.debrid.DebridClient import io.skjaere.debridav.debrid.DebridProvider @@ -18,15 +23,23 @@ import io.skjaere.debridav.debrid.client.StreamableLinkPreparable import io.skjaere.debridav.debrid.client.premiumize.model.CacheCheckResponse import io.skjaere.debridav.debrid.client.premiumize.model.SuccessfulDirectDownloadResponse import io.skjaere.debridav.fs.CachedFile +import kotlin.reflect.KClass +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.jsonPrimitive import org.slf4j.Logger import org.slf4j.LoggerFactory -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.stereotype.Component import java.time.Clock import java.time.Instant +@Serializable +private data class PremiumizeAccountResponse( + val status: String, + val message: String? = null +) + @Component -@ConditionalOnExpression("#{'\${debridav.debrid-clients}'.contains('premiumize')}") class PremiumizeClient( private val premiumizeConfiguration: PremiumizeConfigurationProperties, override val httpClient: HttpClient, @@ -34,6 +47,7 @@ class PremiumizeClient( debridavConfigurationProperties: DebridavConfigurationProperties, premiumizeRateLimiter: RateLimiter ) : DebridCachedTorrentClient, + ConfigurationTester, StreamableLinkPreparable by DefaultStreamableLinkPreparer( httpClient, debridavConfigurationProperties, @@ -41,12 +55,6 @@ class PremiumizeClient( ) { private val logger = LoggerFactory.getLogger(DebridClient::class.java) - init { - require(premiumizeConfiguration.apiKey.isNotEmpty()) { - "Missing API key for Premiumize" - } - } - @Suppress("TooGenericExceptionCaught") override suspend fun isCached(magnet: TorrentMagnet): Boolean { val resp = httpClient @@ -92,8 +100,11 @@ class PremiumizeClient( set(HttpHeaders.Accept, "application/json") } } - - if (resp.status != HttpStatusCode.OK) { + if (!resp.status.isSuccess()) { + throwDebridProviderException(resp, "/transfer/directdl") + } + val json: JsonObject = resp.body() + if (json["status"]?.jsonPrimitive?.content == "error") { throwDebridProviderException(resp, "/transfer/directdl") } return resp.body() @@ -116,4 +127,29 @@ class PremiumizeClient( override fun logger(): Logger { return logger } + + override val configurationClass: KClass<*> = PremiumizeConfigurationProperties::class + override val label: String = "Premiumize" + + @Suppress("TooGenericExceptionCaught") + override suspend fun test(overrides: Map): TestResult = try { + val baseUrl = overrides["premiumize.base-url"] ?: premiumizeConfiguration.baseUrl + val apiKey = overrides["premiumize.api-key"] ?: premiumizeConfiguration.apiKey + + val response = httpClient.get("$baseUrl/account/info?apikey=$apiKey") { + accept(ContentType.Application.Json) + } + if (!response.status.isSuccess()) { + TestResult(success = false, message = "HTTP ${response.status.value}") + } else { + val body = response.body() + if (body.status == "success") { + TestResult(success = true, message = "Connected successfully") + } else { + TestResult(success = false, message = body.message ?: "Authentication failed") + } + } + } catch (e: Exception) { + TestResult(success = false, message = e.message ?: "Unknown error") + } } diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/PremiumizeConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/PremiumizeConfigurationProperties.kt index c6093191..09281fd2 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/PremiumizeConfigurationProperties.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/PremiumizeConfigurationProperties.kt @@ -1,9 +1,12 @@ package io.skjaere.debridav.debrid.client.premiumize +import io.skjaere.debridav.config.ConfigProperty import org.springframework.boot.context.properties.ConfigurationProperties @ConfigurationProperties(prefix = "premiumize") -class PremiumizeConfigurationProperties( - val baseUrl: String, - val apiKey: String -) +class PremiumizeConfigurationProperties { + @ConfigProperty(name = "Base URL", description = "Premiumize base URL", advanced = true) + lateinit var baseUrl: String + @ConfigProperty(name = "API Key", description = "Premiumize API key", sensitive = true) + lateinit var apiKey: String +} diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/model/PremiumizeConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/model/PremiumizeConfiguration.kt index 6c5fa16d..15f58eb3 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/model/PremiumizeConfiguration.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/model/PremiumizeConfiguration.kt @@ -3,7 +3,6 @@ package io.skjaere.debridav.debrid.client.premiumize.model import io.github.resilience4j.ratelimiter.RateLimiter import io.github.resilience4j.ratelimiter.RateLimiterConfig import io.github.resilience4j.ratelimiter.RateLimiterRegistry -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import java.time.Duration @@ -13,7 +12,6 @@ private const val PERIOD_LIMIT = 999 @Configuration class PremiumizeConfiguration { @Bean - @ConditionalOnExpression("#{'\${debridav.debrid-clients}'.contains('premiumize')}") fun premiumizeRateLimiter(rateLimiterRegistry: RateLimiterRegistry): RateLimiter { val rateLimiterConfig = RateLimiterConfig.custom() .limitRefreshPeriod(Duration.ofMinutes(1)) diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridActuatorEndpoint.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridActuatorEndpoint.kt index b33548c2..92453787 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridActuatorEndpoint.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridActuatorEndpoint.kt @@ -3,12 +3,10 @@ package io.skjaere.debridav.debrid.client.realdebrid import org.springframework.boot.actuate.endpoint.annotation.Endpoint import org.springframework.boot.actuate.endpoint.annotation.ReadOperation import org.springframework.boot.actuate.endpoint.annotation.WriteOperation -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.stereotype.Component @Component @Endpoint(id = "realdebrid") -@ConditionalOnExpression("#{'\${debridav.debrid-clients}'.contains('real_debrid')}") class RealDebridActuatorEndpoint( private val realDebridClient: RealDebridClient ) { diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridClient.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridClient.kt index 8b61b120..eae7a542 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridClient.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridClient.kt @@ -13,12 +13,15 @@ import io.ktor.client.request.head import io.ktor.client.request.headers import io.ktor.client.request.post import io.ktor.client.request.setBody +import io.ktor.client.statement.bodyAsText import io.ktor.http.ContentType import io.ktor.http.HttpHeaders import io.ktor.http.HttpStatusCode import io.ktor.http.Parameters import io.ktor.http.contentType import io.ktor.http.isSuccess +import io.skjaere.debridav.config.ConfigurationTester +import io.skjaere.debridav.config.TestResult import io.skjaere.debridav.configuration.DebridavConfigurationProperties import io.skjaere.debridav.debrid.DebridProvider import io.skjaere.debridav.debrid.TorrentMagnet @@ -39,6 +42,7 @@ import io.skjaere.debridav.debrid.client.realdebrid.support.RealDebridDownloadSe import io.skjaere.debridav.debrid.client.realdebrid.support.RealDebridTorrentService import io.skjaere.debridav.fs.CachedFile import io.skjaere.debridav.torrent.TorrentService +import kotlin.reflect.KClass import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope @@ -47,7 +51,6 @@ import kotlinx.coroutines.runBlocking import kotlinx.serialization.json.Json import org.slf4j.Logger import org.slf4j.LoggerFactory -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.scheduling.annotation.Scheduled import org.springframework.stereotype.Component import org.springframework.transaction.annotation.Transactional @@ -59,16 +62,15 @@ private const val LINK_ID_MAP_KEY = "linkId" private const val TORRENT_ID_MAP_KEY = "torrentId" @Component -@ConditionalOnExpression($$"#{'${debridav.debrid-clients}'.contains('real_debrid')}") @Suppress("TooManyFunctions") class RealDebridClient( private val realDebridConfigurationProperties: RealDebridConfigurationProperties, - debridavConfigurationProperties: DebridavConfigurationProperties, + private val debridavConfigurationProperties: DebridavConfigurationProperties, override val httpClient: HttpClient, private val realDebridTorrentService: RealDebridTorrentService, private val realDebridDownloadService: RealDebridDownloadService, private val realDebridRateLimiter: RateLimiter -) : DebridCachedTorrentClient, StreamableLinkPreparable by DefaultStreamableLinkPreparer( +) : DebridCachedTorrentClient, ConfigurationTester, StreamableLinkPreparable by DefaultStreamableLinkPreparer( httpClient, debridavConfigurationProperties, realDebridRateLimiter @@ -78,16 +80,15 @@ class RealDebridClient( var torrentImportEnabled = realDebridConfigurationProperties.syncEnabled - init { - require(realDebridConfigurationProperties.apiKey.isNotEmpty()) { - "Missing API key for Real Debrid" - } - } + private fun isRealDebridConfigured(): Boolean = + DebridProvider.REAL_DEBRID in debridavConfigurationProperties.debridClients && + realDebridConfigurationProperties.apiKey.isNotBlank() @Scheduled( initialDelay = 0, fixedRateString = "\${real-debrid.sync-poll-rate}" ) fun syncTorrentsTask() { + if (!isRealDebridConfigured()) return if (torrentImportEnabled) { runBlocking { launch { @@ -246,7 +247,10 @@ class RealDebridClient( } } if (!resp.status.isSuccess()) { - logger.error("error getting torrent info for id: $id: ${resp.status} ${resp.body()}") + // Don't try to deserialize the error body as TorrentsInfo — kotlinx.serialization + // would throw a SerializationException with no useful message and the real HTTP + // failure would never reach the caller. Funnel through the standard exception path. + throwDebridProviderException(resp, "/torrents/info/$id") } resp.body() } @@ -333,8 +337,11 @@ class RealDebridClient( SuccessfulUnrestrictLinkResponse(entity) } } else { - val responseBody = response.body>() - logger.warn("could not unrestrict link: $link because") + val responseBody = runCatching { response.body>() }.getOrDefault(emptyMap()) + logger.warn( + "could not unrestrict link {}: HTTP {} error={} error_code={}", + link, response.status.value, responseBody["error"], responseBody["error_code"] + ) ErrorUnrestrictLinkResponse(responseBody["error"]) } } @@ -361,31 +368,45 @@ class RealDebridClient( override suspend fun getStreamableLink(key: TorrentMagnet, cachedFile: CachedFile): String? { - //return realDebridDownloadService.getDownloadByLink(cachedFile.params!![LINK_ID_MAP_KEY]!!) - return realDebridDownloadService.getDownloadByHashAndFilenameAndSize( + val existing = realDebridDownloadService.getDownloadByHashAndFilenameAndSize( cachedFile.path!!, cachedFile.size!!, key.getHash()!! - )?.let { realDebridDownload -> - if (isLinkAlive(realDebridDownload.download!!)) { - realDebridDownload.link - } else { - deleteDownload(realDebridDownload.downloadId!!) - realDebridDownloadService.deleteDownload(realDebridDownload) - null + ) + if (existing?.download != null && isDownloadUrlAlive(existing.download!!)) { + return existing.download + } + if (existing != null) { + logger.info( + "RD download {} (link={}) is no longer alive — deleting and refreshing", + existing.downloadId, existing.link + ) + existing.downloadId?.let { id -> runCatching { deleteDownload(id) } } + realDebridDownloadService.deleteDownload(existing) + } + // Reuse the cached share link when we have one — it's the same value + // getFreshRealDebridLink would resolve from the torrent, but skipping + // that lookup means one less round-trip. + val shareLink = existing?.link + ?: getFreshRealDebridLink(key, cachedFile.path!!, cachedFile.size!!) + return shareLink?.let { + when (val result = unrestrictLink(it)) { + is SuccessfulUnrestrictLinkResponse -> result.realDebridDownloadEntity.download + else -> null } - } ?: run { - getFreshRealDebridLink(key, cachedFile.path!!, cachedFile.size!!) - ?.let { - val unrestrictResult = unrestrictLink(it) - when (unrestrictResult) { - is SuccessfulUnrestrictLinkResponse -> unrestrictResult.realDebridDownloadEntity.link - else -> null - } - } } } + @Suppress("TooGenericExceptionCaught") + private suspend fun isDownloadUrlAlive(url: String): Boolean = try { + realDebridRateLimiter.executeSuspendFunction { + httpClient.head(url).status.isSuccess() + } + } catch (e: Exception) { + logger.debug("HEAD on RD download URL failed; treating as dead: {}", e.message) + false + } + suspend fun getFreshRealDebridLink(magnet: TorrentMagnet, filename: String, filesize: Long): String? { val torrents = realDebridTorrentService.getTorrentsByHash(magnet.getHash()!!) if (torrents.size > 1) { @@ -402,7 +423,24 @@ class RealDebridClient( return x } - private suspend fun isLinkAlive(link: String): Boolean { - return realDebridRateLimiter.executeSuspendFunction { httpClient.head(link).status.isSuccess() } + override val configurationClass: KClass<*> = RealDebridConfigurationProperties::class + override val label: String = "Real-Debrid" + + @Suppress("TooGenericExceptionCaught") + override suspend fun test(overrides: Map): TestResult = try { + val baseUrl = overrides["real-debrid.base-url"] ?: realDebridConfigurationProperties.baseUrl + val apiKey = overrides["real-debrid.api-key"] ?: realDebridConfigurationProperties.apiKey + + val response = httpClient.get("$baseUrl/user") { + accept(ContentType.Application.Json) + bearerAuth(apiKey) + } + if (response.status.isSuccess()) { + TestResult(success = true, message = "Connected successfully") + } else { + TestResult(success = false, message = "HTTP ${response.status.value}: ${response.bodyAsText()}") + } + } catch (e: Exception) { + TestResult(success = false, message = e.message ?: "Unknown error") } } diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridConfiguration.kt index 4e4b54d7..270675bd 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridConfiguration.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridConfiguration.kt @@ -3,7 +3,6 @@ package io.skjaere.debridav.debrid.client.realdebrid import io.github.resilience4j.ratelimiter.RateLimiter import io.github.resilience4j.ratelimiter.RateLimiterConfig import io.github.resilience4j.ratelimiter.RateLimiterRegistry -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import java.time.Duration @@ -16,7 +15,6 @@ private const val TIMEOUT = 5L @Configuration class RealDebridConfiguration { @Bean - @ConditionalOnExpression("#{'\${debridav.debrid-clients}'.contains('real_debrid')}") fun realDebridRateLimiter(rateLimiterRegistry: RateLimiterRegistry): RateLimiter { val rateLimiterConfig = RateLimiterConfig.custom() .limitRefreshPeriod(Duration.ofMinutes(WINDOW_DURATION_MINUTES)) diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridConfigurationProperties.kt index cfeec123..e7a54472 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridConfigurationProperties.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridConfigurationProperties.kt @@ -1,10 +1,16 @@ package io.skjaere.debridav.debrid.client.realdebrid +import io.skjaere.debridav.config.ConfigProperty import org.springframework.boot.context.properties.ConfigurationProperties @ConfigurationProperties(prefix = "real-debrid") -class RealDebridConfigurationProperties( - val apiKey: String, - var baseUrl: String, - val syncEnabled: Boolean, -) +class RealDebridConfigurationProperties { + @ConfigProperty(name = "API Key", description = "Real-Debrid API key", sensitive = true) + lateinit var apiKey: String + @ConfigProperty(name = "Base URL", description = "Real-Debrid base URL", advanced = true) + lateinit var baseUrl: String + @ConfigProperty(name = "Sync Enabled", description = "Enable Real-Debrid sync") + var syncEnabled: Boolean = false + @ConfigProperty(name = "Sync Poll Rate", description = "Real-Debrid sync poll rate") + var syncPollRate: String = "PT24H" +} diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/model/RealDebridTorrent.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/model/RealDebridTorrent.kt index 17228617..0fa4ef48 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/model/RealDebridTorrent.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/model/RealDebridTorrent.kt @@ -29,7 +29,10 @@ open class RealDebridTorrentEntity { @ElementCollection open var links: List = emptyList() - @OneToMany(cascade = [CascadeType.ALL], targetEntity = RealDebridTorrentFile::class) + @OneToMany( + cascade = [CascadeType.PERSIST, CascadeType.MERGE, CascadeType.REMOVE], + targetEntity = RealDebridTorrentFile::class, + ) open var files: List = emptyList() } diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/model/RealDebridTorrentRepository.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/model/RealDebridTorrentRepository.kt index bb48fa27..0acc3d62 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/model/RealDebridTorrentRepository.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/model/RealDebridTorrentRepository.kt @@ -1,9 +1,12 @@ package io.skjaere.debridav.debrid.client.realdebrid.model import org.springframework.data.repository.CrudRepository +import org.springframework.transaction.annotation.Transactional interface RealDebridTorrentRepository : CrudRepository { fun findTorrentsByHashIgnoreCase(hash: String): List fun getByTorrentIdIgnoreCase(torrentId: String): RealDebridTorrentEntity? + + @Transactional fun deleteByTorrentIdIgnoreCase(torrentId: String) } diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/support/RealDebridDownloadService.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/support/RealDebridDownloadService.kt index 97e5509d..a21e8a51 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/support/RealDebridDownloadService.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/support/RealDebridDownloadService.kt @@ -17,13 +17,11 @@ import io.skjaere.debridav.debrid.client.realdebrid.model.RealDebridDownloadRepo import io.skjaere.debridav.torrent.TorrentHash import jakarta.transaction.Transactional import kotlinx.coroutines.runBlocking -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.stereotype.Service private const val BULK_SIZE = 100 @Service -@ConditionalOnExpression("#{'\${debridav.debrid-clients}'.contains('real_debrid')}") class RealDebridDownloadService( private val realDebridDownloadRepository: RealDebridDownloadRepository, private val realDebridConfigurationProperties: RealDebridConfigurationProperties, @@ -32,11 +30,27 @@ class RealDebridDownloadService( ) { @Transactional fun syncDownloadsToDatabase(): Unit = runBlocking { - realDebridDownloadRepository.deleteAll() - getListOfDownloads().asSequence() - .map { mapDownloadToRdtEntity(it) } - .toList() - .let { realDebridDownloadRepository.saveAll(it) } + val remoteDownloads = getListOfDownloads() + val remoteDownloadIds = remoteDownloads.map { it.id }.toSet() + + // Remove locally-stored downloads that no longer exist on RD + realDebridDownloadRepository.findAll().forEach { local -> + if (local.downloadId !in remoteDownloadIds) { + realDebridDownloadRepository.delete(local) + } + } + + // Upsert downloads from RD + remoteDownloads.forEach { download -> + val existing = realDebridDownloadRepository.getByDownloadIdIgnoreCase(download.id) + if (existing != null) { + updateDownloadValues(existing, download) + .let { realDebridDownloadRepository.save(it) } + } else { + mapDownloadToRdtEntity(download) + .let { realDebridDownloadRepository.save(it) } + } + } } suspend fun saveDownload(realDebridDownload: RealDebridDownload): RealDebridDownloadEntity { diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/support/RealDebridTorrentService.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/support/RealDebridTorrentService.kt index 3c788590..0e12e27e 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/support/RealDebridTorrentService.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/support/RealDebridTorrentService.kt @@ -17,13 +17,11 @@ import io.skjaere.debridav.debrid.client.realdebrid.model.TorrentsInfo import io.skjaere.debridav.torrent.TorrentHash import jakarta.transaction.Transactional import kotlinx.coroutines.runBlocking -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.stereotype.Component private const val BULK_SIZE = 100 @Component -@ConditionalOnExpression("#{'\${debridav.debrid-clients}'.contains('real_debrid')}") class RealDebridTorrentService( private val realDebridConfigurationProperties: RealDebridConfigurationProperties, private val realDebridTorrentRepository: RealDebridTorrentRepository, @@ -59,11 +57,26 @@ class RealDebridTorrentService( @Transactional fun syncTorrentListToDatabase(): Unit = runBlocking { - realDebridTorrentRepository.deleteAll() - getListOfTorrents().asSequence() - .map { mapTorrentInfoToRdtEntity(it) } - .toList() - .let { realDebridTorrentRepository.saveAll(it) } + val remoteTorrents = getListOfTorrents() + val remoteTorrentIds = remoteTorrents.map { it.id }.toSet() + + // Remove locally-stored torrents that no longer exist on RD + realDebridTorrentRepository.findAll().forEach { local -> + if (local.torrentId !in remoteTorrentIds) { + realDebridTorrentRepository.delete(local) + } + } + + // Upsert torrents from RD + remoteTorrents.forEach { info -> + val existing = realDebridTorrentRepository.getByTorrentIdIgnoreCase(info.id) + val entity = existing ?: RealDebridTorrentEntity() + entity.torrentId = info.id + entity.name = info.filename + entity.hash = info.hash + entity.links = info.links + realDebridTorrentRepository.save(entity) + } } private fun mapTorrentInfoToRdtEntity(info: TorrentsResponseItem): RealDebridTorrentEntity { diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxClient.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxClient.kt index 8339b7a7..199872e7 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxClient.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxClient.kt @@ -21,6 +21,9 @@ import io.ktor.http.isSuccess import io.ktor.http.parameters import io.ktor.http.userAgent import io.milton.http.Range +import io.ktor.client.statement.bodyAsText +import io.skjaere.debridav.config.ConfigurationTester +import io.skjaere.debridav.config.TestResult import io.skjaere.debridav.configuration.DebridavConfigurationProperties import io.skjaere.debridav.debrid.DebridProvider import io.skjaere.debridav.debrid.TorrentMagnet @@ -35,10 +38,10 @@ import io.skjaere.debridav.fs.CachedFile import org.apache.commons.io.FileUtils import org.slf4j.Logger import org.slf4j.LoggerFactory -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.stereotype.Component import java.time.Duration import java.time.Instant +import kotlin.reflect.KClass const val RATE_LIMIT_WINDOW_SIZE_SECONDS = 59L const val RATE_LIMIT_REQUESTS_IN_WINDOW = 60 @@ -46,13 +49,12 @@ const val RATE_LIMIT_TIMEOUT_SECONDS = 5L const val USER_AGENT = "DebriDav/0.9.2 (https://github.com/skjaere/DebriDav)" @Component -@ConditionalOnExpression("#{'\${debridav.debrid-clients}'.contains('torbox')}") class TorBoxClient( private val torboxHttpClient: HttpClient, private val torBoxConfiguration: TorBoxConfigurationProperties, private val debridavConfigurationProperties: DebridavConfigurationProperties, rateLimiterRegistry: RateLimiterRegistry -) : DebridCachedTorrentClient, StreamableLinkPreparable { +) : DebridCachedTorrentClient, ConfigurationTester, StreamableLinkPreparable { companion object { const val TORRENT_ID_KEY = "torrent_id" @@ -244,4 +246,26 @@ class TorBoxClient( } }.status.isSuccess() } + + override val configurationClass: KClass<*> = TorBoxConfigurationProperties::class + override val label: String = "TorBox" + + @Suppress("TooGenericExceptionCaught") + override suspend fun test(overrides: Map): TestResult = try { + val baseUrl = overrides["torbox.base-url"] ?: torBoxConfiguration.baseUrl + val version = overrides["torbox.version"] ?: torBoxConfiguration.version + val apiKey = overrides["torbox.api-key"] ?: torBoxConfiguration.apiKey + + val response = torboxHttpClient.get("$baseUrl/$version/api/user/me") { + accept(ContentType.Application.Json) + bearerAuth(apiKey) + } + if (response.status.isSuccess()) { + TestResult(success = true, message = "Connected successfully") + } else { + TestResult(success = false, message = "HTTP ${response.status.value}: ${response.bodyAsText()}") + } + } catch (e: Exception) { + TestResult(success = false, message = e.message ?: "Unknown error") + } } diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxConfigurationProperties.kt index f4db357b..e111ddac 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxConfigurationProperties.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxConfigurationProperties.kt @@ -1,13 +1,18 @@ package io.skjaere.debridav.debrid.client.torbox +import io.skjaere.debridav.config.ConfigProperty import org.springframework.boot.context.properties.ConfigurationProperties @ConfigurationProperties(prefix = "torbox") -class TorBoxConfigurationProperties( - val apiKey: String, - val baseUrl: String, - val version: String, - val requestTimeoutMillis: Long, - val socketTimeoutMillis: Long, - - ) +class TorBoxConfigurationProperties { + @ConfigProperty(name = "API Key", description = "TorBox API key", sensitive = true) + lateinit var apiKey: String + @ConfigProperty(name = "Base URL", description = "TorBox base URL", advanced = true) + lateinit var baseUrl: String + @ConfigProperty(name = "API Version", description = "TorBox API version") + lateinit var version: String + @ConfigProperty(name = "Request Timeout", description = "TorBox request timeout in ms") + var requestTimeoutMillis: Long = 0 + @ConfigProperty(name = "Socket Timeout", description = "TorBox socket timeout in ms") + var socketTimeoutMillis: Long = 0 +} diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxHttpClientConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxHttpClientConfiguration.kt index 8c9de883..21c203ee 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxHttpClientConfiguration.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxHttpClientConfiguration.kt @@ -13,7 +13,6 @@ import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.serialization.json.Json import org.slf4j.LoggerFactory -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration @@ -27,7 +26,6 @@ class TorBoxHttpClientConfiguration { private val logger = LoggerFactory.getLogger(TorBoxHttpClientConfiguration::class.java) @Bean - @ConditionalOnExpression("#{'\${debridav.debrid-clients}'.contains('torbox')}") fun torboxHttpClient(): HttpClient { val client = HttpClient(CIO) { install(ContentNegotiation) { diff --git a/src/main/kotlin/io/skjaere/debridav/fs/DatabaseFileService.kt b/src/main/kotlin/io/skjaere/debridav/fs/DatabaseFileService.kt index 66adf3fb..35a450b2 100644 --- a/src/main/kotlin/io/skjaere/debridav/fs/DatabaseFileService.kt +++ b/src/main/kotlin/io/skjaere/debridav/fs/DatabaseFileService.kt @@ -2,6 +2,7 @@ package io.skjaere.debridav.fs import io.ipfs.multibase.Base58 import io.skjaere.debridav.configuration.DebridavConfigurationProperties +import io.skjaere.debridav.rclone.FileSystemChangedEvent import io.skjaere.debridav.repository.DebridFileContentsRepository import io.skjaere.debridav.repository.UsenetRepository import io.skjaere.debridav.torrent.TorrentRepository @@ -9,31 +10,33 @@ import jakarta.persistence.EntityManager import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock import kotlinx.coroutines.withContext import org.apache.commons.lang3.Strings import org.hibernate.engine.jdbc.proxy.BlobProxy import org.slf4j.LoggerFactory +import org.springframework.context.ApplicationEventPublisher import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional import java.io.InputStream import java.time.Instant +import java.util.concurrent.locks.ReentrantLock +import kotlin.concurrent.withLock private const val ROOT_NODE = "ROOT" private const val MEGABYTE = 1024 * 1024 @Service +@Suppress("TooManyFunctions") class DatabaseFileService( private val debridFileRepository: DebridFileContentsRepository, private val debridavConfigurationProperties: DebridavConfigurationProperties, private val torrentRepository: TorrentRepository, private val usenetRepository: UsenetRepository, private val entityManager: EntityManager, + private val eventPublisher: ApplicationEventPublisher, ) { private val logger = LoggerFactory.getLogger(DatabaseFileService::class.java) - private val lock = Mutex() + private val lock = ReentrantLock() private val defaultDirectories = listOf("/", "/downloads", "/tv", "/movies") init { @@ -47,12 +50,61 @@ class DatabaseFileService( @Transactional fun createDebridFile( path: String, hash: String, debridFileContents: DebridFileContents - ): RemotelyCachedEntity = runBlocking { + ): RemotelyCachedEntity { val directory = getOrCreateDirectory(path.substringBeforeLast("/")) val name = path.substringAfterLast("/") + val entity = buildDebridFileEntity(path, name, hash, directory, debridFileContents) + emitChange(parentOf(path)) + return entity + } + + /** + * Batch variant of [createDebridFile]. Pre-resolves the unique parent directories + * once instead of N times AND prefetches existing entities at the (directory, name) + * pairs in a single query, so the per-file existence check doesn't trigger an + * auto-flush on every iteration (which would defeat hibernate.jdbc.batch_size). + */ + @Transactional + fun createDebridFiles( + files: List>, + hash: String, + ): List { + val parentByPath = files.associate { (path, _) -> path to path.substringBeforeLast("/") } + val dirCache = parentByPath.values.toSet().associateWith { getOrCreateDirectory(it) } + + // One query for all potential collisions; filter to exact (dir, name) hits. + val targets = files.map { (path, _) -> + dirCache.getValue(parentByPath.getValue(path)) to path.substringAfterLast("/") + } + val existingByKey = if (targets.isEmpty()) { + emptyMap() + } else { + debridFileRepository.findAllByDirectoryInAndNameIn( + targets.map { it.first }.distinct(), + targets.map { it.second }.distinct(), + ).associateBy { it.directory!! to it.name!! } + } + + val emittedParents = mutableSetOf() + return files.map { (path, contents) -> + val name = path.substringAfterLast("/") + val directory = dirCache.getValue(parentByPath.getValue(path)) + val entity = buildDebridFileEntity(path, name, hash, directory, contents, existingByKey[directory to name]) + emittedParents.add(parentOf(path)) + entity + }.also { emitChanges(emittedParents) } + } + private fun buildDebridFileEntity( + path: String, + name: String, + hash: String, + directory: DbDirectory, + contents: DebridFileContents, + existing: DbEntity? = debridFileRepository.findByDirectoryAndName(directory, name), + ): RemotelyCachedEntity { // Overwrite file if it exists - debridFileRepository.findByDirectoryAndName(directory, name)?.let { + existing?.let { it as? RemotelyCachedEntity ?: error("type ${it.javaClass.simpleName} exists at path $path") when (it.contents) { is DebridCachedTorrentContent -> debridFileRepository.unlinkFileFromTorrents(it) @@ -62,16 +114,15 @@ class DatabaseFileService( debridFileRepository.deleteDbEntityByHash(it.hash!!) // TODO: why doesn't debridFileRepository.delete() work? } val fileEntity = RemotelyCachedEntity() - fileEntity.name = path.substringAfterLast("/") + fileEntity.name = name fileEntity.lastModified = Instant.now().toEpochMilli() - fileEntity.size = debridFileContents.size - fileEntity.mimeType = debridFileContents.mimeType + fileEntity.size = contents.size + fileEntity.mimeType = contents.mimeType fileEntity.directory = directory - fileEntity.contents = debridFileContents + fileEntity.contents = contents fileEntity.hash = hash - logger.debug("Creating ${directory.path}/${fileEntity.name}") - fileEntity + return fileEntity } @Transactional @@ -117,6 +168,7 @@ class DatabaseFileService( is RemotelyCachedEntity -> moveFile(destination, dbItem, name) is LocalEntity -> moveFile(destination, dbItem, name) is DbDirectory -> { + val oldParent = parentOfDirectory(dbItem) dbItem.name = name debridFileRepository.save(dbItem) if (directoriesHaveSameParent(dbItem.fileSystemPath()!!, destination)) { @@ -129,6 +181,7 @@ class DatabaseFileService( ) } + emitChanges(setOfNotNull(oldParent, destination)) } } } @@ -138,14 +191,20 @@ class DatabaseFileService( destination: String, dbFile: DbEntity, name: String ) { if (dbFile is DbDirectory) error("entity is directory") + val oldParent = dbFile.directory?.fileSystemPath() val destinationDirectory = getOrCreateDirectory(destination) dbFile.directory = destinationDirectory dbFile.name = name debridFileRepository.save(dbFile) + emitChanges(setOfNotNull(oldParent, destination)) } @Transactional fun deleteFile(file: DbEntity) { + val parent = when (file) { + is DbDirectory -> parentOfDirectory(file) + else -> file.directory?.fileSystemPath() + } when (file) { is RemotelyCachedEntity -> deleteRemotelyCachedEntity(file) is DbDirectory -> debridFileRepository.delete(file) @@ -154,6 +213,7 @@ class DatabaseFileService( debridFileRepository.delete(file) } } + parent?.let { emitChange(it) } } private fun deleteLargeObjectForLocalEntity(file: LocalEntity) { @@ -239,7 +299,9 @@ class DatabaseFileService( localFile.directory = directory localFile.lastModified = System.currentTimeMillis() - return debridFileRepository.save(localFile) + val saved = debridFileRepository.save(localFile) + emitChange(parentOf(path)) + return saved } @@ -257,7 +319,11 @@ class DatabaseFileService( return getOrCreateDirectory(if (path != "/") Strings.CS.removeEnd(path, "/") else path) } - @Transactional + // No @Transactional: Spring binds the transaction to a thread, but a suspend + // function can resume on a different thread (especially with the explicit + // withContext(Dispatchers.IO) below), so the annotation was a no-op here. + // The two repository calls are read-only and run in their own short-lived + // Spring Data read transactions, which is what we want. suspend fun getChildren(directory: DbDirectory): List = withContext(Dispatchers.IO) { listOf( async { debridFileRepository.getChildrenByDirectory(directory) }, @@ -265,19 +331,17 @@ class DatabaseFileService( } @Transactional - fun getOrCreateDirectory(path: String): DbDirectory = runBlocking { - lock.withLock { - getDirectoryTreePaths(path).map { - val directoryEntity = debridFileRepository.getDirectoryByPath(it.pathToLtree()) - if (directoryEntity == null) { - val newDirectoryEntity = DbDirectory() - newDirectoryEntity.path = it.pathToLtree() - newDirectoryEntity.name = if (it != "/") it.substringAfterLast("/") else null - newDirectoryEntity.lastModified = Instant.now().toEpochMilli() - debridFileRepository.save(newDirectoryEntity) - } else directoryEntity - }.last() - } + fun getOrCreateDirectory(path: String): DbDirectory = lock.withLock { + getDirectoryTreePaths(path).map { + val directoryEntity = debridFileRepository.getDirectoryByPath(it.pathToLtree()) + if (directoryEntity == null) { + val newDirectoryEntity = DbDirectory() + newDirectoryEntity.path = it.pathToLtree() + newDirectoryEntity.name = if (it != "/") it.substringAfterLast("/") else null + newDirectoryEntity.lastModified = Instant.now().toEpochMilli() + debridFileRepository.save(newDirectoryEntity) + } else directoryEntity + }.last() } @@ -315,4 +379,19 @@ class DatabaseFileService( private fun directoriesHaveSameParent(first: String, second: String): Boolean { return first.getDirectoryFromPath() == second } + + private fun parentOf(filePath: String): String = filePath.getDirectoryFromPath() + + private fun parentOfDirectory(dir: DbDirectory): String? = + dir.fileSystemPath()?.getDirectoryFromPath() + + private fun emitChange(path: String) { + eventPublisher.publishEvent(FileSystemChangedEvent(setOf(path))) + } + + private fun emitChanges(paths: Set) { + if (paths.isNotEmpty()) { + eventPublisher.publishEvent(FileSystemChangedEvent(paths)) + } + } } diff --git a/src/main/kotlin/io/skjaere/debridav/fs/DbItem.kt b/src/main/kotlin/io/skjaere/debridav/fs/DbItem.kt index 5676809c..0bf04520 100644 --- a/src/main/kotlin/io/skjaere/debridav/fs/DbItem.kt +++ b/src/main/kotlin/io/skjaere/debridav/fs/DbItem.kt @@ -29,7 +29,10 @@ import org.hibernate.annotations.Type @DiscriminatorColumn(name = "db_item_type", discriminatorType = DiscriminatorType.STRING) @Table( name = "db_item", - indexes = [Index(name = "directory_path", columnList = "path")], + indexes = [ + Index(name = "directory_path", columnList = "path"), + Index(name = "idx_db_item_directory_id", columnList = "directory_id"), + ], uniqueConstraints = [UniqueConstraint(columnNames = arrayOf("directory_id", "name"))] ) abstract class DbEntity { @@ -61,11 +64,21 @@ open class DbDirectory : DbEntity() { ?.filter { it != "ROOT" } ?.joinToString("/") { Base58.decode(it).decodeToString() } ?.let { "/$it" } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is DbDirectory) return false + return path != null && path == other.path + } + + override fun hashCode(): Int = path?.hashCode() ?: 0 } @Entity open class RemotelyCachedEntity : DbEntity() { - @OneToOne(cascade = [CascadeType.ALL]) + // Lifecycle cascades only — drop REFRESH and DETACH, which the audit flagged + // as risky for an exclusively-owned child like this one. + @OneToOne(cascade = [CascadeType.PERSIST, CascadeType.MERGE, CascadeType.REMOVE]) @JoinColumn(name = "debrid_file_contents_id") open var contents: DebridFileContents? = null @@ -100,7 +113,10 @@ open class RemotelyCachedEntity : DbEntity() { @Entity open class LocalEntity : DbEntity() { - @OneToOne(fetch = FetchType.LAZY, cascade = [(CascadeType.ALL)]) + @OneToOne( + fetch = FetchType.LAZY, + cascade = [CascadeType.PERSIST, CascadeType.MERGE, CascadeType.REMOVE], + ) @JoinColumn(name = "blob_id") open var blob: Blob? = null } diff --git a/src/main/kotlin/io/skjaere/debridav/fs/DebridFileContents.kt b/src/main/kotlin/io/skjaere/debridav/fs/DebridFileContents.kt index 7ba11d86..5c429cef 100644 --- a/src/main/kotlin/io/skjaere/debridav/fs/DebridFileContents.kt +++ b/src/main/kotlin/io/skjaere/debridav/fs/DebridFileContents.kt @@ -8,6 +8,8 @@ import com.fasterxml.jackson.annotation.JsonTypeName import io.hypersistence.utils.hibernate.type.json.JsonBinaryType import io.skjaere.debridav.debrid.DebridProvider import io.skjaere.debridav.usenet.nzb.NzbDocumentEntity +import org.hibernate.annotations.OnDelete +import org.hibernate.annotations.OnDeleteAction import jakarta.persistence.Column import jakarta.persistence.DiscriminatorColumn import jakarta.persistence.DiscriminatorType @@ -105,6 +107,7 @@ open class DebridCachedUsenetReleaseContent() : DebridFileContents() { open class NzbContents : DebridFileContents() { @ManyToOne(fetch = FetchType.EAGER) @JoinColumn(name = "nzb_document_id") + @OnDelete(action = OnDeleteAction.CASCADE) open var nzbDocument: NzbDocumentEntity? = null } diff --git a/src/main/kotlin/io/skjaere/debridav/fs/FileController.kt b/src/main/kotlin/io/skjaere/debridav/fs/FileController.kt new file mode 100644 index 00000000..5f548e7f --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/fs/FileController.kt @@ -0,0 +1,140 @@ +package io.skjaere.debridav.fs + +import io.skjaere.debridav.config.auth.JwtService +import kotlinx.coroutines.runBlocking +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RequestParam +import org.springframework.web.bind.annotation.RestController + +@RestController +@RequestMapping("/api/v1/files") +class FileController( + private val databaseFileService: DatabaseFileService, + private val jwtService: JwtService +) { + @Suppress("ReturnCount") + @GetMapping("/stream-url") + fun streamUrl(@RequestParam path: String): ResponseEntity { + val entity = databaseFileService.getFileAtPath(path) + ?: return ResponseEntity.notFound().build() + + if (entity is DbDirectory) { + return ResponseEntity.badRequest().build() + } + + val token = jwtService.generateStreamToken(path) + return ResponseEntity.ok( + StreamUrlDto( + url = "/api/v1/stream/t/$token", + expiresIn = JwtService.STREAM_TOKEN_EXPIRY_SECONDS + ) + ) + } + + @Suppress("ReturnCount") + @GetMapping("/detail") + fun detail(@RequestParam path: String): ResponseEntity { + val entity = databaseFileService.getFileAtPath(path) + ?: return ResponseEntity.notFound().build() + + if (entity is DbDirectory) { + return ResponseEntity.badRequest().build() + } + + val dto = when (entity) { + is RemotelyCachedEntity -> buildRemoteDetail(entity, path) + is LocalEntity -> buildLocalDetail(entity, path) + else -> return ResponseEntity.badRequest().build() + } + + return ResponseEntity.ok(dto) + } + + private fun buildRemoteDetail(entity: RemotelyCachedEntity, path: String): FileDetailDto { + val contents = entity.contents + val fileType = when (contents) { + is DebridCachedTorrentContent -> FileType.TORRENT + is DebridCachedUsenetReleaseContent -> FileType.USENET_RELEASE + is NzbContents -> FileType.NZB + else -> FileType.LOCAL + } + val providerStatus = contents?.debridLinks?.mapNotNull { link -> + val provider = link.provider ?: return@mapNotNull null + val status = when (link) { + is CachedFile -> ProviderCacheStatus.CACHED + is MissingFile -> ProviderCacheStatus.MISSING + is ProviderError -> ProviderCacheStatus.PROVIDER_ERROR + is ClientError -> ProviderCacheStatus.CLIENT_ERROR + is NetworkError -> ProviderCacheStatus.NETWORK_ERROR + else -> ProviderCacheStatus.UNKNOWN_ERROR + } + ProviderStatusDto( + provider = provider, + status = status, + lastChecked = link.lastChecked + ) + } + return FileDetailDto( + name = entity.name ?: "", + path = path, + size = entity.size, + lastModified = entity.lastModified, + mimeType = entity.mimeType, + fileType = fileType, + hash = entity.hash, + providerStatus = providerStatus + ) + } + + private fun buildLocalDetail(entity: LocalEntity, path: String): FileDetailDto { + return FileDetailDto( + name = entity.name ?: "", + path = path, + size = entity.size, + lastModified = entity.lastModified, + mimeType = entity.mimeType, + fileType = FileType.LOCAL, + hash = null, + providerStatus = null + ) + } + + @Suppress("ReturnCount") + @GetMapping + fun list(@RequestParam(defaultValue = "/") path: String): ResponseEntity> { + val entity = databaseFileService.getFileAtPath(path) + ?: return ResponseEntity.notFound().build() + + if (entity !is DbDirectory) { + return ResponseEntity.badRequest().build() + } + + val children = runBlocking { databaseFileService.getChildren(entity) } + + val entries = children.mapNotNull { child -> + val name = child.name ?: return@mapNotNull null + when (child) { + is DbDirectory -> FileEntryDto( + name = name, + path = child.fileSystemPath() ?: path, + isDirectory = true, + size = null, + lastModified = child.lastModified, + mimeType = null + ) + else -> FileEntryDto( + name = name, + path = "${path.trimEnd('/')}/$name", + isDirectory = false, + size = child.size, + lastModified = child.lastModified, + mimeType = child.mimeType + ) + } + }.sortedWith(compareByDescending { it.isDirectory }.thenBy { it.name.lowercase() }) + + return ResponseEntity.ok(entries) + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/fs/FileDetailDto.kt b/src/main/kotlin/io/skjaere/debridav/fs/FileDetailDto.kt new file mode 100644 index 00000000..60e57076 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/fs/FileDetailDto.kt @@ -0,0 +1,28 @@ +package io.skjaere.debridav.fs + +import io.skjaere.debridav.debrid.DebridProvider + +data class FileDetailDto( + val name: String, + val path: String, + val size: Long?, + val lastModified: Long?, + val mimeType: String?, + val fileType: FileType, + val hash: String?, + val providerStatus: List? +) + +enum class FileType { + TORRENT, USENET_RELEASE, NZB, LOCAL +} + +data class ProviderStatusDto( + val provider: DebridProvider, + val status: ProviderCacheStatus, + val lastChecked: Long? +) + +enum class ProviderCacheStatus { + CACHED, MISSING, PROVIDER_ERROR, CLIENT_ERROR, NETWORK_ERROR, UNKNOWN_ERROR +} diff --git a/src/main/kotlin/io/skjaere/debridav/fs/FileEntryDto.kt b/src/main/kotlin/io/skjaere/debridav/fs/FileEntryDto.kt new file mode 100644 index 00000000..9d1e2f5b --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/fs/FileEntryDto.kt @@ -0,0 +1,10 @@ +package io.skjaere.debridav.fs + +data class FileEntryDto( + val name: String, + val path: String, + val isDirectory: Boolean, + val size: Long?, + val lastModified: Long?, + val mimeType: String? +) diff --git a/src/main/kotlin/io/skjaere/debridav/fs/FileSystemFileService.kt b/src/main/kotlin/io/skjaere/debridav/fs/FileSystemFileService.kt deleted file mode 100644 index d31b199f..00000000 --- a/src/main/kotlin/io/skjaere/debridav/fs/FileSystemFileService.kt +++ /dev/null @@ -1,158 +0,0 @@ -/* -package io.skjaere.debridav.fs - -import com.google.common.cache.CacheBuilder -import com.google.common.cache.CacheLoader -import com.google.common.cache.LoadingCache -import io.milton.resource.Resource -import io.skjaere.debridav.configuration.DebridavConfiguration -import io.skjaere.debridav.resource.DebridFileResource -import io.skjaere.debridav.resource.DirectoryResource -import io.skjaere.debridav.resource.FileResource -import io.skjaere.debridav.resource.StreamableResourceFactory -import jakarta.annotation.PostConstruct -import kotlinx.serialization.SerializationException -import kotlinx.serialization.encodeToString -import kotlinx.serialization.json.Json -import org.slf4j.LoggerFactory -import org.springframework.stereotype.Service -import java.io.File -import java.io.InputStream -import java.nio.file.Files -import java.nio.file.StandardCopyOption -import javax.naming.ConfigurationException - -@Service -class FileSystemFileService( - private val debridavConfiguration: DebridavConfiguration, - - ) { - companion object { - private const val CACHE_SIZE: Long = 1000 - } - - private val logger = LoggerFactory.getLogger(FileSystemFileService::class.java) - - private val cache: LoadingCache = CacheBuilder.newBuilder() - .maximumSize(CACHE_SIZE) - .build(CacheLoader.from { path -> loadContentsFromFile(path) }) - - @PostConstruct - fun postConstruct() { - if (debridavConfiguration.rootPath.endsWith("/")) { - throw ConfigurationException( - "debridav.root-path: ${debridavConfiguration.rootPath} should not contain a trailing slash" - ) - } - } - - fun createDebridFile( - path: String, - debridFileContents: DebridFileContents - ): File { - return createLocalFile( - "${debridavConfiguration.rootPath}/$path.debridfile", - Json.encodeToString(debridFileContents).byteInputStream() - ) - } - - fun createLocalFile( - directory: String, - inputStream: InputStream - ): File { - val file = File(directory) - return writeFile(file, inputStream) - } - - private fun writeFile(file: File, inputStream: InputStream): File { - if (file.exists()) { - file.delete() - } - if (!Files.exists(file.toPath().parent)) { - Files.createDirectories(file.toPath().parent) - } - file.createNewFile() - inputStream.transferTo(file.outputStream()) - return file - } - - fun moveFile(path: String, destinationDirectory: String, name: String) { - val src = File(path) - val destination = File("$destinationDirectory/$name") - - if (!destination.parentFile.exists()) { - destination.parentFile.mkdirs() - } - Files.move( - src.toPath(), - destination.toPath(), - StandardCopyOption.REPLACE_EXISTING - ) - cache.getIfPresent(path)?.let { - cache.invalidate(path) - cache.put(src.path, it) - } - } - - fun deleteFile(file: File) { - cache.invalidate(file.path) - file.delete() - } - - fun createDirectory(path: String, resourceFactory: StreamableResourceFactory): DirectoryResource { - val file = File(path) - - if (!Files.exists(file.toPath())) { - Files.createDirectory(file.toPath()) - } - - return DirectoryResource(file, resourceFactory, this) - } - - fun getFileAtPath(path: String): File? { - val file = File("${debridavConfiguration.rootPath}$path") - if (file.exists()) return file - return null - } - - fun moveResource(item: Resource, destination: String, name: String) { - when (item) { - is FileResource -> moveFile(item.file.path, destination, name) - is DebridFileResource -> moveFile(item.file.path, destination, "$name.debridfile") - is DirectoryResource -> moveFile(item.directory.path, destination, name) - } - } - - fun getSizeOfCachedContent(debridFile: File): Long { - return cache.get(debridFile.path)!!.size - } - - fun writeContentsToFile(file: File, debridFileContents: DebridFileContents) { - file.writeText(Json.encodeToString(debridFileContents)) - cache.put(file.path, debridFileContents) - } - - fun getDebridFileContents(file: File): DebridFileContents = cache.get(file.path)!! - - fun handleNoLongerCachedFile(file: File) { - if (debridavConfiguration.shouldDeleteNonWorkingFiles) { - logger.info("file ${file.name} is no longer cached. Deleting...") - file.delete() - } - } - - - private fun loadContentsFromFile(path: String): DebridFileContents? { - return if (File(path).exists()) { - try { - Json.decodeFromString(File(path).readText(Charsets.UTF_8)) - } catch (e: SerializationException) { - logger.error("Error deserializing contents of debrid file: $path", e) - return null - } - } else { - null - } - } -} -*/ diff --git a/src/main/kotlin/io/skjaere/debridav/fs/StreamController.kt b/src/main/kotlin/io/skjaere/debridav/fs/StreamController.kt new file mode 100644 index 00000000..7a83aa17 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/fs/StreamController.kt @@ -0,0 +1,92 @@ +package io.skjaere.debridav.fs + +import io.milton.http.Range +import io.milton.resource.GetableResource +import io.skjaere.debridav.config.auth.JwtService +import io.skjaere.debridav.resource.StreamableResourceFactory +import jakarta.servlet.http.HttpServletRequest +import jakarta.servlet.http.HttpServletResponse +import org.springframework.http.HttpStatus +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@RestController +@RequestMapping("/api/v1/stream") +class StreamController( + private val jwtService: JwtService, + private val databaseFileService: DatabaseFileService, + private val streamableResourceFactory: StreamableResourceFactory +) { + @Suppress("ReturnCount") + @GetMapping("/t/{token}") + fun streamByToken( + @PathVariable token: String, + request: HttpServletRequest, + response: HttpServletResponse + ) { + val path = jwtService.validateStreamToken(token) + if (path == null) { + response.status = HttpStatus.UNAUTHORIZED.value() + response.contentType = "application/json" + response.writer.write("""{"error":"Invalid or expired token"}""") + return + } + + val entity = databaseFileService.getFileAtPath(path) + if (entity == null || entity is DbDirectory) { + response.status = HttpStatus.NOT_FOUND.value() + response.contentType = "application/json" + response.writer.write("""{"error":"File not found"}""") + return + } + + val resource = streamableResourceFactory.toFileResource(entity) as? GetableResource + if (resource == null) { + response.status = HttpStatus.NOT_FOUND.value() + response.contentType = "application/json" + response.writer.write("""{"error":"File not found"}""") + return + } + + val contentLength = resource.contentLength + val contentType = resource.getContentType(null) ?: "application/octet-stream" + val rangeHeader = request.getHeader("Range") + val range = parseRangeHeader(rangeHeader, contentLength) + + response.contentType = contentType + response.setHeader("Accept-Ranges", "bytes") + + if (range != null) { + val start = range.start ?: 0 + val finish = range.finish ?: (contentLength - 1) + response.status = HttpServletResponse.SC_PARTIAL_CONTENT + response.setHeader("Content-Range", "bytes $start-$finish/$contentLength") + response.setContentLengthLong(finish - start + 1) + } else { + response.status = HttpServletResponse.SC_OK + response.setContentLengthLong(contentLength) + } + + resource.sendContent(response.outputStream, range, null, contentType) + } + + @Suppress("ReturnCount") + private fun parseRangeHeader(header: String?, contentLength: Long): Range? { + if (header == null || !header.startsWith("bytes=")) return null + val rangeSpec = header.removePrefix("bytes=") + val parts = rangeSpec.split("-", limit = 2) + if (parts.size != 2) return null + + val start = parts[0].toLongOrNull() + val end = parts[1].toLongOrNull() + + return when { + start != null && end != null -> Range(start, end) + start != null -> Range(start, contentLength - 1) + end != null -> Range(contentLength - end, contentLength - 1) + else -> null + } + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/fs/StreamUrlDto.kt b/src/main/kotlin/io/skjaere/debridav/fs/StreamUrlDto.kt new file mode 100644 index 00000000..2ae8d9f8 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/fs/StreamUrlDto.kt @@ -0,0 +1,3 @@ +package io.skjaere.debridav.fs + +data class StreamUrlDto(val url: String, val expiresIn: Long) diff --git a/src/main/kotlin/io/skjaere/debridav/fs/import/FileImport.kt b/src/main/kotlin/io/skjaere/debridav/fs/import/FileImport.kt deleted file mode 100644 index 666ce0c5..00000000 --- a/src/main/kotlin/io/skjaere/debridav/fs/import/FileImport.kt +++ /dev/null @@ -1,30 +0,0 @@ -package io.skjaere.debridav.fs.import - -import jakarta.persistence.Column -import jakarta.persistence.Entity -import jakarta.persistence.GeneratedValue -import jakarta.persistence.GenerationType -import jakarta.persistence.Id -import jakarta.persistence.Index -import jakarta.persistence.Table - -@Entity -@Table( - name = "import_registry", - indexes = [Index(name = "imported_files", columnList = "path")] -) -open class FileImport() { - @Id - @GeneratedValue(strategy = GenerationType.AUTO) - open var id: Long? = null - - @Column( - unique = true, - length = 2048 - ) - open var path: String? = null - - constructor(path: String) : this() { - this.path = path - } -} diff --git a/src/main/kotlin/io/skjaere/debridav/fs/import/FileSystemImportService.kt b/src/main/kotlin/io/skjaere/debridav/fs/import/FileSystemImportService.kt deleted file mode 100644 index 2d58256b..00000000 --- a/src/main/kotlin/io/skjaere/debridav/fs/import/FileSystemImportService.kt +++ /dev/null @@ -1,328 +0,0 @@ -package io.skjaere.debridav.fs.import - -import io.skjaere.debridav.category.Category -import io.skjaere.debridav.category.CategoryService -import io.skjaere.debridav.configuration.DebridavConfigurationProperties -import io.skjaere.debridav.debrid.DebridProvider -import io.skjaere.debridav.debrid.TorrentMagnet -import io.skjaere.debridav.debrid.model.CachedFile -import io.skjaere.debridav.debrid.model.ClientError -import io.skjaere.debridav.debrid.model.MissingFile -import io.skjaere.debridav.debrid.model.NetworkError -import io.skjaere.debridav.debrid.model.ProviderError -import io.skjaere.debridav.fs.Blob -import io.skjaere.debridav.fs.DatabaseFileService -import io.skjaere.debridav.fs.DbEntity -import io.skjaere.debridav.fs.DebridCachedTorrentContent -import io.skjaere.debridav.fs.DebridCachedUsenetReleaseContent -import io.skjaere.debridav.fs.LocalEntity -import io.skjaere.debridav.fs.RemotelyCachedEntity -import io.skjaere.debridav.fs.legacy.DebridFileContents -import io.skjaere.debridav.fs.legacy.DebridFileContents.Type -import io.skjaere.debridav.repository.DebridFileContentsRepository -import io.skjaere.debridav.repository.UsenetRepository -import io.skjaere.debridav.torrent.Torrent -import io.skjaere.debridav.torrent.TorrentRepository -import io.skjaere.debridav.torrent.TorrentService -import io.skjaere.debridav.usenet.UsenetDownload -import io.skjaere.debridav.usenet.UsenetDownloadStatus -import kotlin.io.path.exists -import kotlin.io.path.isRegularFile -import kotlin.io.path.name -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.ExperimentalCoroutinesApi -import kotlinx.coroutines.channels.ReceiveChannel -import kotlinx.coroutines.channels.consumeEach -import kotlinx.coroutines.channels.produce -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.launch -import kotlinx.coroutines.runBlocking -import kotlinx.serialization.json.Json -import org.hibernate.engine.jdbc.proxy.BlobProxy -import org.slf4j.LoggerFactory -import org.springframework.boot.health.contributor.Health -import org.springframework.boot.health.contributor.HealthIndicator -import org.springframework.boot.context.event.ApplicationReadyEvent -import org.springframework.context.event.EventListener -import org.springframework.dao.DataIntegrityViolationException -import org.springframework.stereotype.Service -import org.springframework.transaction.PlatformTransactionManager -import org.springframework.transaction.support.TransactionTemplate -import java.io.File -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths - - -@Service -@Suppress("LongParameterList") -class FileSystemImportService( - private val databaseFileService: DatabaseFileService, - private val debridavConfigurationProperties: DebridavConfigurationProperties, - private val usenetRepository: UsenetRepository, - private val importRegistryRepository: ImportRegistryRepository, - private val torrentRepository: TorrentRepository, - private val debridFileContentsRepository: DebridFileContentsRepository, - platformTransactionManager: PlatformTransactionManager, - categoryService: CategoryService -) : HealthIndicator { - private val logger = LoggerFactory.getLogger(DatabaseFileService::class.java) - private val ignoredFiles = listOf("lb-db.mv.db") - private val importCategory: Category = - categoryService.findByName("imported") ?: categoryService.createCategory("imported") - private val transactionTemplate = TransactionTemplate(platformTransactionManager) - private var isImporting = true - - @EventListener(ApplicationReadyEvent::class) - fun startImport() { - if (debridavConfigurationProperties.enableFileImportOnStartup) { - runBlocking { - importDebridFilesFromFileSystem() - } - } - isImporting = false - } - - suspend fun importDebridFilesFromFileSystem() = coroutineScope { - launch { - saveEntity( - deserializeFiles( - getFlowOfFilesToImport() - ) - ) - } - } - - @OptIn(ExperimentalCoroutinesApi::class) - suspend fun CoroutineScope.deserializeFiles(channel: ReceiveChannel): ReceiveChannel = - produce { - channel.consumeEach { ctx -> - if (ctx.file.isDebridFile()) { - send( - ctx.copy( - fileContents = Json.decodeFromString( - DebridFileContents.serializer(), - ctx.path.toFile().readText() - ) - ) - ) - } else send(ctx) - } - } - - @OptIn(ExperimentalCoroutinesApi::class) - suspend fun CoroutineScope.getFlowOfFilesToImport(): ReceiveChannel = - this.produce { - if (!Path.of(debridavConfigurationProperties.rootPath).exists()) { - logger.warn( - "Can't start import. Root directory does not exist: " + - debridavConfigurationProperties.rootPath - ) - return@produce - } - Files - .walk(Paths.get(debridavConfigurationProperties.rootPath)) - .use { files -> - val filesList = files.toList() - filesList.asSequence() - .filter { it.isRegularFile() } - .filter { !ignoredFiles.contains(it.name.substringAfterLast('/')) } - .filter { !importRegistryRepository.existsByPath(it.toString()) } - .forEach { path -> send(ImportContext(path, path.toFile(), null)) } - } - - } - - @OptIn(ExperimentalCoroutinesApi::class) - private suspend fun saveEntity(channel: ReceiveChannel) { - channel.consumeEach { ctx -> - try { - transactionTemplate.execute { - val entity = mapFileToDbEntity(ctx) - when (entity) { - is RemotelyCachedEntity -> saveRemotelyCachedEntity(entity, ctx) - is LocalEntity -> debridFileContentsRepository.save(entity) - } - importRegistryRepository.save(FileImport(ctx.path.toString())) - logger.info("${ctx.path} was successfully imported") - } - } catch (e: DataIntegrityViolationException) { - logger.error("An error occurred during import of ${ctx.path}:${e.message}") - } - } - } - - private fun saveRemotelyCachedEntity( - entity: RemotelyCachedEntity, - ctx: ImportContext - ) { - when (entity.contents) { - is DebridCachedTorrentContent -> saveTorrentEntity(ctx, entity) - is DebridCachedUsenetReleaseContent -> saveUsenetEntity(ctx, entity) - } - } - - private fun saveTorrentEntity(ctx: ImportContext, entity: DbEntity) { - val torrent: Torrent = TorrentService.getHashFromMagnet( - TorrentMagnet(ctx.fileContents!!.magnet) - )?.let { hash -> - torrentRepository.getByHashIgnoreCase(hash.hash) ?: run { - val newTorrent = Torrent() - newTorrent.name = TorrentService.getNameFromMagnet(TorrentMagnet(ctx.fileContents.magnet)) - newTorrent.hash = hash.hash - newTorrent.category = importCategory - newTorrent.savePath = "" - newTorrent - } - } ?: run { - error("Could not get hash from torrent. File: ${ctx.path} cannot be imported") - } - torrent.files.add(entity as RemotelyCachedEntity) - torrentRepository.save(torrent) - } - - @Suppress("MagicNumber") - private fun saveUsenetEntity(ctx: ImportContext, entity: DbEntity) { - val usenetDownload: UsenetDownload = - usenetRepository.getByName(ctx.fileContents!!.magnet) - ?: run { - val newUsenetDownload = UsenetDownload() - newUsenetDownload.category = importCategory - newUsenetDownload.name = ctx.fileContents.magnet - newUsenetDownload.status = UsenetDownloadStatus.DELETED - newUsenetDownload.percentCompleted = 100.0 - - newUsenetDownload - } - - usenetDownload.debridFiles.add(entity as RemotelyCachedEntity) - usenetRepository.save(usenetDownload) - } - - private fun mapFileToDbEntity(ctx: ImportContext): DbEntity { - val file = ctx.path.toFile() - return if (file.isDebridFile()) { - mapDebridFileToRemotelyCachedItem(ctx) - } else { - mapLocalFileToLocalEntity(ctx.path.toFile()) - } - } - - private fun mapLocalFileToLocalEntity(file: File): LocalEntity { - val entity = LocalEntity() - entity.directory = databaseFileService.getOrCreateDirectory( - file.path - .substringAfterLast(debridavConfigurationProperties.rootPath) - .substringBeforeLast("/") - ) - entity.name = file.name - entity.lastModified = file.lastModified() - entity.size = file.length() - entity.blob = Blob(BlobProxy.generateProxy(file.inputStream(), file.length()), file.length()) - - return entity - } - - private fun mapDebridFileToRemotelyCachedItem(ctx: ImportContext): RemotelyCachedEntity { - val entity = RemotelyCachedEntity() - entity.name = ctx.file.path.getFileNameFromDebridFile() - entity.directory = databaseFileService.getOrCreateDirectory( - ctx.file.path - .substringAfterLast(debridavConfigurationProperties.rootPath) - .substringBeforeLast("/") - ) - entity.lastModified = ctx.file.lastModified() - - return if (ctx.fileContents!!.type == Type.TORRENT_MAGNET) { - mapFileContentsToDebridCachedTorrentContent(ctx.fileContents, entity) - } else if (ctx.fileContents.type == Type.USENET_RELEASE) { - mapFileContentsToDebridCachedUsenetReleaseContent(ctx.fileContents, entity) - } else { - error("unknown type: ${ctx.fileContents.type}") - } - } - - private fun String.getFileNameFromDebridFile(): String = - this.substringAfterLast("/").substringBeforeLast(".debridfile") - - private fun mapFileContentsToDebridCachedUsenetReleaseContent( - deserializedDebridFileContents: DebridFileContents, - entity: RemotelyCachedEntity - ): RemotelyCachedEntity { - val contents = DebridCachedUsenetReleaseContent() - contents.releaseName = deserializedDebridFileContents.magnet - contents.originalPath = deserializedDebridFileContents.originalPath - contents.size = deserializedDebridFileContents.size - contents.debridLinks = - mapLegacyDebridLinksToDebridLinks(deserializedDebridFileContents.debridLinks).toMutableList() - entity.contents = contents - return entity - } - - private fun mapFileContentsToDebridCachedTorrentContent( - deserializedDebridFileContents: DebridFileContents, - entity: RemotelyCachedEntity - ): RemotelyCachedEntity { - val contents = DebridCachedTorrentContent() - contents.magnet = deserializedDebridFileContents.magnet - contents.originalPath = deserializedDebridFileContents.originalPath - contents.size = deserializedDebridFileContents.size - contents.debridLinks = mapLegacyDebridLinksToDebridLinks( - deserializedDebridFileContents.debridLinks - ).toMutableList() - entity.contents = contents - return entity - } - - private fun mapLegacyDebridLinksToDebridLinks( - legacyDebridLinks: List - ): List { - return legacyDebridLinks.map { - when (it) { - is NetworkError -> io.skjaere.debridav.fs.NetworkError(it.provider.toNewProvider(), it.lastChecked) - is CachedFile -> mapLegacyCachedFileToCachedFile(it) - is ClientError -> io.skjaere.debridav.fs.ClientError(it.provider.toNewProvider(), it.lastChecked) - is MissingFile -> io.skjaere.debridav.fs.MissingFile(it.provider.toNewProvider(), it.lastChecked) - is ProviderError -> io.skjaere.debridav.fs.ProviderError( - it.provider.toNewProvider(), - it.lastChecked - ) - } - } - } - - private fun mapLegacyCachedFileToCachedFile(cachedFile: CachedFile): io.skjaere.debridav.fs.CachedFile { - return io.skjaere.debridav.fs.CachedFile( - path = cachedFile.path, - size = cachedFile.size, - mimeType = cachedFile.mimeType, - link = cachedFile.link!!, - params = cachedFile.params, - lastChecked = cachedFile.lastChecked, - provider = cachedFile.provider.toNewProvider() - ) - } - - fun DebridProvider.toNewProvider(): DebridProvider { - return when (this) { - DebridProvider.REAL_DEBRID -> DebridProvider.REAL_DEBRID - DebridProvider.PREMIUMIZE -> DebridProvider.PREMIUMIZE - DebridProvider.EASYNEWS -> DebridProvider.EASYNEWS - DebridProvider.TORBOX -> DebridProvider.TORBOX - } - } - - private fun File.isDebridFile(): Boolean = this.path.endsWith(".debridfile") - - override fun health(): Health = - Health.status( - if (isImporting) "DOWN" else "UP" - ).build() - - - data class ImportContext( - val path: Path, - val file: File, - val fileContents: DebridFileContents? - ) -} diff --git a/src/main/kotlin/io/skjaere/debridav/fs/import/ImportRegistryRepository.kt b/src/main/kotlin/io/skjaere/debridav/fs/import/ImportRegistryRepository.kt deleted file mode 100644 index 735bc8e3..00000000 --- a/src/main/kotlin/io/skjaere/debridav/fs/import/ImportRegistryRepository.kt +++ /dev/null @@ -1,7 +0,0 @@ -package io.skjaere.debridav.fs.import - -import org.springframework.data.repository.CrudRepository - -interface ImportRegistryRepository : CrudRepository { - fun existsByPath(path: String): Boolean -} diff --git a/src/main/kotlin/io/skjaere/debridav/fs/legacy/DebridFileContents.kt b/src/main/kotlin/io/skjaere/debridav/fs/legacy/DebridFileContents.kt deleted file mode 100644 index 396c0cf7..00000000 --- a/src/main/kotlin/io/skjaere/debridav/fs/legacy/DebridFileContents.kt +++ /dev/null @@ -1,45 +0,0 @@ -package io.skjaere.debridav.fs.legacy - -import io.skjaere.debridav.debrid.model.DebridFile -import kotlinx.serialization.Serializable - -@Serializable -data class DebridFileContents( - var originalPath: String, - var size: Long, - var modified: Long, - var magnet: String, - var debridLinks: MutableList, - var type: Type = Type.TORRENT_MAGNET -) { - fun replaceOrAddDebridLink(debridLink: DebridFile) { - if (debridLinks.any { link -> link.provider == debridLink.provider }) { - val index = debridLinks.indexOfFirst { link -> link.provider == debridLink.provider } - debridLinks[index] = debridLink - } else { - debridLinks.add(debridLink) - } - } - - override fun equals(other: Any?): Boolean { - if (other is DebridFileContents) { - return originalPath == other.originalPath && - size == other.size && - magnet == other.magnet && - debridLinks == other.debridLinks - } - - return super.equals(other) - } - - override fun hashCode(): Int { - var result = originalPath.hashCode() - result = 31 * result + size.hashCode() - result = 31 * result + modified.hashCode() - result = 31 * result + magnet.hashCode() - result = 31 * result + debridLinks.hashCode() - return result - } - - enum class Type { USENET_RELEASE, TORRENT_MAGNET } -} diff --git a/src/main/kotlin/io/skjaere/debridav/health/HealthCheckConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/health/HealthCheckConfigurationProperties.kt new file mode 100644 index 00000000..87da2e6d --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/health/HealthCheckConfigurationProperties.kt @@ -0,0 +1,27 @@ +package io.skjaere.debridav.health + +import io.skjaere.debridav.config.ConfigProperty +import org.springframework.boot.context.properties.ConfigurationProperties +import java.time.Duration + +private const val TORRENT_INTERVAL_DAYS = 1L +private const val NZB_INTERVAL_DAYS = 7L +private const val DEFAULT_POLL_RATE_MINUTES = 5L + +@ConfigurationProperties(prefix = "health-check") +class HealthCheckConfigurationProperties { + @ConfigProperty(name = "Repair Enabled", description = "Enable automatic repair of unhealthy torrents and NZBs") + var repairEnabled: Boolean = true + + @ConfigProperty(name = "Torrent Interval", description = "How often to reverify torrent availability") + var torrentInterval: Duration = Duration.ofDays(TORRENT_INTERVAL_DAYS) + + @ConfigProperty(name = "Torrent Poll Rate", description = "How often to poll for torrents needing health checks") + var torrentPollRate: Duration = Duration.ofMinutes(DEFAULT_POLL_RATE_MINUTES) + + @ConfigProperty(name = "NZB Interval", description = "How often to reverify NZB segments") + var nzbInterval: Duration = Duration.ofDays(NZB_INTERVAL_DAYS) + + @ConfigProperty(name = "NZB Poll Rate", description = "How often to poll for NZBs needing health checks") + var nzbPollRate: Duration = Duration.ofMinutes(DEFAULT_POLL_RATE_MINUTES) +} diff --git a/src/main/kotlin/io/skjaere/debridav/health/HealthMetrics.kt b/src/main/kotlin/io/skjaere/debridav/health/HealthMetrics.kt new file mode 100644 index 00000000..44f6eb4f --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/health/HealthMetrics.kt @@ -0,0 +1,56 @@ +package io.skjaere.debridav.health + +import io.micrometer.core.instrument.MeterRegistry +import io.micrometer.core.instrument.Timer +import org.springframework.stereotype.Component + +private const val CHECKS_METRIC = "debridav.health.checks" +private const val REPAIRS_METRIC = "debridav.health.repairs" +private const val CHECK_DURATION_METRIC = "debridav.health.check.duration" +private const val REPAIR_DURATION_METRIC = "debridav.health.repair.duration" + +@Component +class HealthMetrics(private val meterRegistry: MeterRegistry) { + + fun recordCheck(type: HealthType, result: CheckResult) { + meterRegistry.counter(CHECKS_METRIC, "type", type.tag, "result", result.tag).increment() + } + + fun recordRepair(type: HealthType, action: String) { + meterRegistry.counter(REPAIRS_METRIC, "type", type.tag, "action", action).increment() + } + + fun timeCheck(type: HealthType, block: () -> T): T { + val sample = Timer.start(meterRegistry) + try { + return block() + } finally { + sample.stop( + Timer.builder(CHECK_DURATION_METRIC).tag("type", type.tag).register(meterRegistry) + ) + } + } + + fun timeRepair(type: HealthType, block: () -> T): T { + val sample = Timer.start(meterRegistry) + try { + return block() + } finally { + sample.stop( + Timer.builder(REPAIR_DURATION_METRIC).tag("type", type.tag).register(meterRegistry) + ) + } + } + + enum class HealthType(val tag: String) { + NZB("nzb"), + TORRENT("torrent"), + } + + enum class CheckResult(val tag: String) { + OK("ok"), + MISSING("missing"), + FAILURE("failure"), + NOT_FOUND("not_found"), + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/health/HealthQueueController.kt b/src/main/kotlin/io/skjaere/debridav/health/HealthQueueController.kt new file mode 100644 index 00000000..18627ce5 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/health/HealthQueueController.kt @@ -0,0 +1,36 @@ +package io.skjaere.debridav.health + +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RequestParam +import org.springframework.web.bind.annotation.RestController + +@RestController +@RequestMapping("/api/v1/health-queue") +class HealthQueueController(private val healthQueueService: HealthQueueService) { + + @GetMapping("/check") + fun getHealthCheckStatus(): ResponseEntity = + ResponseEntity.ok(healthQueueService.getHealthCheckStatus()) + + @GetMapping("/check/history") + fun getHealthCheckHistory( + @RequestParam(defaultValue = "0") page: Int, + @RequestParam(defaultValue = "20") size: Int, + @RequestParam(defaultValue = "") search: String + ): ResponseEntity = + ResponseEntity.ok(healthQueueService.getHealthCheckHistory(page, size, search)) + + @GetMapping("/repair") + fun getRepairStatus(): ResponseEntity = + ResponseEntity.ok(healthQueueService.getRepairStatus()) + + @GetMapping("/repair/history") + fun getRepairHistory( + @RequestParam(defaultValue = "0") page: Int, + @RequestParam(defaultValue = "20") size: Int, + @RequestParam(defaultValue = "") search: String + ): ResponseEntity = + ResponseEntity.ok(healthQueueService.getRepairHistory(page, size, search)) +} diff --git a/src/main/kotlin/io/skjaere/debridav/health/HealthQueueItemDto.kt b/src/main/kotlin/io/skjaere/debridav/health/HealthQueueItemDto.kt new file mode 100644 index 00000000..935fc5ee --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/health/HealthQueueItemDto.kt @@ -0,0 +1,31 @@ +package io.skjaere.debridav.health + +import java.time.Instant + +data class HealthQueueItemDto( + val msgId: Long, + val documentId: Long, + val name: String?, + val category: String?, + val type: String, + val readCount: Int, + val enqueuedAt: Instant?, + val lastReadAt: Instant?, + val archivedAt: Instant?, + val message: String?, + val action: String? = null +) + +data class HealthQueueStatusResponse( + val pending: List, + val count: Int +) + +data class HealthQueueHistoryResponse( + val content: List, + val page: Int, + val size: Int, + val totalElements: Long, + val totalPages: Int, + val last: Boolean +) diff --git a/src/main/kotlin/io/skjaere/debridav/health/HealthQueueService.kt b/src/main/kotlin/io/skjaere/debridav/health/HealthQueueService.kt new file mode 100644 index 00000000..6f2585bc --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/health/HealthQueueService.kt @@ -0,0 +1,23 @@ +package io.skjaere.debridav.health + +import org.springframework.stereotype.Service + +@Service +class HealthQueueService(private val repository: PgmqHealthQueueRepository) { + + fun getHealthCheckStatus(): HealthQueueStatusResponse { + val pending = repository.getPendingHealthChecks() + return HealthQueueStatusResponse(pending = pending, count = pending.size) + } + + fun getRepairStatus(): HealthQueueStatusResponse { + val pending = repository.getPendingRepairs() + return HealthQueueStatusResponse(pending = pending, count = pending.size) + } + + fun getHealthCheckHistory(page: Int, size: Int, search: String): HealthQueueHistoryResponse = + repository.getHealthCheckHistory(page, size, search) + + fun getRepairHistory(page: Int, size: Int, search: String): HealthQueueHistoryResponse = + repository.getRepairHistory(page, size, search) +} diff --git a/src/main/kotlin/io/skjaere/debridav/health/PgmqHealthQueueRepository.kt b/src/main/kotlin/io/skjaere/debridav/health/PgmqHealthQueueRepository.kt new file mode 100644 index 00000000..96b2b764 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/health/PgmqHealthQueueRepository.kt @@ -0,0 +1,216 @@ +package io.skjaere.debridav.health + +import org.springframework.jdbc.core.JdbcTemplate +import org.springframework.stereotype.Repository +import java.sql.ResultSet + +@Repository +class PgmqHealthQueueRepository(private val jdbc: JdbcTemplate) { + + fun getPendingHealthChecks(): List { + val nzbItems = getPendingNzb("nzb_health_check", false) + val torrentItems = getPendingTorrent("torrent_health_check", false) + return nzbItems + torrentItems + } + + fun getPendingRepairs(): List { + val nzbItems = getPendingNzb("nzb_health_repair", true) + val torrentItems = getPendingTorrent("torrent_health_repair", true) + return nzbItems + torrentItems + } + + fun getHealthCheckHistory(page: Int, size: Int, search: String): HealthQueueHistoryResponse { + val nzbHistory = getArchivedNzb("nzb_health_check", false, search) + val torrentHistory = getArchivedTorrent("torrent_health_check", false, search) + return paginateCombined(nzbHistory + torrentHistory, page, size) + } + + fun getRepairHistory(page: Int, size: Int, search: String): HealthQueueHistoryResponse { + val nzbHistory = getArchivedNzb("nzb_health_repair", true, search) + val torrentHistory = getArchivedTorrent("torrent_health_repair", true, search) + return paginateCombined(nzbHistory + torrentHistory, page, size) + } + + private fun getPendingNzb(queueName: String, hasMessage: Boolean): List { + if (!queueExists(queueName)) return emptyList() + val sql = """ + SELECT q.msg_id, + (q.message->>'nzbDocumentId')::bigint AS document_id, + ${if (hasMessage) "q.message->>'message' AS repair_message," else ""} + d.name AS doc_name, + d.category, + q.read_ct, + q.enqueued_at, + q.vt AS last_read_at + FROM pgmq.q_$queueName q + LEFT JOIN nzb_document d ON d.id = (q.message->>'nzbDocumentId')::bigint + ORDER BY q.msg_id + """.trimIndent() + + return jdbc.query(sql) { rs, _ -> + mapPendingRow(rs, hasMessage, "NZB") + } + } + + private fun getPendingTorrent(queueName: String, hasMessage: Boolean): List { + if (!queueExists(queueName)) return emptyList() + val sql = """ + SELECT q.msg_id, + (q.message->>'torrentId')::bigint AS document_id, + ${if (hasMessage) "q.message->>'message' AS repair_message," else ""} + t.name AS doc_name, + c.name AS category, + q.read_ct, + q.enqueued_at, + q.vt AS last_read_at + FROM pgmq.q_$queueName q + LEFT JOIN torrent t ON t.id = (q.message->>'torrentId')::bigint + LEFT JOIN category c ON c.id = t.category_id + ORDER BY q.msg_id + """.trimIndent() + + return jdbc.query(sql) { rs, _ -> + mapPendingRow(rs, hasMessage, "TORRENT") + } + } + + private fun getArchivedNzb( + queueName: String, + hasMessage: Boolean, + search: String + ): List { + if (!queueExists(queueName, archived = true)) return emptyList() + val searchClause = if (search.isNotBlank()) "AND LOWER(d.name) LIKE ?" else "" + val outcomeJoin = if (hasMessage) { + "LEFT JOIN repair_outcome ro ON ro.queue_name = '$queueName' AND ro.msg_id = q.msg_id" + } else "" + val outcomeSelect = if (hasMessage) ", ro.action AS repair_action" else "" + val sql = """ + SELECT q.msg_id, + (q.message->>'nzbDocumentId')::bigint AS document_id, + ${if (hasMessage) "q.message->>'message' AS repair_message," else ""} + d.name AS doc_name, + d.category, + q.read_ct, + q.enqueued_at, + q.vt AS last_read_at, + q.archived_at + $outcomeSelect + FROM pgmq.a_$queueName q + LEFT JOIN nzb_document d ON d.id = (q.message->>'nzbDocumentId')::bigint + $outcomeJoin + WHERE 1=1 $searchClause + ORDER BY q.archived_at DESC + """.trimIndent() + + val searchParam = if (search.isNotBlank()) "%${search.lowercase()}%" else null + return if (searchParam != null) { + jdbc.query(sql, { rs, _ -> mapArchivedRow(rs, hasMessage, "NZB") }, searchParam) + } else { + jdbc.query(sql) { rs, _ -> mapArchivedRow(rs, hasMessage, "NZB") } + } + } + + private fun getArchivedTorrent( + queueName: String, + hasMessage: Boolean, + search: String + ): List { + if (!queueExists(queueName, archived = true)) return emptyList() + val searchClause = if (search.isNotBlank()) "AND LOWER(t.name) LIKE ?" else "" + val outcomeJoin = if (hasMessage) { + "LEFT JOIN repair_outcome ro ON ro.queue_name = '$queueName' AND ro.msg_id = q.msg_id" + } else "" + val outcomeSelect = if (hasMessage) ", ro.action AS repair_action" else "" + val sql = """ + SELECT q.msg_id, + (q.message->>'torrentId')::bigint AS document_id, + ${if (hasMessage) "q.message->>'message' AS repair_message," else ""} + t.name AS doc_name, + c.name AS category, + q.read_ct, + q.enqueued_at, + q.vt AS last_read_at, + q.archived_at + $outcomeSelect + FROM pgmq.a_$queueName q + LEFT JOIN torrent t ON t.id = (q.message->>'torrentId')::bigint + LEFT JOIN category c ON c.id = t.category_id + $outcomeJoin + WHERE 1=1 $searchClause + ORDER BY q.archived_at DESC + """.trimIndent() + + val searchParam = if (search.isNotBlank()) "%${search.lowercase()}%" else null + return if (searchParam != null) { + jdbc.query(sql, { rs, _ -> mapArchivedRow(rs, hasMessage, "TORRENT") }, searchParam) + } else { + jdbc.query(sql) { rs, _ -> mapArchivedRow(rs, hasMessage, "TORRENT") } + } + } + + private fun paginateCombined( + all: List, + page: Int, + size: Int + ): HealthQueueHistoryResponse { + val sorted = all.sortedByDescending { it.archivedAt } + val totalElements = sorted.size.toLong() + val totalPages = if (size > 0) ((totalElements + size - 1) / size).toInt() else 0 + val offset = page * size + val items = sorted.drop(offset).take(size) + + return HealthQueueHistoryResponse( + content = items, + page = page, + size = size, + totalElements = totalElements, + totalPages = totalPages, + last = page >= totalPages - 1 + ) + } + + private fun queueExists(queueName: String, archived: Boolean = false): Boolean { + val prefix = if (archived) "a_" else "q_" + return try { + jdbc.queryForObject( + "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = 'pgmq' AND table_name = ?)", + Boolean::class.java, + "$prefix$queueName" + ) ?: false + } catch (@Suppress("TooGenericExceptionCaught") _: Exception) { + false + } + } + + private fun mapPendingRow(rs: ResultSet, hasMessage: Boolean, type: String): HealthQueueItemDto { + return HealthQueueItemDto( + msgId = rs.getLong("msg_id"), + documentId = rs.getLong("document_id"), + name = rs.getString("doc_name"), + category = rs.getString("category"), + type = type, + readCount = rs.getInt("read_ct"), + enqueuedAt = rs.getTimestamp("enqueued_at")?.toInstant(), + lastReadAt = rs.getTimestamp("last_read_at")?.toInstant(), + archivedAt = null, + message = if (hasMessage) rs.getString("repair_message") else null + ) + } + + private fun mapArchivedRow(rs: ResultSet, hasMessage: Boolean, type: String): HealthQueueItemDto { + return HealthQueueItemDto( + msgId = rs.getLong("msg_id"), + documentId = rs.getLong("document_id"), + name = rs.getString("doc_name"), + category = rs.getString("category"), + type = type, + readCount = rs.getInt("read_ct"), + enqueuedAt = rs.getTimestamp("enqueued_at")?.toInstant(), + lastReadAt = rs.getTimestamp("last_read_at")?.toInstant(), + archivedAt = rs.getTimestamp("archived_at")?.toInstant(), + message = if (hasMessage) rs.getString("repair_message") else null, + action = if (hasMessage) rs.getString("repair_action") else null + ) + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/health/RepairOutcome.kt b/src/main/kotlin/io/skjaere/debridav/health/RepairOutcome.kt new file mode 100644 index 00000000..9fc982a8 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/health/RepairOutcome.kt @@ -0,0 +1,43 @@ +package io.skjaere.debridav.health + +import jakarta.persistence.Column +import jakarta.persistence.Entity +import jakarta.persistence.EnumType +import jakarta.persistence.Enumerated +import jakarta.persistence.GeneratedValue +import jakarta.persistence.GenerationType +import jakarta.persistence.Id +import jakarta.persistence.Table +import org.springframework.data.repository.CrudRepository +import org.springframework.stereotype.Repository +import java.time.Instant + +enum class RepairAction { + REPAIRED, + DELETED, + SKIPPED +} + +@Entity +@Table(name = "repair_outcome") +open class RepairOutcome { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + open var id: Long? = null + + @Column(name = "queue_name", nullable = false) + open var queueName: String? = null + + @Column(name = "msg_id", nullable = false) + open var msgId: Long? = null + + @Enumerated(EnumType.STRING) + @Column(nullable = false) + open var action: RepairAction? = null + + @Column(name = "created_at", nullable = false) + open var createdAt: Instant = Instant.now() +} + +@Repository +interface RepairOutcomeRepository : CrudRepository diff --git a/src/main/kotlin/io/skjaere/debridav/health/RepairOutcomeService.kt b/src/main/kotlin/io/skjaere/debridav/health/RepairOutcomeService.kt new file mode 100644 index 00000000..52b90816 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/health/RepairOutcomeService.kt @@ -0,0 +1,19 @@ +package io.skjaere.debridav.health + +import org.springframework.stereotype.Service +import org.springframework.transaction.annotation.Propagation +import org.springframework.transaction.annotation.Transactional + +@Service +class RepairOutcomeService( + private val repository: RepairOutcomeRepository +) { + @Transactional(propagation = Propagation.REQUIRES_NEW) + fun record(queueName: String, msgId: Long, action: RepairAction) { + val outcome = RepairOutcome() + outcome.queueName = queueName + outcome.msgId = msgId + outcome.action = action + repository.save(outcome) + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/pgmq/PgmqConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/pgmq/PgmqConfigurationProperties.kt new file mode 100644 index 00000000..d11bfa1e --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/pgmq/PgmqConfigurationProperties.kt @@ -0,0 +1,28 @@ +package io.skjaere.debridav.pgmq + +import org.springframework.boot.context.properties.ConfigurationProperties +import java.time.Duration + +@Suppress("MagicNumber") +@ConfigurationProperties(prefix = "pgmq") +class PgmqConfigurationProperties { + var defaultVisibilityTimeout: Duration = Duration.ofMinutes(5) + var importConcurrency: Int = 2 + var importVisibilityTimeout: Duration = Duration.ofMinutes(10) + var importPollInterval: Duration = Duration.ofSeconds(2) + var healthCheckConcurrency: Int = 1 + var healthCheckVisibilityTimeout: Duration = Duration.ofMinutes(5) + var healthCheckPollInterval: Duration = Duration.ofSeconds(10) + var healthRepairConcurrency: Int = 2 + var healthRepairVisibilityTimeout: Duration = Duration.ofMinutes(2) + var healthRepairPollInterval: Duration = Duration.ofSeconds(5) + var archiveRetention: Duration = Duration.ofDays(30) + var deadLetterRetention: Duration = Duration.ofDays(30) + var maxReadCount: Long = 5 + var torrentHealthCheckConcurrency: Int = 1 + var torrentHealthCheckVisibilityTimeout: Duration = Duration.ofMinutes(5) + var torrentHealthCheckPollInterval: Duration = Duration.ofSeconds(10) + var torrentHealthRepairConcurrency: Int = 2 + var torrentHealthRepairVisibilityTimeout: Duration = Duration.ofMinutes(2) + var torrentHealthRepairPollInterval: Duration = Duration.ofSeconds(5) +} diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqConsumer.kt b/src/main/kotlin/io/skjaere/debridav/pgmq/PgmqConsumer.kt similarity index 58% rename from src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqConsumer.kt rename to src/main/kotlin/io/skjaere/debridav/pgmq/PgmqConsumer.kt index fe754506..1c3c034e 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqConsumer.kt +++ b/src/main/kotlin/io/skjaere/debridav/pgmq/PgmqConsumer.kt @@ -1,16 +1,19 @@ -package io.skjaere.debridav.usenet.pgmq +package io.skjaere.debridav.pgmq +import com.fasterxml.jackson.core.JsonProcessingException import com.fasterxml.jackson.databind.ObjectMapper import com.vdsirotkin.pgmq.PgmqClient +import com.vdsirotkin.pgmq.domain.PgmqEntry import kotlin.time.Duration import kotlin.time.toKotlinDuration import org.slf4j.LoggerFactory import org.springframework.context.SmartLifecycle +import java.time.Instant import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.TimeUnit -@Suppress("MagicNumber") +@Suppress("MagicNumber", "LongParameterList") class PgmqConsumer( private val pgmqClient: PgmqClient, private val objectMapper: ObjectMapper, @@ -19,8 +22,10 @@ class PgmqConsumer( private val concurrency: Int, private val visibilityTimeout: java.time.Duration, private val pollInterval: java.time.Duration, - private val handler: (T) -> Unit + private val maxReadCount: Long = DEFAULT_MAX_READ_COUNT, + private val handler: (T, Long) -> Unit ) : SmartLifecycle { + private val deadLetterQueue = "${queueName}_dlq" private val logger = LoggerFactory.getLogger(PgmqConsumer::class.java) @@ -56,13 +61,28 @@ class PgmqConsumer( val entry = entries.first() try { val message = objectMapper.readValue(entry.message, messageType) - handler(message) + handler(message, entry.messageId) pgmqClient.archive(queueName, entry.messageId) - } catch (@Suppress("TooGenericExceptionCaught") e: Exception) { + } catch (e: JsonProcessingException) { logger.error( - "Error processing message {} from queue '{}': {}", - entry.messageId, queueName, e.message, e + "Malformed message {} on queue '{}'; dead-lettering (retry can't help)", + entry.messageId, queueName, e ) + deadLetter(entry, e) + } catch (@Suppress("TooGenericExceptionCaught") e: Exception) { + val attempt = entry.readCounter + if (attempt >= maxReadCount) { + logger.error( + "Message {} on queue '{}' exhausted {} attempts; dead-lettering", + entry.messageId, queueName, attempt, e + ) + deadLetter(entry, e) + } else { + logger.error( + "Message {} on queue '{}' failed attempt {}/{}; will retry after visibility timeout", + entry.messageId, queueName, attempt, maxReadCount, e + ) + } } } catch (_: InterruptedException) { Thread.currentThread().interrupt() @@ -95,4 +115,30 @@ class PgmqConsumer( override fun isRunning(): Boolean = running override fun getPhase(): Int = Int.MAX_VALUE - 1 + + private fun deadLetter(entry: PgmqEntry, cause: Throwable) { + val envelope = mapOf( + "originalQueue" to queueName, + "originalMessageId" to entry.messageId, + "originalPayload" to entry.message, + "enqueuedAt" to entry.enqueuedAt.toString(), + "readCount" to entry.readCounter, + "failureClass" to cause.javaClass.name, + "failureMessage" to cause.message, + "failedAt" to Instant.now().toString() + ) + try { + pgmqClient.send(deadLetterQueue, envelope) + pgmqClient.archive(queueName, entry.messageId) + } catch (@Suppress("TooGenericExceptionCaught") e: Exception) { + logger.error( + "Failed to dead-letter message {} to queue '{}'; leaving visible for manual inspection", + entry.messageId, deadLetterQueue, e + ) + } + } + + companion object { + const val DEFAULT_MAX_READ_COUNT: Long = 5 + } } diff --git a/src/main/kotlin/io/skjaere/debridav/pgmq/PgmqInfrastructureConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/pgmq/PgmqInfrastructureConfiguration.kt new file mode 100644 index 00000000..90e9d2ad --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/pgmq/PgmqInfrastructureConfiguration.kt @@ -0,0 +1,41 @@ +package io.skjaere.debridav.pgmq + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.KotlinModule +import com.vdsirotkin.pgmq.PgmqClient +import com.vdsirotkin.pgmq.config.PgmqConfiguration +import com.vdsirotkin.pgmq.config.PgmqConnectionFactory +import com.vdsirotkin.pgmq.serialization.JacksonPgmqSerializationProvider +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import javax.sql.DataSource + +@Configuration +class PgmqInfrastructureConfiguration { + + @Bean + fun pgmqConfiguration(props: PgmqConfigurationProperties): PgmqConfiguration = + object : PgmqConfiguration { + override val defaultVisibilityTimeout: java.time.Duration = props.defaultVisibilityTimeout + } + + @Bean + fun pgmqConnectionFactory(dataSource: DataSource): PgmqConnectionFactory = PgmqConnectionFactory { + dataSource.connection + } + + @Bean + fun pgmqObjectMapper(): ObjectMapper = + ObjectMapper().registerModule(KotlinModule.Builder().build()) + + @Bean + fun pgmqSerializationProvider(pgmqObjectMapper: ObjectMapper): JacksonPgmqSerializationProvider = + JacksonPgmqSerializationProvider(pgmqObjectMapper) + + @Bean + fun pgmqClient( + connectionFactory: PgmqConnectionFactory, + serializationProvider: JacksonPgmqSerializationProvider, + configuration: PgmqConfiguration + ): PgmqClient = PgmqClient(connectionFactory, serializationProvider, configuration) +} diff --git a/src/main/kotlin/io/skjaere/debridav/pgmq/PgmqMetricsService.kt b/src/main/kotlin/io/skjaere/debridav/pgmq/PgmqMetricsService.kt new file mode 100644 index 00000000..b2348d51 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/pgmq/PgmqMetricsService.kt @@ -0,0 +1,86 @@ +package io.skjaere.debridav.pgmq + +import io.micrometer.core.instrument.MeterRegistry +import io.micrometer.core.instrument.MultiGauge +import io.micrometer.core.instrument.Tags +import org.slf4j.LoggerFactory +import org.springframework.jdbc.core.JdbcTemplate +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Service + +/** + * Polls PGMQ's `pgmq.metrics_all()` view once per [REFRESH_INTERVAL] and publishes + * queue-depth + message-age gauges per queue. Used by the health dashboard to + * alert on backlog and DLQ fill-up. + */ +@Service +class PgmqMetricsService( + private val jdbc: JdbcTemplate, + meterRegistry: MeterRegistry, +) { + private val logger = LoggerFactory.getLogger(PgmqMetricsService::class.java) + + private val queueLengthGauge = MultiGauge + .builder("debridav.pgmq.queue.length") + .description("Number of visible messages in the PGMQ queue") + .register(meterRegistry) + + private val oldestMsgAgeGauge = MultiGauge + .builder("debridav.pgmq.oldest.message.age.seconds") + .description("Age of the oldest visible message in the queue") + .register(meterRegistry) + + private val totalMessagesGauge = MultiGauge + .builder("debridav.pgmq.messages.total") + .description("Total messages (visible + invisible) on the queue") + .register(meterRegistry) + + @Scheduled(fixedDelayString = REFRESH_INTERVAL, initialDelayString = "PT15S") + fun refresh() { + val rows = try { + jdbc.query( + """ + SELECT queue_name, + queue_length, + COALESCE(oldest_msg_age_sec, 0) AS oldest_msg_age_sec, + total_messages + FROM pgmq.metrics_all() + """.trimIndent() + ) { rs, _ -> + QueueMetrics( + queueName = rs.getString("queue_name"), + queueLength = rs.getLong("queue_length"), + oldestMsgAgeSec = rs.getLong("oldest_msg_age_sec"), + totalMessages = rs.getLong("total_messages"), + ) + } + } catch (@Suppress("TooGenericExceptionCaught") e: Exception) { + logger.warn("Failed to fetch pgmq metrics: {}", e.message) + return + } + + queueLengthGauge.register( + rows.map { MultiGauge.Row.of(Tags.of("queue", it.queueName), it.queueLength.toDouble()) }, + true + ) + oldestMsgAgeGauge.register( + rows.map { MultiGauge.Row.of(Tags.of("queue", it.queueName), it.oldestMsgAgeSec.toDouble()) }, + true + ) + totalMessagesGauge.register( + rows.map { MultiGauge.Row.of(Tags.of("queue", it.queueName), it.totalMessages.toDouble()) }, + true + ) + } + + private data class QueueMetrics( + val queueName: String, + val queueLength: Long, + val oldestMsgAgeSec: Long, + val totalMessages: Long, + ) + + companion object { + private const val REFRESH_INTERVAL = "PT30S" + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/rclone/FileSystemChangedEvent.kt b/src/main/kotlin/io/skjaere/debridav/rclone/FileSystemChangedEvent.kt new file mode 100644 index 00000000..e047545e --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/rclone/FileSystemChangedEvent.kt @@ -0,0 +1,11 @@ +package io.skjaere.debridav.rclone + +/** + * Emitted when files change in the virtual filesystem so external caches + * (currently rclone's VFS via the RC API) can be refreshed. + * + * `paths` holds the directories to refresh — typically the parent of the + * affected file. A move includes both source and destination parents so + * a single event represents the whole operation. + */ +data class FileSystemChangedEvent(val paths: Set) diff --git a/src/main/kotlin/io/skjaere/debridav/rclone/RcloneCacheInvalidator.kt b/src/main/kotlin/io/skjaere/debridav/rclone/RcloneCacheInvalidator.kt new file mode 100644 index 00000000..514fb9c4 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/rclone/RcloneCacheInvalidator.kt @@ -0,0 +1,155 @@ +package io.skjaere.debridav.rclone + +import io.ktor.client.HttpClient +import io.ktor.client.request.header +import io.ktor.client.request.post +import io.ktor.client.request.setBody +import io.ktor.client.statement.bodyAsText +import io.ktor.http.ContentType +import io.ktor.http.HttpHeaders +import io.ktor.http.contentType +import io.ktor.http.isSuccess +import io.skjaere.debridav.configuration.DebridavConfigurationProperties +import jakarta.annotation.PreDestroy +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.launch +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import org.slf4j.LoggerFactory +import org.springframework.context.event.EventListener +import org.springframework.stereotype.Component +import java.util.Base64 +import java.util.concurrent.Executors +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.TimeUnit +import java.util.concurrent.locks.ReentrantLock +import kotlin.concurrent.withLock + +/** + * Listens for [FileSystemChangedEvent]s and POSTs to rclone's `/vfs/refresh` + * RC endpoint so rclone sees new/moved/deleted files without waiting for its + * directory cache to expire. + * + * Events are coalesced within a 500 ms window: the first event in a quiet + * period schedules a flush, subsequent events append to the pending set + * without rescheduling, and at the window's end every unique path gets one + * refresh call. Max latency from event to refresh is one window plus HTTP + * round-trip. + * + * Entirely opt-in. Disabled by default — enable from the UI's Core config + * page (`debridav.rclone-cache-invalidation-enabled`) and set the three + * `DEBRIDAV_RCLONE_RC-*` env vars. If either the toggle is off or the URL + * is blank, events are dropped with no network I/O. + */ +@Component +class RcloneCacheInvalidator( + private val debridavConfig: DebridavConfigurationProperties, + private val rcloneConfig: RcloneConfigurationProperties, + private val httpClient: HttpClient, +) { + private val logger = LoggerFactory.getLogger(javaClass) + + private val lock = ReentrantLock() + private val pending = mutableSetOf() + private var flushScheduled = false + + private val scheduler: ScheduledExecutorService = + Executors.newSingleThreadScheduledExecutor { r -> + Thread(r, "rclone-cache-invalidator").apply { isDaemon = true } + } + private val ioScope = CoroutineScope(SupervisorJob()) + + @EventListener + fun onChange(event: FileSystemChangedEvent) { + if (!debridavConfig.rcloneCacheInvalidationEnabled + || rcloneConfig.rcUrl.isBlank() + || event.paths.isEmpty() + ) return + + // Include every ancestor of each changed path so rclone refreshes + // the parent listings too — otherwise a new directory (e.g. the + // /downloads// created by an NZB import) can't be refreshed + // directly because rclone hasn't seen it yet. + val withAncestors = event.paths.flatMap(::ancestors).toSet() + lock.withLock { + pending.addAll(withAncestors) + if (!flushScheduled) { + flushScheduled = true + scheduler.schedule(::flush, COALESCE_WINDOW_MS, TimeUnit.MILLISECONDS) + } + } + } + + internal fun flush() { + val paths = lock.withLock { + val snap = pending.toSet() + pending.clear() + flushScheduled = false + snap + } + if (paths.isEmpty()) return + // Refresh shallowest-first: rclone has to learn about a directory + // before we can refresh into it. + val ordered = paths.sortedBy { if (it == "/") 0 else it.count { ch -> ch == '/' } } + ioScope.launch { + ordered.forEach { path -> + runCatching { refresh(path) } + .onFailure { logger.warn("rclone refresh failed for '{}': {}", path, it.message) } + } + } + } + + internal fun ancestors(path: String): Set { + if (path == "/" || path.isEmpty()) return setOf("/") + val result = linkedSetOf("/") + val parts = path.trim('/').split('/').filter { it.isNotEmpty() } + val builder = StringBuilder() + for (part in parts) { + builder.append('/').append(part) + result.add(builder.toString()) + } + return result + } + + private suspend fun refresh(dir: String) { + val url = rcloneConfig.rcUrl.trimEnd('/') + "/vfs/refresh" + // rclone's /vfs/refresh rejects dir="/" with "file does not exist" — + // the root is represented by omitting the dir param entirely, which + // refreshes the whole VFS. + val body = if (dir == "/") "{}" else Json.encodeToString( + RefreshRequest.serializer(), + RefreshRequest(dir = dir), + ) + val response = httpClient.post(url) { + contentType(ContentType.Application.Json) + if (rcloneConfig.rcUser.isNotBlank()) { + val creds = "${rcloneConfig.rcUser}:${rcloneConfig.rcPassword}" + val encoded = Base64.getEncoder().encodeToString(creds.toByteArray()) + header(HttpHeaders.Authorization, "Basic $encoded") + } + setBody(body) + } + if (!response.status.isSuccess()) { + logger.warn( + "rclone refresh for '{}' returned {}: {}", + dir, response.status, response.bodyAsText() + ) + } + } + + @PreDestroy + fun shutdown() { + scheduler.shutdown() + runCatching { scheduler.awaitTermination(1, TimeUnit.SECONDS) } + ioScope.cancel() + } + + @Serializable + private data class RefreshRequest(val dir: String, val recursive: String = "false") + + companion object { + const val COALESCE_WINDOW_MS = 500L + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/rclone/RcloneConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/rclone/RcloneConfigurationProperties.kt new file mode 100644 index 00000000..a85d4416 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/rclone/RcloneConfigurationProperties.kt @@ -0,0 +1,18 @@ +package io.skjaere.debridav.rclone + +import org.springframework.boot.context.properties.ConfigurationProperties + +/** + * Rclone remote-control endpoint used by [RcloneCacheInvalidator]. + * + * Env-only on purpose — these are infrastructure details that get set once + * per deployment, not user preferences. The user-facing toggle lives on + * `DebridavConfigurationProperties.rcloneCacheInvalidationEnabled`. + */ +@ConfigurationProperties(prefix = "debridav.rclone") +class RcloneConfigurationProperties { + /** e.g. `http://rclone:5572`. Blank disables the integration. */ + var rcUrl: String = "" + var rcUser: String = "" + var rcPassword: String = "" +} diff --git a/src/main/kotlin/io/skjaere/debridav/repository/DebridFileContentsRepository.kt b/src/main/kotlin/io/skjaere/debridav/repository/DebridFileContentsRepository.kt index 2a6697ce..cd433f6d 100644 --- a/src/main/kotlin/io/skjaere/debridav/repository/DebridFileContentsRepository.kt +++ b/src/main/kotlin/io/skjaere/debridav/repository/DebridFileContentsRepository.kt @@ -2,6 +2,7 @@ package io.skjaere.debridav.repository import io.skjaere.debridav.fs.DbDirectory import io.skjaere.debridav.fs.DbEntity +import io.skjaere.debridav.fs.RemotelyCachedEntity import jakarta.transaction.Transactional import org.springframework.data.jpa.repository.Modifying import org.springframework.data.jpa.repository.Query @@ -11,6 +12,8 @@ import org.springframework.data.repository.CrudRepository interface DebridFileContentsRepository : CrudRepository { fun findByDirectoryAndName(directory: DbDirectory, name: String): DbEntity? + fun findAllByDirectoryInAndNameIn(directories: Collection, names: Collection): List + @Query( "select * from db_item entity where entity.db_item_type='DbDirectory' AND entity.path = CAST(:path AS ltree)", nativeQuery = true @@ -71,19 +74,37 @@ interface DebridFileContentsRepository : CrudRepository { @Query( """ - select jsonb_path_query(debrid_links, '$[*].provider') as provider, - jsonb_path_query(debrid_links, '$[*].\@type') as type, + select jsonb_path_query(debrid_links, '$[*].provider') as provider, + jsonb_path_query(debrid_links, '$[*].\@type') as type, count(*) as count from debrid_cached_torrent_content group by provider, type; """, nativeQuery = true ) fun getLibraryMetricsTorrents(): List> + @Query( + """ + select jsonb_path_query(debrid_links, '$[*].provider') as provider, + jsonb_path_query(debrid_links, '$[*].\@type') as type, + count(*) as count + from debrid_cached_usenet_release_content group by provider, type; + """, nativeQuery = true + ) + fun getLibraryMetricsUsenet(): List> + @Query("select count(*) from DebridCachedTorrentContent ") fun numberOfRemotelyCachedTorrentEntities(): Long @Query("select count(*) from DebridCachedUsenetReleaseContent ") fun numberOfRemotelyCachedUsenetEntities(): Long + + @Query( + "select rce.* from db_item rce " + + "join usenet_download_debrid_files udf on udf.debrid_files_id = rce.id " + + "where udf.usenet_download_id = :usenetDownloadId", + nativeQuery = true + ) + fun findByUsenetDownloadId(usenetDownloadId: Long): List } data class LibraryStats(val provider: String, val type: String, val count: Long) diff --git a/src/main/kotlin/io/skjaere/debridav/repository/NzbImportRepository.kt b/src/main/kotlin/io/skjaere/debridav/repository/NzbImportRepository.kt new file mode 100644 index 00000000..02a44d1d --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/repository/NzbImportRepository.kt @@ -0,0 +1,25 @@ +package io.skjaere.debridav.repository + +import io.skjaere.debridav.usenet.queue.NzbImportRecord +import io.skjaere.debridav.usenet.queue.NzbImportStatus +import jakarta.transaction.Transactional +import org.springframework.data.domain.Page +import org.springframework.data.domain.Pageable +import org.springframework.data.jpa.repository.JpaRepository +import org.springframework.data.jpa.repository.Query + +@Transactional +interface NzbImportRepository : JpaRepository { + fun findByStatusInOrderByUpdatedAtDesc(statuses: Collection): List + fun findByStatusInOrderByIdAsc(statuses: Collection): List + + @Query( + "SELECT r FROM NzbImportRecord r WHERE r.status IN :statuses " + + "AND LOWER(r.name) LIKE LOWER(CONCAT('%', :search, '%'))" + ) + fun findByStatusInAndNameSearch( + statuses: Collection, + search: String, + pageable: Pageable + ): Page +} diff --git a/src/main/kotlin/io/skjaere/debridav/repository/UsenetRepository.kt b/src/main/kotlin/io/skjaere/debridav/repository/UsenetRepository.kt index 8687e58a..16b3982f 100644 --- a/src/main/kotlin/io/skjaere/debridav/repository/UsenetRepository.kt +++ b/src/main/kotlin/io/skjaere/debridav/repository/UsenetRepository.kt @@ -1,16 +1,18 @@ package io.skjaere.debridav.repository import io.skjaere.debridav.usenet.UsenetDownload -import jakarta.transaction.Transactional +import io.skjaere.debridav.usenet.UsenetDownloadStatus +import org.springframework.data.domain.Pageable import org.springframework.data.jpa.repository.Modifying import org.springframework.data.jpa.repository.Query import org.springframework.data.repository.CrudRepository +import org.springframework.transaction.annotation.Transactional -@Transactional interface UsenetRepository : CrudRepository { fun getByName(name: String): UsenetDownload? @Modifying + @Transactional @Query( "update UsenetDownload ud " + "set ud.status=io.skjaere.debridav.usenet.UsenetDownloadStatus.DELETED " + @@ -18,9 +20,18 @@ interface UsenetRepository : CrudRepository { ) fun markUsenetDownloadAsDeleted(usenetDownload: UsenetDownload) + @Transactional fun deleteUsenetDownloadById(id: Long) fun getByHash(hash: String): UsenetDownload? + + @Transactional fun deleteByHashIgnoreCase(hash: String) fun findByCategoryName(categoryName: String): List fun findByNzbDocumentId(nzbDocumentId: Long): UsenetDownload? + + @Query("SELECT u FROM UsenetDownload u ORDER BY u.id DESC") + fun findRecent(pageable: Pageable): List + + @Query("SELECT u FROM UsenetDownload u WHERE u.category.name = :categoryName ORDER BY u.id DESC") + fun findRecentByCategoryName(categoryName: String, pageable: Pageable): List } diff --git a/src/main/kotlin/io/skjaere/debridav/resource/NzbFileResource.kt b/src/main/kotlin/io/skjaere/debridav/resource/NzbFileResource.kt index dbc5ce4e..fddf4f16 100644 --- a/src/main/kotlin/io/skjaere/debridav/resource/NzbFileResource.kt +++ b/src/main/kotlin/io/skjaere/debridav/resource/NzbFileResource.kt @@ -15,14 +15,21 @@ import io.skjaere.nntp.ArticleNotFoundException import io.skjaere.nzbstreamer.NzbStreamer import io.skjaere.nzbstreamer.nzb.NzbDocument import io.skjaere.nzbstreamer.stream.StreamableFile +import kotlin.time.Duration.Companion.milliseconds import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking import org.slf4j.LoggerFactory import java.io.OutputStream import java.time.Instant import java.util.* +import java.util.concurrent.atomic.AtomicBoolean private const val BUFFER_SIZE = 8192 +private const val STALL_TIMEOUT_MS = 5 * 60 * 1000L // 5 minutes class NzbFileResource( val file: RemotelyCachedEntity, @@ -77,14 +84,35 @@ class NzbFileResource( ) try { runBlocking { - nzbStreamer.streamFile(nzbDocument, streamableFile, longRange) { channel -> - val buffer = ByteArray(BUFFER_SIZE) - while (!channel.isClosedForRead) { - val bytesRead = channel.readAvailable(buffer) - if (bytesRead > 0) { - out.write(buffer, 0, bytesRead) + val sawActivity = AtomicBoolean(true) + coroutineScope { + val watchdog = launch { + while (isActive) { + delay(STALL_TIMEOUT_MS.milliseconds) + if (!sawActivity.getAndSet(false)) { + logger.warn( + "NZB stream '{}' stalled (>{}ms without bytes); closing output to unblock write", + streamableFile.path, STALL_TIMEOUT_MS + ) + runCatching { out.close() } + break + } } } + try { + nzbStreamer.streamFile(nzbDocument, streamableFile, longRange) { channel -> + val buffer = ByteArray(BUFFER_SIZE) + while (!channel.isClosedForRead) { + val bytesRead = channel.readAvailable(buffer) + if (bytesRead > 0) { + out.write(buffer, 0, bytesRead) + sawActivity.set(true) + } + } + } + } finally { + watchdog.cancel() + } } } } catch (e: ArticleNotFoundException) { diff --git a/src/main/kotlin/io/skjaere/debridav/resource/StreamableResourceFactory.kt b/src/main/kotlin/io/skjaere/debridav/resource/StreamableResourceFactory.kt index dbd18aa4..1cb78a6e 100644 --- a/src/main/kotlin/io/skjaere/debridav/resource/StreamableResourceFactory.kt +++ b/src/main/kotlin/io/skjaere/debridav/resource/StreamableResourceFactory.kt @@ -34,10 +34,20 @@ class StreamableResourceFactory( @Throws(NotAuthorizedException::class, BadRequestException::class) override fun getResource(host: String?, url: String): Resource? { - val path: Path = Path.path(url) + val path: Path = Path.path(stripWebdavPrefix(url)) return find(path) } + private fun stripWebdavPrefix(url: String): String = when { + url == WEBDAV_PREFIX || url == "$WEBDAV_PREFIX/" -> "/" + url.startsWith("$WEBDAV_PREFIX/") -> url.removePrefix(WEBDAV_PREFIX) + else -> url + } + + companion object { + const val WEBDAV_PREFIX = "/webdav" + } + @Throws(NotAuthorizedException::class, BadRequestException::class) private fun find(path: Path): Resource? { val actualPath = if (path.isRoot) "/" else path.toPath() diff --git a/src/main/kotlin/io/skjaere/debridav/stream/StreamingService.kt b/src/main/kotlin/io/skjaere/debridav/stream/StreamingService.kt index 46c9deb1..7f25c830 100644 --- a/src/main/kotlin/io/skjaere/debridav/stream/StreamingService.kt +++ b/src/main/kotlin/io/skjaere/debridav/stream/StreamingService.kt @@ -4,15 +4,15 @@ import io.ktor.client.call.body import io.ktor.utils.io.ByteReadChannel import io.ktor.utils.io.readAvailable import io.milton.http.Range -import io.prometheus.metrics.core.metrics.Gauge -import io.prometheus.metrics.core.metrics.Histogram -import io.prometheus.metrics.model.registry.PrometheusRegistry +import io.micrometer.core.instrument.MeterRegistry +import io.micrometer.core.instrument.MultiGauge +import io.micrometer.core.instrument.Tags +import io.micrometer.core.instrument.Timer import io.skjaere.debridav.debrid.client.DebridCachedContentClient import io.skjaere.debridav.fs.CachedFile import io.skjaere.debridav.fs.RemotelyCachedEntity import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.cancelChildren import kotlinx.coroutines.channels.Channel @@ -20,10 +20,14 @@ import kotlinx.coroutines.channels.ReceiveChannel import kotlinx.coroutines.channels.consumeEach import kotlinx.coroutines.channels.produce import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.withContext +import kotlinx.coroutines.delay +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch import kotlinx.io.EOFException +import kotlinx.io.IOException import org.apache.catalina.connector.ClientAbortException import org.slf4j.LoggerFactory +import org.springframework.web.context.request.async.AsyncRequestNotUsableException import org.springframework.scheduling.annotation.Scheduled import org.springframework.stereotype.Service import java.io.OutputStream @@ -31,34 +35,28 @@ import java.time.Duration import java.time.Instant import java.util.concurrent.ConcurrentLinkedQueue import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean private const val DEFAULT_BUFFER_SIZE = 256 * 1024 //256kb private const val READ_AHEAD_CHUNKS = 200 // 50Mb private const val STREAMING_METRICS_POLLING_RATE_S = 5L //5 seconds +private const val STALL_TIMEOUT_MS = 5 * 60 * 1000L // 5 minutes @Service class StreamingService( private val debridClients: List, - prometheusRegistry: PrometheusRegistry + private val meterRegistry: MeterRegistry, ) { private val logger = LoggerFactory.getLogger(StreamingService::class.java) - private val outputGauge = - Gauge.builder().name("debridav.output.stream.bitrate").labelNames("file", "client") - .register(prometheusRegistry) - private val inputGauge = Gauge - .builder() - .name("debridav.input.stream.bitrate") - .labelNames("provider", "file", "client") - .register(prometheusRegistry) - private val timeToFirstByteHistogram = - Histogram.builder().help("Time duration between sending request and receiving first byte") - .name("debridav.streaming.time.to.first.byte").labelNames("provider", "client").register(prometheusRegistry) + private val outputGauge = MultiGauge.builder("debridav.output.stream.bitrate") + .register(meterRegistry) + private val inputGauge = MultiGauge.builder("debridav.input.stream.bitrate") + .register(meterRegistry) private val activeOutputStream = ConcurrentLinkedQueue() private val activeInputStreams = ConcurrentLinkedQueue() - @Suppress("TooGenericExceptionCaught") suspend fun streamContents( debridLink: CachedFile, range: Range?, @@ -67,50 +65,11 @@ class StreamingService( client: String, ): StreamResult = coroutineScope { val result = try { - val appliedRange = Range(range?.start ?: 0, range?.finish ?: (debridLink.size!! - 1)) - val inputCounter = ByteCounter() - val outputCounter = ByteCounter() - val inputCtx = InputStreamingContext(inputCounter, debridLink.provider!!, debridLink.path!!, client) - val outputCtx = OutputStreamingContext(outputCounter, remotelyCachedEntity.name!!, client) - activeInputStreams.add(inputCtx) - activeOutputStream.add(outputCtx) - val started = Instant.now() - var ttfbRecorded = false - try { - sendBytesFromHttpStream(debridLink, appliedRange, outputStream) { bytes -> - if (!ttfbRecorded) { - ttfbRecorded = true - timeToFirstByteHistogram.labelValues(debridLink.provider.toString(), client) - .observe(Duration.between(started, Instant.now()).toMillis().toDouble()) - } - inputCounter.add(bytes.toLong()) - outputCounter.add(bytes.toLong()) - } - } finally { - activeOutputStream.removeStream(outputCtx) - activeInputStreams.removeStream(inputCtx) - } - StreamResult.OK - } catch (_: LinkNotFoundException) { - StreamResult.DEAD_LINK - } catch (_: DebridProviderException) { - StreamResult.PROVIDER_ERROR - } catch (_: StreamToClientException) { - StreamResult.IO_ERROR - } catch (_: ReadFromHttpStreamException) { - StreamResult.IO_ERROR - } catch (_: ClientErrorException) { - StreamResult.CLIENT_ERROR - } catch (_: ClientAbortException) { - StreamResult.OK - } catch (e: kotlinx.io.IOException) { - logger.error("IOError occurred during streaming", e) - StreamResult.IO_ERROR + runStreamingPipeline(debridLink, range, outputStream, remotelyCachedEntity, client) } catch (e: CancellationException) { throw e - } catch (e: Exception) { - logger.error("An error occurred during streaming ${debridLink.path}", e) - StreamResult.UNKNOWN_ERROR + } catch (@Suppress("TooGenericExceptionCaught") e: Exception) { + mapToStreamResult(e, debridLink) } finally { this.coroutineContext.cancelChildren() } @@ -118,14 +77,86 @@ class StreamingService( result } + @Suppress("LongParameterList") + private suspend fun runStreamingPipeline( + debridLink: CachedFile, + range: Range?, + outputStream: OutputStream, + remotelyCachedEntity: RemotelyCachedEntity, + client: String, + ): StreamResult = coroutineScope { + val appliedRange = Range(range?.start ?: 0, range?.finish ?: (debridLink.size!! - 1)) + val inputCounter = ByteCounter() + val outputCounter = ByteCounter() + val inputCtx = InputStreamingContext(inputCounter, debridLink.provider!!, debridLink.path!!, client) + val outputCtx = OutputStreamingContext(outputCounter, remotelyCachedEntity.name!!, client) + activeInputStreams.add(inputCtx) + activeOutputStream.add(outputCtx) + val started = Instant.now() + var ttfbRecorded = false + val sawActivity = AtomicBoolean(true) + val watchdog = launch { + while (isActive) { + delay(STALL_TIMEOUT_MS) + if (!sawActivity.getAndSet(false)) { + logger.warn( + "Stream '{}' stalled (>{}ms without bytes); closing output to unblock write", + debridLink.path, STALL_TIMEOUT_MS + ) + runCatching { outputStream.close() } + break + } + } + } + try { + sendBytesFromHttpStream(debridLink, appliedRange, outputStream) { bytes -> + if (!ttfbRecorded) { + ttfbRecorded = true + Timer.builder("debridav.streaming.time.to.first.byte") + .description("Time duration between sending request and receiving first byte") + .tag("provider", debridLink.provider.toString()) + .tag("client", client) + .register(meterRegistry) + .record(Duration.between(started, Instant.now())) + } + inputCounter.add(bytes.toLong()) + outputCounter.add(bytes.toLong()) + sawActivity.set(true) + } + } finally { + watchdog.cancel() + activeOutputStream.removeStream(outputCtx) + activeInputStreams.removeStream(inputCtx) + } + StreamResult.OK + } + + private fun mapToStreamResult(e: Exception, debridLink: CachedFile): StreamResult = when (e) { + is LinkNotFoundException -> StreamResult.DEAD_LINK + is DebridProviderException -> StreamResult.PROVIDER_ERROR + is StreamToClientException -> StreamResult.IO_ERROR + is ReadFromHttpStreamException -> StreamResult.IO_ERROR + is ClientErrorException -> StreamResult.CLIENT_ERROR + is ClientAbortException -> StreamResult.OK + is AsyncRequestNotUsableException -> StreamResult.OK + is IOException -> { + logger.error("IOError occurred during streaming", e) + StreamResult.IO_ERROR + } + else -> { + logger.error("An error occurred during streaming ${debridLink.path}", e) + StreamResult.UNKNOWN_ERROR + } + } + - fun ConcurrentLinkedQueue.removeStream(ctx: OutputStreamingContext) { - outputGauge.remove(ctx.file, ctx.client) + // MultiGauge row removal happens in recordMetrics() via overwrite=true, + // so these helpers just update the backing queues. + private fun ConcurrentLinkedQueue.removeStream(ctx: OutputStreamingContext) { this.remove(ctx) } - fun ConcurrentLinkedQueue.removeStream(ctx: InputStreamingContext) { - inputGauge.remove(ctx.provider.toString(), ctx.file, ctx.client) + private fun ConcurrentLinkedQueue.removeStream(ctx: InputStreamingContext) { if (this.contains(ctx)) { this.remove(ctx) } else { @@ -149,17 +180,19 @@ class StreamingService( coroutineScope { val bufferPool = createByteArrayPool(READ_AHEAD_CHUNKS + 1, DEFAULT_BUFFER_SIZE) val chunkChannel = produceChunks(length, bufferPool, upstreamByteReadChannel) - withContext(Dispatchers.IO) { - chunkChannel.consumeEach { (buffer, bytesRead) -> - outputStream.write(buffer, 0, bytesRead) - onBytesTransferred(bytesRead) - bufferPool.send(buffer) - } + chunkChannel.consumeEach { (buffer, bytesRead) -> + outputStream.write(buffer, 0, bytesRead) + onBytesTransferred(bytesRead) + bufferPool.send(buffer) } } } catch (e: CancellationException) { throw e } catch (_: ClientAbortException) { + } catch (_: AsyncRequestNotUsableException) { + } catch (e: IOException) { + logger.warn("IO error reading from upstream HTTP stream during streaming", e) + throw ReadFromHttpStreamException("IO error reading from upstream HTTP stream", e) } catch (e: Exception) { logger.error("An error occurred during streaming", e) throw StreamToClientException("An error occurred during streaming", e) @@ -198,13 +231,29 @@ class StreamingService( @Scheduled(fixedRate = STREAMING_METRICS_POLLING_RATE_S, timeUnit = TimeUnit.SECONDS) fun recordMetrics() { - activeOutputStream.forEach { - outputGauge.labelValues(it.file, it.client) - .set(it.counter.countAndReset().toDouble().div(STREAMING_METRICS_POLLING_RATE_S)) - } - activeInputStreams.forEach { - inputGauge.labelValues(it.provider.toString(), it.file, it.client) - .set(it.counter.countAndReset().toDouble().div(STREAMING_METRICS_POLLING_RATE_S)) - } + // overwrite=true drops rows for streams no longer in the queue so + // gauges for ended streams disappear from scrape output. + outputGauge.register( + activeOutputStream.map { ctx -> + MultiGauge.Row.of( + Tags.of("file", ctx.file, "client", ctx.client), + ctx.counter.countAndReset().toDouble().div(STREAMING_METRICS_POLLING_RATE_S), + ) + }, + true, + ) + inputGauge.register( + activeInputStreams.map { ctx -> + MultiGauge.Row.of( + Tags.of( + "provider", ctx.provider.toString(), + "file", ctx.file, + "client", ctx.client, + ), + ctx.counter.countAndReset().toDouble().div(STREAMING_METRICS_POLLING_RATE_S), + ) + }, + true, + ) } } diff --git a/src/main/kotlin/io/skjaere/debridav/torrent/QBittorrentEmulationController.kt b/src/main/kotlin/io/skjaere/debridav/torrent/QBittorrentEmulationController.kt index 1d4d2e06..1d164e43 100644 --- a/src/main/kotlin/io/skjaere/debridav/torrent/QBittorrentEmulationController.kt +++ b/src/main/kotlin/io/skjaere/debridav/torrent/QBittorrentEmulationController.kt @@ -96,8 +96,8 @@ class QBittorrentEmulationController( @Suppress("MagicNumber") @GetMapping("/api/v2/torrents/files") fun torrentFiles(@RequestParam hash: TorrentHash): List? { - return torrentService.getTorrentByHash(hash)?.let { - it.files.map { torrentFile -> + return torrentService.getTorrentFilesByHash(hash)?.let { files -> + files.map { torrentFile -> TorrentFilesResponse( 0, torrentFile.contents!!.originalPath!!, diff --git a/src/main/kotlin/io/skjaere/debridav/torrent/Torrent.kt b/src/main/kotlin/io/skjaere/debridav/torrent/Torrent.kt index 12b36a2e..c0fd4be2 100644 --- a/src/main/kotlin/io/skjaere/debridav/torrent/Torrent.kt +++ b/src/main/kotlin/io/skjaere/debridav/torrent/Torrent.kt @@ -9,11 +9,14 @@ import jakarta.persistence.FetchType import jakarta.persistence.GeneratedValue import jakarta.persistence.GenerationType import jakarta.persistence.Id +import jakarta.persistence.Index import jakarta.persistence.ManyToOne import jakarta.persistence.OneToMany +import jakarta.persistence.Table import java.time.Instant @Entity +@Table(indexes = [Index(name = "idx_torrent_category_id", columnList = "category_id")]) open class Torrent { @Id @GeneratedValue(strategy = GenerationType.AUTO) @@ -26,7 +29,7 @@ open class Torrent { @OneToMany( targetEntity = RemotelyCachedEntity::class, cascade = [CascadeType.PERSIST, CascadeType.MERGE], - fetch = FetchType.EAGER, + fetch = FetchType.LAZY, ) open var files: MutableList = mutableListOf() open var created: Instant? = null @@ -37,6 +40,23 @@ open class Torrent { @Column(nullable = false, length = 2048) open var savePath: String? = null open var status: Status = Status.LIVE + + @Column(name = "last_verified") + open var lastVerified: Instant? = null + + @Column(name = "health_check_enqueued_at") + open var healthCheckEnqueuedAt: Instant? = null + + // Equality on the business key (info-hash). Hibernate proxies are subclasses + // of the entity, so the `is Torrent` check works for both real and proxied + // instances. + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is Torrent) return false + return hash != null && hash == other.hash + } + + override fun hashCode(): Int = hash?.hashCode() ?: 0 } enum class Status { LIVE, DELETED } diff --git a/src/main/kotlin/io/skjaere/debridav/torrent/TorrentHealthCheckActuatorEndpoint.kt b/src/main/kotlin/io/skjaere/debridav/torrent/TorrentHealthCheckActuatorEndpoint.kt new file mode 100644 index 00000000..4ce618f0 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/torrent/TorrentHealthCheckActuatorEndpoint.kt @@ -0,0 +1,16 @@ +package io.skjaere.debridav.torrent + +import org.springframework.boot.actuate.endpoint.annotation.Endpoint +import org.springframework.boot.actuate.endpoint.annotation.WriteOperation +import org.springframework.stereotype.Component + +@Component +@Endpoint(id = "torrenthealthcheck") +class TorrentHealthCheckActuatorEndpoint( + private val torrentHealthCheckService: TorrentHealthCheckService +) { + @WriteOperation + fun triggerHealthCheck() { + torrentHealthCheckService.triggerFullHealthCheck() + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/torrent/TorrentHealthCheckService.kt b/src/main/kotlin/io/skjaere/debridav/torrent/TorrentHealthCheckService.kt new file mode 100644 index 00000000..097e88e8 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/torrent/TorrentHealthCheckService.kt @@ -0,0 +1,60 @@ +package io.skjaere.debridav.torrent + +import com.vdsirotkin.pgmq.PgmqClient +import io.skjaere.debridav.health.HealthCheckConfigurationProperties +import io.skjaere.debridav.torrent.pgmq.TorrentHealthCheckMessage +import org.slf4j.LoggerFactory +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Service +import org.springframework.transaction.annotation.Transactional +import java.time.Clock +import java.time.Instant + +@Service +class TorrentHealthCheckService( + private val torrentRepository: TorrentRepository, + private val healthCheckConfigurationProperties: HealthCheckConfigurationProperties, + private val pgmqClient: PgmqClient, + private val clock: Clock +) { + private val logger = LoggerFactory.getLogger(TorrentHealthCheckService::class.java) + + @Scheduled(fixedDelayString = "\${health-check.torrent-poll-rate:PT5M}") + @Transactional + fun checkTorrentHealth() { + val now = Instant.now(clock) + val cutoff = now.minus(healthCheckConfigurationProperties.torrentInterval) + val enqueueCutoff = now.minus(healthCheckConfigurationProperties.torrentInterval) + + val torrentsToVerify = torrentRepository + .findByStatusAndLastVerifiedIsNullOrStatusAndLastVerifiedBefore( + Status.LIVE, Status.LIVE, cutoff + ) + .filter { + it.healthCheckEnqueuedAt == null || it.healthCheckEnqueuedAt!!.isBefore(enqueueCutoff) + } + + if (torrentsToVerify.isEmpty()) return + + logger.debug("Health check: enqueuing {} torrent(s) for verification", torrentsToVerify.size) + + torrentsToVerify.forEach { torrent -> + pgmqClient.send("torrent_health_check", TorrentHealthCheckMessage(torrent.id!!)) + torrent.healthCheckEnqueuedAt = now + torrentRepository.save(torrent) + } + } + + fun triggerFullHealthCheck() { + val torrents = torrentRepository.findByStatus(Status.LIVE) + val now = Instant.now(clock) + + logger.info("Triggering full health check for all {} live torrents", torrents.size) + + torrents.forEach { torrent -> + pgmqClient.send("torrent_health_check", TorrentHealthCheckMessage(torrent.id!!)) + torrent.healthCheckEnqueuedAt = now + torrentRepository.save(torrent) + } + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/torrent/TorrentRepository.kt b/src/main/kotlin/io/skjaere/debridav/torrent/TorrentRepository.kt index 765df5ee..bbaa0b74 100644 --- a/src/main/kotlin/io/skjaere/debridav/torrent/TorrentRepository.kt +++ b/src/main/kotlin/io/skjaere/debridav/torrent/TorrentRepository.kt @@ -6,6 +6,8 @@ import org.springframework.data.jpa.repository.Modifying import org.springframework.data.jpa.repository.Query import org.springframework.data.repository.CrudRepository import org.springframework.stereotype.Repository +import org.springframework.transaction.annotation.Transactional +import java.time.Instant @Repository interface TorrentRepository : CrudRepository { @@ -13,11 +15,21 @@ interface TorrentRepository : CrudRepository { fun getByHashIgnoreCase(hash: String): Torrent? fun findByHashIgnoreCase(hash: String): List + @Transactional fun deleteByHashIgnoreCase(hash: String) @Modifying + @Transactional @Query("update Torrent set status=io.skjaere.debridav.torrent.Status.DELETED where id=:#{#torrent.id}") fun markTorrentAsDeleted(torrent: Torrent) fun getTorrentByFilesContains(file: DbEntity): List + + fun findByStatusAndLastVerifiedIsNullOrStatusAndLastVerifiedBefore( + status1: Status, + status2: Status, + cutoff: Instant + ): List + + fun findByStatus(status: Status): List } diff --git a/src/main/kotlin/io/skjaere/debridav/torrent/TorrentService.kt b/src/main/kotlin/io/skjaere/debridav/torrent/TorrentService.kt index 01f4ec4b..a86f3b68 100644 --- a/src/main/kotlin/io/skjaere/debridav/torrent/TorrentService.kt +++ b/src/main/kotlin/io/skjaere/debridav/torrent/TorrentService.kt @@ -6,6 +6,7 @@ import io.skjaere.debridav.debrid.DebridCachedContentService import io.skjaere.debridav.debrid.TorrentMagnet import io.skjaere.debridav.fs.DatabaseFileService import io.skjaere.debridav.fs.DebridFileContents +import io.skjaere.debridav.fs.RemotelyCachedEntity import jakarta.transaction.Transactional import kotlinx.coroutines.runBlocking import org.slf4j.LoggerFactory @@ -28,18 +29,16 @@ class TorrentService( ) { private val logger = LoggerFactory.getLogger(TorrentService::class.java) - @Transactional fun addTorrent(category: String, torrent: MultipartFile): Boolean { return addMagnet( category, torrentToMagnetConverter.convertTorrentToMagnet(torrent.bytes) ) } - @Transactional - fun addMagnet(category: String, magnet: TorrentMagnet): Boolean = runBlocking { + fun addMagnet(category: String, magnet: TorrentMagnet): Boolean { val debridFileContents = runBlocking { debridService.addContent(magnet) } - if (debridFileContents.isEmpty()) { + return if (debridFileContents.isEmpty()) { logger.info("${getNameFromMagnet(magnet)} is not cached in any debrid services") false } else { @@ -48,6 +47,7 @@ class TorrentService( } } + @Transactional fun createTorrent( cachedFiles: List, categoryName: String, @@ -66,16 +66,12 @@ class TorrentService( torrent.created = Instant.now() torrent.hash = hash.hash torrent.status = Status.LIVE - torrent.savePath = - "${debridavConfigurationProperties.downloadPath}/${torrent.name}" - torrent.files = - cachedFiles.map { - fileService.createDebridFile( - "${debridavConfigurationProperties.downloadPath}/${torrent.name}/${it.originalPath}", - getHashFromMagnet(magnet)!!.hash, - it - ) - }.toMutableList() + val torrentBasePath = "${debridavConfigurationProperties.downloadPath}/${torrent.name}" + torrent.savePath = torrentBasePath + torrent.files = fileService.createDebridFiles( + cachedFiles.map { "$torrentBasePath/${it.originalPath}" to it }, + hash.hash, + ).toMutableList() logger.info("Saving ${torrent.files.count()} files") return torrentRepository.save(torrent) @@ -92,6 +88,13 @@ class TorrentService( return torrentRepository.getByHashIgnoreCase(hash.hash) } + @Transactional + fun getTorrentFilesByHash(hash: TorrentHash): List? { + // Touch files inside the transaction so Hibernate initializes the lazy + // collection before we hand it back to a controller (OSIV is off). + return torrentRepository.getByHashIgnoreCase(hash.hash)?.files?.toList() + } + @Transactional fun deleteTorrentByHash(hash: String) { return torrentRepository.deleteByHashIgnoreCase(hash) diff --git a/src/main/kotlin/io/skjaere/debridav/torrent/pgmq/Messages.kt b/src/main/kotlin/io/skjaere/debridav/torrent/pgmq/Messages.kt new file mode 100644 index 00000000..f965f5d4 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/torrent/pgmq/Messages.kt @@ -0,0 +1,10 @@ +package io.skjaere.debridav.torrent.pgmq + +data class TorrentHealthCheckMessage( + val torrentId: Long +) + +data class TorrentHealthRepairMessage( + val torrentId: Long, + val message: String +) diff --git a/src/main/kotlin/io/skjaere/debridav/torrent/pgmq/TorrentHealthCheckHandler.kt b/src/main/kotlin/io/skjaere/debridav/torrent/pgmq/TorrentHealthCheckHandler.kt new file mode 100644 index 00000000..55b1fb4d --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/torrent/pgmq/TorrentHealthCheckHandler.kt @@ -0,0 +1,87 @@ +package io.skjaere.debridav.torrent.pgmq + +import com.vdsirotkin.pgmq.PgmqClient +import io.skjaere.debridav.debrid.DebridLinkService +import io.skjaere.debridav.fs.MissingFile +import io.skjaere.debridav.fs.ProviderError +import io.skjaere.debridav.health.HealthMetrics +import io.skjaere.debridav.health.HealthMetrics.CheckResult +import io.skjaere.debridav.health.HealthMetrics.HealthType +import io.skjaere.debridav.torrent.TorrentRepository +import kotlinx.coroutines.flow.firstOrNull +import kotlinx.coroutines.runBlocking +import org.slf4j.LoggerFactory +import org.springframework.stereotype.Service +import org.springframework.transaction.annotation.Transactional +import java.time.Clock +import java.time.Instant + +@Service +class TorrentHealthCheckHandler( + private val torrentRepository: TorrentRepository, + private val debridLinkService: DebridLinkService, + private val pgmqClient: PgmqClient, + private val clock: Clock, + private val healthMetrics: HealthMetrics, +) { + private val logger = LoggerFactory.getLogger(TorrentHealthCheckHandler::class.java) + + @Transactional + fun handle(msg: TorrentHealthCheckMessage) { + val torrent = torrentRepository.findById(msg.torrentId).orElse(null) + if (torrent == null) { + logger.warn("Torrent {} not found, skipping health check", msg.torrentId) + healthMetrics.recordCheck(HealthType.TORRENT, CheckResult.NOT_FOUND) + return + } + + healthMetrics.timeCheck(HealthType.TORRENT) { + try { + val files = torrent.files + if (files.isEmpty()) { + logger.debug("Torrent {} has no files, skipping health check", torrent.id) + healthMetrics.recordCheck(HealthType.TORRENT, CheckResult.OK) + return@timeCheck + } + + val unhealthy = files.any { file -> + val contents = file.contents ?: return@any false + val healthyLink = runBlocking { + debridLinkService.getFlowOfDebridLinks(contents) + .firstOrNull { it !is MissingFile && it !is ProviderError } + } + val allUnavailable = healthyLink == null + if (allUnavailable) { + logger.warn( + "Torrent {} file '{}' is unhealthy — all providers returned MissingFile or ProviderError", + torrent.id, file.name + ) + } + allUnavailable + } + + if (unhealthy) { + logger.warn("Torrent {} '{}' is unhealthy, enqueuing for repair", torrent.id, torrent.name) + healthMetrics.recordCheck(HealthType.TORRENT, CheckResult.MISSING) + pgmqClient.send( + "torrent_health_repair", + TorrentHealthRepairMessage( + torrentId = torrent.id!!, + message = "One or more files unavailable from all debrid providers" + ) + ) + } else { + logger.debug("Torrent {} verified successfully", torrent.id) + healthMetrics.recordCheck(HealthType.TORRENT, CheckResult.OK) + } + } catch (@Suppress("TooGenericExceptionCaught") e: Exception) { + logger.error("Unexpected error verifying torrent {}", torrent.id, e) + healthMetrics.recordCheck(HealthType.TORRENT, CheckResult.FAILURE) + } + } + + torrent.lastVerified = Instant.now(clock) + torrent.healthCheckEnqueuedAt = null + torrentRepository.save(torrent) + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/torrent/pgmq/TorrentHealthRepairHandler.kt b/src/main/kotlin/io/skjaere/debridav/torrent/pgmq/TorrentHealthRepairHandler.kt new file mode 100644 index 00000000..d4f8bd2b --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/torrent/pgmq/TorrentHealthRepairHandler.kt @@ -0,0 +1,107 @@ +package io.skjaere.debridav.torrent.pgmq + +import io.skjaere.debridav.arrs.ArrService +import io.skjaere.debridav.fs.DatabaseFileService +import io.skjaere.debridav.health.HealthCheckConfigurationProperties +import io.skjaere.debridav.health.HealthMetrics +import io.skjaere.debridav.health.HealthMetrics.HealthType +import io.skjaere.debridav.health.RepairAction +import io.skjaere.debridav.health.RepairOutcomeService +import io.skjaere.debridav.torrent.Torrent +import io.skjaere.debridav.torrent.TorrentRepository +import kotlinx.coroutines.runBlocking +import org.slf4j.LoggerFactory +import org.springframework.stereotype.Service +import org.springframework.transaction.annotation.Transactional + +@Service +class TorrentHealthRepairHandler( + private val torrentRepository: TorrentRepository, + private val arrService: ArrService, + private val fileService: DatabaseFileService, + private val healthCheckConfig: HealthCheckConfigurationProperties, + private val repairOutcomeService: RepairOutcomeService, + private val healthMetrics: HealthMetrics, +) { + private val logger = LoggerFactory.getLogger(TorrentHealthRepairHandler::class.java) + + @Transactional + fun handle(msg: TorrentHealthRepairMessage, msgId: Long) { + if (!healthCheckConfig.repairEnabled) { + logger.debug("Repair is disabled, skipping torrent {}", msg.torrentId) + repairOutcomeService.record(QUEUE_NAME, msgId, RepairAction.SKIPPED) + healthMetrics.recordRepair(HealthType.TORRENT, RepairAction.SKIPPED.name) + return + } + + val torrent = torrentRepository.findById(msg.torrentId).orElse(null) + if (torrent == null) { + logger.warn("No torrent found for ID {}", msg.torrentId) + healthMetrics.recordRepair(HealthType.TORRENT, "NOT_FOUND") + return + } + + healthMetrics.timeRepair(HealthType.TORRENT) { + executeRepair(msgId, torrent) + } + } + + private fun executeRepair(msgId: Long, torrent: Torrent) { + val category = torrent.category?.name + val hash = torrent.hash + + if (category != null && hash != null && arrService.getClientForCategory(category) != null) { + logger.info( + "Blocklisting torrent hash '{}' for '{}' (category: {})", + hash, torrent.name, category + ) + runBlocking { arrService.blocklist(hash, category) } + + var anyDeleted = false + var anyRepaired = false + torrent.files.forEach { file -> + val fileName = file.name + if (fileName != null) { + logger.info( + "Notifying Arr to delete file and search for '{}' (category: {})", + fileName, category + ) + val found = runBlocking { arrService.deleteFileAndSearch(fileName, category) } + if (!found) { + logger.info( + "Arr could not find '{}', deleting from virtual filesystem", + fileName + ) + fileService.deleteFile(file) + anyDeleted = true + } else { + anyRepaired = true + } + } + } + val action = when { + anyRepaired && !anyDeleted -> RepairAction.REPAIRED + !anyRepaired && anyDeleted -> RepairAction.DELETED + anyRepaired -> RepairAction.REPAIRED + else -> RepairAction.DELETED + } + repairOutcomeService.record(QUEUE_NAME, msgId, action) + healthMetrics.recordRepair(HealthType.TORRENT, action.name) + } else { + logger.info( + "No Arr client for torrent {} (category: {}), deleting all files from virtual filesystem", + torrent.id, category + ) + torrent.files.forEach { file -> + logger.info("Deleting '{}' from virtual filesystem", file.name) + fileService.deleteFile(file) + } + repairOutcomeService.record(QUEUE_NAME, msgId, RepairAction.DELETED) + healthMetrics.recordRepair(HealthType.TORRENT, RepairAction.DELETED.name) + } + } + + companion object { + const val QUEUE_NAME = "torrent_health_repair" + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/torrent/pgmq/TorrentPgmqConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/torrent/pgmq/TorrentPgmqConfiguration.kt new file mode 100644 index 00000000..961cf3b4 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/torrent/pgmq/TorrentPgmqConfiguration.kt @@ -0,0 +1,50 @@ +package io.skjaere.debridav.torrent.pgmq + +import com.fasterxml.jackson.databind.ObjectMapper +import com.vdsirotkin.pgmq.PgmqClient +import io.skjaere.debridav.pgmq.PgmqConfigurationProperties +import io.skjaere.debridav.pgmq.PgmqConsumer +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration + +@Configuration +class TorrentPgmqConfiguration { + + @Bean + fun torrentHealthCheckConsumer( + pgmqClient: PgmqClient, + pgmqObjectMapper: ObjectMapper, + props: PgmqConfigurationProperties, + handler: TorrentHealthCheckHandler + ): PgmqConsumer = PgmqConsumer( + pgmqClient = pgmqClient, + objectMapper = pgmqObjectMapper, + queueName = "torrent_health_check", + messageType = TorrentHealthCheckMessage::class.java, + concurrency = props.torrentHealthCheckConcurrency, + visibilityTimeout = props.torrentHealthCheckVisibilityTimeout, + pollInterval = props.torrentHealthCheckPollInterval, + maxReadCount = props.maxReadCount + ) { msg, _ -> + handler.handle(msg) + } + + @Bean + fun torrentHealthRepairConsumer( + pgmqClient: PgmqClient, + pgmqObjectMapper: ObjectMapper, + props: PgmqConfigurationProperties, + handler: TorrentHealthRepairHandler + ): PgmqConsumer = PgmqConsumer( + pgmqClient = pgmqClient, + objectMapper = pgmqObjectMapper, + queueName = "torrent_health_repair", + messageType = TorrentHealthRepairMessage::class.java, + concurrency = props.torrentHealthRepairConcurrency, + visibilityTimeout = props.torrentHealthRepairVisibilityTimeout, + pollInterval = props.torrentHealthRepairPollInterval, + maxReadCount = props.maxReadCount + ) { msg, msgId -> + handler.handle(msg, msgId) + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/ui/GrafanaDashboardService.kt b/src/main/kotlin/io/skjaere/debridav/ui/GrafanaDashboardService.kt new file mode 100644 index 00000000..393a94d9 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/ui/GrafanaDashboardService.kt @@ -0,0 +1,55 @@ +package io.skjaere.debridav.ui + +import io.ktor.client.HttpClient +import io.ktor.client.call.body +import io.ktor.client.request.accept +import io.ktor.client.request.bearerAuth +import io.ktor.client.request.get +import io.ktor.http.ContentType +import io.ktor.http.isSuccess +import kotlinx.serialization.Serializable +import org.slf4j.LoggerFactory +import org.springframework.stereotype.Service + +private const val DEBRIDAV_FOLDER = "debridav" + +@Service +class GrafanaDashboardService( + private val uiConfig: UiConfigurationProperties, + private val httpClient: HttpClient, +) { + private val logger = LoggerFactory.getLogger(GrafanaDashboardService::class.java) + + suspend fun listDashboards(): List { + val baseUrl = uiConfig.grafana.baseUrl.trimEnd('/') + if (baseUrl.isBlank()) return emptyList() + val apiKey = uiConfig.grafana.apiKey.ifBlank { null } + + return try { + val response = httpClient.get("$baseUrl/api/search?type=dash-db") { + accept(ContentType.Application.Json) + if (apiKey != null) bearerAuth(apiKey) + } + if (response.status.isSuccess()) { + val entries: List = response.body() + entries + .filter { it.folderTitle == DEBRIDAV_FOLDER } + .map { DashboardDto(label = it.title, path = it.url) } + .sortedBy { it.label } + } else { + logger.warn("Grafana returned {} when listing dashboards", response.status) + emptyList() + } + } catch (@Suppress("TooGenericExceptionCaught") e: Exception) { + logger.warn("Failed to fetch dashboards from Grafana at {}: {}", baseUrl, e.message) + emptyList() + } + } + + @Serializable + private data class GrafanaSearchEntry( + val title: String, + val url: String, + val folderTitle: String? = null, + ) +} diff --git a/src/main/kotlin/io/skjaere/debridav/ui/SummaryController.kt b/src/main/kotlin/io/skjaere/debridav/ui/SummaryController.kt new file mode 100644 index 00000000..c765ed30 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/ui/SummaryController.kt @@ -0,0 +1,111 @@ +package io.skjaere.debridav.ui + +import io.micrometer.core.instrument.MeterRegistry +import io.skjaere.debridav.health.HealthQueueService +import io.skjaere.debridav.repository.DebridFileContentsRepository +import io.skjaere.debridav.usenet.queue.UsenetQueueService +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +/** + * Lightweight snapshot of the running system for the frontend's fallback + * overview page (shown when the Grafana monitoring stack isn't deployed). + * + * All values are read from the in-process MeterRegistry and existing queue + * services — no Prometheus scrape required. Polled from the frontend every + * few seconds. + */ +@RestController +@RequestMapping("/api/v1/summary") +class SummaryController( + private val meterRegistry: MeterRegistry, + private val usenetQueueService: UsenetQueueService, + private val healthQueueService: HealthQueueService, + private val debridFileRepository: DebridFileContentsRepository, +) { + @GetMapping + fun getSummary(): ResponseEntity { + val debridStreams = meterRegistry.find("debridav.input.stream.bitrate").gauges().map { g -> + StreamDto( + name = g.id.getTag("file") ?: "(unknown)", + source = g.id.getTag("provider") ?: "debrid", + bitrateBytesPerSec = g.value(), + ) + } + val nntpStreams = meterRegistry.find("nzb.streams.bitrate").gauges().map { g -> + StreamDto( + name = g.id.getTag("name") ?: "(unknown)", + source = "NNTP", + bitrateBytesPerSec = g.value(), + ) + } + val streams = (debridStreams + nntpStreams).sortedByDescending { it.bitrateBytesPerSec } + val activeStreams = ActiveStreamsDto( + count = streams.size, + bitrateBytesPerSec = streams.sumOf { it.bitrateBytesPerSec }, + streams = streams, + ) + + // debridav.library.size lives in the Prometheus native client registry, + // not Micrometer, so we bypass both and query the repository directly. + // Side benefit: fresh values every poll instead of 60s-stale metric ticks. + val torrentCount = debridFileRepository.numberOfRemotelyCachedTorrentEntities() + val usenetCount = debridFileRepository.numberOfRemotelyCachedUsenetEntities() + val library = LibraryDto( + totalFiles = torrentCount + usenetCount, + bySource = listOf( + LibrarySourceDto("torrent", torrentCount), + LibrarySourceDto("usenet", usenetCount), + ).filter { it.files > 0 }, + ) + + val usenetStatus = usenetQueueService.getQueueStatus() + val importQueue = QueueCountsDto( + pending = usenetStatus.pending.size, + processing = usenetStatus.processing.size, + ) + + val healthCheck = healthQueueService.getHealthCheckStatus() + val repair = healthQueueService.getRepairStatus() + val healthQueue = QueueCountsDto( + pending = healthCheck.count + repair.count, + processing = 0, + ) + + val cpu = meterRegistry.find("process.cpu.usage").gauge()?.value() ?: 0.0 + val memory = meterRegistry.find("jvm.memory.used").gauges().sumOf { it.value().toLong() } + val system = SystemDto(cpuUsage = cpu, memoryBytes = memory) + + return ResponseEntity.ok(SummaryDto(activeStreams, library, importQueue, healthQueue, system)) + } +} + +data class SummaryDto( + val activeStreams: ActiveStreamsDto, + val library: LibraryDto, + val importQueue: QueueCountsDto, + val healthQueue: QueueCountsDto, + val system: SystemDto, +) + +data class ActiveStreamsDto( + val count: Int, + val bitrateBytesPerSec: Double, + val streams: List, +) + +data class StreamDto( + val name: String, + val source: String, + val bitrateBytesPerSec: Double, +) + +data class LibraryDto(val totalFiles: Long, val bySource: List) + +data class LibrarySourceDto(val source: String, val files: Long) + +data class QueueCountsDto(val pending: Int, val processing: Int) + +data class SystemDto(val cpuUsage: Double, val memoryBytes: Long) diff --git a/src/main/kotlin/io/skjaere/debridav/ui/UiConfigController.kt b/src/main/kotlin/io/skjaere/debridav/ui/UiConfigController.kt new file mode 100644 index 00000000..3dd59928 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/ui/UiConfigController.kt @@ -0,0 +1,37 @@ +package io.skjaere.debridav.ui + +import kotlinx.coroutines.runBlocking +import org.springframework.boot.info.BuildProperties +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@RestController +@RequestMapping("/api/v1") +class UiConfigController( + private val uiConfig: UiConfigurationProperties, + private val grafanaDashboardService: GrafanaDashboardService, + private val buildProperties: BuildProperties? = null, +) { + @GetMapping("/ui-config") + fun getUiConfig(): ResponseEntity { + val grafana = uiConfig.grafana + .takeIf { it.baseUrl.isNotBlank() } + ?.let { cfg -> GrafanaDto(baseUrl = cfg.baseUrl.trimEnd('/')) } + return ResponseEntity.ok( + UiConfigDto( + grafana = grafana, + version = buildProperties?.version, + ) + ) + } + + @GetMapping("/grafana/dashboards") + fun getGrafanaDashboards(): ResponseEntity> = + ResponseEntity.ok(runBlocking { grafanaDashboardService.listDashboards() }) +} + +data class UiConfigDto(val grafana: GrafanaDto?, val version: String?) +data class GrafanaDto(val baseUrl: String) +data class DashboardDto(val label: String, val path: String) diff --git a/src/main/kotlin/io/skjaere/debridav/ui/UiConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/ui/UiConfigurationProperties.kt new file mode 100644 index 00000000..083e6878 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/ui/UiConfigurationProperties.kt @@ -0,0 +1,13 @@ +package io.skjaere.debridav.ui + +import org.springframework.boot.context.properties.ConfigurationProperties + +@ConfigurationProperties(prefix = "debridav.ui") +class UiConfigurationProperties { + var grafana: GrafanaConfig = GrafanaConfig() +} + +class GrafanaConfig { + var baseUrl: String = "" + var apiKey: String = "" +} diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/NzbHealthCheckActuatorEndpoint.kt b/src/main/kotlin/io/skjaere/debridav/usenet/NzbHealthCheckActuatorEndpoint.kt index 9a369a17..50e87706 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/NzbHealthCheckActuatorEndpoint.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/NzbHealthCheckActuatorEndpoint.kt @@ -2,12 +2,10 @@ package io.skjaere.debridav.usenet import org.springframework.boot.actuate.endpoint.annotation.Endpoint import org.springframework.boot.actuate.endpoint.annotation.WriteOperation -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.stereotype.Component @Component @Endpoint(id = "nzbhealthcheck") -@ConditionalOnProperty("nntp.enabled", havingValue = "true") class NzbHealthCheckActuatorEndpoint( private val nzbHealthCheckService: NzbHealthCheckService ) { diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/NzbHealthCheckService.kt b/src/main/kotlin/io/skjaere/debridav/usenet/NzbHealthCheckService.kt index 040ed3d1..2ef9b75c 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/NzbHealthCheckService.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/NzbHealthCheckService.kt @@ -1,10 +1,10 @@ package io.skjaere.debridav.usenet import com.vdsirotkin.pgmq.PgmqClient +import io.skjaere.debridav.health.HealthCheckConfigurationProperties import io.skjaere.debridav.repository.NzbDocumentRepository import io.skjaere.debridav.usenet.pgmq.NzbHealthCheckMessage import org.slf4j.LoggerFactory -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.scheduling.annotation.Scheduled import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional @@ -12,21 +12,20 @@ import java.time.Clock import java.time.Instant @Service -@ConditionalOnProperty("nntp.enabled", havingValue = "true") class NzbHealthCheckService( private val nzbDocumentRepository: NzbDocumentRepository, - private val nntpConfigurationProperties: NntpConfigurationProperties, + private val healthCheckConfigurationProperties: HealthCheckConfigurationProperties, private val pgmqClient: PgmqClient, private val clock: Clock ) { private val logger = LoggerFactory.getLogger(NzbHealthCheckService::class.java) - @Scheduled(fixedDelayString = "\${nntp.health-check-poll-rate}") + @Scheduled(fixedDelayString = "\${health-check.nzb-poll-rate}") @Transactional fun checkNzbHealth() { val now = Instant.now(clock) - val cutoff = now.minus(nntpConfigurationProperties.healthCheckInterval) - val enqueueCutoff = now.minus(nntpConfigurationProperties.healthCheckInterval) + val cutoff = now.minus(healthCheckConfigurationProperties.nzbInterval) + val enqueueCutoff = now.minus(healthCheckConfigurationProperties.nzbInterval) val nzbsToVerify = nzbDocumentRepository .findByLastVerifiedIsNullOrLastVerifiedBefore(cutoff) .filter { it.healthCheckEnqueuedAt == null || it.healthCheckEnqueuedAt!!.isBefore(enqueueCutoff) } diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/NzbImportService.kt b/src/main/kotlin/io/skjaere/debridav/usenet/NzbImportService.kt index f7e535cd..40ebb0a8 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/NzbImportService.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/NzbImportService.kt @@ -5,7 +5,11 @@ import io.skjaere.debridav.configuration.DebridavConfigurationProperties import io.skjaere.debridav.fs.DatabaseFileService import io.skjaere.debridav.fs.NzbContents import io.skjaere.debridav.repository.NzbDocumentRepository +import io.skjaere.debridav.repository.NzbImportRepository import io.skjaere.debridav.repository.UsenetRepository +import io.skjaere.debridav.usenet.queue.NzbImportFileJson +import io.skjaere.debridav.usenet.queue.NzbImportRecord +import io.skjaere.debridav.usenet.queue.NzbImportStatus import io.skjaere.debridav.usenet.nzb.NzbArchiveType import io.skjaere.debridav.usenet.nzb.NzbDocumentEntity import io.skjaere.debridav.usenet.nzb.NzbFileJson @@ -19,79 +23,140 @@ import io.skjaere.nzbstreamer.metadata.PrepareResult import io.skjaere.nzbstreamer.stream.StreamableFile import kotlinx.coroutines.runBlocking import org.slf4j.LoggerFactory -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.stereotype.Service -import org.springframework.transaction.annotation.Transactional +import org.springframework.transaction.PlatformTransactionManager +import org.springframework.transaction.support.TransactionTemplate import java.time.Instant import java.util.* @Service -@ConditionalOnProperty("nntp.enabled", havingValue = "true") class NzbImportService( private val nzbStreamer: NzbStreamer, private val nzbDocumentRepository: NzbDocumentRepository, private val usenetRepository: UsenetRepository, + private val nzbImportRepository: NzbImportRepository, private val pgmqClient: PgmqClient, private val databaseFileService: DatabaseFileService, - private val debridavConfigurationProperties: DebridavConfigurationProperties + private val debridavConfigurationProperties: DebridavConfigurationProperties, + platformTransactionManager: PlatformTransactionManager ) { + private val transactionTemplate = TransactionTemplate(platformTransactionManager) private val logger = LoggerFactory.getLogger(NzbImportService::class.java) - fun scheduleImport(nzbBytes: ByteArray, usenetDownload: UsenetDownload) { + fun scheduleImport(nzbBytes: ByteArray, usenetDownload: UsenetDownload, nzbImportRecordId: Long) { pgmqClient.send( "nzb_import", NzbImportMessage( nzbBytesBase64 = Base64.getEncoder().encodeToString(nzbBytes), - usenetDownloadId = usenetDownload.id!! + usenetDownloadId = usenetDownload.id!!, + nzbImportRecordId = nzbImportRecordId ) ) } - @Transactional @Suppress("LongMethod", "ReturnCount") fun executeImport(taskData: NzbImportTaskData) { - val usenetDownload = usenetRepository.findById(taskData.usenetDownloadId).orElseThrow { - IllegalStateException("UsenetDownload not found: ${taskData.usenetDownloadId}") - } - try { + // Phase 1: Load entities and mark as IMPORTING in a short transaction. + // We do NOT hold this transaction open during the long NNTP I/O below. + val downloadName = transactionTemplate.execute { + val usenetDownload = usenetRepository.findById(taskData.usenetDownloadId).orElse(null) + ?: run { + logger.warn( + "UsenetDownload ${taskData.usenetDownloadId} not found (may have been deleted), " + + "skipping import" + ) + return@execute null + } + val importRecord = nzbImportRepository.findById(taskData.nzbImportRecordId).orElseThrow { + IllegalStateException("NzbImportRecord not found: ${taskData.nzbImportRecordId}") + } logger.info("Importing ${usenetDownload.name}") - val nzbBytes = Base64.getDecoder().decode(taskData.nzbBytesBase64) - val prepareResult = runBlocking { nzbStreamer.prepare(nzbBytes) } + importRecord.status = NzbImportStatus.IMPORTING + nzbImportRepository.save(importRecord) + usenetDownload.name + } ?: return + + // Phase 2: Perform long-running NNTP I/O outside any database transaction. + // This prevents the transaction from being held open while waiting for the + // network, which would cause ObjectOptimisticLockingFailureException if the + // UsenetDownload row is deleted by another thread during the I/O. + val nzbBytes = Base64.getDecoder().decode(taskData.nzbBytesBase64) + var prepareResult: PrepareResult? = null + var prepareException: Exception? = null + try { + prepareResult = runBlocking { nzbStreamer.prepare(nzbBytes) } + } catch (@Suppress("TooGenericExceptionCaught") e: Exception) { + logger.error("Failed to prepare NZB for download '$downloadName'", e) + prepareException = e + } + + // Phase 3: Re-fetch entities and persist results in a new short transaction. + // Re-fetching avoids operating on stale/detached entities and gracefully handles + // the case where the UsenetDownload was deleted while the I/O was running. + transactionTemplate.execute { + val usenetDownload = usenetRepository.findById(taskData.usenetDownloadId).orElse(null) + val importRecord = nzbImportRepository.findById(taskData.nzbImportRecordId).orElseThrow { + IllegalStateException("NzbImportRecord not found: ${taskData.nzbImportRecordId}") + } + + if (usenetDownload == null) { + logger.warn( + "UsenetDownload ${taskData.usenetDownloadId} ('$downloadName') was deleted " + + "during import; aborting result persistence" + ) + importRecord.status = NzbImportStatus.FAILED + importRecord.errorMessage = "Download was deleted during import" + nzbImportRepository.save(importRecord) + return@execute + } + + when { + prepareException != null -> { + usenetDownload.status = UsenetDownloadStatus.FAILED + importRecord.status = NzbImportStatus.FAILED + importRecord.errorMessage = prepareException.stackTraceToString() + } - when (prepareResult) { - is PrepareResult.MissingArticles -> { + prepareResult is PrepareResult.MissingArticles -> { + val result = prepareResult as PrepareResult.MissingArticles logger.warn( "Articles missing from Usenet for '{}': {}", usenetDownload.name, - prepareResult.message + result.message ) usenetDownload.status = UsenetDownloadStatus.FAILED - return + importRecord.status = NzbImportStatus.FAILED + importRecord.errorMessage = result.message } - is PrepareResult.Failure -> { + prepareResult is PrepareResult.Failure -> { + val result = prepareResult as PrepareResult.Failure logger.error( "NNTP failure importing '{}': {}", usenetDownload.name, - prepareResult.message, - prepareResult.cause + result.message, + result.cause ) usenetDownload.status = UsenetDownloadStatus.FAILED - return + importRecord.status = NzbImportStatus.FAILED + importRecord.errorMessage = result.cause.stackTraceToString() } - is PrepareResult.UnsupportedArchive -> { + prepareResult is PrepareResult.UnsupportedArchive -> { + val result = prepareResult as PrepareResult.UnsupportedArchive logger.warn( "Unsupported archive type for '{}': {}", usenetDownload.name, - prepareResult.message + result.message ) usenetDownload.status = UsenetDownloadStatus.FAILED - return + importRecord.status = NzbImportStatus.FAILED + importRecord.errorMessage = result.message } - is PrepareResult.Success -> { - val metadata = prepareResult.metadata + prepareResult is PrepareResult.Success -> { + val result = prepareResult as PrepareResult.Success + val metadata = result.metadata val streamableFiles = nzbStreamer.resolveStreamableFiles(metadata) val documentEntity = toDocumentEntity(metadata, streamableFiles) documentEntity.category = usenetDownload.category?.name @@ -101,31 +166,44 @@ class NzbImportService( val savedDocument = nzbDocumentRepository.save(documentEntity) usenetDownload.nzbDocument = savedDocument - usenetDownload.debridFiles = savedDocument.streamableFiles.map { sf -> - val nzbContents = NzbContents().apply { - nzbDocument = savedDocument - originalPath = sf.path - size = sf.totalSize - modified = Instant.now().toEpochMilli() - } - val path = "${debridavConfigurationProperties.downloadPath}" + - "/${usenetDownload.name}/${sf.path}" - databaseFileService.createDebridFile( - path, - usenetDownload.hash!!, - nzbContents - ) - }.toMutableList() + val basePath = "${debridavConfigurationProperties.downloadPath}/${usenetDownload.name}" + val downloadHash = usenetDownload.hash!! + usenetDownload.debridFiles = databaseFileService.createDebridFiles( + savedDocument.streamableFiles.map { sf -> + val nzbContents = NzbContents().apply { + nzbDocument = savedDocument + originalPath = sf.path + size = sf.totalSize + modified = Instant.now().toEpochMilli() + } + "$basePath/${sf.path}" to nzbContents + }, + downloadHash, + ).toMutableList() usenetDownload.status = UsenetDownloadStatus.COMPLETED + importRecord.status = NzbImportStatus.COMPLETED + importRecord.size = savedDocument.streamableFiles.sumOf { it.totalSize } + importRecord.archiveType = savedDocument.archiveType.name + importRecord.files = savedDocument.streamableFiles.map { sf -> + NzbImportFileJson( + path = "${debridavConfigurationProperties.downloadPath}" + + "/${usenetDownload.name}/${sf.path}", + size = sf.totalSize + ) + } logger.info("Imported ${usenetDownload.name}") } + + else -> { + usenetDownload.status = UsenetDownloadStatus.FAILED + importRecord.status = NzbImportStatus.FAILED + importRecord.errorMessage = "Unknown error during import prepare phase" + } } - } catch (@Suppress("TooGenericExceptionCaught") e: Exception) { - logger.error("Failed to import NZB for download '${usenetDownload.name}'", e) - usenetDownload.status = UsenetDownloadStatus.FAILED - } finally { + usenetRepository.save(usenetDownload) + nzbImportRepository.save(importRecord) } } diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/NzbImportTaskData.kt b/src/main/kotlin/io/skjaere/debridav/usenet/NzbImportTaskData.kt index 0d87fa89..46f52c91 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/NzbImportTaskData.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/NzbImportTaskData.kt @@ -2,5 +2,6 @@ package io.skjaere.debridav.usenet data class NzbImportTaskData( val nzbBytesBase64: String, - val usenetDownloadId: Long + val usenetDownloadId: Long, + val nzbImportRecordId: Long ) diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/NzbStreamerConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/usenet/NzbStreamerConfiguration.kt index 292247ce..6ecb3cd8 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/NzbStreamerConfiguration.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/NzbStreamerConfiguration.kt @@ -1,57 +1,77 @@ package io.skjaere.debridav.usenet +import io.skjaere.debridav.config.ConfigProperty import io.skjaere.nzbstreamer.NzbStreamer import io.skjaere.nzbstreamer.config.NntpConfig -import io.skjaere.nzbstreamer.config.SeekConfig +import io.skjaere.nzbstreamer.config.StreamingConfig import org.slf4j.LoggerFactory -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.boot.context.properties.ConfigurationProperties import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import java.time.Duration -@Suppress("MagicNumber") -@ConfigurationProperties(prefix = "nntp") -data class NntpConfigurationProperties( - val enabled: Boolean = false, +data class NntpPoolProperties( val host: String = "", val port: Int = 563, val username: String = "", val password: String = "", val useTls: Boolean = true, - val concurrency: Int = 4, val maxConnections: Int = 8, - val readAheadSegments: Int? = null, - val forwardThresholdBytes: Long = 102400L, - val healthCheckInterval: Duration = Duration.ofDays(7), - val healthCheckPollRate: Duration = Duration.ofMinutes(5) + val priority: Int = 0 ) +@Suppress("MagicNumber") +@ConfigurationProperties(prefix = "nntp") +class NntpConfigurationProperties { + @ConfigProperty(name = "Concurrency", description = "NNTP streaming concurrency") + var concurrency: Int = 4 + @ConfigProperty( + name = "Read Ahead Segments", + description = "Segments to prefetch per stream. Leave empty to default to the concurrency value." + ) + var readAheadSegments: Int? = null + var pools: List = emptyList() +} + @Configuration class NzbStreamerConfiguration { private val logger = LoggerFactory.getLogger(NzbStreamerConfiguration::class.java) @Bean - @ConditionalOnProperty("nntp.enabled", havingValue = "true") fun nzbStreamer(props: NntpConfigurationProperties): NzbStreamer { + val nntpConfigs = buildNntpConfigs(props) + val streamingConfig = StreamingConfig( + concurrency = props.concurrency, + readAheadSegments = props.readAheadSegments ?: props.concurrency + ) logger.info( - "Creating NzbStreamer with host='{}', port={}, useTls={}, username='{}', concurrency={}, maxConnections={}", - props.host, props.port, props.useTls, props.username, props.concurrency, props.maxConnections + "Creating NzbStreamer with {} pool(s), concurrency={}", + nntpConfigs.size, streamingConfig.concurrency ) + nntpConfigs.forEachIndexed { index, config -> + logger.info( + " pool[{}]: host='{}', port={}, useTls={}, username='{}', maxConnections={}, priority={}", + index, config.host, config.port, config.useTls, config.username, config.maxConnections, + config.priority + ) + } return NzbStreamer.fromConfig( + nntpConfigs, + streamingConfig + ) + } + + private fun buildNntpConfigs(props: NntpConfigurationProperties): List { + return props.pools.sortedBy { it.priority }.map { pool -> NntpConfig( - host = props.host, - port = props.port, - username = props.username, - password = props.password, - useTls = props.useTls, - concurrency = props.concurrency, - maxConnections = props.maxConnections, - readAheadSegments = props.readAheadSegments ?: props.concurrency - ), - SeekConfig( - forwardThresholdBytes = props.forwardThresholdBytes + host = pool.host, + port = pool.port, + username = pool.username, + password = pool.password, + useTls = pool.useTls, + maxConnections = pool.maxConnections, + priority = pool.priority ) - ) + } } } diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/UsenetDownload.kt b/src/main/kotlin/io/skjaere/debridav/usenet/UsenetDownload.kt index c79dc403..21353c55 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/UsenetDownload.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/UsenetDownload.kt @@ -9,11 +9,14 @@ import jakarta.persistence.FetchType import jakarta.persistence.GeneratedValue import jakarta.persistence.GenerationType import jakarta.persistence.Id +import jakarta.persistence.Index import jakarta.persistence.JoinColumn import jakarta.persistence.ManyToOne import jakarta.persistence.OneToMany +import jakarta.persistence.Table @Entity +@Table(indexes = [Index(name = "idx_usenet_download_category_id", columnList = "category_id")]) open class UsenetDownload { @Id @GeneratedValue(strategy = GenerationType.AUTO) @@ -37,9 +40,19 @@ open class UsenetDownload { @OneToMany( targetEntity = RemotelyCachedEntity::class, cascade = [CascadeType.PERSIST, CascadeType.MERGE], - fetch = FetchType.EAGER, + fetch = FetchType.LAZY, ) open var debridFiles: MutableList = mutableListOf() + + // Equality on the business key (NZB-bytes md5). `is UsenetDownload` matches + // Hibernate proxies as well as real instances. + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other !is UsenetDownload) return false + return hash != null && hash == other.hash + } + + override fun hashCode(): Int = hash?.hashCode() ?: 0 } enum class UsenetDownloadStatus { diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/Messages.kt b/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/Messages.kt index 787e08be..9fb83372 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/Messages.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/Messages.kt @@ -2,7 +2,8 @@ package io.skjaere.debridav.usenet.pgmq data class NzbImportMessage( val nzbBytesBase64: String, - val usenetDownloadId: Long + val usenetDownloadId: Long, + val nzbImportRecordId: Long ) data class NzbHealthCheckMessage( diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/NzbHealthCheckHandler.kt b/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/NzbHealthCheckHandler.kt index 6bba95e9..7df66c79 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/NzbHealthCheckHandler.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/NzbHealthCheckHandler.kt @@ -1,25 +1,27 @@ package io.skjaere.debridav.usenet.pgmq import com.vdsirotkin.pgmq.PgmqClient +import io.skjaere.debridav.health.HealthMetrics +import io.skjaere.debridav.health.HealthMetrics.CheckResult +import io.skjaere.debridav.health.HealthMetrics.HealthType import io.skjaere.debridav.repository.NzbDocumentRepository import io.skjaere.debridav.usenet.nzb.toNzbDocument import io.skjaere.nzbstreamer.NzbStreamer import io.skjaere.nzbstreamer.enrichment.VerificationResult import kotlinx.coroutines.runBlocking import org.slf4j.LoggerFactory -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional import java.time.Clock import java.time.Instant @Service -@ConditionalOnProperty("nntp.enabled", havingValue = "true") class NzbHealthCheckHandler( private val nzbStreamer: NzbStreamer, private val nzbDocumentRepository: NzbDocumentRepository, private val pgmqClient: PgmqClient, - private val clock: Clock + private val clock: Clock, + private val healthMetrics: HealthMetrics, ) { private val logger = LoggerFactory.getLogger(NzbHealthCheckHandler::class.java) @@ -28,42 +30,49 @@ class NzbHealthCheckHandler( val entity = nzbDocumentRepository.findById(msg.nzbDocumentId).orElse(null) if (entity == null) { logger.warn("NZB document {} not found, skipping health check", msg.nzbDocumentId) + healthMetrics.recordCheck(HealthType.NZB, CheckResult.NOT_FOUND) return } - try { - val nzbDocument = entity.toNzbDocument() - when (val result = runBlocking { nzbStreamer.verifySegments(nzbDocument) }) { - is VerificationResult.Success -> { - logger.debug("NZB document {} verified successfully", entity.id) - } + healthMetrics.timeCheck(HealthType.NZB) { + try { + val nzbDocument = entity.toNzbDocument() + when (val result = runBlocking { nzbStreamer.verifySegments(nzbDocument) }) { + is VerificationResult.Success -> { + logger.debug("NZB document {} verified successfully", entity.id) + healthMetrics.recordCheck(HealthType.NZB, CheckResult.OK) + } - is VerificationResult.MissingArticles -> { - logger.warn( - "NZB document {} has missing articles: {}", - entity.id, - result.message - ) - pgmqClient.send( - "nzb_health_repair", - NzbHealthRepairMessage( - nzbDocumentId = entity.id!!, - message = result.message + is VerificationResult.MissingArticles -> { + logger.warn( + "NZB document {} has missing articles: {}", + entity.id, + result.message ) - ) - } + healthMetrics.recordCheck(HealthType.NZB, CheckResult.MISSING) + pgmqClient.send( + "nzb_health_repair", + NzbHealthRepairMessage( + nzbDocumentId = entity.id!!, + message = result.message + ) + ) + } - is VerificationResult.Failure -> { - logger.error( - "NZB document {} verification failed: {}", - entity.id, - result.message, - result.cause - ) + is VerificationResult.Failure -> { + logger.error( + "NZB document {} verification failed: {}", + entity.id, + result.message, + result.cause + ) + healthMetrics.recordCheck(HealthType.NZB, CheckResult.FAILURE) + } } + } catch (@Suppress("TooGenericExceptionCaught") e: Exception) { + logger.error("Unexpected error verifying NZB document {}", entity.id, e) + healthMetrics.recordCheck(HealthType.NZB, CheckResult.FAILURE) } - } catch (@Suppress("TooGenericExceptionCaught") e: Exception) { - logger.error("Unexpected error verifying NZB document {}", entity.id, e) } entity.lastVerified = Instant.now(clock) diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/NzbHealthRepairHandler.kt b/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/NzbHealthRepairHandler.kt index d2f81f3d..3072f1de 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/NzbHealthRepairHandler.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/NzbHealthRepairHandler.kt @@ -1,27 +1,58 @@ package io.skjaere.debridav.usenet.pgmq import io.skjaere.debridav.arrs.ArrService +import io.skjaere.debridav.fs.DatabaseFileService +import io.skjaere.debridav.health.HealthCheckConfigurationProperties +import io.skjaere.debridav.health.HealthMetrics +import io.skjaere.debridav.health.HealthMetrics.HealthType +import io.skjaere.debridav.health.RepairAction +import io.skjaere.debridav.health.RepairOutcomeService import io.skjaere.debridav.repository.NzbDocumentRepository +import io.skjaere.debridav.repository.UsenetRepository +import io.skjaere.debridav.usenet.nzb.NzbDocumentEntity import kotlinx.coroutines.runBlocking import org.slf4j.LoggerFactory -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.stereotype.Service +import org.springframework.transaction.annotation.Transactional @Service -@ConditionalOnProperty("nntp.enabled", havingValue = "true") class NzbHealthRepairHandler( private val nzbDocumentRepository: NzbDocumentRepository, - private val arrService: ArrService + private val usenetRepository: UsenetRepository, + private val arrService: ArrService, + private val fileService: DatabaseFileService, + private val healthCheckConfig: HealthCheckConfigurationProperties, + private val repairOutcomeService: RepairOutcomeService, + private val healthMetrics: HealthMetrics, ) { private val logger = LoggerFactory.getLogger(NzbHealthRepairHandler::class.java) - fun handle(msg: NzbHealthRepairMessage) { + @Transactional + fun handle(msg: NzbHealthRepairMessage, msgId: Long) { + if (!healthCheckConfig.repairEnabled) { + logger.debug("Repair is disabled, skipping NZB document {}", msg.nzbDocumentId) + repairOutcomeService.record(QUEUE_NAME, msgId, RepairAction.SKIPPED) + healthMetrics.recordRepair(HealthType.NZB, RepairAction.SKIPPED.name) + return + } + val nzbDocument = nzbDocumentRepository.findById(msg.nzbDocumentId).orElse(null) if (nzbDocument == null) { logger.warn("No NzbDocument found for ID {}", msg.nzbDocumentId) + healthMetrics.recordRepair(HealthType.NZB, "NOT_FOUND") return } + healthMetrics.timeRepair(HealthType.NZB) { + val action = executeRepair(msgId, nzbDocument) + healthMetrics.recordRepair(HealthType.NZB, action.name) + } + } + + private fun executeRepair( + msgId: Long, + nzbDocument: NzbDocumentEntity + ): RepairAction { val category = nzbDocument.category val name = nzbDocument.name if (category == null || name == null) { @@ -29,17 +60,17 @@ class NzbHealthRepairHandler( "NzbDocument {} missing category or name, cannot notify Arr", nzbDocument.id ) - return + deleteVirtualFiles(nzbDocument.id!!) + repairOutcomeService.record(QUEUE_NAME, msgId, RepairAction.DELETED) + return RepairAction.DELETED } - if (arrService.getClientForCategory(category) != null) { + return if (arrService.getClientForCategory(category) != null) { val downloadId = nzbDocument.downloadId if (downloadId != null) { logger.info( "Blocklisting downloadId '{}' for '{}' (category: {})", - downloadId, - name, - category + downloadId, name, category ) runBlocking { arrService.blocklist(downloadId, category) } } else { @@ -51,10 +82,45 @@ class NzbHealthRepairHandler( logger.info( "Notifying Arr to delete file and search for '{}' (category: {})", - name, - category + name, category ) - runBlocking { arrService.deleteFileAndSearch(name, category) } + val found = runBlocking { arrService.deleteFileAndSearch(name, category) } + val action = if (!found) { + logger.info( + "Arr could not find '{}', deleting from virtual filesystem", + name + ) + deleteVirtualFiles(nzbDocument.id!!) + RepairAction.DELETED + } else { + RepairAction.REPAIRED + } + repairOutcomeService.record(QUEUE_NAME, msgId, action) + action + } else { + logger.info( + "No Arr client for NZB {} (category: {}), deleting files from virtual filesystem", + nzbDocument.id, category + ) + deleteVirtualFiles(nzbDocument.id!!) + repairOutcomeService.record(QUEUE_NAME, msgId, RepairAction.DELETED) + RepairAction.DELETED } } + + private fun deleteVirtualFiles(nzbDocumentId: Long) { + val usenetDownload = usenetRepository.findByNzbDocumentId(nzbDocumentId) + if (usenetDownload != null) { + usenetDownload.debridFiles.forEach { file -> + logger.info("Deleting '{}' from virtual filesystem", file.name) + fileService.deleteFile(file) + } + } else { + logger.warn("No UsenetDownload found for NzbDocument {}, cannot delete virtual files", nzbDocumentId) + } + } + + companion object { + const val QUEUE_NAME = "nzb_health_repair" + } } diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqArchiveCleanupService.kt b/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqArchiveCleanupService.kt new file mode 100644 index 00000000..f90a9fbd --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqArchiveCleanupService.kt @@ -0,0 +1,80 @@ +package io.skjaere.debridav.usenet.pgmq + +import io.skjaere.debridav.pgmq.PgmqConfigurationProperties +import org.slf4j.LoggerFactory +import org.springframework.jdbc.core.JdbcTemplate +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Service +import java.sql.Timestamp +import java.time.Instant + +@Service +class PgmqArchiveCleanupService( + private val jdbc: JdbcTemplate, + private val props: PgmqConfigurationProperties +) { + private val logger = LoggerFactory.getLogger(PgmqArchiveCleanupService::class.java) + + companion object { + private val MAIN_QUEUES = listOf( + "nzb_import", "nzb_health_check", "nzb_health_repair", + "torrent_health_check", "torrent_health_repair" + ) + private val DEAD_LETTER_QUEUES = MAIN_QUEUES.map { "${it}_dlq" } + private val ALL_QUEUES = MAIN_QUEUES + DEAD_LETTER_QUEUES + } + + /** Drops rows from `pgmq.a_` (archive tables) older than [archiveRetention]. */ + @Scheduled(fixedDelayString = "PT1H", initialDelayString = "PT1M") + fun cleanupArchivedMessages() { + val cutoff = Instant.now().minus(props.archiveRetention) + logger.debug("Cleaning up archived PGMQ messages older than {}", cutoff) + + var totalDeleted = 0L + for (queueName in ALL_QUEUES) { + val deleted = jdbc.update( + "DELETE FROM pgmq.a_$queueName WHERE archived_at < ?", + Timestamp.from(cutoff) + ) + if (deleted > 0) { + logger.info("Deleted {} archived messages from queue '{}'", deleted, queueName) + totalDeleted += deleted + } + } + + if (totalDeleted > 0) { + logger.info("Archive cleanup complete: {} total messages removed", totalDeleted) + } else { + logger.debug("Archive cleanup complete: no expired messages found") + } + } + + /** + * Ages out live dead-letter messages: anything sitting in a `*_dlq` queue beyond + * [deadLetterRetention] gets archived so the existing [cleanupArchivedMessages] loop + * can eventually reap it. Gives operators a retention window to investigate poison + * payloads without letting the DLQ grow unbounded. + */ + @Scheduled(fixedDelayString = "PT1H", initialDelayString = "PT2M") + fun archiveStaleDeadLetterMessages() { + val cutoff = Instant.now().minus(props.deadLetterRetention) + logger.debug("Archiving dead-letter messages enqueued before {}", cutoff) + + var totalArchived = 0L + for (queueName in DEAD_LETTER_QUEUES) { + val archived = jdbc.query( + "SELECT pgmq.archive(?, msg_id) AS archived FROM pgmq.q_$queueName WHERE enqueued_at < ?", + { rs, _ -> rs.getBoolean("archived") }, + queueName, Timestamp.from(cutoff) + ).count { it } + if (archived > 0) { + logger.info("Archived {} stale dead-letter messages from queue '{}'", archived, queueName) + totalArchived += archived + } + } + + if (totalArchived > 0) { + logger.info("Dead-letter archive complete: {} total messages moved to archive", totalArchived) + } + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqSpringConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqSpringConfiguration.kt index bec0ff65..da401eb2 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqSpringConfiguration.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqSpringConfiguration.kt @@ -1,65 +1,17 @@ package io.skjaere.debridav.usenet.pgmq import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.kotlin.KotlinModule import com.vdsirotkin.pgmq.PgmqClient -import com.vdsirotkin.pgmq.config.PgmqConfiguration -import com.vdsirotkin.pgmq.config.PgmqConnectionFactory -import com.vdsirotkin.pgmq.serialization.JacksonPgmqSerializationProvider +import io.skjaere.debridav.pgmq.PgmqConfigurationProperties +import io.skjaere.debridav.pgmq.PgmqConsumer import io.skjaere.debridav.usenet.NzbImportService import io.skjaere.debridav.usenet.NzbImportTaskData -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty -import org.springframework.boot.context.properties.ConfigurationProperties import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration -import java.time.Duration -import javax.sql.DataSource - -@Suppress("MagicNumber") -@ConfigurationProperties(prefix = "pgmq") -data class PgmqConfigurationProperties( - val defaultVisibilityTimeout: Duration = Duration.ofMinutes(5), - val importConcurrency: Int = 2, - val importVisibilityTimeout: Duration = Duration.ofMinutes(10), - val importPollInterval: Duration = Duration.ofSeconds(2), - val healthCheckConcurrency: Int = 1, - val healthCheckVisibilityTimeout: Duration = Duration.ofMinutes(5), - val healthCheckPollInterval: Duration = Duration.ofSeconds(10), - val healthRepairConcurrency: Int = 2, - val healthRepairVisibilityTimeout: Duration = Duration.ofMinutes(2), - val healthRepairPollInterval: Duration = Duration.ofSeconds(5) -) @Configuration -@ConditionalOnProperty("nntp.enabled", havingValue = "true") class PgmqSpringConfiguration { - @Bean - fun pgmqConfiguration(props: PgmqConfigurationProperties): PgmqConfiguration = - object : PgmqConfiguration { - override val defaultVisibilityTimeout: java.time.Duration = props.defaultVisibilityTimeout - } - - @Bean - fun pgmqConnectionFactory(dataSource: DataSource): PgmqConnectionFactory = PgmqConnectionFactory { - dataSource.connection - } - - @Bean - fun pgmqObjectMapper(): ObjectMapper = - ObjectMapper().registerModule(KotlinModule.Builder().build()) - - @Bean - fun pgmqSerializationProvider(pgmqObjectMapper: ObjectMapper): JacksonPgmqSerializationProvider = - JacksonPgmqSerializationProvider(pgmqObjectMapper) - - @Bean - fun pgmqClient( - connectionFactory: PgmqConnectionFactory, - serializationProvider: JacksonPgmqSerializationProvider, - configuration: PgmqConfiguration - ): PgmqClient = PgmqClient(connectionFactory, serializationProvider, configuration) - @Bean fun nzbImportConsumer( pgmqClient: PgmqClient, @@ -73,10 +25,11 @@ class PgmqSpringConfiguration { messageType = NzbImportMessage::class.java, concurrency = props.importConcurrency, visibilityTimeout = props.importVisibilityTimeout, - pollInterval = props.importPollInterval - ) { msg -> + pollInterval = props.importPollInterval, + maxReadCount = props.maxReadCount + ) { msg, _ -> nzbImportService.executeImport( - NzbImportTaskData(msg.nzbBytesBase64, msg.usenetDownloadId) + NzbImportTaskData(msg.nzbBytesBase64, msg.usenetDownloadId, msg.nzbImportRecordId) ) } @@ -93,8 +46,9 @@ class PgmqSpringConfiguration { messageType = NzbHealthCheckMessage::class.java, concurrency = props.healthCheckConcurrency, visibilityTimeout = props.healthCheckVisibilityTimeout, - pollInterval = props.healthCheckPollInterval - ) { msg -> + pollInterval = props.healthCheckPollInterval, + maxReadCount = props.maxReadCount + ) { msg, _ -> healthCheckHandler.handle(msg) } @@ -111,8 +65,9 @@ class PgmqSpringConfiguration { messageType = NzbHealthRepairMessage::class.java, concurrency = props.healthRepairConcurrency, visibilityTimeout = props.healthRepairVisibilityTimeout, - pollInterval = props.healthRepairPollInterval - ) { msg -> - healthRepairHandler.handle(msg) + pollInterval = props.healthRepairPollInterval, + maxReadCount = props.maxReadCount + ) { msg, msgId -> + healthRepairHandler.handle(msg, msgId) } } diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/queue/NzbImportRecord.kt b/src/main/kotlin/io/skjaere/debridav/usenet/queue/NzbImportRecord.kt new file mode 100644 index 00000000..ba618f8f --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/usenet/queue/NzbImportRecord.kt @@ -0,0 +1,77 @@ +package io.skjaere.debridav.usenet.queue + +import io.hypersistence.utils.hibernate.type.json.JsonBinaryType +import jakarta.persistence.Column +import jakarta.persistence.Entity +import jakarta.persistence.EnumType +import jakarta.persistence.Enumerated +import jakarta.persistence.GeneratedValue +import jakarta.persistence.GenerationType +import jakarta.persistence.Id +import jakarta.persistence.PrePersist +import jakarta.persistence.PreUpdate +import jakarta.persistence.Table +import org.hibernate.annotations.Type +import java.time.Instant + +@Entity +@Table(name = "nzb_import") +open class NzbImportRecord { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + open var id: Long? = null + + @Column(name = "usenet_download_id") + open var usenetDownloadId: Long? = null + + @Column(nullable = false) + open var name: String = "" + + open var category: String? = null + + @Enumerated(EnumType.STRING) + @Column(nullable = false, length = 30) + open var status: NzbImportStatus = NzbImportStatus.QUEUED + + @Column(name = "archive_type", length = 30) + open var archiveType: String? = null + + @Column(name = "error_message", columnDefinition = "TEXT") + open var errorMessage: String? = null + + @Type(JsonBinaryType::class) + @Column(name = "files", columnDefinition = "jsonb") + open var files: List? = null + + open var size: Long? = null + + @Column(name = "created_at") + open var createdAt: Instant? = null + + @Column(name = "updated_at") + open var updatedAt: Instant? = null + + @PrePersist + fun onPrePersist() { + val now = Instant.now() + createdAt = now + updatedAt = now + } + + @PreUpdate + fun onPreUpdate() { + updatedAt = Instant.now() + } +} + +enum class NzbImportStatus { + QUEUED, + IMPORTING, + COMPLETED, + FAILED +} + +data class NzbImportFileJson( + val path: String, + val size: Long +) : java.io.Serializable diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/queue/QueueItemDto.kt b/src/main/kotlin/io/skjaere/debridav/usenet/queue/QueueItemDto.kt new file mode 100644 index 00000000..3bd33a06 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/usenet/queue/QueueItemDto.kt @@ -0,0 +1,30 @@ +package io.skjaere.debridav.usenet.queue + +import java.time.Instant + +data class QueueItemDto( + val id: Long, + val name: String, + val status: String, + val size: Long?, + val errorMessage: String?, + val updatedAt: Instant?, + val createdAt: Instant?, + val archiveType: String? = null, + val files: List? = null +) + +data class QueueStatusResponse( + val processing: List, + val pending: List, + val history: List +) + +data class HistoryPageResponse( + val content: List, + val page: Int, + val size: Int, + val totalElements: Long, + val totalPages: Int, + val last: Boolean +) diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/queue/UsenetQueueController.kt b/src/main/kotlin/io/skjaere/debridav/usenet/queue/UsenetQueueController.kt new file mode 100644 index 00000000..17293b36 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/usenet/queue/UsenetQueueController.kt @@ -0,0 +1,33 @@ +package io.skjaere.debridav.usenet.queue + +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RequestParam +import org.springframework.web.bind.annotation.RestController + +@RestController +@RequestMapping("/api/v1/queue") +class UsenetQueueController(private val queueService: UsenetQueueService) { + @GetMapping + fun getQueueStatus(): ResponseEntity = + ResponseEntity.ok(queueService.getQueueStatus()) + + @GetMapping("/history") + fun getHistory( + @RequestParam(defaultValue = "0") page: Int, + @RequestParam(defaultValue = "20") size: Int, + @RequestParam(defaultValue = "") search: String, + @RequestParam(defaultValue = "updatedAt") sort: String, + @RequestParam(defaultValue = "desc") direction: String + ): ResponseEntity = + ResponseEntity.ok(queueService.getHistory(page, size, search, sort, direction)) + + @GetMapping("/{id}/files") + fun getItemFiles(@PathVariable id: Long): ResponseEntity> { + val files = queueService.resolveCurrentFilePaths(id) + ?: return ResponseEntity.notFound().build() + return ResponseEntity.ok(files) + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/queue/UsenetQueueService.kt b/src/main/kotlin/io/skjaere/debridav/usenet/queue/UsenetQueueService.kt new file mode 100644 index 00000000..69d751b7 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/usenet/queue/UsenetQueueService.kt @@ -0,0 +1,93 @@ +package io.skjaere.debridav.usenet.queue + +import io.skjaere.debridav.repository.DebridFileContentsRepository +import io.skjaere.debridav.repository.NzbImportRepository +import org.springframework.data.domain.PageRequest +import org.springframework.data.domain.Sort +import org.springframework.stereotype.Service + +@Service +class UsenetQueueService( + private val nzbImportRepository: NzbImportRepository, + private val debridFileContentsRepository: DebridFileContentsRepository +) { + + companion object { + val PROCESSING_STATUSES = listOf( + NzbImportStatus.IMPORTING + ) + + val PENDING_STATUSES = listOf( + NzbImportStatus.QUEUED + ) + + val HISTORY_STATUSES = listOf( + NzbImportStatus.COMPLETED, + NzbImportStatus.FAILED + ) + + private val ALLOWED_SORT_FIELDS = setOf("updatedAt", "name") + } + + fun getQueueStatus(): QueueStatusResponse { + val processing = nzbImportRepository.findByStatusInOrderByUpdatedAtDesc(PROCESSING_STATUSES) + .map { it.toDto() } + val pending = nzbImportRepository.findByStatusInOrderByIdAsc(PENDING_STATUSES) + .map { it.toDto() } + + return QueueStatusResponse( + processing = processing, + pending = pending, + history = emptyList() + ) + } + + fun getHistory(page: Int, size: Int, search: String, sort: String, direction: String): HistoryPageResponse { + val sortField = if (sort in ALLOWED_SORT_FIELDS) sort else "updatedAt" + val sortDir = if (direction.equals("asc", ignoreCase = true)) Sort.Direction.ASC else Sort.Direction.DESC + val pageResult = nzbImportRepository.findByStatusInAndNameSearch( + HISTORY_STATUSES, + search, + PageRequest.of(page, size, Sort.by(sortDir, sortField)) + ) + return HistoryPageResponse( + content = pageResult.content.map { it.toDto() }, + page = pageResult.number, + size = pageResult.size, + totalElements = pageResult.totalElements, + totalPages = pageResult.totalPages, + last = pageResult.isLast + ) + } + + @Suppress("ReturnCount") + fun resolveCurrentFilePaths(importId: Long): List? { + val record = nzbImportRepository.findById(importId).orElse(null) ?: return null + val usenetDownloadId = record.usenetDownloadId ?: return record.files + val dbItems = debridFileContentsRepository.findByUsenetDownloadId(usenetDownloadId) + if (dbItems.isEmpty()) return record.files + val currentFiles = dbItems.mapNotNull { entity -> + val dirPath = entity.directory?.fileSystemPath() ?: return@mapNotNull null + val fileName = entity.name ?: return@mapNotNull null + NzbImportFileJson( + path = "$dirPath/$fileName", + size = entity.size ?: 0L + ) + } + return currentFiles.ifEmpty { record.files } + } + + private fun NzbImportRecord.toDto(): QueueItemDto { + return QueueItemDto( + id = id!!, + name = name, + status = status.name, + size = size, + errorMessage = errorMessage, + updatedAt = updatedAt, + createdAt = createdAt, + archiveType = archiveType, + files = files + ) + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/sabnzbd/SabNzbdService.kt b/src/main/kotlin/io/skjaere/debridav/usenet/sabnzbd/SabNzbdService.kt index fbeddd2b..1b425904 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/sabnzbd/SabNzbdService.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/sabnzbd/SabNzbdService.kt @@ -3,14 +3,19 @@ package io.skjaere.debridav.usenet.sabnzbd import io.skjaere.debridav.category.CategoryService import io.skjaere.debridav.configuration.DebridavConfigurationProperties import io.skjaere.debridav.debrid.DebridCachedContentService +import io.skjaere.debridav.debrid.DebridProvider import io.skjaere.debridav.debrid.UsenetRelease import io.skjaere.debridav.fs.DatabaseFileService import io.skjaere.debridav.fs.DebridFileContents import io.skjaere.debridav.fs.RemotelyCachedEntity +import io.skjaere.debridav.repository.NzbImportRepository import io.skjaere.debridav.repository.UsenetRepository +import io.skjaere.debridav.usenet.NntpConfigurationProperties import io.skjaere.debridav.usenet.NzbImportService import io.skjaere.debridav.usenet.UsenetDownload import io.skjaere.debridav.usenet.UsenetDownloadStatus +import io.skjaere.debridav.usenet.queue.NzbImportRecord +import io.skjaere.debridav.usenet.queue.NzbImportStatus import io.skjaere.debridav.usenet.sabnzbd.model.HistorySlot import io.skjaere.debridav.usenet.sabnzbd.model.ListResponseDownloadSlot import io.skjaere.debridav.usenet.sabnzbd.model.Queue @@ -22,8 +27,6 @@ import io.skjaere.debridav.usenet.sabnzbd.model.SabnzbdFullListResponse import io.skjaere.debridav.usenet.sabnzbd.model.SabnzbdHistory import io.skjaere.debridav.usenet.sabnzbd.model.SabnzbdHistoryResponse import jakarta.transaction.Transactional -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonArray import kotlinx.serialization.json.JsonObject @@ -33,24 +36,33 @@ import org.apache.commons.codec.digest.DigestUtils import org.slf4j.LoggerFactory import org.springframework.core.convert.ConversionService import org.springframework.core.io.ResourceLoader +import org.springframework.data.domain.PageRequest import org.springframework.stereotype.Service import org.springframework.web.multipart.MultipartFile import java.io.InputStream +private const val MAX_HISTORY_SLOTS = 500 + @Service @Suppress("LongParameterList") class SabNzbdService( private val cachedContentService: DebridCachedContentService, private val fileService: DatabaseFileService, private val debridavConfigurationProperties: DebridavConfigurationProperties, + private val nntpConfigurationProperties: NntpConfigurationProperties, private val usenetRepository: UsenetRepository, private val usenetConversionService: ConversionService, private val categoryService: CategoryService, private val resourceLoader: ResourceLoader, - private val nzbImportService: NzbImportService? + private val nzbImportService: NzbImportService, + private val nzbImportRepository: NzbImportRepository ) { private val logger = LoggerFactory.getLogger(SabNzbdService::class.java) + private fun isEasynewsOnlySetup(): Boolean = + DebridProvider.EASYNEWS in debridavConfigurationProperties.debridClients + && nntpConfigurationProperties.pools.none { it.host.isNotBlank() } + @Transactional suspend fun addNzbFile(request: SabnzbdApiRequest): UsenetDownload { val multipartFile = request.name as MultipartFile @@ -58,21 +70,27 @@ class SabNzbdService( val nzbBytes = multipartFile.bytes val hash = nzbBytes.inputStream().md5() - if (nzbImportService != null) { - val usenetDownload = createQueuedUsenetDownload(releaseName, hash, request.cat!!) - nzbImportService.scheduleImport(nzbBytes, usenetDownload) - return usenetDownload + if (isEasynewsOnlySetup()) { + val debridFiles = cachedContentService.addContent(UsenetRelease(releaseName)) + return if (debridFiles.isNotEmpty()) { + val savedDebridFiles = createDebridFilesFromDebridResponse(debridFiles, hash, releaseName) + createCachedUsenetDownload(releaseName, hash, request.cat!!, savedDebridFiles) + } else { + logger.debug("$releaseName is not cached in any available debrid services") + createFailedUsenetDownload(releaseName, hash, request.cat!!) + } } - val debridFiles = cachedContentService.addContent(UsenetRelease(releaseName)) - - return if (debridFiles.isNotEmpty()) { - val savedDebridFiles = createDebridFilesFromDebridResponse(debridFiles, hash, releaseName) - createCachedUsenetDownload(releaseName, hash, request.cat!!, savedDebridFiles) - } else { - logger.debug("$releaseName is not cached in any available debrid services") - createFailedUsenetDownload(releaseName, hash, request.cat!!) + val usenetDownload = createQueuedUsenetDownload(releaseName, hash, request.cat!!) + val importRecord = NzbImportRecord().apply { + usenetDownloadId = usenetDownload.id + name = releaseName + category = request.cat + status = NzbImportStatus.QUEUED } + val savedRecord = nzbImportRepository.save(importRecord) + nzbImportService.scheduleImport(nzbBytes, usenetDownload, savedRecord.id!!) + return usenetDownload } fun InputStream.md5(): String = this.use { inputStream -> @@ -85,10 +103,13 @@ class SabNzbdService( usenetRepository.deleteUsenetDownloadById(request.value!!.toLong()) return SabNzbdHistoryDeleteResponse(true, listOf(request.value)) } else { + // Sonarr/Radarr poll history but only need recent entries to match against + // their pending grabs. Cap at MAX_HISTORY_SLOTS (newest first) so the + // table doesn't OOM us when it grows over time. + val pageable = PageRequest.of(0, MAX_HISTORY_SLOTS) val slots = request.cat?.let { - usenetRepository - .findByCategoryName(request.cat) - } ?: usenetRepository.findAll() + usenetRepository.findRecentByCategoryName(it, pageable) + } ?: usenetRepository.findRecent(pageable) val filteredSlots = slots .filter { it.status?.isCompleted() == true } @@ -104,34 +125,33 @@ class SabNzbdService( } @Transactional - suspend fun queue(request: SabnzbdApiRequest): SabNzbdQueueResponse = withContext(Dispatchers.IO) { + suspend fun queue(request: SabnzbdApiRequest): SabNzbdQueueResponse { if (request.name is String && request.name == "delete") { usenetRepository.findById(request.value!!.toLong()).get().let { usenetDownload -> usenetRepository.markUsenetDownloadAsDeleted(usenetDownload) } - SabNzbdQueueDeleteResponse(true, listOf(request.value)) - } else { - val queueSlots = emptyList() - val queue = Queue( - status = "Downloading", - speedLimit = "0", - speedLimitAbs = "0", - paused = false, - noofSlots = 0, - noofSlotsTotal = 0, - limit = 0, - start = 0, - timeLeft = "0:10:0", - speed = "1 M", - kbPerSec = "100.0", - size = "0", - sizeLeft = "0", - mb = "0", - mbLeft = "0", - slots = queueSlots - ) - SabnzbdFullListResponse(queue) + return SabNzbdQueueDeleteResponse(true, listOf(request.value)) } + val queueSlots = emptyList() + val queue = Queue( + status = "Downloading", + speedLimit = "0", + speedLimitAbs = "0", + paused = false, + noofSlots = 0, + noofSlotsTotal = 0, + limit = 0, + start = 0, + timeLeft = "0:10:0", + speed = "1 M", + kbPerSec = "100.0", + size = "0", + sizeLeft = "0", + mb = "0", + mbLeft = "0", + slots = queueSlots + ) + return SabnzbdFullListResponse(queue) } fun config(): String { @@ -171,14 +191,13 @@ class SabNzbdService( debridFiles: List, hash: String, releaseName: String - ): List = - debridFiles.map { file -> - fileService.createDebridFile( - "${debridavConfigurationProperties.downloadPath}/${releaseName}/${file.originalPath}", - hash, - file - ) - } + ): List { + val basePath = "${debridavConfigurationProperties.downloadPath}/$releaseName" + return fileService.createDebridFiles( + debridFiles.map { "$basePath/${it.originalPath}" to it }, + hash, + ) + } private suspend fun createQueuedUsenetDownload( @@ -221,19 +240,19 @@ class SabNzbdService( hash: String, category: String, createdFiles: List - ): UsenetDownload = withContext(Dispatchers.IO) { - val category = categoryService.getOrCreateCategory(category) + ): UsenetDownload { + val resolvedCategory = categoryService.getOrCreateCategory(category) val usenetDownload = UsenetDownload() usenetDownload.status = UsenetDownloadStatus.COMPLETED usenetDownload.name = releaseName usenetDownload.hash = hash - usenetDownload.category = category + usenetDownload.category = resolvedCategory usenetDownload.storagePath = "${debridavConfigurationProperties.mountPath}${debridavConfigurationProperties.downloadPath}/$releaseName" usenetDownload.percentCompleted = 1.0 usenetDownload.size = createdFiles.first().size usenetDownload.debridFiles.addAll(createdFiles) - usenetRepository.save(usenetDownload) + return usenetRepository.save(usenetDownload) } } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties deleted file mode 100644 index 58a6f866..00000000 --- a/src/main/resources/application.properties +++ /dev/null @@ -1,103 +0,0 @@ -#spring.jpa.generate-ddl=true -#spring.jpa.hibernate.ddl-auto=update -spring.jpa.properties.hibernate.event.merge.entity_copy_observer=allow -logging.level.web=info -spring.servlet.multipart.max-file-size=-1 -spring.servlet.multipart.max-request-size=-1 -management.endpoints.web.exposure.include=health,realdebrid,prometheus,nzbhealthcheck -management.endpoint.health.group.readiness.include=fileSystemImportService -management.endpoint.health.group.liveness.exclude=fileSystemImportService -debridav.root-path=${user.dir}/debridav-files -debridav.download-path=/downloads -debridav.mount-path=/data -debridav.debrid-clients= -debridav.delay-between-retries=200ms -debridav.retries-on-provider-error=1 -debridav.wait-after-missing=24h -debridav.wait-after-network-error=1000ms -debridav.wait-after-provider-error=10m -debridav.wait-after-client-error=1000ms -debridav.should-delete-non-working-files=true -debridav.connect-timeout-milliseconds=5000 -debridav.read-timeout-milliseconds=5000 -debridav.enable-file-import-on-startup=true -debridav.local-entity-max-size-mb=130 -debridav.default-categories= -debridav.torrent-lifetime=1d -# WebDAV Authentication (empty = disabled) -debridav.webdav-username= -debridav.webdav-password= -# Database -debridav.db.host=localhost -debridav.db.port=5432 -debridav.db.database-name=debridav -spring.datasource.username=debridav -spring.datasource.password=debridav -# five hours -spring.datasource.hikari.max-lifetime=180000000 -spring.datasource.hikari.idle-timeout=0 -spring.datasource.url=jdbc:postgresql://${debridav.db.host}:5432/debridav -spring.datasource.hikari.maximum-pool-size=5 -# Premiumize -premiumize.api-key= -premiumize.bas-eurl=https://www.premiumize.me/api -# Real-Debrid -real-debrid.api-key= -real-debrid.base-url=https://api.real-debrid.com/rest/1.0 -real-debrid.sync-enabled=true -real-debrid.sync-poll-rate=PT24H -# TorBox -torbox.api-key= -torbox.base-url=https://api.torbox.app -torbox.version=v1 -torbox.request-timeout-millis=10000 -torbox.socket-timeout-millis=10000 -logging.level.io.milton.http.*=error -# Easynews -easynews.username= -easynews.password= -easynews.api-base-url=https://members.easynews.com -easynews.enabled-for-torrents=true -easynews.rate-limit-window-duration=15s -easynews.allowed-requests-in-window=10 -easynews.connect-timeout=20000 -easynews.socket-timeout=5000 -# Sonarr -sonarr.integration-enabled=false -sonarr.host=localhost -sonarr.port=8990 -sonarr.api-base-path=/api/v3 -sonarr.api-key=1105779a7abb40898567b406442cd927 -sonarr.category=tv-sonarr -radarr.integration-enabled=false -radarr.host=localhost -radarr.port=7878 -radarr.api-base-path=/api/v3 -radarr.api-key=8d273d4f92294234a9cdddba605054e1 -radarr.category=radarr -# NNTP -nntp.enabled=false -#nntp.host= -nntp.port=563 -nntp.username= -nntp.password= -nntp.use-tls=true -nntp.concurrency=4 -nntp.max-connections=60 -nntp.forward-threshold-bytes=102400 -nntp.health-check-interval=7d -nntp.health-check-poll-rate=PT5M -# PGMQ -pgmq.default-visibility-timeout=5m -pgmq.import-concurrency=2 -pgmq.import-visibility-timeout=10m -pgmq.import-poll-interval=2s -pgmq.health-check-concurrency=1 -pgmq.health-check-visibility-timeout=5m -pgmq.health-check-poll-interval=10s -pgmq.health-repair-concurrency=2 -pgmq.health-repair-visibility-timeout=2m -pgmq.health-repair-poll-interval=5s -# Sentry / GlitchTip -sentry.send-default-pii=false -sentry.traces-sample-rate=0 diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml new file mode 100644 index 00000000..b0d6d471 --- /dev/null +++ b/src/main/resources/application.yaml @@ -0,0 +1,150 @@ +#spring: +# jpa: +# generate-ddl: true +# hibernate: +# ddl-auto: update + +spring: + jpa: + open-in-view: false + properties: + hibernate: + # Batch the per-file inserts done when adding a torrent or NZB so the + # JSONB DebridFileContents column is round-tripped to Postgres in one + # statement instead of N. order_inserts/updates groups the same-table + # writes contiguously, which is what makes batching actually trigger. + jdbc.batch_size: 50 + order_inserts: true + order_updates: true + servlet: + multipart: + max-file-size: -1 + max-request-size: -1 + datasource: + username: debridav + password: debridav + url: jdbc:postgresql://${debridav.db.host}:5432/debridav + hikari: + max-lifetime: 180000000 # five hours + idle-timeout: 0 + maximum-pool-size: 5 + flyway: + out-of-order: true + +logging: + level: + web: info + io.milton.http: error + file: + name: logs/debridav.log + +management: + endpoints: + web: + exposure: + include: health,realdebrid,prometheus,nzbhealthcheck,torrenthealthcheck,logfile + +debridav: + download-path: /downloads + mount-path: /data + debrid-clients: + delay-between-retries: 200ms + retries-on-provider-error: 1 + wait-after-missing: 24h + wait-after-network-error: 1000ms + wait-after-provider-error: 10m + wait-after-client-error: 1000ms + should-delete-non-working-files: true + connect-timeout-milliseconds: 5000 + read-timeout-milliseconds: 5000 + local-entity-max-size-mb: 130 + default-categories: + torrent-lifetime: 1d + webdav-username: + webdav-password: + db: + host: localhost + port: 5432 + database-name: debridav + auth: + enabled: false + jwt-secret: ${DEBRIDAV_AUTH_JWT_SECRET:} + token-expiration-hours: 24 + protect-qbittorrent-api: false + protect-sabnzbd-api: false + protect-actuator: false + +health-check: + repair-enabled: true + torrent-interval: P1D + torrent-poll-rate: PT5M + nzb-interval: 7d + nzb-poll-rate: PT5M + +premiumize: + api-key: + bas-eurl: https://www.premiumize.me/api + +real-debrid: + api-key: + base-url: https://api.real-debrid.com/rest/1.0 + sync-enabled: true + sync-poll-rate: PT24H + +torbox: + api-key: + base-url: https://api.torbox.app + version: v1 + request-timeout-millis: 10000 + socket-timeout-millis: 10000 + +easynews: + username: + password: + api-base-url: https://members.easynews.com + enabled-for-torrents: true + rate-limit-window-duration: 15s + allowed-requests-in-window: 10 + connect-timeout: 20000 + socket-timeout: 5000 + +sonarr: + integration-enabled: false + host: localhost + port: 8990 + api-base-path: /api/v3 + api-key: 1105779a7abb40898567b406442cd927 + category: tv-sonarr + +radarr: + integration-enabled: false + host: localhost + port: 7878 + api-base-path: /api/v3 + api-key: 8d273d4f92294234a9cdddba605054e1 + category: radarr + +nntp: + concurrency: 4 + +pgmq: + default-visibility-timeout: 5m + import-concurrency: 2 + import-visibility-timeout: 10m + import-poll-interval: 2s + health-check-concurrency: 1 + health-check-visibility-timeout: 5m + health-check-poll-interval: 10s + health-repair-concurrency: 2 + health-repair-visibility-timeout: 2m + health-repair-poll-interval: 5s + torrent-health-check-concurrency: 1 + torrent-health-check-visibility-timeout: 5m + torrent-health-check-poll-interval: 10s + torrent-health-repair-concurrency: 2 + torrent-health-repair-visibility-timeout: 2m + torrent-health-repair-poll-interval: 5s + +sentry: + send-default-pii: false + traces-sample-rate: 0 diff --git a/src/main/resources/db/migration/V14__config_override_table.sql b/src/main/resources/db/migration/V14__config_override_table.sql new file mode 100644 index 00000000..b7d63609 --- /dev/null +++ b/src/main/resources/db/migration/V14__config_override_table.sql @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS config_override ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + prop_key VARCHAR(255) NOT NULL UNIQUE, + prop_value TEXT, + sensitive BOOLEAN DEFAULT FALSE, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); diff --git a/src/main/resources/db/migration/V16__create_nzb_import_table.sql b/src/main/resources/db/migration/V16__create_nzb_import_table.sql new file mode 100644 index 00000000..1f648a1d --- /dev/null +++ b/src/main/resources/db/migration/V16__create_nzb_import_table.sql @@ -0,0 +1,14 @@ +CREATE TABLE nzb_import ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + usenet_download_id BIGINT REFERENCES usenet_download(id) ON DELETE SET NULL, + name VARCHAR(255) NOT NULL, + category VARCHAR(255), + status VARCHAR(30) NOT NULL DEFAULT 'QUEUED', + archive_type VARCHAR(30), + error_message TEXT, + files JSONB, + size BIGINT, + created_at TIMESTAMP WITH TIME ZONE, + updated_at TIMESTAMP WITH TIME ZONE +); +CREATE INDEX idx_nzb_import_status ON nzb_import (status); diff --git a/src/main/resources/db/migration/V17__torrent_health_and_pgmq_queues.sql b/src/main/resources/db/migration/V17__torrent_health_and_pgmq_queues.sql new file mode 100644 index 00000000..d6711fb1 --- /dev/null +++ b/src/main/resources/db/migration/V17__torrent_health_and_pgmq_queues.sql @@ -0,0 +1,5 @@ +ALTER TABLE torrent ADD COLUMN last_verified TIMESTAMP; +ALTER TABLE torrent ADD COLUMN health_check_enqueued_at TIMESTAMP; + +SELECT pgmq.create('torrent_health_check'); +SELECT pgmq.create('torrent_health_repair'); diff --git a/src/main/resources/db/migration/V18__repair_outcome_table.sql b/src/main/resources/db/migration/V18__repair_outcome_table.sql new file mode 100644 index 00000000..e382cd72 --- /dev/null +++ b/src/main/resources/db/migration/V18__repair_outcome_table.sql @@ -0,0 +1,9 @@ +CREATE TABLE repair_outcome ( + id BIGSERIAL PRIMARY KEY, + queue_name VARCHAR(255) NOT NULL, + msg_id BIGINT NOT NULL, + action VARCHAR(50) NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT NOW() +); + +CREATE INDEX idx_repair_outcome_queue_msg ON repair_outcome (queue_name, msg_id); diff --git a/src/main/resources/db/migration/V19__drop_import_registry.sql b/src/main/resources/db/migration/V19__drop_import_registry.sql new file mode 100644 index 00000000..2a336b56 --- /dev/null +++ b/src/main/resources/db/migration/V19__drop_import_registry.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS import_registry; +DROP SEQUENCE IF EXISTS import_registry_seq; diff --git a/src/main/resources/db/migration/V20__create_pgmq_dlq_queues.sql b/src/main/resources/db/migration/V20__create_pgmq_dlq_queues.sql new file mode 100644 index 00000000..1eba5dfe --- /dev/null +++ b/src/main/resources/db/migration/V20__create_pgmq_dlq_queues.sql @@ -0,0 +1,5 @@ +SELECT pgmq.create('nzb_import_dlq'); +SELECT pgmq.create('nzb_health_check_dlq'); +SELECT pgmq.create('nzb_health_repair_dlq'); +SELECT pgmq.create('torrent_health_check_dlq'); +SELECT pgmq.create('torrent_health_repair_dlq'); diff --git a/src/main/resources/db/migration/V21__cascade_nzb_contents_to_document.sql b/src/main/resources/db/migration/V21__cascade_nzb_contents_to_document.sql new file mode 100644 index 00000000..ae9e8d8c --- /dev/null +++ b/src/main/resources/db/migration/V21__cascade_nzb_contents_to_document.sql @@ -0,0 +1,9 @@ +-- NzbContents has no meaning without its parent NzbDocument, so cascade-delete +-- when the document is removed. The previous NO ACTION constraint forced callers +-- to manually unlink contents before they could delete a document, which made +-- both production cleanup and integration-test teardown brittle. + +ALTER TABLE nzb_contents DROP CONSTRAINT fk_nzb_contents_document; +ALTER TABLE nzb_contents + ADD CONSTRAINT fk_nzb_contents_document + FOREIGN KEY (nzb_document_id) REFERENCES nzb_document (id) ON DELETE CASCADE; diff --git a/src/main/resources/db/migration/V22__add_fk_indexes.sql b/src/main/resources/db/migration/V22__add_fk_indexes.sql new file mode 100644 index 00000000..92d8a7eb --- /dev/null +++ b/src/main/resources/db/migration/V22__add_fk_indexes.sql @@ -0,0 +1,7 @@ +-- Add indexes on FK columns that the application filters on regularly. Postgres +-- does not auto-index FK columns, so deletes from the parent table (e.g. removing +-- a Category) and joins against the child do full table scans without these. + +CREATE INDEX IF NOT EXISTS idx_db_item_directory_id ON db_item (directory_id); +CREATE INDEX IF NOT EXISTS idx_torrent_category_id ON torrent (category_id); +CREATE INDEX IF NOT EXISTS idx_usenet_download_category_id ON usenet_download (category_id); diff --git a/src/test/kotlin/io/skjaere/debridav/rclone/RcloneCacheInvalidatorTest.kt b/src/test/kotlin/io/skjaere/debridav/rclone/RcloneCacheInvalidatorTest.kt new file mode 100644 index 00000000..94236e9b --- /dev/null +++ b/src/test/kotlin/io/skjaere/debridav/rclone/RcloneCacheInvalidatorTest.kt @@ -0,0 +1,151 @@ +package io.skjaere.debridav.rclone + +import io.ktor.client.HttpClient +import io.ktor.client.engine.mock.MockEngine +import io.ktor.client.engine.mock.respond +import io.ktor.http.HttpStatusCode +import io.ktor.http.content.TextContent +import io.ktor.http.headersOf +import io.ktor.utils.io.ByteReadChannel +import io.skjaere.debridav.configuration.DebridavConfigurationProperties +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import java.util.concurrent.ConcurrentLinkedQueue +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +class RcloneCacheInvalidatorTest { + + private val receivedDirs = ConcurrentLinkedQueue() + private val receivedBodies = ConcurrentLinkedQueue() + private lateinit var invalidator: RcloneCacheInvalidator + private lateinit var debridavConfig: DebridavConfigurationProperties + private lateinit var rcloneConfig: RcloneConfigurationProperties + + @BeforeEach + fun setUp() { + receivedDirs.clear() + receivedBodies.clear() + val mockEngine = MockEngine { request -> + val body = (request.body as? TextContent)?.text ?: "" + receivedBodies.add(body) + // extract "dir":"..." — naive parse is fine for a test + Regex("\"dir\"\\s*:\\s*\"([^\"]+)\"").find(body)?.groupValues?.get(1) + ?.let { receivedDirs.add(it) } + respond( + content = ByteReadChannel("{}"), + status = HttpStatusCode.OK, + headers = headersOf("Content-Type", "application/json") + ) + } + val httpClient = HttpClient(mockEngine) + + debridavConfig = DebridavConfigurationProperties().apply { + rcloneCacheInvalidationEnabled = true + } + rcloneConfig = RcloneConfigurationProperties().apply { + rcUrl = "http://rclone:5572" + } + invalidator = RcloneCacheInvalidator(debridavConfig, rcloneConfig, httpClient) + } + + @AfterEach + fun tearDown() { + invalidator.shutdown() + } + + @Test + fun `coalesces multiple events into a single flush per unique path`() { + invalidator.onChange(FileSystemChangedEvent(setOf("/a", "/b"))) + invalidator.onChange(FileSystemChangedEvent(setOf("/b", "/c"))) + invalidator.onChange(FileSystemChangedEvent(setOf("/a"))) + + // Before the window closes, nothing has been sent. + assertEquals(0, receivedDirs.size, "flush should not happen before window expires") + + // Trigger flush deterministically instead of sleeping through the window. + invalidator.flush() + waitForAsyncRefresh() + + assertEquals(setOf("/a", "/b", "/c"), receivedDirs.toSet()) + assertEquals(3, receivedDirs.size, "each unique path refreshed exactly once") + } + + @Test + fun `event after a flush starts a fresh window`() { + invalidator.onChange(FileSystemChangedEvent(setOf("/first"))) + invalidator.flush() + waitForAsyncRefresh() + + invalidator.onChange(FileSystemChangedEvent(setOf("/second"))) + invalidator.flush() + waitForAsyncRefresh() + + assertEquals(listOf("/first", "/second"), receivedDirs.toList()) + } + + @Test + fun `disabled toggle drops events with no HTTP activity`() { + debridavConfig.rcloneCacheInvalidationEnabled = false + invalidator.onChange(FileSystemChangedEvent(setOf("/whatever"))) + invalidator.flush() + waitForAsyncRefresh() + + assertTrue(receivedDirs.isEmpty()) + } + + @Test + fun `blank rcUrl drops events with no HTTP activity`() { + rcloneConfig.rcUrl = "" + invalidator.onChange(FileSystemChangedEvent(setOf("/whatever"))) + invalidator.flush() + waitForAsyncRefresh() + + assertTrue(receivedDirs.isEmpty()) + } + + @Test + fun `emitted path fans out to include all ancestors in refresh order`() { + // A brand-new nested directory (e.g. /downloads/NewRelease) can't be + // refreshed directly — rclone doesn't know it exists until its parent + // is refreshed first. Root is included too so a brand-new top-level + // directory is equally discoverable; the extra root refresh is cheap + // (a tiny listing re-read). + invalidator.onChange(FileSystemChangedEvent(setOf("/downloads/NewRelease"))) + invalidator.flush() + waitForRefreshes(expected = 3) + + assertEquals(3, receivedBodies.size) + assertEquals("{}", receivedBodies.elementAt(0), "root refresh should go first") + assertEquals(listOf("/downloads", "/downloads/NewRelease"), receivedDirs.toList()) + } + + @Test + fun `root path sends an empty body so rclone refreshes the whole VFS`() { + // rclone's /vfs/refresh rejects dir="/" with "file does not exist"; + // omitting the dir param refreshes the whole VFS instead. + invalidator.onChange(FileSystemChangedEvent(setOf("/"))) + invalidator.flush() + waitForAsyncRefresh() + + assertEquals(1, receivedBodies.size) + val body = receivedBodies.first() + assertEquals("{}", body, "root should produce empty-object body, not {\"dir\":\"/\"}") + assertTrue(receivedDirs.isEmpty(), "root path must not be sent as dir") + } + + /** Wait until the expected number of POSTs have landed. */ + private fun waitForRefreshes(expected: Int, timeoutMs: Long = 2000) = runBlocking { + val deadline = System.currentTimeMillis() + timeoutMs + while (receivedBodies.size < expected && System.currentTimeMillis() < deadline) { + kotlinx.coroutines.delay(20) + } + } + + /** Give the IO coroutine scope a moment to execute the posted refreshes. */ + private fun waitForAsyncRefresh() = runBlocking { + kotlinx.coroutines.delay(150) + } +} diff --git a/src/test/kotlin/io/skjaere/debridav/test/ArrServiceTest.kt b/src/test/kotlin/io/skjaere/debridav/test/ArrServiceTest.kt index c2876bbc..025f6cfe 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/ArrServiceTest.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/ArrServiceTest.kt @@ -19,7 +19,7 @@ class ArrServiceTest { fun thatDeleteFileAndSearchCallsClient() = runTest { //given every { sonarrApiClient.getCategory() } returns "tv-sonarr" - coEvery { sonarrApiClient.deleteFileAndSearch(eq("test-item")) } just Runs + coEvery { sonarrApiClient.deleteFileAndSearch(eq("test-item")) } returns true //when underTest.deleteFileAndSearch("test-item", "tv-sonarr") diff --git a/src/test/kotlin/io/skjaere/debridav/test/DebridLinkServiceTest.kt b/src/test/kotlin/io/skjaere/debridav/test/DebridLinkServiceTest.kt index 71c17c82..80173f8f 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/DebridLinkServiceTest.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/DebridLinkServiceTest.kt @@ -20,6 +20,7 @@ import io.skjaere.debridav.debrid.client.model.NetworkErrorGetCachedFilesRespons import io.skjaere.debridav.debrid.client.model.NotCachedGetCachedFilesResponse import io.skjaere.debridav.debrid.client.model.ProviderErrorGetCachedFilesResponse import io.skjaere.debridav.debrid.client.model.SuccessfulGetCachedFilesResponse +import io.skjaere.debridav.debrid.client.DebridCachedContentClient import io.skjaere.debridav.debrid.client.premiumize.PremiumizeClient import io.skjaere.debridav.debrid.client.realdebrid.RealDebridClient import io.skjaere.debridav.debrid.model.DebridProviderError @@ -49,26 +50,24 @@ class DebridLinkServiceTest { private val clock = Clock.fixed(Instant.ofEpochMilli(1730477942L), ZoneId.systemDefault()) private val realDebridClient = mockk() private val debridCachedContentService = mockk() - private val debridClients = listOf(realDebridClient, premiumizeClient) - private val debridavConfigurationProperties = DebridavConfigurationProperties( - mountPath = "${TestContextInitializer.BASE_PATH}/debridav", - debridClients = listOf(DebridProvider.REAL_DEBRID, DebridProvider.PREMIUMIZE), - downloadPath = "${TestContextInitializer.BASE_PATH}/downloads", - rootPath = "${TestContextInitializer.BASE_PATH}/files", - retriesOnProviderError = 3, - waitAfterNetworkError = Duration.ofMillis(10000), - delayBetweenRetries = Duration.ofMillis(1000), - waitAfterMissing = Duration.ofMillis(1000), - waitAfterProviderError = Duration.ofMillis(1000), - readTimeoutMilliseconds = 1000, - connectTimeoutMilliseconds = 1000, - waitAfterClientError = Duration.ofMillis(1000), - shouldDeleteNonWorkingFiles = true, - torrentLifetime = Duration.ofMinutes(1), - enableFileImportOnStartup = false, - defaultCategories = listOf(), + private val debridClients: List = listOf(realDebridClient, premiumizeClient) + private val debridavConfigurationProperties = DebridavConfigurationProperties().apply { + mountPath = "${TestContextInitializer.BASE_PATH}/debridav" + debridClients = listOf(DebridProvider.REAL_DEBRID, DebridProvider.PREMIUMIZE) + downloadPath = "${TestContextInitializer.BASE_PATH}/downloads" + retriesOnProviderError = 3 + waitAfterNetworkError = Duration.ofMillis(10000) + delayBetweenRetries = Duration.ofMillis(1000) + waitAfterMissing = Duration.ofMillis(1000) + waitAfterProviderError = Duration.ofMillis(1000) + readTimeoutMilliseconds = 1000 + connectTimeoutMilliseconds = 1000 + waitAfterClientError = Duration.ofMillis(1000) + shouldDeleteNonWorkingFiles = true + torrentLifetime = Duration.ofMinutes(1) + defaultCategories = listOf() localEntityMaxSizeMb = 1 - ) + } val file = mockk() private val fileService = mockk() diff --git a/src/test/kotlin/io/skjaere/debridav/test/MigrationTest.kt b/src/test/kotlin/io/skjaere/debridav/test/MigrationTest.kt deleted file mode 100644 index dd41450c..00000000 --- a/src/test/kotlin/io/skjaere/debridav/test/MigrationTest.kt +++ /dev/null @@ -1,78 +0,0 @@ -package io.skjaere.debridav.test - -import io.skjaere.debridav.fs.legacy.DebridFileContents -import kotlin.test.assertTrue -import kotlinx.serialization.json.Json -import org.junit.jupiter.api.Test - -class MigrationTest { - - @Test - fun `that old version can be deserialized`() { - val oldJson = """ - { - "originalPath": "/foo/bar.mkv", - "size": 100, - "modified": 1730477942, - "magnet": "magnet:?xt=urn:btih:hash&dn=test&tr=", - "debridLinks": [ - { - "type": "io.skjaere.debridav.debrid.model.CachedFile", - "path": "/foo/bar.mkv", - "size": 100, - "mimeType": "video/mkv", - "link": "http://test.test/bar.mkv", - "provider": "REAL_DEBRID", - "lastChecked": 100 - }, - { - "type": "io.skjaere.debridav.debrid.model.CachedFile", - "path": "/foo/bar.mkv", - "size": 100, - "mimeType": "video/mkv", - "link": "http://test.test/bar.mkv", - "provider": "PREMIUMIZE", - "lastChecked": 100 - } - ] - } - """.trimIndent() - val oldDebridFileContentsDeserialized = Json.decodeFromString(oldJson) - assertTrue(oldDebridFileContentsDeserialized.type == DebridFileContents.Type.TORRENT_MAGNET) - } - - @Test - fun `that new version can be deserialized`() { - val newJson = """ - { - "originalPath": "/foo/bar.mkv", - "size": 100, - "modified": 1730477942, - "magnet": "Release.Name.2024.1080p.GrP", - "type": "USENET_RELEASE", - "debridLinks": [ - { - "type": "io.skjaere.debridav.debrid.model.CachedFile", - "path": "/foo/bar.mkv", - "size": 100, - "mimeType": "video/mkv", - "link": "http://test.test/bar.mkv", - "provider": "REAL_DEBRID", - "lastChecked": 100 - }, - { - "type": "io.skjaere.debridav.debrid.model.CachedFile", - "path": "/foo/bar.mkv", - "size": 100, - "mimeType": "video/mkv", - "link": "http://test.test/bar.mkv", - "provider": "PREMIUMIZE", - "lastChecked": 100 - } - ] - } - """.trimIndent() - val oldDebridFileContentsDeserialized = Json.decodeFromString(newJson) - assertTrue(oldDebridFileContentsDeserialized.type == DebridFileContents.Type.USENET_RELEASE) - } -} diff --git a/src/test/kotlin/io/skjaere/debridav/test/NzbImportServiceTest.kt b/src/test/kotlin/io/skjaere/debridav/test/NzbImportServiceTest.kt index 66305c08..78e6bb7d 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/NzbImportServiceTest.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/NzbImportServiceTest.kt @@ -12,17 +12,19 @@ import io.skjaere.debridav.fs.DatabaseFileService import io.skjaere.debridav.fs.NzbContents import io.skjaere.debridav.fs.RemotelyCachedEntity import io.skjaere.debridav.repository.NzbDocumentRepository +import io.skjaere.debridav.repository.NzbImportRepository import io.skjaere.debridav.repository.UsenetRepository import io.skjaere.debridav.usenet.NzbImportService import io.skjaere.debridav.usenet.NzbImportTaskData import io.skjaere.debridav.usenet.UsenetDownload import io.skjaere.debridav.usenet.UsenetDownloadStatus +import io.skjaere.debridav.usenet.nzb.NzbArchiveType import io.skjaere.debridav.usenet.nzb.NzbDocumentEntity import io.skjaere.debridav.usenet.nzb.StreamableFileJson +import io.skjaere.debridav.usenet.queue.NzbImportRecord +import io.skjaere.debridav.usenet.queue.NzbImportStatus import io.skjaere.nntp.ArticleNotFoundException import io.skjaere.nntp.NntpConnectionException -import io.skjaere.nntp.YencHeaders -import java.io.IOException import io.skjaere.nzbstreamer.NzbStreamer import io.skjaere.nzbstreamer.metadata.ExtractedMetadata import io.skjaere.nzbstreamer.metadata.NzbMetadataResponse @@ -31,7 +33,10 @@ import io.skjaere.nzbstreamer.nzb.NzbDocument import io.skjaere.nzbstreamer.nzb.NzbFile import io.skjaere.nzbstreamer.nzb.NzbSegment import io.skjaere.nzbstreamer.stream.StreamableFile +import io.skjaere.nntp.YencHeaders +import java.io.IOException import org.junit.jupiter.api.Test +import org.springframework.transaction.PlatformTransactionManager import java.time.Duration import java.util.* import kotlin.test.assertEquals @@ -41,31 +46,38 @@ class NzbImportServiceTest { private val nzbStreamer = mockk() private val nzbDocumentRepository = mockk() private val usenetRepository = mockk() + private val nzbImportRepository = mockk() private val pgmqClient = mockk() private val databaseFileService = mockk() - private val config = DebridavConfigurationProperties( - rootPath = "/", - downloadPath = "/downloads", - mountPath = "/data", - debridClients = listOf(DebridProvider.EASYNEWS), - waitAfterMissing = Duration.ZERO, - waitAfterProviderError = Duration.ZERO, - waitAfterNetworkError = Duration.ZERO, - waitAfterClientError = Duration.ZERO, - retriesOnProviderError = 0, - delayBetweenRetries = Duration.ZERO, - connectTimeoutMilliseconds = 5000, - readTimeoutMilliseconds = 30000, - shouldDeleteNonWorkingFiles = false, - torrentLifetime = Duration.ofHours(1), - enableFileImportOnStartup = false, - defaultCategories = emptyList(), + private val config = DebridavConfigurationProperties().apply { + downloadPath = "/downloads" + mountPath = "/data" + debridClients = listOf(DebridProvider.EASYNEWS) + waitAfterMissing = Duration.ZERO + waitAfterProviderError = Duration.ZERO + waitAfterNetworkError = Duration.ZERO + waitAfterClientError = Duration.ZERO + retriesOnProviderError = 0 + delayBetweenRetries = Duration.ZERO + connectTimeoutMilliseconds = 5000 + readTimeoutMilliseconds = 30000 + shouldDeleteNonWorkingFiles = false + torrentLifetime = Duration.ofHours(1) + defaultCategories = emptyList() localEntityMaxSizeMb = 100 - ) + } + + /** + * A no-op PlatformTransactionManager that does not start real DB transactions. + * TransactionTemplate will still execute the callback directly, which is all + * we need for unit tests that already mock the repository methods. + */ + private val platformTransactionManager = mockk(relaxed = true) private val underTest = NzbImportService( nzbStreamer, nzbDocumentRepository, usenetRepository, - pgmqClient, databaseFileService, config + nzbImportRepository, pgmqClient, databaseFileService, config, + platformTransactionManager ) private val nzbBytes = "test".toByteArray() @@ -80,11 +92,21 @@ class NzbImportServiceTest { return download } + private fun createImportRecord(id: Long = 100L): NzbImportRecord { + val record = NzbImportRecord() + record.id = id + record.name = "test-release" + record.status = NzbImportStatus.QUEUED + return record + } + @Test fun `executeImport sets COMPLETED on PrepareResult Success`() { // given val download = createUsenetDownload() + val importRecord = createImportRecord() every { usenetRepository.findById(1L) } returns Optional.of(download) + every { nzbImportRepository.findById(100L) } returns Optional.of(importRecord) val nzbFile = NzbFile( poster = "test", date = 0, subject = "test", @@ -113,6 +135,7 @@ class NzbImportServiceTest { val savedDoc = NzbDocumentEntity().apply { id = 10L + archiveType = NzbArchiveType.RAR streamableFiles = listOf( StreamableFileJson( path = "video.mkv", @@ -127,25 +150,33 @@ class NzbImportServiceTest { every { nzbDocumentRepository.save(any()) } returns savedDoc val debridFile = mockk() - every { databaseFileService.createDebridFile(any(), any(), any()) } returns debridFile + every { databaseFileService.createDebridFiles(any(), any()) } returns listOf(debridFile) val savedSlot = slot() every { usenetRepository.save(capture(savedSlot)) } answers { savedSlot.captured } + val importSlot = slot() + every { nzbImportRepository.save(capture(importSlot)) } answers { importSlot.captured } + // when - underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L)) + underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L, 100L)) // then assertEquals(UsenetDownloadStatus.COMPLETED, savedSlot.captured.status) + assertEquals(NzbImportStatus.COMPLETED, importSlot.captured.status) + assertEquals(4000L, importSlot.captured.size) + assertEquals("RAR", importSlot.captured.archiveType) verify(exactly = 1) { nzbDocumentRepository.save(any()) } - verify(exactly = 1) { databaseFileService.createDebridFile(any(), eq("abc123"), any()) } + verify(exactly = 1) { databaseFileService.createDebridFiles(any(), eq("abc123")) } } @Test fun `executeImport sets FAILED on PrepareResult MissingArticles`() { // given val download = createUsenetDownload() + val importRecord = createImportRecord() every { usenetRepository.findById(1L) } returns Optional.of(download) + every { nzbImportRepository.findById(100L) } returns Optional.of(importRecord) coEvery { nzbStreamer.prepare(any()) } returns PrepareResult.MissingArticles( "Article not found: 430", @@ -155,11 +186,16 @@ class NzbImportServiceTest { val savedSlot = slot() every { usenetRepository.save(capture(savedSlot)) } answers { savedSlot.captured } + val importSlot = slot() + every { nzbImportRepository.save(capture(importSlot)) } answers { importSlot.captured } + // when - underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L)) + underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L, 100L)) // then assertEquals(UsenetDownloadStatus.FAILED, savedSlot.captured.status) + assertEquals(NzbImportStatus.FAILED, importSlot.captured.status) + assertEquals("Article not found: 430", importSlot.captured.errorMessage) verify(exactly = 0) { nzbDocumentRepository.save(any()) } } @@ -167,7 +203,9 @@ class NzbImportServiceTest { fun `executeImport sets FAILED on PrepareResult Failure`() { // given val download = createUsenetDownload() + val importRecord = createImportRecord() every { usenetRepository.findById(1L) } returns Optional.of(download) + every { nzbImportRepository.findById(100L) } returns Optional.of(importRecord) coEvery { nzbStreamer.prepare(any()) } returns PrepareResult.Failure( "Connection refused", @@ -177,11 +215,15 @@ class NzbImportServiceTest { val savedSlot = slot() every { usenetRepository.save(capture(savedSlot)) } answers { savedSlot.captured } + val importSlot = slot() + every { nzbImportRepository.save(capture(importSlot)) } answers { importSlot.captured } + // when - underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L)) + underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L, 100L)) // then assertEquals(UsenetDownloadStatus.FAILED, savedSlot.captured.status) + assertEquals(NzbImportStatus.FAILED, importSlot.captured.status) verify(exactly = 0) { nzbDocumentRepository.save(any()) } } @@ -189,7 +231,9 @@ class NzbImportServiceTest { fun `executeImport sets FAILED on PrepareResult UnsupportedArchive`() { // given val download = createUsenetDownload() + val importRecord = createImportRecord() every { usenetRepository.findById(1L) } returns Optional.of(download) + every { nzbImportRepository.findById(100L) } returns Optional.of(importRecord) coEvery { nzbStreamer.prepare(any()) } returns PrepareResult.UnsupportedArchive( "Unable to detect archive type from filenames or byte signatures", @@ -199,11 +243,15 @@ class NzbImportServiceTest { val savedSlot = slot() every { usenetRepository.save(capture(savedSlot)) } answers { savedSlot.captured } + val importSlot = slot() + every { nzbImportRepository.save(capture(importSlot)) } answers { importSlot.captured } + // when - underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L)) + underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L, 100L)) // then assertEquals(UsenetDownloadStatus.FAILED, savedSlot.captured.status) + assertEquals(NzbImportStatus.FAILED, importSlot.captured.status) verify(exactly = 0) { nzbDocumentRepository.save(any()) } } @@ -211,47 +259,123 @@ class NzbImportServiceTest { fun `executeImport sets FAILED on unexpected exception`() { // given val download = createUsenetDownload() + val importRecord = createImportRecord() every { usenetRepository.findById(1L) } returns Optional.of(download) + every { nzbImportRepository.findById(100L) } returns Optional.of(importRecord) coEvery { nzbStreamer.prepare(any()) } throws RuntimeException("unexpected error") val savedSlot = slot() every { usenetRepository.save(capture(savedSlot)) } answers { savedSlot.captured } + val importSlot = slot() + every { nzbImportRepository.save(capture(importSlot)) } answers { importSlot.captured } + // when - underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L)) + underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L, 100L)) // then assertEquals(UsenetDownloadStatus.FAILED, savedSlot.captured.status) + assertEquals(NzbImportStatus.FAILED, importSlot.captured.status) } @Test - fun `executeImport throws when UsenetDownload not found`() { + fun `executeImport throws when UsenetDownload not found on initial load`() { // given every { usenetRepository.findById(999L) } returns Optional.empty() + // when - should return early without error (download may have been deleted) + underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 999L, 100L)) + + // then - no exception, no save calls + verify(exactly = 0) { usenetRepository.save(any()) } + verify(exactly = 0) { nzbImportRepository.save(any()) } + } + + @Test + fun `executeImport throws when NzbImportRecord not found`() { + // given + val download = createUsenetDownload() + every { usenetRepository.findById(1L) } returns Optional.of(download) + every { nzbImportRepository.findById(999L) } returns Optional.empty() + // when/then kotlin.test.assertFailsWith { - underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 999L)) + underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L, 999L)) } } @Test - fun `executeImport always saves UsenetDownload in finally block`() { + fun `executeImport always saves both records in finally block`() { // given val download = createUsenetDownload() + val importRecord = createImportRecord() every { usenetRepository.findById(1L) } returns Optional.of(download) + every { nzbImportRepository.findById(100L) } returns Optional.of(importRecord) coEvery { nzbStreamer.prepare(any()) } returns PrepareResult.MissingArticles( "missing", ArticleNotFoundException("missing") ) every { usenetRepository.save(any()) } answers { firstArg() } + every { nzbImportRepository.save(any()) } answers { firstArg() } // when - underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L)) + underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L, 100L)) - // then - save is called exactly once (in the finally block) + // then - nzbImportRepository.save is called: once for IMPORTING status (phase 1) + once in phase 3 = 2 + verify(exactly = 2) { nzbImportRepository.save(any()) } verify(exactly = 1) { usenetRepository.save(any()) } + assertEquals(UsenetDownloadStatus.FAILED, download.status) + assertEquals(NzbImportStatus.FAILED, importRecord.status) + } + + /** + * Reproduces the production bug: ObjectOptimisticLockingFailureException when + * the UsenetDownload row is deleted (e.g., via the SABnzbd delete API) while + * the long-running NNTP prepare is in flight. + * + * Before the fix, executeImport held a single @Transactional spanning the entire + * method including the NNTP I/O. If another transaction deleted the UsenetDownload + * row during that I/O, the transaction commit would fail with: + * "Unexpected row count (expected row count 1 but was 0) + * [update usenet_download ... where id=?]" + * + * After the fix, executeImport uses short-lived TransactionTemplate scopes so no + * transaction is held during I/O. The entity is re-fetched in the save phase; if + * it was deleted, the method completes gracefully without throwing. + */ + @Test + fun `executeImport completes gracefully when UsenetDownload is deleted during NNTP IO`() { + // given + val download = createUsenetDownload(id = 102L) + val importRecord = createImportRecord(id = 86L) + + // Phase 1 (load): download exists + // Phase 3 (save): download has been deleted by another thread/request + every { usenetRepository.findById(102L) } returnsMany listOf( + Optional.of(download), + Optional.empty() // simulates concurrent deletion during NNTP I/O + ) + every { nzbImportRepository.findById(86L) } returns Optional.of(importRecord) + + coEvery { nzbStreamer.prepare(any()) } returns PrepareResult.MissingArticles( + "Article not found", + ArticleNotFoundException("Article not found") + ) + + val importSlot = slot() + every { nzbImportRepository.save(capture(importSlot)) } answers { importSlot.captured } + + // when — must NOT throw ObjectOptimisticLockingFailureException + underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 102L, 86L)) + + // then + // UsenetDownload.save is never called because the row is gone + verify(exactly = 0) { usenetRepository.save(any()) } + // The import record is still saved so we have a record of the failure + verify(exactly = 2) { nzbImportRepository.save(any()) } + assertEquals(NzbImportStatus.FAILED, importSlot.captured.status) + assertEquals("Download was deleted during import", importSlot.captured.errorMessage) } } diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/ConfigApiIT.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/ConfigApiIT.kt new file mode 100644 index 00000000..58ca2ac0 --- /dev/null +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/ConfigApiIT.kt @@ -0,0 +1,259 @@ +package io.skjaere.debridav.test.integrationtest + +import io.skjaere.debridav.DebriDavApplication +import io.skjaere.debridav.MiltonConfiguration +import io.skjaere.debridav.config.ConfigOverrideRepository +import io.skjaere.debridav.config.DatabasePropertySourceInitializer +import io.skjaere.debridav.configuration.DebridavConfigurationProperties +import io.skjaere.debridav.debrid.client.premiumize.PremiumizeConfigurationProperties +import io.skjaere.debridav.test.integrationtest.config.IntegrationTestContextConfiguration +import io.skjaere.debridav.test.integrationtest.config.MockServerTest +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.cloud.context.refresh.ContextRefresher +import org.springframework.http.MediaType +import org.springframework.test.web.reactive.server.WebTestClient +import java.time.Duration +import kotlin.test.assertEquals + +@SpringBootTest( + classes = [DebriDavApplication::class, IntegrationTestContextConfiguration::class, MiltonConfiguration::class], + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = [ + "debridav.debrid-clients=premiumize", + "debridav.auth.enabled=false", + "debridav.auth.jwt-secret=test-secret-key-that-is-at-least-256-bits-long-for-hs256" + ] +) +@MockServerTest +class ConfigApiIT { + + @Autowired + private lateinit var webTestClient: WebTestClient + + @Autowired + private lateinit var configOverrideRepository: ConfigOverrideRepository + + @Autowired + private lateinit var debridavConfig: DebridavConfigurationProperties + + @Autowired + private lateinit var premiumizeConfig: PremiumizeConfigurationProperties + + @Autowired + private lateinit var contextRefresher: ContextRefresher + + @Autowired + private lateinit var dbPropertySourceInitializer: DatabasePropertySourceInitializer + + @AfterEach + fun tearDown() { + configOverrideRepository.deleteAll() + val propertySource = dbPropertySourceInitializer.getOrCreatePropertySource() + propertySource.replaceAll(emptyMap()) + contextRefresher.refreshEnvironment() + } + + @Test + fun `list all whitelisted config properties`() { + webTestClient.get() + .uri("/api/v1/config") + .exchange() + .expectStatus().isOk + .expectBody() + .jsonPath("$").isArray + .jsonPath("$.length()").isNotEmpty + .jsonPath("$[?(@.key == 'debridav.download-path')]").exists() + .jsonPath("$[?(@.key == 'debridav.download-path')].name").isEqualTo("Download Path") + .jsonPath("$[?(@.key == 'debridav.download-path')].type").isEqualTo("STRING") + .jsonPath("$[?(@.key == 'debridav.should-delete-non-working-files')].type").isEqualTo("BOOLEAN") + .jsonPath("$[?(@.key == 'debridav.torrent-lifetime')].type").isEqualTo("DURATION") + .jsonPath("$[?(@.key == 'spring.datasource.url')]").doesNotExist() + } + + @Test + fun `get single config property`() { + webTestClient.get() + .uri("/api/v1/config/debridav.download-path") + .exchange() + .expectStatus().isOk + .expectBody() + .jsonPath("$.key").isEqualTo("debridav.download-path") + .jsonPath("$.name").isEqualTo("Download Path") + .jsonPath("$.hasOverride").isEqualTo(false) + .jsonPath("$.group").isEqualTo("debridav") + .jsonPath("$.type").isEqualTo("STRING") + } + + @Test + fun `upsert creates override`() { + webTestClient.put() + .uri("/api/v1/config/debridav.torrent-lifetime") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue("""{"value": "2h"}""") + .exchange() + .expectStatus().isOk + .expectBody() + .jsonPath("$.key").isEqualTo("debridav.torrent-lifetime") + .jsonPath("$.name").isEqualTo("Torrent Lifetime") + .jsonPath("$.hasOverride").isEqualTo(true) + .jsonPath("$.effectiveValue").isEqualTo("2h") + .jsonPath("$.type").isEqualTo("DURATION") + } + + @Test + fun `delete removes override and reverts to default`() { + // First create an override + webTestClient.put() + .uri("/api/v1/config/debridav.torrent-lifetime") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue("""{"value": "2h"}""") + .exchange() + .expectStatus().isOk + + // Then delete it + webTestClient.delete() + .uri("/api/v1/config/debridav.torrent-lifetime") + .exchange() + .expectStatus().isOk + .expectBody() + .jsonPath("$.hasOverride").isEqualTo(false) + } + + @Test + fun `non-whitelisted key returns 400`() { + webTestClient.get() + .uri("/api/v1/config/spring.datasource.url") + .exchange() + .expectStatus().isBadRequest + .expectBody() + .jsonPath("$.error").exists() + } + + @Test + fun `upsert non-whitelisted key returns 400`() { + webTestClient.put() + .uri("/api/v1/config/spring.datasource.url") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue("""{"value": "jdbc:postgresql://evil:5432/db"}""") + .exchange() + .expectStatus().isBadRequest + } + + @Test + fun `sensitive values are masked in responses`() { + // Upsert a sensitive value + webTestClient.put() + .uri("/api/v1/config/premiumize.api-key") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue("""{"value": "my-secret-key"}""") + .exchange() + .expectStatus().isOk + .expectBody() + .jsonPath("$.sensitive").isEqualTo(true) + .jsonPath("$.effectiveValue").isEqualTo("***") + } + + @Test + fun `delete non-existent override returns 404`() { + webTestClient.delete() + .uri("/api/v1/config/debridav.download-path") + .exchange() + .expectStatus().isNotFound + } + + @Test + fun `upsert refreshes config bean at runtime`() { + val originalLifetime = debridavConfig.torrentLifetime + + webTestClient.put() + .uri("/api/v1/config/debridav.torrent-lifetime") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue("""{"value": "2h"}""") + .exchange() + .expectStatus().isOk + + assertEquals(Duration.ofHours(2), debridavConfig.torrentLifetime) + assert(debridavConfig.torrentLifetime != originalLifetime) { + "torrentLifetime should have changed from default $originalLifetime" + } + } + + @Test + fun `delete reverts config bean to default at runtime`() { + val originalLifetime = debridavConfig.torrentLifetime + + // Override + webTestClient.put() + .uri("/api/v1/config/debridav.torrent-lifetime") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue("""{"value": "2h"}""") + .exchange() + .expectStatus().isOk + assertEquals(Duration.ofHours(2), debridavConfig.torrentLifetime) + + // Delete override + webTestClient.delete() + .uri("/api/v1/config/debridav.torrent-lifetime") + .exchange() + .expectStatus().isOk + + assertEquals(originalLifetime, debridavConfig.torrentLifetime) + } + + @Test + fun `upsert refreshes boolean property on config bean`() { + val original = debridavConfig.shouldDeleteNonWorkingFiles + + webTestClient.put() + .uri("/api/v1/config/debridav.should-delete-non-working-files") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue("""{"value": "${!original}"}""") + .exchange() + .expectStatus().isOk + + assertEquals(!original, debridavConfig.shouldDeleteNonWorkingFiles) + } + + @Test + fun `upsert refreshes property on different config bean`() { + webTestClient.put() + .uri("/api/v1/config/premiumize.api-key") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue("""{"value": "new-api-key-12345"}""") + .exchange() + .expectStatus().isOk + + assertEquals("new-api-key-12345", premiumizeConfig.apiKey) + } + + @Test + fun `saving nntp pools twice does not cause duplicate key violation`() { + val poolJson = """[{"host":"news.example.com","port":563,"username":"user",""" + + """"password":"pass","useTls":true,"maxConnections":8,"priority":0}]""" + + // First save + webTestClient.put() + .uri("/api/v1/config/nntp-pools") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue(poolJson) + .exchange() + .expectStatus().isOk + + val updatedPoolJson = """[{"host":"news2.example.com","port":563,"username":"user2",""" + + """"password":"pass2","useTls":true,"maxConnections":4,"priority":0}]""" + + // Second save - should not throw DataIntegrityViolationException + webTestClient.put() + .uri("/api/v1/config/nntp-pools") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue(updatedPoolJson) + .exchange() + .expectStatus().isOk + .expectBody() + .jsonPath("$[0].host").isEqualTo("news2.example.com") + .jsonPath("$[0].maxConnections").isEqualTo(4) + } +} diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/ContentIT.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/ContentIT.kt index 0a1519d7..174bbaca 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/ContentIT.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/ContentIT.kt @@ -60,14 +60,14 @@ class ContentIT { // when / then webTestClient .get() - .uri("testfile.mp4") + .uri("/webdav/testfile.mp4") .exchange() .expectStatus().is2xxSuccessful .expectBody(String::class.java) .isEqualTo("it works!") webTestClient.delete() - .uri("/testfile.mp4") + .uri("/webdav/testfile.mp4") .exchange() .expectStatus().is2xxSuccessful } diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/DebridProviderErrorHandlingIT.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/DebridProviderErrorHandlingIT.kt index 52bda7be..48ece79c 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/DebridProviderErrorHandlingIT.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/DebridProviderErrorHandlingIT.kt @@ -344,7 +344,7 @@ class DebridProviderErrorHandlingIT { webTestClient .mutate().responseTimeout(Duration.ofMillis(30000)).build() .get() - .uri("testfile.mp4") + .uri("/webdav/testfile.mp4") .exchange() .expectStatus().is2xxSuccessful @@ -389,7 +389,7 @@ class DebridProviderErrorHandlingIT { webTestClient .mutate().responseTimeout(Duration.ofMillis(30000)).build() .get() - .uri("testfile.mp4") + .uri("/webdav/testfile.mp4") .exchange() .expectStatus().is2xxSuccessful diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/JwtAuthIT.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/JwtAuthIT.kt new file mode 100644 index 00000000..b95d830f --- /dev/null +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/JwtAuthIT.kt @@ -0,0 +1,124 @@ +package io.skjaere.debridav.test.integrationtest + +import io.skjaere.debridav.DebriDavApplication +import io.skjaere.debridav.MiltonConfiguration +import io.skjaere.debridav.config.ConfigOverrideRepository +import io.skjaere.debridav.config.auth.JwtService +import io.skjaere.debridav.test.integrationtest.config.IntegrationTestContextConfiguration +import io.skjaere.debridav.test.integrationtest.config.MockServerTest +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.http.MediaType +import org.springframework.test.web.reactive.server.WebTestClient + +@SpringBootTest( + classes = [DebriDavApplication::class, IntegrationTestContextConfiguration::class, MiltonConfiguration::class], + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = [ + "debridav.debrid-clients=premiumize", + "debridav.auth.enabled=true", + "debridav.auth.jwt-secret=test-secret-key-that-is-at-least-256-bits-long-for-hs256", + "debridav.webdav-username=admin", + "debridav.webdav-password=secret", + "debridav.auth.protect-qbittorrent-api=true", + "debridav.auth.protect-sabnzbd-api=false" + ] +) +@MockServerTest +class JwtAuthIT { + + @Autowired + private lateinit var webTestClient: WebTestClient + + @Autowired + private lateinit var jwtService: JwtService + + @Autowired + private lateinit var configOverrideRepository: ConfigOverrideRepository + + @AfterEach + fun tearDown() { + configOverrideRepository.deleteAll() + } + + @Test + fun `login with valid credentials returns token`() { + webTestClient.post() + .uri("/api/v1/auth/login") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue("""{"username": "admin", "password": "secret"}""") + .exchange() + .expectStatus().isOk + .expectBody() + .jsonPath("$.token").isNotEmpty + } + + @Test + fun `login with invalid credentials returns 401`() { + webTestClient.post() + .uri("/api/v1/auth/login") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue("""{"username": "admin", "password": "wrong"}""") + .exchange() + .expectStatus().isUnauthorized + } + + @Test + fun `config endpoint requires auth when enabled`() { + webTestClient.get() + .uri("/api/v1/config") + .exchange() + .expectStatus().isUnauthorized + } + + @Test + fun `config endpoint accessible with valid token`() { + val token = jwtService.generateToken("admin") + + webTestClient.get() + .uri("/api/v1/config") + .header("Authorization", "Bearer $token") + .exchange() + .expectStatus().isOk + } + + @Test + fun `config endpoint rejects invalid token`() { + webTestClient.get() + .uri("/api/v1/config") + .header("Authorization", "Bearer invalid-token") + .exchange() + .expectStatus().isUnauthorized + } + + @Test + fun `qbittorrent api requires auth when configured`() { + webTestClient.get() + .uri("/api/v2/app/webapiVersion") + .exchange() + .expectStatus().isUnauthorized + } + + @Test + fun `qbittorrent api accessible with valid token`() { + val token = jwtService.generateToken("admin") + + webTestClient.get() + .uri("/api/v2/app/webapiVersion") + .header("Authorization", "Bearer $token") + .exchange() + .expectStatus().isOk + } + + @Test + fun `auth endpoint is always public`() { + webTestClient.post() + .uri("/api/v1/auth/login") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue("""{"username": "admin", "password": "secret"}""") + .exchange() + .expectStatus().isOk + } +} diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/NzbHealthCheckIT.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/NzbHealthCheckIT.kt index 2b4a43f2..adc5eaeb 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/NzbHealthCheckIT.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/NzbHealthCheckIT.kt @@ -7,12 +7,11 @@ import io.skjaere.debridav.MiltonConfiguration import io.skjaere.debridav.repository.NzbDocumentRepository import io.skjaere.debridav.repository.UsenetRepository import io.skjaere.debridav.test.integrationtest.config.IntegrationTestContextConfiguration -import io.skjaere.debridav.test.integrationtest.config.MockServerTest +import io.skjaere.debridav.test.integrationtest.config.MockServerNntpTest +import io.skjaere.debridav.test.integrationtest.config.awaitSabImportCompletion import io.skjaere.debridav.usenet.NzbHealthCheckService -import io.skjaere.debridav.usenet.sabnzbd.model.SabnzbdFullHistoryResponse import io.skjaere.mocknntp.testcontainer.MockNntpServerContainer import kotlinx.coroutines.runBlocking -import kotlinx.serialization.json.Json import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers.`is` import org.junit.jupiter.api.AfterEach @@ -34,12 +33,11 @@ import org.springframework.web.reactive.function.BodyInserters webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = [ "debridav.debrid-clients=easynews", - "nntp.enabled=true", "sonarr.integration-enabled=true", "sonarr.category=testcat" ] ) -@MockServerTest +@MockServerNntpTest class NzbHealthCheckIT { @Autowired @@ -63,7 +61,6 @@ class NzbHealthCheckIT { @LocalServerPort var randomServerPort: Int = 0 - private val deserializer = Json { ignoreUnknownKeys = true } private val sardine = SardineFactory.begin() private val createdReleases = mutableListOf() @@ -187,42 +184,8 @@ class NzbHealthCheckIT { .expectStatus().is2xxSuccessful } - @Suppress("NestedBlockDepth") - private fun waitForCompletion(releaseName: String) { - val historyParts = MultipartBodyBuilder() - historyParts.part("mode", "history") - historyParts.part("cat", "testcat") - - var completed = false - var lastStatus = "unknown" - var attempts = 0 - while (attempts < 30 && !completed) { - Thread.sleep(1000) - webTestClient.post().uri("/api") - .body(BodyInserters.fromMultipartData(historyParts.build())) - .exchange() - .expectStatus().is2xxSuccessful - .expectBody(String::class.java) - .returnResult().responseBody - ?.let { historyBody -> - val history = deserializer.decodeFromString(historyBody) - val slot = history.history.slots.firstOrNull { it.name == releaseName } - slot?.let { - lastStatus = it.status - if (it.status == "COMPLETED" || it.status == "FAILED") { - completed = it.status == "COMPLETED" - } - } - } - attempts++ - } - - assertThat( - "Import should complete within timeout (last status: $lastStatus)", - completed, - `is`(true) - ) - } + private fun waitForCompletion(releaseName: String) = + webTestClient.awaitSabImportCompletion(releaseName) @Suppress("TooGenericExceptionCaught") private fun waitForMockServerVerification( diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/NzbImportIT.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/NzbImportIT.kt index bb3ba934..63407f48 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/NzbImportIT.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/NzbImportIT.kt @@ -9,12 +9,12 @@ import io.skjaere.debridav.MiltonConfiguration import io.skjaere.debridav.repository.NzbDocumentRepository import io.skjaere.debridav.repository.UsenetRepository import io.skjaere.debridav.test.integrationtest.config.IntegrationTestContextConfiguration -import io.skjaere.debridav.test.integrationtest.config.MockServerTest +import io.skjaere.debridav.test.integrationtest.config.MockServerNntpTest +import io.skjaere.debridav.test.integrationtest.config.awaitSabImportCompletion +import io.skjaere.debridav.test.integrationtest.config.awaitSabImportFailure import io.skjaere.debridav.usenet.nzb.NzbArchiveType -import io.skjaere.debridav.usenet.sabnzbd.model.SabnzbdFullHistoryResponse import io.skjaere.mocknntp.testcontainer.MockNntpServerContainer import kotlinx.coroutines.runBlocking -import kotlinx.serialization.json.Json import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers.hasItem import org.hamcrest.Matchers.hasProperty @@ -33,9 +33,9 @@ import org.springframework.web.reactive.function.BodyInserters @SpringBootTest( classes = [DebriDavApplication::class, IntegrationTestContextConfiguration::class, MiltonConfiguration::class], webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - properties = ["debridav.debrid-clients=easynews", "nntp.enabled=true"] + properties = ["debridav.debrid-clients=easynews"] ) -@MockServerTest +@MockServerNntpTest class NzbImportIT { @Autowired @@ -53,7 +53,6 @@ class NzbImportIT { @LocalServerPort var randomServerPort: Int = 0 - private val deserializer = Json { ignoreUnknownKeys = true } private val sardine = SardineFactory.begin() private val createdReleases = mutableListOf() @@ -64,7 +63,7 @@ class NzbImportIT { for (releaseName in createdReleases) { @Suppress("TooGenericExceptionCaught") try { - sardine.delete("http://localhost:${randomServerPort}/downloads/$releaseName") + sardine.delete("http://localhost:${randomServerPort}/webdav/downloads/$releaseName") } catch (_: Exception) { // directory may not exist if import failed } @@ -162,7 +161,7 @@ class NzbImportIT { // verify WebDAV still has the release directory with the file assertThat( - sardine.list("http://localhost:${randomServerPort}/downloads/$releaseName"), + sardine.list("http://localhost:${randomServerPort}/webdav/downloads/$releaseName"), hasItem(hasProperty("displayName", `is`("testfile.bin"))) ) } @@ -186,13 +185,13 @@ class NzbImportIT { // verify WebDAV has the release directory assertThat( - sardine.list("http://localhost:${randomServerPort}/downloads/"), + sardine.list("http://localhost:${randomServerPort}/webdav/downloads/"), hasItem(hasProperty("displayName", `is`(releaseName))) ) // verify WebDAV has the extracted file inside the release directory assertThat( - sardine.list("http://localhost:${randomServerPort}/downloads/$releaseName"), + sardine.list("http://localhost:${randomServerPort}/webdav/downloads/$releaseName"), hasItem(hasProperty("displayName", `is`("testfile.bin"))) ) @@ -225,80 +224,9 @@ class NzbImportIT { .body(BodyInserters.fromMultipartData(parts.build())).exchange().expectStatus().is2xxSuccessful } - @Suppress("NestedBlockDepth") - private fun waitForFailure(releaseName: String) { - val historyParts = MultipartBodyBuilder() - historyParts.part("mode", "history") - historyParts.part("cat", "testcat") - - var failed = false - var lastStatus = "unknown" - var attempts = 0 - while (attempts < 30 && !failed) { - Thread.sleep(1000) - webTestClient.post().uri("/api") - .body(BodyInserters.fromMultipartData(historyParts.build())) - .exchange() - .expectStatus().is2xxSuccessful - .expectBody(String::class.java) - .returnResult().responseBody - ?.let { historyBody -> - val history = deserializer.decodeFromString(historyBody) - val slot = history.history.slots.firstOrNull { it.name == releaseName } - slot?.let { - lastStatus = it.status - if (it.status == "FAILED") { - failed = true - } - } - } - attempts++ - } - - assertThat( - "Import should fail within timeout (last status: $lastStatus)", - failed, - `is`(true) - ) - } + private fun waitForFailure(releaseName: String) = + webTestClient.awaitSabImportFailure(releaseName) - @Suppress("NestedBlockDepth") - private fun waitForCompletion(releaseName: String) { - val historyParts = MultipartBodyBuilder() - historyParts.part("mode", "history") - historyParts.part("cat", "testcat") - - var completed = false - var haveResponse = false - var lastStatus = "unknown" - var attemps = 0 - while (attemps < 30 && !haveResponse) { - Thread.sleep(1000) - webTestClient.post().uri("/api") - .body(BodyInserters.fromMultipartData(historyParts.build())) - .exchange() - .expectStatus().is2xxSuccessful - .expectBody(String::class.java) - .returnResult().responseBody - ?.let { historyBody -> - val history = deserializer.decodeFromString(historyBody) - val slot = history.history.slots.firstOrNull { it.name == releaseName } - slot?.let { - lastStatus = it.status - haveResponse = true - if (it.status == "COMPLETED" || it.status == "FAILED") { - completed = it.status == "COMPLETED" - break - } - } - } - attemps++ - } - - assertThat( - "Import should complete within timeout (last status: $lastStatus)", - completed, - `is`(true) - ) - } + private fun waitForCompletion(releaseName: String) = + webTestClient.awaitSabImportCompletion(releaseName) } diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/NzbStreamingIT.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/NzbStreamingIT.kt index d582aab3..7d923de7 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/NzbStreamingIT.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/NzbStreamingIT.kt @@ -6,11 +6,10 @@ import io.skjaere.debridav.DebriDavApplication import io.skjaere.debridav.MiltonConfiguration import io.skjaere.debridav.repository.UsenetRepository import io.skjaere.debridav.test.integrationtest.config.IntegrationTestContextConfiguration -import io.skjaere.debridav.test.integrationtest.config.MockServerTest -import io.skjaere.debridav.usenet.sabnzbd.model.SabnzbdFullHistoryResponse +import io.skjaere.debridav.test.integrationtest.config.MockServerNntpTest +import io.skjaere.debridav.test.integrationtest.config.awaitSabImportCompletion import io.skjaere.mocknntp.testcontainer.MockNntpServerContainer import kotlinx.coroutines.runBlocking -import kotlinx.serialization.json.Json import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers.`is` import org.junit.jupiter.api.AfterEach @@ -28,9 +27,9 @@ import java.net.URI @SpringBootTest( classes = [DebriDavApplication::class, IntegrationTestContextConfiguration::class, MiltonConfiguration::class], webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - properties = ["debridav.debrid-clients=easynews", "nntp.enabled=true"] + properties = ["debridav.debrid-clients=easynews"] ) -@MockServerTest +@MockServerNntpTest class NzbStreamingIT { @Autowired @@ -45,7 +44,6 @@ class NzbStreamingIT { @LocalServerPort var randomServerPort: Int = 0 - private val deserializer = Json { ignoreUnknownKeys = true } private val sardine = SardineFactory.begin() @AfterEach @@ -82,7 +80,8 @@ class NzbStreamingIT { waitForCompletion(releaseName) // verify - GET the file via WebDAV and check content matches - val downloadedBytes = sardine.get("http://localhost:$randomServerPort/downloads/$releaseName/testfile.bin") + val downloadedBytes = sardine + .get("http://localhost:$randomServerPort/webdav/downloads/$releaseName/testfile.bin") .use { it.readBytes() } assertThat( @@ -99,7 +98,7 @@ class NzbStreamingIT { // verify - range request returns correct partial content val rangeEnd = 1023 val rangeBytes = getWithRange( - "http://localhost:$randomServerPort/downloads/$releaseName/testfile.bin", + "http://localhost:$randomServerPort/webdav/downloads/$releaseName/testfile.bin", 0, rangeEnd ) @@ -116,47 +115,12 @@ class NzbStreamingIT { ) // cleanup - sardine.delete("http://localhost:$randomServerPort/downloads/$releaseName") + sardine.delete("http://localhost:$randomServerPort/webdav/downloads/$releaseName") usenetRepository.deleteAll() } - @Suppress("LoopWithTooManyJumpStatements") - private fun waitForCompletion(releaseName: String) { - val historyParts = MultipartBodyBuilder() - historyParts.part("mode", "history") - historyParts.part("cat", "testcat") - - var completed = false - var lastStatus = "unknown" - for (attempt in 1..30) { - Thread.sleep(1000) - val historyBody = webTestClient.post().uri("/api") - .body(BodyInserters.fromMultipartData(historyParts.build())) - .exchange() - .expectStatus().is2xxSuccessful - .expectBody(String::class.java) - .returnResult().responseBody ?: continue - - val history = deserializer.decodeFromString(historyBody) - val slot = history.history.slots.firstOrNull { it.name == releaseName } - if (slot != null) { - lastStatus = slot.status - if (slot.status == "COMPLETED") { - completed = true - break - } - if (slot.status == "FAILED") { - break - } - } - } - - assertThat( - "Import should complete within timeout (last status: $lastStatus)", - completed, - `is`(true) - ) - } + private fun waitForCompletion(releaseName: String) = + webTestClient.awaitSabImportCompletion(releaseName) private fun getWithRange(url: String, start: Int, end: Int): ByteArray { val connection = URI(url).toURL().openConnection() as HttpURLConnection diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/NzbStreamingRepairIT.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/NzbStreamingRepairIT.kt index 685b6b54..5f04f69d 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/NzbStreamingRepairIT.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/NzbStreamingRepairIT.kt @@ -7,11 +7,10 @@ import io.skjaere.debridav.MiltonConfiguration import io.skjaere.debridav.repository.NzbDocumentRepository import io.skjaere.debridav.repository.UsenetRepository import io.skjaere.debridav.test.integrationtest.config.IntegrationTestContextConfiguration -import io.skjaere.debridav.test.integrationtest.config.MockServerTest +import io.skjaere.debridav.test.integrationtest.config.MockServerNntpTest +import io.skjaere.debridav.test.integrationtest.config.awaitSabImportCompletion import io.skjaere.mocknntp.testcontainer.MockNntpServerContainer import kotlinx.coroutines.runBlocking -import kotlinx.serialization.json.Json -import io.skjaere.debridav.usenet.sabnzbd.model.SabnzbdFullHistoryResponse import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers.`is` import org.junit.jupiter.api.AfterEach @@ -34,12 +33,11 @@ import org.springframework.web.reactive.function.BodyInserters webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = [ "debridav.debrid-clients=easynews", - "nntp.enabled=true", "sonarr.integration-enabled=true", "sonarr.category=testcat" ] ) -@MockServerTest +@MockServerNntpTest class NzbStreamingRepairIT { @Autowired @@ -60,7 +58,6 @@ class NzbStreamingRepairIT { @LocalServerPort var randomServerPort: Int = 0 - private val deserializer = Json { ignoreUnknownKeys = true } private val sardine = SardineFactory.begin() private val createdReleases = mutableListOf() @@ -79,7 +76,7 @@ class NzbStreamingRepairIT { for (releaseName in createdReleases) { @Suppress("TooGenericExceptionCaught") try { - sardine.delete("http://localhost:$randomServerPort/downloads/$releaseName") + sardine.delete("http://localhost:$randomServerPort/webdav/downloads/$releaseName") } catch (_: Exception) { // directory may not exist if import failed } @@ -143,7 +140,7 @@ class NzbStreamingRepairIT { // when - trigger streaming (WebDAV GET), which hits ArticleNotFoundException @Suppress("TooGenericExceptionCaught") try { - sardine.get("http://localhost:$randomServerPort/downloads/$releaseName/testfile.bin") + sardine.get("http://localhost:$randomServerPort/webdav/downloads/$releaseName/testfile.bin") } catch (_: Exception) { // the stream will fail since the article is missing — that's expected } @@ -187,42 +184,8 @@ class NzbStreamingRepairIT { .expectStatus().is2xxSuccessful } - @Suppress("NestedBlockDepth") - private fun waitForCompletion(releaseName: String) { - val historyParts = MultipartBodyBuilder() - historyParts.part("mode", "history") - historyParts.part("cat", "testcat") - - var completed = false - var lastStatus = "unknown" - var attempts = 0 - while (attempts < 30 && !completed) { - Thread.sleep(1000) - webTestClient.post().uri("/api") - .body(BodyInserters.fromMultipartData(historyParts.build())) - .exchange() - .expectStatus().is2xxSuccessful - .expectBody(String::class.java) - .returnResult().responseBody - ?.let { historyBody -> - val history = deserializer.decodeFromString(historyBody) - val slot = history.history.slots.firstOrNull { it.name == releaseName } - slot?.let { - lastStatus = it.status - if (it.status == "COMPLETED" || it.status == "FAILED") { - completed = it.status == "COMPLETED" - } - } - } - attempts++ - } - - assertThat( - "Import should complete within timeout (last status: $lastStatus)", - completed, - `is`(true) - ) - } + private fun waitForCompletion(releaseName: String) = + webTestClient.awaitSabImportCompletion(releaseName) @Suppress("TooGenericExceptionCaught") private fun waitForMockServerVerification( diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/QBittorrentEmulationIT.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/QBittorrentEmulationIT.kt index 97f8e707..fad93c42 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/QBittorrentEmulationIT.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/QBittorrentEmulationIT.kt @@ -74,7 +74,7 @@ class QBittorrentEmulationIT { fun tearDown() { mockserverClient.reset() try { - sardine.delete("http://localhost:${randomServerPort}/downloads/test") + sardine.delete("http://localhost:${randomServerPort}/webdav/downloads/test") } catch (_: Throwable) { } } @@ -150,19 +150,19 @@ class QBittorrentEmulationIT { (debridFileContents?.debridLinks!!.first() as CachedFile).link ) sardine.move( - "http://localhost:${randomServerPort}/downloads/test/a/b/c/movie.mkv", - "http://localhost:${randomServerPort}/movie.mkv" + "http://localhost:${randomServerPort}/webdav/downloads/test/a/b/c/movie.mkv", + "http://localhost:${randomServerPort}/webdav/movie.mkv" ) assertThat( - sardine.list("http://localhost:${randomServerPort}/"), hasItem( + sardine.list("http://localhost:${randomServerPort}/webdav/"), hasItem( hasProperty( "displayName", `is`("movie.mkv") ) ) ) - sardine.delete("http://localhost:${randomServerPort}/movie.mkv") + sardine.delete("http://localhost:${randomServerPort}/webdav/movie.mkv") assertThat( - sardine.list("http://localhost:${randomServerPort}/"), not( + sardine.list("http://localhost:${randomServerPort}/webdav/"), not( hasItem( hasProperty( "displayName", `is`("/movie.mkv") @@ -325,7 +325,7 @@ class QBittorrentEmulationIT { ) // finally - sardine.delete("http://localhost:${randomServerPort}/downloads/second-name") + sardine.delete("http://localhost:${randomServerPort}/webdav/downloads/second-name") } @Test diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/QueueFileResolutionIT.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/QueueFileResolutionIT.kt new file mode 100644 index 00000000..692d592e --- /dev/null +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/QueueFileResolutionIT.kt @@ -0,0 +1,147 @@ +package io.skjaere.debridav.test.integrationtest + +import com.github.sardine.SardineFactory +import io.skjaere.compressionutils.generation.ContainerType +import io.skjaere.debridav.DebriDavApplication +import io.skjaere.debridav.MiltonConfiguration +import io.skjaere.debridav.fs.DatabaseFileService +import io.skjaere.debridav.fs.DbDirectory +import io.skjaere.debridav.repository.NzbImportRepository +import io.skjaere.debridav.repository.UsenetRepository +import io.skjaere.debridav.test.integrationtest.config.IntegrationTestContextConfiguration +import io.skjaere.debridav.test.integrationtest.config.MockServerNntpTest +import io.skjaere.debridav.test.integrationtest.config.awaitSabImportCompletion +import io.skjaere.debridav.usenet.queue.NzbImportFileJson +import io.skjaere.mocknntp.testcontainer.MockNntpServerContainer +import kotlinx.coroutines.runBlocking +import org.hamcrest.MatcherAssert.assertThat +import org.hamcrest.Matchers.`is` +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.boot.test.web.server.LocalServerPort +import org.springframework.http.MediaType +import org.springframework.http.client.MultipartBodyBuilder +import org.springframework.test.web.reactive.server.WebTestClient +import org.springframework.web.reactive.function.BodyInserters +import tools.jackson.module.kotlin.jacksonObjectMapper + +@SpringBootTest( + classes = [DebriDavApplication::class, IntegrationTestContextConfiguration::class, MiltonConfiguration::class], + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = ["debridav.debrid-clients=easynews"] +) +@MockServerNntpTest +class QueueFileResolutionIT { + + @Autowired + private lateinit var usenetRepository: UsenetRepository + + @Autowired + private lateinit var nzbImportRepository: NzbImportRepository + + @Autowired + private lateinit var databaseFileService: DatabaseFileService + + @Autowired + private lateinit var webTestClient: WebTestClient + + @Autowired + private lateinit var mockNntpServerContainer: MockNntpServerContainer + + @LocalServerPort + var randomServerPort: Int = 0 + + private val sardine = SardineFactory.begin() + + @AfterEach + fun tearDown() { + runBlocking { mockNntpServerContainer.client.clearYencBodyExpectations() } + @Suppress("TooGenericExceptionCaught") + try { + sardine.delete("http://localhost:${randomServerPort}/movies/") + } catch (_: Exception) { } + @Suppress("TooGenericExceptionCaught") + try { + sardine.delete("http://localhost:${randomServerPort}/downloads/queue-file-resolution-test") + } catch (_: Exception) { } + usenetRepository.deleteAll() + nzbImportRepository.deleteAll() + } + + @Test + fun `queue files endpoint returns updated paths after file is moved`() { + val releaseName = "queue-file-resolution-test" + + // given - prepare and import an NZB + val testData = ByteArray(32 * 1024) { (it % 256).toByte() } + val nzbXml = runBlocking { + mockNntpServerContainer.client.prepareArchiveNzb( + fileContents = mapOf("testfile.bin" to testData), + containerType = ContainerType.RAR5 + ) + } + uploadNzb(nzbXml, releaseName) + waitForCompletion(releaseName) + + // find the import record id by name (other tests may leave records behind) + val importId = nzbImportRepository.findAll() + .first { it.name == releaseName }.id!! + + // verify initial file path points to downloads/ + val initialFiles = getQueueItemFiles(importId) + assertThat("Should have one file", initialFiles.size, `is`(1)) + assertThat( + "Initial path should be under /downloads, got: ${initialFiles[0].path}", + initialFiles[0].path.startsWith("/downloads/$releaseName/"), + `is`(true) + ) + + // when - move the release directory using DatabaseFileService + val releaseDir = databaseFileService.getFileAtPath("/downloads/$releaseName") as DbDirectory + databaseFileService.createDirectory("/movies") + databaseFileService.moveResource(releaseDir, "/movies/$releaseName", releaseName) + + // then - queue files endpoint should return the new path + val movedFiles = getQueueItemFiles(importId) + assertThat("Should still have one file", movedFiles.size, `is`(1)) + assertThat( + "Path should now be under /movies, got: ${movedFiles[0].path}", + movedFiles[0].path.startsWith("/movies/$releaseName/"), + `is`(true) + ) + assertThat( + "File name should be preserved", + movedFiles[0].path.endsWith("/testfile.bin"), + `is`(true) + ) + } + + private fun getQueueItemFiles(importId: Long): List { + val body = webTestClient.get().uri("/api/v1/queue/$importId/files") + .exchange() + .expectStatus().is2xxSuccessful + .expectBody(String::class.java) + .returnResult().responseBody!! + + val mapper = jacksonObjectMapper() + val type = mapper.typeFactory + .constructCollectionType(List::class.java, NzbImportFileJson::class.java) + return mapper.readValue(body, type) + } + + private fun uploadNzb(nzbXml: String, releaseName: String) { + val parts = MultipartBodyBuilder() + parts.part("mode", "addfile") + parts.part("cat", "testcat") + parts.part("name", nzbXml.toByteArray(Charsets.UTF_8)) + .header("Content-Disposition", "form-data; name=name; filename=$releaseName.nzb") + + webTestClient.post().uri("/api").contentType(MediaType.APPLICATION_JSON) + .body(BodyInserters.fromMultipartData(parts.build())).exchange().expectStatus().is2xxSuccessful + } + + private fun waitForCompletion(releaseName: String) = + webTestClient.awaitSabImportCompletion(releaseName) +} diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/RealDebridClientIT.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/RealDebridClientIT.kt index 6130b9d6..f3752201 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/RealDebridClientIT.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/RealDebridClientIT.kt @@ -124,7 +124,7 @@ class RealDebridClientIT { // then assertThat( sardine.list( - "http://localhost:${randomServerPort}/downloads/" + + "http://localhost:${randomServerPort}/webdav/downloads/" + "Vengeance.Valley.1951.DVDRip.x264.EAC3-SARTRE/Vengeance.Valley.1951.DVDRip.x264.EAC3-SARTRE" ), hasItem( @@ -138,7 +138,7 @@ class RealDebridClientIT { realdebridTorrentRepository.deleteAll() realDebridDownloadRepository.deleteAll() sardine.delete( - "http://localhost:${randomServerPort}/downloads/" + + "http://localhost:${randomServerPort}/webdav/downloads/" + "Vengeance.Valley.1951.DVDRip.x264.EAC3-SARTRE" ) } @@ -187,7 +187,7 @@ class RealDebridClientIT { // then assertThat( sardine.list( - "http://localhost:${randomServerPort}/downloads/" + + "http://localhost:${randomServerPort}/webdav/downloads/" + "Vengeance.Valley.1951.DVDRip.x264.EAC3-SARTRE/Vengeance.Valley.1951.DVDRip.x264.EAC3-SARTRE" ), hasItem( @@ -199,7 +199,7 @@ class RealDebridClientIT { assertThat( sardine.list( - "http://localhost:${randomServerPort}/downloads/" + + "http://localhost:${randomServerPort}/webdav/downloads/" + "Vengeance.Valley.1951.DVDRip.x264.EAC3-SARTRE/Vengeance.Valley.1951.DVDRip.x264.EAC3-SARTRE" ), hasItem( @@ -213,7 +213,9 @@ class RealDebridClientIT { //finally realdebridTorrentRepository.deleteAll() realDebridDownloadRepository.deleteAll() - sardine.delete("http://localhost:${randomServerPort}/downloads/Vengeance.Valley.1951.DVDRip.x264.EAC3-SARTRE") + sardine.delete( + "http://localhost:${randomServerPort}/webdav/downloads/Vengeance.Valley.1951.DVDRip.x264.EAC3-SARTRE" + ) } @Test @@ -263,7 +265,7 @@ class RealDebridClientIT { //then assertThat( sardine.list( - "http://localhost:${randomServerPort}/downloads/" + + "http://localhost:${randomServerPort}/webdav/downloads/" + "Vengeance.Valley.1951.DVDRip.x264.EAC3-SARTRE/Vengeance.Valley.1951.DVDRip.x264.EAC3-SARTRE" ), hasItem( @@ -279,7 +281,7 @@ class RealDebridClientIT { realdebridTorrentRepository.deleteAll() realDebridDownloadRepository.deleteAll() sardine.delete( - "http://localhost:${randomServerPort}/downloads/" + + "http://localhost:${randomServerPort}/webdav/downloads/" + "Vengeance.Valley.1951.DVDRip.x264.EAC3-SARTRE" ) } @@ -335,7 +337,7 @@ class RealDebridClientIT { .responseTimeout(Duration.ofMillis(3000000)) .build() .get() - .uri("/downloads/testfile.mp4") + .uri("/webdav/downloads/testfile.mp4") .exchange() .expectStatus().is2xxSuccessful() .expectBody().equals("it works!") @@ -346,7 +348,7 @@ class RealDebridClientIT { //finally sardine.delete( - "http://localhost:${randomServerPort}/downloads/testfile.mp4" + "http://localhost:${randomServerPort}/webdav/downloads/testfile.mp4" ) } diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/SabNzbdEmulationIT.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/SabNzbdEmulationIT.kt index c35ed62e..8931ff68 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/SabNzbdEmulationIT.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/SabNzbdEmulationIT.kt @@ -91,7 +91,7 @@ class SabNzbdEmulationIT { // then assertThat( - sardine.list("http://localhost:${randomServerPort}/downloads/"), hasItem( + sardine.list("http://localhost:${randomServerPort}/webdav/downloads/"), hasItem( hasProperty( "displayName", `is`("releaseName") ) @@ -99,7 +99,7 @@ class SabNzbdEmulationIT { ) assertThat( - sardine.list("http://localhost:${randomServerPort}/downloads/releaseName"), hasItem( + sardine.list("http://localhost:${randomServerPort}/webdav/downloads/releaseName"), hasItem( hasProperty( "displayName", `is`("releaseName.mkv") ) @@ -129,7 +129,7 @@ class SabNzbdEmulationIT { .expectStatus().is2xxSuccessful.expectBody().jsonPath("$.history.slots").isEmpty - sardine.delete("http://localhost:${randomServerPort}/downloads/releaseName") + sardine.delete("http://localhost:${randomServerPort}/webdav/downloads/releaseName") } @@ -157,7 +157,7 @@ class SabNzbdEmulationIT { webTestClient.post().uri("/api").contentType(MediaType.APPLICATION_JSON) .body(BodyInserters.fromMultipartData(addNzbParts.build())).exchange().expectStatus().is2xxSuccessful } - sardine.delete("http://localhost:${randomServerPort}/downloads/releaseName") + sardine.delete("http://localhost:${randomServerPort}/webdav/downloads/releaseName") usenetRepository.deleteAll() } @@ -204,7 +204,7 @@ class SabNzbdEmulationIT { .expectStatus().is2xxSuccessful.expectBody() .jsonPath("$.history.slots.length()").isEqualTo(preDeleteHistory.history.slots.size - 1) - sardine.delete("http://localhost:${randomServerPort}/downloads/releaseName") + sardine.delete("http://localhost:${randomServerPort}/webdav/downloads/releaseName") usenetRepository.deleteAll() } @@ -288,8 +288,8 @@ class SabNzbdEmulationIT { ) // finally - sardine.delete("http://localhost:${randomServerPort}/downloads/releaseName") - sardine.delete("http://localhost:${randomServerPort}/downloads/secondReleaseName") + sardine.delete("http://localhost:${randomServerPort}/webdav/downloads/releaseName") + sardine.delete("http://localhost:${randomServerPort}/webdav/downloads/secondReleaseName") usenetRepository.deleteAll() } diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/StreamingEofIT.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/StreamingEofIT.kt new file mode 100644 index 00000000..c780b30d --- /dev/null +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/StreamingEofIT.kt @@ -0,0 +1,115 @@ +package io.skjaere.debridav.test.integrationtest + +import ch.qos.logback.classic.Level +import ch.qos.logback.classic.Logger +import ch.qos.logback.classic.spi.ILoggingEvent +import ch.qos.logback.core.read.ListAppender +import io.skjaere.debridav.DebriDavApplication +import io.skjaere.debridav.MiltonConfiguration +import io.skjaere.debridav.debrid.DebridProvider +import io.skjaere.debridav.fs.CachedFile +import io.skjaere.debridav.fs.DatabaseFileService +import io.skjaere.debridav.repository.DebridFileContentsRepository +import io.skjaere.debridav.stream.StreamingService +import io.skjaere.debridav.test.debridFileContents +import io.skjaere.debridav.test.deepCopy +import io.skjaere.debridav.test.integrationtest.config.ContentStubbingService +import io.skjaere.debridav.test.integrationtest.config.IntegrationTestContextConfiguration +import io.skjaere.debridav.test.integrationtest.config.MockServerTest +import org.apache.commons.codec.digest.DigestUtils +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Test +import org.mockserver.integration.ClientAndServer +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.test.web.reactive.server.WebTestClient +import java.time.Duration +import java.time.Instant +import kotlin.test.assertFalse + +@SpringBootTest( + classes = [DebriDavApplication::class, IntegrationTestContextConfiguration::class, MiltonConfiguration::class], + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT +) +@MockServerTest +class StreamingEofIT { + @Autowired + private lateinit var databaseFileService: DatabaseFileService + + @Autowired + private lateinit var webTestClient: WebTestClient + + @Autowired + private lateinit var contentStubbingService: ContentStubbingService + + @Autowired + lateinit var debridFileContentsRepository: DebridFileContentsRepository + + @Autowired + lateinit var mockserverClient: ClientAndServer + + @AfterEach + fun tearDown() { + mockserverClient.reset() + } + + @Test + fun `that premature EOF from upstream does not result in ERROR log`() { + // given - file claims to be 1000 bytes but server only returns 9 bytes ("it works!") + // This simulates a debrid provider sending fewer bytes than promised, causing premature EOF + val fileContents = debridFileContents.deepCopy() + val hash = DigestUtils.md5Hex("eof-test") + fileContents.size = 1000L + + val debridLink = CachedFile( + "testfile-eof.mp4", + link = "http://localhost:${contentStubbingService.port}/truncatedLink", + size = 1000L, + provider = DebridProvider.PREMIUMIZE, + lastChecked = Instant.now().toEpochMilli(), + params = mapOf(), + mimeType = "video/mp4" + ) + fileContents.debridLinks = mutableListOf(debridLink) + contentStubbingService.mockTruncatedStream() + databaseFileService.createDebridFile("/testfile-eof.mp4", hash, fileContents) + .let { debridFileContentsRepository.save(it) } + + // Capture StreamingService logs to verify no ERROR-level log is generated for EOF + val streamingLogger = LoggerFactory.getLogger(StreamingService::class.java) as Logger + val listAppender = ListAppender() + listAppender.start() + streamingLogger.addAppender(listAppender) + + try { + // when - make request; server sends 9 bytes but we expect 1000, triggering premature EOF + // The EOFException thrown by readAvailable returning -1 before all bytes are consumed + // should be handled gracefully at WARN level (not ERROR), so Sentry does not capture it + try { + webTestClient + .mutate().responseTimeout(Duration.ofMillis(30000)).build() + .get() + .uri("/testfile-eof.mp4") + .exchange() + } catch (_: Exception) { + // Connection may close early due to EOF handling upstream; this is expected + } + + // then - no ERROR log should be generated for premature EOF from upstream HTTP stream. + // Before the fix, EOFException was caught by the generic Exception handler which logged + // at ERROR level, causing Sentry to fire false alerts for normal network conditions. + // After the fix, it is caught by the kotlinx.io.IOException handler and logged at WARN. + val errorLogs = listAppender.list.filter { + it.level == Level.ERROR && it.formattedMessage.contains("An error occurred during streaming") + } + assertFalse( + errorLogs.isNotEmpty(), + "Expected no ERROR log for premature EOF from upstream, but found: " + + errorLogs.map { it.formattedMessage } + ) + } finally { + streamingLogger.detachAppender(listAppender) + } + } +} diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/TorrentHealthCheckIT.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/TorrentHealthCheckIT.kt new file mode 100644 index 00000000..f028c141 --- /dev/null +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/TorrentHealthCheckIT.kt @@ -0,0 +1,352 @@ +package io.skjaere.debridav.test.integrationtest + +import io.skjaere.debridav.DebriDavApplication +import io.skjaere.debridav.MiltonConfiguration +import io.skjaere.debridav.category.Category +import io.skjaere.debridav.category.CategoryRepository +import io.skjaere.debridav.debrid.DebridProvider +import io.skjaere.debridav.fs.DatabaseFileService +import io.skjaere.debridav.fs.DebridCachedTorrentContent +import io.skjaere.debridav.fs.MissingFile +import io.skjaere.debridav.health.RepairAction +import io.skjaere.debridav.health.RepairOutcomeRepository +import io.skjaere.debridav.test.MAGNET +import io.skjaere.debridav.test.integrationtest.config.IntegrationTestContextConfiguration +import io.skjaere.debridav.test.integrationtest.config.MockServerTest +import io.skjaere.debridav.test.integrationtest.config.PremiumizeStubbingService +import io.skjaere.debridav.torrent.Status +import io.skjaere.debridav.torrent.Torrent +import io.skjaere.debridav.torrent.TorrentHealthCheckService +import io.skjaere.debridav.torrent.TorrentRepository +import org.hamcrest.MatcherAssert.assertThat +import org.hamcrest.Matchers.`is` +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockserver.integration.ClientAndServer +import org.mockserver.model.HttpRequest.request +import org.mockserver.model.HttpResponse.response +import org.mockserver.model.MediaType +import org.mockserver.verify.VerificationTimes +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import java.time.Instant + +@SpringBootTest( + classes = [DebriDavApplication::class, IntegrationTestContextConfiguration::class, MiltonConfiguration::class], + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = [ + "debridav.debrid-clients=premiumize", + "sonarr.integration-enabled=true", + "sonarr.category=tv", + "health-check.repair-enabled=true" + ] +) +@MockServerTest +class TorrentHealthCheckIT { + + @Autowired + private lateinit var torrentRepository: TorrentRepository + + @Autowired + private lateinit var categoryRepository: CategoryRepository + + @Autowired + private lateinit var databaseFileService: DatabaseFileService + + @Autowired + private lateinit var torrentHealthCheckService: TorrentHealthCheckService + + @Autowired + private lateinit var premiumizeStubbingService: PremiumizeStubbingService + + @Autowired + private lateinit var repairOutcomeRepository: RepairOutcomeRepository + + @Autowired + private lateinit var mockServer: ClientAndServer + + @BeforeEach + fun setUp() { + torrentRepository.deleteAll() + repairOutcomeRepository.deleteAll() + mockServer.reset() + } + + @AfterEach + fun tearDown() { + torrentRepository.deleteAll() + repairOutcomeRepository.deleteAll() + mockServer.reset() + } + + @Test + @Suppress("LongMethod") + fun `unhealthy torrent triggers Arr blocklist and search`() { + // given — create a torrent with a file whose only debrid link is MissingFile + val category = categoryRepository.findByNameIgnoreCase("tv") + ?: categoryRepository.save(Category("tv", "/data/downloads/tv")) + + val contents = DebridCachedTorrentContent( + originalPath = "movie.mkv", + size = 1_000_000L, + modified = Instant.EPOCH.toEpochMilli(), + magnet = MAGNET, + mimeType = "video/x-matroska", + debridLinks = mutableListOf( + MissingFile(DebridProvider.PREMIUMIZE, Instant.EPOCH.toEpochMilli()) + ) + ) + + val file = databaseFileService.createDebridFile( + "/downloads/tv/test-torrent/movie.mkv", + "testhash123", + contents + ) + databaseFileService.saveDbEntity(file) + + val torrent = Torrent().apply { + name = "test-torrent" + this.category = category + hash = "testhash123" + savePath = "/data/downloads/tv" + status = Status.LIVE + lastVerified = null + } + torrentRepository.save(torrent) + torrent.files = mutableListOf(file) + torrentRepository.save(torrent) + + // given — premiumize says "not cached" when the health check re-verifies + premiumizeStubbingService.mockIsNotCached() + + // given — Sonarr mock: history lookup for blocklisting + mockServer.`when`( + request() + .withMethod("GET") + .withPath("/sonarr/api/v3/history") + ).respond( + response() + .withStatusCode(200) + .withContentType(MediaType.APPLICATION_JSON) + .withBody("""{"page": 1, "pageSize": 1, "totalRecords": 1, "records": [{"id": 42}]}""") + ) + + // given — Sonarr mock: mark history record as failed + mockServer.`when`( + request() + .withMethod("POST") + .withPath("/sonarr/api/v3/history/failed/42") + ).respond( + response().withStatusCode(200) + ) + + // given — Sonarr mock: parse endpoint for deleteFileAndSearch + mockServer.`when`( + request() + .withMethod("GET") + .withPath("/sonarr/api/v3/parse") + ).respond( + response() + .withStatusCode(200) + .withContentType(MediaType.APPLICATION_JSON) + .withBody("""{"episodes": [{"id": 1, "episodeFileId": 10}]}""") + ) + + // given — Sonarr mock: delete episode file + mockServer.`when`( + request() + .withMethod("DELETE") + .withPath("/sonarr/api/v3/episodefile/10") + ).respond( + response().withStatusCode(200) + ) + + // given — Sonarr mock: command (search) + mockServer.`when`( + request() + .withMethod("POST") + .withPath("/sonarr/api/v3/command") + ).respond( + response().withStatusCode(200) + ) + + // when — trigger health check (enqueues to PGMQ, processed asynchronously) + torrentHealthCheckService.triggerFullHealthCheck() + + // then — wait for the full check → repair pipeline to complete + waitForMockServerVerification { + // blocklist: history lookup + mark failed + mockServer.verify( + request() + .withMethod("GET") + .withPath("/sonarr/api/v3/history"), + VerificationTimes.atLeast(1) + ) + mockServer.verify( + request() + .withMethod("POST") + .withPath("/sonarr/api/v3/history/failed/42"), + VerificationTimes.atLeast(1) + ) + + // deleteFileAndSearch: parse + delete + command + mockServer.verify( + request() + .withMethod("GET") + .withPath("/sonarr/api/v3/parse"), + VerificationTimes.atLeast(1) + ) + mockServer.verify( + request() + .withMethod("DELETE") + .withPath("/sonarr/api/v3/episodefile/10"), + VerificationTimes.atLeast(1) + ) + mockServer.verify( + request() + .withMethod("POST") + .withPath("/sonarr/api/v3/command"), + VerificationTimes.atLeast(1) + ) + } + + // then — repair outcome recorded as REPAIRED + waitForCondition("repair outcome recorded") { + repairOutcomeRepository.findAll().toList().any { it.action == RepairAction.REPAIRED } + } + } + + @Test + fun `healthy torrent updates lastVerified without enqueuing repair`() { + // given — create a torrent with a file that has a working CachedFile link + val contents = DebridCachedTorrentContent( + originalPath = "healthy-movie.mkv", + size = 500_000L, + modified = Instant.EPOCH.toEpochMilli(), + magnet = MAGNET, + mimeType = "video/x-matroska", + debridLinks = mutableListOf( + MissingFile(DebridProvider.PREMIUMIZE, Instant.EPOCH.toEpochMilli()) + ) + ) + + val file = databaseFileService.createDebridFile( + "/downloads/misc/healthy-torrent/healthy-movie.mkv", + "healthyhash456", + contents + ) + databaseFileService.saveDbEntity(file) + + val torrent = Torrent().apply { + name = "healthy-torrent" + hash = "healthyhash456" + savePath = "/data/downloads/misc" + status = Status.LIVE + lastVerified = null + } + torrentRepository.save(torrent) + torrent.files = mutableListOf(file) + torrentRepository.save(torrent) + + // given — premiumize returns cached (healthy) + premiumizeStubbingService.mockIsCached() + premiumizeStubbingService.mockCachedContents() + + // when + torrentHealthCheckService.triggerFullHealthCheck() + + // then — torrent gets lastVerified set, no repair messages sent + waitForCondition("lastVerified is set") { + val updated = torrentRepository.findById(torrent.id!!).orElse(null) + updated?.lastVerified != null + } + + val updated = torrentRepository.findById(torrent.id!!).get() + assertThat("lastVerified should be set", updated.lastVerified != null, `is`(true)) + assertThat("healthCheckEnqueuedAt should be cleared", updated.healthCheckEnqueuedAt == null, `is`(true)) + } + + @Test + fun `unhealthy torrent without Arr category deletes files`() { + // given — torrent with no category (no Arr client match) + val contents = DebridCachedTorrentContent( + originalPath = "orphan.mkv", + size = 200_000L, + modified = Instant.EPOCH.toEpochMilli(), + magnet = MAGNET, + mimeType = "video/x-matroska", + debridLinks = mutableListOf( + MissingFile(DebridProvider.PREMIUMIZE, Instant.EPOCH.toEpochMilli()) + ) + ) + + val file = databaseFileService.createDebridFile( + "/downloads/nocategory/orphan-torrent/orphan.mkv", + "orphanhash789", + contents + ) + databaseFileService.saveDbEntity(file) + + val torrent = Torrent().apply { + name = "orphan-torrent" + hash = "orphanhash789" + savePath = "/data/downloads/nocategory" + status = Status.LIVE + lastVerified = null + } + torrentRepository.save(torrent) + torrent.files = mutableListOf(file) + torrentRepository.save(torrent) + + // given — premiumize says not cached + premiumizeStubbingService.mockIsNotCached() + + // when + torrentHealthCheckService.triggerFullHealthCheck() + + // then — repair outcome recorded as DELETED (no Arr client to search) + waitForCondition("repair outcome recorded as DELETED") { + repairOutcomeRepository.findAll().toList().any { it.action == RepairAction.DELETED } + } + } + + @Suppress("TooGenericExceptionCaught") + private fun waitForMockServerVerification( + timeoutMs: Long = 30_000, + pollMs: Long = 500, + verification: () -> Unit + ) { + val deadline = System.currentTimeMillis() + timeoutMs + var lastError: Throwable? = null + while (System.currentTimeMillis() < deadline) { + try { + verification() + return + } catch (e: Throwable) { + lastError = e + Thread.sleep(pollMs) + } + } + throw AssertionError("Verification did not pass within ${timeoutMs}ms", lastError) + } + + @Suppress("TooGenericExceptionCaught") + private fun waitForCondition( + description: String, + timeoutMs: Long = 30_000, + pollMs: Long = 500, + condition: () -> Boolean + ) { + val deadline = System.currentTimeMillis() + timeoutMs + while (System.currentTimeMillis() < deadline) { + try { + if (condition()) return + } catch (_: Throwable) { + // ignore and retry + } + Thread.sleep(pollMs) + } + throw AssertionError("Condition '$description' not met within ${timeoutMs}ms") + } +} diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/WebDavAuthenticationIT.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/WebDavAuthenticationIT.kt index 2b403961..649b0d1c 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/WebDavAuthenticationIT.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/WebDavAuthenticationIT.kt @@ -32,7 +32,7 @@ class WebDavAuthenticationIT { fun `that unauthenticated request is rejected`() { val sardine = SardineFactory.begin() assertFailsWith { - sardine.list("http://localhost:${randomServerPort}/") + sardine.list("http://localhost:${randomServerPort}/webdav/") } } @@ -40,14 +40,14 @@ class WebDavAuthenticationIT { fun `that wrong credentials are rejected`() { val sardine = SardineFactory.begin("wronguser", "wrongpass") assertFailsWith { - sardine.list("http://localhost:${randomServerPort}/") + sardine.list("http://localhost:${randomServerPort}/webdav/") } } @Test fun `that correct credentials succeed`() { val sardine = SardineFactory.begin("testuser", "testpass") - val resources = sardine.list("http://localhost:${randomServerPort}/") + val resources = sardine.list("http://localhost:${randomServerPort}/webdav/") assertThat(resources.isEmpty(), `is`(false)) } } diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/WebDavOperationsIT.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/WebDavOperationsIT.kt index 6405f89a..f03a250d 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/WebDavOperationsIT.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/WebDavOperationsIT.kt @@ -16,6 +16,7 @@ import org.hamcrest.Matchers.hasProperty import org.hamcrest.Matchers.hasSize import org.hamcrest.Matchers.`is` import org.hamcrest.Matchers.not +import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired @@ -39,10 +40,17 @@ class WebDavOperationsIT { private val sardine = SardineFactory.begin() + private var baselineEntityCount: Int = -1 + + @BeforeEach + fun captureBaseline() { + baselineEntityCount = debridFileContentsRepository.findAll().toList().size + } + @Test fun thatCreatingFileInRootWorks() { //when - sardine.put("http://localhost:${randomServerPort}/testfile.txt", "test contents".byteInputStream()) + sardine.put("http://localhost:${randomServerPort}/webdav/testfile.txt", "test contents".byteInputStream()) //then val listOfFiles: List = listDirectory("/") @@ -62,7 +70,7 @@ class WebDavOperationsIT { @Test fun thatDeletingFileInRootWorks() { //given - sardine.put("http://localhost:${randomServerPort}/testfile.txt", "test contents".byteInputStream()) + sardine.put("http://localhost:${randomServerPort}/webdav/testfile.txt", "test contents".byteInputStream()) val listOfFiles: List = listDirectory("/") assertThat( listOfFiles, hasItem( @@ -91,7 +99,7 @@ class WebDavOperationsIT { @Test fun thatCreatingDirectoryInRootWorks() { //when - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory") val listOfFiles: List = listDirectory("/") //then @@ -111,7 +119,7 @@ class WebDavOperationsIT { @Test fun thatRenamingEmptyDirectoryInRootWorks() { //given - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory") assertThat( listDirectory("/"), hasItem( hasProperty( @@ -122,8 +130,8 @@ class WebDavOperationsIT { //when sardine.move( - "http://localhost:${randomServerPort}/testDirectory", - "http://localhost:${randomServerPort}/movedTestDirectory" + "http://localhost:${randomServerPort}/webdav/testDirectory", + "http://localhost:${randomServerPort}/webdav/movedTestDirectory" ) //then @@ -143,9 +151,9 @@ class WebDavOperationsIT { @Test fun thatRenamingPopulatedDirectoryInRootWorks() { //given - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory") sardine.put( - "http://localhost:${randomServerPort}/testDirectory/testfile.txt", + "http://localhost:${randomServerPort}/webdav/testDirectory/testfile.txt", "test contents".byteInputStream() ) assertThat( @@ -165,8 +173,8 @@ class WebDavOperationsIT { //when sardine.move( - "http://localhost:${randomServerPort}/testDirectory", - "http://localhost:${randomServerPort}/movedTestDirectory" + "http://localhost:${randomServerPort}/webdav/testDirectory", + "http://localhost:${randomServerPort}/webdav/movedTestDirectory" ) //then @@ -193,8 +201,8 @@ class WebDavOperationsIT { @Test fun thatRenamingEmptyDirectoryInBranchWorks() { //given - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory") - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory/nestedDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory/nestedDirectory") assertThat( listDirectory("/"), hasItem( hasProperty( @@ -212,8 +220,8 @@ class WebDavOperationsIT { //when sardine.move( - "http://localhost:${randomServerPort}/testDirectory/nestedDirectory", - "http://localhost:${randomServerPort}/testDirectory/renamedNestedDirectory" + "http://localhost:${randomServerPort}/webdav/testDirectory/nestedDirectory", + "http://localhost:${randomServerPort}/webdav/testDirectory/renamedNestedDirectory" ) //then @@ -233,10 +241,10 @@ class WebDavOperationsIT { @Test fun thatRenamingPopulatedDirectoryInBranchWorks() { //given - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory/") - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory/nestedDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory/") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory/nestedDirectory") sardine.put( - "http://localhost:${randomServerPort}/testDirectory/nestedDirectory/testfile.txt", + "http://localhost:${randomServerPort}/webdav/testDirectory/nestedDirectory/testfile.txt", "test contents".byteInputStream() ) assertThat( @@ -256,8 +264,8 @@ class WebDavOperationsIT { //when sardine.move( - "http://localhost:${randomServerPort}/testDirectory/nestedDirectory", - "http://localhost:${randomServerPort}/testDirectory/renamedNestedDirectory" + "http://localhost:${randomServerPort}/webdav/testDirectory/nestedDirectory", + "http://localhost:${randomServerPort}/webdav/testDirectory/renamedNestedDirectory" ) //then @@ -277,9 +285,9 @@ class WebDavOperationsIT { @Test fun thatMovingDirectoryToNestedDirectoryWorks() { //given - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory") - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory/nestedTestDirectory") - sardine.createDirectory("http://localhost:${randomServerPort}/directoryToBeMoved") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory/nestedTestDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/directoryToBeMoved") assertThat( listDirectory("/"), allOf( hasItem( @@ -304,8 +312,8 @@ class WebDavOperationsIT { //when sardine.move( - "http://localhost:${randomServerPort}/directoryToBeMoved", - "http://localhost:${randomServerPort}/testDirectory/nestedTestDirectory/directoryToBeMoved" + "http://localhost:${randomServerPort}/webdav/directoryToBeMoved", + "http://localhost:${randomServerPort}/webdav/testDirectory/nestedTestDirectory/directoryToBeMoved" ) //then @@ -335,10 +343,10 @@ class WebDavOperationsIT { @Test fun thatMovingDirectoryWithFilesWorks() { //given - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory") - sardine.createDirectory("http://localhost:${randomServerPort}/destinationDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/destinationDirectory") sardine.put( - "http://localhost:${randomServerPort}/testDirectory/testfile.txt", + "http://localhost:${randomServerPort}/webdav/testDirectory/testfile.txt", "test contents".byteInputStream() ) assertThat( @@ -365,8 +373,8 @@ class WebDavOperationsIT { //when sardine.move( - "http://localhost:${randomServerPort}/testDirectory", - "http://localhost:${randomServerPort}/destinationDirectory/testDirectory" + "http://localhost:${randomServerPort}/webdav/testDirectory", + "http://localhost:${randomServerPort}/webdav/destinationDirectory/testDirectory" ) //then @@ -396,9 +404,9 @@ class WebDavOperationsIT { @Test fun thatMovingDirectoryWithSubdirectoriesWorks() { //given - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory") - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory/subDirectory") - sardine.createDirectory("http://localhost:${randomServerPort}/destinationDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory/subDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/destinationDirectory") assertThat( listDirectory("/"), allOf( @@ -417,8 +425,8 @@ class WebDavOperationsIT { //when sardine.move( - "http://localhost:${randomServerPort}/testDirectory", - "http://localhost:${randomServerPort}/destinationDirectory/testDirectory" + "http://localhost:${randomServerPort}/webdav/testDirectory", + "http://localhost:${randomServerPort}/webdav/destinationDirectory/testDirectory" ) //then @@ -455,16 +463,16 @@ class WebDavOperationsIT { @Test fun thatMovingLocalEntityWorks() { //given - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory") sardine.put( - "http://localhost:${randomServerPort}/testfile.txt", + "http://localhost:${randomServerPort}/webdav/testfile.txt", "test contents".byteInputStream() ) //when sardine.move( - "http://localhost:${randomServerPort}/testfile.txt", - "http://localhost:${randomServerPort}/testDirectory/testfile.txt", + "http://localhost:${randomServerPort}/webdav/testfile.txt", + "http://localhost:${randomServerPort}/webdav/testDirectory/testfile.txt", ) // then @@ -482,9 +490,11 @@ class WebDavOperationsIT { @Test fun thatDeletingDirectoryBranchWorks() { //given - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory") - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory/subDirectory") - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory/subDirectory/secondSubDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory/subDirectory") + sardine.createDirectory( + "http://localhost:${randomServerPort}/webdav/testDirectory/subDirectory/secondSubDirectory" + ) assertThat( listDirectory("/"), allOf( @@ -515,7 +525,7 @@ class WebDavOperationsIT { ) //when - sardine.delete("http://localhost:${randomServerPort}/testDirectory") + sardine.delete("http://localhost:${randomServerPort}/webdav/testDirectory") //then assertThat( @@ -534,13 +544,13 @@ class WebDavOperationsIT { fun thatReadingLocalEntityWorks() { // given sardine.put( - "http://localhost:${randomServerPort}/testDirectory/testfile.txt", + "http://localhost:${randomServerPort}/webdav/testDirectory/testfile.txt", "test contents".byteInputStream() ) // when val response = sardine.get( - "http://localhost:${randomServerPort}/testDirectory/testfile.txt" + "http://localhost:${randomServerPort}/webdav/testDirectory/testfile.txt" ).readAllBytes().decodeToString() assertThat(response, `is`("test contents")) @@ -552,13 +562,13 @@ class WebDavOperationsIT { fun thatReadingLocalEntityWithRangeWorks() { // given sardine.put( - "http://localhost:${randomServerPort}/testDirectory/testfile.txt", + "http://localhost:${randomServerPort}/webdav/testDirectory/testfile.txt", "test contents".byteInputStream() ) // when val response = sardine.get( - "http://localhost:${randomServerPort}/testDirectory/testfile.txt", + "http://localhost:${randomServerPort}/webdav/testDirectory/testfile.txt", mapOf( "Range" to "bytes=0-0", ) @@ -575,7 +585,7 @@ class WebDavOperationsIT { val contents = IntRange(0, (1024 * 1024 * 2)).map { Byte.MIN_VALUE }.toByteArray() assertFailsWith { sardine.put( - "http://localhost:${randomServerPort}/testfile.txt", + "http://localhost:${randomServerPort}/webdav/testfile.txt", contents.inputStream() ) } @@ -584,17 +594,16 @@ class WebDavOperationsIT { private fun assertReset() { debridFileContentsRepository.findAll() .toList().let { - if (it.size != 4) { + if (it.size != baselineEntityCount) { it.forEach { logger.error("item found ${it.name}") } } - assertThat(it, hasSize(4)) + assertThat(it, hasSize(baselineEntityCount)) } - } private fun listDirectory(path: String): List = - sardine.list("http://localhost:${randomServerPort}/$path") + sardine.list("http://localhost:${randomServerPort}/webdav/${path.trimStart('/')}") private fun deleteFile(path: String) = - sardine.delete("http://localhost:${randomServerPort}/$path") + sardine.delete("http://localhost:${randomServerPort}/webdav/${path.trimStart('/')}") } diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/ContentStubbingService.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/ContentStubbingService.kt index b4e5c1b7..7a62f641 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/ContentStubbingService.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/ContentStubbingService.kt @@ -2,6 +2,7 @@ package io.skjaere.debridav.test.integrationtest.config import org.mockserver.client.MockServerClient import org.mockserver.matchers.Times +import org.mockserver.model.Delay import org.mockserver.model.Header import org.mockserver.model.HttpRequest import org.mockserver.model.HttpResponse @@ -63,7 +64,7 @@ class ContentStubbingService(@Value("\${mockserver.port}") val port: Int) { //Times.exactly(1) ).respond( HttpResponse.response() - .withDelay(org.mockserver.model.Delay.milliseconds(500)) + .withDelay(Delay.milliseconds(500)) .withStatusCode(206) .withBody(oneHundredKilobytes) .withHeader(Header("content-range", "bytes $startByte-$endByte/${endByte + 1}")) @@ -155,6 +156,33 @@ class ContentStubbingService(@Value("\${mockserver.port}") val port: Int) { ) } + fun mockTruncatedStream() { + MockServerClient( + "localhost", + port + ).`when`( + HttpRequest.request() + .withMethod("GET") + .withPath("/truncatedLink"), + Times.exactly(1) + ).respond( + HttpResponse.response() + .withStatusCode(200) + .withBody("it works!".toByteArray()) + ) + MockServerClient( + "localhost", + port + ).`when`( + HttpRequest.request() + .withMethod("HEAD") + .withPath("/truncatedLink") + ).respond( + HttpResponse.response() + .withStatusCode(200) + ) + } + fun mockDeadLink() { MockServerClient( "localhost", diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/MockServerNntpTest.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/MockServerNntpTest.kt new file mode 100644 index 00000000..11890f3b --- /dev/null +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/MockServerNntpTest.kt @@ -0,0 +1,18 @@ +package io.skjaere.debridav.test.integrationtest.config + +import org.springframework.boot.webtestclient.autoconfigure.AutoConfigureWebTestClient +import org.springframework.test.context.ContextConfiguration + +/** + * Same wiring as [MockServerTest] but adds [NntpPoolInitializer] so the NNTP pool + * properties are populated from the mock NNTP container. Use on tests that exercise + * the NZB import or streaming path. + */ +@Target(AnnotationTarget.CLASS) +@Retention(AnnotationRetention.RUNTIME) +@ContextConfiguration( + initializers = [TestContextInitializer::class, NntpPoolInitializer::class], + classes = [PremiumizeStubbingService::class] +) +@AutoConfigureWebTestClient +annotation class MockServerNntpTest diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/NntpPoolInitializer.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/NntpPoolInitializer.kt new file mode 100644 index 00000000..9105399d --- /dev/null +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/NntpPoolInitializer.kt @@ -0,0 +1,22 @@ +package io.skjaere.debridav.test.integrationtest.config + +import org.springframework.boot.test.util.TestPropertyValues +import org.springframework.context.ApplicationContextInitializer +import org.springframework.context.ConfigurableApplicationContext + +/** + * Wires the mock NNTP server (started by [TestContextInitializer]) into `nntp.pools[0]`. + * Apply via `@ContextConfiguration(initializers = [NntpPoolInitializer::class])` on tests + * that exercise the NZB import / streaming path. Tests that don't need NNTP (e.g. + * easynews-only paths) should omit it so [SabNzbdService.isEasynewsOnlySetup] returns true. + */ +class NntpPoolInitializer : ApplicationContextInitializer { + override fun initialize(applicationContext: ConfigurableApplicationContext) { + val container = TestContextInitializer.mockNntpServerContainer + TestPropertyValues.of( + "nntp.pools[0].host=${container.nntpHost}", + "nntp.pools[0].port=${container.nntpPort}", + "nntp.pools[0].use-tls=false", + ).applyTo(applicationContext) + } +} diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/SabImportAwait.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/SabImportAwait.kt new file mode 100644 index 00000000..d3cfaf2e --- /dev/null +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/SabImportAwait.kt @@ -0,0 +1,54 @@ +package io.skjaere.debridav.test.integrationtest.config + +import io.skjaere.debridav.usenet.sabnzbd.model.SabnzbdFullHistoryResponse +import kotlinx.serialization.json.Json +import org.awaitility.Awaitility.await +import org.springframework.http.client.MultipartBodyBuilder +import org.springframework.test.web.reactive.server.WebTestClient +import org.springframework.web.reactive.function.BodyInserters +import java.time.Duration + +private val deserializer = Json { ignoreUnknownKeys = true } + +/** + * Polls the SABnzbd `/api?mode=history` endpoint until the named release reports + * COMPLETED. Throws if the slot reports FAILED, or if the timeout elapses without + * a COMPLETED status. + */ +fun WebTestClient.awaitSabImportCompletion( + releaseName: String, + timeout: Duration = Duration.ofSeconds(30), +) = awaitSabImportStatus(releaseName, expected = "COMPLETED", terminalErrors = setOf("FAILED"), timeout = timeout) + +/** + * Polls the SABnzbd `/api?mode=history` endpoint until the named release reports + * FAILED. Throws if the slot reports COMPLETED, or if the timeout elapses. + */ +fun WebTestClient.awaitSabImportFailure( + releaseName: String, + timeout: Duration = Duration.ofSeconds(30), +) = awaitSabImportStatus(releaseName, expected = "FAILED", terminalErrors = setOf("COMPLETED"), timeout = timeout) + +private fun WebTestClient.awaitSabImportStatus( + releaseName: String, + expected: String, + terminalErrors: Set, + timeout: Duration, +) { + await().atMost(timeout).until { + val historyParts = MultipartBodyBuilder().apply { part("mode", "history") } + val body = post().uri("/api") + .body(BodyInserters.fromMultipartData(historyParts.build())) + .exchange() + .expectStatus().is2xxSuccessful + .expectBody(String::class.java) + .returnResult().responseBody ?: return@until false + val slot = deserializer.decodeFromString(body) + .history.slots.firstOrNull { it.name == releaseName } + when (slot?.status) { + expected -> true + in terminalErrors -> error("Expected $expected but got ${slot?.status} for $releaseName") + else -> false + } + } +} diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/TestContextInitializer.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/TestContextInitializer.kt index 9ea0e8a0..437d1432 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/TestContextInitializer.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/TestContextInitializer.kt @@ -19,11 +19,16 @@ import java.io.File class TestContextInitializer : ApplicationContextInitializer { companion object { const val BASE_PATH = "/tmp/debridavtests" + val postgreSQLContainer: PostgreSQLContainer = PostgreSQLContainer(DockerImageName.parse("postgres:16-alpine")) .withUsername("postgres") .withPassword("postgres") .withDatabaseName("debridav") + // Each cached Spring context holds a Hikari pool open; with ~12 IT + // contexts and the default max_connections=100, the suite hits + // "FATAL: sorry, too many clients already". 300 buys plenty of headroom. + .withCommand("postgres", "-c", "max_connections=300") val mockNntpServerContainer: MockNntpServerContainer = MockNntpServerContainer() } @@ -37,7 +42,7 @@ class TestContextInitializer : ApplicationContextInitializer() { @@ -45,6 +50,9 @@ class TestContextInitializer : ApplicationContextInitializer