From e21718fabc0c7301296d5a6d1b7617c95641101c Mon Sep 17 00:00:00 2001 From: Skjaere Date: Sat, 28 Feb 2026 10:30:00 +0100 Subject: [PATCH 01/61] refactor(config): @ConfigProperty + runtime overrides via DB-backed property source Introduces a @ConfigProperty annotation that flags properties as user-configurable, combined with a DatabasePropertySource that lets runtime overrides persist to postgres and refresh Spring beans without a restart. Includes: - @ConfigProperty annotations + advanced flag + property type metadata - File browser API behind the config API - Per-provider test / verification endpoints - Migration of application.properties to YAML - Conversion of @ConfigurationProperties from data class to mutable class - spring-cloud-context for runtime rebinding - Removal of @ConditionalOnExpression from debrid client beans Co-Authored-By: Claude Opus 4.7 (1M context) --- build.gradle.kts | 7 + gradle/libs.versions.toml | 10 + .../skjaere/debridav/arrs/ArrConfiguration.kt | 12 +- .../arrs/RadarrConfigurationProperties.kt | 24 +- .../arrs/SonarrConfigurationProperties.kt | 23 +- .../debridav/arrs/client/RadarrApiClient.kt | 29 ++- .../debridav/arrs/client/SonarrApiClient.kt | 29 ++- .../config/ConfigApiExceptionHandler.kt | 22 ++ .../skjaere/debridav/config/ConfigOverride.kt | 32 +++ .../config/ConfigOverrideController.kt | 68 ++++++ .../debridav/config/ConfigOverrideDto.kt | 14 ++ .../config/ConfigOverrideRepository.kt | 8 + .../debridav/config/ConfigOverrideService.kt | 231 ++++++++++++++++++ .../skjaere/debridav/config/ConfigProperty.kt | 11 + .../debridav/config/ConfigPropertyRegistry.kt | 116 +++++++++ .../debridav/config/ConfigTestResultDto.kt | 14 ++ .../debridav/config/ConfigurationTester.kt | 14 ++ .../debridav/config/DatabasePropertySource.kt | 18 ++ .../DatabasePropertySourceInitializer.kt | 41 ++++ .../auth/AuthConfigurationProperties.kt | 13 + .../debridav/config/auth/AuthController.kt | 39 +++ .../config/auth/JwtAuthenticationFilter.kt | 32 +++ .../debridav/config/auth/JwtService.kt | 42 ++++ .../config/auth/SecurityConfiguration.kt | 91 +++++++ .../DbConfigurationProperties.kt | 10 + .../DebridavConfigurationProperties.kt | 97 ++++++-- .../debrid/client/easynews/EasynewsClient.kt | 37 ++- .../EasynewsConfigurationProperties.kt | 30 ++- .../client/premiumize/PremiumizeClient.kt | 40 ++- .../PremiumizeConfigurationProperties.kt | 11 +- .../model/PremiumizeConfiguration.kt | 2 - .../realdebrid/RealDebridActuatorEndpoint.kt | 2 - .../client/realdebrid/RealDebridClient.kt | 28 ++- .../realdebrid/RealDebridConfiguration.kt | 2 - .../RealDebridConfigurationProperties.kt | 16 +- .../support/RealDebridDownloadService.kt | 2 - .../support/RealDebridTorrentService.kt | 2 - .../debrid/client/torbox/TorBoxClient.kt | 29 ++- .../torbox/TorBoxConfigurationProperties.kt | 21 +- .../torbox/TorBoxHttpClientConfiguration.kt | 2 - .../io/skjaere/debridav/fs/FileController.kt | 111 +++++++++ .../io/skjaere/debridav/fs/FileDetailDto.kt | 28 +++ .../io/skjaere/debridav/fs/FileEntryDto.kt | 10 + .../usenet/NzbStreamerConfiguration.kt | 29 +-- .../usenet/pgmq/PgmqSpringConfiguration.kt | 24 +- src/main/resources/application.properties | 103 -------- src/main/resources/application.yaml | 147 +++++++++++ .../migration/V14__config_override_table.sql | 8 + .../debridav/test/DebridLinkServiceTest.kt | 39 +-- .../debridav/test/NzbImportServiceTest.kt | 36 +-- .../test/integrationtest/ConfigApiIT.kt | 231 ++++++++++++++++++ src/test/resources/application.properties | 87 ------- src/test/resources/application.yaml | 124 ++++++++++ 53 files changed, 1892 insertions(+), 356 deletions(-) create mode 100644 src/main/kotlin/io/skjaere/debridav/config/ConfigApiExceptionHandler.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/config/ConfigOverride.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideController.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideDto.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideRepository.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideService.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/config/ConfigProperty.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/config/ConfigPropertyRegistry.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/config/ConfigTestResultDto.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/config/ConfigurationTester.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/config/DatabasePropertySource.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/config/DatabasePropertySourceInitializer.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/config/auth/AuthConfigurationProperties.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/config/auth/AuthController.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/config/auth/JwtAuthenticationFilter.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/config/auth/JwtService.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/config/auth/SecurityConfiguration.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/configuration/DbConfigurationProperties.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/fs/FileController.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/fs/FileDetailDto.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/fs/FileEntryDto.kt delete mode 100644 src/main/resources/application.properties create mode 100644 src/main/resources/application.yaml create mode 100644 src/main/resources/db/migration/V14__config_override_table.sql create mode 100644 src/test/kotlin/io/skjaere/debridav/test/integrationtest/ConfigApiIT.kt delete mode 100644 src/test/resources/application.properties create mode 100644 src/test/resources/application.yaml diff --git a/build.gradle.kts b/build.gradle.kts index e37963d0..9666c141 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -61,6 +61,7 @@ tasks.jacocoTestReport { dependencies { implementation(platform(org.springframework.boot.gradle.plugin.SpringBootPlugin.BOM_COORDINATES)) + implementation(platform("org.springframework.cloud:spring-cloud-dependencies:${libs.versions.spring.cloud.get()}")) implementation(libs.spring.boot.starter.webmvc) implementation(libs.jackson.module.kotlin) @@ -97,6 +98,11 @@ dependencies { implementation(libs.sentry.spring.boot) implementation(libs.sentry.logback) implementation(libs.pgmq.kotlin.jvm) + implementation(libs.spring.cloud.context) + implementation(libs.spring.boot.starter.security) + implementation(libs.jjwt.api) + runtimeOnly(libs.jjwt.impl) + runtimeOnly(libs.jjwt.jackson) implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310") implementation("com.fasterxml.jackson.module:jackson-module-kotlin") @@ -114,6 +120,7 @@ dependencies { testImplementation(libs.sardine) testImplementation(libs.ktor.client.mock) testImplementation(libs.mock.nntp.server) + testImplementation(libs.spring.boot.starter.security.test) } java { diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index a92fdcf1..d893cbce 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -4,6 +4,7 @@ kotlinx-coroutines = "1.10.2" kotlinx-serialization = "1.10.0" ktor = "3.5.0-eap-1584" spring-boot = "4.0.3" +spring-cloud = "2025.1.1" mockk = "1.14.9" resilience4j = "2.3.0" testcontainers = "2.0.3" @@ -21,6 +22,7 @@ sentry = "8.33.0" nzb-streamer = "v0.6.0" pgmq-kotlin = "0.1.0" mock-nntp-server = "v0.2.0" +jjwt = "0.12.6" [libraries] # Kotlin @@ -100,3 +102,11 @@ sardine = { module = "com.github.lookfirst:sardine", version.ref = "sardine" } nzb-streamer = { module = "com.github.skjaere:nzb-streamer", version.ref = "nzb-streamer" } pgmq-kotlin-jvm = { module = "com.vdsirotkin.pgmq:pgmq-kotlin-jvm", version.ref = "pgmq-kotlin" } mock-nntp-server = { module = "com.github.skjaere:mock-nntp-server", version.ref = "mock-nntp-server" } + +# Security +spring-boot-starter-security = { module = "org.springframework.boot:spring-boot-starter-security", version.ref = "spring-boot" } +spring-boot-starter-security-test = { module = "org.springframework.boot:spring-boot-security-test", version.ref = "spring-boot" } +spring-cloud-context = { module = "org.springframework.cloud:spring-cloud-context" } +jjwt-api = { module = "io.jsonwebtoken:jjwt-api", version.ref = "jjwt" } +jjwt-impl = { module = "io.jsonwebtoken:jjwt-impl", version.ref = "jjwt" } +jjwt-jackson = { module = "io.jsonwebtoken:jjwt-jackson", version.ref = "jjwt" } diff --git a/src/main/kotlin/io/skjaere/debridav/arrs/ArrConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/arrs/ArrConfiguration.kt index 8f42df81..12320818 100644 --- a/src/main/kotlin/io/skjaere/debridav/arrs/ArrConfiguration.kt +++ b/src/main/kotlin/io/skjaere/debridav/arrs/ArrConfiguration.kt @@ -1,12 +1,12 @@ package io.skjaere.debridav.arrs interface ArrConfiguration { - val host: String - val port: Int - val apiBasePath: String - val apiKey: String - val category: String - val integrationEnabled: Boolean + var host: String + var port: Int + var apiBasePath: String + var apiKey: String + var category: String + var integrationEnabled: Boolean fun getApiBaseUrl(): String = "http://$host:$port$apiBasePath" } diff --git a/src/main/kotlin/io/skjaere/debridav/arrs/RadarrConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/arrs/RadarrConfigurationProperties.kt index 1fd1426a..61be041e 100644 --- a/src/main/kotlin/io/skjaere/debridav/arrs/RadarrConfigurationProperties.kt +++ b/src/main/kotlin/io/skjaere/debridav/arrs/RadarrConfigurationProperties.kt @@ -1,14 +1,20 @@ package io.skjaere.debridav.arrs +import io.skjaere.debridav.config.ConfigProperty import org.springframework.boot.context.properties.ConfigurationProperties @ConfigurationProperties(prefix = "radarr") -data class RadarrConfigurationProperties( - override val integrationEnabled: Boolean, - override val host: String, - override val port: Int = 7878, - override val apiBasePath: String = "/api/v3", - override val apiKey: String, - override val category: String, - ): ArrConfiguration - +class RadarrConfigurationProperties : ArrConfiguration { + @ConfigProperty(name = "Integration Enabled", description = "Enable Radarr integration") + override var integrationEnabled: Boolean = false + @ConfigProperty(name = "Host", description = "Radarr host") + override var host: String = "" + @ConfigProperty(name = "Port", description = "Radarr port") + override var port: Int = 7878 + @ConfigProperty(name = "API Base Path", description = "Radarr API base path", advanced = true) + override var apiBasePath: String = "/api/v3" + @ConfigProperty(name = "API Key", description = "Radarr API key", sensitive = true) + override var apiKey: String = "" + @ConfigProperty(name = "Category", description = "Radarr category") + override var category: String = "" +} diff --git a/src/main/kotlin/io/skjaere/debridav/arrs/SonarrConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/arrs/SonarrConfigurationProperties.kt index 4836adc4..ca65f043 100644 --- a/src/main/kotlin/io/skjaere/debridav/arrs/SonarrConfigurationProperties.kt +++ b/src/main/kotlin/io/skjaere/debridav/arrs/SonarrConfigurationProperties.kt @@ -1,13 +1,20 @@ package io.skjaere.debridav.arrs +import io.skjaere.debridav.config.ConfigProperty import org.springframework.boot.context.properties.ConfigurationProperties @ConfigurationProperties(prefix = "sonarr") -data class SonarrConfigurationProperties( - override val integrationEnabled: Boolean, - override val host: String, - override val port: Int = 8989, - override val apiBasePath: String = "/api/v3", - override val apiKey: String, - override val category: String, -): ArrConfiguration +class SonarrConfigurationProperties : ArrConfiguration { + @ConfigProperty(name = "Integration Enabled", description = "Enable Sonarr integration") + override var integrationEnabled: Boolean = false + @ConfigProperty(name = "Host", description = "Sonarr host") + override var host: String = "" + @ConfigProperty(name = "Port", description = "Sonarr port") + override var port: Int = 8989 + @ConfigProperty(name = "API Base Path", description = "Sonarr API base path", advanced = true) + override var apiBasePath: String = "/api/v3" + @ConfigProperty(name = "API Key", description = "Sonarr API key", sensitive = true) + override var apiKey: String = "" + @ConfigProperty(name = "Category", description = "Sonarr category") + override var category: String = "" +} diff --git a/src/main/kotlin/io/skjaere/debridav/arrs/client/RadarrApiClient.kt b/src/main/kotlin/io/skjaere/debridav/arrs/client/RadarrApiClient.kt index a021ad54..728215ec 100644 --- a/src/main/kotlin/io/skjaere/debridav/arrs/client/RadarrApiClient.kt +++ b/src/main/kotlin/io/skjaere/debridav/arrs/client/RadarrApiClient.kt @@ -4,6 +4,7 @@ import io.ktor.client.HttpClient import io.ktor.client.call.body import io.ktor.client.request.accept import io.ktor.client.request.delete +import io.ktor.client.request.get import io.ktor.client.request.header import io.ktor.client.request.post import io.ktor.client.request.setBody @@ -13,9 +14,12 @@ import io.ktor.http.contentType import io.ktor.http.isSuccess import io.skjaere.debridav.arrs.RadarrConfigurationProperties import io.skjaere.debridav.arrs.client.models.radarr.RadarrParseResponse +import io.skjaere.debridav.config.ConfigurationTester +import io.skjaere.debridav.config.TestResult import org.slf4j.LoggerFactory import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.stereotype.Component +import kotlin.reflect.KClass @Component @ConditionalOnExpression("\${radarr.integration-enabled:true}") @@ -23,7 +27,7 @@ class RadarrApiClient( private val httpClient: HttpClient, private val radarrConfigurationProperties: RadarrConfigurationProperties ) : BaseArrClient by DefaultBaseArrClient(httpClient, radarrConfigurationProperties), - ArrClient { + ArrClient, ConfigurationTester { private val logger = LoggerFactory.getLogger(RadarrApiClient::class.java) override suspend fun getItemIdFromName(name: String): Long { @@ -71,4 +75,27 @@ class RadarrApiClient( ) } } + + override val configurationClass: KClass<*> = RadarrConfigurationProperties::class + override val label: String = "Radarr" + + override suspend fun test(overrides: Map): TestResult = try { + val host = overrides["radarr.host"] ?: radarrConfigurationProperties.host + val port = overrides["radarr.port"]?.toIntOrNull() ?: radarrConfigurationProperties.port + val apiBasePath = overrides["radarr.api-base-path"] ?: radarrConfigurationProperties.apiBasePath + val apiKey = overrides["radarr.api-key"] ?: radarrConfigurationProperties.apiKey + val baseUrl = "http://$host:$port$apiBasePath" + + val response = httpClient.get("$baseUrl/system/status") { + accept(ContentType.Application.Json) + header("X-Api-Key", apiKey) + } + if (response.status.isSuccess()) { + TestResult(success = true, message = "Connected successfully") + } else { + TestResult(success = false, message = "HTTP ${response.status.value}: ${response.bodyAsText()}") + } + } catch (e: Exception) { + TestResult(success = false, message = e.message ?: "Unknown error") + } } diff --git a/src/main/kotlin/io/skjaere/debridav/arrs/client/SonarrApiClient.kt b/src/main/kotlin/io/skjaere/debridav/arrs/client/SonarrApiClient.kt index cc87d861..7558294c 100644 --- a/src/main/kotlin/io/skjaere/debridav/arrs/client/SonarrApiClient.kt +++ b/src/main/kotlin/io/skjaere/debridav/arrs/client/SonarrApiClient.kt @@ -4,6 +4,7 @@ import io.ktor.client.HttpClient import io.ktor.client.call.body import io.ktor.client.request.accept import io.ktor.client.request.delete +import io.ktor.client.request.get import io.ktor.client.request.header import io.ktor.client.request.post import io.ktor.client.request.setBody @@ -13,9 +14,12 @@ import io.ktor.http.contentType import io.ktor.http.isSuccess import io.skjaere.debridav.arrs.SonarrConfigurationProperties import io.skjaere.debridav.arrs.client.models.sonarr.SonarrParseResponse +import io.skjaere.debridav.config.ConfigurationTester +import io.skjaere.debridav.config.TestResult import org.slf4j.LoggerFactory import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.stereotype.Component +import kotlin.reflect.KClass @Component @ConditionalOnExpression($$"${sonarr.integration-enabled:true}") @@ -23,7 +27,7 @@ class SonarrApiClient( private val httpClient: HttpClient, private val sonarrConfigurationProperties: SonarrConfigurationProperties ) : BaseArrClient by DefaultBaseArrClient(httpClient, sonarrConfigurationProperties), - ArrClient { + ArrClient, ConfigurationTester { private val logger = LoggerFactory.getLogger(SonarrApiClient::class.java) override suspend fun getItemIdFromName(name: String): Long? { @@ -76,4 +80,27 @@ class SonarrApiClient( ) } } + + override val configurationClass: KClass<*> = SonarrConfigurationProperties::class + override val label: String = "Sonarr" + + override suspend fun test(overrides: Map): TestResult = try { + val host = overrides["sonarr.host"] ?: sonarrConfigurationProperties.host + val port = overrides["sonarr.port"]?.toIntOrNull() ?: sonarrConfigurationProperties.port + val apiBasePath = overrides["sonarr.api-base-path"] ?: sonarrConfigurationProperties.apiBasePath + val apiKey = overrides["sonarr.api-key"] ?: sonarrConfigurationProperties.apiKey + val baseUrl = "http://$host:$port$apiBasePath" + + val response = httpClient.get("$baseUrl/system/status") { + accept(ContentType.Application.Json) + header("X-Api-Key", apiKey) + } + if (response.status.isSuccess()) { + TestResult(success = true, message = "Connected successfully") + } else { + TestResult(success = false, message = "HTTP ${response.status.value}: ${response.bodyAsText()}") + } + } catch (e: Exception) { + TestResult(success = false, message = e.message ?: "Unknown error") + } } diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigApiExceptionHandler.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigApiExceptionHandler.kt new file mode 100644 index 00000000..112333cf --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigApiExceptionHandler.kt @@ -0,0 +1,22 @@ +package io.skjaere.debridav.config + +import org.springframework.http.HttpStatus +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.ExceptionHandler +import org.springframework.web.bind.annotation.RestControllerAdvice + +@RestControllerAdvice(assignableTypes = [ConfigOverrideController::class]) +class ConfigApiExceptionHandler { + + @ExceptionHandler(KeyNotWhitelistedException::class) + fun handleNotWhitelisted(ex: KeyNotWhitelistedException): ResponseEntity = + ResponseEntity.status(HttpStatus.BAD_REQUEST) + .body(ErrorResponse(ex.message ?: "Key not whitelisted")) + + @ExceptionHandler(OverrideNotFoundException::class) + fun handleNotFound(ex: OverrideNotFoundException): ResponseEntity = + ResponseEntity.status(HttpStatus.NOT_FOUND) + .body(ErrorResponse(ex.message ?: "Override not found")) +} + +data class ErrorResponse(val error: String) diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigOverride.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverride.kt new file mode 100644 index 00000000..712c4cc6 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverride.kt @@ -0,0 +1,32 @@ +package io.skjaere.debridav.config + +import jakarta.persistence.Column +import jakarta.persistence.Entity +import jakarta.persistence.GeneratedValue +import jakarta.persistence.GenerationType +import jakarta.persistence.Id +import jakarta.persistence.Table +import java.time.Instant + +@Entity +@Table(name = "config_override") +open class ConfigOverride { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + open var id: Long? = null + + @Column(name = "prop_key", nullable = false, unique = true) + open var propKey: String = "" + + @Column(name = "prop_value", columnDefinition = "TEXT") + open var propValue: String? = null + + @Column(name = "sensitive") + open var sensitive: Boolean = false + + @Column(name = "created_at") + open var createdAt: Instant? = null + + @Column(name = "updated_at") + open var updatedAt: Instant? = null +} diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideController.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideController.kt new file mode 100644 index 00000000..33f1f232 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideController.kt @@ -0,0 +1,68 @@ +package io.skjaere.debridav.config + +import kotlinx.coroutines.runBlocking +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.DeleteMapping +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.PutMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@RestController +@RequestMapping("/api/v1/config") +class ConfigOverrideController( + private val service: ConfigOverrideService, + private val registry: ConfigPropertyRegistry +) { + @GetMapping + fun listAll(): ResponseEntity> = + ResponseEntity.ok(service.listAll()) + + @GetMapping("/{key}") + fun get(@PathVariable key: String): ResponseEntity = + ResponseEntity.ok(service.getEffective(key)) + + @PutMapping("/{key}") + fun upsert( + @PathVariable key: String, + @RequestBody body: UpsertRequest + ): ResponseEntity = + ResponseEntity.ok(service.upsert(key, body.value)) + + @DeleteMapping("/{key}") + fun delete(@PathVariable key: String): ResponseEntity = + ResponseEntity.ok(service.delete(key)) + + @GetMapping("/testable") + fun listTestable(): ResponseEntity> = + ResponseEntity.ok(registry.getTestablePrefixes()) + + @PostMapping("/test/{prefix}") + fun test( + @PathVariable prefix: String, + @RequestBody(required = false) body: TestRequest? + ): ResponseEntity { + val tester = registry.getTester(prefix) + ?: return ResponseEntity.notFound().build() + + val start = System.currentTimeMillis() + val result = runBlocking { tester.test(body?.overrides ?: emptyMap()) } + val durationMs = System.currentTimeMillis() - start + + return ResponseEntity.ok( + ConfigTestResultDto( + prefix = prefix, + label = tester.label, + success = result.success, + message = result.message, + durationMs = durationMs + ) + ) + } +} + +data class UpsertRequest(val value: String? = null) +data class TestRequest(val overrides: Map = emptyMap()) diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideDto.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideDto.kt new file mode 100644 index 00000000..d3b55771 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideDto.kt @@ -0,0 +1,14 @@ +package io.skjaere.debridav.config + +data class ConfigOverrideDto( + val key: String, + val name: String?, + val effectiveValue: String?, + val defaultValue: String?, + val hasOverride: Boolean, + val sensitive: Boolean, + val group: String, + val description: String, + val type: String, + val advanced: Boolean +) diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideRepository.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideRepository.kt new file mode 100644 index 00000000..971c5598 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideRepository.kt @@ -0,0 +1,8 @@ +package io.skjaere.debridav.config + +import org.springframework.data.repository.CrudRepository + +interface ConfigOverrideRepository : CrudRepository { + fun findByPropKey(key: String): ConfigOverride? + fun findAllByPropKeyIn(keys: Collection): List +} diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideService.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideService.kt new file mode 100644 index 00000000..f1bf3aca --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideService.kt @@ -0,0 +1,231 @@ +package io.skjaere.debridav.config + +import io.skjaere.debridav.usenet.NntpConfigurationProperties +import io.skjaere.debridav.usenet.NntpPoolProperties +import io.skjaere.nzbstreamer.NzbStreamer +import io.skjaere.nzbstreamer.config.NntpConfig +import jakarta.transaction.Transactional +import org.slf4j.LoggerFactory +import org.springframework.cloud.context.refresh.ContextRefresher +import org.springframework.core.env.ConfigurableEnvironment +import org.springframework.core.env.EnumerablePropertySource +import org.springframework.stereotype.Service +import java.time.Instant + +@Service +class ConfigOverrideService( + private val repository: ConfigOverrideRepository, + private val environment: ConfigurableEnvironment, + private val registry: ConfigPropertyRegistry, + private val nntpConfig: NntpConfigurationProperties, + private val contextRefresher: ContextRefresher, + private val dbPropertySourceInitializer: DatabasePropertySourceInitializer, + private val nzbStreamer: NzbStreamer? = null +) { + companion object { + private const val MASKED = "***" + } + + fun listAll(): List { + val overrides = repository.findAllByPropKeyIn(registry.properties.keys) + .associateBy { it.propKey } + + return registry.properties.map { (key, meta) -> + val override = overrides[key] + val defaultValue = getDefaultValue(key) + val effectiveValue = override?.propValue ?: defaultValue + + ConfigOverrideDto( + key = key, + name = meta.name, + effectiveValue = if (meta.sensitive) effectiveValue?.let { MASKED } else effectiveValue, + defaultValue = if (meta.sensitive) defaultValue?.let { MASKED } else defaultValue, + hasOverride = override != null, + sensitive = meta.sensitive, + group = meta.group, + description = meta.description, + type = meta.type, + advanced = meta.advanced + ) + } + } + + fun getEffective(key: String): ConfigOverrideDto { + val meta = registry.getMeta(key) + ?: throw KeyNotWhitelistedException(key) + + val override = repository.findByPropKey(key) + val defaultValue = getDefaultValue(key) + val effectiveValue = override?.propValue ?: defaultValue + + return ConfigOverrideDto( + key = key, + name = meta.name, + effectiveValue = if (meta.sensitive) effectiveValue?.let { MASKED } else effectiveValue, + defaultValue = if (meta.sensitive) defaultValue?.let { MASKED } else defaultValue, + hasOverride = override != null, + sensitive = meta.sensitive, + group = meta.group, + description = meta.description, + type = meta.type, + advanced = meta.advanced + ) + } + + fun upsert(key: String, value: String?): ConfigOverrideDto { + val meta = registry.getMeta(key) + ?: throw KeyNotWhitelistedException(key) + + val now = Instant.now() + val entity = repository.findByPropKey(key) ?: ConfigOverride().apply { + propKey = key + createdAt = now + } + entity.propValue = value + entity.sensitive = meta.sensitive + entity.updatedAt = now + repository.save(entity) + + refreshEnvironment() + + return getEffective(key) + } + + fun delete(key: String): ConfigOverrideDto { + if (!registry.isWhitelisted(key)) { + throw KeyNotWhitelistedException(key) + } + + val override = repository.findByPropKey(key) + ?: throw OverrideNotFoundException(key) + repository.delete(override) + + refreshEnvironment() + + return getEffective(key) + } + + private fun refreshEnvironment() { + val propertySource = dbPropertySourceInitializer.getOrCreatePropertySource() + val overrides = repository.findAll().associate { it.propKey to (it.propValue ?: "") } + propertySource.replaceAll(overrides) + contextRefresher.refreshEnvironment() + logger.info("Refreshed environment with {} database override(s)", overrides.size) + } + + private fun getDefaultValue(key: String): String? { + for (source in environment.propertySources) { + if (source.name == DatabasePropertySource.NAME) continue + if (source is EnumerablePropertySource<*>) { + val value = source.getProperty(key) + if (value != null) return value.toString() + } else { + val value = source.getProperty(key) + if (value != null) return value.toString() + } + } + return null + } + + fun getNntpPools(): List { + val overrides = repository.findAllByPropKeyStartingWith(POOL_PREFIX) + val pools = if (overrides.isEmpty()) { + nntpConfig.pools.map { it.toDto() } + } else { + parsePoolOverrides(overrides) + } + return pools.sortedBy { it.priority } + } + + @Transactional + fun saveNntpPools(pools: List) { + repository.deleteAllByPropKeyStartingWith(POOL_PREFIX) + val now = Instant.now() + pools.forEachIndexed { i, pool -> + val entries = mapOf( + "nntp.pools[$i].host" to pool.host, + "nntp.pools[$i].port" to pool.port.toString(), + "nntp.pools[$i].username" to pool.username, + "nntp.pools[$i].password" to pool.password, + "nntp.pools[$i].use-tls" to pool.useTls.toString(), + "nntp.pools[$i].max-connections" to pool.maxConnections.toString(), + "nntp.pools[$i].priority" to pool.priority.toString() + ) + for ((key, value) in entries) { + val entity = ConfigOverride().apply { + propKey = key + propValue = value + sensitive = key.endsWith(".password") + createdAt = now + updatedAt = now + } + repository.save(entity) + } + } + syncRunningPools(pools) + } + + private fun syncRunningPools(saved: List) { + if (nzbStreamer == null) return + val savedConfigs = saved.map { it.toNntpConfig() }.toSet() + val runningConfigs = nzbStreamer.getPoolConfigs().toSet() + + val toRemove = runningConfigs - savedConfigs + val toAdd = savedConfigs - runningConfigs + + toRemove.forEach { nzbStreamer.removePool(it) } + toAdd.forEach { nzbStreamer.addPool(it) } + + if (toRemove.isNotEmpty() || toAdd.isNotEmpty()) { + logger.info("Synced NNTP pools: removed={}, added={}", toRemove.size, toAdd.size) + } + } + + private fun NntpPoolDto.toNntpConfig() = NntpConfig( + host = host, + port = port, + username = username, + password = password, + useTls = useTls, + maxConnections = maxConnections, + priority = priority + ) + + private fun parsePoolOverrides(overrides: List): List { + val poolMap = mutableMapOf>() + val regex = Regex("""nntp\.pools\[(\d+)]\.(.+)""") + for (ov in overrides) { + val match = regex.matchEntire(ov.propKey) ?: continue + val index = match.groupValues[1].toInt() + val field = match.groupValues[2] + poolMap.getOrPut(index) { mutableMapOf() }[field] = ov.propValue ?: "" + } + return poolMap.toSortedMap().map { (_, fields) -> + NntpPoolDto( + host = fields["host"] ?: "", + port = fields["port"]?.toIntOrNull() ?: 563, + username = fields["username"] ?: "", + password = fields["password"] ?: "", + useTls = fields["use-tls"]?.toBooleanStrictOrNull() ?: true, + maxConnections = fields["max-connections"]?.toIntOrNull() ?: 8, + priority = fields["priority"]?.toIntOrNull() ?: 0 + ) + } + } + + private fun NntpPoolProperties.toDto() = NntpPoolDto( + host = host, + port = port, + username = username, + password = password, + useTls = useTls, + maxConnections = maxConnections, + priority = priority + ) +} + +class KeyNotWhitelistedException(val key: String) : + RuntimeException("Property key '$key' is not whitelisted for override") + +class OverrideNotFoundException(val key: String) : + RuntimeException("No override found for key '$key'") diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigProperty.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigProperty.kt new file mode 100644 index 00000000..7bb38ccf --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigProperty.kt @@ -0,0 +1,11 @@ +package io.skjaere.debridav.config + +@Target(AnnotationTarget.VALUE_PARAMETER, AnnotationTarget.PROPERTY, AnnotationTarget.FIELD) +@Retention(AnnotationRetention.RUNTIME) +annotation class ConfigProperty( + val name: String, + val description: String = "", + val sensitive: Boolean = false, + val group: String = "", + val advanced: Boolean = false +) diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigPropertyRegistry.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigPropertyRegistry.kt new file mode 100644 index 00000000..4442b389 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigPropertyRegistry.kt @@ -0,0 +1,116 @@ +package io.skjaere.debridav.config + +import jakarta.annotation.PostConstruct +import org.slf4j.LoggerFactory +import org.springframework.boot.context.properties.ConfigurationProperties +import org.springframework.context.ApplicationContext +import org.springframework.stereotype.Component +import java.time.Duration +import kotlin.reflect.full.findAnnotation +import kotlin.reflect.full.memberProperties + +data class ConfigPropertyMeta( + val name: String, + val description: String, + val sensitive: Boolean = false, + val group: String, + val type: String = "STRING", + val advanced: Boolean = false +) + +@Component +class ConfigPropertyRegistry( + private val applicationContext: ApplicationContext +) { + private val logger = LoggerFactory.getLogger(ConfigPropertyRegistry::class.java) + private val _properties = mutableMapOf() + private val _testers = mutableMapOf() + + val properties: Map get() = _properties + + @PostConstruct + fun init() { + val beanNames = applicationContext.getBeanNamesForAnnotation(ConfigurationProperties::class.java) + for (beanName in beanNames) { + val beanType = applicationContext.getType(beanName) ?: continue + val prefix = beanType.getAnnotation(ConfigurationProperties::class.java)?.prefix + ?: continue + + for (prop in beanType.kotlin.memberProperties) { + val annotation = prop.findAnnotation() ?: continue + + val kebabName = camelToKebab(prop.name) + val key = "$prefix.$kebabName" + val group = annotation.group.ifEmpty { deriveGroup(prefix) } + val type = when (prop.returnType.classifier) { + Boolean::class -> "BOOLEAN" + Int::class -> "INT" + Long::class -> "LONG" + Duration::class -> "DURATION" + List::class -> "STRING_LIST" + else -> "STRING" + } + + _properties[key] = ConfigPropertyMeta( + name = annotation.name, + description = annotation.description, + sensitive = annotation.sensitive, + group = group, + type = type, + advanced = annotation.advanced + ) + } + } + discoverTesters() + } + + private fun discoverTesters() { + val testers = applicationContext.getBeansOfType(ConfigurationTester::class.java).values + for (tester in testers) { + val prefix = tester.configurationClass.java + .getAnnotation(ConfigurationProperties::class.java) + ?.prefix + if (prefix != null) { + _testers[prefix] = tester + logger.info("Registered configuration tester '{}' for prefix '{}'", tester.label, prefix) + } else { + logger.warn( + "ConfigurationTester '{}' targets {} which has no @ConfigurationProperties annotation", + tester.label, tester.configurationClass + ) + } + } + } + + fun getTester(prefix: String): ConfigurationTester? = _testers[prefix] + + fun getTestablePrefixes(): List = _testers.map { (prefix, tester) -> + TestablePrefixDto(prefix = prefix, label = tester.label) + } + + fun isWhitelisted(key: String): Boolean = _properties.containsKey(key) + + fun getMeta(key: String): ConfigPropertyMeta? = _properties[key] + + companion object { + private val PROVIDER_PREFIXES = setOf("premiumize", "real-debrid", "torbox", "easynews") + private val ARR_PREFIXES = setOf("sonarr", "radarr") + + fun camelToKebab(name: String): String = buildString { + for ((i, ch) in name.withIndex()) { + if (ch.isUpperCase()) { + if (i > 0) append('-') + append(ch.lowercaseChar()) + } else { + append(ch) + } + } + } + + fun deriveGroup(prefix: String): String = when (prefix) { + in PROVIDER_PREFIXES -> "providers" + in ARR_PREFIXES -> "arrs" + else -> prefix + } + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigTestResultDto.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigTestResultDto.kt new file mode 100644 index 00000000..a4525690 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigTestResultDto.kt @@ -0,0 +1,14 @@ +package io.skjaere.debridav.config + +data class ConfigTestResultDto( + val prefix: String, + val label: String, + val success: Boolean, + val message: String, + val durationMs: Long +) + +data class TestablePrefixDto( + val prefix: String, + val label: String +) diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigurationTester.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigurationTester.kt new file mode 100644 index 00000000..cbae29a7 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigurationTester.kt @@ -0,0 +1,14 @@ +package io.skjaere.debridav.config + +import kotlin.reflect.KClass + +interface ConfigurationTester { + val configurationClass: KClass<*> + val label: String + suspend fun test(overrides: Map = emptyMap()): TestResult +} + +data class TestResult( + val success: Boolean, + val message: String +) diff --git a/src/main/kotlin/io/skjaere/debridav/config/DatabasePropertySource.kt b/src/main/kotlin/io/skjaere/debridav/config/DatabasePropertySource.kt new file mode 100644 index 00000000..15648f06 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/DatabasePropertySource.kt @@ -0,0 +1,18 @@ +package io.skjaere.debridav.config + +import org.springframework.core.env.MapPropertySource +import java.util.concurrent.ConcurrentHashMap + +class DatabasePropertySource( + private val map: ConcurrentHashMap = ConcurrentHashMap() +) : MapPropertySource(NAME, map) { + + fun replaceAll(overrides: Map) { + map.clear() + map.putAll(overrides) + } + + companion object { + const val NAME = "databaseOverrides" + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/config/DatabasePropertySourceInitializer.kt b/src/main/kotlin/io/skjaere/debridav/config/DatabasePropertySourceInitializer.kt new file mode 100644 index 00000000..4fecd913 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/DatabasePropertySourceInitializer.kt @@ -0,0 +1,41 @@ +package io.skjaere.debridav.config + +import org.slf4j.LoggerFactory +import org.springframework.boot.context.event.ApplicationReadyEvent +import org.springframework.cloud.context.refresh.ContextRefresher +import org.springframework.context.ApplicationListener +import org.springframework.core.env.ConfigurableEnvironment +import org.springframework.stereotype.Component + +@Component +class DatabasePropertySourceInitializer( + private val environment: ConfigurableEnvironment, + private val repository: ConfigOverrideRepository, + private val contextRefresher: ContextRefresher +) : ApplicationListener { + + private val logger = LoggerFactory.getLogger(DatabasePropertySourceInitializer::class.java) + + override fun onApplicationEvent(event: ApplicationReadyEvent) { + val propertySource = getOrCreatePropertySource() + val overrides = repository.findAll().associate { it.propKey to (it.propValue ?: "") } + if (overrides.isNotEmpty()) { + propertySource.replaceAll(overrides) + contextRefresher.refreshEnvironment() + logger.info("Loaded {} database config override(s) and refreshed environment", overrides.size) + } else { + logger.info("No database config overrides found") + } + } + + fun getOrCreatePropertySource(): DatabasePropertySource { + val sources = environment.propertySources + val existing = sources.get(DatabasePropertySource.NAME) + if (existing != null) { + return existing as DatabasePropertySource + } + val propertySource = DatabasePropertySource() + sources.addFirst(propertySource) + return propertySource + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/config/auth/AuthConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/config/auth/AuthConfigurationProperties.kt new file mode 100644 index 00000000..eb2c190d --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/auth/AuthConfigurationProperties.kt @@ -0,0 +1,13 @@ +package io.skjaere.debridav.config.auth + +import org.springframework.boot.context.properties.ConfigurationProperties + +@ConfigurationProperties(prefix = "debridav.auth") +class AuthConfigurationProperties { + var enabled: Boolean = false + var jwtSecret: String = "" + var tokenExpirationHours: Long = 24 + var protectQbittorrentApi: Boolean = false + var protectSabnzbdApi: Boolean = false + var protectActuator: Boolean = false +} diff --git a/src/main/kotlin/io/skjaere/debridav/config/auth/AuthController.kt b/src/main/kotlin/io/skjaere/debridav/config/auth/AuthController.kt new file mode 100644 index 00000000..a3fa1339 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/auth/AuthController.kt @@ -0,0 +1,39 @@ +package io.skjaere.debridav.config.auth + +import io.skjaere.debridav.configuration.DebridavConfigurationProperties +import org.springframework.http.HttpStatus +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@RestController +@RequestMapping("/api/v1/auth") +class AuthController( + private val jwtService: JwtService, + private val debridavConfig: DebridavConfigurationProperties +) { + @PostMapping("/login") + fun login(@RequestBody request: LoginRequest): ResponseEntity { + val expectedUsername = debridavConfig.webdavUsername + val expectedPassword = debridavConfig.webdavPassword + + if (expectedUsername.isNullOrBlank() || expectedPassword.isNullOrBlank()) { + return ResponseEntity.status(HttpStatus.UNAUTHORIZED) + .body(ErrorBody("No credentials configured")) + } + + if (request.username != expectedUsername || request.password != expectedPassword) { + return ResponseEntity.status(HttpStatus.UNAUTHORIZED) + .body(ErrorBody("Invalid credentials")) + } + + val token = jwtService.generateToken(request.username) + return ResponseEntity.ok(LoginResponse(token)) + } +} + +data class LoginRequest(val username: String, val password: String) +data class LoginResponse(val token: String) +data class ErrorBody(val error: String) diff --git a/src/main/kotlin/io/skjaere/debridav/config/auth/JwtAuthenticationFilter.kt b/src/main/kotlin/io/skjaere/debridav/config/auth/JwtAuthenticationFilter.kt new file mode 100644 index 00000000..e241b64e --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/auth/JwtAuthenticationFilter.kt @@ -0,0 +1,32 @@ +package io.skjaere.debridav.config.auth + +import jakarta.servlet.FilterChain +import jakarta.servlet.http.HttpServletRequest +import jakarta.servlet.http.HttpServletResponse +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken +import org.springframework.security.core.context.SecurityContextHolder +import org.springframework.stereotype.Component +import org.springframework.web.filter.OncePerRequestFilter + +@Component +class JwtAuthenticationFilter( + private val jwtService: JwtService +) : OncePerRequestFilter() { + + override fun doFilterInternal( + request: HttpServletRequest, + response: HttpServletResponse, + filterChain: FilterChain + ) { + val authHeader = request.getHeader("Authorization") + if (authHeader != null && authHeader.startsWith("Bearer ")) { + val token = authHeader.substring(7) + val username = jwtService.validateTokenAndGetUsername(token) + if (username != null && SecurityContextHolder.getContext().authentication == null) { + val auth = UsernamePasswordAuthenticationToken(username, null, emptyList()) + SecurityContextHolder.getContext().authentication = auth + } + } + filterChain.doFilter(request, response) + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/config/auth/JwtService.kt b/src/main/kotlin/io/skjaere/debridav/config/auth/JwtService.kt new file mode 100644 index 00000000..db162547 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/auth/JwtService.kt @@ -0,0 +1,42 @@ +package io.skjaere.debridav.config.auth + +import io.jsonwebtoken.JwtException +import io.jsonwebtoken.Jwts +import io.jsonwebtoken.security.Keys +import org.springframework.stereotype.Service +import java.util.Date +import javax.crypto.SecretKey + +@Service +class JwtService( + private val authConfig: AuthConfigurationProperties +) { + private val key: SecretKey by lazy { + Keys.hmacShaKeyFor(authConfig.jwtSecret.toByteArray()) + } + + fun generateToken(username: String): String { + val now = Date() + val expiration = Date(now.time + authConfig.tokenExpirationHours * 3600 * 1000) + + return Jwts.builder() + .subject(username) + .issuedAt(now) + .expiration(expiration) + .signWith(key) + .compact() + } + + fun validateTokenAndGetUsername(token: String): String? = try { + Jwts.parser() + .verifyWith(key) + .build() + .parseSignedClaims(token) + .payload + .subject + } catch (_: JwtException) { + null + } catch (_: IllegalArgumentException) { + null + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/config/auth/SecurityConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/config/auth/SecurityConfiguration.kt new file mode 100644 index 00000000..36bb3f5a --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/auth/SecurityConfiguration.kt @@ -0,0 +1,91 @@ +package io.skjaere.debridav.config.auth + +import jakarta.servlet.http.HttpServletRequest +import jakarta.servlet.http.HttpServletResponse +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.core.annotation.Order +import org.springframework.http.HttpStatus +import org.springframework.security.config.annotation.web.builders.HttpSecurity +import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity +import org.springframework.security.config.http.SessionCreationPolicy +import org.springframework.security.core.AuthenticationException +import org.springframework.security.web.AuthenticationEntryPoint +import org.springframework.security.web.SecurityFilterChain +import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter +import org.springframework.security.web.firewall.HttpFirewall +import org.springframework.security.web.firewall.StrictHttpFirewall + +@Configuration +@EnableWebSecurity +class SecurityConfiguration( + private val jwtAuthenticationFilter: JwtAuthenticationFilter, + private val authConfig: AuthConfigurationProperties +) { + @Bean + @Order(1) + fun apiSecurityFilterChain(http: HttpSecurity): SecurityFilterChain { + http + .securityMatcher("/api/**", "/actuator/**") + .csrf { it.disable() } + .sessionManagement { it.sessionCreationPolicy(SessionCreationPolicy.STATELESS) } + .exceptionHandling { it.authenticationEntryPoint(unauthorizedEntryPoint()) } + .addFilterBefore(jwtAuthenticationFilter, UsernamePasswordAuthenticationFilter::class.java) + .authorizeHttpRequests { auth -> + // Auth endpoints are always public + auth.requestMatchers("/api/v1/auth/**").permitAll() + + // Config API is protected when auth is enabled + if (authConfig.enabled) { + auth.requestMatchers("/api/v1/config/**").authenticated() + } else { + auth.requestMatchers("/api/v1/config/**").permitAll() + } + + // Conditionally protect qBittorrent API + if (authConfig.protectQbittorrentApi) { + auth.requestMatchers("/api/v2/**").authenticated() + } + + // Conditionally protect SABnzbd API + if (authConfig.protectSabnzbdApi) { + auth.requestMatchers("/api").authenticated() + } + + // Conditionally protect actuator + if (authConfig.protectActuator) { + auth.requestMatchers("/actuator/**").authenticated() + } + + // All other API/actuator paths are public + auth.anyRequest().permitAll() + } + + return http.build() + } + + @Bean + @Order(2) + fun webDavSecurityFilterChain(http: HttpSecurity): SecurityFilterChain { + http + .csrf { it.disable() } + .authorizeHttpRequests { it.anyRequest().permitAll() } + + return http.build() + } + + @Bean + fun httpFirewall(): HttpFirewall { + val firewall = StrictHttpFirewall() + // Allow WebDAV methods (PROPFIND, MKCOL, COPY, MOVE, LOCK, UNLOCK, PROPPATCH) + firewall.setUnsafeAllowAnyHttpMethod(true) + return firewall + } + + private fun unauthorizedEntryPoint() = AuthenticationEntryPoint { + _: HttpServletRequest, response: HttpServletResponse, _: AuthenticationException -> + response.status = HttpStatus.UNAUTHORIZED.value() + response.contentType = "application/json" + response.writer.write("""{"error":"Unauthorized"}""") + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/configuration/DbConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/configuration/DbConfigurationProperties.kt new file mode 100644 index 00000000..fd0229f4 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/configuration/DbConfigurationProperties.kt @@ -0,0 +1,10 @@ +package io.skjaere.debridav.configuration + +import org.springframework.boot.context.properties.ConfigurationProperties + +@ConfigurationProperties(prefix = "debridav.db") +class DbConfigurationProperties { + var host: String = "localhost" + var port: Int = 5432 + var databaseName: String = "debridav" +} diff --git a/src/main/kotlin/io/skjaere/debridav/configuration/DebridavConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/configuration/DebridavConfigurationProperties.kt index e8993330..3d8c9af3 100644 --- a/src/main/kotlin/io/skjaere/debridav/configuration/DebridavConfigurationProperties.kt +++ b/src/main/kotlin/io/skjaere/debridav/configuration/DebridavConfigurationProperties.kt @@ -1,36 +1,79 @@ package io.skjaere.debridav.configuration +import io.skjaere.debridav.config.ConfigProperty import io.skjaere.debridav.debrid.DebridProvider import org.springframework.boot.context.properties.ConfigurationProperties import java.time.Duration @ConfigurationProperties(prefix = "debridav") -data class DebridavConfigurationProperties( - val rootPath: String, - val downloadPath: String, - val mountPath: String, - var debridClients: List, - val waitAfterMissing: Duration, - val waitAfterProviderError: Duration, - val waitAfterNetworkError: Duration, - val waitAfterClientError: Duration, - val retriesOnProviderError: Long, - val delayBetweenRetries: Duration, - val connectTimeoutMilliseconds: Long, - val readTimeoutMilliseconds: Long, - val shouldDeleteNonWorkingFiles: Boolean, - val torrentLifetime: Duration, - val enableFileImportOnStartup: Boolean, - val defaultCategories: List, - val localEntityMaxSizeMb: Int, - val webdavUsername: String? = null, - val webdavPassword: String? = null, -) { - fun isWebdavAuthEnabled(): Boolean = !webdavUsername.isNullOrBlank() && !webdavPassword.isNullOrBlank() +class DebridavConfigurationProperties { + lateinit var rootPath: String + + @ConfigProperty(name = "Download Path", description = "Download path") + lateinit var downloadPath: String + + @ConfigProperty(name = "Mount Path", description = "Mount path") + lateinit var mountPath: String + + @ConfigProperty(name = "Debrid Clients", description = "Enabled debrid providers (comma-separated)") + var debridClients: List = emptyList() + + @ConfigProperty(name = "Wait After Missing", description = "Wait duration after missing file", advanced = true) + var waitAfterMissing: Duration = Duration.ZERO + + @ConfigProperty( + name = "Wait After Provider Error", + description = "Wait duration after provider error", + advanced = true + ) + var waitAfterProviderError: Duration = Duration.ZERO + + @ConfigProperty( + name = "Wait After Network Error", + description = "Wait duration after network error", + advanced = true + ) + var waitAfterNetworkError: Duration = Duration.ZERO + + @ConfigProperty(name = "Wait After Client Error", description = "Wait duration after client error", advanced = true) + var waitAfterClientError: Duration = Duration.ZERO + + @ConfigProperty( + name = "Retries on Provider Error", + description = "Number of retries on provider error", + advanced = true + ) + var retriesOnProviderError: Long = 0 + + @ConfigProperty(name = "Delay Between Retries", description = "Delay between retries", advanced = true) + var delayBetweenRetries: Duration = Duration.ZERO - init { - require(debridClients.isNotEmpty()) { - "No debrid providers defined" - } - } + @ConfigProperty(name = "Connect Timeout", description = "HTTP connect timeout in ms", advanced = true) + var connectTimeoutMilliseconds: Long = 0 + + @ConfigProperty(name = "Read Timeout", description = "HTTP read timeout in ms", advanced = true) + var readTimeoutMilliseconds: Long = 0 + + @ConfigProperty(name = "Delete Non-Working Files", description = "Delete non-working files", advanced = true) + var shouldDeleteNonWorkingFiles: Boolean = false + + @ConfigProperty(name = "Torrent Lifetime", description = "Torrent lifetime duration", advanced = true) + var torrentLifetime: Duration = Duration.ZERO + + @ConfigProperty(name = "File Import on Startup", description = "Enable file import on startup", advanced = true) + var enableFileImportOnStartup: Boolean = false + + @ConfigProperty(name = "Default Categories", description = "Default categories (comma-separated)", advanced = true) + var defaultCategories: List = emptyList() + + @ConfigProperty(name = "Max Local Entity Size (MB)", description = "Max local entity size in MB", advanced = true) + var localEntityMaxSizeMb: Int = 0 + + @ConfigProperty(name = "WebDAV Username", description = "WebDAV username", group = "webdav") + var webdavUsername: String? = null + + @ConfigProperty(name = "WebDAV Password", description = "WebDAV password", sensitive = true, group = "webdav") + var webdavPassword: String? = null + + fun isWebdavAuthEnabled(): Boolean = !webdavUsername.isNullOrBlank() && !webdavPassword.isNullOrBlank() } diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/easynews/EasynewsClient.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/easynews/EasynewsClient.kt index d5b60539..86ae3b6d 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/easynews/EasynewsClient.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/easynews/EasynewsClient.kt @@ -22,6 +22,8 @@ import io.ktor.http.ContentType import io.ktor.http.HttpHeaders import io.ktor.http.HttpHeaders.Authorization import io.ktor.http.isSuccess +import io.skjaere.debridav.config.ConfigurationTester +import io.skjaere.debridav.config.TestResult import io.milton.http.Range import io.skjaere.debridav.debrid.CachedContentKey import io.skjaere.debridav.debrid.DebridProvider @@ -36,13 +38,13 @@ import kotlinx.coroutines.flow.flowOf import kotlinx.serialization.json.Json import org.slf4j.Logger import org.slf4j.LoggerFactory -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.stereotype.Component import java.net.URLEncoder import java.nio.charset.StandardCharsets import java.time.Duration import java.time.Instant import java.util.* +import kotlin.reflect.KClass const val TIMEOUT_MS = 5_000L const val RETRIES = 3 @@ -55,14 +57,13 @@ private const val RATE_LIMITER_TIMEOUT = 5L @Component @Suppress("UnusedPrivateProperty", "TooManyFunctions") -@ConditionalOnExpression("#{'\${debridav.debrid-clients}'.contains('easynews')}") class EasynewsClient( override val httpClient: HttpClient, private val easynewsConfiguration: EasynewsConfigurationProperties, private val easynewsReleaseNameMatchingService: EasynewsReleaseNameMatchingService, rateLimiterRegistry: RateLimiterRegistry, retryRegistry: RetryRegistry -) : DebridCachedContentClient { +) : DebridCachedContentClient, ConfigurationTester { private val jsonParser = Json { ignoreUnknownKeys = true } private val logger = LoggerFactory.getLogger(EasynewsClient::class.java) private val auth = getBasicAuth() @@ -387,4 +388,34 @@ class EasynewsClient( override fun logger(): Logger { return logger } + + override val configurationClass: KClass<*> = EasynewsConfigurationProperties::class + override val label: String = "Easynews" + + override suspend fun test(overrides: Map): TestResult = try { + val apiBaseUrl = overrides["easynews.api-base-url"] ?: easynewsConfiguration.apiBaseUrl + val username = overrides["easynews.username"] ?: easynewsConfiguration.username + val password = overrides["easynews.password"] ?: easynewsConfiguration.password + + val credentials = "$username:$password" + val testAuth = "Basic ${Base64.getEncoder().encodeToString(credentials.toByteArray())}" + + val response = httpClient.get("$apiBaseUrl/2.0/search/solr-search/") { + url { + parameters.append("gps", "test") + parameters.append("pby", "1") + } + headers { + append(Authorization, testAuth) + accept(ContentType.Application.Json) + } + } + if (response.status.isSuccess()) { + TestResult(success = true, message = "Connected successfully") + } else { + TestResult(success = false, message = "HTTP ${response.status.value}") + } + } catch (e: Exception) { + TestResult(success = false, message = e.message ?: "Unknown error") + } } diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/easynews/EasynewsConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/easynews/EasynewsConfigurationProperties.kt index ac6bafef..84865800 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/easynews/EasynewsConfigurationProperties.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/easynews/EasynewsConfigurationProperties.kt @@ -1,17 +1,25 @@ package io.skjaere.debridav.debrid.client.easynews +import io.skjaere.debridav.config.ConfigProperty import org.springframework.boot.context.properties.ConfigurationProperties import java.time.Duration @ConfigurationProperties(prefix = "easynews") -data class EasynewsConfigurationProperties( - val apiBaseUrl: String, - val username: String, - val password: String, - val enabledForTorrents: Boolean, - val rateLimitWindowDuration: Duration, - val allowedRequestsInWindow: Int, - val connectTimeout: Int, - val socketTimeout: Int -) - +class EasynewsConfigurationProperties { + @ConfigProperty(name = "API Base URL", description = "Easynews API base URL", advanced = true) + lateinit var apiBaseUrl: String + @ConfigProperty(name = "Username", description = "Easynews username") + lateinit var username: String + @ConfigProperty(name = "Password", description = "Easynews password", sensitive = true) + lateinit var password: String + @ConfigProperty(name = "Enabled for Torrents", description = "Enable Easynews for torrents") + var enabledForTorrents: Boolean = false + @ConfigProperty(name = "Rate Limit Window", description = "Easynews rate limit window") + var rateLimitWindowDuration: Duration = Duration.ZERO + @ConfigProperty(name = "Allowed Requests in Window", description = "Easynews allowed requests per window") + var allowedRequestsInWindow: Int = 0 + @ConfigProperty(name = "Connect Timeout", description = "Easynews connect timeout") + var connectTimeout: Int = 0 + @ConfigProperty(name = "Socket Timeout", description = "Easynews socket timeout") + var socketTimeout: Int = 0 +} diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/PremiumizeClient.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/PremiumizeClient.kt index 0e61f94f..cd3475b2 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/PremiumizeClient.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/PremiumizeClient.kt @@ -3,11 +3,16 @@ package io.skjaere.debridav.debrid.client.premiumize import io.github.resilience4j.ratelimiter.RateLimiter import io.ktor.client.HttpClient import io.ktor.client.call.body +import io.ktor.client.request.accept import io.ktor.client.request.get import io.ktor.client.request.post +import io.ktor.http.ContentType import io.ktor.http.HttpHeaders import io.ktor.http.HttpStatusCode import io.ktor.http.headers +import io.ktor.http.isSuccess +import io.skjaere.debridav.config.ConfigurationTester +import io.skjaere.debridav.config.TestResult import io.skjaere.debridav.configuration.DebridavConfigurationProperties import io.skjaere.debridav.debrid.DebridClient import io.skjaere.debridav.debrid.DebridProvider @@ -18,15 +23,21 @@ import io.skjaere.debridav.debrid.client.StreamableLinkPreparable import io.skjaere.debridav.debrid.client.premiumize.model.CacheCheckResponse import io.skjaere.debridav.debrid.client.premiumize.model.SuccessfulDirectDownloadResponse import io.skjaere.debridav.fs.CachedFile +import kotlinx.serialization.Serializable import org.slf4j.Logger import org.slf4j.LoggerFactory -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.stereotype.Component import java.time.Clock import java.time.Instant +import kotlin.reflect.KClass + +@Serializable +private data class PremiumizeAccountResponse( + val status: String, + val message: String? = null +) @Component -@ConditionalOnExpression("#{'\${debridav.debrid-clients}'.contains('premiumize')}") class PremiumizeClient( private val premiumizeConfiguration: PremiumizeConfigurationProperties, override val httpClient: HttpClient, @@ -34,6 +45,7 @@ class PremiumizeClient( debridavConfigurationProperties: DebridavConfigurationProperties, premiumizeRateLimiter: RateLimiter ) : DebridCachedTorrentClient, + ConfigurationTester, StreamableLinkPreparable by DefaultStreamableLinkPreparer( httpClient, debridavConfigurationProperties, @@ -116,4 +128,28 @@ class PremiumizeClient( override fun logger(): Logger { return logger } + + override val configurationClass: KClass<*> = PremiumizeConfigurationProperties::class + override val label: String = "Premiumize" + + override suspend fun test(overrides: Map): TestResult = try { + val baseUrl = overrides["premiumize.base-url"] ?: premiumizeConfiguration.baseUrl + val apiKey = overrides["premiumize.api-key"] ?: premiumizeConfiguration.apiKey + + val response = httpClient.get("$baseUrl/account/info?apikey=$apiKey") { + accept(ContentType.Application.Json) + } + if (!response.status.isSuccess()) { + TestResult(success = false, message = "HTTP ${response.status.value}") + } else { + val body = response.body() + if (body.status == "success") { + TestResult(success = true, message = "Connected successfully") + } else { + TestResult(success = false, message = body.message ?: "Authentication failed") + } + } + } catch (e: Exception) { + TestResult(success = false, message = e.message ?: "Unknown error") + } } diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/PremiumizeConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/PremiumizeConfigurationProperties.kt index c6093191..09281fd2 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/PremiumizeConfigurationProperties.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/PremiumizeConfigurationProperties.kt @@ -1,9 +1,12 @@ package io.skjaere.debridav.debrid.client.premiumize +import io.skjaere.debridav.config.ConfigProperty import org.springframework.boot.context.properties.ConfigurationProperties @ConfigurationProperties(prefix = "premiumize") -class PremiumizeConfigurationProperties( - val baseUrl: String, - val apiKey: String -) +class PremiumizeConfigurationProperties { + @ConfigProperty(name = "Base URL", description = "Premiumize base URL", advanced = true) + lateinit var baseUrl: String + @ConfigProperty(name = "API Key", description = "Premiumize API key", sensitive = true) + lateinit var apiKey: String +} diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/model/PremiumizeConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/model/PremiumizeConfiguration.kt index 6c5fa16d..15f58eb3 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/model/PremiumizeConfiguration.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/model/PremiumizeConfiguration.kt @@ -3,7 +3,6 @@ package io.skjaere.debridav.debrid.client.premiumize.model import io.github.resilience4j.ratelimiter.RateLimiter import io.github.resilience4j.ratelimiter.RateLimiterConfig import io.github.resilience4j.ratelimiter.RateLimiterRegistry -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import java.time.Duration @@ -13,7 +12,6 @@ private const val PERIOD_LIMIT = 999 @Configuration class PremiumizeConfiguration { @Bean - @ConditionalOnExpression("#{'\${debridav.debrid-clients}'.contains('premiumize')}") fun premiumizeRateLimiter(rateLimiterRegistry: RateLimiterRegistry): RateLimiter { val rateLimiterConfig = RateLimiterConfig.custom() .limitRefreshPeriod(Duration.ofMinutes(1)) diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridActuatorEndpoint.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridActuatorEndpoint.kt index b33548c2..92453787 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridActuatorEndpoint.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridActuatorEndpoint.kt @@ -3,12 +3,10 @@ package io.skjaere.debridav.debrid.client.realdebrid import org.springframework.boot.actuate.endpoint.annotation.Endpoint import org.springframework.boot.actuate.endpoint.annotation.ReadOperation import org.springframework.boot.actuate.endpoint.annotation.WriteOperation -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.stereotype.Component @Component @Endpoint(id = "realdebrid") -@ConditionalOnExpression("#{'\${debridav.debrid-clients}'.contains('real_debrid')}") class RealDebridActuatorEndpoint( private val realDebridClient: RealDebridClient ) { diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridClient.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridClient.kt index 8b61b120..256e2a4c 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridClient.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridClient.kt @@ -19,6 +19,9 @@ import io.ktor.http.HttpStatusCode import io.ktor.http.Parameters import io.ktor.http.contentType import io.ktor.http.isSuccess +import io.ktor.client.statement.bodyAsText +import io.skjaere.debridav.config.ConfigurationTester +import io.skjaere.debridav.config.TestResult import io.skjaere.debridav.configuration.DebridavConfigurationProperties import io.skjaere.debridav.debrid.DebridProvider import io.skjaere.debridav.debrid.TorrentMagnet @@ -47,11 +50,11 @@ import kotlinx.coroutines.runBlocking import kotlinx.serialization.json.Json import org.slf4j.Logger import org.slf4j.LoggerFactory -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.scheduling.annotation.Scheduled import org.springframework.stereotype.Component import org.springframework.transaction.annotation.Transactional import java.time.Instant +import kotlin.reflect.KClass private const val CREATED = 204 private const val NOT_FOUND = 404 @@ -59,7 +62,6 @@ private const val LINK_ID_MAP_KEY = "linkId" private const val TORRENT_ID_MAP_KEY = "torrentId" @Component -@ConditionalOnExpression($$"#{'${debridav.debrid-clients}'.contains('real_debrid')}") @Suppress("TooManyFunctions") class RealDebridClient( private val realDebridConfigurationProperties: RealDebridConfigurationProperties, @@ -68,7 +70,7 @@ class RealDebridClient( private val realDebridTorrentService: RealDebridTorrentService, private val realDebridDownloadService: RealDebridDownloadService, private val realDebridRateLimiter: RateLimiter -) : DebridCachedTorrentClient, StreamableLinkPreparable by DefaultStreamableLinkPreparer( +) : DebridCachedTorrentClient, ConfigurationTester, StreamableLinkPreparable by DefaultStreamableLinkPreparer( httpClient, debridavConfigurationProperties, realDebridRateLimiter @@ -405,4 +407,24 @@ class RealDebridClient( private suspend fun isLinkAlive(link: String): Boolean { return realDebridRateLimiter.executeSuspendFunction { httpClient.head(link).status.isSuccess() } } + + override val configurationClass: KClass<*> = RealDebridConfigurationProperties::class + override val label: String = "Real-Debrid" + + override suspend fun test(overrides: Map): TestResult = try { + val baseUrl = overrides["real-debrid.base-url"] ?: realDebridConfigurationProperties.baseUrl + val apiKey = overrides["real-debrid.api-key"] ?: realDebridConfigurationProperties.apiKey + + val response = httpClient.get("$baseUrl/user") { + accept(ContentType.Application.Json) + bearerAuth(apiKey) + } + if (response.status.isSuccess()) { + TestResult(success = true, message = "Connected successfully") + } else { + TestResult(success = false, message = "HTTP ${response.status.value}: ${response.bodyAsText()}") + } + } catch (e: Exception) { + TestResult(success = false, message = e.message ?: "Unknown error") + } } diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridConfiguration.kt index 4e4b54d7..270675bd 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridConfiguration.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridConfiguration.kt @@ -3,7 +3,6 @@ package io.skjaere.debridav.debrid.client.realdebrid import io.github.resilience4j.ratelimiter.RateLimiter import io.github.resilience4j.ratelimiter.RateLimiterConfig import io.github.resilience4j.ratelimiter.RateLimiterRegistry -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import java.time.Duration @@ -16,7 +15,6 @@ private const val TIMEOUT = 5L @Configuration class RealDebridConfiguration { @Bean - @ConditionalOnExpression("#{'\${debridav.debrid-clients}'.contains('real_debrid')}") fun realDebridRateLimiter(rateLimiterRegistry: RateLimiterRegistry): RateLimiter { val rateLimiterConfig = RateLimiterConfig.custom() .limitRefreshPeriod(Duration.ofMinutes(WINDOW_DURATION_MINUTES)) diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridConfigurationProperties.kt index cfeec123..e7a54472 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridConfigurationProperties.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridConfigurationProperties.kt @@ -1,10 +1,16 @@ package io.skjaere.debridav.debrid.client.realdebrid +import io.skjaere.debridav.config.ConfigProperty import org.springframework.boot.context.properties.ConfigurationProperties @ConfigurationProperties(prefix = "real-debrid") -class RealDebridConfigurationProperties( - val apiKey: String, - var baseUrl: String, - val syncEnabled: Boolean, -) +class RealDebridConfigurationProperties { + @ConfigProperty(name = "API Key", description = "Real-Debrid API key", sensitive = true) + lateinit var apiKey: String + @ConfigProperty(name = "Base URL", description = "Real-Debrid base URL", advanced = true) + lateinit var baseUrl: String + @ConfigProperty(name = "Sync Enabled", description = "Enable Real-Debrid sync") + var syncEnabled: Boolean = false + @ConfigProperty(name = "Sync Poll Rate", description = "Real-Debrid sync poll rate") + var syncPollRate: String = "PT24H" +} diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/support/RealDebridDownloadService.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/support/RealDebridDownloadService.kt index 97e5509d..91abf5f6 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/support/RealDebridDownloadService.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/support/RealDebridDownloadService.kt @@ -17,13 +17,11 @@ import io.skjaere.debridav.debrid.client.realdebrid.model.RealDebridDownloadRepo import io.skjaere.debridav.torrent.TorrentHash import jakarta.transaction.Transactional import kotlinx.coroutines.runBlocking -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.stereotype.Service private const val BULK_SIZE = 100 @Service -@ConditionalOnExpression("#{'\${debridav.debrid-clients}'.contains('real_debrid')}") class RealDebridDownloadService( private val realDebridDownloadRepository: RealDebridDownloadRepository, private val realDebridConfigurationProperties: RealDebridConfigurationProperties, diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/support/RealDebridTorrentService.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/support/RealDebridTorrentService.kt index 3c788590..609a9cb6 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/support/RealDebridTorrentService.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/support/RealDebridTorrentService.kt @@ -17,13 +17,11 @@ import io.skjaere.debridav.debrid.client.realdebrid.model.TorrentsInfo import io.skjaere.debridav.torrent.TorrentHash import jakarta.transaction.Transactional import kotlinx.coroutines.runBlocking -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.stereotype.Component private const val BULK_SIZE = 100 @Component -@ConditionalOnExpression("#{'\${debridav.debrid-clients}'.contains('real_debrid')}") class RealDebridTorrentService( private val realDebridConfigurationProperties: RealDebridConfigurationProperties, private val realDebridTorrentRepository: RealDebridTorrentRepository, diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxClient.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxClient.kt index 8339b7a7..61ea7ef4 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxClient.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxClient.kt @@ -21,6 +21,9 @@ import io.ktor.http.isSuccess import io.ktor.http.parameters import io.ktor.http.userAgent import io.milton.http.Range +import io.ktor.client.statement.bodyAsText +import io.skjaere.debridav.config.ConfigurationTester +import io.skjaere.debridav.config.TestResult import io.skjaere.debridav.configuration.DebridavConfigurationProperties import io.skjaere.debridav.debrid.DebridProvider import io.skjaere.debridav.debrid.TorrentMagnet @@ -35,10 +38,10 @@ import io.skjaere.debridav.fs.CachedFile import org.apache.commons.io.FileUtils import org.slf4j.Logger import org.slf4j.LoggerFactory -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.stereotype.Component import java.time.Duration import java.time.Instant +import kotlin.reflect.KClass const val RATE_LIMIT_WINDOW_SIZE_SECONDS = 59L const val RATE_LIMIT_REQUESTS_IN_WINDOW = 60 @@ -46,13 +49,12 @@ const val RATE_LIMIT_TIMEOUT_SECONDS = 5L const val USER_AGENT = "DebriDav/0.9.2 (https://github.com/skjaere/DebriDav)" @Component -@ConditionalOnExpression("#{'\${debridav.debrid-clients}'.contains('torbox')}") class TorBoxClient( private val torboxHttpClient: HttpClient, private val torBoxConfiguration: TorBoxConfigurationProperties, private val debridavConfigurationProperties: DebridavConfigurationProperties, rateLimiterRegistry: RateLimiterRegistry -) : DebridCachedTorrentClient, StreamableLinkPreparable { +) : DebridCachedTorrentClient, ConfigurationTester, StreamableLinkPreparable { companion object { const val TORRENT_ID_KEY = "torrent_id" @@ -244,4 +246,25 @@ class TorBoxClient( } }.status.isSuccess() } + + override val configurationClass: KClass<*> = TorBoxConfigurationProperties::class + override val label: String = "TorBox" + + override suspend fun test(overrides: Map): TestResult = try { + val baseUrl = overrides["torbox.base-url"] ?: torBoxConfiguration.baseUrl + val version = overrides["torbox.version"] ?: torBoxConfiguration.version + val apiKey = overrides["torbox.api-key"] ?: torBoxConfiguration.apiKey + + val response = torboxHttpClient.get("$baseUrl/$version/api/user/me") { + accept(ContentType.Application.Json) + bearerAuth(apiKey) + } + if (response.status.isSuccess()) { + TestResult(success = true, message = "Connected successfully") + } else { + TestResult(success = false, message = "HTTP ${response.status.value}: ${response.bodyAsText()}") + } + } catch (e: Exception) { + TestResult(success = false, message = e.message ?: "Unknown error") + } } diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxConfigurationProperties.kt index f4db357b..e111ddac 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxConfigurationProperties.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxConfigurationProperties.kt @@ -1,13 +1,18 @@ package io.skjaere.debridav.debrid.client.torbox +import io.skjaere.debridav.config.ConfigProperty import org.springframework.boot.context.properties.ConfigurationProperties @ConfigurationProperties(prefix = "torbox") -class TorBoxConfigurationProperties( - val apiKey: String, - val baseUrl: String, - val version: String, - val requestTimeoutMillis: Long, - val socketTimeoutMillis: Long, - - ) +class TorBoxConfigurationProperties { + @ConfigProperty(name = "API Key", description = "TorBox API key", sensitive = true) + lateinit var apiKey: String + @ConfigProperty(name = "Base URL", description = "TorBox base URL", advanced = true) + lateinit var baseUrl: String + @ConfigProperty(name = "API Version", description = "TorBox API version") + lateinit var version: String + @ConfigProperty(name = "Request Timeout", description = "TorBox request timeout in ms") + var requestTimeoutMillis: Long = 0 + @ConfigProperty(name = "Socket Timeout", description = "TorBox socket timeout in ms") + var socketTimeoutMillis: Long = 0 +} diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxHttpClientConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxHttpClientConfiguration.kt index 8c9de883..21c203ee 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxHttpClientConfiguration.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxHttpClientConfiguration.kt @@ -13,7 +13,6 @@ import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import kotlinx.serialization.json.Json import org.slf4j.LoggerFactory -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration @@ -27,7 +26,6 @@ class TorBoxHttpClientConfiguration { private val logger = LoggerFactory.getLogger(TorBoxHttpClientConfiguration::class.java) @Bean - @ConditionalOnExpression("#{'\${debridav.debrid-clients}'.contains('torbox')}") fun torboxHttpClient(): HttpClient { val client = HttpClient(CIO) { install(ContentNegotiation) { diff --git a/src/main/kotlin/io/skjaere/debridav/fs/FileController.kt b/src/main/kotlin/io/skjaere/debridav/fs/FileController.kt new file mode 100644 index 00000000..ecf37393 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/fs/FileController.kt @@ -0,0 +1,111 @@ +package io.skjaere.debridav.fs + +import kotlinx.coroutines.runBlocking +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RequestParam +import org.springframework.web.bind.annotation.RestController + +@RestController +@RequestMapping("/api/v1/files") +class FileController( + private val databaseFileService: DatabaseFileService +) { + @GetMapping("/detail") + fun detail(@RequestParam path: String): ResponseEntity { + val entity = databaseFileService.getFileAtPath(path) + ?: return ResponseEntity.notFound().build() + + if (entity is DbDirectory) { + return ResponseEntity.badRequest().build() + } + + val dto = when (entity) { + is RemotelyCachedEntity -> { + val contents = entity.contents + val fileType = when (contents) { + is DebridCachedTorrentContent -> FileType.TORRENT + is DebridCachedUsenetReleaseContent -> FileType.USENET_RELEASE + is NzbContents -> FileType.NZB + else -> FileType.LOCAL + } + val providerStatus = contents?.debridLinks?.mapNotNull { link -> + val provider = link.provider ?: return@mapNotNull null + val status = when (link) { + is CachedFile -> ProviderCacheStatus.CACHED + is MissingFile -> ProviderCacheStatus.MISSING + is ProviderError -> ProviderCacheStatus.PROVIDER_ERROR + is ClientError -> ProviderCacheStatus.CLIENT_ERROR + is NetworkError -> ProviderCacheStatus.NETWORK_ERROR + else -> ProviderCacheStatus.UNKNOWN_ERROR + } + ProviderStatusDto( + provider = provider, + status = status, + lastChecked = link.lastChecked + ) + } + FileDetailDto( + name = entity.name ?: "", + path = path, + size = entity.size, + lastModified = entity.lastModified, + mimeType = entity.mimeType, + fileType = fileType, + hash = entity.hash, + providerStatus = providerStatus + ) + } + is LocalEntity -> FileDetailDto( + name = entity.name ?: "", + path = path, + size = entity.size, + lastModified = entity.lastModified, + mimeType = entity.mimeType, + fileType = FileType.LOCAL, + hash = null, + providerStatus = null + ) + else -> return ResponseEntity.badRequest().build() + } + + return ResponseEntity.ok(dto) + } + + @GetMapping + fun list(@RequestParam(defaultValue = "/") path: String): ResponseEntity> { + val entity = databaseFileService.getFileAtPath(path) + ?: return ResponseEntity.notFound().build() + + if (entity !is DbDirectory) { + return ResponseEntity.badRequest().build() + } + + val children = runBlocking { databaseFileService.getChildren(entity) } + + val entries = children.mapNotNull { child -> + val name = child.name ?: return@mapNotNull null + when (child) { + is DbDirectory -> FileEntryDto( + name = name, + path = child.fileSystemPath() ?: path, + isDirectory = true, + size = null, + lastModified = child.lastModified, + mimeType = null + ) + else -> FileEntryDto( + name = name, + path = "${path.trimEnd('/')}/$name", + isDirectory = false, + size = child.size, + lastModified = child.lastModified, + mimeType = child.mimeType + ) + } + }.sortedWith(compareByDescending { it.isDirectory }.thenBy { it.name.lowercase() }) + + return ResponseEntity.ok(entries) + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/fs/FileDetailDto.kt b/src/main/kotlin/io/skjaere/debridav/fs/FileDetailDto.kt new file mode 100644 index 00000000..60e57076 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/fs/FileDetailDto.kt @@ -0,0 +1,28 @@ +package io.skjaere.debridav.fs + +import io.skjaere.debridav.debrid.DebridProvider + +data class FileDetailDto( + val name: String, + val path: String, + val size: Long?, + val lastModified: Long?, + val mimeType: String?, + val fileType: FileType, + val hash: String?, + val providerStatus: List? +) + +enum class FileType { + TORRENT, USENET_RELEASE, NZB, LOCAL +} + +data class ProviderStatusDto( + val provider: DebridProvider, + val status: ProviderCacheStatus, + val lastChecked: Long? +) + +enum class ProviderCacheStatus { + CACHED, MISSING, PROVIDER_ERROR, CLIENT_ERROR, NETWORK_ERROR, UNKNOWN_ERROR +} diff --git a/src/main/kotlin/io/skjaere/debridav/fs/FileEntryDto.kt b/src/main/kotlin/io/skjaere/debridav/fs/FileEntryDto.kt new file mode 100644 index 00000000..9d1e2f5b --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/fs/FileEntryDto.kt @@ -0,0 +1,10 @@ +package io.skjaere.debridav.fs + +data class FileEntryDto( + val name: String, + val path: String, + val isDirectory: Boolean, + val size: Long?, + val lastModified: Long?, + val mimeType: String? +) diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/NzbStreamerConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/usenet/NzbStreamerConfiguration.kt index 292247ce..bb28e76b 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/NzbStreamerConfiguration.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/NzbStreamerConfiguration.kt @@ -1,5 +1,6 @@ package io.skjaere.debridav.usenet +import io.skjaere.debridav.config.ConfigProperty import io.skjaere.nzbstreamer.NzbStreamer import io.skjaere.nzbstreamer.config.NntpConfig import io.skjaere.nzbstreamer.config.SeekConfig @@ -12,20 +13,20 @@ import java.time.Duration @Suppress("MagicNumber") @ConfigurationProperties(prefix = "nntp") -data class NntpConfigurationProperties( - val enabled: Boolean = false, - val host: String = "", - val port: Int = 563, - val username: String = "", - val password: String = "", - val useTls: Boolean = true, - val concurrency: Int = 4, - val maxConnections: Int = 8, - val readAheadSegments: Int? = null, - val forwardThresholdBytes: Long = 102400L, - val healthCheckInterval: Duration = Duration.ofDays(7), - val healthCheckPollRate: Duration = Duration.ofMinutes(5) -) +class NntpConfigurationProperties { + @ConfigProperty(name = "Enabled", description = "Enable NNTP") + var enabled: Boolean = false + @ConfigProperty(name = "Concurrency", description = "NNTP streaming concurrency") + var concurrency: Int = 4 + var readAheadSegments: Int? = null + @ConfigProperty(name = "Forward Threshold Bytes", description = "NNTP forward threshold bytes") + var forwardThresholdBytes: Long = 102400L + @ConfigProperty(name = "Health Check Interval", description = "NNTP health check interval") + var healthCheckInterval: Duration = Duration.ofDays(7) + @ConfigProperty(name = "Health Check Poll Rate", description = "NNTP health check poll rate") + var healthCheckPollRate: Duration = Duration.ofMinutes(5) + var pools: List = emptyList() +} @Configuration class NzbStreamerConfiguration { diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqSpringConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqSpringConfiguration.kt index bec0ff65..84533ad2 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqSpringConfiguration.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqSpringConfiguration.kt @@ -17,18 +17,18 @@ import javax.sql.DataSource @Suppress("MagicNumber") @ConfigurationProperties(prefix = "pgmq") -data class PgmqConfigurationProperties( - val defaultVisibilityTimeout: Duration = Duration.ofMinutes(5), - val importConcurrency: Int = 2, - val importVisibilityTimeout: Duration = Duration.ofMinutes(10), - val importPollInterval: Duration = Duration.ofSeconds(2), - val healthCheckConcurrency: Int = 1, - val healthCheckVisibilityTimeout: Duration = Duration.ofMinutes(5), - val healthCheckPollInterval: Duration = Duration.ofSeconds(10), - val healthRepairConcurrency: Int = 2, - val healthRepairVisibilityTimeout: Duration = Duration.ofMinutes(2), - val healthRepairPollInterval: Duration = Duration.ofSeconds(5) -) +class PgmqConfigurationProperties { + var defaultVisibilityTimeout: Duration = Duration.ofMinutes(5) + var importConcurrency: Int = 2 + var importVisibilityTimeout: Duration = Duration.ofMinutes(10) + var importPollInterval: Duration = Duration.ofSeconds(2) + var healthCheckConcurrency: Int = 1 + var healthCheckVisibilityTimeout: Duration = Duration.ofMinutes(5) + var healthCheckPollInterval: Duration = Duration.ofSeconds(10) + var healthRepairConcurrency: Int = 2 + var healthRepairVisibilityTimeout: Duration = Duration.ofMinutes(2) + var healthRepairPollInterval: Duration = Duration.ofSeconds(5) +} @Configuration @ConditionalOnProperty("nntp.enabled", havingValue = "true") diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties deleted file mode 100644 index 58a6f866..00000000 --- a/src/main/resources/application.properties +++ /dev/null @@ -1,103 +0,0 @@ -#spring.jpa.generate-ddl=true -#spring.jpa.hibernate.ddl-auto=update -spring.jpa.properties.hibernate.event.merge.entity_copy_observer=allow -logging.level.web=info -spring.servlet.multipart.max-file-size=-1 -spring.servlet.multipart.max-request-size=-1 -management.endpoints.web.exposure.include=health,realdebrid,prometheus,nzbhealthcheck -management.endpoint.health.group.readiness.include=fileSystemImportService -management.endpoint.health.group.liveness.exclude=fileSystemImportService -debridav.root-path=${user.dir}/debridav-files -debridav.download-path=/downloads -debridav.mount-path=/data -debridav.debrid-clients= -debridav.delay-between-retries=200ms -debridav.retries-on-provider-error=1 -debridav.wait-after-missing=24h -debridav.wait-after-network-error=1000ms -debridav.wait-after-provider-error=10m -debridav.wait-after-client-error=1000ms -debridav.should-delete-non-working-files=true -debridav.connect-timeout-milliseconds=5000 -debridav.read-timeout-milliseconds=5000 -debridav.enable-file-import-on-startup=true -debridav.local-entity-max-size-mb=130 -debridav.default-categories= -debridav.torrent-lifetime=1d -# WebDAV Authentication (empty = disabled) -debridav.webdav-username= -debridav.webdav-password= -# Database -debridav.db.host=localhost -debridav.db.port=5432 -debridav.db.database-name=debridav -spring.datasource.username=debridav -spring.datasource.password=debridav -# five hours -spring.datasource.hikari.max-lifetime=180000000 -spring.datasource.hikari.idle-timeout=0 -spring.datasource.url=jdbc:postgresql://${debridav.db.host}:5432/debridav -spring.datasource.hikari.maximum-pool-size=5 -# Premiumize -premiumize.api-key= -premiumize.bas-eurl=https://www.premiumize.me/api -# Real-Debrid -real-debrid.api-key= -real-debrid.base-url=https://api.real-debrid.com/rest/1.0 -real-debrid.sync-enabled=true -real-debrid.sync-poll-rate=PT24H -# TorBox -torbox.api-key= -torbox.base-url=https://api.torbox.app -torbox.version=v1 -torbox.request-timeout-millis=10000 -torbox.socket-timeout-millis=10000 -logging.level.io.milton.http.*=error -# Easynews -easynews.username= -easynews.password= -easynews.api-base-url=https://members.easynews.com -easynews.enabled-for-torrents=true -easynews.rate-limit-window-duration=15s -easynews.allowed-requests-in-window=10 -easynews.connect-timeout=20000 -easynews.socket-timeout=5000 -# Sonarr -sonarr.integration-enabled=false -sonarr.host=localhost -sonarr.port=8990 -sonarr.api-base-path=/api/v3 -sonarr.api-key=1105779a7abb40898567b406442cd927 -sonarr.category=tv-sonarr -radarr.integration-enabled=false -radarr.host=localhost -radarr.port=7878 -radarr.api-base-path=/api/v3 -radarr.api-key=8d273d4f92294234a9cdddba605054e1 -radarr.category=radarr -# NNTP -nntp.enabled=false -#nntp.host= -nntp.port=563 -nntp.username= -nntp.password= -nntp.use-tls=true -nntp.concurrency=4 -nntp.max-connections=60 -nntp.forward-threshold-bytes=102400 -nntp.health-check-interval=7d -nntp.health-check-poll-rate=PT5M -# PGMQ -pgmq.default-visibility-timeout=5m -pgmq.import-concurrency=2 -pgmq.import-visibility-timeout=10m -pgmq.import-poll-interval=2s -pgmq.health-check-concurrency=1 -pgmq.health-check-visibility-timeout=5m -pgmq.health-check-poll-interval=10s -pgmq.health-repair-concurrency=2 -pgmq.health-repair-visibility-timeout=2m -pgmq.health-repair-poll-interval=5s -# Sentry / GlitchTip -sentry.send-default-pii=false -sentry.traces-sample-rate=0 diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml new file mode 100644 index 00000000..20a8d76d --- /dev/null +++ b/src/main/resources/application.yaml @@ -0,0 +1,147 @@ +#spring: +# jpa: +# generate-ddl: true +# hibernate: +# ddl-auto: update + +spring: + jpa: + properties: + hibernate: + event: + merge: + entity_copy_observer: allow + servlet: + multipart: + max-file-size: -1 + max-request-size: -1 + datasource: + username: debridav + password: debridav + url: jdbc:postgresql://${debridav.db.host}:5432/debridav + hikari: + max-lifetime: 180000000 # five hours + idle-timeout: 0 + maximum-pool-size: 5 + +logging: + level: + web: info + io.milton.http: error + +management: + endpoints: + web: + exposure: + include: health,realdebrid,prometheus,nzbhealthcheck + endpoint: + health: + group: + readiness: + include: fileSystemImportService + liveness: + exclude: fileSystemImportService + +debridav: + root-path: ${user.dir}/debridav-files + download-path: /downloads + mount-path: /data + debrid-clients: + delay-between-retries: 200ms + retries-on-provider-error: 1 + wait-after-missing: 24h + wait-after-network-error: 1000ms + wait-after-provider-error: 10m + wait-after-client-error: 1000ms + should-delete-non-working-files: true + connect-timeout-milliseconds: 5000 + read-timeout-milliseconds: 5000 + enable-file-import-on-startup: true + local-entity-max-size-mb: 130 + default-categories: + torrent-lifetime: 1d + webdav-username: + webdav-password: + db: + host: localhost + port: 5432 + database-name: debridav + auth: + enabled: false + jwt-secret: ${DEBRIDAV_AUTH_JWT_SECRET:} + token-expiration-hours: 24 + protect-qbittorrent-api: false + protect-sabnzbd-api: false + protect-actuator: false + +premiumize: + api-key: + bas-eurl: https://www.premiumize.me/api + +real-debrid: + api-key: + base-url: https://api.real-debrid.com/rest/1.0 + sync-enabled: true + sync-poll-rate: PT24H + +torbox: + api-key: + base-url: https://api.torbox.app + version: v1 + request-timeout-millis: 10000 + socket-timeout-millis: 10000 + +easynews: + username: + password: + api-base-url: https://members.easynews.com + enabled-for-torrents: true + rate-limit-window-duration: 15s + allowed-requests-in-window: 10 + connect-timeout: 20000 + socket-timeout: 5000 + +sonarr: + integration-enabled: false + host: localhost + port: 8990 + api-base-path: /api/v3 + api-key: 1105779a7abb40898567b406442cd927 + category: tv-sonarr + +radarr: + integration-enabled: false + host: localhost + port: 7878 + api-base-path: /api/v3 + api-key: 8d273d4f92294234a9cdddba605054e1 + category: radarr + +nntp: + enabled: false + #host: + port: 563 + username: + password: + use-tls: true + concurrency: 4 + max-connections: 60 + forward-threshold-bytes: 102400 + health-check-interval: 7d + health-check-poll-rate: PT5M + +pgmq: + default-visibility-timeout: 5m + import-concurrency: 2 + import-visibility-timeout: 10m + import-poll-interval: 2s + health-check-concurrency: 1 + health-check-visibility-timeout: 5m + health-check-poll-interval: 10s + health-repair-concurrency: 2 + health-repair-visibility-timeout: 2m + health-repair-poll-interval: 5s + +sentry: + send-default-pii: false + traces-sample-rate: 0 diff --git a/src/main/resources/db/migration/V14__config_override_table.sql b/src/main/resources/db/migration/V14__config_override_table.sql new file mode 100644 index 00000000..b7d63609 --- /dev/null +++ b/src/main/resources/db/migration/V14__config_override_table.sql @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS config_override ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + prop_key VARCHAR(255) NOT NULL UNIQUE, + prop_value TEXT, + sensitive BOOLEAN DEFAULT FALSE, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); diff --git a/src/test/kotlin/io/skjaere/debridav/test/DebridLinkServiceTest.kt b/src/test/kotlin/io/skjaere/debridav/test/DebridLinkServiceTest.kt index 71c17c82..b316918f 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/DebridLinkServiceTest.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/DebridLinkServiceTest.kt @@ -20,6 +20,7 @@ import io.skjaere.debridav.debrid.client.model.NetworkErrorGetCachedFilesRespons import io.skjaere.debridav.debrid.client.model.NotCachedGetCachedFilesResponse import io.skjaere.debridav.debrid.client.model.ProviderErrorGetCachedFilesResponse import io.skjaere.debridav.debrid.client.model.SuccessfulGetCachedFilesResponse +import io.skjaere.debridav.debrid.client.DebridCachedContentClient import io.skjaere.debridav.debrid.client.premiumize.PremiumizeClient import io.skjaere.debridav.debrid.client.realdebrid.RealDebridClient import io.skjaere.debridav.debrid.model.DebridProviderError @@ -49,26 +50,26 @@ class DebridLinkServiceTest { private val clock = Clock.fixed(Instant.ofEpochMilli(1730477942L), ZoneId.systemDefault()) private val realDebridClient = mockk() private val debridCachedContentService = mockk() - private val debridClients = listOf(realDebridClient, premiumizeClient) - private val debridavConfigurationProperties = DebridavConfigurationProperties( - mountPath = "${TestContextInitializer.BASE_PATH}/debridav", - debridClients = listOf(DebridProvider.REAL_DEBRID, DebridProvider.PREMIUMIZE), - downloadPath = "${TestContextInitializer.BASE_PATH}/downloads", - rootPath = "${TestContextInitializer.BASE_PATH}/files", - retriesOnProviderError = 3, - waitAfterNetworkError = Duration.ofMillis(10000), - delayBetweenRetries = Duration.ofMillis(1000), - waitAfterMissing = Duration.ofMillis(1000), - waitAfterProviderError = Duration.ofMillis(1000), - readTimeoutMilliseconds = 1000, - connectTimeoutMilliseconds = 1000, - waitAfterClientError = Duration.ofMillis(1000), - shouldDeleteNonWorkingFiles = true, - torrentLifetime = Duration.ofMinutes(1), - enableFileImportOnStartup = false, - defaultCategories = listOf(), + private val debridClients: List = listOf(realDebridClient, premiumizeClient) + private val debridavConfigurationProperties = DebridavConfigurationProperties().apply { + mountPath = "${TestContextInitializer.BASE_PATH}/debridav" + debridClients = listOf(DebridProvider.REAL_DEBRID, DebridProvider.PREMIUMIZE) + downloadPath = "${TestContextInitializer.BASE_PATH}/downloads" + rootPath = "${TestContextInitializer.BASE_PATH}/files" + retriesOnProviderError = 3 + waitAfterNetworkError = Duration.ofMillis(10000) + delayBetweenRetries = Duration.ofMillis(1000) + waitAfterMissing = Duration.ofMillis(1000) + waitAfterProviderError = Duration.ofMillis(1000) + readTimeoutMilliseconds = 1000 + connectTimeoutMilliseconds = 1000 + waitAfterClientError = Duration.ofMillis(1000) + shouldDeleteNonWorkingFiles = true + torrentLifetime = Duration.ofMinutes(1) + enableFileImportOnStartup = false + defaultCategories = listOf() localEntityMaxSizeMb = 1 - ) + } val file = mockk() private val fileService = mockk() diff --git a/src/test/kotlin/io/skjaere/debridav/test/NzbImportServiceTest.kt b/src/test/kotlin/io/skjaere/debridav/test/NzbImportServiceTest.kt index 66305c08..98dc91cc 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/NzbImportServiceTest.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/NzbImportServiceTest.kt @@ -43,25 +43,25 @@ class NzbImportServiceTest { private val usenetRepository = mockk() private val pgmqClient = mockk() private val databaseFileService = mockk() - private val config = DebridavConfigurationProperties( - rootPath = "/", - downloadPath = "/downloads", - mountPath = "/data", - debridClients = listOf(DebridProvider.EASYNEWS), - waitAfterMissing = Duration.ZERO, - waitAfterProviderError = Duration.ZERO, - waitAfterNetworkError = Duration.ZERO, - waitAfterClientError = Duration.ZERO, - retriesOnProviderError = 0, - delayBetweenRetries = Duration.ZERO, - connectTimeoutMilliseconds = 5000, - readTimeoutMilliseconds = 30000, - shouldDeleteNonWorkingFiles = false, - torrentLifetime = Duration.ofHours(1), - enableFileImportOnStartup = false, - defaultCategories = emptyList(), + private val config = DebridavConfigurationProperties().apply { + rootPath = "/" + downloadPath = "/downloads" + mountPath = "/data" + debridClients = listOf(DebridProvider.EASYNEWS) + waitAfterMissing = Duration.ZERO + waitAfterProviderError = Duration.ZERO + waitAfterNetworkError = Duration.ZERO + waitAfterClientError = Duration.ZERO + retriesOnProviderError = 0 + delayBetweenRetries = Duration.ZERO + connectTimeoutMilliseconds = 5000 + readTimeoutMilliseconds = 30000 + shouldDeleteNonWorkingFiles = false + torrentLifetime = Duration.ofHours(1) + enableFileImportOnStartup = false + defaultCategories = emptyList() localEntityMaxSizeMb = 100 - ) + } private val underTest = NzbImportService( nzbStreamer, nzbDocumentRepository, usenetRepository, diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/ConfigApiIT.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/ConfigApiIT.kt new file mode 100644 index 00000000..242ba77c --- /dev/null +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/ConfigApiIT.kt @@ -0,0 +1,231 @@ +package io.skjaere.debridav.test.integrationtest + +import io.skjaere.debridav.DebriDavApplication +import io.skjaere.debridav.MiltonConfiguration +import io.skjaere.debridav.config.ConfigOverrideRepository +import io.skjaere.debridav.config.DatabasePropertySourceInitializer +import io.skjaere.debridav.configuration.DebridavConfigurationProperties +import io.skjaere.debridav.debrid.client.premiumize.PremiumizeConfigurationProperties +import io.skjaere.debridav.test.integrationtest.config.IntegrationTestContextConfiguration +import io.skjaere.debridav.test.integrationtest.config.MockServerTest +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.cloud.context.refresh.ContextRefresher +import org.springframework.http.MediaType +import org.springframework.test.web.reactive.server.WebTestClient +import java.time.Duration +import kotlin.test.assertEquals + +@SpringBootTest( + classes = [DebriDavApplication::class, IntegrationTestContextConfiguration::class, MiltonConfiguration::class], + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = [ + "debridav.debrid-clients=premiumize", + "debridav.auth.enabled=false", + "debridav.auth.jwt-secret=test-secret-key-that-is-at-least-256-bits-long-for-hs256" + ] +) +@MockServerTest +class ConfigApiIT { + + @Autowired + private lateinit var webTestClient: WebTestClient + + @Autowired + private lateinit var configOverrideRepository: ConfigOverrideRepository + + @Autowired + private lateinit var debridavConfig: DebridavConfigurationProperties + + @Autowired + private lateinit var premiumizeConfig: PremiumizeConfigurationProperties + + @Autowired + private lateinit var contextRefresher: ContextRefresher + + @Autowired + private lateinit var dbPropertySourceInitializer: DatabasePropertySourceInitializer + + @AfterEach + fun tearDown() { + configOverrideRepository.deleteAll() + val propertySource = dbPropertySourceInitializer.getOrCreatePropertySource() + propertySource.replaceAll(emptyMap()) + contextRefresher.refreshEnvironment() + } + + @Test + fun `list all whitelisted config properties`() { + webTestClient.get() + .uri("/api/v1/config") + .exchange() + .expectStatus().isOk + .expectBody() + .jsonPath("$").isArray + .jsonPath("$.length()").isNotEmpty + .jsonPath("$[?(@.key == 'debridav.root-path')]").exists() + .jsonPath("$[?(@.key == 'debridav.root-path')].name").isEqualTo("Root Path") + .jsonPath("$[?(@.key == 'debridav.root-path')].type").isEqualTo("STRING") + .jsonPath("$[?(@.key == 'debridav.should-delete-non-working-files')].type").isEqualTo("BOOLEAN") + .jsonPath("$[?(@.key == 'debridav.torrent-lifetime')].type").isEqualTo("DURATION") + .jsonPath("$[?(@.key == 'spring.datasource.url')]").doesNotExist() + } + + @Test + fun `get single config property`() { + webTestClient.get() + .uri("/api/v1/config/debridav.root-path") + .exchange() + .expectStatus().isOk + .expectBody() + .jsonPath("$.key").isEqualTo("debridav.root-path") + .jsonPath("$.name").isEqualTo("Root Path") + .jsonPath("$.hasOverride").isEqualTo(false) + .jsonPath("$.group").isEqualTo("debridav") + .jsonPath("$.type").isEqualTo("STRING") + } + + @Test + fun `upsert creates override`() { + webTestClient.put() + .uri("/api/v1/config/debridav.torrent-lifetime") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue("""{"value": "2h"}""") + .exchange() + .expectStatus().isOk + .expectBody() + .jsonPath("$.key").isEqualTo("debridav.torrent-lifetime") + .jsonPath("$.name").isEqualTo("Torrent Lifetime") + .jsonPath("$.hasOverride").isEqualTo(true) + .jsonPath("$.effectiveValue").isEqualTo("2h") + .jsonPath("$.type").isEqualTo("DURATION") + } + + @Test + fun `delete removes override and reverts to default`() { + // First create an override + webTestClient.put() + .uri("/api/v1/config/debridav.torrent-lifetime") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue("""{"value": "2h"}""") + .exchange() + .expectStatus().isOk + + // Then delete it + webTestClient.delete() + .uri("/api/v1/config/debridav.torrent-lifetime") + .exchange() + .expectStatus().isOk + .expectBody() + .jsonPath("$.hasOverride").isEqualTo(false) + } + + @Test + fun `non-whitelisted key returns 400`() { + webTestClient.get() + .uri("/api/v1/config/spring.datasource.url") + .exchange() + .expectStatus().isBadRequest + .expectBody() + .jsonPath("$.error").exists() + } + + @Test + fun `upsert non-whitelisted key returns 400`() { + webTestClient.put() + .uri("/api/v1/config/spring.datasource.url") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue("""{"value": "jdbc:postgresql://evil:5432/db"}""") + .exchange() + .expectStatus().isBadRequest + } + + @Test + fun `sensitive values are masked in responses`() { + // Upsert a sensitive value + webTestClient.put() + .uri("/api/v1/config/premiumize.api-key") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue("""{"value": "my-secret-key"}""") + .exchange() + .expectStatus().isOk + .expectBody() + .jsonPath("$.sensitive").isEqualTo(true) + .jsonPath("$.effectiveValue").isEqualTo("***") + } + + @Test + fun `delete non-existent override returns 404`() { + webTestClient.delete() + .uri("/api/v1/config/debridav.root-path") + .exchange() + .expectStatus().isNotFound + } + + @Test + fun `upsert refreshes config bean at runtime`() { + val originalLifetime = debridavConfig.torrentLifetime + + webTestClient.put() + .uri("/api/v1/config/debridav.torrent-lifetime") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue("""{"value": "2h"}""") + .exchange() + .expectStatus().isOk + + assertEquals(Duration.ofHours(2), debridavConfig.torrentLifetime) + assert(debridavConfig.torrentLifetime != originalLifetime) { + "torrentLifetime should have changed from default $originalLifetime" + } + } + + @Test + fun `delete reverts config bean to default at runtime`() { + val originalLifetime = debridavConfig.torrentLifetime + + // Override + webTestClient.put() + .uri("/api/v1/config/debridav.torrent-lifetime") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue("""{"value": "2h"}""") + .exchange() + .expectStatus().isOk + assertEquals(Duration.ofHours(2), debridavConfig.torrentLifetime) + + // Delete override + webTestClient.delete() + .uri("/api/v1/config/debridav.torrent-lifetime") + .exchange() + .expectStatus().isOk + + assertEquals(originalLifetime, debridavConfig.torrentLifetime) + } + + @Test + fun `upsert refreshes boolean property on config bean`() { + val original = debridavConfig.shouldDeleteNonWorkingFiles + + webTestClient.put() + .uri("/api/v1/config/debridav.should-delete-non-working-files") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue("""{"value": "${!original}"}""") + .exchange() + .expectStatus().isOk + + assertEquals(!original, debridavConfig.shouldDeleteNonWorkingFiles) + } + + @Test + fun `upsert refreshes property on different config bean`() { + webTestClient.put() + .uri("/api/v1/config/premiumize.api-key") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue("""{"value": "new-api-key-12345"}""") + .exchange() + .expectStatus().isOk + + assertEquals("new-api-key-12345", premiumizeConfig.apiKey) + } +} diff --git a/src/test/resources/application.properties b/src/test/resources/application.properties deleted file mode 100644 index 81c5a287..00000000 --- a/src/test/resources/application.properties +++ /dev/null @@ -1,87 +0,0 @@ -#spring.jpa.generate-ddl=true -#spring.jpa.hibernate.ddl-auto=update -logging.level.web=debug -spring.servlet.multipart.max-file-size=10MB -spring.servlet.multipart.max-request-size=10MB -management.endpoints.web.exposure.include=realdebrid -debridav.root-path=/tmp/debridavtests -debridav.download-path=/downloads -debridav.mount-path=/data -debridav.debrid-clients=premiumize,real_debrid,easynews -debridav.delay-between-retries=1000ms -debridav.retries-on-provider-error=3 -debridav.wait-after-missing=24h -debridav.wait-after-network-error=1000ms -debridav.wait-after-provider-error=1000ms -debridav.wait-after-client-error=1000ms -debridav.should-delete-non-working-files=true -debridav.connect-timeout-milliseconds=1000 -debridav.read-timeout-milliseconds=1000 -debridav.enable-file-import-on-startup=false -debridav.local-entity-max-size-mb=1 -debridav.default-categories=test1,test2 -# Database -debridav.db.host=localhost -debridav.db.port=5432 -debridav.db.database-name=debridav -debridav.db.username=debridav -debridav.db.password=debridav -spring.datasource.url=jdbc:postgresql://${debridav.db.host}:${debridav.db.port}/${debridav.db.name}?user=${debridav.db.username}&password=${debridav.db.password} -premiumize.api-key=asd -premiumize.bas-eurl=https://www.premiumize.me/api/ -real-debrid.api-key=asd -real-debrid.base-url=https://api.real-debrid.com/rest/1.0/ -real-debrid.sync-enabled=false -real-debrid.sync-poll-rate=PT4H -debridav.torrent-lifetime=1h -debridav.webdav-username= -debridav.webdav-password= -easynews.username=asd -easynews.password=asd -easynews.api-base-url=https://members.easynews.com/3.0/api -easynews.enabled-for-torrents=true -easynews.rate-limit-window-duration=15s -easynews.allowed-requests-in-window=100000 -easynews.connect-timeout=20000 -easynews.socket-timeout=5000 -# TorBox -torbox.api-key= -torbox.base-url=https://api.torbox.app -torbox.version=v1 -torbox.request-timeout-millis=5000 -torbox.socket-timeout-millis=5000 -# Sonarr -sonarr.integration-enabled=true -sonarr.host=http://localhost -sonarr.port=8989 -sonarr.api-base-path=/api/v3 -sonarr.api-key=1105779a7abb40898567b406442cd927 -sonarr.category=tv-sonarr -radarr.integration-enabled=true -radarr.host=http://localhost -radarr.port=7878 -radarr.api-base-path=/api/v3 -radarr.api-key=8d273d4f92294234a9cdddba605054e1 -radarr.category=radarr -# NNTP -nntp.enabled=false -#nntp.host= -nntp.port=563 -nntp.username= -nntp.password= -nntp.use-tls=true -nntp.concurrency=4 -nntp.forward-threshold-bytes=102400 -nntp.health-check-interval=7d -nntp.health-check-poll-rate=PT5M -# PGMQ -pgmq.default-visibility-timeout=5m -pgmq.import-concurrency=2 -pgmq.import-visibility-timeout=10m -pgmq.import-poll-interval=2s -pgmq.health-check-concurrency=1 -pgmq.health-check-visibility-timeout=5m -pgmq.health-check-poll-interval=10s -pgmq.health-repair-concurrency=2 -pgmq.health-repair-visibility-timeout=2m -pgmq.health-repair-poll-interval=5s \ No newline at end of file diff --git a/src/test/resources/application.yaml b/src/test/resources/application.yaml new file mode 100644 index 00000000..a63c6065 --- /dev/null +++ b/src/test/resources/application.yaml @@ -0,0 +1,124 @@ +#spring: +# jpa: +# generate-ddl: true +# hibernate: +# ddl-auto: update + +logging: + level: + web: debug + +spring: + servlet: + multipart: + max-file-size: 10MB + max-request-size: 10MB + datasource: + url: jdbc:postgresql://${debridav.db.host}:${debridav.db.port}/${debridav.db.name}?user=${debridav.db.username}&password=${debridav.db.password} + +management: + endpoints: + web: + exposure: + include: realdebrid + +debridav: + root-path: /tmp/debridavtests + download-path: /downloads + mount-path: /data + debrid-clients: premiumize,real_debrid,easynews + delay-between-retries: 1000ms + retries-on-provider-error: 3 + wait-after-missing: 24h + wait-after-network-error: 1000ms + wait-after-provider-error: 1000ms + wait-after-client-error: 1000ms + should-delete-non-working-files: true + connect-timeout-milliseconds: 1000 + read-timeout-milliseconds: 1000 + enable-file-import-on-startup: false + local-entity-max-size-mb: 1 + default-categories: test1,test2 + torrent-lifetime: 1h + webdav-username: + webdav-password: + db: + host: localhost + port: 5432 + database-name: debridav + username: debridav + password: debridav + auth: + enabled: false + jwt-secret: test-secret-key-that-is-at-least-256-bits-long-for-hs256 + token-expiration-hours: 24 + protect-qbittorrent-api: false + protect-sabnzbd-api: false + protect-actuator: false + +premiumize: + api-key: asd + bas-eurl: https://www.premiumize.me/api/ + +real-debrid: + api-key: asd + base-url: https://api.real-debrid.com/rest/1.0/ + sync-enabled: false + sync-poll-rate: PT4H + +torbox: + api-key: + base-url: https://api.torbox.app + version: v1 + request-timeout-millis: 5000 + socket-timeout-millis: 5000 + +easynews: + username: asd + password: asd + api-base-url: https://members.easynews.com/3.0/api + enabled-for-torrents: true + rate-limit-window-duration: 15s + allowed-requests-in-window: 100000 + connect-timeout: 20000 + socket-timeout: 5000 + +sonarr: + integration-enabled: true + host: http://localhost + port: 8989 + api-base-path: /api/v3 + api-key: 1105779a7abb40898567b406442cd927 + category: tv-sonarr + +radarr: + integration-enabled: true + host: http://localhost + port: 7878 + api-base-path: /api/v3 + api-key: 8d273d4f92294234a9cdddba605054e1 + category: radarr + +nntp: + enabled: false + #host: + port: 563 + username: + password: + use-tls: true + concurrency: 4 + forward-threshold-bytes: 102400 + health-check-interval: 7d + health-check-poll-rate: PT5M + +pgmq: + default-visibility-timeout: 5m + import-concurrency: 2 + import-visibility-timeout: 10m + import-poll-interval: 2s + health-check-concurrency: 1 + health-check-visibility-timeout: 5m + health-check-poll-interval: 10s + health-repair-concurrency: 2 + health-repair-visibility-timeout: 2m + health-repair-poll-interval: 5s From 77616df6ced43624b72a7ed3ba80d95f8e9ca22b Mon Sep 17 00:00:00 2001 From: Skjaere Date: Sat, 28 Feb 2026 14:15:00 +0100 Subject: [PATCH 02/61] feat(nntp): multi-pool support with runtime add/remove + priority fallback Migrates the NNTP config from a flat single-server shape to a pools list, each with host/port/credentials/TLS/maxConnections and a priority for fill/fallback. Adds CRUD endpoints for pools plus a test endpoint, and plumbs runtime updates through so pool changes applied via the config API take effect without a restart. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../config/ConfigOverrideController.kt | 30 ++++++++- .../config/ConfigOverrideRepository.kt | 2 + .../debridav/config/ConfigOverrideService.kt | 32 ++------- .../io/skjaere/debridav/config/NntpPoolDto.kt | 11 +++ .../skjaere/debridav/config/NntpPoolTester.kt | 35 ++++++++++ .../usenet/NzbStreamerConfiguration.kt | 67 +++++++++++++------ src/main/resources/application.yaml | 13 ++-- .../config/TestContextInitializer.kt | 6 +- src/test/resources/application.yaml | 12 ++-- 9 files changed, 145 insertions(+), 63 deletions(-) create mode 100644 src/main/kotlin/io/skjaere/debridav/config/NntpPoolDto.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/config/NntpPoolTester.kt diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideController.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideController.kt index 33f1f232..a607e2cd 100644 --- a/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideController.kt +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideController.kt @@ -15,7 +15,8 @@ import org.springframework.web.bind.annotation.RestController @RequestMapping("/api/v1/config") class ConfigOverrideController( private val service: ConfigOverrideService, - private val registry: ConfigPropertyRegistry + private val registry: ConfigPropertyRegistry, + private val nntpPoolTester: NntpPoolTester ) { @GetMapping fun listAll(): ResponseEntity> = @@ -40,6 +41,33 @@ class ConfigOverrideController( fun listTestable(): ResponseEntity> = ResponseEntity.ok(registry.getTestablePrefixes()) + @GetMapping("/nntp-pools") + fun getNntpPools(): ResponseEntity> = + ResponseEntity.ok(service.getNntpPools()) + + @PutMapping("/nntp-pools") + fun saveNntpPools(@RequestBody pools: List): ResponseEntity> { + service.saveNntpPools(pools) + return ResponseEntity.ok(service.getNntpPools()) + } + + @PostMapping("/nntp-pools/test") + fun testNntpPool(@RequestBody pool: NntpPoolDto): ResponseEntity { + val start = System.currentTimeMillis() + val result = runBlocking { nntpPoolTester.test(pool) } + val durationMs = System.currentTimeMillis() - start + + return ResponseEntity.ok( + ConfigTestResultDto( + prefix = "nntp", + label = "NNTP Pool", + success = result.success, + message = result.message, + durationMs = durationMs + ) + ) + } + @PostMapping("/test/{prefix}") fun test( @PathVariable prefix: String, diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideRepository.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideRepository.kt index 971c5598..1a4e57dc 100644 --- a/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideRepository.kt +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideRepository.kt @@ -5,4 +5,6 @@ import org.springframework.data.repository.CrudRepository interface ConfigOverrideRepository : CrudRepository { fun findByPropKey(key: String): ConfigOverride? fun findAllByPropKeyIn(keys: Collection): List + fun findAllByPropKeyStartingWith(prefix: String): List + fun deleteAllByPropKeyStartingWith(prefix: String) } diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideService.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideService.kt index f1bf3aca..55a9bb97 100644 --- a/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideService.kt +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideService.kt @@ -6,24 +6,22 @@ import io.skjaere.nzbstreamer.NzbStreamer import io.skjaere.nzbstreamer.config.NntpConfig import jakarta.transaction.Transactional import org.slf4j.LoggerFactory -import org.springframework.cloud.context.refresh.ContextRefresher -import org.springframework.core.env.ConfigurableEnvironment -import org.springframework.core.env.EnumerablePropertySource +import org.springframework.core.env.Environment import org.springframework.stereotype.Service import java.time.Instant @Service class ConfigOverrideService( private val repository: ConfigOverrideRepository, - private val environment: ConfigurableEnvironment, + private val environment: Environment, private val registry: ConfigPropertyRegistry, private val nntpConfig: NntpConfigurationProperties, - private val contextRefresher: ContextRefresher, - private val dbPropertySourceInitializer: DatabasePropertySourceInitializer, private val nzbStreamer: NzbStreamer? = null ) { + private val logger = LoggerFactory.getLogger(ConfigOverrideService::class.java) companion object { private const val MASKED = "***" + private const val POOL_PREFIX = "nntp.pools[" } fun listAll(): List { @@ -105,28 +103,6 @@ class ConfigOverrideService( return getEffective(key) } - private fun refreshEnvironment() { - val propertySource = dbPropertySourceInitializer.getOrCreatePropertySource() - val overrides = repository.findAll().associate { it.propKey to (it.propValue ?: "") } - propertySource.replaceAll(overrides) - contextRefresher.refreshEnvironment() - logger.info("Refreshed environment with {} database override(s)", overrides.size) - } - - private fun getDefaultValue(key: String): String? { - for (source in environment.propertySources) { - if (source.name == DatabasePropertySource.NAME) continue - if (source is EnumerablePropertySource<*>) { - val value = source.getProperty(key) - if (value != null) return value.toString() - } else { - val value = source.getProperty(key) - if (value != null) return value.toString() - } - } - return null - } - fun getNntpPools(): List { val overrides = repository.findAllByPropKeyStartingWith(POOL_PREFIX) val pools = if (overrides.isEmpty()) { diff --git a/src/main/kotlin/io/skjaere/debridav/config/NntpPoolDto.kt b/src/main/kotlin/io/skjaere/debridav/config/NntpPoolDto.kt new file mode 100644 index 00000000..05383fb6 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/NntpPoolDto.kt @@ -0,0 +1,11 @@ +package io.skjaere.debridav.config + +data class NntpPoolDto( + val host: String = "", + val port: Int = 563, + val username: String = "", + val password: String = "", + val useTls: Boolean = true, + val maxConnections: Int = 8, + val priority: Int = 0 +) diff --git a/src/main/kotlin/io/skjaere/debridav/config/NntpPoolTester.kt b/src/main/kotlin/io/skjaere/debridav/config/NntpPoolTester.kt new file mode 100644 index 00000000..d2f60ef2 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/config/NntpPoolTester.kt @@ -0,0 +1,35 @@ +package io.skjaere.debridav.config + +import io.ktor.network.selector.SelectorManager +import io.skjaere.nntp.NntpAuthenticationException +import io.skjaere.nntp.NntpClient +import io.skjaere.nntp.NntpException +import kotlinx.coroutines.Dispatchers +import org.springframework.stereotype.Service + +@Service +class NntpPoolTester { + suspend fun test(pool: NntpPoolDto): TestResult { + val selectorManager = SelectorManager(Dispatchers.IO) + try { + val client = NntpClient.connect( + host = pool.host, + port = pool.port, + selectorManager = selectorManager, + useTls = pool.useTls, + username = pool.username, + password = pool.password + ) + client.use { it.quit() } + return TestResult(success = true, message = "Connected successfully") + } catch (e: NntpAuthenticationException) { + return TestResult(success = false, message = "Authentication failed: ${e.message}") + } catch (e: NntpException) { + return TestResult(success = false, message = e.message ?: "NNTP error") + } catch (e: Exception) { + return TestResult(success = false, message = e.message ?: "Connection failed") + } finally { + selectorManager.close() + } + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/NzbStreamerConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/usenet/NzbStreamerConfiguration.kt index bb28e76b..bfc0ae7d 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/NzbStreamerConfiguration.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/NzbStreamerConfiguration.kt @@ -4,6 +4,7 @@ import io.skjaere.debridav.config.ConfigProperty import io.skjaere.nzbstreamer.NzbStreamer import io.skjaere.nzbstreamer.config.NntpConfig import io.skjaere.nzbstreamer.config.SeekConfig +import io.skjaere.nzbstreamer.config.StreamingConfig import org.slf4j.LoggerFactory import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.boot.context.properties.ConfigurationProperties @@ -11,22 +12,32 @@ import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration import java.time.Duration +data class NntpPoolProperties( + val host: String = "", + val port: Int = 563, + val username: String = "", + val password: String = "", + val useTls: Boolean = true, + val maxConnections: Int = 8, + val priority: Int = 0 +) + @Suppress("MagicNumber") @ConfigurationProperties(prefix = "nntp") class NntpConfigurationProperties { @ConfigProperty(name = "Enabled", description = "Enable NNTP") - var enabled: Boolean = false + val enabled: Boolean = false, @ConfigProperty(name = "Concurrency", description = "NNTP streaming concurrency") - var concurrency: Int = 4 - var readAheadSegments: Int? = null + val concurrency: Int = 4, + val readAheadSegments: Int? = null, @ConfigProperty(name = "Forward Threshold Bytes", description = "NNTP forward threshold bytes") var forwardThresholdBytes: Long = 102400L @ConfigProperty(name = "Health Check Interval", description = "NNTP health check interval") var healthCheckInterval: Duration = Duration.ofDays(7) @ConfigProperty(name = "Health Check Poll Rate", description = "NNTP health check poll rate") - var healthCheckPollRate: Duration = Duration.ofMinutes(5) - var pools: List = emptyList() -} + val healthCheckPollRate: Duration = Duration.ofMinutes(5), + val pools: List = emptyList() +) @Configuration class NzbStreamerConfiguration { @@ -35,24 +46,40 @@ class NzbStreamerConfiguration { @Bean @ConditionalOnProperty("nntp.enabled", havingValue = "true") fun nzbStreamer(props: NntpConfigurationProperties): NzbStreamer { + val nntpConfigs = buildNntpConfigs(props) + val streamingConfig = StreamingConfig( + concurrency = props.concurrency, + readAheadSegments = props.readAheadSegments ?: props.concurrency + ) logger.info( - "Creating NzbStreamer with host='{}', port={}, useTls={}, username='{}', concurrency={}, maxConnections={}", - props.host, props.port, props.useTls, props.username, props.concurrency, props.maxConnections + "Creating NzbStreamer with {} pool(s), concurrency={}", + nntpConfigs.size, streamingConfig.concurrency ) + nntpConfigs.forEachIndexed { index, config -> + logger.info( + " pool[{}]: host='{}', port={}, useTls={}, username='{}', maxConnections={}, priority={}", + index, config.host, config.port, config.useTls, config.username, config.maxConnections, + config.priority + ) + } return NzbStreamer.fromConfig( + nntpConfigs, + streamingConfig, + SeekConfig(forwardThresholdBytes = props.forwardThresholdBytes) + ) + } + + private fun buildNntpConfigs(props: NntpConfigurationProperties): List { + return props.pools.sortedBy { it.priority }.map { pool -> NntpConfig( - host = props.host, - port = props.port, - username = props.username, - password = props.password, - useTls = props.useTls, - concurrency = props.concurrency, - maxConnections = props.maxConnections, - readAheadSegments = props.readAheadSegments ?: props.concurrency - ), - SeekConfig( - forwardThresholdBytes = props.forwardThresholdBytes + host = pool.host, + port = pool.port, + username = pool.username, + password = pool.password, + useTls = pool.useTls, + maxConnections = pool.maxConnections, + priority = pool.priority ) - ) + } } } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 20a8d76d..d1344ab1 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -119,16 +119,17 @@ radarr: nntp: enabled: false - #host: - port: 563 - username: - password: - use-tls: true concurrency: 4 - max-connections: 60 forward-threshold-bytes: 102400 health-check-interval: 7d health-check-poll-rate: PT5M + pools: + - host: + port: 563 + username: + password: + use-tls: true + max-connections: 60 pgmq: default-visibility-timeout: 5m diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/TestContextInitializer.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/TestContextInitializer.kt index 9ea0e8a0..f612c97c 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/TestContextInitializer.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/TestContextInitializer.kt @@ -59,9 +59,9 @@ class TestContextInitializer : ApplicationContextInitializer Date: Sun, 1 Mar 2026 11:00:00 +0100 Subject: [PATCH 03/61] feat(auth): JWT authentication for API + temporary file access tokens Adds a JWT-based authentication layer that can protect the API and the qBittorrent / SABnzbd / actuator endpoints behind opt-in flags. Also generates short-lived tokens for temporary file access (used by the frontend file browser). Excludes Spring's default UserDetailsService auto-config so the generated-password warning no longer appears at startup. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../skjaere/debridav/DebriDavApplication.kt | 3 +- .../arrs/RadarrConfigurationProperties.kt | 4 +- .../arrs/SonarrConfigurationProperties.kt | 4 +- .../debridav/config/auth/JwtService.kt | 30 +++++ .../config/auth/SecurityConfiguration.kt | 3 +- .../io/skjaere/debridav/fs/FileController.kt | 22 +++- .../skjaere/debridav/fs/StreamController.kt | 90 +++++++++++++ .../io/skjaere/debridav/fs/StreamUrlDto.kt | 3 + .../test/integrationtest/JwtAuthIT.kt | 124 ++++++++++++++++++ 9 files changed, 276 insertions(+), 7 deletions(-) create mode 100644 src/main/kotlin/io/skjaere/debridav/fs/StreamController.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/fs/StreamUrlDto.kt create mode 100644 src/test/kotlin/io/skjaere/debridav/test/integrationtest/JwtAuthIT.kt diff --git a/src/main/kotlin/io/skjaere/debridav/DebriDavApplication.kt b/src/main/kotlin/io/skjaere/debridav/DebriDavApplication.kt index 982ec24a..04937fc4 100644 --- a/src/main/kotlin/io/skjaere/debridav/DebriDavApplication.kt +++ b/src/main/kotlin/io/skjaere/debridav/DebriDavApplication.kt @@ -1,9 +1,10 @@ package io.skjaere.debridav import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.security.autoconfigure.UserDetailsServiceAutoConfiguration import org.springframework.boot.runApplication -@SpringBootApplication +@SpringBootApplication(exclude = [UserDetailsServiceAutoConfiguration::class]) class DebriDavApplication @Suppress("SpreadOperator") diff --git a/src/main/kotlin/io/skjaere/debridav/arrs/RadarrConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/arrs/RadarrConfigurationProperties.kt index 61be041e..1903db47 100644 --- a/src/main/kotlin/io/skjaere/debridav/arrs/RadarrConfigurationProperties.kt +++ b/src/main/kotlin/io/skjaere/debridav/arrs/RadarrConfigurationProperties.kt @@ -10,9 +10,9 @@ class RadarrConfigurationProperties : ArrConfiguration { @ConfigProperty(name = "Host", description = "Radarr host") override var host: String = "" @ConfigProperty(name = "Port", description = "Radarr port") - override var port: Int = 7878 + override val port: Int = 7878, @ConfigProperty(name = "API Base Path", description = "Radarr API base path", advanced = true) - override var apiBasePath: String = "/api/v3" + override val apiBasePath: String = "/api/v3", @ConfigProperty(name = "API Key", description = "Radarr API key", sensitive = true) override var apiKey: String = "" @ConfigProperty(name = "Category", description = "Radarr category") diff --git a/src/main/kotlin/io/skjaere/debridav/arrs/SonarrConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/arrs/SonarrConfigurationProperties.kt index ca65f043..8420501b 100644 --- a/src/main/kotlin/io/skjaere/debridav/arrs/SonarrConfigurationProperties.kt +++ b/src/main/kotlin/io/skjaere/debridav/arrs/SonarrConfigurationProperties.kt @@ -10,9 +10,9 @@ class SonarrConfigurationProperties : ArrConfiguration { @ConfigProperty(name = "Host", description = "Sonarr host") override var host: String = "" @ConfigProperty(name = "Port", description = "Sonarr port") - override var port: Int = 8989 + override val port: Int = 8989, @ConfigProperty(name = "API Base Path", description = "Sonarr API base path", advanced = true) - override var apiBasePath: String = "/api/v3" + override val apiBasePath: String = "/api/v3", @ConfigProperty(name = "API Key", description = "Sonarr API key", sensitive = true) override var apiKey: String = "" @ConfigProperty(name = "Category", description = "Sonarr category") diff --git a/src/main/kotlin/io/skjaere/debridav/config/auth/JwtService.kt b/src/main/kotlin/io/skjaere/debridav/config/auth/JwtService.kt index db162547..096c3573 100644 --- a/src/main/kotlin/io/skjaere/debridav/config/auth/JwtService.kt +++ b/src/main/kotlin/io/skjaere/debridav/config/auth/JwtService.kt @@ -39,4 +39,34 @@ class JwtService( } catch (_: IllegalArgumentException) { null } + + fun generateStreamToken(path: String): String { + val now = Date() + val expiration = Date(now.time + STREAM_TOKEN_EXPIRY_SECONDS * 1000) + + return Jwts.builder() + .subject(path) + .claim("type", "stream") + .issuedAt(now) + .expiration(expiration) + .signWith(key) + .compact() + } + + fun validateStreamToken(token: String): String? = try { + val claims = Jwts.parser() + .verifyWith(key) + .build() + .parseSignedClaims(token) + .payload + if (claims["type"] == "stream") claims.subject else null + } catch (_: JwtException) { + null + } catch (_: IllegalArgumentException) { + null + } + + companion object { + const val STREAM_TOKEN_EXPIRY_SECONDS = 86400L // 24 hours + } } diff --git a/src/main/kotlin/io/skjaere/debridav/config/auth/SecurityConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/config/auth/SecurityConfiguration.kt index 36bb3f5a..bfea1f9d 100644 --- a/src/main/kotlin/io/skjaere/debridav/config/auth/SecurityConfiguration.kt +++ b/src/main/kotlin/io/skjaere/debridav/config/auth/SecurityConfiguration.kt @@ -32,8 +32,9 @@ class SecurityConfiguration( .exceptionHandling { it.authenticationEntryPoint(unauthorizedEntryPoint()) } .addFilterBefore(jwtAuthenticationFilter, UsernamePasswordAuthenticationFilter::class.java) .authorizeHttpRequests { auth -> - // Auth endpoints are always public + // Auth and stream endpoints are always public auth.requestMatchers("/api/v1/auth/**").permitAll() + auth.requestMatchers("/api/v1/stream/**").permitAll() // Config API is protected when auth is enabled if (authConfig.enabled) { diff --git a/src/main/kotlin/io/skjaere/debridav/fs/FileController.kt b/src/main/kotlin/io/skjaere/debridav/fs/FileController.kt index ecf37393..0b6095f8 100644 --- a/src/main/kotlin/io/skjaere/debridav/fs/FileController.kt +++ b/src/main/kotlin/io/skjaere/debridav/fs/FileController.kt @@ -1,5 +1,6 @@ package io.skjaere.debridav.fs +import io.skjaere.debridav.config.auth.JwtService import kotlinx.coroutines.runBlocking import org.springframework.http.ResponseEntity import org.springframework.web.bind.annotation.GetMapping @@ -10,8 +11,27 @@ import org.springframework.web.bind.annotation.RestController @RestController @RequestMapping("/api/v1/files") class FileController( - private val databaseFileService: DatabaseFileService + private val databaseFileService: DatabaseFileService, + private val jwtService: JwtService ) { + @GetMapping("/stream-url") + fun streamUrl(@RequestParam path: String): ResponseEntity { + val entity = databaseFileService.getFileAtPath(path) + ?: return ResponseEntity.notFound().build() + + if (entity is DbDirectory) { + return ResponseEntity.badRequest().build() + } + + val token = jwtService.generateStreamToken(path) + return ResponseEntity.ok( + StreamUrlDto( + url = "/api/v1/stream/t/$token", + expiresIn = JwtService.STREAM_TOKEN_EXPIRY_SECONDS + ) + ) + } + @GetMapping("/detail") fun detail(@RequestParam path: String): ResponseEntity { val entity = databaseFileService.getFileAtPath(path) diff --git a/src/main/kotlin/io/skjaere/debridav/fs/StreamController.kt b/src/main/kotlin/io/skjaere/debridav/fs/StreamController.kt new file mode 100644 index 00000000..f902dd3f --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/fs/StreamController.kt @@ -0,0 +1,90 @@ +package io.skjaere.debridav.fs + +import io.milton.http.Range +import io.milton.resource.GetableResource +import io.skjaere.debridav.config.auth.JwtService +import io.skjaere.debridav.resource.StreamableResourceFactory +import jakarta.servlet.http.HttpServletRequest +import jakarta.servlet.http.HttpServletResponse +import org.springframework.http.HttpStatus +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@RestController +@RequestMapping("/api/v1/stream") +class StreamController( + private val jwtService: JwtService, + private val databaseFileService: DatabaseFileService, + private val streamableResourceFactory: StreamableResourceFactory +) { + @GetMapping("/t/{token}") + fun streamByToken( + @PathVariable token: String, + request: HttpServletRequest, + response: HttpServletResponse + ) { + val path = jwtService.validateStreamToken(token) + if (path == null) { + response.status = HttpStatus.UNAUTHORIZED.value() + response.contentType = "application/json" + response.writer.write("""{"error":"Invalid or expired token"}""") + return + } + + val entity = databaseFileService.getFileAtPath(path) + if (entity == null || entity is DbDirectory) { + response.status = HttpStatus.NOT_FOUND.value() + response.contentType = "application/json" + response.writer.write("""{"error":"File not found"}""") + return + } + + val resource = streamableResourceFactory.toFileResource(entity) as? GetableResource + if (resource == null) { + response.status = HttpStatus.NOT_FOUND.value() + response.contentType = "application/json" + response.writer.write("""{"error":"File not found"}""") + return + } + + val contentLength = resource.contentLength + val contentType = resource.getContentType(null) ?: "application/octet-stream" + val rangeHeader = request.getHeader("Range") + val range = parseRangeHeader(rangeHeader, contentLength) + + response.contentType = contentType + response.setHeader("Accept-Ranges", "bytes") + + if (range != null) { + val start = range.start ?: 0 + val finish = range.finish ?: (contentLength - 1) + response.status = HttpServletResponse.SC_PARTIAL_CONTENT + response.setHeader("Content-Range", "bytes $start-$finish/$contentLength") + response.setContentLengthLong(finish - start + 1) + } else { + response.status = HttpServletResponse.SC_OK + response.setContentLengthLong(contentLength) + } + + resource.sendContent(response.outputStream, range, null, contentType) + } + + private fun parseRangeHeader(header: String?, contentLength: Long): Range? { + if (header == null || !header.startsWith("bytes=")) return null + val rangeSpec = header.removePrefix("bytes=") + val parts = rangeSpec.split("-", limit = 2) + if (parts.size != 2) return null + + val start = parts[0].toLongOrNull() + val end = parts[1].toLongOrNull() + + return when { + start != null && end != null -> Range(start, end) + start != null -> Range(start, contentLength - 1) + end != null -> Range(contentLength - end, contentLength - 1) + else -> null + } + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/fs/StreamUrlDto.kt b/src/main/kotlin/io/skjaere/debridav/fs/StreamUrlDto.kt new file mode 100644 index 00000000..2ae8d9f8 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/fs/StreamUrlDto.kt @@ -0,0 +1,3 @@ +package io.skjaere.debridav.fs + +data class StreamUrlDto(val url: String, val expiresIn: Long) diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/JwtAuthIT.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/JwtAuthIT.kt new file mode 100644 index 00000000..b95d830f --- /dev/null +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/JwtAuthIT.kt @@ -0,0 +1,124 @@ +package io.skjaere.debridav.test.integrationtest + +import io.skjaere.debridav.DebriDavApplication +import io.skjaere.debridav.MiltonConfiguration +import io.skjaere.debridav.config.ConfigOverrideRepository +import io.skjaere.debridav.config.auth.JwtService +import io.skjaere.debridav.test.integrationtest.config.IntegrationTestContextConfiguration +import io.skjaere.debridav.test.integrationtest.config.MockServerTest +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.http.MediaType +import org.springframework.test.web.reactive.server.WebTestClient + +@SpringBootTest( + classes = [DebriDavApplication::class, IntegrationTestContextConfiguration::class, MiltonConfiguration::class], + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = [ + "debridav.debrid-clients=premiumize", + "debridav.auth.enabled=true", + "debridav.auth.jwt-secret=test-secret-key-that-is-at-least-256-bits-long-for-hs256", + "debridav.webdav-username=admin", + "debridav.webdav-password=secret", + "debridav.auth.protect-qbittorrent-api=true", + "debridav.auth.protect-sabnzbd-api=false" + ] +) +@MockServerTest +class JwtAuthIT { + + @Autowired + private lateinit var webTestClient: WebTestClient + + @Autowired + private lateinit var jwtService: JwtService + + @Autowired + private lateinit var configOverrideRepository: ConfigOverrideRepository + + @AfterEach + fun tearDown() { + configOverrideRepository.deleteAll() + } + + @Test + fun `login with valid credentials returns token`() { + webTestClient.post() + .uri("/api/v1/auth/login") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue("""{"username": "admin", "password": "secret"}""") + .exchange() + .expectStatus().isOk + .expectBody() + .jsonPath("$.token").isNotEmpty + } + + @Test + fun `login with invalid credentials returns 401`() { + webTestClient.post() + .uri("/api/v1/auth/login") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue("""{"username": "admin", "password": "wrong"}""") + .exchange() + .expectStatus().isUnauthorized + } + + @Test + fun `config endpoint requires auth when enabled`() { + webTestClient.get() + .uri("/api/v1/config") + .exchange() + .expectStatus().isUnauthorized + } + + @Test + fun `config endpoint accessible with valid token`() { + val token = jwtService.generateToken("admin") + + webTestClient.get() + .uri("/api/v1/config") + .header("Authorization", "Bearer $token") + .exchange() + .expectStatus().isOk + } + + @Test + fun `config endpoint rejects invalid token`() { + webTestClient.get() + .uri("/api/v1/config") + .header("Authorization", "Bearer invalid-token") + .exchange() + .expectStatus().isUnauthorized + } + + @Test + fun `qbittorrent api requires auth when configured`() { + webTestClient.get() + .uri("/api/v2/app/webapiVersion") + .exchange() + .expectStatus().isUnauthorized + } + + @Test + fun `qbittorrent api accessible with valid token`() { + val token = jwtService.generateToken("admin") + + webTestClient.get() + .uri("/api/v2/app/webapiVersion") + .header("Authorization", "Bearer $token") + .exchange() + .expectStatus().isOk + } + + @Test + fun `auth endpoint is always public`() { + webTestClient.post() + .uri("/api/v1/auth/login") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue("""{"username": "admin", "password": "secret"}""") + .exchange() + .expectStatus().isOk + } +} From 076e73be6abc01932e58057f43ab69a12c4bb773 Mon Sep 17 00:00:00 2001 From: Skjaere Date: Sun, 1 Mar 2026 16:30:00 +0100 Subject: [PATCH 04/61] feat(nzb): async import queue + paginated history API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extracts NZB import tracking into a dedicated NzbImportRecord entity so the import pipeline can be observed and paginated. Adds a sortable/searchable /api/v1/imports history endpoint, exposes the current queue status with an ARTICLES_MISSING terminal state, and bumps nzb-streamer through 0.7.0 → 0.7.3. Co-Authored-By: Claude Opus 4.7 (1M context) --- gradle/libs.versions.toml | 2 +- .../config/auth/SecurityConfiguration.kt | 2 + .../DebridavConfigurationProperties.kt | 1 + .../repository/NzbImportRepository.kt | 24 +++++ .../debridav/repository/UsenetRepository.kt | 1 + .../debridav/usenet/NzbImportService.kt | 37 +++++++- .../debridav/usenet/NzbImportTaskData.kt | 3 +- .../skjaere/debridav/usenet/UsenetDownload.kt | 5 +- .../skjaere/debridav/usenet/pgmq/Messages.kt | 3 +- .../usenet/pgmq/PgmqSpringConfiguration.kt | 2 +- .../debridav/usenet/queue/NzbImportRecord.kt | 78 ++++++++++++++++ .../debridav/usenet/queue/QueueItemDto.kt | 30 +++++++ .../usenet/queue/UsenetQueueController.kt | 32 +++++++ .../usenet/queue/UsenetQueueService.kt | 71 +++++++++++++++ .../debridav/usenet/sabnzbd/SabNzbdService.kt | 15 +++- src/main/resources/application.yaml | 2 + .../V16__create_nzb_import_table.sql | 14 +++ .../debridav/test/NzbImportServiceTest.kt | 90 ++++++++++++++++--- 18 files changed, 389 insertions(+), 23 deletions(-) create mode 100644 src/main/kotlin/io/skjaere/debridav/repository/NzbImportRepository.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/usenet/queue/NzbImportRecord.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/usenet/queue/QueueItemDto.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/usenet/queue/UsenetQueueController.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/usenet/queue/UsenetQueueService.kt create mode 100644 src/main/resources/db/migration/V16__create_nzb_import_table.sql diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index d893cbce..72847202 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -19,7 +19,7 @@ logstash-logback = "7.4" hamcrest = "3.0" sardine = "5.13" sentry = "8.33.0" -nzb-streamer = "v0.6.0" +nzb-streamer = "v0.7.3" pgmq-kotlin = "0.1.0" mock-nntp-server = "v0.2.0" jjwt = "0.12.6" diff --git a/src/main/kotlin/io/skjaere/debridav/config/auth/SecurityConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/config/auth/SecurityConfiguration.kt index bfea1f9d..86d99d78 100644 --- a/src/main/kotlin/io/skjaere/debridav/config/auth/SecurityConfiguration.kt +++ b/src/main/kotlin/io/skjaere/debridav/config/auth/SecurityConfiguration.kt @@ -39,8 +39,10 @@ class SecurityConfiguration( // Config API is protected when auth is enabled if (authConfig.enabled) { auth.requestMatchers("/api/v1/config/**").authenticated() + auth.requestMatchers("/api/v1/queue/**").authenticated() } else { auth.requestMatchers("/api/v1/config/**").permitAll() + auth.requestMatchers("/api/v1/queue/**").permitAll() } // Conditionally protect qBittorrent API diff --git a/src/main/kotlin/io/skjaere/debridav/configuration/DebridavConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/configuration/DebridavConfigurationProperties.kt index 3d8c9af3..a5df6e00 100644 --- a/src/main/kotlin/io/skjaere/debridav/configuration/DebridavConfigurationProperties.kt +++ b/src/main/kotlin/io/skjaere/debridav/configuration/DebridavConfigurationProperties.kt @@ -7,6 +7,7 @@ import java.time.Duration @ConfigurationProperties(prefix = "debridav") class DebridavConfigurationProperties { + @ConfigProperty(name = "Root Path", description = "Root path") lateinit var rootPath: String @ConfigProperty(name = "Download Path", description = "Download path") diff --git a/src/main/kotlin/io/skjaere/debridav/repository/NzbImportRepository.kt b/src/main/kotlin/io/skjaere/debridav/repository/NzbImportRepository.kt new file mode 100644 index 00000000..b83d469b --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/repository/NzbImportRepository.kt @@ -0,0 +1,24 @@ +package io.skjaere.debridav.repository + +import io.skjaere.debridav.usenet.queue.NzbImportRecord +import io.skjaere.debridav.usenet.queue.NzbImportStatus +import jakarta.transaction.Transactional +import org.springframework.data.domain.Page +import org.springframework.data.domain.Pageable +import org.springframework.data.jpa.repository.JpaRepository +import org.springframework.data.jpa.repository.Query + +@Transactional +interface NzbImportRepository : JpaRepository { + fun findByStatusInOrderByUpdatedAtDesc(statuses: Collection): List + fun findByStatusInOrderByIdAsc(statuses: Collection): List + + @Query( + "SELECT r FROM NzbImportRecord r WHERE r.status IN :statuses AND LOWER(r.name) LIKE LOWER(CONCAT('%', :search, '%'))" + ) + fun findByStatusInAndNameSearch( + statuses: Collection, + search: String, + pageable: Pageable + ): Page +} diff --git a/src/main/kotlin/io/skjaere/debridav/repository/UsenetRepository.kt b/src/main/kotlin/io/skjaere/debridav/repository/UsenetRepository.kt index 8687e58a..60afd7f6 100644 --- a/src/main/kotlin/io/skjaere/debridav/repository/UsenetRepository.kt +++ b/src/main/kotlin/io/skjaere/debridav/repository/UsenetRepository.kt @@ -1,6 +1,7 @@ package io.skjaere.debridav.repository import io.skjaere.debridav.usenet.UsenetDownload +import io.skjaere.debridav.usenet.UsenetDownloadStatus import jakarta.transaction.Transactional import org.springframework.data.jpa.repository.Modifying import org.springframework.data.jpa.repository.Query diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/NzbImportService.kt b/src/main/kotlin/io/skjaere/debridav/usenet/NzbImportService.kt index f7e535cd..da6cdb7b 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/NzbImportService.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/NzbImportService.kt @@ -5,7 +5,11 @@ import io.skjaere.debridav.configuration.DebridavConfigurationProperties import io.skjaere.debridav.fs.DatabaseFileService import io.skjaere.debridav.fs.NzbContents import io.skjaere.debridav.repository.NzbDocumentRepository +import io.skjaere.debridav.repository.NzbImportRepository import io.skjaere.debridav.repository.UsenetRepository +import io.skjaere.debridav.usenet.queue.NzbImportFileJson +import io.skjaere.debridav.usenet.queue.NzbImportRecord +import io.skjaere.debridav.usenet.queue.NzbImportStatus import io.skjaere.debridav.usenet.nzb.NzbArchiveType import io.skjaere.debridav.usenet.nzb.NzbDocumentEntity import io.skjaere.debridav.usenet.nzb.NzbFileJson @@ -31,18 +35,20 @@ class NzbImportService( private val nzbStreamer: NzbStreamer, private val nzbDocumentRepository: NzbDocumentRepository, private val usenetRepository: UsenetRepository, + private val nzbImportRepository: NzbImportRepository, private val pgmqClient: PgmqClient, private val databaseFileService: DatabaseFileService, private val debridavConfigurationProperties: DebridavConfigurationProperties ) { private val logger = LoggerFactory.getLogger(NzbImportService::class.java) - fun scheduleImport(nzbBytes: ByteArray, usenetDownload: UsenetDownload) { + fun scheduleImport(nzbBytes: ByteArray, usenetDownload: UsenetDownload, nzbImportRecordId: Long) { pgmqClient.send( "nzb_import", NzbImportMessage( nzbBytesBase64 = Base64.getEncoder().encodeToString(nzbBytes), - usenetDownloadId = usenetDownload.id!! + usenetDownloadId = usenetDownload.id!!, + nzbImportRecordId = nzbImportRecordId ) ) } @@ -53,8 +59,14 @@ class NzbImportService( val usenetDownload = usenetRepository.findById(taskData.usenetDownloadId).orElseThrow { IllegalStateException("UsenetDownload not found: ${taskData.usenetDownloadId}") } + val importRecord = nzbImportRepository.findById(taskData.nzbImportRecordId).orElseThrow { + IllegalStateException("NzbImportRecord not found: ${taskData.nzbImportRecordId}") + } try { logger.info("Importing ${usenetDownload.name}") + importRecord.status = NzbImportStatus.IMPORTING + nzbImportRepository.save(importRecord) + val nzbBytes = Base64.getDecoder().decode(taskData.nzbBytesBase64) val prepareResult = runBlocking { nzbStreamer.prepare(nzbBytes) } @@ -65,7 +77,9 @@ class NzbImportService( usenetDownload.name, prepareResult.message ) - usenetDownload.status = UsenetDownloadStatus.FAILED + usenetDownload.status = UsenetDownloadStatus.ARTICLES_MISSING + importRecord.status = NzbImportStatus.ARTICLES_MISSING + importRecord.errorMessage = prepareResult.message return } @@ -77,6 +91,8 @@ class NzbImportService( prepareResult.cause ) usenetDownload.status = UsenetDownloadStatus.FAILED + importRecord.status = NzbImportStatus.FAILED + importRecord.errorMessage = prepareResult.cause.stackTraceToString() return } @@ -87,6 +103,8 @@ class NzbImportService( prepareResult.message ) usenetDownload.status = UsenetDownloadStatus.FAILED + importRecord.status = NzbImportStatus.FAILED + importRecord.errorMessage = prepareResult.message return } @@ -118,14 +136,27 @@ class NzbImportService( }.toMutableList() usenetDownload.status = UsenetDownloadStatus.COMPLETED + importRecord.status = NzbImportStatus.COMPLETED + importRecord.size = savedDocument.streamableFiles.sumOf { it.totalSize } + importRecord.archiveType = savedDocument.archiveType.name + importRecord.files = savedDocument.streamableFiles.map { sf -> + NzbImportFileJson( + path = "${debridavConfigurationProperties.downloadPath}" + + "/${usenetDownload.name}/${sf.path}", + size = sf.totalSize + ) + } logger.info("Imported ${usenetDownload.name}") } } } catch (@Suppress("TooGenericExceptionCaught") e: Exception) { logger.error("Failed to import NZB for download '${usenetDownload.name}'", e) usenetDownload.status = UsenetDownloadStatus.FAILED + importRecord.status = NzbImportStatus.FAILED + importRecord.errorMessage = e.stackTraceToString() } finally { usenetRepository.save(usenetDownload) + nzbImportRepository.save(importRecord) } } diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/NzbImportTaskData.kt b/src/main/kotlin/io/skjaere/debridav/usenet/NzbImportTaskData.kt index 0d87fa89..46f52c91 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/NzbImportTaskData.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/NzbImportTaskData.kt @@ -2,5 +2,6 @@ package io.skjaere.debridav.usenet data class NzbImportTaskData( val nzbBytesBase64: String, - val usenetDownloadId: Long + val usenetDownloadId: Long, + val nzbImportRecordId: Long ) diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/UsenetDownload.kt b/src/main/kotlin/io/skjaere/debridav/usenet/UsenetDownload.kt index c79dc403..7e950697 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/UsenetDownload.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/UsenetDownload.kt @@ -44,9 +44,9 @@ open class UsenetDownload { enum class UsenetDownloadStatus { CREATED, QUEUED, DOWNLOADING, EXTRACTING, COMPLETED, FAILED, VERIFYING, - DELETED, CACHED, REPAIRING, POST_PROCESSING, VALIDATING; + DELETED, CACHED, REPAIRING, POST_PROCESSING, VALIDATING, ARTICLES_MISSING; - fun isCompleted(): Boolean = this == COMPLETED || this == CACHED || this == FAILED + fun isCompleted(): Boolean = this == COMPLETED || this == CACHED || this == FAILED || this == ARTICLES_MISSING } enum class SabnzbdUsenetDownloadStatus { @@ -67,6 +67,7 @@ enum class SabnzbdUsenetDownloadStatus { UsenetDownloadStatus.REPAIRING -> REPAIRING UsenetDownloadStatus.VALIDATING -> VERIFYING UsenetDownloadStatus.POST_PROCESSING -> VERIFYING + UsenetDownloadStatus.ARTICLES_MISSING -> FAILED } } diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/Messages.kt b/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/Messages.kt index 787e08be..9fb83372 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/Messages.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/Messages.kt @@ -2,7 +2,8 @@ package io.skjaere.debridav.usenet.pgmq data class NzbImportMessage( val nzbBytesBase64: String, - val usenetDownloadId: Long + val usenetDownloadId: Long, + val nzbImportRecordId: Long ) data class NzbHealthCheckMessage( diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqSpringConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqSpringConfiguration.kt index 84533ad2..8fa4feb8 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqSpringConfiguration.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqSpringConfiguration.kt @@ -76,7 +76,7 @@ class PgmqSpringConfiguration { pollInterval = props.importPollInterval ) { msg -> nzbImportService.executeImport( - NzbImportTaskData(msg.nzbBytesBase64, msg.usenetDownloadId) + NzbImportTaskData(msg.nzbBytesBase64, msg.usenetDownloadId, msg.nzbImportRecordId) ) } diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/queue/NzbImportRecord.kt b/src/main/kotlin/io/skjaere/debridav/usenet/queue/NzbImportRecord.kt new file mode 100644 index 00000000..f6687632 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/usenet/queue/NzbImportRecord.kt @@ -0,0 +1,78 @@ +package io.skjaere.debridav.usenet.queue + +import io.hypersistence.utils.hibernate.type.json.JsonBinaryType +import jakarta.persistence.Column +import jakarta.persistence.Entity +import jakarta.persistence.EnumType +import jakarta.persistence.Enumerated +import jakarta.persistence.GeneratedValue +import jakarta.persistence.GenerationType +import jakarta.persistence.Id +import jakarta.persistence.PrePersist +import jakarta.persistence.PreUpdate +import jakarta.persistence.Table +import org.hibernate.annotations.Type +import java.time.Instant + +@Entity +@Table(name = "nzb_import") +open class NzbImportRecord { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + open var id: Long? = null + + @Column(name = "usenet_download_id") + open var usenetDownloadId: Long? = null + + @Column(nullable = false) + open var name: String = "" + + open var category: String? = null + + @Enumerated(EnumType.STRING) + @Column(nullable = false, length = 30) + open var status: NzbImportStatus = NzbImportStatus.QUEUED + + @Column(name = "archive_type", length = 30) + open var archiveType: String? = null + + @Column(name = "error_message", columnDefinition = "TEXT") + open var errorMessage: String? = null + + @Type(JsonBinaryType::class) + @Column(name = "files", columnDefinition = "jsonb") + open var files: List? = null + + open var size: Long? = null + + @Column(name = "created_at") + open var createdAt: Instant? = null + + @Column(name = "updated_at") + open var updatedAt: Instant? = null + + @PrePersist + fun onPrePersist() { + val now = Instant.now() + createdAt = now + updatedAt = now + } + + @PreUpdate + fun onPreUpdate() { + updatedAt = Instant.now() + } +} + +enum class NzbImportStatus { + QUEUED, + IMPORTING, + COMPLETED, + FAILED, + ARTICLES_MISSING +} + +data class NzbImportFileJson( + val path: String, + val size: Long +) : java.io.Serializable diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/queue/QueueItemDto.kt b/src/main/kotlin/io/skjaere/debridav/usenet/queue/QueueItemDto.kt new file mode 100644 index 00000000..3bd33a06 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/usenet/queue/QueueItemDto.kt @@ -0,0 +1,30 @@ +package io.skjaere.debridav.usenet.queue + +import java.time.Instant + +data class QueueItemDto( + val id: Long, + val name: String, + val status: String, + val size: Long?, + val errorMessage: String?, + val updatedAt: Instant?, + val createdAt: Instant?, + val archiveType: String? = null, + val files: List? = null +) + +data class QueueStatusResponse( + val processing: List, + val pending: List, + val history: List +) + +data class HistoryPageResponse( + val content: List, + val page: Int, + val size: Int, + val totalElements: Long, + val totalPages: Int, + val last: Boolean +) diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/queue/UsenetQueueController.kt b/src/main/kotlin/io/skjaere/debridav/usenet/queue/UsenetQueueController.kt new file mode 100644 index 00000000..8f1bf3bd --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/usenet/queue/UsenetQueueController.kt @@ -0,0 +1,32 @@ +package io.skjaere.debridav.usenet.queue + +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RequestParam +import org.springframework.web.bind.annotation.RestController + +@RestController +@RequestMapping("/api/v1/queue") +class UsenetQueueController(private val queueService: UsenetQueueService) { + @GetMapping + fun getQueueStatus(): ResponseEntity = + ResponseEntity.ok(queueService.getQueueStatus()) + + @GetMapping("/history") + fun getHistory( + @RequestParam(defaultValue = "0") page: Int, + @RequestParam(defaultValue = "20") size: Int, + @RequestParam(defaultValue = "") search: String, + @RequestParam(defaultValue = "updatedAt") sort: String, + @RequestParam(defaultValue = "desc") direction: String + ): ResponseEntity = + ResponseEntity.ok(queueService.getHistory(page, size, search, sort, direction)) + + @GetMapping("/{id}/files") + fun getItemFiles(@PathVariable id: Long): ResponseEntity> { + val files = queueService.resolveCurrentFilePaths(id) + ?: return ResponseEntity.notFound().build() + return ResponseEntity.ok(files) + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/queue/UsenetQueueService.kt b/src/main/kotlin/io/skjaere/debridav/usenet/queue/UsenetQueueService.kt new file mode 100644 index 00000000..841a7e1c --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/usenet/queue/UsenetQueueService.kt @@ -0,0 +1,71 @@ +package io.skjaere.debridav.usenet.queue + +import io.skjaere.debridav.repository.NzbImportRepository +import org.springframework.data.domain.PageRequest +import org.springframework.data.domain.Sort +import org.springframework.stereotype.Service + +@Service +class UsenetQueueService(private val nzbImportRepository: NzbImportRepository) { + + companion object { + val PROCESSING_STATUSES = listOf( + NzbImportStatus.IMPORTING + ) + + val PENDING_STATUSES = listOf( + NzbImportStatus.QUEUED + ) + + val HISTORY_STATUSES = listOf( + NzbImportStatus.COMPLETED, + NzbImportStatus.FAILED, + NzbImportStatus.ARTICLES_MISSING + ) + + private val ALLOWED_SORT_FIELDS = setOf("updatedAt", "name") + } + + fun getQueueStatus(): QueueStatusResponse { + val processing = nzbImportRepository.findByStatusInOrderByUpdatedAtDesc(PROCESSING_STATUSES) + .map { it.toDto() } + val pending = nzbImportRepository.findByStatusInOrderByIdAsc(PENDING_STATUSES) + .map { it.toDto() } + + return QueueStatusResponse( + processing = processing, + pending = pending, + history = emptyList() + ) + } + + fun getHistory(page: Int, size: Int, search: String, sort: String, direction: String): HistoryPageResponse { + val sortField = if (sort in ALLOWED_SORT_FIELDS) sort else "updatedAt" + val sortDir = if (direction.equals("asc", ignoreCase = true)) Sort.Direction.ASC else Sort.Direction.DESC + val pageResult = nzbImportRepository.findByStatusInAndNameSearch( + HISTORY_STATUSES, + search, + PageRequest.of(page, size, Sort.by(sortDir, sortField)) + ) + return HistoryPageResponse( + content = pageResult.content.map { it.toDto() }, + page = pageResult.number, + size = pageResult.size, + totalElements = pageResult.totalElements, + totalPages = pageResult.totalPages, + last = pageResult.isLast + ) + } + + private fun NzbImportRecord.toDto() = QueueItemDto( + id = id!!, + name = name, + status = status.name, + size = size, + errorMessage = errorMessage, + updatedAt = updatedAt, + createdAt = createdAt, + archiveType = archiveType, + files = files + ) +} diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/sabnzbd/SabNzbdService.kt b/src/main/kotlin/io/skjaere/debridav/usenet/sabnzbd/SabNzbdService.kt index fbeddd2b..5795e525 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/sabnzbd/SabNzbdService.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/sabnzbd/SabNzbdService.kt @@ -7,10 +7,13 @@ import io.skjaere.debridav.debrid.UsenetRelease import io.skjaere.debridav.fs.DatabaseFileService import io.skjaere.debridav.fs.DebridFileContents import io.skjaere.debridav.fs.RemotelyCachedEntity +import io.skjaere.debridav.repository.NzbImportRepository import io.skjaere.debridav.repository.UsenetRepository import io.skjaere.debridav.usenet.NzbImportService import io.skjaere.debridav.usenet.UsenetDownload import io.skjaere.debridav.usenet.UsenetDownloadStatus +import io.skjaere.debridav.usenet.queue.NzbImportRecord +import io.skjaere.debridav.usenet.queue.NzbImportStatus import io.skjaere.debridav.usenet.sabnzbd.model.HistorySlot import io.skjaere.debridav.usenet.sabnzbd.model.ListResponseDownloadSlot import io.skjaere.debridav.usenet.sabnzbd.model.Queue @@ -47,7 +50,8 @@ class SabNzbdService( private val usenetConversionService: ConversionService, private val categoryService: CategoryService, private val resourceLoader: ResourceLoader, - private val nzbImportService: NzbImportService? + private val nzbImportService: NzbImportService?, + private val nzbImportRepository: NzbImportRepository? ) { private val logger = LoggerFactory.getLogger(SabNzbdService::class.java) @@ -60,7 +64,14 @@ class SabNzbdService( if (nzbImportService != null) { val usenetDownload = createQueuedUsenetDownload(releaseName, hash, request.cat!!) - nzbImportService.scheduleImport(nzbBytes, usenetDownload) + val importRecord = NzbImportRecord().apply { + usenetDownloadId = usenetDownload.id + name = releaseName + category = request.cat + status = NzbImportStatus.QUEUED + } + val savedRecord = nzbImportRepository!!.save(importRecord) + nzbImportService.scheduleImport(nzbBytes, usenetDownload, savedRecord.id!!) return usenetDownload } diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index d1344ab1..73b4947e 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -23,6 +23,8 @@ spring: max-lifetime: 180000000 # five hours idle-timeout: 0 maximum-pool-size: 5 + flyway: + out-of-order: true logging: level: diff --git a/src/main/resources/db/migration/V16__create_nzb_import_table.sql b/src/main/resources/db/migration/V16__create_nzb_import_table.sql new file mode 100644 index 00000000..1f648a1d --- /dev/null +++ b/src/main/resources/db/migration/V16__create_nzb_import_table.sql @@ -0,0 +1,14 @@ +CREATE TABLE nzb_import ( + id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + usenet_download_id BIGINT REFERENCES usenet_download(id) ON DELETE SET NULL, + name VARCHAR(255) NOT NULL, + category VARCHAR(255), + status VARCHAR(30) NOT NULL DEFAULT 'QUEUED', + archive_type VARCHAR(30), + error_message TEXT, + files JSONB, + size BIGINT, + created_at TIMESTAMP WITH TIME ZONE, + updated_at TIMESTAMP WITH TIME ZONE +); +CREATE INDEX idx_nzb_import_status ON nzb_import (status); diff --git a/src/test/kotlin/io/skjaere/debridav/test/NzbImportServiceTest.kt b/src/test/kotlin/io/skjaere/debridav/test/NzbImportServiceTest.kt index 98dc91cc..13e0941a 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/NzbImportServiceTest.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/NzbImportServiceTest.kt @@ -12,13 +12,17 @@ import io.skjaere.debridav.fs.DatabaseFileService import io.skjaere.debridav.fs.NzbContents import io.skjaere.debridav.fs.RemotelyCachedEntity import io.skjaere.debridav.repository.NzbDocumentRepository +import io.skjaere.debridav.repository.NzbImportRepository import io.skjaere.debridav.repository.UsenetRepository import io.skjaere.debridav.usenet.NzbImportService import io.skjaere.debridav.usenet.NzbImportTaskData import io.skjaere.debridav.usenet.UsenetDownload import io.skjaere.debridav.usenet.UsenetDownloadStatus +import io.skjaere.debridav.usenet.nzb.NzbArchiveType import io.skjaere.debridav.usenet.nzb.NzbDocumentEntity import io.skjaere.debridav.usenet.nzb.StreamableFileJson +import io.skjaere.debridav.usenet.queue.NzbImportRecord +import io.skjaere.debridav.usenet.queue.NzbImportStatus import io.skjaere.nntp.ArticleNotFoundException import io.skjaere.nntp.NntpConnectionException import io.skjaere.nntp.YencHeaders @@ -41,6 +45,7 @@ class NzbImportServiceTest { private val nzbStreamer = mockk() private val nzbDocumentRepository = mockk() private val usenetRepository = mockk() + private val nzbImportRepository = mockk() private val pgmqClient = mockk() private val databaseFileService = mockk() private val config = DebridavConfigurationProperties().apply { @@ -65,7 +70,7 @@ class NzbImportServiceTest { private val underTest = NzbImportService( nzbStreamer, nzbDocumentRepository, usenetRepository, - pgmqClient, databaseFileService, config + nzbImportRepository, pgmqClient, databaseFileService, config ) private val nzbBytes = "test".toByteArray() @@ -80,11 +85,21 @@ class NzbImportServiceTest { return download } + private fun createImportRecord(id: Long = 100L): NzbImportRecord { + val record = NzbImportRecord() + record.id = id + record.name = "test-release" + record.status = NzbImportStatus.QUEUED + return record + } + @Test fun `executeImport sets COMPLETED on PrepareResult Success`() { // given val download = createUsenetDownload() + val importRecord = createImportRecord() every { usenetRepository.findById(1L) } returns Optional.of(download) + every { nzbImportRepository.findById(100L) } returns Optional.of(importRecord) val nzbFile = NzbFile( poster = "test", date = 0, subject = "test", @@ -113,6 +128,7 @@ class NzbImportServiceTest { val savedDoc = NzbDocumentEntity().apply { id = 10L + archiveType = NzbArchiveType.RAR streamableFiles = listOf( StreamableFileJson( path = "video.mkv", @@ -132,20 +148,28 @@ class NzbImportServiceTest { val savedSlot = slot() every { usenetRepository.save(capture(savedSlot)) } answers { savedSlot.captured } + val importSlot = slot() + every { nzbImportRepository.save(capture(importSlot)) } answers { importSlot.captured } + // when - underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L)) + underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L, 100L)) // then assertEquals(UsenetDownloadStatus.COMPLETED, savedSlot.captured.status) + assertEquals(NzbImportStatus.COMPLETED, importSlot.captured.status) + assertEquals(4000L, importSlot.captured.size) + assertEquals("RAR", importSlot.captured.archiveType) verify(exactly = 1) { nzbDocumentRepository.save(any()) } verify(exactly = 1) { databaseFileService.createDebridFile(any(), eq("abc123"), any()) } } @Test - fun `executeImport sets FAILED on PrepareResult MissingArticles`() { + fun `executeImport sets ARTICLES_MISSING on PrepareResult MissingArticles`() { // given val download = createUsenetDownload() + val importRecord = createImportRecord() every { usenetRepository.findById(1L) } returns Optional.of(download) + every { nzbImportRepository.findById(100L) } returns Optional.of(importRecord) coEvery { nzbStreamer.prepare(any()) } returns PrepareResult.MissingArticles( "Article not found: 430", @@ -155,11 +179,16 @@ class NzbImportServiceTest { val savedSlot = slot() every { usenetRepository.save(capture(savedSlot)) } answers { savedSlot.captured } + val importSlot = slot() + every { nzbImportRepository.save(capture(importSlot)) } answers { importSlot.captured } + // when - underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L)) + underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L, 100L)) // then - assertEquals(UsenetDownloadStatus.FAILED, savedSlot.captured.status) + assertEquals(UsenetDownloadStatus.ARTICLES_MISSING, savedSlot.captured.status) + assertEquals(NzbImportStatus.ARTICLES_MISSING, importSlot.captured.status) + assertEquals("Article not found: 430", importSlot.captured.errorMessage) verify(exactly = 0) { nzbDocumentRepository.save(any()) } } @@ -167,7 +196,9 @@ class NzbImportServiceTest { fun `executeImport sets FAILED on PrepareResult Failure`() { // given val download = createUsenetDownload() + val importRecord = createImportRecord() every { usenetRepository.findById(1L) } returns Optional.of(download) + every { nzbImportRepository.findById(100L) } returns Optional.of(importRecord) coEvery { nzbStreamer.prepare(any()) } returns PrepareResult.Failure( "Connection refused", @@ -177,11 +208,15 @@ class NzbImportServiceTest { val savedSlot = slot() every { usenetRepository.save(capture(savedSlot)) } answers { savedSlot.captured } + val importSlot = slot() + every { nzbImportRepository.save(capture(importSlot)) } answers { importSlot.captured } + // when - underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L)) + underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L, 100L)) // then assertEquals(UsenetDownloadStatus.FAILED, savedSlot.captured.status) + assertEquals(NzbImportStatus.FAILED, importSlot.captured.status) verify(exactly = 0) { nzbDocumentRepository.save(any()) } } @@ -189,7 +224,9 @@ class NzbImportServiceTest { fun `executeImport sets FAILED on PrepareResult UnsupportedArchive`() { // given val download = createUsenetDownload() + val importRecord = createImportRecord() every { usenetRepository.findById(1L) } returns Optional.of(download) + every { nzbImportRepository.findById(100L) } returns Optional.of(importRecord) coEvery { nzbStreamer.prepare(any()) } returns PrepareResult.UnsupportedArchive( "Unable to detect archive type from filenames or byte signatures", @@ -199,11 +236,15 @@ class NzbImportServiceTest { val savedSlot = slot() every { usenetRepository.save(capture(savedSlot)) } answers { savedSlot.captured } + val importSlot = slot() + every { nzbImportRepository.save(capture(importSlot)) } answers { importSlot.captured } + // when - underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L)) + underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L, 100L)) // then assertEquals(UsenetDownloadStatus.FAILED, savedSlot.captured.status) + assertEquals(NzbImportStatus.FAILED, importSlot.captured.status) verify(exactly = 0) { nzbDocumentRepository.save(any()) } } @@ -211,18 +252,24 @@ class NzbImportServiceTest { fun `executeImport sets FAILED on unexpected exception`() { // given val download = createUsenetDownload() + val importRecord = createImportRecord() every { usenetRepository.findById(1L) } returns Optional.of(download) + every { nzbImportRepository.findById(100L) } returns Optional.of(importRecord) coEvery { nzbStreamer.prepare(any()) } throws RuntimeException("unexpected error") val savedSlot = slot() every { usenetRepository.save(capture(savedSlot)) } answers { savedSlot.captured } + val importSlot = slot() + every { nzbImportRepository.save(capture(importSlot)) } answers { importSlot.captured } + // when - underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L)) + underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L, 100L)) // then assertEquals(UsenetDownloadStatus.FAILED, savedSlot.captured.status) + assertEquals(NzbImportStatus.FAILED, importSlot.captured.status) } @Test @@ -232,26 +279,45 @@ class NzbImportServiceTest { // when/then kotlin.test.assertFailsWith { - underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 999L)) + underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 999L, 100L)) + } + } + + @Test + fun `executeImport throws when NzbImportRecord not found`() { + // given + val download = createUsenetDownload() + every { usenetRepository.findById(1L) } returns Optional.of(download) + every { nzbImportRepository.findById(999L) } returns Optional.empty() + + // when/then + kotlin.test.assertFailsWith { + underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L, 999L)) } } @Test - fun `executeImport always saves UsenetDownload in finally block`() { + fun `executeImport always saves both records in finally block`() { // given val download = createUsenetDownload() + val importRecord = createImportRecord() every { usenetRepository.findById(1L) } returns Optional.of(download) + every { nzbImportRepository.findById(100L) } returns Optional.of(importRecord) coEvery { nzbStreamer.prepare(any()) } returns PrepareResult.MissingArticles( "missing", ArticleNotFoundException("missing") ) every { usenetRepository.save(any()) } answers { firstArg() } + every { nzbImportRepository.save(any()) } answers { firstArg() } // when - underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L)) + underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L, 100L)) - // then - save is called exactly once (in the finally block) + // then - save is called: once for IMPORTING status + once in finally block = 2 + verify(exactly = 2) { nzbImportRepository.save(any()) } verify(exactly = 1) { usenetRepository.save(any()) } + assertEquals(UsenetDownloadStatus.ARTICLES_MISSING, download.status) + assertEquals(NzbImportStatus.ARTICLES_MISSING, importRecord.status) } } From c26c955285f47a1e23285bb717c3a7eb85330605 Mon Sep 17 00:00:00 2001 From: Skjaere Date: Sat, 7 Mar 2026 11:00:00 +0100 Subject: [PATCH 05/61] fix(reliability): glitchtip-reported bugs + Premiumize 200-with-error Bundles several small fixes surfaced via glitchtip / observability: - Premiumize returning errors inside 200 responses now parses cleanly - Wrapped ClientAbortException no longer escalates to ERROR - ObjectOptimisticLockingFailureException on concurrent updates - DataIntegrityViolationException on duplicate-key conflicts - EOFException on abruptly closed streams Plus housekeeping: - Downgrade ktor from 3.5.0-eap-1584 to stable 3.4.1 - Extract FileController.detail() into focused private methods - Suppress noisy detekt warnings across the codebase Co-Authored-By: Claude Opus 4.7 (1M context) --- gradle/libs.versions.toml | 2 +- mcp-config.json | 13 ++ .../arrs/RadarrConfigurationProperties.kt | 3 +- .../arrs/SonarrConfigurationProperties.kt | 3 +- .../debridav/arrs/client/RadarrApiClient.kt | 1 + .../debridav/arrs/client/SonarrApiClient.kt | 1 + .../config/ConfigOverrideRepository.kt | 4 +- .../debridav/config/ConfigOverrideService.kt | 24 ++++ .../debridav/config/ConfigPropertyRegistry.kt | 1 + .../skjaere/debridav/config/NntpPoolTester.kt | 1 + .../auth/AuthConfigurationProperties.kt | 1 + .../debridav/config/auth/AuthController.kt | 1 + .../config/auth/JwtAuthenticationFilter.kt | 1 + .../debridav/config/auth/JwtService.kt | 2 + .../DbConfigurationProperties.kt | 1 + .../debrid/client/easynews/EasynewsClient.kt | 1 + .../client/premiumize/PremiumizeClient.kt | 12 +- .../client/realdebrid/RealDebridClient.kt | 1 + .../debrid/client/torbox/TorBoxClient.kt | 1 + .../io/skjaere/debridav/fs/FileController.kt | 99 ++++++++------- .../skjaere/debridav/fs/StreamController.kt | 2 + .../debridav/stream/StreamingService.kt | 7 ++ .../debridav/usenet/NzbImportService.kt | 118 +++++++++++++----- .../debridav/test/NzbImportServiceTest.kt | 78 ++++++++++-- .../test/integrationtest/ConfigApiIT.kt | 28 +++++ .../test/integrationtest/StreamingEofIT.kt | 115 +++++++++++++++++ .../config/ContentStubbingService.kt | 27 ++++ .../config/TestContextInitializer.kt | 21 ++-- 28 files changed, 464 insertions(+), 105 deletions(-) create mode 100644 mcp-config.json create mode 100644 src/test/kotlin/io/skjaere/debridav/test/integrationtest/StreamingEofIT.kt diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 72847202..53d02932 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -2,7 +2,7 @@ kotlin = "2.3.10" kotlinx-coroutines = "1.10.2" kotlinx-serialization = "1.10.0" -ktor = "3.5.0-eap-1584" +ktor = "3.4.1" spring-boot = "4.0.3" spring-cloud = "2025.1.1" mockk = "1.14.9" diff --git a/mcp-config.json b/mcp-config.json new file mode 100644 index 00000000..1a579708 --- /dev/null +++ b/mcp-config.json @@ -0,0 +1,13 @@ +{ + "mcpServers": { + "jvm-debugger": { + "command": "java", + "args": [ + "--add-modules", + "jdk.jdi", + "-jar", + "/opt/judi/judi.jar" + ] + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/io/skjaere/debridav/arrs/RadarrConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/arrs/RadarrConfigurationProperties.kt index 1903db47..05d06bbe 100644 --- a/src/main/kotlin/io/skjaere/debridav/arrs/RadarrConfigurationProperties.kt +++ b/src/main/kotlin/io/skjaere/debridav/arrs/RadarrConfigurationProperties.kt @@ -10,7 +10,8 @@ class RadarrConfigurationProperties : ArrConfiguration { @ConfigProperty(name = "Host", description = "Radarr host") override var host: String = "" @ConfigProperty(name = "Port", description = "Radarr port") - override val port: Int = 7878, + @Suppress("MagicNumber") + override var port: Int = 7878 @ConfigProperty(name = "API Base Path", description = "Radarr API base path", advanced = true) override val apiBasePath: String = "/api/v3", @ConfigProperty(name = "API Key", description = "Radarr API key", sensitive = true) diff --git a/src/main/kotlin/io/skjaere/debridav/arrs/SonarrConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/arrs/SonarrConfigurationProperties.kt index 8420501b..cb2531b3 100644 --- a/src/main/kotlin/io/skjaere/debridav/arrs/SonarrConfigurationProperties.kt +++ b/src/main/kotlin/io/skjaere/debridav/arrs/SonarrConfigurationProperties.kt @@ -10,7 +10,8 @@ class SonarrConfigurationProperties : ArrConfiguration { @ConfigProperty(name = "Host", description = "Sonarr host") override var host: String = "" @ConfigProperty(name = "Port", description = "Sonarr port") - override val port: Int = 8989, + @Suppress("MagicNumber") + override var port: Int = 8989 @ConfigProperty(name = "API Base Path", description = "Sonarr API base path", advanced = true) override val apiBasePath: String = "/api/v3", @ConfigProperty(name = "API Key", description = "Sonarr API key", sensitive = true) diff --git a/src/main/kotlin/io/skjaere/debridav/arrs/client/RadarrApiClient.kt b/src/main/kotlin/io/skjaere/debridav/arrs/client/RadarrApiClient.kt index 728215ec..ed202b5d 100644 --- a/src/main/kotlin/io/skjaere/debridav/arrs/client/RadarrApiClient.kt +++ b/src/main/kotlin/io/skjaere/debridav/arrs/client/RadarrApiClient.kt @@ -79,6 +79,7 @@ class RadarrApiClient( override val configurationClass: KClass<*> = RadarrConfigurationProperties::class override val label: String = "Radarr" + @Suppress("TooGenericExceptionCaught") override suspend fun test(overrides: Map): TestResult = try { val host = overrides["radarr.host"] ?: radarrConfigurationProperties.host val port = overrides["radarr.port"]?.toIntOrNull() ?: radarrConfigurationProperties.port diff --git a/src/main/kotlin/io/skjaere/debridav/arrs/client/SonarrApiClient.kt b/src/main/kotlin/io/skjaere/debridav/arrs/client/SonarrApiClient.kt index 7558294c..68be1137 100644 --- a/src/main/kotlin/io/skjaere/debridav/arrs/client/SonarrApiClient.kt +++ b/src/main/kotlin/io/skjaere/debridav/arrs/client/SonarrApiClient.kt @@ -84,6 +84,7 @@ class SonarrApiClient( override val configurationClass: KClass<*> = SonarrConfigurationProperties::class override val label: String = "Sonarr" + @Suppress("TooGenericExceptionCaught") override suspend fun test(overrides: Map): TestResult = try { val host = overrides["sonarr.host"] ?: sonarrConfigurationProperties.host val port = overrides["sonarr.port"]?.toIntOrNull() ?: sonarrConfigurationProperties.port diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideRepository.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideRepository.kt index 1a4e57dc..aafa64e7 100644 --- a/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideRepository.kt +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideRepository.kt @@ -1,8 +1,8 @@ package io.skjaere.debridav.config -import org.springframework.data.repository.CrudRepository +import org.springframework.data.jpa.repository.JpaRepository -interface ConfigOverrideRepository : CrudRepository { +interface ConfigOverrideRepository : JpaRepository { fun findByPropKey(key: String): ConfigOverride? fun findAllByPropKeyIn(keys: Collection): List fun findAllByPropKeyStartingWith(prefix: String): List diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideService.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideService.kt index 55a9bb97..08dfde0f 100644 --- a/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideService.kt +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideService.kt @@ -103,6 +103,29 @@ class ConfigOverrideService( return getEffective(key) } + private fun refreshEnvironment() { + val propertySource = dbPropertySourceInitializer.getOrCreatePropertySource() + val overrides = repository.findAll().associate { it.propKey to (it.propValue ?: "") } + propertySource.replaceAll(overrides) + contextRefresher.refreshEnvironment() + logger.info("Refreshed environment with {} database override(s)", overrides.size) + } + + @Suppress("ReturnCount") + private fun getDefaultValue(key: String): String? { + for (source in environment.propertySources) { + if (source.name == DatabasePropertySource.NAME) continue + if (source is EnumerablePropertySource<*>) { + val value = source.getProperty(key) + if (value != null) return value.toString() + } else { + val value = source.getProperty(key) + if (value != null) return value.toString() + } + } + return null + } + fun getNntpPools(): List { val overrides = repository.findAllByPropKeyStartingWith(POOL_PREFIX) val pools = if (overrides.isEmpty()) { @@ -116,6 +139,7 @@ class ConfigOverrideService( @Transactional fun saveNntpPools(pools: List) { repository.deleteAllByPropKeyStartingWith(POOL_PREFIX) + repository.flush() val now = Instant.now() pools.forEachIndexed { i, pool -> val entries = mapOf( diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigPropertyRegistry.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigPropertyRegistry.kt index 4442b389..5d0f5354 100644 --- a/src/main/kotlin/io/skjaere/debridav/config/ConfigPropertyRegistry.kt +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigPropertyRegistry.kt @@ -28,6 +28,7 @@ class ConfigPropertyRegistry( val properties: Map get() = _properties + @Suppress("LoopWithTooManyJumpStatements") @PostConstruct fun init() { val beanNames = applicationContext.getBeanNamesForAnnotation(ConfigurationProperties::class.java) diff --git a/src/main/kotlin/io/skjaere/debridav/config/NntpPoolTester.kt b/src/main/kotlin/io/skjaere/debridav/config/NntpPoolTester.kt index d2f60ef2..0eb53987 100644 --- a/src/main/kotlin/io/skjaere/debridav/config/NntpPoolTester.kt +++ b/src/main/kotlin/io/skjaere/debridav/config/NntpPoolTester.kt @@ -9,6 +9,7 @@ import org.springframework.stereotype.Service @Service class NntpPoolTester { + @Suppress("TooGenericExceptionCaught", "ReturnCount") suspend fun test(pool: NntpPoolDto): TestResult { val selectorManager = SelectorManager(Dispatchers.IO) try { diff --git a/src/main/kotlin/io/skjaere/debridav/config/auth/AuthConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/config/auth/AuthConfigurationProperties.kt index eb2c190d..98a0a358 100644 --- a/src/main/kotlin/io/skjaere/debridav/config/auth/AuthConfigurationProperties.kt +++ b/src/main/kotlin/io/skjaere/debridav/config/auth/AuthConfigurationProperties.kt @@ -6,6 +6,7 @@ import org.springframework.boot.context.properties.ConfigurationProperties class AuthConfigurationProperties { var enabled: Boolean = false var jwtSecret: String = "" + @Suppress("MagicNumber") var tokenExpirationHours: Long = 24 var protectQbittorrentApi: Boolean = false var protectSabnzbdApi: Boolean = false diff --git a/src/main/kotlin/io/skjaere/debridav/config/auth/AuthController.kt b/src/main/kotlin/io/skjaere/debridav/config/auth/AuthController.kt index a3fa1339..d1bc4fd7 100644 --- a/src/main/kotlin/io/skjaere/debridav/config/auth/AuthController.kt +++ b/src/main/kotlin/io/skjaere/debridav/config/auth/AuthController.kt @@ -14,6 +14,7 @@ class AuthController( private val jwtService: JwtService, private val debridavConfig: DebridavConfigurationProperties ) { + @Suppress("ReturnCount") @PostMapping("/login") fun login(@RequestBody request: LoginRequest): ResponseEntity { val expectedUsername = debridavConfig.webdavUsername diff --git a/src/main/kotlin/io/skjaere/debridav/config/auth/JwtAuthenticationFilter.kt b/src/main/kotlin/io/skjaere/debridav/config/auth/JwtAuthenticationFilter.kt index e241b64e..1e6bc6e6 100644 --- a/src/main/kotlin/io/skjaere/debridav/config/auth/JwtAuthenticationFilter.kt +++ b/src/main/kotlin/io/skjaere/debridav/config/auth/JwtAuthenticationFilter.kt @@ -20,6 +20,7 @@ class JwtAuthenticationFilter( ) { val authHeader = request.getHeader("Authorization") if (authHeader != null && authHeader.startsWith("Bearer ")) { + @Suppress("MagicNumber") val token = authHeader.substring(7) val username = jwtService.validateTokenAndGetUsername(token) if (username != null && SecurityContextHolder.getContext().authentication == null) { diff --git a/src/main/kotlin/io/skjaere/debridav/config/auth/JwtService.kt b/src/main/kotlin/io/skjaere/debridav/config/auth/JwtService.kt index 096c3573..b4b1c962 100644 --- a/src/main/kotlin/io/skjaere/debridav/config/auth/JwtService.kt +++ b/src/main/kotlin/io/skjaere/debridav/config/auth/JwtService.kt @@ -15,6 +15,7 @@ class JwtService( Keys.hmacShaKeyFor(authConfig.jwtSecret.toByteArray()) } + @Suppress("MagicNumber") fun generateToken(username: String): String { val now = Date() val expiration = Date(now.time + authConfig.tokenExpirationHours * 3600 * 1000) @@ -40,6 +41,7 @@ class JwtService( null } + @Suppress("MagicNumber") fun generateStreamToken(path: String): String { val now = Date() val expiration = Date(now.time + STREAM_TOKEN_EXPIRY_SECONDS * 1000) diff --git a/src/main/kotlin/io/skjaere/debridav/configuration/DbConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/configuration/DbConfigurationProperties.kt index fd0229f4..d0f66fd0 100644 --- a/src/main/kotlin/io/skjaere/debridav/configuration/DbConfigurationProperties.kt +++ b/src/main/kotlin/io/skjaere/debridav/configuration/DbConfigurationProperties.kt @@ -5,6 +5,7 @@ import org.springframework.boot.context.properties.ConfigurationProperties @ConfigurationProperties(prefix = "debridav.db") class DbConfigurationProperties { var host: String = "localhost" + @Suppress("MagicNumber") var port: Int = 5432 var databaseName: String = "debridav" } diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/easynews/EasynewsClient.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/easynews/EasynewsClient.kt index 86ae3b6d..16b0dbc3 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/easynews/EasynewsClient.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/easynews/EasynewsClient.kt @@ -392,6 +392,7 @@ class EasynewsClient( override val configurationClass: KClass<*> = EasynewsConfigurationProperties::class override val label: String = "Easynews" + @Suppress("TooGenericExceptionCaught") override suspend fun test(overrides: Map): TestResult = try { val apiBaseUrl = overrides["easynews.api-base-url"] ?: easynewsConfiguration.apiBaseUrl val username = overrides["easynews.username"] ?: easynewsConfiguration.username diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/PremiumizeClient.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/PremiumizeClient.kt index cd3475b2..10971bb4 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/PremiumizeClient.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/premiumize/PremiumizeClient.kt @@ -23,13 +23,15 @@ import io.skjaere.debridav.debrid.client.StreamableLinkPreparable import io.skjaere.debridav.debrid.client.premiumize.model.CacheCheckResponse import io.skjaere.debridav.debrid.client.premiumize.model.SuccessfulDirectDownloadResponse import io.skjaere.debridav.fs.CachedFile +import kotlin.reflect.KClass import kotlinx.serialization.Serializable +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.jsonPrimitive import org.slf4j.Logger import org.slf4j.LoggerFactory import org.springframework.stereotype.Component import java.time.Clock import java.time.Instant -import kotlin.reflect.KClass @Serializable private data class PremiumizeAccountResponse( @@ -104,8 +106,11 @@ class PremiumizeClient( set(HttpHeaders.Accept, "application/json") } } - - if (resp.status != HttpStatusCode.OK) { + if (!resp.status.isSuccess()) { + throwDebridProviderException(resp, "/transfer/directdl") + } + val json: JsonObject = resp.body() + if (json["status"]?.jsonPrimitive?.content == "error") { throwDebridProviderException(resp, "/transfer/directdl") } return resp.body() @@ -132,6 +137,7 @@ class PremiumizeClient( override val configurationClass: KClass<*> = PremiumizeConfigurationProperties::class override val label: String = "Premiumize" + @Suppress("TooGenericExceptionCaught") override suspend fun test(overrides: Map): TestResult = try { val baseUrl = overrides["premiumize.base-url"] ?: premiumizeConfiguration.baseUrl val apiKey = overrides["premiumize.api-key"] ?: premiumizeConfiguration.apiKey diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridClient.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridClient.kt index 256e2a4c..e1a5223d 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridClient.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridClient.kt @@ -411,6 +411,7 @@ class RealDebridClient( override val configurationClass: KClass<*> = RealDebridConfigurationProperties::class override val label: String = "Real-Debrid" + @Suppress("TooGenericExceptionCaught") override suspend fun test(overrides: Map): TestResult = try { val baseUrl = overrides["real-debrid.base-url"] ?: realDebridConfigurationProperties.baseUrl val apiKey = overrides["real-debrid.api-key"] ?: realDebridConfigurationProperties.apiKey diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxClient.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxClient.kt index 61ea7ef4..199872e7 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxClient.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/torbox/TorBoxClient.kt @@ -250,6 +250,7 @@ class TorBoxClient( override val configurationClass: KClass<*> = TorBoxConfigurationProperties::class override val label: String = "TorBox" + @Suppress("TooGenericExceptionCaught") override suspend fun test(overrides: Map): TestResult = try { val baseUrl = overrides["torbox.base-url"] ?: torBoxConfiguration.baseUrl val version = overrides["torbox.version"] ?: torBoxConfiguration.version diff --git a/src/main/kotlin/io/skjaere/debridav/fs/FileController.kt b/src/main/kotlin/io/skjaere/debridav/fs/FileController.kt index 0b6095f8..5f548e7f 100644 --- a/src/main/kotlin/io/skjaere/debridav/fs/FileController.kt +++ b/src/main/kotlin/io/skjaere/debridav/fs/FileController.kt @@ -14,6 +14,7 @@ class FileController( private val databaseFileService: DatabaseFileService, private val jwtService: JwtService ) { + @Suppress("ReturnCount") @GetMapping("/stream-url") fun streamUrl(@RequestParam path: String): ResponseEntity { val entity = databaseFileService.getFileAtPath(path) @@ -32,6 +33,7 @@ class FileController( ) } + @Suppress("ReturnCount") @GetMapping("/detail") fun detail(@RequestParam path: String): ResponseEntity { val entity = databaseFileService.getFileAtPath(path) @@ -42,57 +44,64 @@ class FileController( } val dto = when (entity) { - is RemotelyCachedEntity -> { - val contents = entity.contents - val fileType = when (contents) { - is DebridCachedTorrentContent -> FileType.TORRENT - is DebridCachedUsenetReleaseContent -> FileType.USENET_RELEASE - is NzbContents -> FileType.NZB - else -> FileType.LOCAL - } - val providerStatus = contents?.debridLinks?.mapNotNull { link -> - val provider = link.provider ?: return@mapNotNull null - val status = when (link) { - is CachedFile -> ProviderCacheStatus.CACHED - is MissingFile -> ProviderCacheStatus.MISSING - is ProviderError -> ProviderCacheStatus.PROVIDER_ERROR - is ClientError -> ProviderCacheStatus.CLIENT_ERROR - is NetworkError -> ProviderCacheStatus.NETWORK_ERROR - else -> ProviderCacheStatus.UNKNOWN_ERROR - } - ProviderStatusDto( - provider = provider, - status = status, - lastChecked = link.lastChecked - ) - } - FileDetailDto( - name = entity.name ?: "", - path = path, - size = entity.size, - lastModified = entity.lastModified, - mimeType = entity.mimeType, - fileType = fileType, - hash = entity.hash, - providerStatus = providerStatus - ) - } - is LocalEntity -> FileDetailDto( - name = entity.name ?: "", - path = path, - size = entity.size, - lastModified = entity.lastModified, - mimeType = entity.mimeType, - fileType = FileType.LOCAL, - hash = null, - providerStatus = null - ) + is RemotelyCachedEntity -> buildRemoteDetail(entity, path) + is LocalEntity -> buildLocalDetail(entity, path) else -> return ResponseEntity.badRequest().build() } return ResponseEntity.ok(dto) } + private fun buildRemoteDetail(entity: RemotelyCachedEntity, path: String): FileDetailDto { + val contents = entity.contents + val fileType = when (contents) { + is DebridCachedTorrentContent -> FileType.TORRENT + is DebridCachedUsenetReleaseContent -> FileType.USENET_RELEASE + is NzbContents -> FileType.NZB + else -> FileType.LOCAL + } + val providerStatus = contents?.debridLinks?.mapNotNull { link -> + val provider = link.provider ?: return@mapNotNull null + val status = when (link) { + is CachedFile -> ProviderCacheStatus.CACHED + is MissingFile -> ProviderCacheStatus.MISSING + is ProviderError -> ProviderCacheStatus.PROVIDER_ERROR + is ClientError -> ProviderCacheStatus.CLIENT_ERROR + is NetworkError -> ProviderCacheStatus.NETWORK_ERROR + else -> ProviderCacheStatus.UNKNOWN_ERROR + } + ProviderStatusDto( + provider = provider, + status = status, + lastChecked = link.lastChecked + ) + } + return FileDetailDto( + name = entity.name ?: "", + path = path, + size = entity.size, + lastModified = entity.lastModified, + mimeType = entity.mimeType, + fileType = fileType, + hash = entity.hash, + providerStatus = providerStatus + ) + } + + private fun buildLocalDetail(entity: LocalEntity, path: String): FileDetailDto { + return FileDetailDto( + name = entity.name ?: "", + path = path, + size = entity.size, + lastModified = entity.lastModified, + mimeType = entity.mimeType, + fileType = FileType.LOCAL, + hash = null, + providerStatus = null + ) + } + + @Suppress("ReturnCount") @GetMapping fun list(@RequestParam(defaultValue = "/") path: String): ResponseEntity> { val entity = databaseFileService.getFileAtPath(path) diff --git a/src/main/kotlin/io/skjaere/debridav/fs/StreamController.kt b/src/main/kotlin/io/skjaere/debridav/fs/StreamController.kt index f902dd3f..7a83aa17 100644 --- a/src/main/kotlin/io/skjaere/debridav/fs/StreamController.kt +++ b/src/main/kotlin/io/skjaere/debridav/fs/StreamController.kt @@ -19,6 +19,7 @@ class StreamController( private val databaseFileService: DatabaseFileService, private val streamableResourceFactory: StreamableResourceFactory ) { + @Suppress("ReturnCount") @GetMapping("/t/{token}") fun streamByToken( @PathVariable token: String, @@ -71,6 +72,7 @@ class StreamController( resource.sendContent(response.outputStream, range, null, contentType) } + @Suppress("ReturnCount") private fun parseRangeHeader(header: String?, contentLength: Long): Range? { if (header == null || !header.startsWith("bytes=")) return null val rangeSpec = header.removePrefix("bytes=") diff --git a/src/main/kotlin/io/skjaere/debridav/stream/StreamingService.kt b/src/main/kotlin/io/skjaere/debridav/stream/StreamingService.kt index 46c9deb1..8dd79fdc 100644 --- a/src/main/kotlin/io/skjaere/debridav/stream/StreamingService.kt +++ b/src/main/kotlin/io/skjaere/debridav/stream/StreamingService.kt @@ -24,6 +24,7 @@ import kotlinx.coroutines.withContext import kotlinx.io.EOFException import org.apache.catalina.connector.ClientAbortException import org.slf4j.LoggerFactory +import org.springframework.web.context.request.async.AsyncRequestNotUsableException import org.springframework.scheduling.annotation.Scheduled import org.springframework.stereotype.Service import java.io.OutputStream @@ -103,6 +104,8 @@ class StreamingService( StreamResult.CLIENT_ERROR } catch (_: ClientAbortException) { StreamResult.OK + } catch (_: AsyncRequestNotUsableException) { + StreamResult.OK } catch (e: kotlinx.io.IOException) { logger.error("IOError occurred during streaming", e) StreamResult.IO_ERROR @@ -160,6 +163,10 @@ class StreamingService( } catch (e: CancellationException) { throw e } catch (_: ClientAbortException) { + } catch (_: AsyncRequestNotUsableException) { + } catch (e: kotlinx.io.IOException) { + logger.warn("IO error reading from upstream HTTP stream during streaming", e) + throw ReadFromHttpStreamException("IO error reading from upstream HTTP stream", e) } catch (e: Exception) { logger.error("An error occurred during streaming", e) throw StreamToClientException("An error occurred during streaming", e) diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/NzbImportService.kt b/src/main/kotlin/io/skjaere/debridav/usenet/NzbImportService.kt index da6cdb7b..64a4587e 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/NzbImportService.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/NzbImportService.kt @@ -25,7 +25,8 @@ import kotlinx.coroutines.runBlocking import org.slf4j.LoggerFactory import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.stereotype.Service -import org.springframework.transaction.annotation.Transactional +import org.springframework.transaction.PlatformTransactionManager +import org.springframework.transaction.support.TransactionTemplate import java.time.Instant import java.util.* @@ -38,8 +39,10 @@ class NzbImportService( private val nzbImportRepository: NzbImportRepository, private val pgmqClient: PgmqClient, private val databaseFileService: DatabaseFileService, - private val debridavConfigurationProperties: DebridavConfigurationProperties + private val debridavConfigurationProperties: DebridavConfigurationProperties, + platformTransactionManager: PlatformTransactionManager ) { + private val transactionTemplate = TransactionTemplate(platformTransactionManager) private val logger = LoggerFactory.getLogger(NzbImportService::class.java) fun scheduleImport(nzbBytes: ByteArray, usenetDownload: UsenetDownload, nzbImportRecordId: Long) { @@ -53,63 +56,109 @@ class NzbImportService( ) } - @Transactional @Suppress("LongMethod", "ReturnCount") fun executeImport(taskData: NzbImportTaskData) { - val usenetDownload = usenetRepository.findById(taskData.usenetDownloadId).orElseThrow { - IllegalStateException("UsenetDownload not found: ${taskData.usenetDownloadId}") - } - val importRecord = nzbImportRepository.findById(taskData.nzbImportRecordId).orElseThrow { - IllegalStateException("NzbImportRecord not found: ${taskData.nzbImportRecordId}") - } - try { + // Phase 1: Load entities and mark as IMPORTING in a short transaction. + // We do NOT hold this transaction open during the long NNTP I/O below. + val downloadName = transactionTemplate.execute { + val usenetDownload = usenetRepository.findById(taskData.usenetDownloadId).orElse(null) + ?: run { + logger.warn( + "UsenetDownload ${taskData.usenetDownloadId} not found (may have been deleted), " + + "skipping import" + ) + return@execute null + } + val importRecord = nzbImportRepository.findById(taskData.nzbImportRecordId).orElseThrow { + IllegalStateException("NzbImportRecord not found: ${taskData.nzbImportRecordId}") + } logger.info("Importing ${usenetDownload.name}") importRecord.status = NzbImportStatus.IMPORTING nzbImportRepository.save(importRecord) + usenetDownload.name + } ?: return + + // Phase 2: Perform long-running NNTP I/O outside any database transaction. + // This prevents the transaction from being held open while waiting for the + // network, which would cause ObjectOptimisticLockingFailureException if the + // UsenetDownload row is deleted by another thread during the I/O. + val nzbBytes = Base64.getDecoder().decode(taskData.nzbBytesBase64) + var prepareResult: PrepareResult? = null + var prepareException: Exception? = null + try { + prepareResult = runBlocking { nzbStreamer.prepare(nzbBytes) } + } catch (@Suppress("TooGenericExceptionCaught") e: Exception) { + logger.error("Failed to prepare NZB for download '$downloadName'", e) + prepareException = e + } + + // Phase 3: Re-fetch entities and persist results in a new short transaction. + // Re-fetching avoids operating on stale/detached entities and gracefully handles + // the case where the UsenetDownload was deleted while the I/O was running. + transactionTemplate.execute { + val usenetDownload = usenetRepository.findById(taskData.usenetDownloadId).orElse(null) + val importRecord = nzbImportRepository.findById(taskData.nzbImportRecordId).orElseThrow { + IllegalStateException("NzbImportRecord not found: ${taskData.nzbImportRecordId}") + } + + if (usenetDownload == null) { + logger.warn( + "UsenetDownload ${taskData.usenetDownloadId} ('$downloadName') was deleted " + + "during import; aborting result persistence" + ) + importRecord.status = NzbImportStatus.FAILED + importRecord.errorMessage = "Download was deleted during import" + nzbImportRepository.save(importRecord) + return@execute + } - val nzbBytes = Base64.getDecoder().decode(taskData.nzbBytesBase64) - val prepareResult = runBlocking { nzbStreamer.prepare(nzbBytes) } + when { + prepareException != null -> { + usenetDownload.status = UsenetDownloadStatus.FAILED + importRecord.status = NzbImportStatus.FAILED + importRecord.errorMessage = prepareException.stackTraceToString() + } - when (prepareResult) { - is PrepareResult.MissingArticles -> { + prepareResult is PrepareResult.MissingArticles -> { + val result = prepareResult as PrepareResult.MissingArticles logger.warn( "Articles missing from Usenet for '{}': {}", usenetDownload.name, - prepareResult.message + result.message ) usenetDownload.status = UsenetDownloadStatus.ARTICLES_MISSING importRecord.status = NzbImportStatus.ARTICLES_MISSING - importRecord.errorMessage = prepareResult.message - return + importRecord.errorMessage = result.message } - is PrepareResult.Failure -> { + prepareResult is PrepareResult.Failure -> { + val result = prepareResult as PrepareResult.Failure logger.error( "NNTP failure importing '{}': {}", usenetDownload.name, - prepareResult.message, - prepareResult.cause + result.message, + result.cause ) usenetDownload.status = UsenetDownloadStatus.FAILED importRecord.status = NzbImportStatus.FAILED - importRecord.errorMessage = prepareResult.cause.stackTraceToString() - return + importRecord.errorMessage = result.cause.stackTraceToString() } - is PrepareResult.UnsupportedArchive -> { + prepareResult is PrepareResult.UnsupportedArchive -> { + val result = prepareResult as PrepareResult.UnsupportedArchive logger.warn( "Unsupported archive type for '{}': {}", usenetDownload.name, - prepareResult.message + result.message ) usenetDownload.status = UsenetDownloadStatus.FAILED importRecord.status = NzbImportStatus.FAILED - importRecord.errorMessage = prepareResult.message - return + importRecord.errorMessage = result.message } - is PrepareResult.Success -> { - val metadata = prepareResult.metadata + prepareResult is PrepareResult.Success -> { + val result = prepareResult as PrepareResult.Success + val metadata = result.metadata val streamableFiles = nzbStreamer.resolveStreamableFiles(metadata) val documentEntity = toDocumentEntity(metadata, streamableFiles) documentEntity.category = usenetDownload.category?.name @@ -148,13 +197,14 @@ class NzbImportService( } logger.info("Imported ${usenetDownload.name}") } + + else -> { + usenetDownload.status = UsenetDownloadStatus.FAILED + importRecord.status = NzbImportStatus.FAILED + importRecord.errorMessage = "Unknown error during import prepare phase" + } } - } catch (@Suppress("TooGenericExceptionCaught") e: Exception) { - logger.error("Failed to import NZB for download '${usenetDownload.name}'", e) - usenetDownload.status = UsenetDownloadStatus.FAILED - importRecord.status = NzbImportStatus.FAILED - importRecord.errorMessage = e.stackTraceToString() - } finally { + usenetRepository.save(usenetDownload) nzbImportRepository.save(importRecord) } diff --git a/src/test/kotlin/io/skjaere/debridav/test/NzbImportServiceTest.kt b/src/test/kotlin/io/skjaere/debridav/test/NzbImportServiceTest.kt index 13e0941a..0e8841f4 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/NzbImportServiceTest.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/NzbImportServiceTest.kt @@ -25,8 +25,6 @@ import io.skjaere.debridav.usenet.queue.NzbImportRecord import io.skjaere.debridav.usenet.queue.NzbImportStatus import io.skjaere.nntp.ArticleNotFoundException import io.skjaere.nntp.NntpConnectionException -import io.skjaere.nntp.YencHeaders -import java.io.IOException import io.skjaere.nzbstreamer.NzbStreamer import io.skjaere.nzbstreamer.metadata.ExtractedMetadata import io.skjaere.nzbstreamer.metadata.NzbMetadataResponse @@ -35,7 +33,10 @@ import io.skjaere.nzbstreamer.nzb.NzbDocument import io.skjaere.nzbstreamer.nzb.NzbFile import io.skjaere.nzbstreamer.nzb.NzbSegment import io.skjaere.nzbstreamer.stream.StreamableFile +import io.skjaere.nntp.YencHeaders +import java.io.IOException import org.junit.jupiter.api.Test +import org.springframework.transaction.PlatformTransactionManager import java.time.Duration import java.util.* import kotlin.test.assertEquals @@ -68,9 +69,17 @@ class NzbImportServiceTest { localEntityMaxSizeMb = 100 } + /** + * A no-op PlatformTransactionManager that does not start real DB transactions. + * TransactionTemplate will still execute the callback directly, which is all + * we need for unit tests that already mock the repository methods. + */ + private val platformTransactionManager = mockk(relaxed = true) + private val underTest = NzbImportService( nzbStreamer, nzbDocumentRepository, usenetRepository, - nzbImportRepository, pgmqClient, databaseFileService, config + nzbImportRepository, pgmqClient, databaseFileService, config, + platformTransactionManager ) private val nzbBytes = "test".toByteArray() @@ -273,14 +282,16 @@ class NzbImportServiceTest { } @Test - fun `executeImport throws when UsenetDownload not found`() { + fun `executeImport throws when UsenetDownload not found on initial load`() { // given every { usenetRepository.findById(999L) } returns Optional.empty() - // when/then - kotlin.test.assertFailsWith { - underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 999L, 100L)) - } + // when - should return early without error (download may have been deleted) + underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 999L, 100L)) + + // then - no exception, no save calls + verify(exactly = 0) { usenetRepository.save(any()) } + verify(exactly = 0) { nzbImportRepository.save(any()) } } @Test @@ -314,10 +325,59 @@ class NzbImportServiceTest { // when underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L, 100L)) - // then - save is called: once for IMPORTING status + once in finally block = 2 + // then - nzbImportRepository.save is called: once for IMPORTING status (phase 1) + once in phase 3 = 2 verify(exactly = 2) { nzbImportRepository.save(any()) } verify(exactly = 1) { usenetRepository.save(any()) } assertEquals(UsenetDownloadStatus.ARTICLES_MISSING, download.status) assertEquals(NzbImportStatus.ARTICLES_MISSING, importRecord.status) } + + /** + * Reproduces the production bug: ObjectOptimisticLockingFailureException when + * the UsenetDownload row is deleted (e.g., via the SABnzbd delete API) while + * the long-running NNTP prepare is in flight. + * + * Before the fix, executeImport held a single @Transactional spanning the entire + * method including the NNTP I/O. If another transaction deleted the UsenetDownload + * row during that I/O, the transaction commit would fail with: + * "Unexpected row count (expected row count 1 but was 0) + * [update usenet_download ... where id=?]" + * + * After the fix, executeImport uses short-lived TransactionTemplate scopes so no + * transaction is held during I/O. The entity is re-fetched in the save phase; if + * it was deleted, the method completes gracefully without throwing. + */ + @Test + fun `executeImport completes gracefully when UsenetDownload is deleted during NNTP IO`() { + // given + val download = createUsenetDownload(id = 102L) + val importRecord = createImportRecord(id = 86L) + + // Phase 1 (load): download exists + // Phase 3 (save): download has been deleted by another thread/request + every { usenetRepository.findById(102L) } returnsMany listOf( + Optional.of(download), + Optional.empty() // simulates concurrent deletion during NNTP I/O + ) + every { nzbImportRepository.findById(86L) } returns Optional.of(importRecord) + + coEvery { nzbStreamer.prepare(any()) } returns PrepareResult.MissingArticles( + "Article not found", + ArticleNotFoundException("Article not found") + ) + + val importSlot = slot() + every { nzbImportRepository.save(capture(importSlot)) } answers { importSlot.captured } + + // when — must NOT throw ObjectOptimisticLockingFailureException + underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 102L, 86L)) + + // then + // UsenetDownload.save is never called because the row is gone + verify(exactly = 0) { usenetRepository.save(any()) } + // The import record is still saved so we have a record of the failure + verify(exactly = 2) { nzbImportRepository.save(any()) } + assertEquals(NzbImportStatus.FAILED, importSlot.captured.status) + assertEquals("Download was deleted during import", importSlot.captured.errorMessage) + } } diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/ConfigApiIT.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/ConfigApiIT.kt index 242ba77c..2fa40fe8 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/ConfigApiIT.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/ConfigApiIT.kt @@ -228,4 +228,32 @@ class ConfigApiIT { assertEquals("new-api-key-12345", premiumizeConfig.apiKey) } + + @Test + fun `saving nntp pools twice does not cause duplicate key violation`() { + val poolJson = """[{"host":"news.example.com","port":563,"username":"user",""" + + """"password":"pass","useTls":true,"maxConnections":8,"priority":0}]""" + + // First save + webTestClient.put() + .uri("/api/v1/config/nntp-pools") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue(poolJson) + .exchange() + .expectStatus().isOk + + val updatedPoolJson = """[{"host":"news2.example.com","port":563,"username":"user2",""" + + """"password":"pass2","useTls":true,"maxConnections":4,"priority":0}]""" + + // Second save - should not throw DataIntegrityViolationException + webTestClient.put() + .uri("/api/v1/config/nntp-pools") + .contentType(MediaType.APPLICATION_JSON) + .bodyValue(updatedPoolJson) + .exchange() + .expectStatus().isOk + .expectBody() + .jsonPath("$[0].host").isEqualTo("news2.example.com") + .jsonPath("$[0].maxConnections").isEqualTo(4) + } } diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/StreamingEofIT.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/StreamingEofIT.kt new file mode 100644 index 00000000..c780b30d --- /dev/null +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/StreamingEofIT.kt @@ -0,0 +1,115 @@ +package io.skjaere.debridav.test.integrationtest + +import ch.qos.logback.classic.Level +import ch.qos.logback.classic.Logger +import ch.qos.logback.classic.spi.ILoggingEvent +import ch.qos.logback.core.read.ListAppender +import io.skjaere.debridav.DebriDavApplication +import io.skjaere.debridav.MiltonConfiguration +import io.skjaere.debridav.debrid.DebridProvider +import io.skjaere.debridav.fs.CachedFile +import io.skjaere.debridav.fs.DatabaseFileService +import io.skjaere.debridav.repository.DebridFileContentsRepository +import io.skjaere.debridav.stream.StreamingService +import io.skjaere.debridav.test.debridFileContents +import io.skjaere.debridav.test.deepCopy +import io.skjaere.debridav.test.integrationtest.config.ContentStubbingService +import io.skjaere.debridav.test.integrationtest.config.IntegrationTestContextConfiguration +import io.skjaere.debridav.test.integrationtest.config.MockServerTest +import org.apache.commons.codec.digest.DigestUtils +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Test +import org.mockserver.integration.ClientAndServer +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.test.web.reactive.server.WebTestClient +import java.time.Duration +import java.time.Instant +import kotlin.test.assertFalse + +@SpringBootTest( + classes = [DebriDavApplication::class, IntegrationTestContextConfiguration::class, MiltonConfiguration::class], + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT +) +@MockServerTest +class StreamingEofIT { + @Autowired + private lateinit var databaseFileService: DatabaseFileService + + @Autowired + private lateinit var webTestClient: WebTestClient + + @Autowired + private lateinit var contentStubbingService: ContentStubbingService + + @Autowired + lateinit var debridFileContentsRepository: DebridFileContentsRepository + + @Autowired + lateinit var mockserverClient: ClientAndServer + + @AfterEach + fun tearDown() { + mockserverClient.reset() + } + + @Test + fun `that premature EOF from upstream does not result in ERROR log`() { + // given - file claims to be 1000 bytes but server only returns 9 bytes ("it works!") + // This simulates a debrid provider sending fewer bytes than promised, causing premature EOF + val fileContents = debridFileContents.deepCopy() + val hash = DigestUtils.md5Hex("eof-test") + fileContents.size = 1000L + + val debridLink = CachedFile( + "testfile-eof.mp4", + link = "http://localhost:${contentStubbingService.port}/truncatedLink", + size = 1000L, + provider = DebridProvider.PREMIUMIZE, + lastChecked = Instant.now().toEpochMilli(), + params = mapOf(), + mimeType = "video/mp4" + ) + fileContents.debridLinks = mutableListOf(debridLink) + contentStubbingService.mockTruncatedStream() + databaseFileService.createDebridFile("/testfile-eof.mp4", hash, fileContents) + .let { debridFileContentsRepository.save(it) } + + // Capture StreamingService logs to verify no ERROR-level log is generated for EOF + val streamingLogger = LoggerFactory.getLogger(StreamingService::class.java) as Logger + val listAppender = ListAppender() + listAppender.start() + streamingLogger.addAppender(listAppender) + + try { + // when - make request; server sends 9 bytes but we expect 1000, triggering premature EOF + // The EOFException thrown by readAvailable returning -1 before all bytes are consumed + // should be handled gracefully at WARN level (not ERROR), so Sentry does not capture it + try { + webTestClient + .mutate().responseTimeout(Duration.ofMillis(30000)).build() + .get() + .uri("/testfile-eof.mp4") + .exchange() + } catch (_: Exception) { + // Connection may close early due to EOF handling upstream; this is expected + } + + // then - no ERROR log should be generated for premature EOF from upstream HTTP stream. + // Before the fix, EOFException was caught by the generic Exception handler which logged + // at ERROR level, causing Sentry to fire false alerts for normal network conditions. + // After the fix, it is caught by the kotlinx.io.IOException handler and logged at WARN. + val errorLogs = listAppender.list.filter { + it.level == Level.ERROR && it.formattedMessage.contains("An error occurred during streaming") + } + assertFalse( + errorLogs.isNotEmpty(), + "Expected no ERROR log for premature EOF from upstream, but found: " + + errorLogs.map { it.formattedMessage } + ) + } finally { + streamingLogger.detachAppender(listAppender) + } + } +} diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/ContentStubbingService.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/ContentStubbingService.kt index b4e5c1b7..30e03cd8 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/ContentStubbingService.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/ContentStubbingService.kt @@ -155,6 +155,33 @@ class ContentStubbingService(@Value("\${mockserver.port}") val port: Int) { ) } + fun mockTruncatedStream() { + MockServerClient( + "localhost", + port + ).`when`( + HttpRequest.request() + .withMethod("GET") + .withPath("/truncatedLink"), + Times.exactly(1) + ).respond( + HttpResponse.response() + .withStatusCode(200) + .withBody("it works!".toByteArray()) + ) + MockServerClient( + "localhost", + port + ).`when`( + HttpRequest.request() + .withMethod("HEAD") + .withPath("/truncatedLink") + ).respond( + HttpResponse.response() + .withStatusCode(200) + ) + } + fun mockDeadLink() { MockServerClient( "localhost", diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/TestContextInitializer.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/TestContextInitializer.kt index f612c97c..099ec282 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/TestContextInitializer.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/TestContextInitializer.kt @@ -1,6 +1,5 @@ package io.skjaere.debridav.test.integrationtest.config -import io.skjaere.mocknntp.testcontainer.MockNntpServerContainer import org.apache.commons.io.FileUtils import org.mockserver.configuration.Configuration import org.mockserver.integration.ClientAndServer @@ -12,13 +11,12 @@ import org.springframework.context.ApplicationListener import org.springframework.context.ConfigurableApplicationContext import org.springframework.context.event.ContextClosedEvent import org.springframework.test.util.TestSocketUtils -import org.testcontainers.postgresql.PostgreSQLContainer -import org.testcontainers.utility.DockerImageName import java.io.File class TestContextInitializer : ApplicationContextInitializer { companion object { const val BASE_PATH = "/tmp/debridavtests" + val postgreSQLContainer: PostgreSQLContainer = PostgreSQLContainer(DockerImageName.parse("postgres:16-alpine")) .withUsername("postgres") @@ -37,7 +35,7 @@ class TestContextInitializer : ApplicationContextInitializer() { @@ -45,6 +43,11 @@ class TestContextInitializer : ApplicationContextInitializer Date: Sat, 7 Mar 2026 15:45:00 +0100 Subject: [PATCH 06/61] fix(config): sync pools/timeouts after runtime overrides apply Two related fixes to how runtime config overrides take effect: - NNTP pools now re-sync with NzbStreamer after DB overrides are loaded on startup (previously pools stored in overrides were ignored until the next save) - HttpClient connect-timeout now reads the current config at request time instead of at bean creation, so overrides actually apply Co-Authored-By: Claude Opus 4.7 (1M context) --- .../io/skjaere/debridav/DebridavConfiguration.kt | 15 ++++++++++++--- .../debridav/config/ConfigOverrideService.kt | 4 ++++ .../config/DatabasePropertySourceInitializer.kt | 5 ++++- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/io/skjaere/debridav/DebridavConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/DebridavConfiguration.kt index f0721638..93a6bd0f 100644 --- a/src/main/kotlin/io/skjaere/debridav/DebridavConfiguration.kt +++ b/src/main/kotlin/io/skjaere/debridav/DebridavConfiguration.kt @@ -3,7 +3,9 @@ package io.skjaere.debridav import io.ktor.client.HttpClient import io.ktor.client.engine.cio.CIO import io.ktor.client.plugins.HttpTimeout +import io.ktor.client.plugins.api.createClientPlugin import io.ktor.client.plugins.contentnegotiation.ContentNegotiation +import io.ktor.client.plugins.timeout import io.ktor.serialization.kotlinx.json.json import io.milton.servlet.SpringMiltonFilter import io.skjaere.debridav.configuration.DebridavConfigurationProperties @@ -44,9 +46,7 @@ class DebridavConfiguration { @Bean fun httpClient(debridavConfigurationProperties: DebridavConfigurationProperties): HttpClient = HttpClient(CIO) { - install(HttpTimeout) { - connectTimeoutMillis = debridavConfigurationProperties.connectTimeoutMilliseconds - } + install(HttpTimeout) install(ContentNegotiation) { json( Json { @@ -56,6 +56,15 @@ class DebridavConfiguration { } ) } + install( + createClientPlugin("DynamicConnectTimeout") { + onRequest { request, _ -> + request.timeout { + connectTimeoutMillis = debridavConfigurationProperties.connectTimeoutMilliseconds + } + } + } + ) } @Bean diff --git a/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideService.kt b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideService.kt index 08dfde0f..1d3e796d 100644 --- a/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideService.kt +++ b/src/main/kotlin/io/skjaere/debridav/config/ConfigOverrideService.kt @@ -165,6 +165,10 @@ class ConfigOverrideService( syncRunningPools(pools) } + fun syncRunningNntpPools() { + syncRunningPools(getNntpPools()) + } + private fun syncRunningPools(saved: List) { if (nzbStreamer == null) return val savedConfigs = saved.map { it.toNntpConfig() }.toSet() diff --git a/src/main/kotlin/io/skjaere/debridav/config/DatabasePropertySourceInitializer.kt b/src/main/kotlin/io/skjaere/debridav/config/DatabasePropertySourceInitializer.kt index 4fecd913..58fcfc6a 100644 --- a/src/main/kotlin/io/skjaere/debridav/config/DatabasePropertySourceInitializer.kt +++ b/src/main/kotlin/io/skjaere/debridav/config/DatabasePropertySourceInitializer.kt @@ -4,6 +4,7 @@ import org.slf4j.LoggerFactory import org.springframework.boot.context.event.ApplicationReadyEvent import org.springframework.cloud.context.refresh.ContextRefresher import org.springframework.context.ApplicationListener +import org.springframework.context.annotation.Lazy import org.springframework.core.env.ConfigurableEnvironment import org.springframework.stereotype.Component @@ -11,7 +12,8 @@ import org.springframework.stereotype.Component class DatabasePropertySourceInitializer( private val environment: ConfigurableEnvironment, private val repository: ConfigOverrideRepository, - private val contextRefresher: ContextRefresher + private val contextRefresher: ContextRefresher, + @Lazy private val configOverrideService: ConfigOverrideService ) : ApplicationListener { private val logger = LoggerFactory.getLogger(DatabasePropertySourceInitializer::class.java) @@ -23,6 +25,7 @@ class DatabasePropertySourceInitializer( propertySource.replaceAll(overrides) contextRefresher.refreshEnvironment() logger.info("Loaded {} database config override(s) and refreshed environment", overrides.size) + configOverrideService.syncRunningNntpPools() } else { logger.info("No database config overrides found") } From b4e179a7f2fabae583338b88e114dc5dd4b07a1e Mon Sep 17 00:00:00 2001 From: Skjaere Date: Sun, 8 Mar 2026 12:00:00 +0100 Subject: [PATCH 07/61] fix(queue): lazy file-path resolution in queue responses When a file moves in the Arr, the queue response used to return stale paths cached at enqueue time. Resolves paths lazily via a dedicated endpoint that queries db_item directly, so queue responses always reflect the current on-disk layout. Also derives SABnzbd history storage path from the actual file location and tightens a few test isolation issues the changes surfaced. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../DebridFileContentsRepository.kt | 9 + .../usenet/queue/UsenetQueueController.kt | 11 +- .../usenet/queue/UsenetQueueService.kt | 49 +++-- .../integrationtest/QueueFileResolutionIT.kt | 183 ++++++++++++++++++ .../integrationtest/WebDavOperationsIT.kt | 13 +- .../config/TestContextInitializer.kt | 16 +- 6 files changed, 247 insertions(+), 34 deletions(-) create mode 100644 src/test/kotlin/io/skjaere/debridav/test/integrationtest/QueueFileResolutionIT.kt diff --git a/src/main/kotlin/io/skjaere/debridav/repository/DebridFileContentsRepository.kt b/src/main/kotlin/io/skjaere/debridav/repository/DebridFileContentsRepository.kt index 2a6697ce..5ad6011b 100644 --- a/src/main/kotlin/io/skjaere/debridav/repository/DebridFileContentsRepository.kt +++ b/src/main/kotlin/io/skjaere/debridav/repository/DebridFileContentsRepository.kt @@ -2,6 +2,7 @@ package io.skjaere.debridav.repository import io.skjaere.debridav.fs.DbDirectory import io.skjaere.debridav.fs.DbEntity +import io.skjaere.debridav.fs.RemotelyCachedEntity import jakarta.transaction.Transactional import org.springframework.data.jpa.repository.Modifying import org.springframework.data.jpa.repository.Query @@ -84,6 +85,14 @@ interface DebridFileContentsRepository : CrudRepository { @Query("select count(*) from DebridCachedUsenetReleaseContent ") fun numberOfRemotelyCachedUsenetEntities(): Long + + @Query( + "select rce.* from db_item rce " + + "join usenet_download_debrid_files udf on udf.debrid_files_id = rce.id " + + "where udf.usenet_download_id = :usenetDownloadId", + nativeQuery = true + ) + fun findByUsenetDownloadId(usenetDownloadId: Long): List } data class LibraryStats(val provider: String, val type: String, val count: Long) diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/queue/UsenetQueueController.kt b/src/main/kotlin/io/skjaere/debridav/usenet/queue/UsenetQueueController.kt index 8f1bf3bd..9a80e929 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/queue/UsenetQueueController.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/queue/UsenetQueueController.kt @@ -2,6 +2,7 @@ package io.skjaere.debridav.usenet.queue import org.springframework.http.ResponseEntity import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RequestParam import org.springframework.web.bind.annotation.RestController @@ -13,16 +14,6 @@ class UsenetQueueController(private val queueService: UsenetQueueService) { fun getQueueStatus(): ResponseEntity = ResponseEntity.ok(queueService.getQueueStatus()) - @GetMapping("/history") - fun getHistory( - @RequestParam(defaultValue = "0") page: Int, - @RequestParam(defaultValue = "20") size: Int, - @RequestParam(defaultValue = "") search: String, - @RequestParam(defaultValue = "updatedAt") sort: String, - @RequestParam(defaultValue = "desc") direction: String - ): ResponseEntity = - ResponseEntity.ok(queueService.getHistory(page, size, search, sort, direction)) - @GetMapping("/{id}/files") fun getItemFiles(@PathVariable id: Long): ResponseEntity> { val files = queueService.resolveCurrentFilePaths(id) diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/queue/UsenetQueueService.kt b/src/main/kotlin/io/skjaere/debridav/usenet/queue/UsenetQueueService.kt index 841a7e1c..f5a72a7f 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/queue/UsenetQueueService.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/queue/UsenetQueueService.kt @@ -1,12 +1,14 @@ package io.skjaere.debridav.usenet.queue +import io.skjaere.debridav.repository.DebridFileContentsRepository import io.skjaere.debridav.repository.NzbImportRepository -import org.springframework.data.domain.PageRequest -import org.springframework.data.domain.Sort import org.springframework.stereotype.Service @Service -class UsenetQueueService(private val nzbImportRepository: NzbImportRepository) { +class UsenetQueueService( + private val nzbImportRepository: NzbImportRepository, + private val debridFileContentsRepository: DebridFileContentsRepository +) { companion object { val PROCESSING_STATUSES = listOf( @@ -57,15 +59,34 @@ class UsenetQueueService(private val nzbImportRepository: NzbImportRepository) { ) } - private fun NzbImportRecord.toDto() = QueueItemDto( - id = id!!, - name = name, - status = status.name, - size = size, - errorMessage = errorMessage, - updatedAt = updatedAt, - createdAt = createdAt, - archiveType = archiveType, - files = files - ) + @Suppress("ReturnCount") + fun resolveCurrentFilePaths(importId: Long): List? { + val record = nzbImportRepository.findById(importId).orElse(null) ?: return null + val usenetDownloadId = record.usenetDownloadId ?: return record.files + val dbItems = debridFileContentsRepository.findByUsenetDownloadId(usenetDownloadId) + if (dbItems.isEmpty()) return record.files + val currentFiles = dbItems.mapNotNull { entity -> + val dirPath = entity.directory?.fileSystemPath() ?: return@mapNotNull null + val fileName = entity.name ?: return@mapNotNull null + NzbImportFileJson( + path = "$dirPath/$fileName", + size = entity.size ?: 0L + ) + } + return currentFiles.ifEmpty { record.files } + } + + private fun NzbImportRecord.toDto(): QueueItemDto { + return QueueItemDto( + id = id!!, + name = name, + status = status.name, + size = size, + errorMessage = errorMessage, + updatedAt = updatedAt, + createdAt = createdAt, + archiveType = archiveType, + files = files + ) + } } diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/QueueFileResolutionIT.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/QueueFileResolutionIT.kt new file mode 100644 index 00000000..63df74dc --- /dev/null +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/QueueFileResolutionIT.kt @@ -0,0 +1,183 @@ +package io.skjaere.debridav.test.integrationtest + +import com.github.sardine.SardineFactory +import io.skjaere.compressionutils.generation.ContainerType +import io.skjaere.debridav.DebriDavApplication +import io.skjaere.debridav.MiltonConfiguration +import io.skjaere.debridav.fs.DatabaseFileService +import io.skjaere.debridav.fs.DbDirectory +import io.skjaere.debridav.repository.NzbImportRepository +import io.skjaere.debridav.repository.UsenetRepository +import io.skjaere.debridav.test.integrationtest.config.IntegrationTestContextConfiguration +import io.skjaere.debridav.test.integrationtest.config.MockServerTest +import io.skjaere.debridav.usenet.queue.NzbImportFileJson +import io.skjaere.debridav.usenet.sabnzbd.model.SabnzbdFullHistoryResponse +import io.skjaere.mocknntp.testcontainer.MockNntpServerContainer +import kotlinx.coroutines.runBlocking +import kotlinx.serialization.json.Json +import org.hamcrest.MatcherAssert.assertThat +import org.hamcrest.Matchers.`is` +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.boot.test.web.server.LocalServerPort +import org.springframework.http.MediaType +import org.springframework.http.client.MultipartBodyBuilder +import org.springframework.test.web.reactive.server.WebTestClient +import org.springframework.web.reactive.function.BodyInserters +import tools.jackson.module.kotlin.jacksonObjectMapper + +@SpringBootTest( + classes = [DebriDavApplication::class, IntegrationTestContextConfiguration::class, MiltonConfiguration::class], + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = ["debridav.debrid-clients=easynews", "nntp.enabled=true"] +) +@MockServerTest +class QueueFileResolutionIT { + + @Autowired + private lateinit var usenetRepository: UsenetRepository + + @Autowired + private lateinit var nzbImportRepository: NzbImportRepository + + @Autowired + private lateinit var databaseFileService: DatabaseFileService + + @Autowired + private lateinit var webTestClient: WebTestClient + + @Autowired + private lateinit var mockNntpServerContainer: MockNntpServerContainer + + @LocalServerPort + var randomServerPort: Int = 0 + + private val deserializer = Json { ignoreUnknownKeys = true } + private val sardine = SardineFactory.begin() + + @AfterEach + fun tearDown() { + runBlocking { mockNntpServerContainer.client.clearYencBodyExpectations() } + @Suppress("TooGenericExceptionCaught") + try { + sardine.delete("http://localhost:${randomServerPort}/movies/") + } catch (_: Exception) { } + @Suppress("TooGenericExceptionCaught") + try { + sardine.delete("http://localhost:${randomServerPort}/downloads/queue-file-resolution-test") + } catch (_: Exception) { } + usenetRepository.deleteAll() + nzbImportRepository.deleteAll() + } + + @Test + fun `queue files endpoint returns updated paths after file is moved`() { + val releaseName = "queue-file-resolution-test" + + // given - prepare and import an NZB + val testData = ByteArray(32 * 1024) { (it % 256).toByte() } + val nzbXml = runBlocking { + mockNntpServerContainer.client.prepareArchiveNzb( + fileContents = mapOf("testfile.bin" to testData), + containerType = ContainerType.RAR5 + ) + } + uploadNzb(nzbXml, releaseName) + waitForCompletion(releaseName) + + // find the import record id by name (other tests may leave records behind) + val importId = nzbImportRepository.findAll() + .first { it.name == releaseName }.id!! + + // verify initial file path points to downloads/ + val initialFiles = getQueueItemFiles(importId) + assertThat("Should have one file", initialFiles.size, `is`(1)) + assertThat( + "Initial path should be under /downloads, got: ${initialFiles[0].path}", + initialFiles[0].path.startsWith("/downloads/$releaseName/"), + `is`(true) + ) + + // when - move the release directory using DatabaseFileService + val releaseDir = databaseFileService.getFileAtPath("/downloads/$releaseName") as DbDirectory + databaseFileService.createDirectory("/movies") + databaseFileService.moveResource(releaseDir, "/movies/$releaseName", releaseName) + + // then - queue files endpoint should return the new path + val movedFiles = getQueueItemFiles(importId) + assertThat("Should still have one file", movedFiles.size, `is`(1)) + assertThat( + "Path should now be under /movies, got: ${movedFiles[0].path}", + movedFiles[0].path.startsWith("/movies/$releaseName/"), + `is`(true) + ) + assertThat( + "File name should be preserved", + movedFiles[0].path.endsWith("/testfile.bin"), + `is`(true) + ) + } + + private fun getQueueItemFiles(importId: Long): List { + val body = webTestClient.get().uri("/api/v1/queue/$importId/files") + .exchange() + .expectStatus().is2xxSuccessful + .expectBody(String::class.java) + .returnResult().responseBody!! + + val mapper = jacksonObjectMapper() + val type = mapper.typeFactory + .constructCollectionType(List::class.java, NzbImportFileJson::class.java) + return mapper.readValue(body, type) + } + + private fun uploadNzb(nzbXml: String, releaseName: String) { + val parts = MultipartBodyBuilder() + parts.part("mode", "addfile") + parts.part("cat", "testcat") + parts.part("name", nzbXml.toByteArray(Charsets.UTF_8)) + .header("Content-Disposition", "form-data; name=name; filename=$releaseName.nzb") + + webTestClient.post().uri("/api").contentType(MediaType.APPLICATION_JSON) + .body(BodyInserters.fromMultipartData(parts.build())).exchange().expectStatus().is2xxSuccessful + } + + @Suppress("NestedBlockDepth") + private fun waitForCompletion(releaseName: String) { + val historyParts = MultipartBodyBuilder() + historyParts.part("mode", "history") + historyParts.part("cat", "testcat") + + var completed = false + var lastStatus = "unknown" + var attempts = 0 + while (attempts < 30 && !completed) { + Thread.sleep(1000) + webTestClient.post().uri("/api") + .body(BodyInserters.fromMultipartData(historyParts.build())) + .exchange() + .expectStatus().is2xxSuccessful + .expectBody(String::class.java) + .returnResult().responseBody + ?.let { historyBody -> + val history = deserializer.decodeFromString(historyBody) + val slot = history.history.slots.firstOrNull { it.name == releaseName } + slot?.let { + lastStatus = it.status + if (it.status == "COMPLETED" || it.status == "FAILED") { + completed = it.status == "COMPLETED" + } + } + } + attempts++ + } + + assertThat( + "Import should complete within timeout (last status: $lastStatus)", + completed, + `is`(true) + ) + } +} diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/WebDavOperationsIT.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/WebDavOperationsIT.kt index 6405f89a..4a40969b 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/WebDavOperationsIT.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/WebDavOperationsIT.kt @@ -16,6 +16,7 @@ import org.hamcrest.Matchers.hasProperty import org.hamcrest.Matchers.hasSize import org.hamcrest.Matchers.`is` import org.hamcrest.Matchers.not +import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired @@ -39,6 +40,13 @@ class WebDavOperationsIT { private val sardine = SardineFactory.begin() + private var baselineEntityCount: Int = -1 + + @BeforeEach + fun captureBaseline() { + baselineEntityCount = debridFileContentsRepository.findAll().toList().size + } + @Test fun thatCreatingFileInRootWorks() { //when @@ -584,12 +592,11 @@ class WebDavOperationsIT { private fun assertReset() { debridFileContentsRepository.findAll() .toList().let { - if (it.size != 4) { + if (it.size != baselineEntityCount) { it.forEach { logger.error("item found ${it.name}") } } - assertThat(it, hasSize(4)) + assertThat(it, hasSize(baselineEntityCount)) } - } private fun listDirectory(path: String): List = diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/TestContextInitializer.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/TestContextInitializer.kt index 099ec282..2d9ce5e1 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/TestContextInitializer.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/config/TestContextInitializer.kt @@ -1,5 +1,6 @@ package io.skjaere.debridav.test.integrationtest.config +import io.skjaere.mocknntp.testcontainer.MockNntpServerContainer import org.apache.commons.io.FileUtils import org.mockserver.configuration.Configuration import org.mockserver.integration.ClientAndServer @@ -11,12 +12,13 @@ import org.springframework.context.ApplicationListener import org.springframework.context.ConfigurableApplicationContext import org.springframework.context.event.ContextClosedEvent import org.springframework.test.util.TestSocketUtils +import org.testcontainers.postgresql.PostgreSQLContainer +import org.testcontainers.utility.DockerImageName import java.io.File class TestContextInitializer : ApplicationContextInitializer { companion object { const val BASE_PATH = "/tmp/debridavtests" - val postgreSQLContainer: PostgreSQLContainer = PostgreSQLContainer(DockerImageName.parse("postgres:16-alpine")) .withUsername("postgres") @@ -35,7 +37,7 @@ class TestContextInitializer : ApplicationContextInitializer() { @@ -58,12 +60,12 @@ class TestContextInitializer : ApplicationContextInitializer Date: Sun, 8 Mar 2026 17:30:00 +0100 Subject: [PATCH 08/61] feat(health): torrent health check + repair pipeline Adds a full health-check and repair pipeline for torrents on top of PGMQ: periodic re-verification of cached links, short-circuit on the first healthy debrid provider, and automatic Arr-side blocklist + search on failure. Migrates the health check surface to a dedicated REST API, adds scheduled PGMQ archive cleanup, and retires ARTICLES_MISSING in favour of a single FAILED terminal state. Bumps nzb-streamer to 0.7.4. Co-Authored-By: Claude Opus 4.7 (1M context) --- gradle/libs.versions.toml | 2 +- .../io/skjaere/debridav/arrs/ArrService.kt | 6 +- .../skjaere/debridav/arrs/client/ArrClient.kt | 2 +- .../debridav/arrs/client/RadarrApiClient.kt | 8 +- .../debridav/arrs/client/SonarrApiClient.kt | 5 +- .../DebridavConfigurationProperties.kt | 14 + .../debridav/debrid/DebridLinkService.kt | 2 +- .../debridav/health/HealthQueueController.kt | 36 ++ .../debridav/health/HealthQueueItemDto.kt | 31 ++ .../debridav/health/HealthQueueService.kt | 23 ++ .../health/PgmqHealthQueueRepository.kt | 216 +++++++++++ .../health/RepairConfigurationProperties.kt | 10 + .../skjaere/debridav/health/RepairOutcome.kt | 43 +++ .../debridav/health/RepairOutcomeService.kt | 19 + .../pgmq/PgmqConfigurationProperties.kt | 26 ++ .../{usenet => }/pgmq/PgmqConsumer.kt | 6 +- .../pgmq/PgmqInfrastructureConfiguration.kt | 41 ++ .../io/skjaere/debridav/torrent/Torrent.kt | 6 + .../TorrentHealthCheckActuatorEndpoint.kt | 16 + .../torrent/TorrentHealthCheckService.kt | 60 +++ .../debridav/torrent/TorrentRepository.kt | 9 + .../skjaere/debridav/torrent/pgmq/Messages.kt | 10 + .../torrent/pgmq/TorrentHealthCheckHandler.kt | 76 ++++ .../pgmq/TorrentHealthRepairHandler.kt | 93 +++++ .../torrent/pgmq/TorrentPgmqConfiguration.kt | 48 +++ .../debridav/usenet/NzbImportService.kt | 4 +- .../skjaere/debridav/usenet/UsenetDownload.kt | 5 +- .../usenet/pgmq/NzbHealthRepairHandler.kt | 65 +++- .../usenet/pgmq/PgmqArchiveCleanupService.kt | 48 +++ .../usenet/pgmq/PgmqSpringConfiguration.kt | 58 +-- .../debridav/usenet/queue/NzbImportRecord.kt | 3 +- .../usenet/queue/UsenetQueueService.kt | 3 +- src/main/resources/application.yaml | 13 +- .../V17__torrent_health_and_pgmq_queues.sql | 5 + .../migration/V18__repair_outcome_table.sql | 9 + .../skjaere/debridav/test/ArrServiceTest.kt | 2 +- .../debridav/test/NzbImportServiceTest.kt | 10 +- .../integrationtest/TorrentHealthCheckIT.kt | 352 ++++++++++++++++++ 38 files changed, 1297 insertions(+), 88 deletions(-) create mode 100644 src/main/kotlin/io/skjaere/debridav/health/HealthQueueController.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/health/HealthQueueItemDto.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/health/HealthQueueService.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/health/PgmqHealthQueueRepository.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/health/RepairConfigurationProperties.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/health/RepairOutcome.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/health/RepairOutcomeService.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/pgmq/PgmqConfigurationProperties.kt rename src/main/kotlin/io/skjaere/debridav/{usenet => }/pgmq/PgmqConsumer.kt (96%) create mode 100644 src/main/kotlin/io/skjaere/debridav/pgmq/PgmqInfrastructureConfiguration.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/torrent/TorrentHealthCheckActuatorEndpoint.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/torrent/TorrentHealthCheckService.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/torrent/pgmq/Messages.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/torrent/pgmq/TorrentHealthCheckHandler.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/torrent/pgmq/TorrentHealthRepairHandler.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/torrent/pgmq/TorrentPgmqConfiguration.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqArchiveCleanupService.kt create mode 100644 src/main/resources/db/migration/V17__torrent_health_and_pgmq_queues.sql create mode 100644 src/main/resources/db/migration/V18__repair_outcome_table.sql create mode 100644 src/test/kotlin/io/skjaere/debridav/test/integrationtest/TorrentHealthCheckIT.kt diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 53d02932..82774c6c 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -19,7 +19,7 @@ logstash-logback = "7.4" hamcrest = "3.0" sardine = "5.13" sentry = "8.33.0" -nzb-streamer = "v0.7.3" +nzb-streamer = "v0.7.4" pgmq-kotlin = "0.1.0" mock-nntp-server = "v0.2.0" jjwt = "0.12.6" diff --git a/src/main/kotlin/io/skjaere/debridav/arrs/ArrService.kt b/src/main/kotlin/io/skjaere/debridav/arrs/ArrService.kt index 88002299..7a7406ba 100644 --- a/src/main/kotlin/io/skjaere/debridav/arrs/ArrService.kt +++ b/src/main/kotlin/io/skjaere/debridav/arrs/ArrService.kt @@ -13,11 +13,11 @@ class ArrService( fun getClientForCategory(category: String): ArrClient? = arrClients.firstOrNull { it.getCategory() == category } - suspend fun deleteFileAndSearch(itemName: String, category: String) { + suspend fun deleteFileAndSearch(itemName: String, category: String): Boolean { logger.info("Deleting file and triggering search for {} in Arrs", itemName) - getClientForCategory(category)?.let { client -> + return getClientForCategory(category)?.let { client -> client.deleteFileAndSearch(itemName) - } + } ?: false } suspend fun blocklist(downloadId: String, category: String) { diff --git a/src/main/kotlin/io/skjaere/debridav/arrs/client/ArrClient.kt b/src/main/kotlin/io/skjaere/debridav/arrs/client/ArrClient.kt index 3699f311..131a87fd 100644 --- a/src/main/kotlin/io/skjaere/debridav/arrs/client/ArrClient.kt +++ b/src/main/kotlin/io/skjaere/debridav/arrs/client/ArrClient.kt @@ -3,5 +3,5 @@ package io.skjaere.debridav.arrs.client interface ArrClient : BaseArrClient { suspend fun getItemIdFromName(name: String): Long? fun getCategory(): String - suspend fun deleteFileAndSearch(name: String) + suspend fun deleteFileAndSearch(name: String): Boolean } diff --git a/src/main/kotlin/io/skjaere/debridav/arrs/client/RadarrApiClient.kt b/src/main/kotlin/io/skjaere/debridav/arrs/client/RadarrApiClient.kt index ed202b5d..46b4ef53 100644 --- a/src/main/kotlin/io/skjaere/debridav/arrs/client/RadarrApiClient.kt +++ b/src/main/kotlin/io/skjaere/debridav/arrs/client/RadarrApiClient.kt @@ -36,10 +36,15 @@ class RadarrApiClient( override fun getCategory(): String = radarrConfigurationProperties.category - override suspend fun deleteFileAndSearch(name: String) { + override suspend fun deleteFileAndSearch(name: String): Boolean { val parseResponse = parse(name).body() val movie = parseResponse.movie + if (movie.id == 0L) { + logger.warn("No movie found for '{}' in Radarr", name) + return false + } + if (movie.movieFileId > 0) { logger.info("Deleting movie file {} for '{}'", movie.movieFileId, name) val deleteResponse = httpClient.delete( @@ -74,6 +79,7 @@ class RadarrApiClient( searchResponse.bodyAsText() ) } + return true } override val configurationClass: KClass<*> = RadarrConfigurationProperties::class diff --git a/src/main/kotlin/io/skjaere/debridav/arrs/client/SonarrApiClient.kt b/src/main/kotlin/io/skjaere/debridav/arrs/client/SonarrApiClient.kt index 68be1137..e56aba8b 100644 --- a/src/main/kotlin/io/skjaere/debridav/arrs/client/SonarrApiClient.kt +++ b/src/main/kotlin/io/skjaere/debridav/arrs/client/SonarrApiClient.kt @@ -36,12 +36,12 @@ class SonarrApiClient( override fun getCategory(): String = sonarrConfigurationProperties.category - override suspend fun deleteFileAndSearch(name: String) { + override suspend fun deleteFileAndSearch(name: String): Boolean { val parseResponse = parse(name).body() val episodes = parseResponse.episodes if (episodes.isEmpty()) { logger.warn("No episodes found for '{}' in Sonarr", name) - return + return false } episodes.filter { it.episodeFileId > 0 }.forEach { episode -> @@ -79,6 +79,7 @@ class SonarrApiClient( searchResponse.bodyAsText() ) } + return true } override val configurationClass: KClass<*> = SonarrConfigurationProperties::class diff --git a/src/main/kotlin/io/skjaere/debridav/configuration/DebridavConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/configuration/DebridavConfigurationProperties.kt index a5df6e00..a6c1af17 100644 --- a/src/main/kotlin/io/skjaere/debridav/configuration/DebridavConfigurationProperties.kt +++ b/src/main/kotlin/io/skjaere/debridav/configuration/DebridavConfigurationProperties.kt @@ -70,6 +70,20 @@ class DebridavConfigurationProperties { @ConfigProperty(name = "Max Local Entity Size (MB)", description = "Max local entity size in MB", advanced = true) var localEntityMaxSizeMb: Int = 0 + @ConfigProperty( + name = "Torrent Health Check Interval", + description = "How often to reverify torrent availability", + advanced = true + ) + var torrentHealthCheckInterval: Duration = Duration.ofDays(1) + + @ConfigProperty( + name = "Torrent Health Check Poll Rate", + description = "How often to poll for torrents needing health checks", + advanced = true + ) + var torrentHealthCheckPollRate: Duration = Duration.ofMinutes(5) + @ConfigProperty(name = "WebDAV Username", description = "WebDAV username", group = "webdav") var webdavUsername: String? = null diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/DebridLinkService.kt b/src/main/kotlin/io/skjaere/debridav/debrid/DebridLinkService.kt index f3f41a7a..f18e4c29 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/DebridLinkService.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/DebridLinkService.kt @@ -151,7 +151,7 @@ class DebridLinkService( } } - private suspend fun getFlowOfDebridLinks(debridFileContents: DebridFileContents): Flow = flow { + suspend fun getFlowOfDebridLinks(debridFileContents: DebridFileContents): Flow = flow { debridavConfigurationProperties.debridClients .map { debridClients.getClient(it) } .map { debridClient -> diff --git a/src/main/kotlin/io/skjaere/debridav/health/HealthQueueController.kt b/src/main/kotlin/io/skjaere/debridav/health/HealthQueueController.kt new file mode 100644 index 00000000..18627ce5 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/health/HealthQueueController.kt @@ -0,0 +1,36 @@ +package io.skjaere.debridav.health + +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RequestParam +import org.springframework.web.bind.annotation.RestController + +@RestController +@RequestMapping("/api/v1/health-queue") +class HealthQueueController(private val healthQueueService: HealthQueueService) { + + @GetMapping("/check") + fun getHealthCheckStatus(): ResponseEntity = + ResponseEntity.ok(healthQueueService.getHealthCheckStatus()) + + @GetMapping("/check/history") + fun getHealthCheckHistory( + @RequestParam(defaultValue = "0") page: Int, + @RequestParam(defaultValue = "20") size: Int, + @RequestParam(defaultValue = "") search: String + ): ResponseEntity = + ResponseEntity.ok(healthQueueService.getHealthCheckHistory(page, size, search)) + + @GetMapping("/repair") + fun getRepairStatus(): ResponseEntity = + ResponseEntity.ok(healthQueueService.getRepairStatus()) + + @GetMapping("/repair/history") + fun getRepairHistory( + @RequestParam(defaultValue = "0") page: Int, + @RequestParam(defaultValue = "20") size: Int, + @RequestParam(defaultValue = "") search: String + ): ResponseEntity = + ResponseEntity.ok(healthQueueService.getRepairHistory(page, size, search)) +} diff --git a/src/main/kotlin/io/skjaere/debridav/health/HealthQueueItemDto.kt b/src/main/kotlin/io/skjaere/debridav/health/HealthQueueItemDto.kt new file mode 100644 index 00000000..935fc5ee --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/health/HealthQueueItemDto.kt @@ -0,0 +1,31 @@ +package io.skjaere.debridav.health + +import java.time.Instant + +data class HealthQueueItemDto( + val msgId: Long, + val documentId: Long, + val name: String?, + val category: String?, + val type: String, + val readCount: Int, + val enqueuedAt: Instant?, + val lastReadAt: Instant?, + val archivedAt: Instant?, + val message: String?, + val action: String? = null +) + +data class HealthQueueStatusResponse( + val pending: List, + val count: Int +) + +data class HealthQueueHistoryResponse( + val content: List, + val page: Int, + val size: Int, + val totalElements: Long, + val totalPages: Int, + val last: Boolean +) diff --git a/src/main/kotlin/io/skjaere/debridav/health/HealthQueueService.kt b/src/main/kotlin/io/skjaere/debridav/health/HealthQueueService.kt new file mode 100644 index 00000000..6f2585bc --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/health/HealthQueueService.kt @@ -0,0 +1,23 @@ +package io.skjaere.debridav.health + +import org.springframework.stereotype.Service + +@Service +class HealthQueueService(private val repository: PgmqHealthQueueRepository) { + + fun getHealthCheckStatus(): HealthQueueStatusResponse { + val pending = repository.getPendingHealthChecks() + return HealthQueueStatusResponse(pending = pending, count = pending.size) + } + + fun getRepairStatus(): HealthQueueStatusResponse { + val pending = repository.getPendingRepairs() + return HealthQueueStatusResponse(pending = pending, count = pending.size) + } + + fun getHealthCheckHistory(page: Int, size: Int, search: String): HealthQueueHistoryResponse = + repository.getHealthCheckHistory(page, size, search) + + fun getRepairHistory(page: Int, size: Int, search: String): HealthQueueHistoryResponse = + repository.getRepairHistory(page, size, search) +} diff --git a/src/main/kotlin/io/skjaere/debridav/health/PgmqHealthQueueRepository.kt b/src/main/kotlin/io/skjaere/debridav/health/PgmqHealthQueueRepository.kt new file mode 100644 index 00000000..96b2b764 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/health/PgmqHealthQueueRepository.kt @@ -0,0 +1,216 @@ +package io.skjaere.debridav.health + +import org.springframework.jdbc.core.JdbcTemplate +import org.springframework.stereotype.Repository +import java.sql.ResultSet + +@Repository +class PgmqHealthQueueRepository(private val jdbc: JdbcTemplate) { + + fun getPendingHealthChecks(): List { + val nzbItems = getPendingNzb("nzb_health_check", false) + val torrentItems = getPendingTorrent("torrent_health_check", false) + return nzbItems + torrentItems + } + + fun getPendingRepairs(): List { + val nzbItems = getPendingNzb("nzb_health_repair", true) + val torrentItems = getPendingTorrent("torrent_health_repair", true) + return nzbItems + torrentItems + } + + fun getHealthCheckHistory(page: Int, size: Int, search: String): HealthQueueHistoryResponse { + val nzbHistory = getArchivedNzb("nzb_health_check", false, search) + val torrentHistory = getArchivedTorrent("torrent_health_check", false, search) + return paginateCombined(nzbHistory + torrentHistory, page, size) + } + + fun getRepairHistory(page: Int, size: Int, search: String): HealthQueueHistoryResponse { + val nzbHistory = getArchivedNzb("nzb_health_repair", true, search) + val torrentHistory = getArchivedTorrent("torrent_health_repair", true, search) + return paginateCombined(nzbHistory + torrentHistory, page, size) + } + + private fun getPendingNzb(queueName: String, hasMessage: Boolean): List { + if (!queueExists(queueName)) return emptyList() + val sql = """ + SELECT q.msg_id, + (q.message->>'nzbDocumentId')::bigint AS document_id, + ${if (hasMessage) "q.message->>'message' AS repair_message," else ""} + d.name AS doc_name, + d.category, + q.read_ct, + q.enqueued_at, + q.vt AS last_read_at + FROM pgmq.q_$queueName q + LEFT JOIN nzb_document d ON d.id = (q.message->>'nzbDocumentId')::bigint + ORDER BY q.msg_id + """.trimIndent() + + return jdbc.query(sql) { rs, _ -> + mapPendingRow(rs, hasMessage, "NZB") + } + } + + private fun getPendingTorrent(queueName: String, hasMessage: Boolean): List { + if (!queueExists(queueName)) return emptyList() + val sql = """ + SELECT q.msg_id, + (q.message->>'torrentId')::bigint AS document_id, + ${if (hasMessage) "q.message->>'message' AS repair_message," else ""} + t.name AS doc_name, + c.name AS category, + q.read_ct, + q.enqueued_at, + q.vt AS last_read_at + FROM pgmq.q_$queueName q + LEFT JOIN torrent t ON t.id = (q.message->>'torrentId')::bigint + LEFT JOIN category c ON c.id = t.category_id + ORDER BY q.msg_id + """.trimIndent() + + return jdbc.query(sql) { rs, _ -> + mapPendingRow(rs, hasMessage, "TORRENT") + } + } + + private fun getArchivedNzb( + queueName: String, + hasMessage: Boolean, + search: String + ): List { + if (!queueExists(queueName, archived = true)) return emptyList() + val searchClause = if (search.isNotBlank()) "AND LOWER(d.name) LIKE ?" else "" + val outcomeJoin = if (hasMessage) { + "LEFT JOIN repair_outcome ro ON ro.queue_name = '$queueName' AND ro.msg_id = q.msg_id" + } else "" + val outcomeSelect = if (hasMessage) ", ro.action AS repair_action" else "" + val sql = """ + SELECT q.msg_id, + (q.message->>'nzbDocumentId')::bigint AS document_id, + ${if (hasMessage) "q.message->>'message' AS repair_message," else ""} + d.name AS doc_name, + d.category, + q.read_ct, + q.enqueued_at, + q.vt AS last_read_at, + q.archived_at + $outcomeSelect + FROM pgmq.a_$queueName q + LEFT JOIN nzb_document d ON d.id = (q.message->>'nzbDocumentId')::bigint + $outcomeJoin + WHERE 1=1 $searchClause + ORDER BY q.archived_at DESC + """.trimIndent() + + val searchParam = if (search.isNotBlank()) "%${search.lowercase()}%" else null + return if (searchParam != null) { + jdbc.query(sql, { rs, _ -> mapArchivedRow(rs, hasMessage, "NZB") }, searchParam) + } else { + jdbc.query(sql) { rs, _ -> mapArchivedRow(rs, hasMessage, "NZB") } + } + } + + private fun getArchivedTorrent( + queueName: String, + hasMessage: Boolean, + search: String + ): List { + if (!queueExists(queueName, archived = true)) return emptyList() + val searchClause = if (search.isNotBlank()) "AND LOWER(t.name) LIKE ?" else "" + val outcomeJoin = if (hasMessage) { + "LEFT JOIN repair_outcome ro ON ro.queue_name = '$queueName' AND ro.msg_id = q.msg_id" + } else "" + val outcomeSelect = if (hasMessage) ", ro.action AS repair_action" else "" + val sql = """ + SELECT q.msg_id, + (q.message->>'torrentId')::bigint AS document_id, + ${if (hasMessage) "q.message->>'message' AS repair_message," else ""} + t.name AS doc_name, + c.name AS category, + q.read_ct, + q.enqueued_at, + q.vt AS last_read_at, + q.archived_at + $outcomeSelect + FROM pgmq.a_$queueName q + LEFT JOIN torrent t ON t.id = (q.message->>'torrentId')::bigint + LEFT JOIN category c ON c.id = t.category_id + $outcomeJoin + WHERE 1=1 $searchClause + ORDER BY q.archived_at DESC + """.trimIndent() + + val searchParam = if (search.isNotBlank()) "%${search.lowercase()}%" else null + return if (searchParam != null) { + jdbc.query(sql, { rs, _ -> mapArchivedRow(rs, hasMessage, "TORRENT") }, searchParam) + } else { + jdbc.query(sql) { rs, _ -> mapArchivedRow(rs, hasMessage, "TORRENT") } + } + } + + private fun paginateCombined( + all: List, + page: Int, + size: Int + ): HealthQueueHistoryResponse { + val sorted = all.sortedByDescending { it.archivedAt } + val totalElements = sorted.size.toLong() + val totalPages = if (size > 0) ((totalElements + size - 1) / size).toInt() else 0 + val offset = page * size + val items = sorted.drop(offset).take(size) + + return HealthQueueHistoryResponse( + content = items, + page = page, + size = size, + totalElements = totalElements, + totalPages = totalPages, + last = page >= totalPages - 1 + ) + } + + private fun queueExists(queueName: String, archived: Boolean = false): Boolean { + val prefix = if (archived) "a_" else "q_" + return try { + jdbc.queryForObject( + "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = 'pgmq' AND table_name = ?)", + Boolean::class.java, + "$prefix$queueName" + ) ?: false + } catch (@Suppress("TooGenericExceptionCaught") _: Exception) { + false + } + } + + private fun mapPendingRow(rs: ResultSet, hasMessage: Boolean, type: String): HealthQueueItemDto { + return HealthQueueItemDto( + msgId = rs.getLong("msg_id"), + documentId = rs.getLong("document_id"), + name = rs.getString("doc_name"), + category = rs.getString("category"), + type = type, + readCount = rs.getInt("read_ct"), + enqueuedAt = rs.getTimestamp("enqueued_at")?.toInstant(), + lastReadAt = rs.getTimestamp("last_read_at")?.toInstant(), + archivedAt = null, + message = if (hasMessage) rs.getString("repair_message") else null + ) + } + + private fun mapArchivedRow(rs: ResultSet, hasMessage: Boolean, type: String): HealthQueueItemDto { + return HealthQueueItemDto( + msgId = rs.getLong("msg_id"), + documentId = rs.getLong("document_id"), + name = rs.getString("doc_name"), + category = rs.getString("category"), + type = type, + readCount = rs.getInt("read_ct"), + enqueuedAt = rs.getTimestamp("enqueued_at")?.toInstant(), + lastReadAt = rs.getTimestamp("last_read_at")?.toInstant(), + archivedAt = rs.getTimestamp("archived_at")?.toInstant(), + message = if (hasMessage) rs.getString("repair_message") else null, + action = if (hasMessage) rs.getString("repair_action") else null + ) + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/health/RepairConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/health/RepairConfigurationProperties.kt new file mode 100644 index 00000000..f35275f3 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/health/RepairConfigurationProperties.kt @@ -0,0 +1,10 @@ +package io.skjaere.debridav.health + +import io.skjaere.debridav.config.ConfigProperty +import org.springframework.boot.context.properties.ConfigurationProperties + +@ConfigurationProperties(prefix = "repair") +class RepairConfigurationProperties { + @ConfigProperty(name = "Enabled", description = "Enable automatic repair of unhealthy torrents and NZBs") + var enabled: Boolean = true +} diff --git a/src/main/kotlin/io/skjaere/debridav/health/RepairOutcome.kt b/src/main/kotlin/io/skjaere/debridav/health/RepairOutcome.kt new file mode 100644 index 00000000..9fc982a8 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/health/RepairOutcome.kt @@ -0,0 +1,43 @@ +package io.skjaere.debridav.health + +import jakarta.persistence.Column +import jakarta.persistence.Entity +import jakarta.persistence.EnumType +import jakarta.persistence.Enumerated +import jakarta.persistence.GeneratedValue +import jakarta.persistence.GenerationType +import jakarta.persistence.Id +import jakarta.persistence.Table +import org.springframework.data.repository.CrudRepository +import org.springframework.stereotype.Repository +import java.time.Instant + +enum class RepairAction { + REPAIRED, + DELETED, + SKIPPED +} + +@Entity +@Table(name = "repair_outcome") +open class RepairOutcome { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + open var id: Long? = null + + @Column(name = "queue_name", nullable = false) + open var queueName: String? = null + + @Column(name = "msg_id", nullable = false) + open var msgId: Long? = null + + @Enumerated(EnumType.STRING) + @Column(nullable = false) + open var action: RepairAction? = null + + @Column(name = "created_at", nullable = false) + open var createdAt: Instant = Instant.now() +} + +@Repository +interface RepairOutcomeRepository : CrudRepository diff --git a/src/main/kotlin/io/skjaere/debridav/health/RepairOutcomeService.kt b/src/main/kotlin/io/skjaere/debridav/health/RepairOutcomeService.kt new file mode 100644 index 00000000..52b90816 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/health/RepairOutcomeService.kt @@ -0,0 +1,19 @@ +package io.skjaere.debridav.health + +import org.springframework.stereotype.Service +import org.springframework.transaction.annotation.Propagation +import org.springframework.transaction.annotation.Transactional + +@Service +class RepairOutcomeService( + private val repository: RepairOutcomeRepository +) { + @Transactional(propagation = Propagation.REQUIRES_NEW) + fun record(queueName: String, msgId: Long, action: RepairAction) { + val outcome = RepairOutcome() + outcome.queueName = queueName + outcome.msgId = msgId + outcome.action = action + repository.save(outcome) + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/pgmq/PgmqConfigurationProperties.kt b/src/main/kotlin/io/skjaere/debridav/pgmq/PgmqConfigurationProperties.kt new file mode 100644 index 00000000..79abde35 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/pgmq/PgmqConfigurationProperties.kt @@ -0,0 +1,26 @@ +package io.skjaere.debridav.pgmq + +import org.springframework.boot.context.properties.ConfigurationProperties +import java.time.Duration + +@Suppress("MagicNumber") +@ConfigurationProperties(prefix = "pgmq") +class PgmqConfigurationProperties { + var defaultVisibilityTimeout: Duration = Duration.ofMinutes(5) + var importConcurrency: Int = 2 + var importVisibilityTimeout: Duration = Duration.ofMinutes(10) + var importPollInterval: Duration = Duration.ofSeconds(2) + var healthCheckConcurrency: Int = 1 + var healthCheckVisibilityTimeout: Duration = Duration.ofMinutes(5) + var healthCheckPollInterval: Duration = Duration.ofSeconds(10) + var healthRepairConcurrency: Int = 2 + var healthRepairVisibilityTimeout: Duration = Duration.ofMinutes(2) + var healthRepairPollInterval: Duration = Duration.ofSeconds(5) + var archiveRetention: Duration = Duration.ofDays(30) + var torrentHealthCheckConcurrency: Int = 1 + var torrentHealthCheckVisibilityTimeout: Duration = Duration.ofMinutes(5) + var torrentHealthCheckPollInterval: Duration = Duration.ofSeconds(10) + var torrentHealthRepairConcurrency: Int = 2 + var torrentHealthRepairVisibilityTimeout: Duration = Duration.ofMinutes(2) + var torrentHealthRepairPollInterval: Duration = Duration.ofSeconds(5) +} diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqConsumer.kt b/src/main/kotlin/io/skjaere/debridav/pgmq/PgmqConsumer.kt similarity index 96% rename from src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqConsumer.kt rename to src/main/kotlin/io/skjaere/debridav/pgmq/PgmqConsumer.kt index fe754506..1d301ffe 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqConsumer.kt +++ b/src/main/kotlin/io/skjaere/debridav/pgmq/PgmqConsumer.kt @@ -1,4 +1,4 @@ -package io.skjaere.debridav.usenet.pgmq +package io.skjaere.debridav.pgmq import com.fasterxml.jackson.databind.ObjectMapper import com.vdsirotkin.pgmq.PgmqClient @@ -19,7 +19,7 @@ class PgmqConsumer( private val concurrency: Int, private val visibilityTimeout: java.time.Duration, private val pollInterval: java.time.Duration, - private val handler: (T) -> Unit + private val handler: (T, Long) -> Unit ) : SmartLifecycle { private val logger = LoggerFactory.getLogger(PgmqConsumer::class.java) @@ -56,7 +56,7 @@ class PgmqConsumer( val entry = entries.first() try { val message = objectMapper.readValue(entry.message, messageType) - handler(message) + handler(message, entry.messageId) pgmqClient.archive(queueName, entry.messageId) } catch (@Suppress("TooGenericExceptionCaught") e: Exception) { logger.error( diff --git a/src/main/kotlin/io/skjaere/debridav/pgmq/PgmqInfrastructureConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/pgmq/PgmqInfrastructureConfiguration.kt new file mode 100644 index 00000000..90e9d2ad --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/pgmq/PgmqInfrastructureConfiguration.kt @@ -0,0 +1,41 @@ +package io.skjaere.debridav.pgmq + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.KotlinModule +import com.vdsirotkin.pgmq.PgmqClient +import com.vdsirotkin.pgmq.config.PgmqConfiguration +import com.vdsirotkin.pgmq.config.PgmqConnectionFactory +import com.vdsirotkin.pgmq.serialization.JacksonPgmqSerializationProvider +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import javax.sql.DataSource + +@Configuration +class PgmqInfrastructureConfiguration { + + @Bean + fun pgmqConfiguration(props: PgmqConfigurationProperties): PgmqConfiguration = + object : PgmqConfiguration { + override val defaultVisibilityTimeout: java.time.Duration = props.defaultVisibilityTimeout + } + + @Bean + fun pgmqConnectionFactory(dataSource: DataSource): PgmqConnectionFactory = PgmqConnectionFactory { + dataSource.connection + } + + @Bean + fun pgmqObjectMapper(): ObjectMapper = + ObjectMapper().registerModule(KotlinModule.Builder().build()) + + @Bean + fun pgmqSerializationProvider(pgmqObjectMapper: ObjectMapper): JacksonPgmqSerializationProvider = + JacksonPgmqSerializationProvider(pgmqObjectMapper) + + @Bean + fun pgmqClient( + connectionFactory: PgmqConnectionFactory, + serializationProvider: JacksonPgmqSerializationProvider, + configuration: PgmqConfiguration + ): PgmqClient = PgmqClient(connectionFactory, serializationProvider, configuration) +} diff --git a/src/main/kotlin/io/skjaere/debridav/torrent/Torrent.kt b/src/main/kotlin/io/skjaere/debridav/torrent/Torrent.kt index 12b36a2e..731820f5 100644 --- a/src/main/kotlin/io/skjaere/debridav/torrent/Torrent.kt +++ b/src/main/kotlin/io/skjaere/debridav/torrent/Torrent.kt @@ -37,6 +37,12 @@ open class Torrent { @Column(nullable = false, length = 2048) open var savePath: String? = null open var status: Status = Status.LIVE + + @Column(name = "last_verified") + open var lastVerified: Instant? = null + + @Column(name = "health_check_enqueued_at") + open var healthCheckEnqueuedAt: Instant? = null } enum class Status { LIVE, DELETED } diff --git a/src/main/kotlin/io/skjaere/debridav/torrent/TorrentHealthCheckActuatorEndpoint.kt b/src/main/kotlin/io/skjaere/debridav/torrent/TorrentHealthCheckActuatorEndpoint.kt new file mode 100644 index 00000000..4ce618f0 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/torrent/TorrentHealthCheckActuatorEndpoint.kt @@ -0,0 +1,16 @@ +package io.skjaere.debridav.torrent + +import org.springframework.boot.actuate.endpoint.annotation.Endpoint +import org.springframework.boot.actuate.endpoint.annotation.WriteOperation +import org.springframework.stereotype.Component + +@Component +@Endpoint(id = "torrenthealthcheck") +class TorrentHealthCheckActuatorEndpoint( + private val torrentHealthCheckService: TorrentHealthCheckService +) { + @WriteOperation + fun triggerHealthCheck() { + torrentHealthCheckService.triggerFullHealthCheck() + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/torrent/TorrentHealthCheckService.kt b/src/main/kotlin/io/skjaere/debridav/torrent/TorrentHealthCheckService.kt new file mode 100644 index 00000000..3574dcfb --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/torrent/TorrentHealthCheckService.kt @@ -0,0 +1,60 @@ +package io.skjaere.debridav.torrent + +import com.vdsirotkin.pgmq.PgmqClient +import io.skjaere.debridav.configuration.DebridavConfigurationProperties +import io.skjaere.debridav.torrent.pgmq.TorrentHealthCheckMessage +import org.slf4j.LoggerFactory +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Service +import org.springframework.transaction.annotation.Transactional +import java.time.Clock +import java.time.Instant + +@Service +class TorrentHealthCheckService( + private val torrentRepository: TorrentRepository, + private val debridavConfigurationProperties: DebridavConfigurationProperties, + private val pgmqClient: PgmqClient, + private val clock: Clock +) { + private val logger = LoggerFactory.getLogger(TorrentHealthCheckService::class.java) + + @Scheduled(fixedDelayString = "\${debridav.torrent-health-check-poll-rate:PT5M}") + @Transactional + fun checkTorrentHealth() { + val now = Instant.now(clock) + val cutoff = now.minus(debridavConfigurationProperties.torrentHealthCheckInterval) + val enqueueCutoff = now.minus(debridavConfigurationProperties.torrentHealthCheckInterval) + + val torrentsToVerify = torrentRepository + .findByStatusAndLastVerifiedIsNullOrStatusAndLastVerifiedBefore( + Status.LIVE, Status.LIVE, cutoff + ) + .filter { + it.healthCheckEnqueuedAt == null || it.healthCheckEnqueuedAt!!.isBefore(enqueueCutoff) + } + + if (torrentsToVerify.isEmpty()) return + + logger.debug("Health check: enqueuing {} torrent(s) for verification", torrentsToVerify.size) + + torrentsToVerify.forEach { torrent -> + pgmqClient.send("torrent_health_check", TorrentHealthCheckMessage(torrent.id!!)) + torrent.healthCheckEnqueuedAt = now + torrentRepository.save(torrent) + } + } + + fun triggerFullHealthCheck() { + val torrents = torrentRepository.findByStatus(Status.LIVE) + val now = Instant.now(clock) + + logger.info("Triggering full health check for all {} live torrents", torrents.size) + + torrents.forEach { torrent -> + pgmqClient.send("torrent_health_check", TorrentHealthCheckMessage(torrent.id!!)) + torrent.healthCheckEnqueuedAt = now + torrentRepository.save(torrent) + } + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/torrent/TorrentRepository.kt b/src/main/kotlin/io/skjaere/debridav/torrent/TorrentRepository.kt index 765df5ee..b9555a7f 100644 --- a/src/main/kotlin/io/skjaere/debridav/torrent/TorrentRepository.kt +++ b/src/main/kotlin/io/skjaere/debridav/torrent/TorrentRepository.kt @@ -6,6 +6,7 @@ import org.springframework.data.jpa.repository.Modifying import org.springframework.data.jpa.repository.Query import org.springframework.data.repository.CrudRepository import org.springframework.stereotype.Repository +import java.time.Instant @Repository interface TorrentRepository : CrudRepository { @@ -20,4 +21,12 @@ interface TorrentRepository : CrudRepository { fun markTorrentAsDeleted(torrent: Torrent) fun getTorrentByFilesContains(file: DbEntity): List + + fun findByStatusAndLastVerifiedIsNullOrStatusAndLastVerifiedBefore( + status1: Status, + status2: Status, + cutoff: Instant + ): List + + fun findByStatus(status: Status): List } diff --git a/src/main/kotlin/io/skjaere/debridav/torrent/pgmq/Messages.kt b/src/main/kotlin/io/skjaere/debridav/torrent/pgmq/Messages.kt new file mode 100644 index 00000000..f965f5d4 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/torrent/pgmq/Messages.kt @@ -0,0 +1,10 @@ +package io.skjaere.debridav.torrent.pgmq + +data class TorrentHealthCheckMessage( + val torrentId: Long +) + +data class TorrentHealthRepairMessage( + val torrentId: Long, + val message: String +) diff --git a/src/main/kotlin/io/skjaere/debridav/torrent/pgmq/TorrentHealthCheckHandler.kt b/src/main/kotlin/io/skjaere/debridav/torrent/pgmq/TorrentHealthCheckHandler.kt new file mode 100644 index 00000000..ad9a5eaf --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/torrent/pgmq/TorrentHealthCheckHandler.kt @@ -0,0 +1,76 @@ +package io.skjaere.debridav.torrent.pgmq + +import com.vdsirotkin.pgmq.PgmqClient +import io.skjaere.debridav.debrid.DebridLinkService +import io.skjaere.debridav.fs.MissingFile +import io.skjaere.debridav.fs.ProviderError +import io.skjaere.debridav.torrent.TorrentRepository +import kotlinx.coroutines.flow.firstOrNull +import kotlinx.coroutines.runBlocking +import org.slf4j.LoggerFactory +import org.springframework.stereotype.Service +import org.springframework.transaction.annotation.Transactional +import java.time.Clock +import java.time.Instant + +@Service +class TorrentHealthCheckHandler( + private val torrentRepository: TorrentRepository, + private val debridLinkService: DebridLinkService, + private val pgmqClient: PgmqClient, + private val clock: Clock +) { + private val logger = LoggerFactory.getLogger(TorrentHealthCheckHandler::class.java) + + @Transactional + fun handle(msg: TorrentHealthCheckMessage) { + val torrent = torrentRepository.findById(msg.torrentId).orElse(null) + if (torrent == null) { + logger.warn("Torrent {} not found, skipping health check", msg.torrentId) + return + } + + try { + val files = torrent.files + if (files.isEmpty()) { + logger.debug("Torrent {} has no files, skipping health check", torrent.id) + return + } + + val unhealthy = files.any { file -> + val contents = file.contents ?: return@any false + val healthyLink = runBlocking { + debridLinkService.getFlowOfDebridLinks(contents) + .firstOrNull { it !is MissingFile && it !is ProviderError } + } + val allUnavailable = healthyLink == null + if (allUnavailable) { + logger.warn( + "Torrent {} file '{}' is unhealthy — all providers returned MissingFile or ProviderError", + torrent.id, file.name + ) + } + allUnavailable + } + + if (unhealthy) { + logger.warn("Torrent {} '{}' is unhealthy, enqueuing for repair", torrent.id, torrent.name) + pgmqClient.send( + "torrent_health_repair", + TorrentHealthRepairMessage( + torrentId = torrent.id!!, + message = "One or more files unavailable from all debrid providers" + ) + ) + } else { + logger.debug("Torrent {} verified successfully", torrent.id) + } + } catch (@Suppress("TooGenericExceptionCaught") e: Exception) { + logger.error("Unexpected error verifying torrent {}", torrent.id, e) + } + + torrent.lastVerified = Instant.now(clock) + torrent.healthCheckEnqueuedAt = null + torrentRepository.save(torrent) + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/torrent/pgmq/TorrentHealthRepairHandler.kt b/src/main/kotlin/io/skjaere/debridav/torrent/pgmq/TorrentHealthRepairHandler.kt new file mode 100644 index 00000000..02b8a01a --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/torrent/pgmq/TorrentHealthRepairHandler.kt @@ -0,0 +1,93 @@ +package io.skjaere.debridav.torrent.pgmq + +import io.skjaere.debridav.arrs.ArrService +import io.skjaere.debridav.fs.DatabaseFileService +import io.skjaere.debridav.health.RepairAction +import io.skjaere.debridav.health.RepairConfigurationProperties +import io.skjaere.debridav.health.RepairOutcomeService +import io.skjaere.debridav.torrent.TorrentRepository +import kotlinx.coroutines.runBlocking +import org.slf4j.LoggerFactory +import org.springframework.stereotype.Service +import org.springframework.transaction.annotation.Transactional + +@Service +class TorrentHealthRepairHandler( + private val torrentRepository: TorrentRepository, + private val arrService: ArrService, + private val fileService: DatabaseFileService, + private val repairConfig: RepairConfigurationProperties, + private val repairOutcomeService: RepairOutcomeService +) { + private val logger = LoggerFactory.getLogger(TorrentHealthRepairHandler::class.java) + + @Transactional + fun handle(msg: TorrentHealthRepairMessage, msgId: Long) { + if (!repairConfig.enabled) { + logger.debug("Repair is disabled, skipping torrent {}", msg.torrentId) + repairOutcomeService.record(QUEUE_NAME, msgId, RepairAction.SKIPPED) + return + } + + val torrent = torrentRepository.findById(msg.torrentId).orElse(null) + if (torrent == null) { + logger.warn("No torrent found for ID {}", msg.torrentId) + return + } + + val category = torrent.category?.name + val hash = torrent.hash + + if (category != null && hash != null && arrService.getClientForCategory(category) != null) { + logger.info( + "Blocklisting torrent hash '{}' for '{}' (category: {})", + hash, torrent.name, category + ) + runBlocking { arrService.blocklist(hash, category) } + + var anyDeleted = false + var anyRepaired = false + torrent.files.forEach { file -> + val fileName = file.name + if (fileName != null) { + logger.info( + "Notifying Arr to delete file and search for '{}' (category: {})", + fileName, category + ) + val found = runBlocking { arrService.deleteFileAndSearch(fileName, category) } + if (!found) { + logger.info( + "Arr could not find '{}', deleting from virtual filesystem", + fileName + ) + fileService.deleteFile(file) + anyDeleted = true + } else { + anyRepaired = true + } + } + } + val action = when { + anyRepaired && !anyDeleted -> RepairAction.REPAIRED + !anyRepaired && anyDeleted -> RepairAction.DELETED + anyRepaired -> RepairAction.REPAIRED + else -> RepairAction.DELETED + } + repairOutcomeService.record(QUEUE_NAME, msgId, action) + } else { + logger.info( + "No Arr client for torrent {} (category: {}), deleting all files from virtual filesystem", + torrent.id, category + ) + torrent.files.forEach { file -> + logger.info("Deleting '{}' from virtual filesystem", file.name) + fileService.deleteFile(file) + } + repairOutcomeService.record(QUEUE_NAME, msgId, RepairAction.DELETED) + } + } + + companion object { + const val QUEUE_NAME = "torrent_health_repair" + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/torrent/pgmq/TorrentPgmqConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/torrent/pgmq/TorrentPgmqConfiguration.kt new file mode 100644 index 00000000..ba4b4050 --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/torrent/pgmq/TorrentPgmqConfiguration.kt @@ -0,0 +1,48 @@ +package io.skjaere.debridav.torrent.pgmq + +import com.fasterxml.jackson.databind.ObjectMapper +import com.vdsirotkin.pgmq.PgmqClient +import io.skjaere.debridav.pgmq.PgmqConfigurationProperties +import io.skjaere.debridav.pgmq.PgmqConsumer +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration + +@Configuration +class TorrentPgmqConfiguration { + + @Bean + fun torrentHealthCheckConsumer( + pgmqClient: PgmqClient, + pgmqObjectMapper: ObjectMapper, + props: PgmqConfigurationProperties, + handler: TorrentHealthCheckHandler + ): PgmqConsumer = PgmqConsumer( + pgmqClient = pgmqClient, + objectMapper = pgmqObjectMapper, + queueName = "torrent_health_check", + messageType = TorrentHealthCheckMessage::class.java, + concurrency = props.torrentHealthCheckConcurrency, + visibilityTimeout = props.torrentHealthCheckVisibilityTimeout, + pollInterval = props.torrentHealthCheckPollInterval + ) { msg, _ -> + handler.handle(msg) + } + + @Bean + fun torrentHealthRepairConsumer( + pgmqClient: PgmqClient, + pgmqObjectMapper: ObjectMapper, + props: PgmqConfigurationProperties, + handler: TorrentHealthRepairHandler + ): PgmqConsumer = PgmqConsumer( + pgmqClient = pgmqClient, + objectMapper = pgmqObjectMapper, + queueName = "torrent_health_repair", + messageType = TorrentHealthRepairMessage::class.java, + concurrency = props.torrentHealthRepairConcurrency, + visibilityTimeout = props.torrentHealthRepairVisibilityTimeout, + pollInterval = props.torrentHealthRepairPollInterval + ) { msg, msgId -> + handler.handle(msg, msgId) + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/NzbImportService.kt b/src/main/kotlin/io/skjaere/debridav/usenet/NzbImportService.kt index 64a4587e..d82081c2 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/NzbImportService.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/NzbImportService.kt @@ -126,8 +126,8 @@ class NzbImportService( usenetDownload.name, result.message ) - usenetDownload.status = UsenetDownloadStatus.ARTICLES_MISSING - importRecord.status = NzbImportStatus.ARTICLES_MISSING + usenetDownload.status = UsenetDownloadStatus.FAILED + importRecord.status = NzbImportStatus.FAILED importRecord.errorMessage = result.message } diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/UsenetDownload.kt b/src/main/kotlin/io/skjaere/debridav/usenet/UsenetDownload.kt index 7e950697..c79dc403 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/UsenetDownload.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/UsenetDownload.kt @@ -44,9 +44,9 @@ open class UsenetDownload { enum class UsenetDownloadStatus { CREATED, QUEUED, DOWNLOADING, EXTRACTING, COMPLETED, FAILED, VERIFYING, - DELETED, CACHED, REPAIRING, POST_PROCESSING, VALIDATING, ARTICLES_MISSING; + DELETED, CACHED, REPAIRING, POST_PROCESSING, VALIDATING; - fun isCompleted(): Boolean = this == COMPLETED || this == CACHED || this == FAILED || this == ARTICLES_MISSING + fun isCompleted(): Boolean = this == COMPLETED || this == CACHED || this == FAILED } enum class SabnzbdUsenetDownloadStatus { @@ -67,7 +67,6 @@ enum class SabnzbdUsenetDownloadStatus { UsenetDownloadStatus.REPAIRING -> REPAIRING UsenetDownloadStatus.VALIDATING -> VERIFYING UsenetDownloadStatus.POST_PROCESSING -> VERIFYING - UsenetDownloadStatus.ARTICLES_MISSING -> FAILED } } diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/NzbHealthRepairHandler.kt b/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/NzbHealthRepairHandler.kt index d2f81f3d..58bddd90 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/NzbHealthRepairHandler.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/NzbHealthRepairHandler.kt @@ -1,21 +1,38 @@ package io.skjaere.debridav.usenet.pgmq import io.skjaere.debridav.arrs.ArrService +import io.skjaere.debridav.fs.DatabaseFileService +import io.skjaere.debridav.health.RepairAction +import io.skjaere.debridav.health.RepairConfigurationProperties +import io.skjaere.debridav.health.RepairOutcomeService import io.skjaere.debridav.repository.NzbDocumentRepository +import io.skjaere.debridav.repository.UsenetRepository import kotlinx.coroutines.runBlocking import org.slf4j.LoggerFactory import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty import org.springframework.stereotype.Service +import org.springframework.transaction.annotation.Transactional @Service @ConditionalOnProperty("nntp.enabled", havingValue = "true") class NzbHealthRepairHandler( private val nzbDocumentRepository: NzbDocumentRepository, - private val arrService: ArrService + private val usenetRepository: UsenetRepository, + private val arrService: ArrService, + private val fileService: DatabaseFileService, + private val repairConfig: RepairConfigurationProperties, + private val repairOutcomeService: RepairOutcomeService ) { private val logger = LoggerFactory.getLogger(NzbHealthRepairHandler::class.java) - fun handle(msg: NzbHealthRepairMessage) { + @Transactional + fun handle(msg: NzbHealthRepairMessage, msgId: Long) { + if (!repairConfig.enabled) { + logger.debug("Repair is disabled, skipping NZB document {}", msg.nzbDocumentId) + repairOutcomeService.record(QUEUE_NAME, msgId, RepairAction.SKIPPED) + return + } + val nzbDocument = nzbDocumentRepository.findById(msg.nzbDocumentId).orElse(null) if (nzbDocument == null) { logger.warn("No NzbDocument found for ID {}", msg.nzbDocumentId) @@ -29,6 +46,8 @@ class NzbHealthRepairHandler( "NzbDocument {} missing category or name, cannot notify Arr", nzbDocument.id ) + deleteVirtualFiles(nzbDocument.id!!) + repairOutcomeService.record(QUEUE_NAME, msgId, RepairAction.DELETED) return } @@ -37,9 +56,7 @@ class NzbHealthRepairHandler( if (downloadId != null) { logger.info( "Blocklisting downloadId '{}' for '{}' (category: {})", - downloadId, - name, - category + downloadId, name, category ) runBlocking { arrService.blocklist(downloadId, category) } } else { @@ -51,10 +68,42 @@ class NzbHealthRepairHandler( logger.info( "Notifying Arr to delete file and search for '{}' (category: {})", - name, - category + name, category ) - runBlocking { arrService.deleteFileAndSearch(name, category) } + val found = runBlocking { arrService.deleteFileAndSearch(name, category) } + if (!found) { + logger.info( + "Arr could not find '{}', deleting from virtual filesystem", + name + ) + deleteVirtualFiles(nzbDocument.id!!) + repairOutcomeService.record(QUEUE_NAME, msgId, RepairAction.DELETED) + } else { + repairOutcomeService.record(QUEUE_NAME, msgId, RepairAction.REPAIRED) + } + } else { + logger.info( + "No Arr client for NZB {} (category: {}), deleting files from virtual filesystem", + nzbDocument.id, category + ) + deleteVirtualFiles(nzbDocument.id!!) + repairOutcomeService.record(QUEUE_NAME, msgId, RepairAction.DELETED) } } + + private fun deleteVirtualFiles(nzbDocumentId: Long) { + val usenetDownload = usenetRepository.findByNzbDocumentId(nzbDocumentId) + if (usenetDownload != null) { + usenetDownload.debridFiles.forEach { file -> + logger.info("Deleting '{}' from virtual filesystem", file.name) + fileService.deleteFile(file) + } + } else { + logger.warn("No UsenetDownload found for NzbDocument {}, cannot delete virtual files", nzbDocumentId) + } + } + + companion object { + const val QUEUE_NAME = "nzb_health_repair" + } } diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqArchiveCleanupService.kt b/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqArchiveCleanupService.kt new file mode 100644 index 00000000..be3db2ad --- /dev/null +++ b/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqArchiveCleanupService.kt @@ -0,0 +1,48 @@ +package io.skjaere.debridav.usenet.pgmq + +import io.skjaere.debridav.pgmq.PgmqConfigurationProperties +import org.slf4j.LoggerFactory +import org.springframework.jdbc.core.JdbcTemplate +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Service +import java.sql.Timestamp +import java.time.Instant + +@Service +class PgmqArchiveCleanupService( + private val jdbc: JdbcTemplate, + private val props: PgmqConfigurationProperties +) { + private val logger = LoggerFactory.getLogger(PgmqArchiveCleanupService::class.java) + + companion object { + private val QUEUE_NAMES = listOf( + "nzb_import", "nzb_health_check", "nzb_health_repair", + "torrent_health_check", "torrent_health_repair" + ) + } + + @Scheduled(fixedDelayString = "PT1H", initialDelayString = "PT1M") + fun cleanupArchivedMessages() { + val cutoff = Instant.now().minus(props.archiveRetention) + logger.debug("Cleaning up archived PGMQ messages older than {}", cutoff) + + var totalDeleted = 0L + for (queueName in QUEUE_NAMES) { + val deleted = jdbc.update( + "DELETE FROM pgmq.a_$queueName WHERE archived_at < ?", + Timestamp.from(cutoff) + ) + if (deleted > 0) { + logger.info("Deleted {} archived messages from queue '{}'", deleted, queueName) + totalDeleted += deleted + } + } + + if (totalDeleted > 0) { + logger.info("Archive cleanup complete: {} total messages removed", totalDeleted) + } else { + logger.debug("Archive cleanup complete: no expired messages found") + } + } +} diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqSpringConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqSpringConfiguration.kt index 8fa4feb8..0c18ec0a 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqSpringConfiguration.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/pgmq/PgmqSpringConfiguration.kt @@ -1,65 +1,19 @@ package io.skjaere.debridav.usenet.pgmq import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.kotlin.KotlinModule import com.vdsirotkin.pgmq.PgmqClient -import com.vdsirotkin.pgmq.config.PgmqConfiguration -import com.vdsirotkin.pgmq.config.PgmqConnectionFactory -import com.vdsirotkin.pgmq.serialization.JacksonPgmqSerializationProvider +import io.skjaere.debridav.pgmq.PgmqConfigurationProperties +import io.skjaere.debridav.pgmq.PgmqConsumer import io.skjaere.debridav.usenet.NzbImportService import io.skjaere.debridav.usenet.NzbImportTaskData import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty -import org.springframework.boot.context.properties.ConfigurationProperties import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration -import java.time.Duration -import javax.sql.DataSource - -@Suppress("MagicNumber") -@ConfigurationProperties(prefix = "pgmq") -class PgmqConfigurationProperties { - var defaultVisibilityTimeout: Duration = Duration.ofMinutes(5) - var importConcurrency: Int = 2 - var importVisibilityTimeout: Duration = Duration.ofMinutes(10) - var importPollInterval: Duration = Duration.ofSeconds(2) - var healthCheckConcurrency: Int = 1 - var healthCheckVisibilityTimeout: Duration = Duration.ofMinutes(5) - var healthCheckPollInterval: Duration = Duration.ofSeconds(10) - var healthRepairConcurrency: Int = 2 - var healthRepairVisibilityTimeout: Duration = Duration.ofMinutes(2) - var healthRepairPollInterval: Duration = Duration.ofSeconds(5) -} @Configuration @ConditionalOnProperty("nntp.enabled", havingValue = "true") class PgmqSpringConfiguration { - @Bean - fun pgmqConfiguration(props: PgmqConfigurationProperties): PgmqConfiguration = - object : PgmqConfiguration { - override val defaultVisibilityTimeout: java.time.Duration = props.defaultVisibilityTimeout - } - - @Bean - fun pgmqConnectionFactory(dataSource: DataSource): PgmqConnectionFactory = PgmqConnectionFactory { - dataSource.connection - } - - @Bean - fun pgmqObjectMapper(): ObjectMapper = - ObjectMapper().registerModule(KotlinModule.Builder().build()) - - @Bean - fun pgmqSerializationProvider(pgmqObjectMapper: ObjectMapper): JacksonPgmqSerializationProvider = - JacksonPgmqSerializationProvider(pgmqObjectMapper) - - @Bean - fun pgmqClient( - connectionFactory: PgmqConnectionFactory, - serializationProvider: JacksonPgmqSerializationProvider, - configuration: PgmqConfiguration - ): PgmqClient = PgmqClient(connectionFactory, serializationProvider, configuration) - @Bean fun nzbImportConsumer( pgmqClient: PgmqClient, @@ -74,7 +28,7 @@ class PgmqSpringConfiguration { concurrency = props.importConcurrency, visibilityTimeout = props.importVisibilityTimeout, pollInterval = props.importPollInterval - ) { msg -> + ) { msg, _ -> nzbImportService.executeImport( NzbImportTaskData(msg.nzbBytesBase64, msg.usenetDownloadId, msg.nzbImportRecordId) ) @@ -94,7 +48,7 @@ class PgmqSpringConfiguration { concurrency = props.healthCheckConcurrency, visibilityTimeout = props.healthCheckVisibilityTimeout, pollInterval = props.healthCheckPollInterval - ) { msg -> + ) { msg, _ -> healthCheckHandler.handle(msg) } @@ -112,7 +66,7 @@ class PgmqSpringConfiguration { concurrency = props.healthRepairConcurrency, visibilityTimeout = props.healthRepairVisibilityTimeout, pollInterval = props.healthRepairPollInterval - ) { msg -> - healthRepairHandler.handle(msg) + ) { msg, msgId -> + healthRepairHandler.handle(msg, msgId) } } diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/queue/NzbImportRecord.kt b/src/main/kotlin/io/skjaere/debridav/usenet/queue/NzbImportRecord.kt index f6687632..ba618f8f 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/queue/NzbImportRecord.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/queue/NzbImportRecord.kt @@ -68,8 +68,7 @@ enum class NzbImportStatus { QUEUED, IMPORTING, COMPLETED, - FAILED, - ARTICLES_MISSING + FAILED } data class NzbImportFileJson( diff --git a/src/main/kotlin/io/skjaere/debridav/usenet/queue/UsenetQueueService.kt b/src/main/kotlin/io/skjaere/debridav/usenet/queue/UsenetQueueService.kt index f5a72a7f..210f7f5a 100644 --- a/src/main/kotlin/io/skjaere/debridav/usenet/queue/UsenetQueueService.kt +++ b/src/main/kotlin/io/skjaere/debridav/usenet/queue/UsenetQueueService.kt @@ -21,8 +21,7 @@ class UsenetQueueService( val HISTORY_STATUSES = listOf( NzbImportStatus.COMPLETED, - NzbImportStatus.FAILED, - NzbImportStatus.ARTICLES_MISSING + NzbImportStatus.FAILED ) private val ALLOWED_SORT_FIELDS = setOf("updatedAt", "name") diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 73b4947e..6d611b00 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -35,7 +35,7 @@ management: endpoints: web: exposure: - include: health,realdebrid,prometheus,nzbhealthcheck + include: health,realdebrid,prometheus,nzbhealthcheck,torrenthealthcheck endpoint: health: group: @@ -62,6 +62,8 @@ debridav: local-entity-max-size-mb: 130 default-categories: torrent-lifetime: 1d + torrent-health-check-interval: P1D + torrent-health-check-poll-rate: PT5M webdav-username: webdav-password: db: @@ -76,6 +78,9 @@ debridav: protect-sabnzbd-api: false protect-actuator: false +repair: + enabled: true + premiumize: api-key: bas-eurl: https://www.premiumize.me/api @@ -144,6 +149,12 @@ pgmq: health-repair-concurrency: 2 health-repair-visibility-timeout: 2m health-repair-poll-interval: 5s + torrent-health-check-concurrency: 1 + torrent-health-check-visibility-timeout: 5m + torrent-health-check-poll-interval: 10s + torrent-health-repair-concurrency: 2 + torrent-health-repair-visibility-timeout: 2m + torrent-health-repair-poll-interval: 5s sentry: send-default-pii: false diff --git a/src/main/resources/db/migration/V17__torrent_health_and_pgmq_queues.sql b/src/main/resources/db/migration/V17__torrent_health_and_pgmq_queues.sql new file mode 100644 index 00000000..d6711fb1 --- /dev/null +++ b/src/main/resources/db/migration/V17__torrent_health_and_pgmq_queues.sql @@ -0,0 +1,5 @@ +ALTER TABLE torrent ADD COLUMN last_verified TIMESTAMP; +ALTER TABLE torrent ADD COLUMN health_check_enqueued_at TIMESTAMP; + +SELECT pgmq.create('torrent_health_check'); +SELECT pgmq.create('torrent_health_repair'); diff --git a/src/main/resources/db/migration/V18__repair_outcome_table.sql b/src/main/resources/db/migration/V18__repair_outcome_table.sql new file mode 100644 index 00000000..e382cd72 --- /dev/null +++ b/src/main/resources/db/migration/V18__repair_outcome_table.sql @@ -0,0 +1,9 @@ +CREATE TABLE repair_outcome ( + id BIGSERIAL PRIMARY KEY, + queue_name VARCHAR(255) NOT NULL, + msg_id BIGINT NOT NULL, + action VARCHAR(50) NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT NOW() +); + +CREATE INDEX idx_repair_outcome_queue_msg ON repair_outcome (queue_name, msg_id); diff --git a/src/test/kotlin/io/skjaere/debridav/test/ArrServiceTest.kt b/src/test/kotlin/io/skjaere/debridav/test/ArrServiceTest.kt index c2876bbc..025f6cfe 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/ArrServiceTest.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/ArrServiceTest.kt @@ -19,7 +19,7 @@ class ArrServiceTest { fun thatDeleteFileAndSearchCallsClient() = runTest { //given every { sonarrApiClient.getCategory() } returns "tv-sonarr" - coEvery { sonarrApiClient.deleteFileAndSearch(eq("test-item")) } just Runs + coEvery { sonarrApiClient.deleteFileAndSearch(eq("test-item")) } returns true //when underTest.deleteFileAndSearch("test-item", "tv-sonarr") diff --git a/src/test/kotlin/io/skjaere/debridav/test/NzbImportServiceTest.kt b/src/test/kotlin/io/skjaere/debridav/test/NzbImportServiceTest.kt index 0e8841f4..37023a9f 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/NzbImportServiceTest.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/NzbImportServiceTest.kt @@ -173,7 +173,7 @@ class NzbImportServiceTest { } @Test - fun `executeImport sets ARTICLES_MISSING on PrepareResult MissingArticles`() { + fun `executeImport sets FAILED on PrepareResult MissingArticles`() { // given val download = createUsenetDownload() val importRecord = createImportRecord() @@ -195,8 +195,8 @@ class NzbImportServiceTest { underTest.executeImport(NzbImportTaskData(nzbBytesBase64, 1L, 100L)) // then - assertEquals(UsenetDownloadStatus.ARTICLES_MISSING, savedSlot.captured.status) - assertEquals(NzbImportStatus.ARTICLES_MISSING, importSlot.captured.status) + assertEquals(UsenetDownloadStatus.FAILED, savedSlot.captured.status) + assertEquals(NzbImportStatus.FAILED, importSlot.captured.status) assertEquals("Article not found: 430", importSlot.captured.errorMessage) verify(exactly = 0) { nzbDocumentRepository.save(any()) } } @@ -328,8 +328,8 @@ class NzbImportServiceTest { // then - nzbImportRepository.save is called: once for IMPORTING status (phase 1) + once in phase 3 = 2 verify(exactly = 2) { nzbImportRepository.save(any()) } verify(exactly = 1) { usenetRepository.save(any()) } - assertEquals(UsenetDownloadStatus.ARTICLES_MISSING, download.status) - assertEquals(NzbImportStatus.ARTICLES_MISSING, importRecord.status) + assertEquals(UsenetDownloadStatus.FAILED, download.status) + assertEquals(NzbImportStatus.FAILED, importRecord.status) } /** diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/TorrentHealthCheckIT.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/TorrentHealthCheckIT.kt new file mode 100644 index 00000000..b8482f56 --- /dev/null +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/TorrentHealthCheckIT.kt @@ -0,0 +1,352 @@ +package io.skjaere.debridav.test.integrationtest + +import io.skjaere.debridav.DebriDavApplication +import io.skjaere.debridav.MiltonConfiguration +import io.skjaere.debridav.category.Category +import io.skjaere.debridav.category.CategoryRepository +import io.skjaere.debridav.debrid.DebridProvider +import io.skjaere.debridav.fs.DatabaseFileService +import io.skjaere.debridav.fs.DebridCachedTorrentContent +import io.skjaere.debridav.fs.MissingFile +import io.skjaere.debridav.health.RepairAction +import io.skjaere.debridav.health.RepairOutcomeRepository +import io.skjaere.debridav.test.MAGNET +import io.skjaere.debridav.test.integrationtest.config.IntegrationTestContextConfiguration +import io.skjaere.debridav.test.integrationtest.config.MockServerTest +import io.skjaere.debridav.test.integrationtest.config.PremiumizeStubbingService +import io.skjaere.debridav.torrent.Status +import io.skjaere.debridav.torrent.Torrent +import io.skjaere.debridav.torrent.TorrentHealthCheckService +import io.skjaere.debridav.torrent.TorrentRepository +import org.hamcrest.MatcherAssert.assertThat +import org.hamcrest.Matchers.`is` +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockserver.integration.ClientAndServer +import org.mockserver.model.HttpRequest.request +import org.mockserver.model.HttpResponse.response +import org.mockserver.model.MediaType +import org.mockserver.verify.VerificationTimes +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import java.time.Instant + +@SpringBootTest( + classes = [DebriDavApplication::class, IntegrationTestContextConfiguration::class, MiltonConfiguration::class], + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = [ + "debridav.debrid-clients=premiumize", + "sonarr.integration-enabled=true", + "sonarr.category=tv", + "repair.enabled=true" + ] +) +@MockServerTest +class TorrentHealthCheckIT { + + @Autowired + private lateinit var torrentRepository: TorrentRepository + + @Autowired + private lateinit var categoryRepository: CategoryRepository + + @Autowired + private lateinit var databaseFileService: DatabaseFileService + + @Autowired + private lateinit var torrentHealthCheckService: TorrentHealthCheckService + + @Autowired + private lateinit var premiumizeStubbingService: PremiumizeStubbingService + + @Autowired + private lateinit var repairOutcomeRepository: RepairOutcomeRepository + + @Autowired + private lateinit var mockServer: ClientAndServer + + @BeforeEach + fun setUp() { + torrentRepository.deleteAll() + repairOutcomeRepository.deleteAll() + mockServer.reset() + } + + @AfterEach + fun tearDown() { + torrentRepository.deleteAll() + repairOutcomeRepository.deleteAll() + mockServer.reset() + } + + @Test + @Suppress("LongMethod") + fun `unhealthy torrent triggers Arr blocklist and search`() { + // given — create a torrent with a file whose only debrid link is MissingFile + val category = categoryRepository.findByNameIgnoreCase("tv") + ?: categoryRepository.save(Category("tv", "/data/downloads/tv")) + + val contents = DebridCachedTorrentContent( + originalPath = "movie.mkv", + size = 1_000_000L, + modified = Instant.EPOCH.toEpochMilli(), + magnet = MAGNET, + mimeType = "video/x-matroska", + debridLinks = mutableListOf( + MissingFile(DebridProvider.PREMIUMIZE, Instant.EPOCH.toEpochMilli()) + ) + ) + + val file = databaseFileService.createDebridFile( + "/downloads/tv/test-torrent/movie.mkv", + "testhash123", + contents + ) + databaseFileService.saveDbEntity(file) + + val torrent = Torrent().apply { + name = "test-torrent" + this.category = category + hash = "testhash123" + savePath = "/data/downloads/tv" + status = Status.LIVE + lastVerified = null + } + torrentRepository.save(torrent) + torrent.files = mutableListOf(file) + torrentRepository.save(torrent) + + // given — premiumize says "not cached" when the health check re-verifies + premiumizeStubbingService.mockIsNotCached() + + // given — Sonarr mock: history lookup for blocklisting + mockServer.`when`( + request() + .withMethod("GET") + .withPath("/sonarr/api/v3/history") + ).respond( + response() + .withStatusCode(200) + .withContentType(MediaType.APPLICATION_JSON) + .withBody("""{"page": 1, "pageSize": 1, "totalRecords": 1, "records": [{"id": 42}]}""") + ) + + // given — Sonarr mock: mark history record as failed + mockServer.`when`( + request() + .withMethod("POST") + .withPath("/sonarr/api/v3/history/failed/42") + ).respond( + response().withStatusCode(200) + ) + + // given — Sonarr mock: parse endpoint for deleteFileAndSearch + mockServer.`when`( + request() + .withMethod("GET") + .withPath("/sonarr/api/v3/parse") + ).respond( + response() + .withStatusCode(200) + .withContentType(MediaType.APPLICATION_JSON) + .withBody("""{"episodes": [{"id": 1, "episodeFileId": 10}]}""") + ) + + // given — Sonarr mock: delete episode file + mockServer.`when`( + request() + .withMethod("DELETE") + .withPath("/sonarr/api/v3/episodefile/10") + ).respond( + response().withStatusCode(200) + ) + + // given — Sonarr mock: command (search) + mockServer.`when`( + request() + .withMethod("POST") + .withPath("/sonarr/api/v3/command") + ).respond( + response().withStatusCode(200) + ) + + // when — trigger health check (enqueues to PGMQ, processed asynchronously) + torrentHealthCheckService.triggerFullHealthCheck() + + // then — wait for the full check → repair pipeline to complete + waitForMockServerVerification { + // blocklist: history lookup + mark failed + mockServer.verify( + request() + .withMethod("GET") + .withPath("/sonarr/api/v3/history"), + VerificationTimes.atLeast(1) + ) + mockServer.verify( + request() + .withMethod("POST") + .withPath("/sonarr/api/v3/history/failed/42"), + VerificationTimes.atLeast(1) + ) + + // deleteFileAndSearch: parse + delete + command + mockServer.verify( + request() + .withMethod("GET") + .withPath("/sonarr/api/v3/parse"), + VerificationTimes.atLeast(1) + ) + mockServer.verify( + request() + .withMethod("DELETE") + .withPath("/sonarr/api/v3/episodefile/10"), + VerificationTimes.atLeast(1) + ) + mockServer.verify( + request() + .withMethod("POST") + .withPath("/sonarr/api/v3/command"), + VerificationTimes.atLeast(1) + ) + } + + // then — repair outcome recorded as REPAIRED + waitForCondition("repair outcome recorded") { + repairOutcomeRepository.findAll().toList().any { it.action == RepairAction.REPAIRED } + } + } + + @Test + fun `healthy torrent updates lastVerified without enqueuing repair`() { + // given — create a torrent with a file that has a working CachedFile link + val contents = DebridCachedTorrentContent( + originalPath = "healthy-movie.mkv", + size = 500_000L, + modified = Instant.EPOCH.toEpochMilli(), + magnet = MAGNET, + mimeType = "video/x-matroska", + debridLinks = mutableListOf( + MissingFile(DebridProvider.PREMIUMIZE, Instant.EPOCH.toEpochMilli()) + ) + ) + + val file = databaseFileService.createDebridFile( + "/downloads/misc/healthy-torrent/healthy-movie.mkv", + "healthyhash456", + contents + ) + databaseFileService.saveDbEntity(file) + + val torrent = Torrent().apply { + name = "healthy-torrent" + hash = "healthyhash456" + savePath = "/data/downloads/misc" + status = Status.LIVE + lastVerified = null + } + torrentRepository.save(torrent) + torrent.files = mutableListOf(file) + torrentRepository.save(torrent) + + // given — premiumize returns cached (healthy) + premiumizeStubbingService.mockIsCached() + premiumizeStubbingService.mockCachedContents() + + // when + torrentHealthCheckService.triggerFullHealthCheck() + + // then — torrent gets lastVerified set, no repair messages sent + waitForCondition("lastVerified is set") { + val updated = torrentRepository.findById(torrent.id!!).orElse(null) + updated?.lastVerified != null + } + + val updated = torrentRepository.findById(torrent.id!!).get() + assertThat("lastVerified should be set", updated.lastVerified != null, `is`(true)) + assertThat("healthCheckEnqueuedAt should be cleared", updated.healthCheckEnqueuedAt == null, `is`(true)) + } + + @Test + fun `unhealthy torrent without Arr category deletes files`() { + // given — torrent with no category (no Arr client match) + val contents = DebridCachedTorrentContent( + originalPath = "orphan.mkv", + size = 200_000L, + modified = Instant.EPOCH.toEpochMilli(), + magnet = MAGNET, + mimeType = "video/x-matroska", + debridLinks = mutableListOf( + MissingFile(DebridProvider.PREMIUMIZE, Instant.EPOCH.toEpochMilli()) + ) + ) + + val file = databaseFileService.createDebridFile( + "/downloads/nocategory/orphan-torrent/orphan.mkv", + "orphanhash789", + contents + ) + databaseFileService.saveDbEntity(file) + + val torrent = Torrent().apply { + name = "orphan-torrent" + hash = "orphanhash789" + savePath = "/data/downloads/nocategory" + status = Status.LIVE + lastVerified = null + } + torrentRepository.save(torrent) + torrent.files = mutableListOf(file) + torrentRepository.save(torrent) + + // given — premiumize says not cached + premiumizeStubbingService.mockIsNotCached() + + // when + torrentHealthCheckService.triggerFullHealthCheck() + + // then — repair outcome recorded as DELETED (no Arr client to search) + waitForCondition("repair outcome recorded as DELETED") { + repairOutcomeRepository.findAll().toList().any { it.action == RepairAction.DELETED } + } + } + + @Suppress("TooGenericExceptionCaught") + private fun waitForMockServerVerification( + timeoutMs: Long = 30_000, + pollMs: Long = 500, + verification: () -> Unit + ) { + val deadline = System.currentTimeMillis() + timeoutMs + var lastError: Throwable? = null + while (System.currentTimeMillis() < deadline) { + try { + verification() + return + } catch (e: Throwable) { + lastError = e + Thread.sleep(pollMs) + } + } + throw AssertionError("Verification did not pass within ${timeoutMs}ms", lastError) + } + + @Suppress("TooGenericExceptionCaught") + private fun waitForCondition( + description: String, + timeoutMs: Long = 30_000, + pollMs: Long = 500, + condition: () -> Boolean + ) { + val deadline = System.currentTimeMillis() + timeoutMs + while (System.currentTimeMillis() < deadline) { + try { + if (condition()) return + } catch (_: Throwable) { + // ignore and retry + } + Thread.sleep(pollMs) + } + throw AssertionError("Condition '$description' not met within ${timeoutMs}ms") + } +} From 0af95638bb6ae8d338ceceee66ae79790724e455 Mon Sep 17 00:00:00 2001 From: Skjaere Date: Sat, 14 Mar 2026 14:00:00 +0100 Subject: [PATCH 09/61] fix(realdebrid): link handling + sync improvements + CLAUDE.md docs Improves Real-Debrid integration reliability around link-not-found / stale-link scenarios and sync consistency. Adds reference CLAUDE.md sections documenting both the Real-Debrid and NNTP/Usenet integrations so future work has a contentful starting point. Co-Authored-By: Claude Opus 4.7 (1M context) --- CLAUDE.md | 279 ++++++++++++++++++ .../client/realdebrid/RealDebridClient.kt | 16 +- .../support/RealDebridDownloadService.kt | 26 +- .../support/RealDebridTorrentService.kt | 25 +- 4 files changed, 322 insertions(+), 24 deletions(-) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..b5e6de70 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,279 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Build and Test Commands + +```bash +# Build +./gradlew build # Full build with tests +./gradlew bootJar # Build Spring Boot JAR +./gradlew bootRun # Run application directly +./gradlew jibDockerBuild # Build Docker image locally + +# Test +./gradlew test # Run all tests +./gradlew test --tests "io.skjaere.debridav.test.SomeTest" # Run single test class +./gradlew test --tests "*SomeTest.testMethod" # Run single test method + +# Other +./gradlew compileKotlin # Compile only (no tests) +``` + +## Technology Stack + +- **Kotlin 2.3.0** with **Java 25** (virtual threads via Loom) +- **Spring Boot 4.0.0** with Spring Data JPA +- **PostgreSQL** with Flyway migrations +- **Ktor 3.3.3** for HTTP client operations +- **Milton 4.0.4** for WebDAV protocol +- **Kotlin Coroutines** with custom `Dispatchers.LOOM` for virtual thread integration + +## Architecture Overview + +DebriDAV creates a WebDAV-mountable virtual filesystem backed by debrid service providers. It emulates the qBittorrent and SABnzbd APIs for integration with Sonarr/Radarr. + +### Core Modules + +| Package | Purpose | +|---------|---------| +| `debrid/client/` | Provider implementations (RealDebrid, Premiumize, Easynews, TorBox) with abstract `DebridClient` | +| `fs/` | Virtual filesystem layer - `DatabaseFileService` manages file hierarchy using PostgreSQL LTree | +| `resource/` | WebDAV resource factory connecting Milton to the virtual FS | +| `torrent/` | qBittorrent API emulation (`QBittorrentEmulationController`) | +| `nntp/` | Usenet/NZB support with streaming RAR parsing and Yenc decompression | +| `archive/` | RAR file parsing (`Rar4Parser`) for metadata extraction | +| `cache/` | Byte-range caching for metadata extraction (`FileChunkCachingService`) | +| `arrs/` | Sonarr/Radarr integration services | + +### Data Flow + +1. Sonarr/Radarr send requests to qBittorrent-emulated API +2. Torrent/NZB content checked against debrid provider caches +3. Cached content registered in PostgreSQL as virtual files +4. WebDAV server exposes virtual filesystem for media server mounting +5. On file access, content streamed from debrid provider with chunk caching + +### Database + +- PostgreSQL required (uses LTree extension for hierarchical paths) +- Entities: `Torrent`, `UsenetEntity`, `DebridFileContents`, `FileChunk` +- Migrations in `src/main/resources/db/migration/` (V1-V11) + +## Code Patterns + +**Configuration**: Use `@ConfigurationProperties` classes in `DebridavConfiguration.kt`. Properties defined in `application.properties`. + +**Async operations**: Use Kotlin coroutines with `Dispatchers.LOOM` for blocking I/O: +```kotlin +withContext(Dispatchers.LOOM) { + // blocking operation +} +``` + +**Transactions**: Use `TransactionTemplate` for explicit transaction boundaries in services. + +**Testing**: Integration tests use TestContainers (PostgreSQL), MockServer for HTTP, MockK for mocking. Tests in `src/test/kotlin/io/skjaere/debridav/test/`. + +## Real-Debrid Integration + +### Key Files + +| File | Purpose | +|------|---------| +| `debrid/client/realdebrid/RealDebridClient.kt` | Main client — cache checking, torrent management, link unrestriction | +| `debrid/client/realdebrid/support/RealDebridTorrentService.kt` | Torrent sync and DB persistence | +| `debrid/client/realdebrid/support/RealDebridDownloadService.kt` | Download sync and DB persistence | +| `debrid/client/realdebrid/RealDebridConfigurationProperties.kt` | Configuration properties (`real-debrid.*`) | +| `debrid/client/realdebrid/RealDebridConfiguration.kt` | Spring bean config including Resilience4j rate limiter | +| `debrid/client/realdebrid/RealDebridActuatorEndpoint.kt` | Actuator endpoint for toggling torrent import at runtime | + +### Class Hierarchy + +`RealDebridClient` extends `DebridCachedTorrentClient` and `DebridCachedContentClient`, implements `StreamableLinkPreparable` (delegated to `DefaultStreamableLinkPreparer`) and `ConfigurationTester`. + +### API Endpoints Used + +All calls go to `https://api.real-debrid.com/rest/1.0` (configurable). Authentication is via Bearer token (`real-debrid.api-key`). + +| Endpoint | Method | Purpose | +|----------|--------|---------| +| `/torrents/addMagnet` | POST | Submit magnet link (form-encoded `magnet=...`) | +| `/torrents/` | GET | List user's torrents (paginated, 100/page) | +| `/torrents/info/{id}` | GET | Get torrent info with files and links | +| `/torrents/selectFiles/{id}` | POST | Select files from torrent (form-encoded `files=1,2,3`) | +| `/torrents/delete/{id}` | DELETE | Remove torrent from account | +| `/unrestrict/link` | POST | Convert RD share link → direct download URL | +| `/downloads` | GET | List user's downloads (paginated, 100/page) | +| `/downloads/delete/{id}` | DELETE | Remove download from account | +| `/user` | GET | Validate API key (used by `ConfigurationTester`) | + +### End-to-End Flow + +**Phase 1 — Torrent Addition:** +1. `getCachedFiles(magnet)` checks DB for existing torrent by info hash +2. If not found: `POST /torrents/addMagnet` → `GET /torrents/info/{id}` → save to `RealDebridTorrentEntity` + +**Phase 2 — File Selection:** +1. `getIdsToSelect()` filters for video files (`.mp4`, `.mkv`, `.avi`, `.ts`) +2. `POST /torrents/selectFiles/{id}` with selected file IDs +3. `GET /torrents/info/{id}` to retrieve links for selected files +4. If no links available (not cached): DELETE torrent, return empty list + +**Phase 3 — Link Unrestriction:** +1. For each file link, check DB for existing `RealDebridDownloadEntity` +2. If not found: `POST /unrestrict/link` → returns direct download URL, saved to DB +3. Returns `List` with path, download URL, MIME type, and params (`torrentId`, `linkId`) + +**Phase 4 — Streaming:** +1. `getStreamableLink()` looks up download by hash + filename + size in DB +2. `isLinkAlive()` — HEAD request to download URL (rate-limited, cached 5 min) +3. If alive: return URL. If dead: delete and fetch fresh link via unrestrict +4. `DefaultStreamableLinkPreparer` builds Ktor HTTP GET with byte-range headers for seeking support + +### Rate Limiting + +Resilience4j `RateLimiter`: **249 requests per 1 minute** (just under RD's ~250/min limit), 5-second timeout per acquisition. + +### Scheduled Sync + +`syncTorrentsTask()` runs on a configurable schedule (`real-debrid.sync-poll-rate`, default `PT24H`): +- Clears and re-fetches all `RealDebridTorrentEntity` records (paginated `/torrents/`) +- Clears and re-fetches all `RealDebridDownloadEntity` records (paginated `/downloads`) +- Can be toggled at runtime via the actuator endpoint + +### Database Entities + +**`RealDebridTorrentEntity`**: `torrentId` (indexed), `name`, `hash` (indexed), `links` (ElementCollection), `files` (one-to-many `TorrentsInfoFile`) + +**`RealDebridDownloadEntity`**: `downloadId` (indexed), `filename`, `mimeType`, `fileSize`, `link` (RD share link), `host`, `download` (actual URL), `chunks`, `streamable` + +**Key query**: `getDownloadByHashAndFilenameAndSize()` — native SQL joining downloads → torrent links → torrents to find a download by torrent hash + filename + file size. + +### Error Handling + +- `isCached()` always returns `true` (RD doesn't expose a cache-check API; availability is determined during file selection) +- HTTP 4xx → `DebridClientError`, 5xx → `DebridProviderError` +- `addMagnet` failures return `FailedAddMagnetResponse` with reason (not thrown) +- `unrestrict` failures logged as warnings, return `ErrorUnrestrictLinkResponse` +- Configurable retries in `DebridCachedContentService` (default 1, 200ms delay) + +## NNTP/Usenet Integration + +### External Artifacts + +| Artifact | Version | Purpose | +|----------|---------|---------| +| `com.github.skjaere:nzb-streamer` | 0.7.0 | NZB parsing, NNTP article fetching, Yenc decompression, RAR/7zip archive parsing, and file streaming | +| `com.github.skjaere:mock-nntp-server` | 0.2.0 | Test-only mock NNTP server | + +**nzb-streamer** internally depends on **ktor-nntp-client** (a Ktor-based NNTP protocol client) for connecting to Usenet servers, fetching articles by message-ID, and managing connection pools with TLS support and server priority failover. + +### Key Files + +| File | Purpose | +|------|---------| +| `usenet/NzbStreamerConfiguration.kt` | NNTP server pool config, creates `NzbStreamer` bean | +| `usenet/NzbImportService.kt` | Orchestrates NZB import: parse → extract metadata → register in filesystem | +| `usenet/NzbHealthCheckService.kt` | Scheduled verification that NZB segments still exist on Usenet | +| `usenet/sabnzbd/SabnzbdApiController.kt` | SABnzbd API emulation endpoints | +| `usenet/sabnzbd/SabNzbdService.kt` | NZB handling and SABnzbd response building | +| `usenet/pgmq/PgmqSpringConfiguration.kt` | PostgreSQL message queue setup (3 queues) | +| `usenet/pgmq/PgmqConsumer.kt` | Generic message consumer loop | +| `usenet/pgmq/NzbHealthCheckHandler.kt` | Processes health check messages | +| `usenet/pgmq/NzbHealthRepairHandler.kt` | Blocklists failed NZBs in Sonarr/Radarr | +| `usenet/nzb/NzbDocumentEntity.kt` | JPA entity storing parsed NZB metadata as JSONB | +| `usenet/UsenetDownload.kt` | JPA entity tracking download status | +| `usenet/queue/NzbImportRecord.kt` | JPA entity tracking import queue status | +| `resource/NzbFileResource.kt` | WebDAV resource for streaming NZB files via Milton | + +### Configuration Properties + +**`nntp.*`** (all conditional on `nntp.enabled=true`): +- `enabled` — enable/disable NNTP support +- `concurrency` (default 4) — concurrent NNTP streaming threads +- `forwardThresholdBytes` (default 102400) — byte threshold for forward seeking +- `healthCheckInterval` (default 7 days) — how often to reverify NZB segments +- `healthCheckPollRate` (default 5 min) — poll rate for health check scheduling +- `pools` — list of NNTP server pools, each with: `host`, `port`, `username`, `password`, `useTls`, `maxConnections`, `priority` + +**`pgmq.*`**: +- `importConcurrency` (default 2) — workers processing NZB imports +- `importVisibilityTimeout` (default 10 min) — message lock duration +- `importPollInterval` (default 2 sec) — queue poll rate +- `healthCheckConcurrency` (default 1), `healthRepairConcurrency` (default 2) + +### End-to-End Flow + +**Phase 1 — NZB Upload (SABnzbd API emulation):** +1. Sonarr/Radarr POST NZB file to `/api?mode=addfile` +2. `SabNzbdService` creates `UsenetDownload` (QUEUED) and `NzbImportRecord` (QUEUED) +3. Sends `NzbImportMessage` (NZB bytes as Base64) to PGMQ `nzb_import` queue +4. Returns immediately to caller + +**Phase 2 — Async Import (PGMQ consumer):** +1. `PgmqConsumer` picks up message from `nzb_import` queue +2. `NzbImportService.executeImport()`: + - Decodes NZB bytes, calls `nzbStreamer.prepare(nzbBytes)` + - nzb-streamer parses NZB XML, fetches initial articles from NNTP servers via ktor-nntp-client + - Yenc-decodes article bodies, parses RAR/7zip archive headers to extract file metadata + - Returns `PrepareResult`: `Success`, `MissingArticles`, `Failure`, or `UnsupportedArchive` +3. On success: `nzbStreamer.resolveStreamableFiles(metadata)` → list of files with volume/offset info +4. Creates `NzbDocumentEntity` (files + streamableFiles stored as JSONB), `NzbContents` per file, and `RemotelyCachedEntity` entries in the virtual filesystem +5. Updates `UsenetDownload.status` → COMPLETED + +**Phase 3 — File Streaming (WebDAV access):** +1. Media server accesses file via WebDAV +2. `StreamableResourceFactory` creates `NzbFileResource` from `NzbContents` entity +3. `NzbFileResource.sendContent()` calls `nzbStreamer.streamFile(nzbDocument, streamableFile, range)` +4. nzb-streamer fetches NNTP articles on-demand, Yenc-decodes, reconstructs archive data, and streams the extracted file content via a `ByteReadChannel` +5. Supports byte-range requests for seeking/scrubbing + +**Phase 4 — Health Check & Repair:** +1. `NzbHealthCheckService` runs on schedule, finds NZB documents not verified within `healthCheckInterval` +2. Sends `NzbHealthCheckMessage` to PGMQ `nzb_health_check` queue +3. `NzbHealthCheckHandler` calls `nzbStreamer.verifySegments()` to check article availability +4. If articles missing: sends `NzbHealthRepairMessage` to `nzb_health_repair` queue +5. `NzbHealthRepairHandler` blocklists the download in Sonarr/Radarr and triggers a new search + +### Message Queue Architecture (PGMQ) + +Three PostgreSQL-backed queues (installed via Flyway migration `V12__install_pgmq.sql`): + +| Queue | Message Type | Handler | Concurrency | +|-------|-------------|---------|-------------| +| `nzb_import` | `NzbImportMessage` | `NzbImportService` | 2 workers | +| `nzb_health_check` | `NzbHealthCheckMessage` | `NzbHealthCheckHandler` | 1 worker | +| `nzb_health_repair` | `NzbHealthRepairMessage` | `NzbHealthRepairHandler` | 2 workers | + +### Supported Archive Types + +nzb-streamer handles: `RAW`, `RAR`, `SEVEN_ZIP`, `RAR_IN_SEVEN_ZIP`, `RAR_IN_RAR`, `SEVEN_ZIP_IN_RAR`, `SEVEN_ZIP_IN_SEVEN_ZIP` (nested archives). + +### Database Entities + +**`NzbDocumentEntity`** (table `nzb_document`): `files` (JSONB — Yenc headers + segment article IDs), `streamableFiles` (JSONB — file paths with volume/offset positions), `archiveType`, `lastVerified`, `name`, `category` + +**`UsenetDownload`**: `status` (QUEUED → DOWNLOADING → COMPLETED/FAILED/ARTICLES_MISSING), `name`, `hash` (MD5 of NZB), `size`, `category`, references `NzbDocumentEntity` + +**`NzbImportRecord`** (table `nzb_import`): tracks import queue status with `status`, `archiveType`, `errorMessage`, `files` (JSONB), timestamps + +**`NzbContents`** (extends `DebridFileContents`): `originalPath`, `size`, `mimeType`, references `NzbDocumentEntity` + +### SABnzbd API Emulation + +Endpoints at `/api` (emulating SABnzbd v4.4.0): +- `mode=addfile` — multipart NZB upload +- `mode=queue` — returns queue status +- `mode=history` — completed/failed downloads from DB +- `mode=get_config` — categories and configuration +- `mode=version` — returns "4.4.0" +- `mode=fullstatus` — static status with configured paths + +## Configuration + +Key properties in `application.properties`: +- `debridav.debrid-clients` - Enabled providers (real-debrid, premiumize, easynews, torbox) +- `debridav.root-path` - WebDAV root path +- `debridav.download-path` - Download directory path +- Provider-specific API keys and settings diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridClient.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridClient.kt index e1a5223d..dff87711 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridClient.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/RealDebridClient.kt @@ -9,7 +9,6 @@ import io.ktor.client.request.bearerAuth import io.ktor.client.request.delete import io.ktor.client.request.forms.FormDataContent import io.ktor.client.request.get -import io.ktor.client.request.head import io.ktor.client.request.headers import io.ktor.client.request.post import io.ktor.client.request.setBody @@ -363,25 +362,18 @@ class RealDebridClient( override suspend fun getStreamableLink(key: TorrentMagnet, cachedFile: CachedFile): String? { - //return realDebridDownloadService.getDownloadByLink(cachedFile.params!![LINK_ID_MAP_KEY]!!) return realDebridDownloadService.getDownloadByHashAndFilenameAndSize( cachedFile.path!!, cachedFile.size!!, key.getHash()!! )?.let { realDebridDownload -> - if (isLinkAlive(realDebridDownload.download!!)) { - realDebridDownload.link - } else { - deleteDownload(realDebridDownload.downloadId!!) - realDebridDownloadService.deleteDownload(realDebridDownload) - null - } + realDebridDownload.download } ?: run { getFreshRealDebridLink(key, cachedFile.path!!, cachedFile.size!!) ?.let { val unrestrictResult = unrestrictLink(it) when (unrestrictResult) { - is SuccessfulUnrestrictLinkResponse -> unrestrictResult.realDebridDownloadEntity.link + is SuccessfulUnrestrictLinkResponse -> unrestrictResult.realDebridDownloadEntity.download else -> null } } @@ -404,10 +396,6 @@ class RealDebridClient( return x } - private suspend fun isLinkAlive(link: String): Boolean { - return realDebridRateLimiter.executeSuspendFunction { httpClient.head(link).status.isSuccess() } - } - override val configurationClass: KClass<*> = RealDebridConfigurationProperties::class override val label: String = "Real-Debrid" diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/support/RealDebridDownloadService.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/support/RealDebridDownloadService.kt index 91abf5f6..a21e8a51 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/support/RealDebridDownloadService.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/support/RealDebridDownloadService.kt @@ -30,11 +30,27 @@ class RealDebridDownloadService( ) { @Transactional fun syncDownloadsToDatabase(): Unit = runBlocking { - realDebridDownloadRepository.deleteAll() - getListOfDownloads().asSequence() - .map { mapDownloadToRdtEntity(it) } - .toList() - .let { realDebridDownloadRepository.saveAll(it) } + val remoteDownloads = getListOfDownloads() + val remoteDownloadIds = remoteDownloads.map { it.id }.toSet() + + // Remove locally-stored downloads that no longer exist on RD + realDebridDownloadRepository.findAll().forEach { local -> + if (local.downloadId !in remoteDownloadIds) { + realDebridDownloadRepository.delete(local) + } + } + + // Upsert downloads from RD + remoteDownloads.forEach { download -> + val existing = realDebridDownloadRepository.getByDownloadIdIgnoreCase(download.id) + if (existing != null) { + updateDownloadValues(existing, download) + .let { realDebridDownloadRepository.save(it) } + } else { + mapDownloadToRdtEntity(download) + .let { realDebridDownloadRepository.save(it) } + } + } } suspend fun saveDownload(realDebridDownload: RealDebridDownload): RealDebridDownloadEntity { diff --git a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/support/RealDebridTorrentService.kt b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/support/RealDebridTorrentService.kt index 609a9cb6..0e12e27e 100644 --- a/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/support/RealDebridTorrentService.kt +++ b/src/main/kotlin/io/skjaere/debridav/debrid/client/realdebrid/support/RealDebridTorrentService.kt @@ -57,11 +57,26 @@ class RealDebridTorrentService( @Transactional fun syncTorrentListToDatabase(): Unit = runBlocking { - realDebridTorrentRepository.deleteAll() - getListOfTorrents().asSequence() - .map { mapTorrentInfoToRdtEntity(it) } - .toList() - .let { realDebridTorrentRepository.saveAll(it) } + val remoteTorrents = getListOfTorrents() + val remoteTorrentIds = remoteTorrents.map { it.id }.toSet() + + // Remove locally-stored torrents that no longer exist on RD + realDebridTorrentRepository.findAll().forEach { local -> + if (local.torrentId !in remoteTorrentIds) { + realDebridTorrentRepository.delete(local) + } + } + + // Upsert torrents from RD + remoteTorrents.forEach { info -> + val existing = realDebridTorrentRepository.getByTorrentIdIgnoreCase(info.id) + val entity = existing ?: RealDebridTorrentEntity() + entity.torrentId = info.id + entity.name = info.filename + entity.hash = info.hash + entity.links = info.links + realDebridTorrentRepository.save(entity) + } } private fun mapTorrentInfoToRdtEntity(info: TorrentsResponseItem): RealDebridTorrentEntity { From c96ef53ece480b1649951d4c161722405fed3aab Mon Sep 17 00:00:00 2001 From: Skjaere Date: Sat, 11 Apr 2026 10:30:00 +0200 Subject: [PATCH 10/61] feat(webdav): move WebDAV API under /webdav prefix Previously WebDAV was mounted at the root, which meant any non-API path had to be disambiguated between WebDAV and the Spring MVC controllers. Moving it under /webdav cleanly separates the WebDAV surface from REST / static resources. BREAKING CHANGE: WebDAV clients (rclone, media servers) must update their URL from / to /webdav. Co-Authored-By: Claude Opus 4.7 (1M context) --- dev/rclone.conf | 2 +- .../skjaere/debridav/DebridavConfiguration.kt | 3 +- .../resource/StreamableResourceFactory.kt | 12 ++- .../integrationtest/QBittorrentEmulationIT.kt | 14 +-- .../integrationtest/WebDavAuthenticationIT.kt | 6 +- .../integrationtest/WebDavOperationsIT.kt | 98 +++++++++---------- 6 files changed, 72 insertions(+), 63 deletions(-) diff --git a/dev/rclone.conf b/dev/rclone.conf index 300802a1..7ddadf30 100644 --- a/dev/rclone.conf +++ b/dev/rclone.conf @@ -1,5 +1,5 @@ [debridav] type = webdav -url = http://172.17.0.1:8080/ +url = http://172.17.0.1:8080/webdav/ vendor = other pacer_min_sleep = 0 diff --git a/src/main/kotlin/io/skjaere/debridav/DebridavConfiguration.kt b/src/main/kotlin/io/skjaere/debridav/DebridavConfiguration.kt index 93a6bd0f..881eaed3 100644 --- a/src/main/kotlin/io/skjaere/debridav/DebridavConfiguration.kt +++ b/src/main/kotlin/io/skjaere/debridav/DebridavConfiguration.kt @@ -27,8 +27,7 @@ class DebridavConfiguration { fun miltonFilterFilterRegistrationBean(): FilterRegistrationBean { val registration = FilterRegistrationBean(SpringMiltonFilter()) registration.setName("MiltonFilter") - registration.addUrlPatterns("/*") - registration.addInitParameter("milton.exclude.paths", "/files,/api,/version,/sabnzbd,/actuator") + registration.addUrlPatterns("/webdav/*") registration.addInitParameter( "resource.factory.class", "io.skjaere.debrid.resource.StreamableResourceFactory" ) diff --git a/src/main/kotlin/io/skjaere/debridav/resource/StreamableResourceFactory.kt b/src/main/kotlin/io/skjaere/debridav/resource/StreamableResourceFactory.kt index dbd18aa4..1cb78a6e 100644 --- a/src/main/kotlin/io/skjaere/debridav/resource/StreamableResourceFactory.kt +++ b/src/main/kotlin/io/skjaere/debridav/resource/StreamableResourceFactory.kt @@ -34,10 +34,20 @@ class StreamableResourceFactory( @Throws(NotAuthorizedException::class, BadRequestException::class) override fun getResource(host: String?, url: String): Resource? { - val path: Path = Path.path(url) + val path: Path = Path.path(stripWebdavPrefix(url)) return find(path) } + private fun stripWebdavPrefix(url: String): String = when { + url == WEBDAV_PREFIX || url == "$WEBDAV_PREFIX/" -> "/" + url.startsWith("$WEBDAV_PREFIX/") -> url.removePrefix(WEBDAV_PREFIX) + else -> url + } + + companion object { + const val WEBDAV_PREFIX = "/webdav" + } + @Throws(NotAuthorizedException::class, BadRequestException::class) private fun find(path: Path): Resource? { val actualPath = if (path.isRoot) "/" else path.toPath() diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/QBittorrentEmulationIT.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/QBittorrentEmulationIT.kt index 97f8e707..fad93c42 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/QBittorrentEmulationIT.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/QBittorrentEmulationIT.kt @@ -74,7 +74,7 @@ class QBittorrentEmulationIT { fun tearDown() { mockserverClient.reset() try { - sardine.delete("http://localhost:${randomServerPort}/downloads/test") + sardine.delete("http://localhost:${randomServerPort}/webdav/downloads/test") } catch (_: Throwable) { } } @@ -150,19 +150,19 @@ class QBittorrentEmulationIT { (debridFileContents?.debridLinks!!.first() as CachedFile).link ) sardine.move( - "http://localhost:${randomServerPort}/downloads/test/a/b/c/movie.mkv", - "http://localhost:${randomServerPort}/movie.mkv" + "http://localhost:${randomServerPort}/webdav/downloads/test/a/b/c/movie.mkv", + "http://localhost:${randomServerPort}/webdav/movie.mkv" ) assertThat( - sardine.list("http://localhost:${randomServerPort}/"), hasItem( + sardine.list("http://localhost:${randomServerPort}/webdav/"), hasItem( hasProperty( "displayName", `is`("movie.mkv") ) ) ) - sardine.delete("http://localhost:${randomServerPort}/movie.mkv") + sardine.delete("http://localhost:${randomServerPort}/webdav/movie.mkv") assertThat( - sardine.list("http://localhost:${randomServerPort}/"), not( + sardine.list("http://localhost:${randomServerPort}/webdav/"), not( hasItem( hasProperty( "displayName", `is`("/movie.mkv") @@ -325,7 +325,7 @@ class QBittorrentEmulationIT { ) // finally - sardine.delete("http://localhost:${randomServerPort}/downloads/second-name") + sardine.delete("http://localhost:${randomServerPort}/webdav/downloads/second-name") } @Test diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/WebDavAuthenticationIT.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/WebDavAuthenticationIT.kt index 2b403961..649b0d1c 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/WebDavAuthenticationIT.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/WebDavAuthenticationIT.kt @@ -32,7 +32,7 @@ class WebDavAuthenticationIT { fun `that unauthenticated request is rejected`() { val sardine = SardineFactory.begin() assertFailsWith { - sardine.list("http://localhost:${randomServerPort}/") + sardine.list("http://localhost:${randomServerPort}/webdav/") } } @@ -40,14 +40,14 @@ class WebDavAuthenticationIT { fun `that wrong credentials are rejected`() { val sardine = SardineFactory.begin("wronguser", "wrongpass") assertFailsWith { - sardine.list("http://localhost:${randomServerPort}/") + sardine.list("http://localhost:${randomServerPort}/webdav/") } } @Test fun `that correct credentials succeed`() { val sardine = SardineFactory.begin("testuser", "testpass") - val resources = sardine.list("http://localhost:${randomServerPort}/") + val resources = sardine.list("http://localhost:${randomServerPort}/webdav/") assertThat(resources.isEmpty(), `is`(false)) } } diff --git a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/WebDavOperationsIT.kt b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/WebDavOperationsIT.kt index 4a40969b..3c23abae 100644 --- a/src/test/kotlin/io/skjaere/debridav/test/integrationtest/WebDavOperationsIT.kt +++ b/src/test/kotlin/io/skjaere/debridav/test/integrationtest/WebDavOperationsIT.kt @@ -50,7 +50,7 @@ class WebDavOperationsIT { @Test fun thatCreatingFileInRootWorks() { //when - sardine.put("http://localhost:${randomServerPort}/testfile.txt", "test contents".byteInputStream()) + sardine.put("http://localhost:${randomServerPort}/webdav/testfile.txt", "test contents".byteInputStream()) //then val listOfFiles: List = listDirectory("/") @@ -70,7 +70,7 @@ class WebDavOperationsIT { @Test fun thatDeletingFileInRootWorks() { //given - sardine.put("http://localhost:${randomServerPort}/testfile.txt", "test contents".byteInputStream()) + sardine.put("http://localhost:${randomServerPort}/webdav/testfile.txt", "test contents".byteInputStream()) val listOfFiles: List = listDirectory("/") assertThat( listOfFiles, hasItem( @@ -99,7 +99,7 @@ class WebDavOperationsIT { @Test fun thatCreatingDirectoryInRootWorks() { //when - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory") val listOfFiles: List = listDirectory("/") //then @@ -119,7 +119,7 @@ class WebDavOperationsIT { @Test fun thatRenamingEmptyDirectoryInRootWorks() { //given - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory") assertThat( listDirectory("/"), hasItem( hasProperty( @@ -130,8 +130,8 @@ class WebDavOperationsIT { //when sardine.move( - "http://localhost:${randomServerPort}/testDirectory", - "http://localhost:${randomServerPort}/movedTestDirectory" + "http://localhost:${randomServerPort}/webdav/testDirectory", + "http://localhost:${randomServerPort}/webdav/movedTestDirectory" ) //then @@ -151,9 +151,9 @@ class WebDavOperationsIT { @Test fun thatRenamingPopulatedDirectoryInRootWorks() { //given - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory") sardine.put( - "http://localhost:${randomServerPort}/testDirectory/testfile.txt", + "http://localhost:${randomServerPort}/webdav/testDirectory/testfile.txt", "test contents".byteInputStream() ) assertThat( @@ -173,8 +173,8 @@ class WebDavOperationsIT { //when sardine.move( - "http://localhost:${randomServerPort}/testDirectory", - "http://localhost:${randomServerPort}/movedTestDirectory" + "http://localhost:${randomServerPort}/webdav/testDirectory", + "http://localhost:${randomServerPort}/webdav/movedTestDirectory" ) //then @@ -201,8 +201,8 @@ class WebDavOperationsIT { @Test fun thatRenamingEmptyDirectoryInBranchWorks() { //given - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory") - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory/nestedDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory/nestedDirectory") assertThat( listDirectory("/"), hasItem( hasProperty( @@ -220,8 +220,8 @@ class WebDavOperationsIT { //when sardine.move( - "http://localhost:${randomServerPort}/testDirectory/nestedDirectory", - "http://localhost:${randomServerPort}/testDirectory/renamedNestedDirectory" + "http://localhost:${randomServerPort}/webdav/testDirectory/nestedDirectory", + "http://localhost:${randomServerPort}/webdav/testDirectory/renamedNestedDirectory" ) //then @@ -241,10 +241,10 @@ class WebDavOperationsIT { @Test fun thatRenamingPopulatedDirectoryInBranchWorks() { //given - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory/") - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory/nestedDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory/") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory/nestedDirectory") sardine.put( - "http://localhost:${randomServerPort}/testDirectory/nestedDirectory/testfile.txt", + "http://localhost:${randomServerPort}/webdav/testDirectory/nestedDirectory/testfile.txt", "test contents".byteInputStream() ) assertThat( @@ -264,8 +264,8 @@ class WebDavOperationsIT { //when sardine.move( - "http://localhost:${randomServerPort}/testDirectory/nestedDirectory", - "http://localhost:${randomServerPort}/testDirectory/renamedNestedDirectory" + "http://localhost:${randomServerPort}/webdav/testDirectory/nestedDirectory", + "http://localhost:${randomServerPort}/webdav/testDirectory/renamedNestedDirectory" ) //then @@ -285,9 +285,9 @@ class WebDavOperationsIT { @Test fun thatMovingDirectoryToNestedDirectoryWorks() { //given - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory") - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory/nestedTestDirectory") - sardine.createDirectory("http://localhost:${randomServerPort}/directoryToBeMoved") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory/nestedTestDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/directoryToBeMoved") assertThat( listDirectory("/"), allOf( hasItem( @@ -312,8 +312,8 @@ class WebDavOperationsIT { //when sardine.move( - "http://localhost:${randomServerPort}/directoryToBeMoved", - "http://localhost:${randomServerPort}/testDirectory/nestedTestDirectory/directoryToBeMoved" + "http://localhost:${randomServerPort}/webdav/directoryToBeMoved", + "http://localhost:${randomServerPort}/webdav/testDirectory/nestedTestDirectory/directoryToBeMoved" ) //then @@ -343,10 +343,10 @@ class WebDavOperationsIT { @Test fun thatMovingDirectoryWithFilesWorks() { //given - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory") - sardine.createDirectory("http://localhost:${randomServerPort}/destinationDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/destinationDirectory") sardine.put( - "http://localhost:${randomServerPort}/testDirectory/testfile.txt", + "http://localhost:${randomServerPort}/webdav/testDirectory/testfile.txt", "test contents".byteInputStream() ) assertThat( @@ -373,8 +373,8 @@ class WebDavOperationsIT { //when sardine.move( - "http://localhost:${randomServerPort}/testDirectory", - "http://localhost:${randomServerPort}/destinationDirectory/testDirectory" + "http://localhost:${randomServerPort}/webdav/testDirectory", + "http://localhost:${randomServerPort}/webdav/destinationDirectory/testDirectory" ) //then @@ -404,9 +404,9 @@ class WebDavOperationsIT { @Test fun thatMovingDirectoryWithSubdirectoriesWorks() { //given - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory") - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory/subDirectory") - sardine.createDirectory("http://localhost:${randomServerPort}/destinationDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory/subDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/destinationDirectory") assertThat( listDirectory("/"), allOf( @@ -425,8 +425,8 @@ class WebDavOperationsIT { //when sardine.move( - "http://localhost:${randomServerPort}/testDirectory", - "http://localhost:${randomServerPort}/destinationDirectory/testDirectory" + "http://localhost:${randomServerPort}/webdav/testDirectory", + "http://localhost:${randomServerPort}/webdav/destinationDirectory/testDirectory" ) //then @@ -463,16 +463,16 @@ class WebDavOperationsIT { @Test fun thatMovingLocalEntityWorks() { //given - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory") sardine.put( - "http://localhost:${randomServerPort}/testfile.txt", + "http://localhost:${randomServerPort}/webdav/testfile.txt", "test contents".byteInputStream() ) //when sardine.move( - "http://localhost:${randomServerPort}/testfile.txt", - "http://localhost:${randomServerPort}/testDirectory/testfile.txt", + "http://localhost:${randomServerPort}/webdav/testfile.txt", + "http://localhost:${randomServerPort}/webdav/testDirectory/testfile.txt", ) // then @@ -490,9 +490,9 @@ class WebDavOperationsIT { @Test fun thatDeletingDirectoryBranchWorks() { //given - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory") - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory/subDirectory") - sardine.createDirectory("http://localhost:${randomServerPort}/testDirectory/subDirectory/secondSubDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory/subDirectory") + sardine.createDirectory("http://localhost:${randomServerPort}/webdav/testDirectory/subDirectory/secondSubDirectory") assertThat( listDirectory("/"), allOf( @@ -523,7 +523,7 @@ class WebDavOperationsIT { ) //when - sardine.delete("http://localhost:${randomServerPort}/testDirectory") + sardine.delete("http://localhost:${randomServerPort}/webdav/testDirectory") //then assertThat( @@ -542,13 +542,13 @@ class WebDavOperationsIT { fun thatReadingLocalEntityWorks() { // given sardine.put( - "http://localhost:${randomServerPort}/testDirectory/testfile.txt", + "http://localhost:${randomServerPort}/webdav/testDirectory/testfile.txt", "test contents".byteInputStream() ) // when val response = sardine.get( - "http://localhost:${randomServerPort}/testDirectory/testfile.txt" + "http://localhost:${randomServerPort}/webdav/testDirectory/testfile.txt" ).readAllBytes().decodeToString() assertThat(response, `is`("test contents")) @@ -560,13 +560,13 @@ class WebDavOperationsIT { fun thatReadingLocalEntityWithRangeWorks() { // given sardine.put( - "http://localhost:${randomServerPort}/testDirectory/testfile.txt", + "http://localhost:${randomServerPort}/webdav/testDirectory/testfile.txt", "test contents".byteInputStream() ) // when val response = sardine.get( - "http://localhost:${randomServerPort}/testDirectory/testfile.txt", + "http://localhost:${randomServerPort}/webdav/testDirectory/testfile.txt", mapOf( "Range" to "bytes=0-0", ) @@ -583,7 +583,7 @@ class WebDavOperationsIT { val contents = IntRange(0, (1024 * 1024 * 2)).map { Byte.MIN_VALUE }.toByteArray() assertFailsWith { sardine.put( - "http://localhost:${randomServerPort}/testfile.txt", + "http://localhost:${randomServerPort}/webdav/testfile.txt", contents.inputStream() ) } @@ -600,8 +600,8 @@ class WebDavOperationsIT { } private fun listDirectory(path: String): List = - sardine.list("http://localhost:${randomServerPort}/$path") + sardine.list("http://localhost:${randomServerPort}/webdav/${path.trimStart('/')}") private fun deleteFile(path: String) = - sardine.delete("http://localhost:${randomServerPort}/$path") + sardine.delete("http://localhost:${randomServerPort}/webdav/${path.trimStart('/')}") } From 5f1bfea605373b461e042559efb91727196f3a00 Mon Sep 17 00:00:00 2001 From: Skjaere Date: Sat, 11 Apr 2026 15:00:00 +0200 Subject: [PATCH 11/61] build: bundle frontend into Spring Boot JAR via git submodule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit debridav-frontend is now tracked as a git submodule and its built `dist/` gets copied into Spring Boot's static resources at build time. Users no longer need to deploy the frontend as a separate container — a single jib-built image serves both the API and the UI. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/ci.yml | 2 + .github/workflows/release-please.yml | 2 + .gitmodules | 3 ++ build.gradle.kts | 63 ++++++++++++++++++++++++++++ debridav-frontend | 1 + 5 files changed, 71 insertions(+) create mode 100644 .gitmodules create mode 160000 debridav-frontend diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4cffe17..70a4adbc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + submodules: true - uses: actions/setup-java@v4 with: diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 096f464f..9e3ff140 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -26,6 +26,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + submodules: true - uses: actions/setup-java@v4 with: diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..f80adeb3 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "debridav-frontend"] + path = debridav-frontend + url = https://github.com/skjaere/debridav-frontend.git diff --git a/build.gradle.kts b/build.gradle.kts index 9666c141..06b02e9b 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,3 +1,4 @@ +import com.github.gradle.node.npm.task.NpmTask import com.google.cloud.tools.jib.gradle.JibTask import dev.detekt.gradle.Detekt import dev.detekt.gradle.DetektCreateBaselineTask @@ -20,6 +21,7 @@ plugins { id("org.springframework.boot") version "4.0.3" id("com.google.cloud.tools.jib") version "3.5.3" id("io.github.simonhauck.release") version "1.5.1" + id("com.github.node-gradle.node") version "7.0.2" } application { @@ -165,6 +167,67 @@ tasks.withType().configureEach { notCompatibleWithConfigurationCache("because https://github.com/GoogleContainerTools/jib/issues/3132") } +// --- Frontend build --- +// Builds the React frontend (debridav-frontend submodule) and bundles its +// static output into the Spring Boot JAR under /static/, so the backend +// serves the UI at /. Pin is tracked as a git submodule — run +// `git submodule update --init` on a fresh clone (or +// `actions/checkout@v4` with `submodules: true` in CI). Skipped if the +// submodule isn't checked out or -PskipFrontend=true is passed. + +val frontendDir = file("debridav-frontend") +val frontendStaticOutput = layout.buildDirectory.dir("generated/frontend/static") +val skipFrontend = providers.gradleProperty("skipFrontend").map { it == "true" }.orElse(false) +val hasFrontend = frontendDir.resolve("package.json").exists() + +node { + version.set("22.12.0") + download.set(true) + workDir.set(layout.buildDirectory.dir("nodejs")) + npmWorkDir.set(layout.buildDirectory.dir("npm")) + nodeProjectDir.set(frontendDir) +} + +tasks.npmInstall { + onlyIf { !skipFrontend.get() && hasFrontend } +} + +val frontendBuild by tasks.registering(NpmTask::class) { + description = "Build frontend static assets" + group = "frontend" + onlyIf { !skipFrontend.get() && hasFrontend } + dependsOn(tasks.npmInstall) + args.set(listOf("run", "build")) + inputs.dir(frontendDir.resolve("src")).optional() + inputs.dir(frontendDir.resolve("public")).optional() + inputs.files( + frontendDir.resolve("package.json"), + frontendDir.resolve("vite.config.ts"), + frontendDir.resolve("tsconfig.json"), + frontendDir.resolve("tsconfig.app.json"), + frontendDir.resolve("tsconfig.node.json"), + frontendDir.resolve("index.html"), + ).optional() + outputs.dir(frontendDir.resolve("dist")) +} + +val copyFrontend by tasks.registering(Copy::class) { + description = "Copy built frontend into resources" + group = "frontend" + onlyIf { !skipFrontend.get() && hasFrontend } + dependsOn(frontendBuild) + from(frontendDir.resolve("dist")) + into(frontendStaticOutput) +} + +sourceSets.main { + resources.srcDir(layout.buildDirectory.dir("generated/frontend")) +} + +tasks.processResources { + dependsOn(copyFrontend) +} + jib { from { platforms { diff --git a/debridav-frontend b/debridav-frontend new file mode 160000 index 00000000..a276a869 --- /dev/null +++ b/debridav-frontend @@ -0,0 +1 @@ +Subproject commit a276a869ef9990159253fed44a2bf19dea15a833 From b521f6f8b9cea54dd881689bfc818b2d11c196ac Mon Sep 17 00:00:00 2001 From: Skjaere Date: Sun, 12 Apr 2026 11:30:00 +0200 Subject: [PATCH 12/61] docs(example): overhaul docker-compose example + monitoring stack Replaces the ad-hoc example/ with a curated docker-compose example that layers a minimal stack, an arrs override, and a full monitoring override (Prometheus + Grafana + dashboards + scraparr for Sonarr/Radarr metrics + postgres-exporter). Includes: - Security scope section in the example README - Rclone and NNTP dashboards ported from debridav-ci - Dracula palette for threshold colors, NNTP streams panel, platform tidy-up - Iterative compose fixes (network labels, healthcheck, Grafana iframe embedding, rclone AppArmor/userns/mount defaults, scraparr image) - UI feature-flag endpoint for the frontend to branch on - Runtime NNTP pool empty-list + blank-host handling - NNTP treated as implicitly enabled when any pool is configured; NNTP_ENABLED env var dropped Co-Authored-By: Claude Opus 4.7 (1M context) --- debridav-frontend | 2 +- example/.env.example | 80 + example/README.md | 153 ++ example/docker-compose.arrs.yml | 56 + example/docker-compose.monitoring.yml | 99 + example/docker-compose.yml | 142 ++ .../provisioning/dashboards/default.yaml | 4 +- .../grafana/provisioning/dashboards/nntp.json | 458 ++++ .../provisioning/dashboards/platform.json | 1654 +++++++++++++++ .../provisioning/dashboards/rclone.json | 739 +++++++ .../provisioning/dashboards/scraparr.json | 431 ++++ .../{custom-queries => }/queries.yaml | 0 example/monitoring/prometheus.yml | 22 + {example => example_old}/.env | 0 {example => example_old}/QUICKSTART.md | 0 {example => example_old}/docker-compose.yaml | 0 {example => example_old}/monitoring/.env | 0 .../monitoring/MONITORING.md | 0 .../monitoring/docker-compose.yaml | 0 example_old/monitoring/grafana/defaults.ini | 1847 +++++++++++++++++ .../provisioning/dashboards/default.yaml | 10 + .../dashboards/definitions/platform.json | 0 .../provisioning/datasources/datasources.yaml | 7 + .../pg-exporter/custom-queries/queries.yaml | 2 + .../monitoring/prometheus/prometheus.yml | 0 .../Definitions/Custom/torbox.yml | 0 .../Definitions/Custom/torrentio.yml | 0 {example => example_old}/rclone.conf | 0 .../debridav/config/ConfigOverrideService.kt | 27 +- .../skjaere/debridav/ui/UiConfigController.kt | 29 + .../debridav/ui/UiConfigurationProperties.kt | 18 + .../usenet/NzbHealthCheckActuatorEndpoint.kt | 2 - .../debridav/usenet/NzbHealthCheckService.kt | 2 - .../debridav/usenet/NzbImportService.kt | 2 - .../usenet/NzbStreamerConfiguration.kt | 4 - .../usenet/pgmq/NzbHealthCheckHandler.kt | 2 - .../usenet/pgmq/NzbHealthRepairHandler.kt | 2 - .../usenet/pgmq/PgmqSpringConfiguration.kt | 2 - 38 files changed, 5771 insertions(+), 25 deletions(-) create mode 100644 example/.env.example create mode 100644 example/README.md create mode 100644 example/docker-compose.arrs.yml create mode 100644 example/docker-compose.monitoring.yml create mode 100644 example/docker-compose.yml create mode 100644 example/monitoring/grafana/provisioning/dashboards/nntp.json create mode 100644 example/monitoring/grafana/provisioning/dashboards/platform.json create mode 100644 example/monitoring/grafana/provisioning/dashboards/rclone.json create mode 100644 example/monitoring/grafana/provisioning/dashboards/scraparr.json rename example/monitoring/pg-exporter/{custom-queries => }/queries.yaml (100%) create mode 100644 example/monitoring/prometheus.yml rename {example => example_old}/.env (100%) rename {example => example_old}/QUICKSTART.md (100%) rename {example => example_old}/docker-compose.yaml (100%) rename {example => example_old}/monitoring/.env (100%) rename {example => example_old}/monitoring/MONITORING.md (100%) rename {example => example_old}/monitoring/docker-compose.yaml (100%) create mode 100644 example_old/monitoring/grafana/defaults.ini create mode 100644 example_old/monitoring/grafana/provisioning/dashboards/default.yaml rename {example => example_old}/monitoring/grafana/provisioning/dashboards/definitions/platform.json (100%) create mode 100644 example_old/monitoring/grafana/provisioning/datasources/datasources.yaml create mode 100644 example_old/monitoring/pg-exporter/custom-queries/queries.yaml rename {example => example_old}/monitoring/prometheus/prometheus.yml (100%) rename {example => example_old}/prowlarr-config/Definitions/Custom/torbox.yml (100%) rename {example => example_old}/prowlarr-config/Definitions/Custom/torrentio.yml (100%) rename {example => example_old}/rclone.conf (100%) create mode 100644 src/main/kotlin/io/skjaere/debridav/ui/UiConfigController.kt create mode 100644 src/main/kotlin/io/skjaere/debridav/ui/UiConfigurationProperties.kt diff --git a/debridav-frontend b/debridav-frontend index a276a869..10a468bc 160000 --- a/debridav-frontend +++ b/debridav-frontend @@ -1 +1 @@ -Subproject commit a276a869ef9990159253fed44a2bf19dea15a833 +Subproject commit 10a468bcbddacc43222e848fe0cee7a001640dca diff --git a/example/.env.example b/example/.env.example new file mode 100644 index 00000000..b5381e32 --- /dev/null +++ b/example/.env.example @@ -0,0 +1,80 @@ +# Most debridav settings — providers, *arr integration, NNTP pools, +# retry timings — are editable at runtime from the UI's Configuration +# pages. This file only needs the bootstrap essentials. + +# --- Required --- + +# Host:container UID/GID for file ownership (defaults to 1000) +PUID=1000 +PGID=1000 +TZ=Etc/UTC + +# Host directory where rclone will mount the WebDAV filesystem. +# Your media server (Jellyfin/Plex) reads from this path. +# Must already exist and be writable by PUID/PGID. +# +# Defaults to $HOME/debridav. On Ubuntu 23.10+ the AppArmor profile +# for fusermount3 only allows FUSE mounts under user home directories +# out of the box, so keep this path under $HOME unless you've loosened +# that profile on the host. +#RCLONE_MOUNT_PATH=/home/you/debridav + +# Database password (picked once, kept stable; stored in the pgdata volume) +POSTGRES_PASSWORD=changeme + +# WebDAV basic auth (rclone + anything mounting the WebDAV needs these) +DEBRIDAV_WEBDAV_USERNAME=debridav +DEBRIDAV_WEBDAV_PASSWORD=changeme + +# --- Optional pre-fill --- +# You can leave everything below blank and set it from the UI after first login. +# Anything set here becomes the default before a UI override is saved. + +# Comma-separated providers to enable on boot (e.g. real_debrid,torbox) +DEBRIDAV_DEBRID_CLIENTS= + +# Provider API keys (UI-editable) +REAL_DEBRID_API_KEY= +PREMIUMIZE_API_KEY= +TORBOX_API_KEY= +EASYNEWS_USERNAME= +EASYNEWS_PASSWORD= + +# Usenet (UI-editable). NNTP is enabled implicitly when at least one +# pool is configured — either via NNTP_HOST + credentials here, or by +# adding a pool from the UI's Configuration → NNTP → Server Pools tab. +NNTP_HOST= +NNTP_PORT=563 +NNTP_USERNAME= +NNTP_PASSWORD= +NNTP_USE_TLS=true + +# *arr integration (UI-editable) +SONARR_INTEGRATION_ENABLED=false +SONARR_HOST=sonarr +SONARR_PORT=8989 +SONARR_API_KEY= + +RADARR_INTEGRATION_ENABLED=false +RADARR_HOST=radarr +RADARR_PORT=7878 +RADARR_API_KEY= + +# --- Port overrides (only if you have conflicts) --- +DEBRIDAV_PORT=8080 +RCLONE_METRICS_PORT=9002 +RCLONE_RC_PORT=5572 +# *arrs stack (only used with docker-compose.arrs.yml) +SONARR_PORT_HOST=8989 +RADARR_PORT_HOST=7878 +PROWLARR_PORT_HOST=9696 +# Monitoring stack (only used with docker-compose.monitoring.yml) +PROMETHEUS_PORT=9090 +GRAFANA_PORT=3000 +GRAFANA_ADMIN_USER=admin +GRAFANA_ADMIN_PASSWORD=admin + +# URL the browser uses to reach Grafana for iframed dashboards. +# Defaults to localhost:3000 — if you access debridav from another +# machine, change this to e.g. http://your-server:3000 +UI_GRAFANA_BASEURL=http://localhost:3000 diff --git a/example/README.md b/example/README.md new file mode 100644 index 00000000..8834d215 --- /dev/null +++ b/example/README.md @@ -0,0 +1,153 @@ +# DebriDAV — Docker Compose example + +Three stacks in one directory, stackable via Compose overrides: + +- **Minimal** (`docker-compose.yml`) — debridav backend + Postgres + rclone mount. Enough to serve the WebDAV, run the UI, and have your media server read from a mounted directory. +- **Arrs** (`docker-compose.arrs.yml`) — adds Sonarr, Radarr, and Prowlarr co-located on the same network, with `/data` pointed at the same rclone mount. +- **Monitoring** (`docker-compose.monitoring.yml`) — adds Prometheus, Grafana (with pre-provisioned debridav dashboards), Postgres exporter, and cAdvisor. + +Arrs and monitoring files are *overrides*: you run them alongside the base file, not instead of it. They stack — combine any or all. + +## Quick start + +```bash +cp .env.example .env +# Edit .env: set POSTGRES_PASSWORD, DEBRIDAV_WEBDAV_USERNAME/PASSWORD, +# and RCLONE_MOUNT_PATH. Everything else can be configured from the UI. + +docker compose up -d +``` + +Wait ~30s for the backend to migrate the database, then: + +- **UI** → http://localhost:8080/ (bundled with the backend JAR) +- **WebDAV** → http://localhost:8080/webdav/ (basic auth: `DEBRIDAV_WEBDAV_USERNAME` / `..._PASSWORD`) +- **Mounted filesystem** → the path you set as `RCLONE_MOUNT_PATH` on the host + +Open the UI, head to the **Configuration** pages, and enable the debrid providers you use (add API keys, add an NNTP pool if you use Usenet, wire up Sonarr/Radarr). The settings persist in the database — no restart needed. + +Point Jellyfin/Plex at `RCLONE_MOUNT_PATH` for media. Point your *arrs at the debridav backend (qBittorrent-compatible API on `:8080`, SABnzbd-compatible API on `:8080/api`). + +## With *arrs (Sonarr / Radarr / Prowlarr) + +```bash +docker compose -f docker-compose.yml -f docker-compose.arrs.yml up -d +``` + +- **Sonarr** → http://localhost:8989 +- **Radarr** → http://localhost:7878 +- **Prowlarr** → http://localhost:9696 + +All three read their media from the same rclone mount as debridav (`/home/debridav/data` inside the containers). Configs live in named Docker volumes (`sonarr-config`, `radarr-config`, `prowlarr-config`). + +On first run, inside each *arr UI: + +- Add debridav as the download client — qBittorrent host `http://debridav:8080`, SABnzbd host `http://debridav:8080/api` for Usenet. +- Set the root/library folder to `/home/debridav/data/tv` (Sonarr) or `/home/debridav/data/movies` (Radarr). debridav creates these on demand. +- In Prowlarr, connect to Sonarr/Radarr via `http://sonarr:8989` / `http://radarr:7878`. + +If you also want debridav to push cleanup actions back to the *arrs (blocklist + research on failed downloads), set `SONARR_INTEGRATION_ENABLED=true` and `SONARR_API_KEY=…` in `.env` (same for Radarr). + +## With monitoring + +```bash +docker compose -f docker-compose.yml -f docker-compose.monitoring.yml up -d +``` + +- **Grafana** → http://localhost:3000 (default admin/admin; override in `.env`) +- **Prometheus** → http://localhost:9090 + +The monitoring stack also includes **scraparr**, a Prometheus exporter for Sonarr/Radarr. It's only useful when the arrs override is also running and `SONARR_API_KEY` / `RADARR_API_KEY` are set in `.env` — otherwise the Sonarr & Radarr dashboard renders empty. + +## Everything together + +```bash +docker compose \ + -f docker-compose.yml \ + -f docker-compose.arrs.yml \ + -f docker-compose.monitoring.yml \ + up -d +``` + +A tip: alias long invocations. E.g. in your shell: + +```bash +alias dc-full="docker compose -f docker-compose.yml -f docker-compose.arrs.yml -f docker-compose.monitoring.yml" +dc-full up -d +dc-full logs -f debridav +``` + +## Host FUSE prerequisites + +The `rclone` container mounts the WebDAV filesystem via FUSE. Two host-side requirements: + +- `/dev/fuse` accessible (default on most distros). +- `user_allow_other` in `/etc/fuse.conf` — required because the mount uses `--allow-other` so other containers (Jellyfin/Plex, the *arrs) can read it: + ```bash + grep -q '^user_allow_other' /etc/fuse.conf || echo 'user_allow_other' | sudo tee -a /etc/fuse.conf + ``` + +**Ubuntu 23.10+ note.** The default AppArmor profile for `fusermount3` only allows FUSE mounts under user home directories. `RCLONE_MOUNT_PATH` therefore defaults to `$HOME/debridav`. If you need the mount elsewhere (e.g. `/srv/debridav` or a separate disk), loosen the profile on the host: + +```bash +sudo ln -s /etc/apparmor.d/fusermount3 /etc/apparmor.d/disable/ +sudo apparmor_parser -R /etc/apparmor.d/fusermount3 +``` + +Or edit `/etc/apparmor.d/fusermount3` and change the `-> @{HOME}/**/` rule to `-> /**/`, then `sudo apparmor_parser -r /etc/apparmor.d/fusermount3`. + +## Security scope + +This compose stack assumes you're running it on a private network. debridav's +built-in JWT auth protects the UI and its API-key endpoints, but everything +else — Grafana (anonymous Viewer), Prometheus, cAdvisor, rclone's RC port, +and the *arrs' web UIs — is published with its defaults. + +If you plan to expose any of this to the internet, put the whole stack behind +your own reverse proxy (Traefik, Caddy) and IdP (Authelia, Authentik, etc.). +TLS, forward-auth, rate limiting, and fine-grained access control are out of +scope for this example. + +## Config at boot vs. in the UI + +Only these need to be set in `.env` — they're the bootstrap essentials, required before the UI comes up: + +| Variable | What it is | +|---|---| +| `POSTGRES_PASSWORD` | Picked once, kept stable. Stored in the `debridav-pgdata` volume. | +| `DEBRIDAV_WEBDAV_USERNAME` / `_PASSWORD` | Basic auth for the WebDAV endpoint. rclone and anyone mounting the filesystem needs these. | +| `RCLONE_MOUNT_PATH` | Host path where rclone mounts the WebDAV filesystem. Must exist and be writable by `PUID:PGID`. | + +Everything else — which debrid providers are enabled, provider API keys, NNTP pools, *arr integration, cache sizes, retry timings — is editable from the UI's **Configuration** pages at runtime. Changes persist in the database and take effect without a restart. + +If you want to pre-seed any of that (e.g. to not have to click through the UI on a fresh deploy) the `.env.example` file has optional variables for all of them. + +## Volumes + +- `debridav-data` — backend's metadata filesystem (lightweight) +- `debridav-pgdata` — Postgres data dir (the important one; back this up) +- `prometheus-data`, `grafana-data` — only exist with the monitoring override + +All are named Docker volumes; inspect with `docker volume ls | grep debridav`. To wipe everything: `docker compose down -v`. + +## Updating + +```bash +docker compose pull debridav +docker compose up -d debridav +``` + +Flyway migrations run on every backend startup. + +## Mounting from outside the compose stack + +If you want to run rclone on the host OS (not in a container) — e.g. to mount debridav on a NAS or under systemd — here's an equivalent `rclone.conf` entry: + +```ini +[debridav] +type = webdav +url = http://:8080/webdav/ +vendor = other +user = +pass = +``` diff --git a/example/docker-compose.arrs.yml b/example/docker-compose.arrs.yml new file mode 100644 index 00000000..786eafa8 --- /dev/null +++ b/example/docker-compose.arrs.yml @@ -0,0 +1,56 @@ +services: + sonarr: + image: lscr.io/linuxserver/sonarr:latest + container_name: debridav-sonarr + restart: unless-stopped + environment: + PUID: ${PUID:-1000} + PGID: ${PGID:-1000} + TZ: ${TZ:-Etc/UTC} + volumes: + - sonarr-config:/config + - ${RCLONE_MOUNT_PATH:-$HOME/debridav}:/home/debridav/data:rshared + ports: + - "${SONARR_PORT_HOST:-8989}:8989" + depends_on: + - rclone + networks: + - debridav-network + + radarr: + image: lscr.io/linuxserver/radarr:latest + container_name: debridav-radarr + restart: unless-stopped + environment: + PUID: ${PUID:-1000} + PGID: ${PGID:-1000} + TZ: ${TZ:-Etc/UTC} + volumes: + - radarr-config:/config + - ${RCLONE_MOUNT_PATH:-$HOME/debridav}:/home/debridav/data:rshared + ports: + - "${RADARR_PORT_HOST:-7878}:7878" + depends_on: + - rclone + networks: + - debridav-network + + prowlarr: + image: lscr.io/linuxserver/prowlarr:latest + container_name: debridav-prowlarr + restart: unless-stopped + environment: + PUID: ${PUID:-1000} + PGID: ${PGID:-1000} + TZ: ${TZ:-Etc/UTC} + volumes: + - prowlarr-config:/config + ports: + - "${PROWLARR_PORT_HOST:-9696}:9696" + networks: + - debridav-network + +volumes: + sonarr-config: + radarr-config: + prowlarr-config: diff --git a/example/docker-compose.monitoring.yml b/example/docker-compose.monitoring.yml new file mode 100644 index 00000000..0d99aed7 --- /dev/null +++ b/example/docker-compose.monitoring.yml @@ -0,0 +1,99 @@ +services: + # Extend the base debridav service with Grafana config so the UI's + # Dashboard tab can embed the dashboards from this stack. + debridav: + environment: + DEBRIDAV_UI_GRAFANA_BASEURL: ${UI_GRAFANA_BASEURL:-http://localhost:3000} + DEBRIDAV_UI_GRAFANA_DASHBOARDS_0_LABEL: Platform + DEBRIDAV_UI_GRAFANA_DASHBOARDS_0_PATH: /d/f6415a14-af35-4d81-8efb-9a018b0e3ed3/debridav-platform + DEBRIDAV_UI_GRAFANA_DASHBOARDS_1_LABEL: Rclone + DEBRIDAV_UI_GRAFANA_DASHBOARDS_1_PATH: /d/rclone-mounts-dashboard/rclone-mounts + DEBRIDAV_UI_GRAFANA_DASHBOARDS_2_LABEL: NNTP + DEBRIDAV_UI_GRAFANA_DASHBOARDS_2_PATH: /d/nntp-pool-dashboard/nntp-connection-pool + DEBRIDAV_UI_GRAFANA_DASHBOARDS_3_LABEL: Sonarr & Radarr + DEBRIDAV_UI_GRAFANA_DASHBOARDS_3_PATH: /d/scraparr-dashboard/sonarr-radarr + + prometheus: + image: prom/prometheus:v2.54.1 + container_name: debridav-prometheus + restart: unless-stopped + volumes: + - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus-data:/prometheus + ports: + - "${PROMETHEUS_PORT:-9090}:9090" + networks: + - debridav-network + + grafana: + image: grafana/grafana:11.2.0 + container_name: debridav-grafana + restart: unless-stopped + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin} + # Allow the debridav UI to embed dashboards via iframe + GF_SECURITY_ALLOW_EMBEDDING: "true" + # Anonymous read-only access so embedded iframes don't hit a login wall + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer + volumes: + - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - ./monitoring/grafana/defaults.ini:/etc/grafana/defaults.ini:ro + - grafana-data:/var/lib/grafana + ports: + - "${GRAFANA_PORT:-3000}:3000" + networks: + - debridav-network + + postgres-exporter: + image: quay.io/prometheuscommunity/postgres-exporter:v0.15.0 + container_name: debridav-postgres-exporter + restart: unless-stopped + environment: + DATA_SOURCE_URI: postgres:5432/debridav?sslmode=disable + DATA_SOURCE_USER: debridav + DATA_SOURCE_PASS: ${POSTGRES_PASSWORD} + PG_EXPORTER_EXTEND_QUERY_PATH: /custom-queries/queries.yaml + volumes: + - ./monitoring/pg-exporter:/custom-queries:ro + depends_on: + postgres: + condition: service_healthy + networks: + - debridav-network + + # Scrapes Sonarr/Radarr APIs and re-exposes as Prometheus metrics. + # Only useful when the arrs compose override is also running and + # SONARR_API_KEY / RADARR_API_KEY are set in .env. + scraparr: + image: ghcr.io/thecfu/scraparr:3.0.3 + container_name: debridav-scraparr + restart: unless-stopped + environment: + SONARR_URL: ${SONARR_URL_INTERNAL:-http://sonarr:8989} + SONARR_API_KEY: ${SONARR_API_KEY:-} + RADARR_URL: ${RADARR_URL_INTERNAL:-http://radarr:7878} + RADARR_API_KEY: ${RADARR_API_KEY:-} + networks: + - debridav-network + + cadvisor: + image: gcr.io/cadvisor/cadvisor:v0.52.1 + container_name: debridav-cadvisor + restart: unless-stopped + privileged: true + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker:/var/lib/docker:ro + - /dev/disk:/dev/disk:ro + devices: + - /dev/kmsg + networks: + - debridav-network + +volumes: + prometheus-data: + grafana-data: diff --git a/example/docker-compose.yml b/example/docker-compose.yml new file mode 100644 index 00000000..f414159a --- /dev/null +++ b/example/docker-compose.yml @@ -0,0 +1,142 @@ +services: + debridav: + image: ghcr.io/skjaere/debridav:latest + container_name: debridav + restart: unless-stopped + environment: + PUID: ${PUID:-1000} + PGID: ${PGID:-1000} + TZ: ${TZ:-Etc/UTC} + DEBRIDAV_ROOTPATH: /data/debridav + DEBRIDAV_DOWNLOADPATH: /downloads + DEBRIDAV_MOUNTPATH: /home/debridav/data + DEBRIDAV_DEBRIDCLIENTS: ${DEBRIDAV_DEBRID_CLIENTS} + DEBRIDAV_WEBDAV_USERNAME: ${DEBRIDAV_WEBDAV_USERNAME} + DEBRIDAV_WEBDAV_PASSWORD: ${DEBRIDAV_WEBDAV_PASSWORD} + SPRING_DATASOURCE_URL: jdbc:postgresql://postgres:5432/debridav?user=debridav&password=${POSTGRES_PASSWORD} + # Debrid providers (leave blank for unused ones) + REALDEBRID_APIKEY: ${REAL_DEBRID_API_KEY:-} + PREMIUMIZE_APIKEY: ${PREMIUMIZE_API_KEY:-} + TORBOX_APIKEY: ${TORBOX_API_KEY:-} + EASYNEWS_USERNAME: ${EASYNEWS_USERNAME:-} + EASYNEWS_PASSWORD: ${EASYNEWS_PASSWORD:-} + # NNTP (enabled implicitly when at least one pool is configured) + NNTP_POOLS_0_HOST: ${NNTP_HOST:-} + NNTP_POOLS_0_PORT: ${NNTP_PORT:-563} + NNTP_POOLS_0_USERNAME: ${NNTP_USERNAME:-} + NNTP_POOLS_0_PASSWORD: ${NNTP_PASSWORD:-} + NNTP_POOLS_0_USE_TLS: ${NNTP_USE_TLS:-true} + # *arr integration (optional) + SONARR_INTEGRATIONENABLED: ${SONARR_INTEGRATION_ENABLED:-false} + SONARR_HOST: ${SONARR_HOST:-} + SONARR_PORT: ${SONARR_PORT:-8989} + SONARR_APIKEY: ${SONARR_API_KEY:-} + RADARR_INTEGRATIONENABLED: ${RADARR_INTEGRATION_ENABLED:-false} + RADARR_HOST: ${RADARR_HOST:-} + RADARR_PORT: ${RADARR_PORT:-7878} + RADARR_APIKEY: ${RADARR_API_KEY:-} + volumes: + - debridav-data:/data/debridav + ports: + - "${DEBRIDAV_PORT:-8080}:8080" + # Image is trimmed JRE — no curl/wget. Use bash's /dev/tcp to probe + # the readiness actuator directly. + healthcheck: + test: + - CMD-SHELL + - | + bash -c ' + exec 3<>/dev/tcp/localhost/8080 && + printf "GET /actuator/health/readiness HTTP/1.0\r\nHost: localhost\r\n\r\n" >&3 && + grep -q "200 " <&3 + ' + interval: 5s + timeout: 3s + start_period: 10s + retries: 60 + depends_on: + postgres: + condition: service_healthy + networks: + - debridav-network + + postgres: + image: postgres:17 + container_name: debridav-postgres + restart: unless-stopped + environment: + POSTGRES_USER: debridav + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: debridav + PGDATA: /var/lib/postgresql/data/pgdata + volumes: + - debridav-pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U debridav -d debridav"] + interval: 2s + timeout: 5s + retries: 30 + networks: + - debridav-network + + rclone: + image: rclone/rclone:latest + container_name: debridav-rclone + restart: unless-stopped + environment: + TZ: ${TZ:-Etc/UTC} + PUID: ${PUID:-1000} + PGID: ${PGID:-1000} + WEBDAV_USER: ${DEBRIDAV_WEBDAV_USERNAME} + WEBDAV_PASS_PLAIN: ${DEBRIDAV_WEBDAV_PASSWORD} + volumes: + - ${RCLONE_MOUNT_PATH:-$HOME/debridav}:/home/debridav/data:rshared + cap_add: + - SYS_ADMIN + security_opt: + - apparmor:unconfined + devices: + - /dev/fuse:/dev/fuse:rwm + # Writes an rclone.conf at startup with the WebDAV password obscured, + # then mounts. Keeping the config in a file (vs. env vars) matches the + # pattern most rclone users are familiar with. + entrypoint: /bin/sh + command: + - -c + - | + OBSCURED=$$(rclone obscure "$$WEBDAV_PASS_PLAIN") + mkdir -p /config/rclone + cat > /config/rclone/rclone.conf <` (for example `Grafana/9.0.0`). +user_agent = + +#################################### Analytics ########################### +[analytics] +# Server reporting, sends usage counters to stats.grafana.org every 24 hours. +# No ip addresses are being tracked, only simple counters to track +# running instances, dashboard and error counts. It is very helpful to us. +# Change this option to false to disable reporting. +reporting_enabled = true + +# The name of the distributor of the Grafana instance. Ex hosted-grafana, grafana-labs +reporting_distributor = grafana-labs + +# Set to false to disable all checks to https://grafana.com +# for new versions of grafana. The check is used +# in some UI views to notify that a grafana update exists. +# This option does not cause any auto updates, nor send any information +# only a GET request to https://grafana.com/api/grafana/versions/stable to get the latest version. +check_for_updates = true + +# Set to false to disable all checks to https://grafana.com +# for new versions of plugins. The check is used +# in some UI views to notify that a plugin update exists. +# This option does not cause any auto updates, nor send any information +# only a GET request to https://grafana.com to get the latest versions. +check_for_plugin_updates = true + +# Google Analytics universal tracking code, only enabled if you specify an id here +google_analytics_ua_id = + +# Google Analytics 4 tracking code, only enabled if you specify an id here +google_analytics_4_id = + +# When Google Analytics 4 Enhanced event measurement is enabled, we will try to avoid sending duplicate events and let Google Analytics 4 detect navigation changes, etc. +google_analytics_4_send_manual_page_views = false + +# Google Tag Manager ID, only enabled if you specify an id here +google_tag_manager_id = + +# Rudderstack write key, enabled only if rudderstack_data_plane_url is also set +rudderstack_write_key = + +# Rudderstack data plane url, enabled only if rudderstack_write_key is also set +rudderstack_data_plane_url = + +# Rudderstack SDK url, optional, only valid if rudderstack_write_key and rudderstack_data_plane_url is also set +rudderstack_sdk_url = + +# Rudderstack Config url, optional, used by Rudderstack SDK to fetch source config +rudderstack_config_url = + +# Rudderstack Integrations URL, optional. Only valid if you pass the SDK version 1.1 or higher +rudderstack_integrations_url = + +# Intercom secret, optional, used to hash user_id before passing to Intercom via Rudderstack +intercom_secret = + +# Application Insights connection string. Specify an URL string to enable this feature. +application_insights_connection_string = + +# Optional. Specifies an Application Insights endpoint URL where the endpoint string is wrapped in backticks ``. +application_insights_endpoint_url = + +# Controls if the UI contains any links to user feedback forms +feedback_links_enabled = true + +#################################### Security ############################ +[security] +# disable creation of admin user on first start of grafana +disable_initial_admin_creation = false + +# default admin user, created on startup +admin_user = admin + +# default admin password, can be changed before first start of grafana, or in profile settings +admin_password = admin + +# default admin email, created on startup +admin_email = admin@localhost + +# used for signing +secret_key = SW2YcwTIb9zpOOhoPsMm + +# current key provider used for envelope encryption, default to static value specified by secret_key +encryption_provider = secretKey.v1 + +# list of configured key providers, space separated (Enterprise only): e.g., awskms.v1 azurekv.v1 +available_encryption_providers = + +# disable gravatar profile images +disable_gravatar = false + +# data source proxy whitelist (ip_or_domain:port separated by spaces) +data_source_proxy_whitelist = + +# disable protection against brute force login attempts +disable_brute_force_login_protection = false + +# set to true if you host Grafana behind HTTPS. default is false. +cookie_secure = false + +# set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict", "none" and "disabled" +cookie_samesite = lax + +# set to true if you want to allow browsers to render Grafana in a ,