Reduce maintenance notification noise

backend/alembic/versions/0005_indexer_health_notification_cooldown.py

@@ -0,0 +1,39 @@
+"""Track indexer health notification cooldown.
+
+Revision ID: 0005_indexer_notify_cooldown
+Revises: 0004_subscription_search_cadence
+Create Date: 2026-07-07
+"""
+
+from __future__ import annotations
+
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = "0005_indexer_notify_cooldown"
+down_revision = "0004_subscription_search_cadence"
+branch_labels = None
+depends_on = None
+
+
+def _has_table(table_name: str) -> bool:
+    bind = op.get_bind()
+    return sa.inspect(bind).has_table(table_name)
+
+
+def _has_column(table_name: str, column_name: str) -> bool:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    return any(column["name"] == column_name for column in inspector.get_columns(table_name))
+
+
+def upgrade() -> None:
+    if not _has_table("indexer_site_health") or _has_column("indexer_site_health", "last_notified_at"):
+        return
+    op.add_column("indexer_site_health", sa.Column("last_notified_at", sa.Text(), nullable=True))
+
+
+def downgrade() -> None:
+    if _has_table("indexer_site_health") and _has_column("indexer_site_health", "last_notified_at"):
+        op.drop_column("indexer_site_health", "last_notified_at")

backend/app/db/repositories/indexer_site_health_repository.py

@@ -23,6 +23,7 @@ def _to_model(row: IndexerSiteHealthORM) -> IndexerSiteHealthStatus:
                 "consecutive_failures": row.consecutive_failures,
                 "last_error_message": row.last_error_message,
                 "notify_pending": bool(row.notify_pending),
+                "last_notified_at": row.last_notified_at,
                 "client_type": row.client_type,
             }
         )
@@ -51,6 +52,7 @@ def upsert(self, status: IndexerSiteHealthStatus) -> IndexerSiteHealthStatus:
             row.consecutive_failures = status.consecutive_failures
             row.last_error_message = status.last_error_message
             row.notify_pending = bool(status.notify_pending)
+            row.last_notified_at = status.last_notified_at.isoformat() if status.last_notified_at else None
             row.client_type = status.client_type
             session.commit()
             return status

backend/app/db/sql/models.py

@@ -440,6 +440,7 @@ class IndexerSiteHealthORM(Base):
     consecutive_failures: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
     last_error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
     notify_pending: Mapped[bool] = mapped_column(Integer, nullable=False, default=0)
+    last_notified_at: Mapped[str | None] = mapped_column(Text, nullable=True)
     client_type: Mapped[str] = mapped_column(Text, nullable=False, default="jackett")
 
     __table_args__ = (

backend/app/schemas/runtime/indexer_site_health.py

@@ -16,6 +16,7 @@ class IndexerSiteHealthStatus(BaseModel):
     consecutive_failures: int = 0
     last_error_message: Optional[str] = None
     notify_pending: bool = False
+    last_notified_at: Optional[datetime] = None
     client_type: str = "jackett"
 
 

backend/app/services/application/workflows/media_server_sync/season_runner.py

@@ -23,6 +23,8 @@ async def sync_one_season(
         state: MediaServerSyncState,
         now: float,
         sync_cfg: MediaServerSyncConfig,
+        *,
+        initial_sync: bool | None = None,
     ) -> bool:
         media = await media_service.info(media_id, season_number=season_number)
         if not media:
@@ -79,7 +81,10 @@ async def sync_one_season(
                 error=str(exc),
             )
             raise
-        if self._should_emit_scheduler_completed_event(needs.missing_flags):
+        if self._should_emit_scheduler_completed_event(
+            state.last_success_at is None if initial_sync is None else initial_sync,
+            needs.missing_flags,
+        ):
             emit_media_server_sync_events(
                 EventType.MEDIA_SERVER_SYNC_COMPLETED,
                 media,
@@ -129,8 +134,8 @@ async def apply_updates(
         )
 
     @staticmethod
-    def _should_emit_scheduler_completed_event(missing_flags: list[str]) -> bool:
-        return set(missing_flags) != {"stale"}
+    def _should_emit_scheduler_completed_event(initial_sync: bool, missing_flags: list[str]) -> bool:
+        return initial_sync and set(missing_flags) != {"stale"}
 
 
 media_server_sync_season_runner = MediaServerSyncSeasonRunner()

backend/app/services/application/workflows/media_server_sync/service.py

@@ -426,6 +426,7 @@ async def _sync_one(
         sync_cfg: MediaServerSyncConfig,
     ) -> MediaServerSyncItemResult:
         state = media_server_sync_state.get_or_create_state(media_server.id, media_id)
+        initial_sync = state.last_success_at is None
         out = MediaServerSyncItemResult(media_server_id=media_server.id, media_id=media_id)
         try:
             directory_ids = media_server_sync_config.directory_ids_for_media_server(media_server.id)
@@ -443,6 +444,7 @@ async def _sync_one(
                     state,
                     now,
                     sync_cfg,
+                    initial_sync=initial_sync,
                 )
                 out.updated = out.updated or season_updated
             return out

backend/app/services/config/indexer_client_settings.py

@@ -2,7 +2,7 @@
 
 import logging
 from collections.abc import Mapping
-from datetime import datetime
+from datetime import datetime, timedelta
 
 from app.db.repositories.indexer_site_health_repository import IndexerSiteHealthRepository
 from app.db.repositories.settings_sqlite_repository import SettingsSqliteRepository
@@ -16,6 +16,7 @@
 
 
 INDEXER_SITE_FAILURE_NOTIFY_THRESHOLD = 3
+INDEXER_SITE_FAILURE_NOTIFY_COOLDOWN = timedelta(hours=24)
 logger = logging.getLogger("app.services.config.indexer_client_settings")
 
 
@@ -29,7 +30,7 @@ def _get_record(self, indexer_id: str, site_id: str) -> IndexerSiteHealthStatus
     def _upsert(self, status: IndexerSiteHealthStatus) -> IndexerSiteHealthStatus:
         return self._repo.upsert(status)
 
-    def _emit_unhealthy_event(self, status: IndexerSiteHealthStatus) -> None:
+    def _emit_unhealthy_event(self, status: IndexerSiteHealthStatus) -> bool:
         try:
             event_service.emit(
                 EventCreate(
@@ -52,13 +53,15 @@ def _emit_unhealthy_event(self, status: IndexerSiteHealthStatus) -> None:
                     correlation_id=f"indexer:{status.indexer_id}:site:{status.site_id}:unhealthy",
                 )
             )
+            return True
         except Exception as exc:
             logger.error(
                 "Failed to emit indexer site unhealthy event for %s/%s: %s",
                 status.indexer_id,
                 status.site_id,
                 exc,
             )
+            return False
 
     def record_outcomes(self, outcomes: list[IndexerSiteSearchOutcome]) -> None:
         for outcome in outcomes:
@@ -103,6 +106,7 @@ def record_success(
             consecutive_failures=0,
             last_error_message=None,
             notify_pending=False,
+            last_notified_at=current.last_notified_at if current else None,
             client_type=client_type,
         )
         return self._upsert(status)
@@ -121,6 +125,10 @@ def record_failure(
         now = datetime.now()
         previous_failures = current.consecutive_failures if current else 0
         consecutive_failures = previous_failures + 1
+        should_emit = (
+            consecutive_failures >= INDEXER_SITE_FAILURE_NOTIFY_THRESHOLD
+            and self._notify_cooldown_elapsed(current.last_notified_at if current else None, now)
+        )
         status = IndexerSiteHealthStatus(
             indexer_id=indexer_id,
             indexer_name=indexer_name,
@@ -133,13 +141,19 @@ def record_failure(
             consecutive_failures=consecutive_failures,
             last_error_message=error_message,
             notify_pending=consecutive_failures >= INDEXER_SITE_FAILURE_NOTIFY_THRESHOLD,
+            last_notified_at=current.last_notified_at if current else None,
             client_type=client_type,
         )
         saved = self._upsert(status)
-        if previous_failures < INDEXER_SITE_FAILURE_NOTIFY_THRESHOLD <= consecutive_failures:
-            self._emit_unhealthy_event(saved)
+        if should_emit and self._emit_unhealthy_event(saved):
+            saved.last_notified_at = now
+            saved = self._upsert(saved)
         return saved
 
+    @staticmethod
+    def _notify_cooldown_elapsed(last_notified_at: datetime | None, now: datetime) -> bool:
+        return last_notified_at is None or (now - last_notified_at) >= INDEXER_SITE_FAILURE_NOTIFY_COOLDOWN
+
     def list_by_indexer(self, indexer_id: str) -> list[IndexerSiteHealthStatus]:
         return self._repo.list_by_indexer(indexer_id)
 

backend/tests/test_indexer_site_health_alerts.py

@@ -1,3 +1,5 @@
+from datetime import datetime, timedelta
+
 from app.schemas.constants.event_types import EventTypes
 from app.schemas.domain.event import EventLevel
 from app.schemas.runtime.indexer_site_health import IndexerSiteHealthStatus
@@ -103,3 +105,107 @@ def test_indexer_site_success_clears_notify_pending_and_allows_future_threshold(
     assert status is not None
     assert status.consecutive_failures == 3
     assert status.notify_pending is True
+
+
+def test_indexer_site_unhealthy_event_respects_notification_cooldown(monkeypatch):
+    emitted_events = []
+    monkeypatch.setattr(
+        "app.services.config.indexer_client_settings.event_service.emit",
+        lambda event: emitted_events.append(event),
+    )
+    state = IndexerSiteHealthState(repo=_FakeIndexerSiteHealthRepository())
+
+    for _ in range(3):
+        state.record_failure(
+            indexer_id="prowlarr",
+            indexer_name="Prowlarr",
+            site_id="audiences",
+            site_name="Audiences",
+            error_message="disabled",
+        )
+    state.record_success(
+        indexer_id="prowlarr",
+        indexer_name="Prowlarr",
+        site_id="audiences",
+        site_name="Audiences",
+    )
+    for _ in range(3):
+        state.record_failure(
+            indexer_id="prowlarr",
+            indexer_name="Prowlarr",
+            site_id="audiences",
+            site_name="Audiences",
+            error_message="disabled again",
+        )
+
+    assert len(emitted_events) == 1
+
+
+def test_indexer_site_unhealthy_event_repeats_after_notification_cooldown(monkeypatch):
+    emitted_events = []
+    monkeypatch.setattr(
+        "app.services.config.indexer_client_settings.event_service.emit",
+        lambda event: emitted_events.append(event),
+    )
+    repo = _FakeIndexerSiteHealthRepository()
+    state = IndexerSiteHealthState(repo=repo)
+    old_notification = datetime.now() - timedelta(hours=25)
+    repo.upsert(
+        IndexerSiteHealthStatus(
+            indexer_id="prowlarr",
+            indexer_name="Prowlarr",
+            site_id="audiences",
+            site_name="Audiences",
+            status="unhealthy",
+            consecutive_failures=7,
+            notify_pending=True,
+            last_notified_at=old_notification,
+        )
+    )
+
+    state.record_failure(
+        indexer_id="prowlarr",
+        indexer_name="Prowlarr",
+        site_id="audiences",
+        site_name="Audiences",
+        error_message="still disabled",
+    )
+
+    assert len(emitted_events) == 1
+
+
+def test_indexer_site_unhealthy_event_failure_does_not_start_cooldown(monkeypatch):
+    attempts = []
+
+    def fake_emit(event):
+        attempts.append(event)
+        if len(attempts) == 1:
+            raise RuntimeError("event store unavailable")
+
+    monkeypatch.setattr(
+        "app.services.config.indexer_client_settings.event_service.emit",
+        fake_emit,
+    )
+    state = IndexerSiteHealthState(repo=_FakeIndexerSiteHealthRepository())
+
+    for _ in range(3):
+        status = state.record_failure(
+            indexer_id="prowlarr",
+            indexer_name="Prowlarr",
+            site_id="audiences",
+            site_name="Audiences",
+            error_message="disabled",
+        )
+
+    assert status.last_notified_at is None
+
+    status = state.record_failure(
+        indexer_id="prowlarr",
+        indexer_name="Prowlarr",
+        site_id="audiences",
+        site_name="Audiences",
+        error_message="still disabled",
+    )
+
+    assert len(attempts) == 2
+    assert status.last_notified_at is not None