feat: default site backups

admin/frontend/dashboard/src/components/sites/BackupConfigDialog.vue

@@ -105,7 +105,7 @@ const open = async () => {
   show.value = true
   try {
     const data = await sitesApi.backups.schedule.get(props.siteName)
-    isEnabled.value = !!data.schedule
+    isEnabled.value = !!data.schedule || !!data.retention?.calendar_rotation
     applySchedule(data.schedule)
     applyRetention(data.retention)
   } catch (e) {

admin/frontend/dashboard/src/components/sites/Backups.vue

@@ -41,13 +41,15 @@ const error = ref('')
 
 const configRef = ref<InstanceType<typeof BackupConfigDialog> | null>(null)
 const config = ref<BackupSchedule | null>(null)
-const enabled = computed(() => !!config.value?.schedule)
+const isDefault = computed(() => !!config.value?.retention?.calendar_rotation)
+const enabled = computed(() => !!config.value?.schedule || isDefault.value)
 
-const scheduleSummary = computed(() =>
-  enabled.value
+const scheduleSummary = computed(() => {
+  if (isDefault.value) return 'Daily, with the default retention policy.'
+  return enabled.value
     ? `${cronToLabel(config.value?.schedule)}.`
-    : 'Manual backups are kept until you delete them.',
-)
+    : 'Manual backups are kept until you delete them.'
+})
 
 const loadConfig = async () => {
   try {

docs/configuration.md

@@ -262,6 +262,12 @@ Set `proxy.cloudflare = true` when site domains use Cloudflare's proxy. nginx th
 
 `[frappe_cloud] url` is the Frappe Cloud that a site restore copies backups from. The default is `https://cloud.frappe.io`. Set it to use another Frappe Cloud, for example a local Press during development.
 
+### Scheduled Backups
+
+Each bench has one hourly `0 * * * *` cron job for scheduled backups. It is installed with the first enrolled site. Sites without a custom backup schedule use this default. New sites are enrolled, including sites created from a backup, and the `enroll_sites_in_scheduled_backups` patch enrolls existing sites that have no backup schedule. Each run reads the last 24 hours of the bench audit log once, then queues a logical backup of the database and site config for each enrolled site with no successful backup in the last 23 hours. Measuring 23 hours keeps each site's backup at the same hour every day. A backup also includes public and private files when the site has no file backup that day (UTC). Physical backups are not scheduled. After six failed backups in 24 hours, the job skips the site until the count drops. The sixth failure since the last successful backup adds a "Backups of <site> are failing" notification to the bench feed. Scheduled backups stay on disk in `sites/<site>/backups`, outside the directory that Frappe prunes. When S3 is configured, each backup is also uploaded and expired runs are deleted from S3. The host must have cron and the Pilot task worker running. A scheduled backup does not start while another task holds the site.
+
+After each backup, Pilot clears expired runs with the default GFS policy: every run is kept for 7 days, Sunday runs for 28 days, runs on the first day of a month for 366 days, and January 1 runs for 3,653 days. The newest run is always kept. The `backup_retention` object in `site_config.json` marks this policy with `scheme: "gfs"` and `calendar_rotation: true`. A custom schedule set in the dashboard or through the backup schedule API replaces the default. It installs a cron job for the site and stores a GFS or FIFO retention record without `calendar_rotation`, so the bench job skips the site and the count-based policy clears its runs. Turning off automated backups removes the schedule and the retention record, and the site then keeps only manual backups.
+
 ### Telemetry
 
 `[telemetry]` is one credential for both shippers. `endpoint` is the base URL of the region's Datum, with no path: metrics append `/v1/ingest` and logs append `/v1/logs/ingest`. `token` is the JWT Central mints for the region, which Datum tells apart by route rather than by token, so one serves both. `logs_enabled` and `metrics_enabled` switch each shipper independently.

pilot/config/backup.py

@@ -14,6 +14,7 @@ class BackupConfig:
     keep_weekly: int = 5  # one per week across the current month
     keep_monthly: int = 12  # one per month, past year
     keep_yearly: int = 5  # one per year, kept a few years
+    calendar_rotation: bool = False
 
     @property
     def counts(self) -> dict[str, int]:

pilot/core/bench/__init__.py

@@ -12,6 +12,7 @@
 if TYPE_CHECKING:
     from pilot.config import S3Config, SiteConfig
     from pilot.core.app import App, NewAppOptions, RevisionPin
+    from pilot.core.bench.backups import BenchBackups
     from pilot.core.bench.hostname_aliases import HostnameAliases
     from pilot.core.bench.migration.store import MigrationStore
     from pilot.core.bench.processes import BenchProcesses
@@ -103,6 +104,12 @@ def processes(self) -> "BenchProcesses":
 
         return BenchProcesses(self)
 
+    @cached_property
+    def backups(self) -> "BenchBackups":
+        from pilot.core.bench.backups import BenchBackups
+
+        return BenchBackups(self)
+
     @cached_property
     def notifications(self) -> "NotificationStore":
         from pilot.core.notification import NotificationStore

pilot/core/bench/audit_log.py

@@ -4,6 +4,7 @@
 
 from collections.abc import Callable
 from dataclasses import dataclass
+from datetime import datetime
 
 from pilot.internal.jsonl_log import JsonlLog, utc_now
 
@@ -67,10 +68,14 @@ def __init__(self, bench) -> None:
     def append(self, entry_type: str, entry: dict) -> None:
         self._log.append({"type": entry_type, "logged_at": utc_now(), **entry})
 
-    def entries(self, entry_type=None, site=None, status=None, jti=None, limit=None) -> list[dict]:
+    def entries(
+        self, entry_type=None, site=None, status=None, jti=None, limit=None, since: datetime | None = None
+    ) -> list[dict]:
         """Return matching records newest first across weekly files."""
         matched = []
         for record in self._log.read_newest_first():
+            if since is not None and datetime.fromisoformat(record["logged_at"]) < since:
+                break
             if self._matches(record, entry_type, site, status, jti):
                 matched.append(record)
                 if limit is not None and len(matched) >= limit:

pilot/core/bench/backups.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+import sys
+from collections import defaultdict
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from pilot.core.bench.audit_log import AuditLog
+
+if TYPE_CHECKING:
+    from pilot.core.bench import Bench
+
+CRON_JOB_KEY = "scheduled-backups"
+CRON_SCHEDULE = "0 * * * *"
+
+
+class BenchBackups:
+    """The hourly scheduled backup poll for every site on a bench."""
+
+    def __init__(self, bench: Bench) -> None:
+        self.bench = bench
+
+    def install_schedule(self) -> None:
+        from pilot.managers.cron import CronManager, cron_module_command
+
+        command = cron_module_command(
+            "pilot.core.bench.backups", [self.bench.path], self.bench.logs_path / "scheduled-backups.log"
+        )
+        CronManager(self.bench.path).set_schedule(CRON_JOB_KEY, CRON_SCHEDULE, command)
+
+    def get_recent_history(self, now: datetime) -> dict[str, list[tuple[datetime, dict]]]:
+        """Backup records from the last 24 hours, newest first, by site. One audit log read."""
+        history = defaultdict(list)
+        for entry in AuditLog(self.bench).entries(entry_type="backup", since=now - timedelta(days=1)):
+            if entry.get("event") == "backup":
+                history[entry["site"]].append((datetime.fromisoformat(entry["logged_at"]), entry))
+        return history
+
+    def queue_scheduled(self, now: datetime | None = None) -> None:
+        now = now or datetime.now(UTC)
+        history = self.get_recent_history(now)
+        for site in self.bench.sites():
+            site.backups.queue_scheduled(now, history.get(site.config.name, []))
+
+
+if __name__ == "__main__":
+    from pilot.core.bench import Bench
+
+    BenchBackups(Bench(Path(sys.argv[1]))).queue_scheduled()

pilot/core/notification/events.py

@@ -107,6 +107,22 @@ def record_alert(
         return False
 
 
+def backups_failing(bench: "Bench", site: str, failures: int) -> None:
+    """Raised once when a site reaches the failure limit since its last successful backup."""
+    try:
+        bench.notifications.create(
+            f"Backups of {site} are failing",
+            category="Sites",
+            event="backups_failing",
+            severity="Error",
+            message=f"{failures} backup attempts failed since the last successful backup.",
+            site=site,
+            action_route=f"/sites/{site}",
+        )
+    except Exception as exc:
+        logging.warning("Notification skipped for failing backups of %s: %s", site, exc)
+
+
 def _readable(command: str) -> str:
     """`backup-site` -> `Backup site`, so the title reads like a sentence."""
     return command.replace("-", " ").capitalize()

pilot/core/site/__init__.py

@@ -329,11 +329,15 @@ def provision(
         admin_password: str,
         db_type: str | None = None,
         on_progress: Callable[[str], None] = lambda message: None,
+        *,
+        schedule_backups: bool = True,
     ) -> "Site":
         """Create a site, install apps, reload nginx, and issue TLS if needed."""
         from pilot.core.site.provisioning import SiteProvisioner
 
-        return SiteProvisioner(bench, name, apps, admin_password, db_type).provision(on_progress)
+        return SiteProvisioner(bench, name, apps, admin_password, db_type).provision(
+            on_progress, schedule_backups=schedule_backups
+        )
 
     def set_ssl(self, enabled: bool) -> None:
         set_site_ssl_flag(self.bench.sites_path, self.config.name, enabled)

pilot/core/site/backups.py

@@ -2,9 +2,12 @@
 
 import re
 from dataclasses import asdict
+from datetime import datetime, timedelta
+from hashlib import sha256
 from pathlib import Path
 from typing import TYPE_CHECKING
 
+from pilot.config.backup import BackupConfig
 from pilot.config.site_backup import clear_retention, read_retention, write_retention
 from pilot.core.site.retention import BackupRetentionPolicy
 from pilot.integrations.s3.backups import OffsiteBackup
@@ -13,6 +16,7 @@
 if TYPE_CHECKING:
     from pilot.core.site import Site
 
+MAX_DAILY_FAILURES = 6
 _TS_RE = re.compile(r"^(\d{8}_\d{6})")
 
 
@@ -153,7 +157,7 @@ def schedule(self) -> dict:
         retention = read_retention(self.site.path / "site_config.json")
         return {"schedule": schedule, "retention": asdict(retention) if retention else None}
 
-    def set_schedule(self, schedule: str, retention) -> dict:
+    def set_schedule(self, schedule: str, retention: BackupConfig) -> dict:
         from pilot.managers.cron import CronManager
 
         CronManager(self.site.bench.path).set_schedule(
@@ -164,6 +168,68 @@ def set_schedule(self, schedule: str, retention) -> dict:
         write_retention(self.site.path / "site_config.json", retention)
         return self.schedule()
 
+    def enable_default_schedule(self) -> None:
+        """Default backups for a site without a custom schedule. `set_schedule` replaces them.
+        The bench job goes in first, so a failed install leaves the site unenrolled for a retry."""
+        self.site.bench.backups.install_schedule()
+        write_retention(
+            self.site.path / "site_config.json",
+            BackupConfig(keep_weekly=4, keep_yearly=10, calendar_rotation=True),
+        )
+
+    @property
+    def is_default_schedule_enabled(self) -> bool:
+        retention = read_retention(self.site.path / "site_config.json")
+        return retention is not None and retention.calendar_rotation
+
+    def get_recent_backup_history(self, now: datetime) -> list[tuple[datetime, dict]]:
+        return self.site.bench.backups.get_recent_history(now).get(self.site.config.name, [])
+
+    def get_failures_since_last_success(self, now: datetime) -> int:
+        failures = 0
+        for _, entry in self.get_recent_backup_history(now):
+            if entry["status"] == "success":
+                break
+            failures += entry["status"] == "failed"
+        return failures
+
+    def get_scheduled_backup(
+        self, now: datetime, history: list[tuple[datetime, dict]] | None = None
+    ) -> dict[str, bool] | None:
+        """Daily logical backup, due 23 hours after the last success so it keeps its hour. Files once a day."""
+        if not self.is_default_schedule_enabled:
+            return None
+
+        backups = self.get_recent_backup_history(now) if history is None else history
+        successful = [(when, entry) for when, entry in backups if entry["status"] == "success"]
+        failures = sum(entry["status"] == "failed" for _, entry in backups)
+        if failures >= MAX_DAILY_FAILURES or any(when > now - timedelta(hours=23) for when, _ in successful):
+            return None
+
+        has_file_backup_today = any(
+            entry.get("with_files") and when.date() == now.date() for when, entry in successful
+        )
+        return {"with_files": not has_file_backup_today}
+
+    def queue_scheduled(self, now: datetime, history: list[tuple[datetime, dict]] | None = None) -> None:
+        from pilot.exceptions import TaskConflictError
+        from pilot.tasks.backup_site import BackupSiteTask
+
+        if self.get_scheduled_backup(now, history) is None:
+            return
+
+        slot = f"{self.site.config.name}:{now:%Y%m%d%H}"
+        try:
+            BackupSiteTask.queue(
+                self.site.bench,
+                site=self.site.config.name,
+                automatic=True,
+                idempotency_key=f"scheduled-backup:{sha256(slot.encode()).hexdigest()}",
+                resource_key=f"site:{self.site.config.name.lower()}",
+            )
+        except TaskConflictError:
+            print(f"Skipped scheduled backup: {self.site.config.name} has an active site task.")
+
     def move_schedule(self, old_name: str) -> None:
         """Carry a renamed site's backup schedule over from its old name."""
         from pilot.managers.cron import CronManager

pilot/core/site/provisioning.py

@@ -29,7 +29,7 @@ def __init__(
         self.admin_password = admin_password
         self.db_type = db_type
 
-    def provision(self, on_progress: Callable[[str], None]) -> "Site":
+    def provision(self, on_progress: Callable[[str], None], *, schedule_backups: bool = True) -> "Site":
         from pilot.core.site import Site
 
         via_wildcard = validate_new_site(self.bench, self.name, self.apps)
@@ -57,12 +57,14 @@ def provision(self, on_progress: Callable[[str], None]) -> "Site":
         self.write_pilot_communication_config(site)
         self.bench.write_common_site_config()
         site.enable_scheduler()
-        on_progress(f"\nSite '{self.name}' created successfully.")
         self.build_missing_assets()
         self.add_to_hosts(site)
         self.reload_nginx()
         if origin_tls:
             self.obtain_cert(site, on_progress)
+        if schedule_backups:
+            site.backups.enable_default_schedule()
+        on_progress(f"\nSite '{self.name}' created successfully.")
         return site
 
     def write_email_sender_name(self, site: "Site") -> None:
@@ -203,12 +205,13 @@ def provision_from_backup(
     from pilot.core.site import Site
     from pilot.core.site.apps import SiteApps
 
-    site = Site.provision(bench, name, [], admin_password, on_progress=on_progress)
+    site = Site.provision(bench, name, [], admin_password, on_progress=on_progress, schedule_backups=False)
     on_progress(f"Restoring backup: {db_file}")
     site.restore(db_file, public_files, private_files)
     # The restored database decides which apps the site runs.
     SiteApps(site).restore_installed_apps_copy()
     bench.processes.apply()
+    site.backups.enable_default_schedule()
     return site
 
 

pilot/core/site/retention.py

@@ -1,6 +1,6 @@
 """Decide which backup runs to prune. Pure logic, no I/O."""
 
-from datetime import datetime
+from datetime import date, datetime
 
 from pilot.config import SCHEME_FIFO, BackupConfig
 
@@ -13,17 +13,37 @@ class BackupRetentionPolicy:
     def __init__(self, config: BackupConfig) -> None:
         self.config = config
 
-    def select_deletions(self, timestamps: list[str]) -> list[str]:
+    def select_deletions(self, timestamps: list[str], today: date | None = None) -> list[str]:
         runs = self._parse(timestamps)
         if len(runs) <= 1:
             return []
-        keep = self._keep_fifo(runs) if self.config.scheme == SCHEME_FIFO else self._keep_gfs(runs)
+        if self.config.scheme == SCHEME_FIFO:
+            keep = self._keep_fifo(runs)
+        elif self.config.calendar_rotation:
+            keep = self._keep_calendar_gfs(runs, today or date.today())
+        else:
+            keep = self._keep_gfs(runs)
         keep.add(runs[0][1])  # newest run is never pruned
         return [ts for _, ts in runs if ts not in keep]
 
     def _keep_fifo(self, runs: list[tuple[datetime, str]]) -> set[str]:
         return {ts for _, ts in runs[: self.config.keep_last]}
 
+    def _keep_calendar_gfs(self, runs: list[tuple[datetime, str]], today: date) -> set[str]:
+        """Keep every run on an anchor day while its calendar window is open."""
+        keep = set()
+        for when, timestamp in runs:
+            age = (today - when.date()).days
+            tiers = (
+                (7, True),
+                (28, when.weekday() == 6),
+                (366, when.day == 1),
+                (3653, when.month == 1 and when.day == 1),
+            )
+            if any(anchor and age <= window for window, anchor in tiers):
+                keep.add(timestamp)
+        return keep
+
     def _keep_gfs(self, runs: list[tuple[datetime, str]]) -> set[str]:
         config = self.config
         keep: set[str] = set()

pilot/managers/cron.py

@@ -56,6 +56,8 @@ def set_schedule(self, job_key: str, cron_expr: str, command: str) -> None:
             lines = self._read_crontab()
             marker = self._marker(job_key)
             entry = f"{cron_expr} {command}"
+            if marker in lines and lines[lines.index(marker) + 1 :][:1] == [entry]:
+                return
             try:
                 i = lines.index(marker)
             except ValueError: