diff --git a/sunbeam-python/sunbeam/commands/backup_restore.py b/sunbeam-python/sunbeam/commands/backup_restore.py new file mode 100644 index 000000000..e48feb755 --- /dev/null +++ b/sunbeam-python/sunbeam/commands/backup_restore.py @@ -0,0 +1,583 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""``sunbeam backup``, ``sunbeam restore`` and ``sunbeam list-backups`` commands.""" + +import logging +import sys +from datetime import datetime, timezone + +import click +from rich.console import Console +from rich.table import Table + +from sunbeam.core.common import get_step_message, run_plan +from sunbeam.core.deployment import Deployment +from sunbeam.core.openstack import OPENSTACK_MODEL +from sunbeam.core.questions import ConfirmQuestion, Question +from sunbeam.steps.backup_restore import ( + BACKUP_COMPONENTS, + DEFAULT_BACKUP_TIMEOUT, + DEFAULT_RESTORE_TIMEOUT, + RESTORE_TIME_FORMAT, + BackupInventory, + BackupResult, + DiscoverBackupApplicationsStep, + ListBackupsStep, + ResolveActionTargetsStep, + RestoreResult, + RestoreStep, + RunBackupStep, + ValidateStep, + WriteBackupInventoryManifestStep, + WriteBackupManifestStep, +) + +LOG = logging.getLogger(__name__) +console = Console() + +EXIT_SUCCESS = 0 +EXIT_PARTIAL = 1 +EXIT_FAILURE = 2 + +CONTINUE_BACKUP_QUESTION = ConfirmQuestion( + "Continue and back up the remaining components?", + default_value=False, + description=( + "Some discovered applications were skipped because they are not" + " ready for backup." + ), +) + +CONTINUE_RESTORE_READY_QUESTION = ConfirmQuestion( + "Continue and restore the remaining components?", + default_value=False, + description=( + "Some discovered applications were skipped because they are not" + " ready for restore. Partial restores may result in " + "dangling OpenStack objects. " + ), +) + +CONTINUE_RESTORE_FILTER_QUESTION = ConfirmQuestion( + "Continue and restore the remaining components?", + default_value=False, + description=( + "Some discovered applications have missing or failed backups.\n" + "- Applications without successful backups will be skipped.\n" + "- Applications with mixed backup results will restore from successful " + "backups only.\n" + "Partial restores may result in dangling OpenStack objects." + ), +) + +START_RESTORE_QUESTION = ConfirmQuestion( + "Start restore?", + default_value=False, +) + + +def _confirm_or_abort(question: Question, no_prompt: bool) -> None: + """Ask a confirmation question, aborting the command if declined.""" + if no_prompt: + return + question.console = console + question.show_hint = True + if not question.ask(): + raise click.Abort() + + +def _components_banner() -> str: + return ",".join(c.name for c in BACKUP_COMPONENTS) + + +def _discover_apps(jhelper, model: str) -> dict[str, list[str]]: + """Discover applications for every registered backup component.""" + results = run_plan( + [DiscoverBackupApplicationsStep(jhelper, model, BACKUP_COMPONENTS)], console + ) + return get_step_message(results, DiscoverBackupApplicationsStep) + + +def _validate_apps( + jhelper, + discovered: dict[str, list[str]], + model: str, + force: bool, +) -> tuple[dict[str, list[str]], bool]: + """Run the validation step and warn on every skipped application.""" + results = run_plan( + [ValidateStep(jhelper, discovered, model=model, force=force)], console + ) + outcome = get_step_message(results, ValidateStep) + valid = outcome["valid"] + failures = outcome["failures"] + + for app in sorted(failures): + reasons = ", ".join(failures[app]) + console.print( + f"[yellow]Warning:[/yellow] {app} is not ready for backup" + f" ({reasons}) and will be skipped." + ) + + return valid, bool(failures) + + +def _list_backup_inventory( + jhelper, + discovered: dict[str, list[str]], + model: str, + timeout: int, +) -> list[BackupInventory]: + """List available backups for the given targets.""" + results = run_plan( + [ + ResolveActionTargetsStep( + jhelper, + discovered, + action=lambda component: component.list_action, + model=model, + ) + ], + console, + ) + resolved = get_step_message(results, ResolveActionTargetsStep) + targets = resolved["targets"] + unresolved_targets = resolved["unresolved"] + + results = run_plan( + [ + ListBackupsStep( + jhelper, + targets, + timeout=timeout, + model=model, + ) + ], + console, + ) + inventories = get_step_message(results, ListBackupsStep) + for unresolved in unresolved_targets: + inventories.append( + BackupInventory( + app=unresolved["app"], + unit="-", + component=unresolved["component"], + error="Could not resolve target.", + ) + ) + return inventories + + +def _print_inventory(results: list[BackupInventory]) -> None: + table = Table() + table.add_column("Application") + table.add_column("Component") + table.add_column("Backup IDs") + table.add_column("Status") + + for result in sorted(results, key=lambda inventory: inventory.app): + if result.backups: + ordered = sorted(result.backups, key=lambda b: b.backup_id, reverse=True) + backup_ids = "\n".join(b.backup_id for b in ordered) + statuses = "\n".join( + "[green]ok[/green]" + if b.success is True + else "[red]failed[/red]" + if b.success is False + else "-" + for b in ordered + ) + else: + backup_ids = "-" + statuses = "[red]failed[/red]" if result.error else "-" + table.add_row(result.app, result.component, backup_ids, statuses) + + console.print(table) + + +def _print_backup_summary(results: list[BackupResult]) -> None: + table = Table() + table.add_column("Application") + table.add_column("Component") + table.add_column("Backup ID") + table.add_column("Status") + for result in results: + if result.backup is None: + table.add_row(result.app, result.component, "-", "-") + continue + status = ( + "[green]ok[/green]" + if result.backup.success is True + else "[red]failed[/red]" + if result.backup.success is False + else "-" + ) + table.add_row( + result.app, result.component, result.backup.backup_id or "-", status + ) + console.print(table) + + +def _filter_restore_targets( + discovered: dict[str, list[str]], + inventory: list[BackupInventory], +) -> tuple[dict[str, list[str]], bool]: + """Warn on missing inventory and keep only restorable targets.""" + partially_failed_apps = { + entry.app + for entry in inventory + if entry.backups + and any(b.success is True for b in entry.backups) + and any(b.success is False for b in entry.backups) + } + + failed_inventory_by_app: dict[str, BackupInventory] = { + entry.app: entry + for entry in inventory + if entry.error is not None + or not entry.backups + or not any(b.success is True for b in entry.backups) + } + + for entry, inv in failed_inventory_by_app.items(): + if inv.error: + console.print( + "[yellow]Warning:[/yellow] Failed to list backups for " + f"{entry}: {inv.error}" + ) + elif not inv.backups: + console.print( + f"[yellow]Warning:[/yellow] No backups available for {entry}." + ) + elif any(b.success is False for b in inv.backups): + console.print( + f"[yellow]Warning:[/yellow] Some backups failed for {entry}." + f" No successful backups are available for restore." + ) + else: + console.print( + f"[yellow]Warning:[/yellow] No successful backups available for" + f" {entry}." + ) + + for app in sorted(partially_failed_apps): + console.print( + f"[yellow]Warning:[/yellow] Some backups for {app} failed." + " Only successful backups will be considered for restore" + " (possible out-of-band state)." + ) + + restorable: dict[str, list[str]] = {} + for component, apps in discovered.items(): + for app in apps: + if app in failed_inventory_by_app: + continue + restorable.setdefault(component, []).append(app) + return restorable, bool(failed_inventory_by_app) or bool(partially_failed_apps) + + +def _warn_restore_to_time_fallback_targets( + discovered: dict[str, list[str]], + restore_to_time: str | None, +) -> list[str]: + """When PITR is requested, report apps that will fall back to latest backup.""" + if restore_to_time is None: + return [] + + supports_restore_to_time = { + component.name: component.restore_to_time_param is not None + for component in BACKUP_COMPONENTS + } + + fallback_apps: list[str] = [] + for component, apps in discovered.items(): + if supports_restore_to_time.get(component, False): + continue + fallback_apps.extend(apps) + + return sorted(fallback_apps) + + +def _validate_restore_to_time( + ctx: click.Context, param: click.Parameter, value: str | None +) -> str | None: + if value is None: + return None + try: + datetime.strptime(value, RESTORE_TIME_FORMAT) + except ValueError: + raise click.BadParameter( + f"expected format 'YYYY-MM-DD HH:MM:SS', got {value!r}" + ) + return value + + +@click.command() +@click.option( + "--force", + is_flag=True, + default=False, + show_default=True, + help=( + "Back up applications whose cluster health cannot be verified, using the" + " leader unit. May capture stale data; use with caution." + ), +) +@click.option( + "--timeout", + default=DEFAULT_BACKUP_TIMEOUT, + show_default=True, + help="Time in seconds to wait for each backup to complete.", +) +@click.option( + "--no-prompt", is_flag=True, default=False, help="Do not prompt for confirmation." +) +@click.pass_context +def backup(ctx: click.Context, force: bool, timeout: int, no_prompt: bool) -> None: + """Create backups of stateful Sunbeam applications (MySQL and Vault).""" + deployment: Deployment = ctx.obj + jhelper = deployment.get_juju_helper() + model = OPENSTACK_MODEL + + console.print( + f"[bold]Backing up \\[{_components_banner()}] in model '{model}'...[/bold]" + ) + + discovered = _discover_apps(jhelper, model) + if not any(discovered.values()): + console.print("No applications found to back up. Exiting.") + sys.exit(EXIT_FAILURE) + + discovered, was_filtered = _validate_apps(jhelper, discovered, model, force=force) + if was_filtered: + _confirm_or_abort(CONTINUE_BACKUP_QUESTION, no_prompt) + + if not any(discovered.values()): + console.print("No applications remain to back up after validation. Exiting.") + sys.exit(EXIT_FAILURE) + + dispatched_at = datetime.now(timezone.utc).strftime(RESTORE_TIME_FORMAT) + console.print(f"Dispatching backups at {dispatched_at} UTC...") + backup_results = run_plan( + [RunBackupStep(jhelper, discovered, force=force, timeout=timeout, model=model)], + console, + ) + results: list[BackupResult] = get_step_message(backup_results, RunBackupStep) + + if not results: + console.print( + "Could not resolve a backup target for any application. Re-run with" + " --force to back up on leader units regardless of cluster health." + ) + sys.exit(EXIT_FAILURE) + + _print_backup_summary(results) + + manifest_results = run_plan( + [WriteBackupManifestStep(results, dispatched_at)], console + ) + manifest_path = get_step_message(manifest_results, WriteBackupManifestStep) + if manifest_path: + console.print(f"Backup manifest written to: {manifest_path}") + + succeeded = sum(1 for r in results if r.backup is not None and r.backup.success) + failed = sum(1 for r in results if r.error is not None) + console.print(f"Backup summary: {succeeded} succeeded, {failed} failed.") + + if failed == 0: + sys.exit(EXIT_SUCCESS) + + console.print( + "[yellow]Warning:[/yellow] one or more backups failed or timed out. A partial" + " restore from a set may result in dangling OpenStack objects." + ) + + if succeeded == 0: + sys.exit(EXIT_FAILURE) + sys.exit(EXIT_PARTIAL) + + +@click.command("list-backups") +@click.option( + "--timeout", + default=DEFAULT_BACKUP_TIMEOUT, + show_default=True, + help="Time in seconds to wait for each list action to complete.", +) +@click.pass_context +def list_backups(ctx: click.Context, timeout: int) -> None: + """List backup IDs from stateful Sunbeam applications.""" + deployment: Deployment = ctx.obj + jhelper = deployment.get_juju_helper() + model = OPENSTACK_MODEL + + console.print( + f"[bold]Listing backups for \\[{_components_banner()}]" + f" in model '{model}'...[/bold]" + ) + + discovered = _discover_apps(jhelper, model) + discovered, _ = _validate_apps(jhelper, discovered, model, force=False) + + if not any(discovered.values()): + console.print("No applications found to list backups from. Exiting.") + sys.exit(EXIT_FAILURE) + + listed_at = datetime.now(timezone.utc).strftime(RESTORE_TIME_FORMAT) + inventory = _list_backup_inventory(jhelper, discovered, model, timeout) + + failed_inventory = sorted( + (entry for entry in inventory if entry.error), + key=lambda entry: entry.app, + ) + for entry in failed_inventory: + console.print( + f"[yellow]Warning:[/yellow] Failed to list backups for" + f" {entry.app}: {entry.error}" + ) + + _print_inventory(inventory) + + manifest_results = run_plan( + [WriteBackupInventoryManifestStep(inventory, listed_at)], console + ) + manifest_path = get_step_message(manifest_results, WriteBackupInventoryManifestStep) + if manifest_path: + console.print(f"Backup inventory manifest written to: {manifest_path}") + + if failed_inventory: + sys.exit(EXIT_FAILURE) + + sys.exit(EXIT_SUCCESS) + + +@click.command() +@click.option( + "--restore-to-time", + default=None, + callback=_validate_restore_to_time, + help="Point-in-time to restore to, formatted 'YYYY-MM-DD HH:MM:SS'.", +) +@click.option( + "--force", + is_flag=True, + default=False, + show_default=True, + help="Proceed with restore despite cluster health concerns.", +) +@click.option( + "--timeout", + default=DEFAULT_RESTORE_TIMEOUT, + show_default=True, + help="Time in seconds to wait for restore operations to complete.", +) +@click.option( + "--no-prompt", is_flag=True, default=False, help="Do not prompt for confirmation." +) +@click.pass_context +def restore( + ctx: click.Context, + restore_to_time: str | None, + force: bool, + timeout: int, + no_prompt: bool, +) -> None: + """Restore stateful Sunbeam applications from a backup.""" + deployment: Deployment = ctx.obj + jhelper = deployment.get_juju_helper() + model = OPENSTACK_MODEL + + console.print( + f"[bold]Restoring \\[{_components_banner()}]" + f" in model '{model}' from backup...[/bold]" + ) + + discovered = _discover_apps(jhelper, model) + if not any(discovered.values()): + console.print("No applications found to restore. Exiting.") + sys.exit(EXIT_FAILURE) + + discovered, was_filtered = _validate_apps(jhelper, discovered, model, force=force) + if was_filtered: + _confirm_or_abort(CONTINUE_RESTORE_READY_QUESTION, no_prompt) + + inventory = _list_backup_inventory(jhelper, discovered, model, timeout) + _print_inventory(inventory) + + unresolved_restore_targets = [ + entry + for entry in inventory + if entry.unit == "-" and entry.error == "Could not resolve target." + ] + if unresolved_restore_targets: + for entry in unresolved_restore_targets: + console.print( + f"[red]Error:[/red] Could not resolve restore target for {entry.app}." + ) + sys.exit(EXIT_FAILURE) + + discovered, was_filtered = _filter_restore_targets(discovered, inventory) + + if was_filtered: + _confirm_or_abort(CONTINUE_RESTORE_FILTER_QUESTION, no_prompt) + + if not any(discovered.values()): + console.print("No applications remain to restore after validation. Exiting.") + sys.exit(EXIT_FAILURE) + + fallback_for_restore_to_time = _warn_restore_to_time_fallback_targets( + discovered, restore_to_time + ) + for app in fallback_for_restore_to_time: + console.print( + f"[yellow]Warning:[/yellow] {app} does not support" + " --restore-to-time. Restoring latest available backup instead." + ) + + if restore_to_time: + START_RESTORE_QUESTION.description = ( + f"Restore will be performed to the point-in-time {restore_to_time} UTC." + ) + else: + START_RESTORE_QUESTION.description = ( + "Restore will be performed to the latest available backup." + ) + _confirm_or_abort(START_RESTORE_QUESTION, no_prompt) + + restore_results = run_plan( + [ + RestoreStep( + jhelper, + discovered, + restore_to_time=restore_to_time, + timeout=timeout, + model=model, + ) + ], + console, + ) + results: list[RestoreResult] = get_step_message(restore_results, RestoreStep) + + for result in results: + if not result.success: + reverted = " Reverted." if result.reverted else "" + rollback_failed = ( + f" Rollback failed: {result.rollback_error}." + if result.rollback_error + else "" + ) + console.print( + f"[red]Error:[/red] {result.app} restore failed:" + f" {result.error}.{reverted}{rollback_failed}" + ) + + succeeded = sum(1 for r in results if r.success) + failed = sum(1 for r in results if not r.success) + console.print(f"Restore summary: {succeeded} succeeded, {failed} failed.") + + if failed == 0: + sys.exit(EXIT_SUCCESS) + if succeeded == 0: + sys.exit(EXIT_FAILURE) + sys.exit(EXIT_PARTIAL) diff --git a/sunbeam-python/sunbeam/core/juju.py b/sunbeam-python/sunbeam/core/juju.py index e1f095f8d..c74499443 100644 --- a/sunbeam-python/sunbeam/core/juju.py +++ b/sunbeam-python/sunbeam/core/juju.py @@ -857,6 +857,23 @@ def run_action( raise ActionFailedException(str(task)) return task.results + def get_application_actions(self, application: str, model: str) -> list[str]: + """Return action names available for an application in a model.""" + with self._model(model) as juju: + try: + actions = self.cli( + "actions", + application, + include_controller=False, + juju=juju, + ) + except jubilant.CLIError as e: + raise JujuException(str(e)) from e + + if not isinstance(actions, dict): + return [] + return sorted(actions.keys()) + def add_secret(self, model: str, name: str, data: dict, info: str) -> str: """Add secret to the model. diff --git a/sunbeam-python/sunbeam/features/disaster_recovery/__init__.py b/sunbeam-python/sunbeam/features/disaster_recovery/__init__.py new file mode 100644 index 000000000..50f658a6a --- /dev/null +++ b/sunbeam-python/sunbeam/features/disaster_recovery/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 diff --git a/sunbeam-python/sunbeam/features/disaster_recovery/etc/deploy-disaster-recovery/main.tf b/sunbeam-python/sunbeam/features/disaster_recovery/etc/deploy-disaster-recovery/main.tf new file mode 100644 index 000000000..40e0754e9 --- /dev/null +++ b/sunbeam-python/sunbeam/features/disaster_recovery/etc/deploy-disaster-recovery/main.tf @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +terraform { + required_providers { + juju = { + source = "juju/juju" + version = "= 1.3.1" + } + } +} + +provider "juju" {} + +data "juju_model" "openstack_model" { + uuid = var.openstack-model-uuid +} + +resource "juju_application" "s3_integrator" { + for_each = var.enable-disaster-recovery ? toset(var.s3-integrator-apps) : [] + name = each.value + model_uuid = data.juju_model.openstack_model.uuid + + charm { + name = "s3-integrator" + channel = var.s3-integrator-channel + revision = var.s3-integrator-revision + } + + config = contains(keys(var.s3-integrator-secret-data), each.value) ? merge( + lookup(var.s3-integrator-config, each.value, {}), + { credentials = juju_secret.s3_credentials[each.value].secret_uri } + ) : lookup(var.s3-integrator-config, each.value, {}) +} + +resource "juju_secret" "s3_credentials" { + for_each = var.enable-disaster-recovery ? var.s3-integrator-secret-data : {} + model_uuid = data.juju_model.openstack_model.uuid + name = "${each.key}-credentials" + value = each.value +} + +resource "juju_access_secret" "s3_credentials_access" { + for_each = var.enable-disaster-recovery ? var.s3-integrator-secret-data : {} + model_uuid = data.juju_model.openstack_model.uuid + secret_id = juju_secret.s3_credentials[each.key].secret_id + applications = [each.key] + + depends_on = [juju_application.s3_integrator] +} + +resource "juju_integration" "s3_integrations" { + for_each = var.enable-disaster-recovery ? var.s3-integrations : {} + model_uuid = data.juju_model.openstack_model.uuid + + application { + name = each.value.integrator_app + endpoint = "s3-credentials" + } + + application { + name = each.key + endpoint = each.value.target_endpoint + } + + depends_on = [juju_access_secret.s3_credentials_access] +} diff --git a/sunbeam-python/sunbeam/features/disaster_recovery/etc/deploy-disaster-recovery/variables.tf b/sunbeam-python/sunbeam/features/disaster_recovery/etc/deploy-disaster-recovery/variables.tf new file mode 100644 index 000000000..268f65113 --- /dev/null +++ b/sunbeam-python/sunbeam/features/disaster_recovery/etc/deploy-disaster-recovery/variables.tf @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +variable "openstack-model-uuid" { + description = "UUID of the OpenStack Juju model" + type = string +} + +variable "enable-disaster-recovery" { + description = "Enable disaster recovery resources" + type = bool + default = false +} + +variable "s3-integrator-channel" { + description = "Channel to use for deployment of s3-integrator charm" + type = string + default = "2/stable" +} + +variable "s3-integrator-revision" { + description = "Charm revision for s3-integrator deployment" + type = number + default = null +} + +variable "s3-integrator-config" { + description = "Operator config for s3-integrator deployment" + type = map(map(string)) + default = {} +} + +variable "s3-integrator-secret-data" { + description = "Per-app secret payload for s3-integrator credentials" + type = map(map(string)) + default = {} +} + +variable "s3-integrator-apps" { + description = "Per-application s3-integrator app names to deploy" + type = list(string) + default = [] +} + +variable "s3-integrations" { + description = "Map of target app -> per-app s3-integrator app for relation wiring" + type = map(object({ + integrator_app = string + target_endpoint = string + })) + default = {} +} diff --git a/sunbeam-python/sunbeam/features/disaster_recovery/feature.py b/sunbeam-python/sunbeam/features/disaster_recovery/feature.py new file mode 100644 index 000000000..2d034a6cc --- /dev/null +++ b/sunbeam-python/sunbeam/features/disaster_recovery/feature.py @@ -0,0 +1,450 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +import logging +from dataclasses import dataclass +from pathlib import Path + +import click +import pydantic +from jubilant.statustypes import AppStatus +from packaging.version import Version +from rich.console import Console + +from sunbeam.core.deployment import Deployment +from sunbeam.core.juju import ApplicationStatusOverlay, JujuException, JujuHelper +from sunbeam.core.manifest import ( + CharmManifest, + FeatureConfig, + SoftwareConfig, + TerraformManifest, +) +from sunbeam.core.openstack import OPENSTACK_MODEL +from sunbeam.core.questions import ( + ConfirmQuestion, + PasswordPromptQuestion, + PromptQuestion, +) +from sunbeam.features.interface.v1.openstack import ( + DatabaseTopology, + OpenStackControlPlaneFeature, + TerraformPlanLocation, +) +from sunbeam.steps.backup_restore import ( + BACKUP_COMPONENTS, + S3_ENDPOINT, + S3_RELATION_VALIDATION_CHECK, +) +from sunbeam.utils import click_option_show_hints, pass_method_obj +from sunbeam.versions import S3_INTEGRATOR_CHANNEL + +_MANAGED_S3_KEY = "managed-s3-integrations" # {target_app: integrator_app} +S3_INTEGRATOR_CHARM = "s3-integrator" +DEFAULT_S3_PATH = "/" +DEFAULT_S3_REGION = "us-east-2" +DEFAULT_S3_ENDPOINT = "https://s3.us-east-2.amazonaws.com" + +console = Console() +LOG = logging.getLogger(__name__) + + +@dataclass +class S3Integration: + """Data class representing an S3 integration for a target application.""" + + app_name: str + integrator_app: str + target_endpoint: str + + +class DisasterRecoveryFeatureConfig(FeatureConfig): + configure_s3_integrators: bool = pydantic.Field( + default=False, + alias="configure-managed-s3-integrators", + validation_alias="configure_managed_s3_integrators", + ) + bucket: str = "" + path: str = DEFAULT_S3_PATH + region: str = DEFAULT_S3_REGION + endpoint: str = DEFAULT_S3_ENDPOINT + access_key: str = pydantic.Field( + default="", alias="access-key", validation_alias="access_key" + ) + secret_key: str = pydantic.Field( + default="", alias="secret-key", validation_alias="secret_key" + ) + + +class DisasterRecoveryFeature(OpenStackControlPlaneFeature): + version = Version("0.0.1") + + name = "disaster-recovery" + generally_available = False + tf_plan_location = TerraformPlanLocation.FEATURE_REPO + + _s3_integrations_cache: list["S3Integration"] | None = None + + def config_type(self) -> type[DisasterRecoveryFeatureConfig]: + """Return manifest config type for disaster-recovery feature.""" + return DisasterRecoveryFeatureConfig + + def default_software_overrides(self) -> SoftwareConfig: + """Feature software configuration.""" + return SoftwareConfig( + charms={ + S3_INTEGRATOR_CHARM: CharmManifest(channel=S3_INTEGRATOR_CHANNEL), + }, + terraform={ + self.tfplan: TerraformManifest( + source=Path(__file__).parent / "etc" / self.tfplan_dir + ) + }, + ) + + def manifest_attributes_tfvar_map(self) -> dict: + """Manifest attributes terraformvars map.""" + return { + self.tfplan: { + "charms": { + S3_INTEGRATOR_CHARM: { + "channel": "s3-integrator-channel", + "revision": "s3-integrator-revision", + "config": "s3-integrator-config", + } + } + } + } + + def set_application_names(self, deployment: Deployment) -> list: + """Application names handled by the terraform plan.""" + integrations = self._s3_integrations(deployment) + return sorted({integration.integrator_app for integration in integrations}) + + def set_tfvars_on_enable( + self, deployment: Deployment, config: FeatureConfig + ) -> dict: + """Set terraform variables to enable the application.""" + jhelper = deployment.get_juju_helper() + model_uuid = jhelper.get_model_uuid(OPENSTACK_MODEL) + integrations = self._s3_integrations(deployment) + dr_config = self._as_dr_config(config) + return { + "enable-disaster-recovery": True, + "openstack-model-uuid": model_uuid, + "s3-integrator-config": self._s3_integrator_config_tfvar( + dr_config, integrations + ), + "s3-integrator-secret-data": self._s3_integrator_secret_data_tfvar( + dr_config, integrations + ), + "s3-integrator-apps": sorted( + {integration.integrator_app for integration in integrations} + ), + "s3-integrations": { + integration.app_name: { + "integrator_app": integration.integrator_app, + "target_endpoint": integration.target_endpoint, + } + for integration in integrations + }, + } + + def set_tfvars_on_disable(self, deployment: Deployment) -> dict: + """Set terraform variables to disable the application.""" + jhelper = deployment.get_juju_helper() + model_uuid = jhelper.get_model_uuid(OPENSTACK_MODEL) + integrations = self._s3_integrations(deployment) + return { + "enable-disaster-recovery": False, + "openstack-model-uuid": model_uuid, + "s3-integrator-config": {}, + "s3-integrator-secret-data": {}, + "s3-integrator-apps": sorted( + {integration.integrator_app for integration in integrations} + ), + "s3-integrations": { + integration.app_name: { + "integrator_app": integration.integrator_app, + "target_endpoint": integration.target_endpoint, + } + for integration in integrations + }, + } + + def set_tfvars_on_resize( + self, deployment: Deployment, config: FeatureConfig + ) -> dict: + """Set terraform variables to resize the application.""" + return {} + + def get_app_status_overlay_on_enable( + self, deployment: Deployment + ) -> dict[str, ApplicationStatusOverlay]: + """Accept blocked status for DR-managed s3-integrator apps on enable.""" + return { + app_name: {"status": ["active", "blocked"]} + for app_name in self.set_application_names(deployment) + } + + def post_enable( + self, deployment: Deployment, config: FeatureConfig, show_hints: bool + ) -> None: + """Perform post-enable actions for the feature.""" + integrations = self._s3_integrations(deployment) + self._s3_save_managed_integrations(deployment, integrations) + + @click.command() + @click_option_show_hints + @pass_method_obj + def enable_cmd(self, deployment: Deployment, show_hints: bool) -> None: + """Enable disaster recovery service.""" + self._s3_integrations_cache = None + config = self._prompt_s3_configuration(deployment, show_hints) + self.enable_feature(deployment, config, show_hints) + + @click.command() + @click_option_show_hints + @pass_method_obj + def disable_cmd(self, deployment: Deployment, show_hints: bool) -> None: + """Disable disaster recovery service.""" + self._s3_integrations_cache = None + self.disable_feature(deployment, show_hints) + + def _s3_integrations(self, deployment: Deployment) -> list[S3Integration]: + """Compute DR S3 integrations once per command invocation. + + The result is cached on the feature instance so the multiple lifecycle + callbacks in a single enable/disable run reuse the same computation + instead of re-querying Juju relations every time. + """ + cache = getattr(self, "_s3_integrations_cache", None) + if cache is not None: + return cache + jhelper = deployment.get_juju_helper() + apps = jhelper.get_model_status(OPENSTACK_MODEL).apps + managed = self._s3_load_managed_integrations(deployment) + database_topology = self.get_database_topology(deployment) + integrations = self._s3_build_integrations( + jhelper, apps, managed, database_topology + ) + self._s3_integrations_cache = integrations + return integrations + + def _s3_build_integrations( + self, + jhelper: JujuHelper, + apps: dict[str, AppStatus], + managed_integrators: dict[str, str] | None = None, + database_topology: DatabaseTopology | None = None, + ) -> list[S3Integration]: + targets = self._s3_discover_relation_targets(apps, database_topology) + managed_integrators = managed_integrators or {} + integrations: list[S3Integration] = [] + + for app_name in targets: + app_status = apps[app_name] + endpoint = self._s3_target_endpoint_for_app(app_status) + is_related = bool(self._s3_relation_consumers(jhelper, app_name, endpoint)) + expected_integrator = managed_integrators.get( + app_name, self._s3_integrator_app_name(app_name) + ) + owned = app_name in managed_integrators + + if not owned and (is_related or expected_integrator in apps): + console.print( + ( + f"[yellow]Warning:[/yellow] Skipping disaster recovery " + f"configuration for {app_name}: " + "existing S3 setup is managed externally." + ) + ) + continue + + integrations.append( + S3Integration( + app_name=app_name, + integrator_app=expected_integrator, + target_endpoint=endpoint, + ) + ) + return integrations + + def _s3_discover_relation_targets( + self, + apps: dict[str, AppStatus], + database_topology: DatabaseTopology | None = None, + ) -> list[str]: + """Return applications eligible for DR S3 integration.""" + target_charms = set(self._s3_relation_target_components().keys()) + targets: list[str] = [] + for app_name, app_status in apps.items(): + if app_status.charm_name not in target_charms: + continue + if app_status.charm_name == "mysql-k8s": + if database_topology == DatabaseTopology.SINGLE and app_name != "mysql": + continue + if database_topology == DatabaseTopology.MULTI and app_name == "mysql": + continue + targets.append(app_name) + return targets + + def _s3_relation_consumers( + self, jhelper: JujuHelper, app_name: str, endpoint: str + ) -> set[str]: + """Return whether app already has any s3 relation.""" + try: + relation_map = jhelper.get_relation_map(app_name, endpoint, OPENSTACK_MODEL) + except JujuException: + return set() + return {consumer for consumer in relation_map.values() if consumer} + + def _s3_target_endpoint_for_app(self, app_status: AppStatus) -> str: + """Return the S3 endpoint used by an app based on backup component mapping.""" + charm_name = app_status.charm_name + return self._s3_relation_target_components().get(charm_name, S3_ENDPOINT) + + def _s3_relation_target_components(self) -> dict[str, str]: + """Return backup components that require S3 relation validation.""" + return { + component.name: S3_ENDPOINT + for component in BACKUP_COMPONENTS + if any( + check.name == S3_RELATION_VALIDATION_CHECK.name + for check in component.validate_checks + ) + } + + def _s3_integrator_app_name(self, app_name: str) -> str: + """Return per-application s3-integrator app name for a target app.""" + service_name = app_name.removesuffix("-mysql") + return f"{service_name}-s3-integrator" + + def _as_dr_config(self, config: FeatureConfig) -> DisasterRecoveryFeatureConfig: + """Cast generic feature config to DR-specific config with defaults.""" + if isinstance(config, DisasterRecoveryFeatureConfig): + return config + return DisasterRecoveryFeatureConfig() + + def _prompt_s3_configuration( + self, deployment: Deployment, show_hints: bool + ) -> DisasterRecoveryFeatureConfig: + """Prompt for managed s3-integrator configuration.""" + config = DisasterRecoveryFeatureConfig() + if not self._s3_integrations(deployment): + return config + + configure = ConfirmQuestion( + "Configure all s3-integrators?", + default_value=False, + description=( + "Configure bucket, endpoint, region, and credentials for all " + "s3-integrator apps." + ), + ) + configure.console = console + configure.show_hint = show_hints + if not configure.ask(): + return config + + config.configure_s3_integrators = True + config.bucket = self._ask_prompt("S3 bucket", show_hints) + config.path = self._ask_prompt( + "S3 path prefix", show_hints, default=DEFAULT_S3_PATH + ) + config.region = self._ask_prompt( + "S3 region", show_hints, default=DEFAULT_S3_REGION + ) + config.endpoint = self._ask_prompt( + "S3 endpoint", show_hints, default=DEFAULT_S3_ENDPOINT + ) + config.access_key = self._ask_password("S3 access key", show_hints) + config.secret_key = self._ask_password("S3 secret key", show_hints) + self._validate_s3_config(config) + return config + + def _ask_prompt(self, question: str, show_hints: bool, default: str = "") -> str: + prompt: PromptQuestion[str] = PromptQuestion(question, default_value=default) + prompt.console = console + prompt.show_hint = show_hints + value = prompt.ask() or "" + return value.strip() + + def _ask_password(self, question: str, show_hints: bool) -> str: + prompt: PasswordPromptQuestion[str] = PasswordPromptQuestion( + question, password=True + ) + prompt.console = console + prompt.show_hint = show_hints + value = prompt.ask() or "" + return value.strip() + + def _validate_s3_config(self, config: DisasterRecoveryFeatureConfig) -> None: + """Validate prompted S3 configuration.""" + if not config.bucket: + raise click.ClickException("S3 bucket is required.") + if not config.access_key: + raise click.ClickException("S3 access key is required.") + if not config.secret_key: + raise click.ClickException("S3 secret key is required.") + if not config.endpoint.startswith(("http://", "https://")): + raise click.ClickException( + "S3 endpoint must start with http:// or https://." + ) + + def _s3_integrator_config_tfvar( + self, + config: DisasterRecoveryFeatureConfig, + integrations: list[S3Integration], + ) -> dict[str, dict[str, str]]: + """Build per-integrator s3-integrator charm config tfvar.""" + if not config.configure_s3_integrators: + return {} + + base_path = (config.path or DEFAULT_S3_PATH).strip("/") + path_prefix = f"/{base_path}" if base_path else "" + return { + integration.integrator_app: { + "bucket": config.bucket, + "path": f"{path_prefix}/{integration.app_name}", + "region": config.region, + "endpoint": config.endpoint, + } + for integration in integrations + } + + def _s3_integrator_secret_data_tfvar( + self, + config: DisasterRecoveryFeatureConfig, + integrations: list[S3Integration], + ) -> dict[str, dict[str, str]]: + """Build per-integrator secret payload for juju_secret resources.""" + if not config.configure_s3_integrators: + return {} + + return { + integration.integrator_app: { + "access-key": config.access_key, + "secret-key": config.secret_key, + } + for integration in integrations + } + + def _s3_load_managed_integrations(self, deployment: Deployment) -> dict[str, str]: + info = self.get_feature_info(deployment.get_client()) + managed = info.get(_MANAGED_S3_KEY, {}) + if not isinstance(managed, dict): + return {} + return {str(k): str(v) for k, v in managed.items()} + + def _s3_save_managed_integrations( + self, deployment: Deployment, integrations: list[S3Integration] + ) -> None: + self.update_feature_info( + deployment.get_client(), + { + _MANAGED_S3_KEY: { + integration.app_name: integration.integrator_app + for integration in integrations + } + }, + ) diff --git a/sunbeam-python/sunbeam/features/interface/v1/openstack.py b/sunbeam-python/sunbeam/features/interface/v1/openstack.py index 8aa43d3ae..d71c52a1b 100644 --- a/sunbeam-python/sunbeam/features/interface/v1/openstack.py +++ b/sunbeam-python/sunbeam/features/interface/v1/openstack.py @@ -5,7 +5,7 @@ import queue import typing from abc import abstractmethod -from enum import Enum +from enum import Enum, StrEnum from pathlib import Path import click @@ -80,6 +80,13 @@ class TerraformPlanLocation(Enum): FEATURE_REPO = 2 +class DatabaseTopology(StrEnum): + """Database deployment topology.""" + + SINGLE = "single" + MULTI = "multi" + + class OpenStackControlPlaneFeature(EnableDisableFeature, typing.Generic[ConfigType]): """Interface for features to manage OpenStack Control plane components. @@ -166,7 +173,12 @@ def run_enable_plans( [ TerraformInitStep(deployment.get_tfhelper(self.tfplan)), EnableOpenStackApplicationStep( - deployment, config, tfhelper, jhelper, self + deployment, + config, + tfhelper, + jhelper, + self, + overlay=self.get_app_status_overlay_on_enable(deployment), ), ] ) @@ -174,6 +186,16 @@ def run_enable_plans( run_plan(plan, console, show_hints) click.echo(f"OpenStack {self.display_name} application enabled.") + def get_app_status_overlay_on_enable( + self, deployment: Deployment + ) -> dict[str, ApplicationStatusOverlay]: + """Set per-app status overlay while waiting on enable. + + Features can override this to accept additional workload statuses for + specific apps during enablement. + """ + return {} + def pre_disable(self, deployment: Deployment, show_hints: bool) -> None: """Handler to perform tasks before disabling the feature.""" self.pre_checks(deployment) @@ -205,12 +227,12 @@ def get_tfvar_config_key(self) -> str: else: return f"TerraformVars{self.app_name}" - def get_database_topology(self, deployment: Deployment) -> str: + def get_database_topology(self, deployment: Deployment) -> DatabaseTopology: """Returns the database topology of the cluster.""" # Database topology can be set only during bootstrap and cannot be changed. client = deployment.get_client() topology = read_config(client, TOPOLOGY_KEY) - return topology["database"] + return DatabaseTopology(topology["database"]) def get_cluster_topology(self, deployment: Deployment) -> str: """Returns the cluster topology of the cluster.""" diff --git a/sunbeam-python/sunbeam/main.py b/sunbeam-python/sunbeam/main.py index 69efba9d3..96fc35833 100644 --- a/sunbeam-python/sunbeam/main.py +++ b/sunbeam-python/sunbeam/main.py @@ -9,6 +9,7 @@ from snaphelpers import Snap from sunbeam import log +from sunbeam.commands import backup_restore as backup_restore_cmds from sunbeam.commands import configure as configure_cmds from sunbeam.commands import dashboard as dashboard_cmds from sunbeam.commands import generate_cloud_config as generate_cloud_config_cmds @@ -123,6 +124,9 @@ def main(): cli.add_command(launch_cmds.launch) cli.add_command(openrc_cmds.openrc) cli.add_command(dashboard_cmds.dashboard) + cli.add_command(backup_restore_cmds.backup) + cli.add_command(backup_restore_cmds.list_backups) + cli.add_command(backup_restore_cmds.restore) # Add identity group cli.add_command(identity_group) diff --git a/sunbeam-python/sunbeam/steps/backup_restore.py b/sunbeam-python/sunbeam/steps/backup_restore.py new file mode 100644 index 000000000..03b180432 --- /dev/null +++ b/sunbeam-python/sunbeam/steps/backup_restore.py @@ -0,0 +1,1496 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Steps and component registry for ``sunbeam backup/restore``. + +The module composes and +runs the top-level steps defined here. All per-component logic (validation +checks, target resolution, and backup/list/restore/revert plans) lives on the +:class:`BackupComponent` registry and is driven by the wrapper steps below. +""" + +import json +import logging +from abc import ABC, abstractmethod +from collections.abc import Callable, Mapping +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from pathlib import Path + +import yaml +from jubilant.statustypes import AppStatus +from snaphelpers import Snap + +from sunbeam.core.common import SHARE_PATH, BaseStep, Result, ResultType, StepContext +from sunbeam.core.juju import ( + ActionFailedException, + ApplicationNotFoundException, + JujuException, + JujuHelper, + JujuWaitException, + LeaderNotFoundException, + ModelNotFoundException, +) +from sunbeam.core.openstack import OPENSTACK_MODEL + +LOG = logging.getLogger(__name__) + +MYSQL_CHARM = "mysql-k8s" +VAULT_CHARM = "vault-k8s" +BACKUP_ACTION = "create-backup" +RESTORE_ACTION = "restore" +BACKUP_RESULT_ID_KEY = "backup-id" +LIST_BACKUPS_ACTION = "list-backups" +VAULT_RESTORE_ACTION = "restore-backup" +MYSQL_CLUSTER_STATUS_ACTION = "get-cluster-status" +S3_INTERFACE = "s3" +S3_ENDPOINT = "s3-parameters" +DEFAULT_BACKUP_TIMEOUT = 1800 +DEFAULT_RESTORE_TIMEOUT = 1800 +DEFAULT_ACTION_TIMEOUT = 120 +BACKUP_MANIFEST_DIR = SHARE_PATH / "backups" +PAUSE_ACTION = "pause" +RESUME_ACTION = "resume" +RESTORE_TIME_FORMAT = "%Y-%m-%d %H:%M:%S" + + +@dataclass +class ActionTarget: + """An application and the unit chosen to run an action against.""" + + app: str + unit: str + component: str + action: str + + +@dataclass +class BackupOutcome: + """A backup entry in a backup inventory for an application.""" + + backup_id: str + success: bool | None = None + + +@dataclass +class BackupResult: + """The outcome of attempting a backup for an application.""" + + app: str + unit: str + component: str + backup: BackupOutcome | None = None + error: str | None = None + + +@dataclass +class BackupInventory: + """The result of listing backup IDs for an application.""" + + app: str + unit: str + component: str + backups: list[BackupOutcome] | None = None + error: str | None = None + + +@dataclass +class RestoreResult: + """The outcome of attempting a restore for an application.""" + + app: str + component: str + success: bool + error: str | None = None + reverted: bool = False + rollback_error: str | None = None + + +@dataclass +class PreparedRestore: + """A restore and revert plan.""" + + component: "BackupComponent" + target: ActionTarget + plan: list[BaseStep] + revert_plan: list[BaseStep] + + +@dataclass +class ValidationCheck: + """An application-readiness check.""" + + name: str + predicate: Callable[[AppStatus], bool] + forceable: bool = False + + +class BackupComponent(ABC): + """Backup and restore workflow contract for an app.""" + + name: str + backup_action: str = BACKUP_ACTION + restore_action: str = RESTORE_ACTION + list_action: str = LIST_BACKUPS_ACTION + backup_id_param: str = BACKUP_RESULT_ID_KEY + restore_to_time_param: str | None = None + + @property + def validate_checks(self) -> list[ValidationCheck]: + """Return readiness checks applied before backup or restore.""" + return [APP_READY_VALIDATION_CHECK, S3_RELATION_VALIDATION_CHECK] + + @abstractmethod + def resolve_backup_target( + self, jhelper: JujuHelper, app: str, model: str, force: bool + ) -> ActionTarget | None: + """Resolve the unit on which to run a backup action.""" + + @abstractmethod + def parse_backup_list(self, action_result: dict) -> list[BackupOutcome]: + """Parse a component-specific list-backups action result.""" + + def parse_backup(self, action_result: dict) -> BackupOutcome | None: + """Parse create-backup output and return the backup ID.""" + backup_id = action_result.get(BACKUP_RESULT_ID_KEY) + if not isinstance(backup_id, str): + return None + return BackupOutcome(backup_id=backup_id, success=True) + + def build_backup_plan( + self, + jhelper: JujuHelper, + target: ActionTarget, + timeout: int, + model: str, + ) -> list[BaseStep]: + """Build the common single-action backup plan.""" + return [_BackupAppStep(jhelper, self, target, timeout=timeout, model=model)] + + def build_restore_precheck_plan( + self, + jhelper: JujuHelper, + target: ActionTarget, + timeout: int, + model: str, + ) -> list[BaseStep]: + """Return non-destructive checks that must pass before restore.""" + return [] + + def latest_backup_params( + self, + jhelper: JujuHelper, + target: ActionTarget, + timeout: int, + model: str, + ) -> dict[str, str]: + """Resolve the latest successful backup into restore action parameters.""" + list_result = jhelper.run_action( + target.unit, + model, + self.list_action, + timeout=timeout, + ) + latest = _latest_backup(self.parse_backup_list(list_result)) + if latest is None: + raise JujuException(f"No finished backups found for {target.app}.") + return {self.backup_id_param: latest} + + @abstractmethod + def restore_params( + self, + jhelper: JujuHelper, + target: ActionTarget, + restore_to_time: str | None, + timeout: int, + model: str, + ) -> dict[str, str]: + """Build parameters that satisfy this component's restore action contract.""" + + @abstractmethod + def build_restore_plan( + self, + jhelper: JujuHelper, + target: ActionTarget, + restore_to_time: str | None, + timeout: int, + model: str, + ) -> list[BaseStep]: + """Build the operational restore sequence.""" + + def build_restore_revert_plan( + self, + jhelper: JujuHelper, + target: ActionTarget, + timeout: int, + model: str, + ) -> list[BaseStep]: + """Build revert steps for a failed restore.""" + return [] + + +def _component_for(name: str) -> BackupComponent | None: + return next((c for c in BACKUP_COMPONENTS if c.name == name), None) + + +# --------------------------------------------------------------------------- +# Validation predicates +# --------------------------------------------------------------------------- +def _is_app_active(app_status: AppStatus) -> bool: + """Return whether application workload status is active.""" + return app_status.app_status.current == "active" + + +def _is_related_to_s3(app_status: AppStatus) -> bool: + """Return whether the application is related to S3 via the endpoint.""" + endpoint_relations = app_status.relations.get(S3_ENDPOINT, []) + return any(rel.interface == S3_INTERFACE for rel in endpoint_relations) + + +APP_READY_VALIDATION_CHECK: ValidationCheck = ValidationCheck( + name="active", + predicate=_is_app_active, + forceable=True, +) + +S3_RELATION_VALIDATION_CHECK: ValidationCheck = ValidationCheck( + name="s3-relation", + predicate=_is_related_to_s3, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- +def _secondary_unit_from_status(units: list[str], action_result: dict) -> str | None: + """Map a SECONDARY mysql cluster member to a Juju unit name.""" + status = action_result.get("status") or {} + topology = status.get("defaultreplicaset", {}).get("topology", {}) + secondary_labels = [ + label + for label, info in topology.items() + if isinstance(info, dict) and info.get("memberrole", "").lower() == "secondary" + ] + if not secondary_labels: + return None + + for label in secondary_labels: + ordinal = label.split(".")[0].rsplit("-", 1)[-1] + for unit in units: + if unit.rsplit("/", 1)[-1] == ordinal: + return unit + return None + + +def _latest_backup(backups: list[BackupOutcome]) -> str | None: + """Return the lexicographically latest successful backup ID, if present.""" + successful = [b for b in backups if b.success] + if not successful: + return None + return sorted(successful, key=lambda b: b.backup_id)[-1].backup_id + + +# --------------------------------------------------------------------------- +# Atomic action steps +# --------------------------------------------------------------------------- +class _ActionStep(BaseStep): + """Dispatch an action on an application's leader or all units.""" + + def __init__( + self, + jhelper: JujuHelper, + name: str, + description: str, + app: str, + action_name: str, + run_on_all_units: bool = False, + expected_status: list[str] | None = None, + timeout: int = DEFAULT_ACTION_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__(name, description) + self.jhelper = jhelper + self.model = model + self.app = app + self.action_name = action_name + self.run_on_all_units = run_on_all_units + self.timeout = timeout + self.expected_status = expected_status or ["active"] + + def run(self, context: StepContext) -> Result: + """Run action on the application's leader or all units.""" + try: + if self.run_on_all_units: + units = list(self.jhelper.get_application(self.app, self.model).units) + else: + units = [self.jhelper.get_leader_unit(self.app, self.model)] + + for unit in units: + self.jhelper.run_action( + unit, + self.model, + self.action_name, + timeout=self.timeout, + ) + self.jhelper.wait_until_desired_status( + self.model, + apps=[self.app], + status=self.expected_status, + agent_status=["idle"], + timeout=self.timeout, + ) + except ( + ActionFailedException, + LeaderNotFoundException, + ModelNotFoundException, + JujuException, + ) as e: + return Result(ResultType.FAILED, str(e)) + + return Result(ResultType.COMPLETED) + + +class _PauseAppStep(_ActionStep): + """Pause application API services.""" + + def __init__( + self, + jhelper: JujuHelper, + app: str, + timeout=DEFAULT_ACTION_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__( + jhelper, + "Pause App", + "Pause API container services", + app, + PAUSE_ACTION, + run_on_all_units=True, + expected_status=["maintenance"], + timeout=timeout, + model=model, + ) + + +class _ResumeAppStep(_ActionStep): + """Resume application API services.""" + + def __init__( + self, + jhelper: JujuHelper, + app: str, + timeout=DEFAULT_ACTION_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__( + jhelper, + "Resume App", + "Resume API container services", + app, + RESUME_ACTION, + run_on_all_units=True, + expected_status=["active"], + timeout=timeout, + model=model, + ) + + +class _ScaleAppStep(BaseStep): + """Scale an application to a target number of units.""" + + def __init__( + self, + jhelper: JujuHelper, + application: str, + scale: int, + timeout: int = DEFAULT_ACTION_TIMEOUT, + model: str = OPENSTACK_MODEL, + ) -> None: + super().__init__("Scale App", f"Scaling {application} to {scale} unit(s)") + self.jhelper = jhelper + self.application = application + self.scale = scale + self.timeout = timeout + self.model = model + + def run(self, context: StepContext) -> Result: + """Scale the application and wait for it to settle.""" + try: + units = list( + self.jhelper.get_application(self.application, self.model).units + ) + self.jhelper.scale_application(self.model, self.application, self.scale) + + if self.scale == 0: + self.jhelper.wait_units_gone(units, self.model, timeout=self.timeout) + else: + self.jhelper.wait_until_active( + self.model, apps=[self.application], timeout=self.timeout + ) + except ( + ApplicationNotFoundException, + JujuException, + JujuWaitException, + TimeoutError, + ) as e: + return Result(ResultType.FAILED, str(e)) + return Result(ResultType.COMPLETED) + + +class _RestoreAppStep(BaseStep): + """Restore a single application from a backup (atomic restore action).""" + + def __init__( + self, + jhelper: JujuHelper, + component: BackupComponent, + target: ActionTarget, + restore_to_time: str | None = None, + expected_status: list[str] | None = None, + timeout: int = DEFAULT_RESTORE_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__("Restore app", f"Restoring {target.app}") + self.jhelper = jhelper + self.target = target + self.restore_to_time = restore_to_time + self.component = component + self.model = model + self.timeout = timeout + self.expected_status = expected_status or ["active"] + + def run(self, context: StepContext) -> Result: + """Restore an app using latest backup or restore-to-time.""" + try: + leader = self.jhelper.get_leader_unit(self.target.app, self.model) + except (LeaderNotFoundException, JujuException) as e: + return Result(ResultType.FAILED, str(e)) + + try: + params = self.component.restore_params( + self.jhelper, + self.target, + self.restore_to_time, + self.timeout, + self.model, + ) + except (ActionFailedException, JujuException) as e: + return Result(ResultType.FAILED, str(e)) + + try: + self.jhelper.run_action( + leader, + self.model, + self.component.restore_action, + params, + timeout=self.timeout, + ) + self.jhelper.wait_until_desired_status( + self.model, + apps=[self.target.app], + status=self.expected_status, + agent_status=["idle"], + timeout=self.timeout, + ) + except (ActionFailedException, JujuException) as e: + return Result(ResultType.FAILED, str(e)) + + return Result(ResultType.COMPLETED) + + +class _BackupAppStep(BaseStep): + """Dispatch a single application's backup action and capture the outcome.""" + + def __init__( + self, + jhelper: JujuHelper, + component: BackupComponent, + target: ActionTarget, + timeout: int = DEFAULT_BACKUP_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__("Backup app", f"Backing up {target.app}") + self.jhelper = jhelper + self.component = component + self.target = target + self.timeout = timeout + self.model = model + self.result: BackupResult | None = None + + def run(self, context: StepContext) -> Result: + """Dispatch the backup action, recording the outcome on ``self.result``.""" + target = self.target + + try: + action_result = self.jhelper.run_action( + target.unit, + self.model, + target.action, + timeout=self.timeout, + ) + backup = self.component.parse_backup(action_result) + if backup is None: + self.result = BackupResult( + app=target.app, + unit=target.unit, + component=target.component, + error="Backup action completed without backup id.", + ) + return Result(ResultType.FAILED, self.result.error) + self.result = BackupResult( + app=target.app, + unit=target.unit, + component=target.component, + backup=backup, + ) + except (ActionFailedException, JujuException) as e: + message = str(e) + self.result = BackupResult( + app=target.app, + unit=target.unit, + component=target.component, + error=message, + ) + return Result(ResultType.FAILED, message) + + return Result(ResultType.COMPLETED, self.result) + + +class _CheckPauseResumeSupportStep(BaseStep): + """Validate pause/resume action support for a single application.""" + + def __init__( + self, + jhelper: JujuHelper, + app: str, + timeout: int = DEFAULT_ACTION_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__( + "Check pause/resume support", + "Validating pause/resume support before restore", + ) + self.jhelper = jhelper + self.app = app + self.model = model + self.timeout = timeout + + def run(self, context: StepContext) -> Result: + """Check that the application supports pause/resume actions.""" + try: + actions = self.jhelper.get_application_actions(self.app, self.model) + except (JujuException, ModelNotFoundException): + return Result( + ResultType.FAILED, + f"Unable to query actions for {self.app}. No changes have been made.", + ) + + if PAUSE_ACTION not in actions or RESUME_ACTION not in actions: + return Result( + ResultType.FAILED, + f"Control-plane application {self.app} does not support the " + "'pause/resume' action required for restore. " + "No changes have been made.", + ) + + return Result(ResultType.COMPLETED) + + +# --------------------------------------------------------------------------- +# Component workflows +# --------------------------------------------------------------------------- +class MySQLBackupComponent(BackupComponent): + """MySQL backup and restore workflow.""" + + name = MYSQL_CHARM + restore_to_time_param = "restore-to-time" + + @staticmethod + def _related_apps_for_interface(app_status: AppStatus, interface: str) -> set[str]: + """Return related application names for an interface.""" + related_apps: set[str] = set() + for endpoint_relations in app_status.relations.values(): + for relation in endpoint_relations: + if relation.interface != interface: + continue + related_apps.add(relation.related_app) + return related_apps + + def _api_apps_via_routers( + self, + apps: Mapping[str, AppStatus], + mysql_app: str, + router_apps: set[str], + ) -> set[str]: + """Traverse mysql-router relations to resolve control-plane apps.""" + api_apps: set[str] = set() + for router_app in router_apps: + router_status = apps.get(router_app) + if router_status is None: + continue + related_apps = self._related_apps_for_interface( + router_status, "mysql_client" + ) + for related_app in related_apps: + if ( + related_app == mysql_app + or related_app.endswith("-mysql-router") + or related_app not in apps + ): + continue + api_apps.add(related_app) + return api_apps + + def _restore_apps( + self, jhelper: JujuHelper, mysql_app: str, model: str + ) -> tuple[list[str], list[str]]: + """Resolve control-plane and router apps backed by a MySQL application.""" + status = jhelper.get_model_status(model) + mysql_status = status.apps.get(mysql_app) + if mysql_status is None: + raise JujuException(f"MySQL application {mysql_app} not found in model") + + router_apps = self._related_apps_for_interface(mysql_status, "mysql_client") + if not router_apps: + raise JujuException( + f"Could not resolve router applications for MySQL app {mysql_app}" + ) + api_apps = self._api_apps_via_routers(status.apps, mysql_app, router_apps) + + if api_apps: + return sorted(api_apps), sorted(router_apps) + if mysql_app.endswith("-mysql"): + api_app = mysql_app.removesuffix("-mysql") + return [api_app], sorted(router_apps) + + raise JujuException( + f"Could not resolve control-plane applications for MySQL app {mysql_app}" + ) + + @staticmethod + def _current_scale(jhelper: JujuHelper, app: str, model: str) -> int: + """Read the current MySQL unit count, failing if Juju cannot provide it.""" + try: + return len(list(jhelper.get_application(app, model).units)) + except (ApplicationNotFoundException, JujuException): + raise JujuException(f"Could not read current scale for {app}") + + def parse_backup_list(self, action_result: dict) -> list[BackupOutcome]: + """Parse MySQL's tabular list-backups output.""" + backups_text = action_result.get("backups") + if not isinstance(backups_text, str): + return [] + + backups: list[BackupOutcome] = [] + for raw_line in backups_text.splitlines(): + line = raw_line.strip() + if not line or line.startswith("backup-id") or set(line) == {"-"}: + continue + columns = [column.strip() for column in line.split("|")] + if len(columns) < 3: + continue + backups.append( + BackupOutcome( + backup_id=columns[0], + success=columns[2].lower() == "finished", + ) + ) + return backups + + def resolve_backup_target( + self, jhelper: JujuHelper, app: str, model: str, force: bool + ) -> ActionTarget | None: + """Prefer a secondary MySQL unit, with forced leader fallback.""" + try: + leader = jhelper.get_leader_unit(app, model) + units = list(jhelper.get_application(app, model).units) + except (LeaderNotFoundException, ApplicationNotFoundException): + LOG.warning("Could not resolve %s, skipping", app) + return None + + try: + result = jhelper.run_action(leader, model, MYSQL_CLUSTER_STATUS_ACTION) + secondary = _secondary_unit_from_status(units, result) + if secondary is not None: + return ActionTarget(app, secondary, self.name, self.backup_action) + except ActionFailedException as e: + if not force: + LOG.warning( + "Could not resolve backup target for %s, skipping: %s", app, e + ) + return None + LOG.warning( + "Could not resolve backup target for %s, using leader (--force): %s", + app, + e, + ) + + return ActionTarget(app, leader, self.name, self.backup_action) + + def restore_params( + self, + jhelper: JujuHelper, + target: ActionTarget, + restore_to_time: str | None, + timeout: int, + model: str, + ) -> dict[str, str]: + """Use PITR when requested; otherwise restore the latest backup ID.""" + if restore_to_time is not None: + return {"restore-to-time": restore_to_time} + return self.latest_backup_params(jhelper, target, timeout, model) + + def build_restore_precheck_plan( + self, + jhelper: JujuHelper, + target: ActionTarget, + timeout: int, + model: str, + ) -> list[BaseStep]: + """Check pause/resume support for every related control-plane app.""" + api_apps, _ = self._restore_apps(jhelper, target.app, model) + return [ + _CheckPauseResumeSupportStep(jhelper, api_app, timeout=timeout, model=model) + for api_app in api_apps + ] + + def build_restore_plan( + self, + jhelper: JujuHelper, + target: ActionTarget, + restore_to_time: str | None, + timeout: int, + model: str, + ) -> list[BaseStep]: + """Pause clients, restore one MySQL unit, then restore scale and clients.""" + api_apps, router_apps = self._restore_apps(jhelper, target.app, model) + original_scale = self._current_scale(jhelper, target.app, model) + router_scales = { + router_app: self._current_scale(jhelper, router_app, model) + for router_app in router_apps + } + return [ + *[ + _PauseAppStep(jhelper, api_app, timeout=timeout, model=model) + for api_app in api_apps + ], + *[ + _ScaleAppStep(jhelper, router_app, 0, timeout=timeout, model=model) + for router_app in router_apps + ], + _ScaleAppStep(jhelper, target.app, 1, timeout=timeout, model=model), + _RestoreAppStep( + jhelper, + self, + target, + restore_to_time=restore_to_time, + expected_status=["active", "blocked"], + timeout=timeout, + model=model, + ), + _ScaleAppStep( + jhelper, + target.app, + original_scale, + timeout=timeout, + model=model, + ), + *[ + _ScaleAppStep( + jhelper, + router_app, + router_scales[router_app], + timeout=timeout, + model=model, + ) + for router_app in router_apps + ], + *[ + _ResumeAppStep(jhelper, api_app, timeout=timeout, model=model) + for api_app in api_apps + ], + ] + + def build_restore_revert_plan( + self, + jhelper: JujuHelper, + target: ActionTarget, + timeout: int, + model: str, + ) -> list[BaseStep]: + """Restore the original MySQL scale and resume client applications.""" + api_apps, router_apps = self._restore_apps(jhelper, target.app, model) + original_scale = self._current_scale(jhelper, target.app, model) + router_scales = { + router_app: self._current_scale(jhelper, router_app, model) + for router_app in router_apps + } + return [ + _ScaleAppStep( + jhelper, + target.app, + original_scale, + timeout=timeout, + model=model, + ), + *[ + _ScaleAppStep( + jhelper, + router_app, + router_scales[router_app], + timeout=timeout, + model=model, + ) + for router_app in router_apps + ], + *[ + _ResumeAppStep(jhelper, api_app, timeout=timeout, model=model) + for api_app in api_apps + ], + ] + + +class VaultBackupComponent(BackupComponent): + """Vault backup and restore workflow.""" + + name = VAULT_CHARM + restore_action = VAULT_RESTORE_ACTION + + def parse_backup_list(self, action_result: dict) -> list[BackupOutcome]: + """Parse Vault's JSON backup-id list.""" + raw_backup_ids = action_result.get("backup-ids") + if not isinstance(raw_backup_ids, str): + return [] + try: + parsed = json.loads(raw_backup_ids) + except json.JSONDecodeError: + return [] + if not isinstance(parsed, list): + return [] + return [ + BackupOutcome(backup_id=str(backup_id), success=True) + for backup_id in parsed + ] + + def resolve_backup_target( + self, jhelper: JujuHelper, app: str, model: str, force: bool + ) -> ActionTarget | None: + """Resolve Vault backups to the leader unit.""" + try: + leader = jhelper.get_leader_unit(app, model) + except (LeaderNotFoundException, ApplicationNotFoundException): + LOG.warning("Could not resolve %s, skipping", app) + return None + return ActionTarget(app, leader, self.name, self.backup_action) + + def restore_params( + self, + jhelper: JujuHelper, + target: ActionTarget, + restore_to_time: str | None, + timeout: int, + model: str, + ) -> dict[str, str]: + """Vault has no PITR action contract; always use the latest backup ID.""" + return self.latest_backup_params(jhelper, target, timeout, model) + + def build_restore_plan( + self, + jhelper: JujuHelper, + target: ActionTarget, + restore_to_time: str | None, + timeout: int, + model: str, + ) -> list[BaseStep]: + """Restore Vault directly, falling back to latest when PITR is requested.""" + return [ + _RestoreAppStep( + jhelper, + self, + target, + restore_to_time=restore_to_time, + timeout=timeout, + model=model, + ) + ] + + +# --------------------------------------------------------------------------- +# Public component registry +# --------------------------------------------------------------------------- +BACKUP_COMPONENTS: list[BackupComponent] = [ + MySQLBackupComponent(), + VaultBackupComponent(), +] + + +# --------------------------------------------------------------------------- +# Public top-level wrapper steps +# --------------------------------------------------------------------------- +class DiscoverBackupApplicationsStep(BaseStep): + """Discover applications of every registered backup component in the model.""" + + def __init__( + self, + jhelper: JujuHelper, + model: str = OPENSTACK_MODEL, + components: list[BackupComponent] = BACKUP_COMPONENTS, + ): + super().__init__( + "Discover backup applications", + "Discovering stateful applications to back up", + ) + self.jhelper = jhelper + self.components = components + self.model = model + + def run(self, context: StepContext) -> Result: + """Return a mapping of component name to discovered application names.""" + try: + status = self.jhelper.get_model_status(self.model) + except (ModelNotFoundException, JujuException) as e: + return Result(ResultType.FAILED, str(e)) + + discovered: dict[str, list[str]] = {c.name: [] for c in self.components} + for app_name, app_status in status.apps.items(): + charm_name = app_status.charm_name or "" + for component in self.components: + if charm_name == component.name: + discovered[component.name].append(app_name) + + return Result(ResultType.COMPLETED, discovered) + + +class ValidateStep(BaseStep): + """Apply each component's own validation checks to its discovered apps. + + Returns a mapping keyed by component with, per application, the list of + failed check names. Applications with an empty failure list are valid. + """ + + def __init__( + self, + jhelper: JujuHelper, + discovered: dict[str, list[str]], + model: str = OPENSTACK_MODEL, + components: list[BackupComponent] = BACKUP_COMPONENTS, + force: bool = False, + ): + super().__init__("Validate applications", "Validating backup readiness") + self.jhelper = jhelper + self.discovered = discovered + self.model = model + self.components = components + self.force = force + + def run(self, context: StepContext) -> Result: + """Return {'valid': {...}, 'failures': {app: [check names]}}.""" + try: + status = self.jhelper.get_model_status(self.model) + except (ModelNotFoundException, JujuException) as e: + return Result(ResultType.FAILED, str(e)) + + valid: dict[str, list[str]] = {} + failures: dict[str, list[str]] = {} + for component_name, apps in self.discovered.items(): + component = _component_for(component_name) + if component is None: + continue + + valid[component_name] = [] + for app_name in apps: + app_status = status.apps.get(app_name) + failed = self._failed_checks(component, app_status, self.force) + if failed: + failures[app_name] = failed + else: + valid[component_name].append(app_name) + + return Result(ResultType.COMPLETED, {"valid": valid, "failures": failures}) + + @staticmethod + def _failed_checks( + component: BackupComponent, + app_status: AppStatus | None, + force: bool = False, + ) -> list[str]: + if app_status is None: + return [check.name for check in component.validate_checks] or ["present"] + return [ + check.name + for check in component.validate_checks + if not (check.forceable and force) and not check.predicate(app_status) + ] + + +class ResolveActionTargetsStep(BaseStep): + """Resolve every discovered application's leader unit for generic actions.""" + + def __init__( + self, + jhelper: JujuHelper, + discovered: dict[str, list[str]], + action: Callable[[BackupComponent], str], + model: str = OPENSTACK_MODEL, + ): + super().__init__( + "Resolve action targets", + "Resolving units for action targets", + ) + self.jhelper = jhelper + self.discovered = discovered + self.action = action + self.model = model + + def run(self, context: StepContext) -> Result: + """Return resolved targets and unresolved apps across components.""" + targets: list[ActionTarget] = [] + unresolved: list[dict[str, str]] = [] + for component_name, apps in self.discovered.items(): + component = _component_for(component_name) + if component is None: + continue + + for app in apps: + try: + leader = self.jhelper.get_leader_unit(app, self.model) + except (LeaderNotFoundException, ApplicationNotFoundException): + self.update_status(context, f"skipped {app}") + unresolved.append({"app": app, "component": component.name}) + continue + targets.append( + ActionTarget( + app=app, + unit=leader, + component=component.name, + action=self.action(component), + ) + ) + + return Result( + ResultType.COMPLETED, + { + "targets": targets, + "unresolved": sorted(unresolved, key=lambda item: item["app"]), + }, + ) + + +class RunBackupStep(BaseStep): + """Resolve targets per component, run each backup plan, and collect results.""" + + def __init__( + self, + jhelper: JujuHelper, + discovered: dict[str, list[str]], + force: bool = False, + timeout: int = DEFAULT_BACKUP_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__("Run backups", "Dispatching backups across applications") + self.jhelper = jhelper + self.discovered = discovered + self.force = force + self.timeout = timeout + self.model = model + + def _resolve_targets( + self, context: StepContext + ) -> tuple[list[tuple[BackupComponent, ActionTarget]], list[BackupResult]]: + """Resolve one backup target per discovered application.""" + resolved: list[tuple[BackupComponent, ActionTarget]] = [] + failed: list[BackupResult] = [] + for component_name, apps in self.discovered.items(): + component = _component_for(component_name) + if component is None: + continue + + for app in apps: + target = component.resolve_backup_target( + self.jhelper, app, self.model, self.force + ) + if target is None: + self.update_status(context, f"skipped {app}") + failed.append( + BackupResult( + app=app, + unit="-", + component=component.name, + error="Could not resolve backup target.", + ) + ) + continue + resolved.append((component, target)) + return resolved, failed + + def _run_backup_plan( + self, + component: BackupComponent, + target: ActionTarget, + context: StepContext, + ) -> BackupResult: + """Run a component's backup plan and return its BackupResult.""" + plan = component.build_backup_plan( + self.jhelper, target, self.timeout, self.model + ) + result: BackupResult | None = None + + for step in plan: + step_result = step.run(context) + if isinstance(step, _BackupAppStep) and step.result is not None: + result = step.result + if step_result.result_type == ResultType.FAILED: + return result or BackupResult( + app=target.app, + unit=target.unit, + component=target.component, + error=str(step_result.message), + ) + return result or BackupResult( + app=target.app, + unit=target.unit, + component=target.component, + error="Backup plan produced no result.", + ) + + def run(self, context: StepContext) -> Result: + """Resolve targets, run backup plans concurrently, return results.""" + resolved, failed = self._resolve_targets(context) + if not resolved: + return Result(ResultType.COMPLETED, failed) + + results: list[BackupResult] = list(failed) + with ThreadPoolExecutor(max_workers=len(resolved)) as executor: + futures = [ + executor.submit(self._run_backup_plan, component, target, context) + for component, target in resolved + ] + for future in as_completed(futures): + results.append(future.result()) + + return Result(ResultType.COMPLETED, results) + + +class ListBackupsStep(BaseStep): + """Dispatch list-backups actions concurrently and collect inventories.""" + + def __init__( + self, + jhelper: JujuHelper, + targets: list[ActionTarget], + timeout: int = DEFAULT_BACKUP_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__("List backups", "Listing backups across applications") + self.jhelper = jhelper + self.targets = targets + self.timeout = timeout + self.model = model + + def _list_one( + self, + target: ActionTarget, + action_name: str, + parse_backups: Callable[[dict], list[BackupOutcome]] | None, + ) -> BackupInventory: + """Dispatch a single list-backups action and parse backup IDs.""" + try: + result = self.jhelper.run_action( + target.unit, self.model, action_name, timeout=self.timeout + ) + backups = parse_backups(result) if parse_backups is not None else [] + return BackupInventory( + app=target.app, + unit=target.unit, + component=target.component, + backups=backups, + ) + except (ActionFailedException, JujuException) as e: + return BackupInventory( + app=target.app, + unit=target.unit, + component=target.component, + error=str(e), + ) + + def run(self, context: StepContext) -> Result: + """Dispatch list-backups concurrently, returning inventory entries.""" + if not self.targets: + return Result(ResultType.COMPLETED, []) + + inventories: list[BackupInventory] = [] + with ThreadPoolExecutor(max_workers=len(self.targets)) as executor: + futures = [] + for target in self.targets: + component = _component_for(target.component) + if component is None: + continue + + futures.append( + executor.submit( + self._list_one, + target, + component.list_action, + component.parse_backup_list, + ) + ) + for future in as_completed(futures): + inventories.append(future.result()) + + return Result(ResultType.COMPLETED, inventories) + + +class RestoreStep(BaseStep): + """Drive every component's restore plan, reverting on failure.""" + + def __init__( + self, + jhelper: JujuHelper, + discovered: dict[str, list[str]], + restore_to_time: str | None = None, + timeout: int = DEFAULT_RESTORE_TIMEOUT, + model: str = OPENSTACK_MODEL, + ): + super().__init__("Restore applications", "Restoring applications from backup") + self.jhelper = jhelper + self.discovered = discovered + self.restore_to_time = restore_to_time + self.timeout = timeout + self.model = model + + def _run_plan(self, plan: list[BaseStep], context: StepContext) -> None: + """Run a plan of steps, raising RuntimeError on the first failure.""" + for step in plan: + result = step.run(context) + if result.result_type == ResultType.FAILED: + raise RuntimeError(result.message) + + def _resolve_targets(self, context: StepContext) -> list[ActionTarget]: + """Resolve one backup target per discovered application.""" + result = ResolveActionTargetsStep( + self.jhelper, + self.discovered, + action=lambda component: component.restore_action, + model=self.model, + ).run(context) + unresolved = result.message["unresolved"] + if unresolved: + raise RuntimeError( + f"Could not resolve restore target for {unresolved[0]['app']}" + ) + return result.message["targets"] + + def _prepare_restore( + self, + component: BackupComponent, + target: ActionTarget, + ) -> PreparedRestore: + """Build restore and compensation plans before any mutation occurs.""" + plan = component.build_restore_plan( + self.jhelper, + target, + self.restore_to_time, + self.timeout, + self.model, + ) + revert_plan = component.build_restore_revert_plan( + self.jhelper, + target, + self.timeout, + self.model, + ) + return PreparedRestore(component, target, plan, revert_plan) + + def _run_revert_plan(self, plan: list[BaseStep], context: StepContext) -> list[str]: + """Attempt every compensation step and return all failure messages.""" + errors: list[str] = [] + for step in plan: + try: + result = step.run(context) + if result.result_type == ResultType.FAILED: + errors.append(str(result.message)) + except ( + JujuException, + ActionFailedException, + LeaderNotFoundException, + ModelNotFoundException, + ) as e: + errors.append(str(e)) + return errors + + def _restore_one( + self, + prepared: PreparedRestore, + context: StepContext, + ) -> RestoreResult: + try: + self._run_plan(prepared.plan, context) + return RestoreResult( + app=prepared.target.app, + component=prepared.component.name, + success=True, + ) + except ( + RuntimeError, + JujuException, + ActionFailedException, + LeaderNotFoundException, + ModelNotFoundException, + ) as e: + revert_errors = self._run_revert_plan(prepared.revert_plan, context) + rollback_error = "; ".join(revert_errors) or None + if revert_errors: + LOG.warning( + "Revert failed for %s: %s", prepared.target.app, rollback_error + ) + return RestoreResult( + app=prepared.target.app, + component=prepared.component.name, + success=False, + error=str(e), + reverted=bool(prepared.revert_plan) and not revert_errors, + rollback_error=rollback_error, + ) + + def run(self, context: StepContext) -> Result: + """Precheck all targets, then restore each sequentially, aggregating.""" + try: + resolved = self._resolve_targets(context) + targets: list[tuple[BackupComponent, ActionTarget]] = [] + for target in resolved: + component = _component_for(target.component) + if component is not None: + targets.append((component, target)) + + for component, target in targets: + precheck = component.build_restore_precheck_plan( + self.jhelper, target, self.timeout, self.model + ) + self._run_plan(precheck, context) + + prepared = [ + self._prepare_restore(component, target) + for component, target in targets + ] + except ( + RuntimeError, + JujuException, + ActionFailedException, + LeaderNotFoundException, + ModelNotFoundException, + ) as e: + return Result(ResultType.FAILED, str(e)) + + if not prepared: + return Result(ResultType.COMPLETED, []) + + results: list[RestoreResult] = [] + for index, restore in enumerate(prepared): + self.update_status(context, f"restoring {restore.target.app}") + outcome = self._restore_one(restore, context) + results.append(outcome) + if outcome.success: + continue + results.extend( + RestoreResult( + app=pending.target.app, + component=pending.component.name, + success=False, + error=( + "Restore not attempted because restore failed for " + f"{restore.target.app}." + ), + ) + for pending in prepared[index + 1 :] + ) + break + + return Result(ResultType.COMPLETED, results) + + +class WriteBackupManifestStep(BaseStep): + """Write a timestamped manifest of the backup run to the snap share path.""" + + def __init__( + self, + results: list[BackupResult], + dispatched_at: str, + manifest_dir: Path | None = None, + ): + super().__init__("Write backup manifest", "Writing backup manifest") + self.results = results + self.dispatched_at = dispatched_at + self.manifest_dir = manifest_dir + + def run(self, context: StepContext) -> Result: + """Write the manifest and return its path.""" + if self.manifest_dir is not None: + directory = self.manifest_dir + else: + directory = Snap().paths.user_common / BACKUP_MANIFEST_DIR + directory.mkdir(parents=True, exist_ok=True) + + succeeded = sum( + 1 for r in self.results if r.backup is not None and r.backup.success + ) + failed = sum(1 for r in self.results if r.error is not None) + manifest = { + "dispatched_at": self.dispatched_at, + "summary": {"succeeded": succeeded, "failed": failed}, + "results": [asdict(r) for r in self.results], + } + + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") + path = directory / f"backup-manifest-{timestamp}.yaml" + try: + with path.open("w") as manifest_file: + yaml.safe_dump(manifest, manifest_file, sort_keys=False) + except OSError as e: + return Result(ResultType.FAILED, str(e)) + + return Result(ResultType.COMPLETED, str(path)) + + +class WriteBackupInventoryManifestStep(BaseStep): + """Write a timestamped manifest for backup inventory results.""" + + def __init__( + self, + results: list[BackupInventory], + listed_at: str, + manifest_dir: Path | None = None, + ): + super().__init__("Write backup inventory manifest", "Writing backup inventory") + self.results = results + self.listed_at = listed_at + self.manifest_dir = manifest_dir + + def run(self, context: StepContext) -> Result: + """Write the inventory manifest and return its path.""" + if self.manifest_dir is not None: + directory = self.manifest_dir + else: + directory = Snap().paths.user_common / BACKUP_MANIFEST_DIR + directory.mkdir(parents=True, exist_ok=True) + + succeeded = sum( + 1 for r in self.results if r.backups and any(b.success for b in r.backups) + ) + failed = len(self.results) - succeeded + manifest = { + "listed_at": self.listed_at, + "summary": {"succeeded": succeeded, "failed": failed}, + "results": [asdict(r) for r in self.results], + } + + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") + path = directory / f"backup-inventory-{timestamp}.yaml" + try: + with path.open("w") as manifest_file: + yaml.safe_dump(manifest, manifest_file, sort_keys=False) + except OSError as e: + return Result(ResultType.FAILED, str(e)) + + return Result(ResultType.COMPLETED, str(path)) diff --git a/sunbeam-python/sunbeam/versions.py b/sunbeam-python/sunbeam/versions.py index c042f1552..876f0cf21 100644 --- a/sunbeam-python/sunbeam/versions.py +++ b/sunbeam-python/sunbeam/versions.py @@ -36,6 +36,7 @@ def determine_version() -> str: VAULT_CHANNEL = "1.18/stable" CONSUL_CHANNEL = "1.19/stable" K8S_CHANNEL = "1.32/stable" +S3_INTEGRATOR_CHANNEL = "2/stable" MULTUS_CHANNEL = "latest/stable" LXD_CHANNEL = "5.21/stable" CINDER_VOLUME_CHARM = "cinder-volume" diff --git a/sunbeam-python/tests/functional/local/test_backup.py b/sunbeam-python/tests/functional/local/test_backup.py new file mode 100644 index 000000000..19f92a2fb --- /dev/null +++ b/sunbeam-python/tests/functional/local/test_backup.py @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Functional smoke test for ``sunbeam backup``. + +Skipped by default: requires a bootstrapped cloud with the ``openstack`` snap +installed and the object-storage backup prerequisite configured. Consistent with +the rest of the hardware/environment-gated functional suite. +""" + +import os + +import pytest + +from .utils import sunbeam_command + +pytestmark = pytest.mark.skipif( + not os.environ.get("SUNBEAM_FUNCTIONAL_BACKUP"), + reason="requires a bootstrapped cloud; set SUNBEAM_FUNCTIONAL_BACKUP to enable", +) + + +def test_backup_smoke(): + """``sunbeam backup --help`` is available and the command runs.""" + output = sunbeam_command("backup --help", capture_output=True) + assert "Create backups" in output diff --git a/sunbeam-python/tests/unit/sunbeam/commands/test_backup.py b/sunbeam-python/tests/unit/sunbeam/commands/test_backup.py new file mode 100644 index 000000000..472de2fd6 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/commands/test_backup.py @@ -0,0 +1,197 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +from unittest.mock import Mock + +import pytest +from click.testing import CliRunner + +from sunbeam.commands.backup_restore import backup +from sunbeam.core.juju import ActionFailedException + + +def _app_status(charm_name): + app = Mock() + app.charm_name = charm_name + app.units = {} + app.relations = {} + app.app_status.current = "active" + return app + + +def _model_status(apps): + status = Mock() + status.apps = apps + return status + + +@pytest.fixture +def deployment(): + return Mock() + + +@pytest.fixture +def jhelper(deployment): + jhelper = Mock() + deployment.get_juju_helper.return_value = jhelper + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + return jhelper + + +def _leader_only_cluster_status(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + return {"backup-id": f"backup-{unit.replace('/', '-')}"} + + +def _s3_related(app): + relation = Mock() + relation.interface = "s3" + app.relations = {"s3-parameters": [relation]} + return app + + +class TestBackupCommand: + def test_no_applications(self, deployment, jhelper): + jhelper.get_model_status.return_value = _model_status({}) + + result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert "No applications found to back up. Exiting." in result.output + + def test_all_success(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) + vault = _s3_related(_app_status("vault-k8s")) + jhelper.get_model_status.return_value = _model_status( + {"keystone-mysql": mysql, "vault": vault} + ) + jhelper.get_application.return_value = _app_status("mysql-k8s") + jhelper.run_action.side_effect = _leader_only_cluster_status + + result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 0, result.output + assert "2 succeeded, 0 failed" in result.output + + def test_partial_failure(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) + vault = _s3_related(_app_status("vault-k8s")) + jhelper.get_model_status.return_value = _model_status( + {"keystone-mysql": mysql, "vault": vault} + ) + jhelper.get_application.return_value = _app_status("mysql-k8s") + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + if unit == "keystone-mysql/0": + raise ActionFailedException("backup failed") + return {"backup-id": "backup-vault-0"} + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 1, result.output + assert "1 succeeded, 1 failed" in result.output + assert "Warning" in result.output + + def test_all_failed(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + jhelper.get_application.return_value = _app_status("mysql-k8s") + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + raise ActionFailedException("backup failed") + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert "0 succeeded, 1 failed" in result.output + + def test_unrelated_mysql_is_skipped_and_backup_continues(self, deployment, jhelper): + mysql = _app_status("mysql-k8s") # no s3 + vault = _s3_related(_app_status("vault-k8s")) + jhelper.get_model_status.return_value = _model_status( + {"keystone-mysql": mysql, "vault": vault} + ) + jhelper.get_application.return_value = vault + jhelper.run_action.return_value = {"backup-id": "backup-vault-0"} + + result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 0, result.output + assert "is not ready for backup" in result.output + assert "keystone-mysql" in result.output + + def test_declining_confirmation_aborts_backup( + self, deployment, jhelper, monkeypatch + ): + mysql = _app_status("mysql-k8s") # no s3, will be skipped -> prompt + vault = _s3_related(_app_status("vault-k8s")) + jhelper.get_model_status.return_value = _model_status( + {"keystone-mysql": mysql, "vault": vault} + ) + jhelper.get_application.return_value = vault + + monkeypatch.setattr( + "sunbeam.commands.backup_restore.ConfirmQuestion.ask", + lambda self, *a, **k: False, + ) + + result = CliRunner().invoke(backup, obj=deployment) + + assert result.exit_code != 0 + assert "Aborted" in result.output + # No backup action dispatched. + actions = [ + call.args[2] + for call in jhelper.run_action.call_args_list + if len(call.args) > 2 + ] + assert "create-backup" not in actions + + def test_no_supported_apps_left(self, deployment, jhelper): + mysql = _app_status("mysql-k8s") # no s3 + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert "keystone-mysql" in result.output + assert "No applications remain to back up after validation." in result.output + + def test_non_active_target_app_is_skipped(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) + mysql.app_status.current = "blocked" + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + result = CliRunner().invoke(backup, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert "keystone-mysql" in result.output + assert "active" in result.output + assert "No applications remain to back up after validation." in result.output diff --git a/sunbeam-python/tests/unit/sunbeam/commands/test_list_backups.py b/sunbeam-python/tests/unit/sunbeam/commands/test_list_backups.py new file mode 100644 index 000000000..f066b9fd5 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/commands/test_list_backups.py @@ -0,0 +1,185 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +import json +from unittest.mock import Mock + +import pytest +from click.testing import CliRunner + +from sunbeam.commands.backup_restore import list_backups +from sunbeam.core.juju import ActionFailedException, LeaderNotFoundException + + +def _app_status(charm_name): + app = Mock() + app.charm_name = charm_name + app.units = {} + app.relations = {} + app.app_status.current = "active" + return app + + +def _model_status(apps): + status = Mock() + status.apps = apps + return status + + +def _s3_related(app): + relation = Mock() + relation.interface = "s3" + app.relations = {"s3-parameters": [relation]} + return app + + +@pytest.fixture +def deployment(): + return Mock() + + +@pytest.fixture +def jhelper(deployment): + jhelper = Mock() + deployment.get_juju_helper.return_value = jhelper + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + return jhelper + + +class TestListBackupsCommand: + def test_no_applications(self, deployment, jhelper): + jhelper.get_model_status.return_value = _model_status({}) + + result = CliRunner().invoke(list_backups, obj=deployment) + + assert result.exit_code == 2, result.output + assert "No applications found" in result.output + + def test_no_supported_apps_left(self, deployment, jhelper): + mysql = _app_status("mysql-k8s") # no s3 + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + result = CliRunner().invoke(list_backups, obj=deployment) + + assert result.exit_code == 2, result.output + assert "keystone-mysql" in result.output + assert "No applications found to list backups from. Exiting." in result.output + + def test_lists_backups_and_writes_manifest(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) + vault = _s3_related(_app_status("vault-k8s")) + jhelper.get_model_status.return_value = _model_status( + {"keystone-mysql": mysql, "vault": vault} + ) + + def _run_action(unit, model, action, params=None, timeout=None): + if unit == "keystone-mysql/0": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + return { + "backup-ids": json.dumps(["vault-backup-openstack-2026-07-15-00-03-28"]) + } + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(list_backups, obj=deployment) + + assert result.exit_code == 0, result.output + assert "keystone-mysql" in result.output + assert "vault" in result.output + assert "Backup inventory manifest written to" in result.output + + def test_lists_from_leader_only(self, deployment, jhelper): + """list-backups resolves leaders; no cluster-status is queried.""" + mysql = _s3_related(_app_status("mysql-k8s")) + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + def _run_action(unit, model, action, params=None, timeout=None): + assert action != "get-cluster-status" + assert unit == "keystone-mysql/0" + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(list_backups, obj=deployment) + + assert result.exit_code == 0, result.output + actions = [call.args[2] for call in jhelper.run_action.call_args_list] + assert "get-cluster-status" not in actions + + def test_non_active_target_app_is_skipped(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) + mysql.app_status.current = "waiting" + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + result = CliRunner().invoke(list_backups, obj=deployment) + + assert result.exit_code == 2, result.output + assert "keystone-mysql" in result.output + assert "waiting" in result.output or "active" in result.output + + def test_list_action_failures_exit_2_with_details(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) + vault = _s3_related(_app_status("vault-k8s")) + jhelper.get_model_status.return_value = _model_status( + {"keystone-mysql": mysql, "vault": vault} + ) + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "list-backups": + raise ActionFailedException("list failed") + return {} + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(list_backups, obj=deployment) + + assert result.exit_code == 2, result.output + assert "Failed to list backups for" in result.output + assert "keystone-mysql" in result.output + assert "vault" in result.output + + def test_unresolved_targets_exit_2_with_warning(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) + vault = _s3_related(_app_status("vault-k8s")) + jhelper.get_model_status.return_value = _model_status( + {"keystone-mysql": mysql, "vault": vault} + ) + + def _leader(app, model): + if app == "vault": + raise LeaderNotFoundException("no leader") + return f"{app}/0" + + def _run_action(unit, model, action, params=None, timeout=None): + if unit == "keystone-mysql/0": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + return {} + + jhelper.get_leader_unit.side_effect = _leader + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(list_backups, obj=deployment) + + assert result.exit_code == 2, result.output + assert ( + "Failed to list backups for vault: Could not resolve target." + in result.output + ) diff --git a/sunbeam-python/tests/unit/sunbeam/commands/test_restore.py b/sunbeam-python/tests/unit/sunbeam/commands/test_restore.py new file mode 100644 index 000000000..f254c0918 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/commands/test_restore.py @@ -0,0 +1,556 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +import json +from unittest.mock import Mock + +import pytest +from click.testing import CliRunner + +from sunbeam.commands.backup_restore import restore +from sunbeam.core.juju import ( + ActionFailedException, + JujuException, + LeaderNotFoundException, +) +from sunbeam.core.openstack import OPENSTACK_MODEL + + +def _app_status(charm_name): + app = Mock() + app.charm_name = charm_name + app.units = {} + app.relations = {} + app.app_status.current = "active" + return app + + +def _model_status(apps): + apps = dict(apps) + for app_name, app_status in list(apps.items()): + if app_status.charm_name != "mysql-k8s" or not app_name.endswith("-mysql"): + continue + api_app = app_name.removesuffix("-mysql") + router_app = f"{api_app}-mysql-router" + app_status.relations["database"] = [ + Mock(interface="mysql_client", related_app=router_app) + ] + apps.setdefault( + router_app, + Mock( + charm_name="mysql-router-k8s", + relations={ + "database": [ + Mock(interface="mysql_client", related_app=app_name), + Mock(interface="mysql_client", related_app=api_app), + ] + }, + ), + ) + status = Mock() + status.apps = apps + return status + + +def _s3_related(app): + relation = Mock() + relation.interface = "s3" + app.relations = {"s3-parameters": [relation]} + return app + + +def _default_run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + if action == "list-backups" and unit.startswith("keystone-mysql"): + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + if action == "list-backups" and unit.startswith("vault"): + return { + "backup-ids": json.dumps(["vault-backup-openstack-2026-07-15-00-03-28"]) + } + return {} + + +@pytest.fixture +def deployment(): + return Mock() + + +@pytest.fixture +def jhelper(deployment): + jhelper = Mock() + deployment.get_juju_helper.return_value = jhelper + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + mysql = _s3_related(_app_status("mysql-k8s")) + vault = _s3_related(_app_status("vault-k8s")) + jhelper.get_model_status.return_value = _model_status( + { + "keystone-mysql": mysql, + "vault": vault, + "keystone-k8s": _app_status("keystone-k8s"), + } + ) + jhelper.get_application.return_value = _app_status("mysql-k8s") + jhelper.get_application_actions.return_value = ["pause", "resume"] + jhelper.run_action.side_effect = _default_run_action + return jhelper + + +class TestRestoreCommand: + def test_stops_at_pause_guard_and_is_non_destructive(self, deployment, jhelper): + jhelper.get_application_actions.return_value = [] + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 1, result.output + assert "pause/resume" in result.output + jhelper.scale_application.assert_not_called() + + def test_prechecks_pause_resume_for_all_apps_before_any_restore_work( + self, deployment, jhelper + ): + mysql_a = _s3_related(_app_status("mysql-k8s")) + mysql_b = _s3_related(_app_status("mysql-k8s")) + jhelper.get_model_status.return_value = _model_status( + {"keystone-mysql": mysql_a, "nova-mysql": mysql_b} + ) + + def _get_actions(app, model): + if app == "nova": + return [] + return ["pause", "resume"] + + jhelper.get_application_actions.side_effect = _get_actions + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + if action == "list-backups": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + return {} + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 1, result.output + assert "pause/resume" in result.output + assert "nova" in result.output + jhelper.scale_application.assert_not_called() + + restore_actions = { + call.args[2] + for call in jhelper.run_action.call_args_list + if len(call.args) > 2 + } + assert "pause" not in restore_actions + assert "resume" not in restore_actions + assert "restore" not in restore_actions + + def test_invalid_restore_to_time_fails_fast(self, deployment, jhelper): + result = CliRunner().invoke( + restore, ["--restore-to-time", "not-a-date"], obj=deployment + ) + + assert result.exit_code != 0 + assert "YYYY-MM-DD HH:MM:SS" in result.output + jhelper.get_model_status.assert_not_called() + jhelper.scale_application.assert_not_called() + + def test_unrelated_mysql_is_skipped_and_restore_continues( + self, deployment, jhelper + ): + mysql = _app_status("mysql-k8s") # no s3 + vault = _s3_related(_app_status("vault-k8s")) + jhelper.get_model_status.return_value = _model_status( + { + "keystone-mysql": mysql, + "vault": vault, + "keystone-k8s": _app_status("keystone-k8s"), + } + ) + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 0, result.output + assert "is not ready for backup" in result.output + jhelper.scale_application.assert_not_called() + + def test_no_supported_apps_left(self, deployment, jhelper): + mysql = _app_status("mysql-k8s") # no s3 + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert ( + "No applications remain to restore after validation. Exiting." + in result.output + ) + + def test_unrelated_vault_is_skipped_and_restore_continues( + self, deployment, jhelper + ): + mysql = _s3_related(_app_status("mysql-k8s")) + vault = _app_status("vault-k8s") # no s3 + jhelper.get_model_status.return_value = _model_status( + { + "keystone-mysql": mysql, + "vault": vault, + "keystone-k8s": _app_status("keystone-k8s"), + } + ) + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 0, result.output + assert "vault is not ready for backup" in result.output + + def test_pitr_falls_back_to_latest_for_components_without_pitr_support( + self, deployment, jhelper + ): + result = CliRunner().invoke( + restore, + ["--restore-to-time", "2026-07-15 00:00:00", "--no-prompt"], + obj=deployment, + ) + + assert result.exit_code == 0, result.output + assert "vault does not support --restore-to-time." in result.output + assert "Restoring latest available" in result.output + assert "backup instead." in result.output + restore_calls = [ + call + for call in jhelper.run_action.call_args_list + if len(call.args) > 2 and call.args[2] == "restore-backup" + ] + assert len(restore_calls) == 1 + assert restore_calls[0].args[3] == { + "backup-id": "vault-backup-openstack-2026-07-15-00-03-28" + } + + def test_no_backups_found(self, deployment, jhelper): + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + if action == "list-backups" and unit.startswith("keystone-mysql"): + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | failed" + ) + } + if action == "list-backups" and unit.startswith("vault"): + return {"backup-ids": json.dumps([])} + return {} + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert ( + "No applications remain to restore after validation. Exiting." + in result.output + ) + + def test_warns_when_some_backups_failed_but_restore_continues( + self, deployment, jhelper + ): + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + if action == "list-backups" and unit.startswith("keystone-mysql"): + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | failed\n" + "2026-07-14T00:00:00Z | physical | finished" + ) + } + if action == "list-backups" and unit.startswith("vault"): + return { + "backup-ids": json.dumps( + ["vault-backup-openstack-2026-07-15-00-03-28"] + ) + } + return {} + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 0, result.output + assert "Some backups for keystone-mysql failed." in result.output + assert "Only successful backups will be" in result.output + assert "considered for restore" in result.output + + def test_partial_backup_failures_prompt_and_decline_aborts( + self, deployment, jhelper, monkeypatch + ): + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + if action == "list-backups" and unit.startswith("keystone-mysql"): + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | failed\n" + "2026-07-14T00:00:00Z | physical | finished" + ) + } + if action == "list-backups" and unit.startswith("vault"): + return { + "backup-ids": json.dumps( + ["vault-backup-openstack-2026-07-15-00-03-28"] + ) + } + return {} + + jhelper.run_action.side_effect = _run_action + + monkeypatch.setattr( + "sunbeam.commands.backup_restore.ConfirmQuestion.ask", + lambda self, *a, **k: False, + ) + + result = CliRunner().invoke(restore, obj=deployment) + + assert result.exit_code == 1, result.output + assert "Aborted" in result.output + + def test_inventory_lookup_failures_are_reported_and_exit_2( + self, deployment, jhelper + ): + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + if action == "list-backups": + raise ActionFailedException("list failed") + return {} + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert "Failed to list backups for" in result.output + + def test_force_proceeds_when_target_app_is_inactive(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) + mysql.app_status.current = "blocked" + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + if action == "list-backups": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + return {} + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(restore, ["--force", "--no-prompt"], obj=deployment) + + assert result.exit_code == 0, result.output + restore_calls = [ + call + for call in jhelper.run_action.call_args_list + if call.args[2] == "restore" + ] + assert restore_calls + + def test_force_does_not_bypass_inventory_target_resolution_failure( + self, deployment, jhelper + ): + mysql = _s3_related(_app_status("mysql-k8s")) + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + leader_calls = {"keystone-mysql": 0} + + def _leader(app, model): + if app != "keystone-mysql": + return f"{app}/0" + leader_calls[app] += 1 + if leader_calls[app] == 1: + raise LeaderNotFoundException("temporary leader lookup failure") + return "keystone-mysql/0" + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + if action == "list-backups": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + return {} + + jhelper.get_leader_unit.side_effect = _leader + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(restore, ["--force", "--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert "Could not resolve restore target for keystone-mysql" in result.output + assert not any( + call.args[2] == "restore" for call in jhelper.run_action.call_args_list + ) + + def test_mixed_unresolved_target_stops_restore(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) + vault = _s3_related(_app_status("vault-k8s")) + jhelper.get_model_status.return_value = _model_status( + {"keystone-mysql": mysql, "vault": vault} + ) + + def _leader(app, model): + if app == "vault": + raise LeaderNotFoundException("no vault leader") + return f"{app}/0" + + jhelper.get_leader_unit.side_effect = _leader + jhelper.run_action.side_effect = _default_run_action + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert "Could not resolve restore target for vault" in result.output + jhelper.scale_application.assert_not_called() + assert not any( + call.args[2] in {"restore", "restore-backup"} + for call in jhelper.run_action.call_args_list + ) + + def test_force_does_not_bypass_missing_pause_resume_actions( + self, deployment, jhelper + ): + jhelper.get_application_actions.return_value = [] + + result = CliRunner().invoke(restore, ["--force", "--no-prompt"], obj=deployment) + + assert result.exit_code == 1, result.output + assert "pause/resume" in result.output + jhelper.scale_application.assert_not_called() + + def test_non_active_target_app_is_skipped(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) + mysql.app_status.current = "error" + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert "keystone-mysql" in result.output + + def test_mysql_restore_failure_reverts_and_reports(self, deployment, jhelper): + mysql = _s3_related(_app_status("mysql-k8s")) + mysql.units = {"keystone-mysql/0": Mock(), "keystone-mysql/1": Mock()} + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + jhelper.get_application.return_value = mysql + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + if action == "list-backups": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + if action == "restore": + raise JujuException("restore failed") + return {} + + jhelper.run_action.side_effect = _run_action + + result = CliRunner().invoke(restore, ["--no-prompt"], obj=deployment) + + assert result.exit_code == 2, result.output + assert "keystone-mysql" in result.output + assert "restore failed" in result.output + assert [call.args for call in jhelper.scale_application.call_args_list] == [ + (OPENSTACK_MODEL, "keystone-mysql-router", 0), + (OPENSTACK_MODEL, "keystone-mysql", 1), + (OPENSTACK_MODEL, "keystone-mysql", 2), + (OPENSTACK_MODEL, "keystone-mysql-router", 2), + ] diff --git a/sunbeam-python/tests/unit/sunbeam/features/test_disaster_recovery_feature.py b/sunbeam-python/tests/unit/sunbeam/features/test_disaster_recovery_feature.py new file mode 100644 index 000000000..11d9cd9ff --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/features/test_disaster_recovery_feature.py @@ -0,0 +1,506 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +from unittest.mock import Mock, patch + +import click +import pytest + +from sunbeam.core.manifest import FeatureConfig, TerraformManifest +from sunbeam.features.disaster_recovery.feature import ( + DisasterRecoveryFeature, + DisasterRecoveryFeatureConfig, + S3Integration, +) +from sunbeam.features.interface.v1.openstack import ( + DatabaseTopology, + TerraformPlanLocation, +) +from sunbeam.steps.backup_restore import S3_ENDPOINT + + +class TestDisasterRecoveryFeature: + @pytest.mark.parametrize( + ("path", "expected_path"), + [ + ("/", "/mysql"), + ("/backups", "/backups/mysql"), + ("backups", "/backups/mysql"), + ], + ) + def test_s3_integrator_config_normalizes_mysql_paths(self, path, expected_path): + feature = DisasterRecoveryFeature() + config = DisasterRecoveryFeatureConfig( + configure_managed_s3_integrators=True, + path=path, + ) + integrations = [ + S3Integration( + app_name="mysql", + integrator_app="mysql-s3-integrator", + target_endpoint=S3_ENDPOINT, + ) + ] + + tfvar = feature._s3_integrator_config_tfvar(config, integrations) + + assert tfvar["mysql-s3-integrator"]["path"] == expected_path + + def test_feature_metadata(self): + feature = DisasterRecoveryFeature() + + assert feature.name == "disaster-recovery" + assert feature.generally_available is False + assert feature.tf_plan_location == TerraformPlanLocation.FEATURE_REPO + + def test_set_application_names_is_per_target_app(self): + feature = DisasterRecoveryFeature() + deployment = Mock() + deployment.get_client.return_value.cluster.get_config.return_value = ( + '{"database": "multi"}' + ) + jhelper = deployment.get_juju_helper.return_value + status = Mock() + status.apps = { + "keystone-mysql": Mock(charm_name="mysql-k8s"), + "vault": Mock(charm_name="vault-k8s"), + "nova": Mock(charm_name="nova-k8s"), + } + jhelper.get_model_status.return_value = status + jhelper.get_relation_map.return_value = {} + + assert feature.set_application_names(deployment) == [ + "keystone-s3-integrator", + "vault-s3-integrator", + ] + + def test_set_application_names_skips_existing_s3_relation(self): + feature = DisasterRecoveryFeature() + deployment = Mock() + deployment.get_client.return_value.cluster.get_config.return_value = ( + '{"database": "multi"}' + ) + jhelper = deployment.get_juju_helper.return_value + status = Mock() + status.apps = { + "keystone-mysql": Mock(charm_name="mysql-k8s"), + "vault": Mock(charm_name="vault-k8s"), + } + jhelper.get_model_status.return_value = status + jhelper.get_relation_map.side_effect = [{"3": "legacy-dr-integrator"}, {}] + + assert feature.set_application_names(deployment) == ["vault-s3-integrator"] + + def test_single_database_topology_uses_only_shared_mysql(self): + feature = DisasterRecoveryFeature() + deployment = Mock() + deployment.get_client.return_value.cluster.get_config.return_value = "{}" + jhelper = deployment.get_juju_helper.return_value + status = Mock() + status.apps = { + "mysql": Mock(charm_name="mysql-k8s"), + "keystone-mysql": Mock(charm_name="mysql-k8s"), + "vault": Mock(charm_name="vault-k8s"), + } + jhelper.get_model_status.return_value = status + jhelper.get_relation_map.return_value = {} + feature.get_database_topology = Mock(return_value=DatabaseTopology.SINGLE) + + assert feature.set_application_names(deployment) == [ + "mysql-s3-integrator", + "vault-s3-integrator", + ] + + def test_multi_database_topology_uses_only_service_mysql(self): + feature = DisasterRecoveryFeature() + deployment = Mock() + deployment.get_client.return_value.cluster.get_config.return_value = "{}" + jhelper = deployment.get_juju_helper.return_value + status = Mock() + status.apps = { + "mysql": Mock(charm_name="mysql-k8s"), + "keystone-mysql": Mock(charm_name="mysql-k8s"), + "vault": Mock(charm_name="vault-k8s"), + } + jhelper.get_model_status.return_value = status + jhelper.get_relation_map.return_value = {} + feature.get_database_topology = Mock(return_value=DatabaseTopology.MULTI) + + assert feature.set_application_names(deployment) == [ + "keystone-s3-integrator", + "vault-s3-integrator", + ] + + def test_enable_disable_tfvars(self): + feature = DisasterRecoveryFeature() + deployment = Mock() + deployment.get_client.return_value.cluster.get_config.return_value = ( + '{"database": "multi"}' + ) + jhelper = deployment.get_juju_helper.return_value + jhelper.get_model_uuid.return_value = "openstack-uuid" + status = Mock() + status.apps = { + "keystone-mysql": Mock(charm_name="mysql-k8s"), + "vault": Mock(charm_name="vault-k8s"), + } + jhelper.get_model_status.return_value = status + jhelper.get_relation_map.return_value = {} + + assert feature.set_tfvars_on_enable(deployment, FeatureConfig()) == { + "enable-disaster-recovery": True, + "openstack-model-uuid": "openstack-uuid", + "s3-integrator-config": {}, + "s3-integrator-secret-data": {}, + "s3-integrator-apps": [ + "keystone-s3-integrator", + "vault-s3-integrator", + ], + "s3-integrations": { + "keystone-mysql": { + "integrator_app": "keystone-s3-integrator", + "target_endpoint": S3_ENDPOINT, + }, + "vault": { + "integrator_app": "vault-s3-integrator", + "target_endpoint": S3_ENDPOINT, + }, + }, + } + assert feature.set_tfvars_on_disable(deployment) == { + "enable-disaster-recovery": False, + "openstack-model-uuid": "openstack-uuid", + "s3-integrator-config": {}, + "s3-integrator-secret-data": {}, + "s3-integrator-apps": [ + "keystone-s3-integrator", + "vault-s3-integrator", + ], + "s3-integrations": { + "keystone-mysql": { + "integrator_app": "keystone-s3-integrator", + "target_endpoint": S3_ENDPOINT, + }, + "vault": { + "integrator_app": "vault-s3-integrator", + "target_endpoint": S3_ENDPOINT, + }, + }, + } + + def test_set_tfvars_on_enable_includes_per_app_s3_config(self): + feature = DisasterRecoveryFeature() + deployment = Mock() + deployment.get_client.return_value.cluster.get_config.return_value = ( + '{"database": "multi"}' + ) + jhelper = deployment.get_juju_helper.return_value + jhelper.get_model_uuid.return_value = "openstack-uuid" + status = Mock() + status.apps = { + "keystone-mysql": Mock(charm_name="mysql-k8s"), + "vault": Mock(charm_name="vault-k8s"), + } + jhelper.get_model_status.return_value = status + jhelper.get_relation_map.return_value = {} + + config = DisasterRecoveryFeatureConfig( + configure_managed_s3_integrators=True, + bucket="openstack-backups", + path="backups", + region="us-east-2", + endpoint="https://s3.us-east-2.amazonaws.com", + access_key="AKIA...", + secret_key="secret", + ) + + tfvars = feature.set_tfvars_on_enable(deployment, config) + + assert tfvars["s3-integrator-config"] == { + "keystone-s3-integrator": { + "bucket": "openstack-backups", + "path": "/backups/keystone-mysql", + "region": "us-east-2", + "endpoint": "https://s3.us-east-2.amazonaws.com", + }, + "vault-s3-integrator": { + "bucket": "openstack-backups", + "path": "/backups/vault", + "region": "us-east-2", + "endpoint": "https://s3.us-east-2.amazonaws.com", + }, + } + assert tfvars["s3-integrator-secret-data"] == { + "keystone-s3-integrator": { + "access-key": "AKIA...", + "secret-key": "secret", + }, + "vault-s3-integrator": { + "access-key": "AKIA...", + "secret-key": "secret", + }, + } + + def test_prompt_s3_configuration_without_targets_skips_prompt(self): + feature = DisasterRecoveryFeature() + deployment = Mock() + feature._s3_integrations = Mock(return_value=[]) + feature._ask_prompt = Mock() + feature._ask_password = Mock() + config = feature._prompt_s3_configuration(deployment, show_hints=False) + + feature._ask_prompt.assert_not_called() + feature._ask_password.assert_not_called() + assert isinstance(config, DisasterRecoveryFeatureConfig) + assert config.configure_s3_integrators is False + + def test_prompt_s3_configuration_accepted_returns_config(self): + feature = DisasterRecoveryFeature() + deployment = Mock() + feature._s3_integrations = Mock( + return_value=[ + S3Integration( + app_name="keystone-mysql", + integrator_app="keystone-s3-integrator", + target_endpoint=S3_ENDPOINT, + ) + ] + ) + feature._ask_prompt = Mock( + side_effect=[ + "openstack-backups", + "backups", + "us-east-2", + "https://s3.us-east-2.amazonaws.com", + ] + ) + feature._ask_password = Mock(return_value="secret") + feature._validate_s3_config = Mock() + feature.enable_feature = Mock() + + from sunbeam.features.disaster_recovery import feature as dr_feature + + confirm_mock = Mock() + confirm_mock.ask.return_value = True + with patch.object(dr_feature, "ConfirmQuestion", return_value=confirm_mock): + config = feature._prompt_s3_configuration(deployment, show_hints=True) + + assert isinstance(config, DisasterRecoveryFeatureConfig) + assert config.configure_s3_integrators is True + assert config.bucket == "openstack-backups" + assert config.path == "backups" + assert config.region == "us-east-2" + assert config.endpoint == "https://s3.us-east-2.amazonaws.com" + assert config.access_key == "secret" + assert config.secret_key == "secret" + + def test_validate_prompted_s3_config_requires_bucket_access_secret(self): + feature = DisasterRecoveryFeature() + + bad_bucket = DisasterRecoveryFeatureConfig( + configure_s3_integrators=True, + bucket="", + access_key="AKIA", + secret_key="secret", + endpoint="https://s3.us-east-2.amazonaws.com", + ) + with pytest.raises(click.ClickException): + feature._validate_s3_config(bad_bucket) + + bad_access = DisasterRecoveryFeatureConfig( + configure_s3_integrators=True, + bucket="bucket", + access_key="", + secret_key="secret", + endpoint="https://s3.us-east-2.amazonaws.com", + ) + with pytest.raises(click.ClickException): + feature._validate_s3_config(bad_access) + + bad_secret = DisasterRecoveryFeatureConfig( + configure_s3_integrators=True, + bucket="bucket", + access_key="AKIA", + secret_key="", + endpoint="https://s3.us-east-2.amazonaws.com", + ) + with pytest.raises(click.ClickException): + feature._validate_s3_config(bad_secret) + + def test_validate_prompted_s3_config_rejects_bad_endpoint(self): + feature = DisasterRecoveryFeature() + config = DisasterRecoveryFeatureConfig( + configure_s3_integrators=True, + bucket="bucket", + access_key="AKIA", + secret_key="secret", + endpoint="s3.us-east-2.amazonaws.com", + ) + + with pytest.raises(click.ClickException): + feature._validate_s3_config(config) + + def test_integrator_app_name_uses_service_prefix_for_mysql(self): + feature = DisasterRecoveryFeature() + + assert ( + feature._s3_integrator_app_name("keystone-mysql") + == "keystone-s3-integrator" + ) + assert feature._s3_integrator_app_name("vault") == "vault-s3-integrator" + + def test_default_software_overrides_includes_terraform_plan(self): + feature = DisasterRecoveryFeature() + + software = feature.default_software_overrides() + + assert feature.tfplan in software.terraform + assert isinstance(software.terraform[feature.tfplan], TerraformManifest) + assert ( + software.terraform[feature.tfplan].source.name == "deploy-disaster-recovery" + ) + + def test_set_application_status_overlay_on_enable_accepts_blocked(self): + feature = DisasterRecoveryFeature() + deployment = Mock() + feature.set_application_names = Mock(return_value=["keystone-s3-integrator"]) + + overlay = feature.get_app_status_overlay_on_enable(deployment) + + assert overlay == {"keystone-s3-integrator": {"status": ["active", "blocked"]}} + + def test_manifest_attributes_tfvar_map_includes_s3_integrator(self): + feature = DisasterRecoveryFeature() + + tfvar_map = feature.manifest_attributes_tfvar_map() + + assert feature.tfplan in tfvar_map + charm_map = tfvar_map[feature.tfplan]["charms"] + assert "s3-integrator" in charm_map + assert charm_map["s3-integrator"]["channel"] == "s3-integrator-channel" + assert charm_map["s3-integrator"]["revision"] == "s3-integrator-revision" + assert charm_map["s3-integrator"]["config"] == "s3-integrator-config" + + def test_discover_relation_targets_mysql_and_vault_only(self): + feature = DisasterRecoveryFeature() + apps = { + "keystone-mysql": Mock(charm_name="mysql-k8s"), + "vault": Mock(charm_name="vault-k8s"), + "nova": Mock(charm_name="nova-k8s"), + "glance": Mock(charm_name="glance-k8s"), + } + + assert feature._s3_discover_relation_targets(apps) == [ + "keystone-mysql", + "vault", + ] + + def test_discover_relation_targets_ignores_missing_charm_name(self): + feature = DisasterRecoveryFeature() + apps = { + "mystery-app": Mock(), + "vault": Mock(charm_name="vault-k8s"), + } + + assert feature._s3_discover_relation_targets(apps) == ["vault"] + + def test_discover_relation_targets_uses_components_with_s3_validation(self): + feature = DisasterRecoveryFeature() + apps = { + "mysql": Mock(charm_name="mysql-k8s"), + "vault": Mock(charm_name="vault-k8s"), + "random-app": Mock(charm_name="random-charm"), + } + + assert feature._s3_discover_relation_targets(apps) == ["mysql", "vault"] + + def test_build_s3_integrations_map(self): + feature = DisasterRecoveryFeature() + jhelper = Mock() + jhelper.get_relation_map.return_value = {} + apps = { + "keystone-mysql": Mock(charm_name="mysql-k8s"), + "vault": Mock(charm_name="vault-k8s"), + "nova": Mock(charm_name="nova-k8s"), + } + + assert feature._s3_build_integrations(jhelper, apps) == [ + S3Integration( + app_name="keystone-mysql", + integrator_app="keystone-s3-integrator", + target_endpoint=S3_ENDPOINT, + ), + S3Integration( + app_name="vault", + integrator_app="vault-s3-integrator", + target_endpoint=S3_ENDPOINT, + ), + ] + + def test_build_s3_integrations_skips_app_with_non_dr_s3_relation(self): + feature = DisasterRecoveryFeature() + apps = { + "keystone-mysql": Mock(charm_name="mysql-k8s"), + } + jhelper = Mock() + jhelper.get_relation_map.return_value = {"3": "legacy-dr-integrator"} + + assert feature._s3_build_integrations(jhelper, apps) == [] + + def test_build_s3_integrations_skips_preexisting_unmanaged_integrator(self): + feature = DisasterRecoveryFeature() + apps = { + "keystone-mysql": Mock(charm_name="mysql-k8s"), + "keystone-s3-integrator": Mock(charm_name="s3-integrator"), + } + jhelper = Mock() + jhelper.get_relation_map.return_value = {} + + assert feature._s3_build_integrations(jhelper, apps) == [] + + def test_build_s3_integrations_keeps_dr_owned_relation(self): + feature = DisasterRecoveryFeature() + apps = { + "keystone-mysql": Mock(charm_name="mysql-k8s"), + "keystone-s3-integrator": Mock(charm_name="s3-integrator"), + } + jhelper = Mock() + jhelper.get_relation_map.return_value = {"3": "keystone-s3-integrator"} + managed = {"keystone-mysql": "keystone-s3-integrator"} + + assert feature._s3_build_integrations(jhelper, apps, managed) == [ + S3Integration( + app_name="keystone-mysql", + integrator_app="keystone-s3-integrator", + target_endpoint=S3_ENDPOINT, + ) + ] + + def test_build_s3_integrations_skips_unmanaged_relation_matching_name(self): + feature = DisasterRecoveryFeature() + apps = { + "keystone-mysql": Mock(charm_name="mysql-k8s"), + "keystone-s3-integrator": Mock(charm_name="s3-integrator"), + } + jhelper = Mock() + jhelper.get_relation_map.return_value = {"3": "keystone-s3-integrator"} + + # No managed map: pre-existing integrator with the conventional name + # must not be claimed as DR-owned. + assert feature._s3_build_integrations(jhelper, apps) == [] + + def test_build_s3_integrations_without_relations_does_not_skip(self): + feature = DisasterRecoveryFeature() + apps = { + "keystone-mysql": Mock(charm_name="mysql-k8s"), + } + jhelper = Mock() + jhelper.get_relation_map.return_value = {} + + assert feature._s3_build_integrations(jhelper, apps) == [ + S3Integration( + app_name="keystone-mysql", + integrator_app="keystone-s3-integrator", + target_endpoint=S3_ENDPOINT, + ) + ] diff --git a/sunbeam-python/tests/unit/sunbeam/steps/test_backup.py b/sunbeam-python/tests/unit/sunbeam/steps/test_backup.py new file mode 100644 index 000000000..db9bec7ad --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/steps/test_backup.py @@ -0,0 +1,682 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +import json +from unittest.mock import Mock + +import pytest + +from sunbeam.core.common import ResultType +from sunbeam.core.juju import ( + ActionFailedException, + ApplicationNotFoundException, + JujuException, + LeaderNotFoundException, + ModelNotFoundException, +) +from sunbeam.steps.backup_restore import ( + BACKUP_COMPONENTS, + MYSQL_CHARM, + S3_ENDPOINT, + S3_INTERFACE, + VAULT_CHARM, + ActionTarget, + BackupComponent, + BackupInventory, + BackupOutcome, + BackupResult, + DiscoverBackupApplicationsStep, + ListBackupsStep, + MySQLBackupComponent, + ResolveActionTargetsStep, + RunBackupStep, + ValidateStep, + VaultBackupComponent, + WriteBackupInventoryManifestStep, + WriteBackupManifestStep, + _BackupAppStep, + _component_for, +) + + +def _app_status(charm_name, units=None, relations=None): + app = Mock() + app.charm_name = charm_name + app.units = units or {} + app.relations = relations or {} + app.app_status.current = "active" + return app + + +def _model_status(apps): + status = Mock() + status.apps = apps + return status + + +def _cluster_status(secondary_ordinal): + topology = { + "mysql-0.mysql-endpoints": {"memberrole": "PRIMARY"}, + f"mysql-{secondary_ordinal}.mysql-endpoints": {"memberrole": "SECONDARY"}, + } + return {"status": {"defaultreplicaset": {"topology": topology}}} + + +class TestBackupResult: + def test_construction_with_error_does_not_raise(self): + result = BackupResult( + app="keystone-mysql", + unit="keystone-mysql/0", + component=MYSQL_CHARM, + error="boom", + ) + assert result.error == "boom" + assert result.backup is None + + +class TestCurrentScale: + def test_raises_on_read_failure(self): + jhelper = Mock() + jhelper.get_application.side_effect = ApplicationNotFoundException("missing") + + with pytest.raises(JujuException): + MySQLBackupComponent._current_scale(jhelper, "keystone-mysql", "openstack") + + +class TestRegistry: + def test_registry_contains_mysql_and_vault(self): + names = {c.name for c in BACKUP_COMPONENTS} + assert names == {MYSQL_CHARM, VAULT_CHARM} + + def test_components_have_restore_plans(self): + for component in BACKUP_COMPONENTS: + assert component.build_restore_plan is not None + + def test_registry_contains_explicit_component_types(self): + assert isinstance(_component_for(MYSQL_CHARM), MySQLBackupComponent) + assert isinstance(_component_for(VAULT_CHARM), VaultBackupComponent) + + def test_component_pitr_contracts_are_explicit(self): + assert MySQLBackupComponent().restore_to_time_param == "restore-to-time" + assert VaultBackupComponent().restore_to_time_param is None + + +class TestResolveMySQLTarget: + def test_picks_secondary_unit(self): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "keystone-mysql/0" + jhelper.run_action.return_value = _cluster_status(1) + app = _app_status( + "mysql-k8s", units={"keystone-mysql/0": Mock(), "keystone-mysql/1": Mock()} + ) + jhelper.get_application.return_value = app + + target = MySQLBackupComponent().resolve_backup_target( + jhelper, "keystone-mysql", "openstack", force=False + ) + + assert target is not None + assert target.unit == "keystone-mysql/1" + assert target.component == MYSQL_CHARM + + def test_falls_back_to_leader_when_no_secondary(self): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "cinder-mysql/0" + jhelper.run_action.return_value = { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + jhelper.get_application.return_value = _app_status( + "mysql-k8s", units={"cinder-mysql/0": Mock()} + ) + + target = MySQLBackupComponent().resolve_backup_target( + jhelper, "cinder-mysql", "openstack", force=False + ) + + assert target is not None + assert target.unit == "cinder-mysql/0" + + def test_skips_on_action_failure_without_force(self): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "keystone-mysql/0" + jhelper.get_application.return_value = _app_status( + "mysql-k8s", units={"keystone-mysql/0": Mock()} + ) + jhelper.run_action.side_effect = ActionFailedException("failed") + + target = MySQLBackupComponent().resolve_backup_target( + jhelper, "keystone-mysql", "openstack", force=False + ) + + assert target is None + + def test_uses_leader_on_action_failure_with_force(self): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "keystone-mysql/0" + jhelper.get_application.return_value = _app_status( + "mysql-k8s", units={"keystone-mysql/0": Mock()} + ) + jhelper.run_action.side_effect = ActionFailedException("failed") + + target = MySQLBackupComponent().resolve_backup_target( + jhelper, "keystone-mysql", "openstack", force=True + ) + + assert target is not None + assert target.unit == "keystone-mysql/0" + + def test_skips_when_no_leader(self): + jhelper = Mock() + jhelper.get_leader_unit.side_effect = LeaderNotFoundException("no leader") + + target = MySQLBackupComponent().resolve_backup_target( + jhelper, "keystone-mysql", "openstack", force=True + ) + + assert target is None + + +class TestResolveVaultTarget: + def test_resolves_leader(self): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "vault/0" + jhelper.get_application.return_value = _app_status( + "vault-k8s", units={"vault/0": Mock()} + ) + + target = VaultBackupComponent().resolve_backup_target( + jhelper, "vault", "openstack", force=False + ) + + assert target is not None + assert target.unit == "vault/0" + assert target.component == VAULT_CHARM + + def test_skips_when_no_leader(self): + jhelper = Mock() + jhelper.get_leader_unit.side_effect = LeaderNotFoundException("no leader") + + target = VaultBackupComponent().resolve_backup_target( + jhelper, "vault", "openstack", force=False + ) + + assert target is None + + +class TestDiscoverBackupApplicationsStep: + def test_discovers_by_charm_name(self, step_context): + jhelper = Mock() + jhelper.get_model_status.return_value = _model_status( + { + "keystone-mysql": _app_status("mysql-k8s"), + "nova-mysql": _app_status("mysql-k8s"), + "vault": _app_status("vault-k8s"), + "keystone": _app_status("keystone-k8s"), + } + ) + + result = DiscoverBackupApplicationsStep(jhelper).run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert sorted(result.message[MYSQL_CHARM]) == ["keystone-mysql", "nova-mysql"] + assert result.message[VAULT_CHARM] == ["vault"] + + def test_fails_on_model_error(self, step_context): + jhelper = Mock() + jhelper.get_model_status.side_effect = ModelNotFoundException("missing") + + result = DiscoverBackupApplicationsStep(jhelper).run(step_context) + + assert result.result_type == ResultType.FAILED + + def test_non_active_app_is_still_discovered(self, step_context): + """Discovery is state-agnostic; validation filters non-active apps.""" + jhelper = Mock() + mysql = _app_status("mysql-k8s") + mysql.app_status.current = "blocked" + jhelper.get_model_status.return_value = _model_status({"keystone-mysql": mysql}) + + result = DiscoverBackupApplicationsStep(jhelper).run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert result.message[MYSQL_CHARM] == ["keystone-mysql"] + + +class TestResolveActionTargetsStep: + def test_resolves_leaders_and_skips_unresolvable(self, step_context): + jhelper = Mock() + + def _leader(app, model): + if app == "broken-mysql": + raise LeaderNotFoundException("no leader") + return f"{app}/0" + + jhelper.get_leader_unit.side_effect = _leader + + discovered = { + MYSQL_CHARM: ["keystone-mysql", "broken-mysql"], + VAULT_CHARM: ["vault"], + } + result = ResolveActionTargetsStep( + jhelper, + discovered, + action=lambda component: component.restore_action, + ).run(step_context) + + assert result.result_type == ResultType.COMPLETED + apps = {t.app for t in result.message["targets"]} + assert apps == {"keystone-mysql", "vault"} + assert all(t.unit.endswith("/0") for t in result.message["targets"]) + actions = {t.app: t.action for t in result.message["targets"]} + assert actions == { + "keystone-mysql": "restore", + "vault": "restore-backup", + } + assert result.message["unresolved"] == [ + {"app": "broken-mysql", "component": MYSQL_CHARM} + ] + + +class TestValidateStep: + def test_partitions_by_active_and_s3(self, step_context): + jhelper = Mock() + s3 = Mock() + s3.interface = S3_INTERFACE + ready = _app_status("mysql-k8s", relations={S3_ENDPOINT: [s3]}) + no_s3 = _app_status("mysql-k8s") + inactive = _app_status("mysql-k8s", relations={S3_ENDPOINT: [s3]}) + inactive.app_status.current = "blocked" + jhelper.get_model_status.return_value = _model_status( + { + "keystone-mysql": ready, + "nova-mysql": no_s3, + "glance-mysql": inactive, + } + ) + + discovered = {MYSQL_CHARM: ["keystone-mysql", "nova-mysql", "glance-mysql"]} + result = ValidateStep(jhelper, discovered).run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert result.message["valid"][MYSQL_CHARM] == ["keystone-mysql"] + assert result.message["failures"]["nova-mysql"] == ["s3-relation"] + assert result.message["failures"]["glance-mysql"] == ["active"] + + def test_missing_app_fails_all_checks(self, step_context): + jhelper = Mock() + jhelper.get_model_status.return_value = _model_status({}) + + discovered = {MYSQL_CHARM: ["keystone-mysql"]} + result = ValidateStep(jhelper, discovered).run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert result.message["valid"][MYSQL_CHARM] == [] + assert "keystone-mysql" in result.message["failures"] + + +class TestBackupAppStep: + def test_success_records_backup_result(self, step_context): + jhelper = Mock() + jhelper.run_action.return_value = {"backup-id": "id-1"} + component = _component_for(MYSQL_CHARM) + target = ActionTarget( + "keystone-mysql", "keystone-mysql/1", MYSQL_CHARM, "create-backup" + ) + + step = _BackupAppStep(jhelper, component, target) + result = step.run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert step.result is not None + assert step.result.backup is not None + assert step.result.backup.success is True + assert step.result.backup.backup_id == "id-1" + + def test_missing_backup_id_marks_step_failed(self, step_context): + jhelper = Mock() + jhelper.run_action.return_value = {} + component = _component_for(MYSQL_CHARM) + target = ActionTarget( + "keystone-mysql", "keystone-mysql/1", MYSQL_CHARM, "create-backup" + ) + + step = _BackupAppStep(jhelper, component, target) + result = step.run(step_context) + + assert result.result_type == ResultType.FAILED + assert step.result is not None + assert step.result.backup is None + assert step.result.error == "Backup action completed without backup id." + + def test_failed_backup_action_records_error(self, step_context): + jhelper = Mock() + jhelper.run_action.side_effect = ActionFailedException( + "timed out waiting for results from: unit nova-mysql/0" + ) + component = _component_for(MYSQL_CHARM) + target = ActionTarget( + "nova-mysql", "nova-mysql/0", MYSQL_CHARM, "create-backup" + ) + + step = _BackupAppStep(jhelper, component, target) + result = step.run(step_context) + + assert result.result_type == ResultType.FAILED + assert step.result is not None + assert step.result.error is not None + assert step.result.backup is None + + +class TestRunBackupsStep: + def test_resolves_and_aggregates_mixed_results(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + jhelper.get_application.side_effect = lambda app, model: _app_status( + "mysql-k8s", units={f"{app}/0": Mock(), f"{app}/1": Mock()} + ) + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return _cluster_status(1) + if unit == "glance-mysql/1": + raise ActionFailedException("backup failed") + return {"backup-id": f"backup-{unit.replace('/', '-')}"} + + jhelper.run_action.side_effect = _run_action + discovered = { + MYSQL_CHARM: ["keystone-mysql", "glance-mysql"], + VAULT_CHARM: ["vault"], + } + + result = RunBackupStep(jhelper, discovered).run(step_context) + + assert result.result_type == ResultType.COMPLETED + by_app = {r.app: r for r in result.message} + assert by_app["keystone-mysql"].backup is not None + assert by_app["keystone-mysql"].backup.success is True + assert by_app["glance-mysql"].backup is None + assert by_app["glance-mysql"].error is not None + assert by_app["vault"].backup is not None + assert by_app["vault"].backup.success is True + + def test_force_does_not_inject_action_params(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + jhelper.get_application.side_effect = lambda app, model: _app_status( + "mysql-k8s", units={f"{app}/0": Mock()} + ) + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + return {"backup-id": "id"} + + jhelper.run_action.side_effect = _run_action + discovered = {MYSQL_CHARM: ["keystone-mysql"], VAULT_CHARM: ["vault"]} + + RunBackupStep(jhelper, discovered, force=True).run(step_context) + + for call in jhelper.run_action.call_args_list: + if call.args[2] != "create-backup": + continue + assert len(call.args) == 3 + + def test_failed_backup_action_returns_error(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + jhelper.get_application.side_effect = lambda app, model: _app_status( + "mysql-k8s", units={f"{app}/0": Mock()} + ) + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + return { + "status": { + "defaultreplicaset": { + "topology": {"mysql-0": {"memberrole": "PRIMARY"}} + } + } + } + raise ActionFailedException( + "timed out waiting for results from: unit nova-mysql/0" + ) + + jhelper.run_action.side_effect = _run_action + discovered = {MYSQL_CHARM: ["nova-mysql"]} + + result = RunBackupStep(jhelper, discovered).run(step_context) + + assert result.result_type == ResultType.COMPLETED + backup_result = result.message[0] + assert backup_result.error is not None + assert backup_result.backup is None + + def test_resolve_target_failure_is_recorded(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + jhelper.get_application.side_effect = lambda app, model: _app_status( + "mysql-k8s", units={f"{app}/0": Mock()} + ) + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "get-cluster-status": + raise ActionFailedException("cluster status unavailable") + return {"backup-id": "id"} + + jhelper.run_action.side_effect = _run_action + discovered = {MYSQL_CHARM: ["nova-mysql"]} + + result = RunBackupStep(jhelper, discovered).run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert len(result.message) == 1 + assert result.message[0].app == "nova-mysql" + assert result.message[0].error == "Could not resolve backup target." + + +class TestListBackupsParsing: + def test_parse_mysql_backup_ids_filters_finished_entries(self): + action_result = { + "backups": ( + "backup-id | backup-type | backup-status\n" + "--------------------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished\n" + "2026-07-14T00:00:00Z | physical | failed" + ) + } + + backups = MySQLBackupComponent().parse_backup_list(action_result) + + assert [b.backup_id for b in backups] == [ + "2026-07-15T00:00:00Z", + "2026-07-14T00:00:00Z", + ] + assert [b.success for b in backups] == [True, False] + + def test_parse_vault_backup_ids_json_array(self): + action_result = { + "backup-ids": json.dumps( + [ + "vault-backup-openstack-2026-07-15-00-03-28", + "vault-backup-openstack-2026-07-14-00-03-28", + ] + ) + } + + backups = VaultBackupComponent().parse_backup_list(action_result) + + assert [b.backup_id for b in backups] == [ + "vault-backup-openstack-2026-07-15-00-03-28", + "vault-backup-openstack-2026-07-14-00-03-28", + ] + assert all(b.success for b in backups) + + +class TestListBackupsStep: + def test_collects_backup_ids_by_target(self, step_context): + jhelper = Mock() + + def _run_action(unit, model, action, params=None, timeout=None): + assert action == "list-backups" + if unit == "keystone-mysql/0": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + return { + "backup-ids": json.dumps(["vault-backup-openstack-2026-07-15-00-03-28"]) + } + + jhelper.run_action.side_effect = _run_action + targets = [ + ActionTarget( + "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "list-backups" + ), + ActionTarget("vault", "vault/0", VAULT_CHARM, "list-backups"), + ] + + result = ListBackupsStep(jhelper, targets).run(step_context) + + assert result.result_type == ResultType.COMPLETED + by_app = {r.app: r for r in result.message} + assert by_app["keystone-mysql"].error is None + assert [b.backup_id for b in by_app["keystone-mysql"].backups] == [ + "2026-07-15T00:00:00Z" + ] + assert by_app["vault"].error is None + assert [b.backup_id for b in by_app["vault"].backups] == [ + "vault-backup-openstack-2026-07-15-00-03-28" + ] + + def test_collects_errors_without_raising(self, step_context): + jhelper = Mock() + jhelper.run_action.side_effect = ActionFailedException("boom") + targets = [ + ActionTarget( + "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "list-backups" + ) + ] + + result = ListBackupsStep(jhelper, targets).run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert result.message[0].backups is None + assert result.message[0].error == "boom" + + +class TestWriteBackupManifestStep: + def test_writes_manifest(self, step_context, tmp_path): + results = [ + BackupResult( + "keystone-mysql", + "keystone-mysql/1", + MYSQL_CHARM, + BackupOutcome("id-1", success=True), + ), + BackupResult("glance-mysql", "glance-mysql/1", MYSQL_CHARM, None, "err"), + ] + step = WriteBackupManifestStep( + results, "2026-04-09T14:22:01+00:00", manifest_dir=tmp_path + ) + + result = step.run(step_context) + + assert result.result_type == ResultType.COMPLETED + written = list(tmp_path.glob("backup-manifest-*.yaml")) + assert len(written) == 1 + import yaml + + data = yaml.safe_load(written[0].read_text()) + assert data["summary"] == {"succeeded": 1, "failed": 1} + assert data["dispatched_at"] == "2026-04-09T14:22:01+00:00" + assert {r["app"] for r in data["results"]} == { + "keystone-mysql", + "glance-mysql", + } + + +class TestWriteBackupInventoryManifestStep: + def test_writes_inventory_manifest(self, step_context, tmp_path): + results = [ + BackupInventory( + app="keystone-mysql", + unit="keystone-mysql/1", + component=MYSQL_CHARM, + backups=[BackupOutcome("2026-07-15T00:00:00Z", success=True)], + ), + BackupInventory( + app="vault", + unit="vault/0", + component=VAULT_CHARM, + error="failed", + ), + ] + step = WriteBackupInventoryManifestStep( + results, "2026-07-15T00:04:28+00:00", manifest_dir=tmp_path + ) + + result = step.run(step_context) + + assert result.result_type == ResultType.COMPLETED + written = list(tmp_path.glob("backup-inventory-*.yaml")) + assert len(written) == 1 + + +class TestExtensibility: + """Adding a component requires an explicit workflow subclass.""" + + def test_new_component_flows_through_generic_pipeline( + self, step_context, monkeypatch + ): + class FakeBackupComponent(BackupComponent): + name = "fake-charm" + + def resolve_backup_target(self, jhelper, app, model, force): + return ActionTarget(app, f"{app}/0", self.name, self.backup_action) + + def parse_backup_list(self, action_result): + return [] + + def restore_params(self, jhelper, target, restore_to_time, timeout, model): + return {} + + def build_restore_plan( + self, jhelper, target, restore_to_time, timeout, model + ): + return [] + + fake = FakeBackupComponent() + components = BACKUP_COMPONENTS + [fake] + monkeypatch.setattr( + "sunbeam.steps.backup_restore.BACKUP_COMPONENTS", components + ) + + jhelper = Mock() + jhelper.get_model_status.return_value = _model_status( + {"my-fake": _app_status("fake-charm")} + ) + jhelper.run_action.return_value = {"backup-id": "fake-backup"} + + discover = DiscoverBackupApplicationsStep(jhelper, components=components).run( + step_context + ) + assert discover.message["fake-charm"] == ["my-fake"] + + run = RunBackupStep(jhelper, {"fake-charm": ["my-fake"]}).run(step_context) + assert run.message[0].component == "fake-charm" + assert run.message[0].backup is not None + assert run.message[0].backup.success is True + assert run.message[0].backup.backup_id == "fake-backup" diff --git a/sunbeam-python/tests/unit/sunbeam/steps/test_restore.py b/sunbeam-python/tests/unit/sunbeam/steps/test_restore.py new file mode 100644 index 000000000..ec1e14d4a --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/steps/test_restore.py @@ -0,0 +1,731 @@ +# SPDX-FileCopyrightText: 2026 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +from unittest.mock import Mock + +import pytest + +from sunbeam.core.common import ResultType +from sunbeam.core.juju import ( + ActionFailedException, + ApplicationNotFoundException, + JujuException, + LeaderNotFoundException, +) +from sunbeam.steps.backup_restore import ( + MYSQL_CHARM, + VAULT_CHARM, + ActionTarget, + MySQLBackupComponent, + RestoreStep, + _ActionStep, + _component_for, + _PauseAppStep, + _RestoreAppStep, + _ResumeAppStep, + _ScaleAppStep, +) + + +def _mysql_component(): + return _component_for(MYSQL_CHARM) + + +def _vault_component(): + return _component_for(VAULT_CHARM) + + +class TestGuardedSteps: + def test_action_step_default_runs_on_leader(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "keystone-k8s/0" + + step = _ActionStep( + jhelper, + name="Action", + description="Run action", + app="keystone-k8s", + action_name="pause", + ) + result = step.run(step_context) + + assert result.result_type == ResultType.COMPLETED + jhelper.get_application.assert_not_called() + jhelper.run_action.assert_called_once_with( + "keystone-k8s/0", + "openstack", + "pause", + timeout=120, + ) + + def test_pause_dispatches_action_on_all_units(self, step_context): + jhelper = Mock() + jhelper.get_application.return_value = Mock( + units={"keystone-k8s/0": Mock(), "keystone-k8s/1": Mock()} + ) + + step = _PauseAppStep(jhelper, app="keystone-k8s") + result = step.run(step_context) + + assert result.result_type == ResultType.COMPLETED + jhelper.get_application_actions.assert_not_called() + jhelper.get_leader_unit.assert_not_called() + assert jhelper.run_action.call_count == 2 + assert jhelper.run_action.call_args_list[0].args == ( + "keystone-k8s/0", + "openstack", + "pause", + ) + assert jhelper.run_action.call_args_list[1].args == ( + "keystone-k8s/1", + "openstack", + "pause", + ) + + def test_resume_dispatches_action_on_all_units(self, step_context): + jhelper = Mock() + jhelper.get_application.return_value = Mock( + units={"keystone-k8s/0": Mock(), "keystone-k8s/1": Mock()} + ) + + step = _ResumeAppStep(jhelper, app="keystone-k8s") + result = step.run(step_context) + + assert result.result_type == ResultType.COMPLETED + jhelper.get_leader_unit.assert_not_called() + assert jhelper.run_action.call_count == 2 + assert jhelper.run_action.call_args_list[0].args == ( + "keystone-k8s/0", + "openstack", + "resume", + ) + assert jhelper.run_action.call_args_list[1].args == ( + "keystone-k8s/1", + "openstack", + "resume", + ) + + def test_restore_mysql_uses_latest_backup_id(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "keystone-mysql/0" + jhelper.run_action.side_effect = [ + { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + }, + {}, + ] + target = ActionTarget( + "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "restore" + ) + result = _RestoreAppStep(jhelper, _mysql_component(), target).run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert jhelper.run_action.call_args_list[1].args[2] == "restore" + assert jhelper.run_action.call_args_list[1].args[3] == { + "backup-id": "2026-07-15T00:00:00Z" + } + + def test_restore_mysql_uses_restore_to_time(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "keystone-mysql/0" + component = _mysql_component() + target = ActionTarget( + "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "restore" + ) + + result = _RestoreAppStep( + jhelper, + component, + target, + restore_to_time="2026-07-15 00:00:00", + ).run(step_context) + + assert result.result_type == ResultType.COMPLETED + jhelper.run_action.assert_called_once_with( + "keystone-mysql/0", + "openstack", + "restore", + {"restore-to-time": "2026-07-15 00:00:00"}, + timeout=1800, + ) + + def test_restore_to_time_falls_back_when_component_does_not_support_it( + self, step_context + ): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "vault/0" + jhelper.run_action.side_effect = [ + {"backup-ids": '["vault-backup-openstack-2026-07-15-00-03-28"]'}, + {}, + ] + target = ActionTarget("vault", "vault/0", VAULT_CHARM, "restore-backup") + + result = _RestoreAppStep( + jhelper, + _vault_component(), + target, + restore_to_time="2026-07-15 00:00:00", + ).run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert jhelper.run_action.call_args_list[1].args[2] == "restore-backup" + assert jhelper.run_action.call_args_list[1].args[3] == { + "backup-id": "vault-backup-openstack-2026-07-15-00-03-28" + } + + def test_restore_vault_uses_latest_backup(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "vault/0" + jhelper.run_action.side_effect = [ + {"backup-ids": '["vault-backup-openstack-2026-07-15-00-03-28"]'}, + {}, + ] + target = ActionTarget("vault", "vault/0", VAULT_CHARM, "restore-backup") + + result = _RestoreAppStep(jhelper, _vault_component(), target).run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert jhelper.run_action.call_args_list[1].args[2] == "restore-backup" + assert jhelper.run_action.call_args_list[1].args[3] == { + "backup-id": "vault-backup-openstack-2026-07-15-00-03-28" + } + + def test_mysql_restore_does_not_retry_action_failure(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "keystone-mysql/0" + jhelper.run_action.side_effect = [ + { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + }, + ActionFailedException("transient restore failure"), + {}, + ] + target = ActionTarget( + "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "restore" + ) + + result = _RestoreAppStep(jhelper, _mysql_component(), target).run(step_context) + + assert result.result_type == ResultType.FAILED + assert jhelper.run_action.call_count == 2 + + def test_vault_restore_does_not_retry_action_failure(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "vault/0" + jhelper.run_action.side_effect = [ + {"backup-ids": '["vault-backup-openstack-2026-07-15-00-03-28"]'}, + ActionFailedException("restore failure"), + ] + target = ActionTarget("vault", "vault/0", VAULT_CHARM, "restore-backup") + + result = _RestoreAppStep(jhelper, _vault_component(), target).run(step_context) + + assert result.result_type == ResultType.FAILED + assert jhelper.run_action.call_count == 2 + + +class TestScaleMySQLStep: + def test_scales_and_waits(self, step_context): + jhelper = Mock() + jhelper.get_application.return_value = Mock(units={"keystone-mysql/0": Mock()}) + result = _ScaleAppStep(jhelper, "keystone-mysql", 1).run(step_context) + assert result.result_type == ResultType.COMPLETED + jhelper.get_application.assert_called_once_with("keystone-mysql", "openstack") + jhelper.scale_application.assert_called_once_with( + "openstack", "keystone-mysql", 1 + ) + jhelper.wait_until_active.assert_called_once() + + def test_returns_failed_on_juju_error(self, step_context): + jhelper = Mock() + jhelper.get_application.return_value = Mock(units={"keystone-mysql/0": Mock()}) + jhelper.scale_application.side_effect = JujuException("boom") + result = _ScaleAppStep(jhelper, "keystone-mysql", 1).run(step_context) + assert result.result_type == ResultType.FAILED + + def test_scale_to_zero_waits_for_existing_units_to_leave(self, step_context): + jhelper = Mock() + jhelper.get_application.return_value = Mock( + units={"keystone-mysql-router/0": Mock(), "keystone-mysql-router/1": Mock()} + ) + + result = _ScaleAppStep(jhelper, "keystone-mysql-router", 0).run(step_context) + + assert result.result_type == ResultType.COMPLETED + jhelper.wait_units_gone.assert_called_once_with( + ["keystone-mysql-router/0", "keystone-mysql-router/1"], + "openstack", + timeout=120, + ) + jhelper.wait_until_active.assert_not_called() + + +class TestMySQLControlPlaneResolution: + def test_resolves_apps_for_shared_mysql_topology(self): + jhelper = Mock() + + mysql_status = Mock( + relations={ + "database": [ + Mock( + interface="mysql_client", + related_app="keystone-mysql-router", + ), + Mock(interface="mysql_client", related_app="nova-mysql-router"), + ] + } + ) + keystone_router_status = Mock( + relations={ + "shared-db": [ + Mock(interface="mysql_client", related_app="mysql"), + Mock(interface="mysql_client", related_app="keystone"), + ] + } + ) + nova_router_status = Mock( + relations={ + "shared-db": [ + Mock(interface="mysql_client", related_app="mysql"), + Mock(interface="mysql_client", related_app="nova"), + ] + } + ) + + jhelper.get_model_status.return_value = Mock( + apps={ + "mysql": mysql_status, + "keystone": Mock(relations={}), + "keystone-mysql-router": keystone_router_status, + "nova": Mock(relations={}), + "nova-mysql-router": nova_router_status, + } + ) + + apps, routers = MySQLBackupComponent()._restore_apps( + jhelper, "mysql", "openstack" + ) + + assert apps == ["keystone", "nova"] + assert routers == ["keystone-mysql-router", "nova-mysql-router"] + + def test_resolves_all_per_service_routers_from_mysql_relations(self): + jhelper = Mock() + + def router_status(): + return Mock( + relations={ + "database": [ + Mock(interface="mysql_client", related_app="nova-mysql"), + Mock(interface="mysql_client", related_app="nova"), + ] + } + ) + + jhelper.get_model_status.return_value = Mock( + apps={ + "nova-mysql": Mock( + relations={ + "database": [ + Mock( + interface="mysql_client", + related_app="nova-mysql-router", + ), + Mock( + interface="mysql_client", + related_app="nova-api-mysql-router", + ), + Mock( + interface="mysql_client", + related_app="nova-cell-mysql-router", + ), + ] + } + ), + "nova-mysql-router": router_status(), + "nova-api-mysql-router": router_status(), + "nova-cell-mysql-router": router_status(), + } + ) + + apps, routers = MySQLBackupComponent()._restore_apps( + jhelper, "nova-mysql", "openstack" + ) + + assert apps == ["nova"] + assert routers == [ + "nova-api-mysql-router", + "nova-cell-mysql-router", + "nova-mysql-router", + ] + + def test_fails_when_relation_is_missing_from_mysql(self): + jhelper = Mock() + jhelper.get_model_status.return_value = Mock( + apps={ + "keystone-mysql": Mock(relations={}), + "keystone-mysql-router": Mock( + relations={ + "database": [ + Mock( + interface="mysql_client", + related_app="keystone-mysql", + ) + ] + } + ), + } + ) + + with pytest.raises(JujuException) as exc: + MySQLBackupComponent()._restore_apps(jhelper, "keystone-mysql", "openstack") + + assert "router applications" in str(exc.value) + + def test_ignores_cross_model_router_consumers(self): + jhelper = Mock() + + def router_status(consumer): + return Mock( + relations={ + "database": [ + Mock(interface="mysql_client", related_app="cinder-mysql"), + Mock(interface="mysql_client", related_app=consumer), + ] + } + ) + + jhelper.get_model_status.return_value = Mock( + apps={ + "cinder": Mock(relations={}), + "cinder-mysql": Mock( + relations={ + "database": [ + Mock( + interface="mysql_client", + related_app="cinder-mysql-router", + ), + Mock( + interface="mysql_client", + related_app="cinder-volume-mysql-router", + ), + ] + } + ), + "cinder-mysql-router": router_status("cinder"), + "cinder-volume-mysql-router": router_status("cinder-volume"), + } + ) + + apps, routers = MySQLBackupComponent()._restore_apps( + jhelper, "cinder-mysql", "openstack" + ) + + assert apps == ["cinder"] + assert routers == [ + "cinder-mysql-router", + "cinder-volume-mysql-router", + ] + + def test_fails_when_per_service_router_cannot_be_resolved(self): + jhelper = Mock() + jhelper.get_model_status.return_value = Mock( + apps={"keystone-mysql": Mock(relations={})} + ) + + with pytest.raises(JujuException) as exc: + MySQLBackupComponent()._restore_apps(jhelper, "keystone-mysql", "openstack") + + assert "router applications" in str(exc.value) + + def test_fails_when_shared_mysql_mapping_cannot_be_resolved(self): + jhelper = Mock() + jhelper.get_model_status.return_value = Mock(apps={"mysql": Mock(relations={})}) + + with pytest.raises(JujuException) as exc: + MySQLBackupComponent()._restore_apps(jhelper, "mysql", "openstack") + + assert "Could not resolve router applications" in str(exc.value) + + +def _finished_backup_action(unit, model, action, params=None, timeout=None): + if action == "list-backups": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + return {} + + +def _set_per_service_mysql_status(jhelper, app="keystone-mysql"): + api_app = app.removesuffix("-mysql") + router_app = f"{api_app}-mysql-router" + jhelper.get_model_status.return_value = Mock( + apps={ + app: Mock( + relations={ + "database": [Mock(interface="mysql_client", related_app=router_app)] + } + ), + router_app: Mock( + relations={ + "database": [ + Mock(interface="mysql_client", related_app=app), + Mock(interface="mysql_client", related_app=api_app), + ] + } + ), + } + ) + + +class TestMySQLRestorePlan: + def test_scales_router_down_before_restore_and_up_before_resume(self): + jhelper = Mock() + _set_per_service_mysql_status(jhelper) + jhelper.get_application.return_value = Mock( + units={"app/0": Mock(), "app/1": Mock()} + ) + target = ActionTarget( + "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "restore" + ) + + plan = MySQLBackupComponent().build_restore_plan( + jhelper, target, restore_to_time=None, timeout=120, model="openstack" + ) + + assert [type(step) for step in plan] == [ + _PauseAppStep, + _ScaleAppStep, + _ScaleAppStep, + _RestoreAppStep, + _ScaleAppStep, + _ScaleAppStep, + _ResumeAppStep, + ] + scale_targets = [ + (step.application, step.scale) + for step in plan + if isinstance(step, _ScaleAppStep) + ] + assert scale_targets == [ + ("keystone-mysql-router", 0), + ("keystone-mysql", 1), + ("keystone-mysql", 2), + ("keystone-mysql-router", 2), + ] + + def test_revert_restores_router_before_resuming_api(self): + jhelper = Mock() + _set_per_service_mysql_status(jhelper) + jhelper.get_application.return_value = Mock( + units={"app/0": Mock(), "app/1": Mock()} + ) + target = ActionTarget( + "keystone-mysql", "keystone-mysql/0", MYSQL_CHARM, "restore" + ) + + plan = MySQLBackupComponent().build_restore_revert_plan( + jhelper, target, timeout=120, model="openstack" + ) + + assert [type(step) for step in plan] == [ + _ScaleAppStep, + _ScaleAppStep, + _ResumeAppStep, + ] + assert plan[1].application == "keystone-mysql-router" + assert plan[1].scale == 2 + + +class TestRestoreStepWrapper: + def test_prechecks_all_before_any_restore_work(self, step_context): + jhelper = Mock() + _set_per_service_mysql_status(jhelper) + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + jhelper.get_application.return_value = Mock( + units={"a/0": Mock(), "a/1": Mock()} + ) + jhelper.get_application_actions.return_value = [] # no pause/resume + jhelper.run_action.side_effect = _finished_backup_action + + discovered = {MYSQL_CHARM: ["keystone-mysql"]} + + result = RestoreStep(jhelper, discovered).run(step_context) + + assert result.result_type == ResultType.FAILED + assert "pause/resume" in result.message + jhelper.scale_application.assert_not_called() + + def test_restores_each_target(self, step_context): + jhelper = Mock() + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + jhelper.get_application.return_value = Mock( + units={"a/0": Mock(), "a/1": Mock()} + ) + jhelper.get_application_actions.return_value = ["pause", "resume"] + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "list-backups": + return {"backup-ids": '["vault-backup-openstack-2026-07-15-00-03-28"]'} + return {} + + jhelper.run_action.side_effect = _run_action + + discovered = {VAULT_CHARM: ["vault"]} + + result = RestoreStep(jhelper, discovered).run(step_context) + + assert result.result_type == ResultType.COMPLETED + assert result.message[0].success is True + + def test_fails_when_restore_target_cannot_be_resolved(self, step_context): + jhelper = Mock() + + def _leader(app, model): + if app == "broken-vault": + raise LeaderNotFoundException("no leader") + return f"{app}/0" + + jhelper.get_leader_unit.side_effect = _leader + + discovered = {VAULT_CHARM: ["broken-vault"]} + + result = RestoreStep(jhelper, discovered).run(step_context) + + assert result.result_type == ResultType.FAILED + assert "Could not resolve restore target for broken-vault" in result.message + + def test_reverts_mysql_on_restore_failure(self, step_context): + jhelper = Mock() + _set_per_service_mysql_status(jhelper) + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + jhelper.get_application.return_value = Mock( + units={"a/0": Mock(), "a/1": Mock()} + ) + jhelper.get_application_actions.return_value = ["pause", "resume"] + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "list-backups": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + if action == "restore": + raise JujuException("restore failed") + return {} + + jhelper.run_action.side_effect = _run_action + + discovered = { + MYSQL_CHARM: ["keystone-mysql"], + VAULT_CHARM: ["vault"], + } + + result = RestoreStep(jhelper, discovered).run(step_context) + + assert result.result_type == ResultType.COMPLETED + outcome = result.message[0] + assert outcome.success is False + assert outcome.reverted is True + assert result.message[1].app == "vault" + assert "not attempted" in result.message[1].error + assert not any( + call.args[2] == "restore-backup" + for call in jhelper.run_action.call_args_list + ) + # scale down to 1, then revert scale back up + scale_calls = [call.args for call in jhelper.scale_application.call_args_list] + assert ("openstack", "keystone-mysql", 1) in scale_calls + + def test_rollback_attempts_resume_after_scale_failure(self, step_context): + jhelper = Mock() + _set_per_service_mysql_status(jhelper) + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + jhelper.get_application.return_value = Mock( + units={"keystone-mysql/0": Mock(), "keystone-mysql/1": Mock()} + ) + jhelper.get_application_actions.return_value = ["pause", "resume"] + + def _scale(model, app, scale): + if app == "keystone-mysql" and scale == 2: + raise JujuException("scale failed") + + jhelper.scale_application.side_effect = _scale + + def _run_action(unit, model, action, params=None, timeout=None): + if action == "list-backups": + return { + "backups": ( + "backup-id | backup-type | backup-status\n" + "---------------------------------------\n" + "2026-07-15T00:00:00Z | physical | finished" + ) + } + if action == "restore": + raise JujuException("restore failed") + return {} + + jhelper.run_action.side_effect = _run_action + + result = RestoreStep( + jhelper, + {MYSQL_CHARM: ["keystone-mysql"], VAULT_CHARM: ["vault"]}, + ).run(step_context) + + outcome = result.message[0] + assert outcome.success is False + assert outcome.reverted is False + assert "restore failed" in outcome.error + assert "scale failed" in outcome.rollback_error + assert any( + call.args[2] == "resume" for call in jhelper.run_action.call_args_list + ) + assert not any( + call.args[2] == "restore-backup" + for call in jhelper.run_action.call_args_list + ) + assert result.message[1].app == "vault" + assert "not attempted" in result.message[1].error + + def test_scale_read_failure_stops_all_restore_work(self, step_context): + jhelper = Mock() + _set_per_service_mysql_status(jhelper) + jhelper.get_leader_unit.side_effect = lambda app, model: f"{app}/0" + jhelper.get_application.side_effect = ApplicationNotFoundException("missing") + jhelper.get_application_actions.return_value = ["pause", "resume"] + + result = RestoreStep( + jhelper, + {MYSQL_CHARM: ["keystone-mysql"], VAULT_CHARM: ["vault"]}, + ).run(step_context) + + assert result.result_type == ResultType.FAILED + assert "Could not read current scale" in result.message + jhelper.run_action.assert_not_called() + + def test_precheck_plan_construction_failure_returns_failed_result( + self, step_context + ): + jhelper = Mock() + jhelper.get_leader_unit.return_value = "mysql/0" + jhelper.get_model_status.return_value = Mock(apps={"mysql": Mock(relations={})}) + + result = RestoreStep(jhelper, {MYSQL_CHARM: ["mysql"]}).run(step_context) + + assert result.result_type == ResultType.FAILED + assert "Could not resolve router applications" in result.message