From 0a31706fae48540d9b358e915a468ef352e7b95e Mon Sep 17 00:00:00 2001 From: German Date: Thu, 28 May 2026 13:34:22 -0700 Subject: [PATCH] test(e2e): add backup expiration cleanup coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #370. Adds e2e coverage for the operator's backup retention path plus reusable helpers, in two complementary specs: - expired → garbage-collected: cluster ready → completed on-demand Backup → MarkExpired(now-1h) → assert both the Backup CR and its underlying VolumeSnapshot are removed. The snapshot assertion closes the more user-visible leak (orphaned storage), exercising the full Backup → CNPG Backup → VolumeSnapshot cascade (operator sets SnapshotOwnerReference=backup). - not-yet-expired → retained: a completed Backup whose status.expiredAt is in the future must NOT be deleted, and neither must its snapshot. Guards user data against an inverted/over-aggressive IsExpired. Helpers: - MarkExpired — status-subresource patch fast-forwarding expiration (avoids the silent no-op when patching through c.Patch on a +kubebuilder:subresource:status type). - WaitForBackupDeleted / WaitForSnapshotsDeletedForBackup — poll until the Backup CR / its labeled VolumeSnapshots are gone. - provisionReadyCluster / createCompletedBackup — shared spec setup that also de-duplicates the on-demand backup spec. Labels: [backup, needs-csi-snapshots, level:medium]. Validation: - go vet ./... clean - go test ./pkg/e2eutils/backup/... pass (incl. new snapshot-deletion unit tests with label/namespace filtering distractors) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6285c813-739d-4ebc-9511-1439884e2c9f Signed-off-by: German Eichberger --- test/e2e/pkg/e2eutils/backup/backup.go | 68 +++++++++++ test/e2e/pkg/e2eutils/backup/backup_test.go | 114 ++++++++++++++++++ test/e2e/pkg/e2eutils/backup/snapshot.go | 33 +++++ .../tests/backup/backup_expiration_test.go | 111 +++++++++++++++++ test/e2e/tests/backup/backup_ondemand_test.go | 33 ++--- test/e2e/tests/backup/helpers_test.go | 46 +++++++ 6 files changed, 379 insertions(+), 26 deletions(-) create mode 100644 test/e2e/tests/backup/backup_expiration_test.go diff --git a/test/e2e/pkg/e2eutils/backup/backup.go b/test/e2e/pkg/e2eutils/backup/backup.go index d7d6424df..36752700f 100644 --- a/test/e2e/pkg/e2eutils/backup/backup.go +++ b/test/e2e/pkg/e2eutils/backup/backup.go @@ -23,6 +23,7 @@ import ( cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" "github.com/cloudnative-pg/cloudnative-pg/tests/utils/envsubst" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/yaml" @@ -205,6 +206,73 @@ func WaitForCompleted(ctx context.Context, c client.Client, ns, name string, tim } } +// MarkExpired patches status.expiredAt on the Backup CR via the +// status subresource so the operator's reconcile loop treats it as +// expired and garbage-collects it. Used by the expired-backup +// cleanup spec to fast-forward expiration without waiting out the +// real retentionDays window. +// +// Trigger semantics: the status patch fires a watch event that the +// Backup controller picks up within seconds, so deletion is +// event-driven rather than bound to the 1-minute periodic requeue in +// backup_controller.go. +// +// Implementation detail: this must use c.Status().Patch — patching +// the same field through c.Patch is a silent no-op because Backup +// declares status as a subresource (see operator/src/api/preview/ +// backup_types.go +kubebuilder:subresource:status). +// +// A fresh Get is performed inside the helper so callers can pass a +// stable (ns, name) without juggling resourceVersion conflicts with +// the controller's own status writers. +func MarkExpired(ctx context.Context, c client.Client, ns, name string, when time.Time) error { + if c == nil { + return errors.New("backup.MarkExpired: client must not be nil") + } + var current previewv1.Backup + if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: name}, ¤t); err != nil { + return fmt.Errorf("get Backup %s/%s: %w", ns, name, err) + } + patch := client.MergeFrom(current.DeepCopy()) + expired := metav1.NewTime(when) + current.Status.ExpiredAt = &expired + if err := c.Status().Patch(ctx, ¤t, patch); err != nil { + return fmt.Errorf("patch status.expiredAt on Backup %s/%s: %w", ns, name, err) + } + return nil +} + +// WaitForBackupDeleted polls until the named Backup no longer exists +// in ns or timeout elapses. Mirrors WaitForPVCDeleted in shape and +// is used by the expired-backup cleanup spec to assert the operator +// removed the Backup CR after MarkExpired flipped status.expiredAt +// into the past. +func WaitForBackupDeleted(ctx context.Context, c client.Client, ns, name string, timeout time.Duration) error { + if c == nil { + return errors.New("backup.WaitForBackupDeleted: client must not be nil") + } + deadline := time.Now().Add(timeout) + for { + var b previewv1.Backup + err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: name}, &b) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return fmt.Errorf("polling Backup %s/%s: %w", ns, name, err) + } + if time.Now().After(deadline) { + return fmt.Errorf("timed out after %s waiting for Backup %s/%s to be deleted (last phase=%q)", + timeout, ns, name, b.Status.Phase) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(DefaultPollInterval): + } + } +} + // Delete issues a foreground delete on the Backup CR and returns when // the object is gone or timeout elapses. Safe to call on a missing // object (IsNotFound is treated as success). diff --git a/test/e2e/pkg/e2eutils/backup/backup_test.go b/test/e2e/pkg/e2eutils/backup/backup_test.go index 5d8b04bb2..e57575cb6 100644 --- a/test/e2e/pkg/e2eutils/backup/backup_test.go +++ b/test/e2e/pkg/e2eutils/backup/backup_test.go @@ -11,6 +11,7 @@ import ( "time" cnpgv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1" + utilslabels "github.com/cloudnative-pg/cloudnative-pg/pkg/utils" snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -207,6 +208,119 @@ func TestPVBelongsToClusterMatchesLabelsOrClaimPrefix(t *testing.T) { } } +func TestMarkExpiredPatchesStatusViaSubresource(t *testing.T) { + t.Parallel() + s := newScheme(t) + bkp := &previewv1.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "b", Namespace: "ns"}, + Spec: previewv1.BackupSpec{Cluster: cnpgv1.LocalObjectReference{Name: "c"}}, + } + c := fakeclient.NewClientBuilder(). + WithScheme(s). + WithObjects(bkp). + WithStatusSubresource(&previewv1.Backup{}). + Build() + + when := time.Date(2000, 1, 2, 3, 4, 5, 0, time.UTC) + if err := MarkExpired(context.Background(), c, "ns", "b", when); err != nil { + t.Fatalf("MarkExpired: %v", err) + } + + var got previewv1.Backup + if err := c.Get(context.Background(), client.ObjectKey{Namespace: "ns", Name: "b"}, &got); err != nil { + t.Fatalf("Get after MarkExpired: %v", err) + } + if got.Status.ExpiredAt == nil { + t.Fatalf("expected status.expiredAt to be set; got nil") + } + if !got.Status.ExpiredAt.Time.Equal(when) { + t.Fatalf("status.expiredAt=%v want %v", got.Status.ExpiredAt.Time, when) + } +} + +func TestMarkExpiredErrorsOnNotFound(t *testing.T) { + t.Parallel() + s := newScheme(t) + c := fakeclient.NewClientBuilder(). + WithScheme(s). + WithStatusSubresource(&previewv1.Backup{}). + Build() + + err := MarkExpired(context.Background(), c, "ns", "missing", time.Unix(0, 0)) + if err == nil { + t.Fatal("expected error for missing Backup, got nil") + } + if !strings.Contains(err.Error(), "missing") { + t.Fatalf("expected error to mention name; got %v", err) + } +} + +func TestWaitForBackupDeletedReturnsWhenMissing(t *testing.T) { + t.Parallel() + s := newScheme(t) + c := fakeclient.NewClientBuilder().WithScheme(s).Build() + if err := WaitForBackupDeleted(context.Background(), c, "ns", "nope", 500*time.Millisecond); err != nil { + t.Fatalf("expected nil for missing Backup, got %v", err) + } +} + +func TestWaitForBackupDeletedTimesOutWhenPresent(t *testing.T) { + t.Parallel() + s := newScheme(t) + bkp := &previewv1.Backup{ + ObjectMeta: metav1.ObjectMeta{Name: "b", Namespace: "ns"}, + Status: previewv1.BackupStatus{Phase: cnpgv1.BackupPhaseCompleted}, + } + c := fakeclient.NewClientBuilder().WithScheme(s).WithObjects(bkp).Build() + err := WaitForBackupDeleted(context.Background(), c, "ns", "b", 500*time.Millisecond) + if err == nil || !strings.Contains(err.Error(), "timed out") { + t.Fatalf("expected timeout error, got %v", err) + } +} + +func TestWaitForSnapshotsDeletedForBackupReturnsWhenNone(t *testing.T) { + t.Parallel() + s := newScheme(t) + // Distractors that must be ignored: a snapshot in the same namespace + // carrying a different backupName, and one with the matching label + // in another namespace. Neither belongs to backup "b" in "ns". + otherBackup := &snapshotv1.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "snap-other", + Namespace: "ns", + Labels: map[string]string{utilslabels.BackupNameLabelName: "other"}, + }, + } + otherNS := &snapshotv1.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "snap-b", + Namespace: "other-ns", + Labels: map[string]string{utilslabels.BackupNameLabelName: "b"}, + }, + } + c := fakeclient.NewClientBuilder().WithScheme(s).WithObjects(otherBackup, otherNS).Build() + if err := WaitForSnapshotsDeletedForBackup(context.Background(), c, "ns", "b", 500*time.Millisecond); err != nil { + t.Fatalf("expected nil when no matching snapshots exist, got %v", err) + } +} + +func TestWaitForSnapshotsDeletedForBackupTimesOutWhenPresent(t *testing.T) { + t.Parallel() + s := newScheme(t) + snap := &snapshotv1.VolumeSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "snap-b", + Namespace: "ns", + Labels: map[string]string{utilslabels.BackupNameLabelName: "b"}, + }, + } + c := fakeclient.NewClientBuilder().WithScheme(s).WithObjects(snap).Build() + err := WaitForSnapshotsDeletedForBackup(context.Background(), c, "ns", "b", 500*time.Millisecond) + if err == nil || !strings.Contains(err.Error(), "timed out") { + t.Fatalf("expected timeout error, got %v", err) + } +} + func TestWaitForPVCDeletedReturnsWhenMissing(t *testing.T) { t.Parallel() s := newScheme(t) diff --git a/test/e2e/pkg/e2eutils/backup/snapshot.go b/test/e2e/pkg/e2eutils/backup/snapshot.go index 621f22f7e..8674f57f2 100644 --- a/test/e2e/pkg/e2eutils/backup/snapshot.go +++ b/test/e2e/pkg/e2eutils/backup/snapshot.go @@ -81,6 +81,39 @@ func WaitForSnapshotForBackup(ctx context.Context, c client.Client, ns, backupNa } } +// WaitForSnapshotsDeletedForBackup polls until no VolumeSnapshot in ns +// carries the cnpg.io/backupName= label, or timeout +// elapses. Used by the expired-backup cleanup spec to assert the +// operator's cascading garbage collection (Backup → CNPG Backup → +// VolumeSnapshot) removes the underlying snapshot, not just the Backup +// CR. A leaked snapshot is the more user-visible failure mode +// (orphaned storage cost), so this closes the loop that +// WaitForBackupDeleted alone leaves open. +func WaitForSnapshotsDeletedForBackup(ctx context.Context, c client.Client, ns, backupName string, timeout time.Duration) error { + if c == nil { + return errors.New("backup.WaitForSnapshotsDeletedForBackup: client must not be nil") + } + deadline := time.Now().Add(timeout) + for { + snaps, err := ListSnapshotsForBackup(ctx, c, ns, backupName) + if err != nil { + return err + } + if len(snaps) == 0 { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("timed out after %s waiting for VolumeSnapshots of backup %s/%s to be deleted (%d still present)", + timeout, ns, backupName, len(snaps)) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(DefaultPollInterval): + } + } +} + // FindRetainedPV returns the first PersistentVolume whose claimRef // points at a PVC in ns for clusterName. The helper prefers PVs that // are in the Released phase (the post-delete state when the reclaim diff --git a/test/e2e/tests/backup/backup_expiration_test.go b/test/e2e/tests/backup/backup_expiration_test.go new file mode 100644 index 000000000..39afa64e1 --- /dev/null +++ b/test/e2e/tests/backup/backup_expiration_test.go @@ -0,0 +1,111 @@ +package backup + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" //nolint:revive + . "github.com/onsi/gomega" //nolint:revive + + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/documentdb/documentdb-operator/test/e2e" + bkp "github.com/documentdb/documentdb-operator/test/e2e/pkg/e2eutils/backup" + "github.com/documentdb/documentdb-operator/test/e2e/pkg/e2eutils/namespaces" + "github.com/documentdb/documentdb-operator/test/e2e/pkg/e2eutils/timeouts" +) + +var _ = Describe("DocumentDB backup — expiration cleanup", + Label(e2e.BackupLabel, e2e.NeedsCSISnapshotsLabel), e2e.MediumLevelLabel, + func() { + const clusterName = "backup-expire" + var ( + ctx context.Context + ns string + c client.Client + ) + + BeforeEach(func() { + e2e.SkipUnlessLevel(e2e.Medium) + ctx = context.Background() + c = e2e.SuiteEnv().Client + skipUnlessCSISnapshotsUsable(ctx, c) + ns = namespaces.NamespaceForSpec(e2e.BackupLabel) + createNamespace(ctx, c, ns) + createCredentialSecret(ctx, c, ns) + // A completed backup marked expired would otherwise race the + // snapshot controller, so every spec here starts from a live + // cluster and lets createCompletedBackup settle the snapshot. + provisionReadyCluster(ctx, c, ns, clusterName) + }) + + It("garbage-collects the Backup CR and its VolumeSnapshot once status.expiredAt is in the past", func() { + const backupName = "backup-expire-1" + createCompletedBackup(ctx, c, ns, clusterName, backupName) + + // A completed CSI backup must have produced a ready + // VolumeSnapshot. Waiting on ReadyToUse (rather than a bare + // list) removes any dependence on CNPG's snapshot-vs-status + // ordering and pins the snapshot identity we expect the + // operator's cascading GC (Backup → CNPG Backup → + // VolumeSnapshot) to reclaim — a leaked snapshot is the more + // user-visible failure than a lingering CR. + snap, err := bkp.WaitForSnapshotForBackup(ctx, c, ns, backupName, + timeouts.For(timeouts.BackupComplete)) + Expect(err).NotTo(HaveOccurred(), + "no ready VolumeSnapshot for completed Backup %s/%s", ns, backupName) + Expect(snap).NotTo(BeNil()) + + // Fast-forward expiration via the status subresource. The + // next reconcile sees IsExpired() == true and deletes the CR. + // This must use the status subresource: Backup declares + // +kubebuilder:subresource:status, so a plain Patch is a + // silent no-op (see MarkExpired). + Expect(bkp.MarkExpired(ctx, c, ns, backupName, time.Now().Add(-1*time.Hour))). + To(Succeed(), "patch status.expiredAt on Backup %s/%s", ns, backupName) + + // The status patch fires a watch event, so deletion is + // event-driven — 3 minutes is comfortable headroom over the + // 1-minute periodic requeue in backup_controller.go. + Expect(bkp.WaitForBackupDeleted(ctx, c, ns, backupName, 3*time.Minute)). + To(Succeed(), "Backup %s/%s was not garbage-collected after expiration", ns, backupName) + Expect(bkp.WaitForSnapshotsDeletedForBackup(ctx, c, ns, backupName, 3*time.Minute)). + To(Succeed(), "VolumeSnapshot for Backup %s/%s leaked after expiration", ns, backupName) + }) + + It("retains a Backup CR whose status.expiredAt is still in the future", func() { + const backupName = "backup-expire-2" + createCompletedBackup(ctx, c, ns, clusterName, backupName) + + // The retention guard is only meaningful if expiredAt is + // actually in the future — otherwise "not deleted" proves + // nothing. createCompletedBackup uses RetentionDays=1, so + // assert the controller populated expiredAt ~1 day out. + b, err := bkp.Get(ctx, c, ns, backupName) + Expect(err).NotTo(HaveOccurred()) + Expect(b.Status.ExpiredAt).NotTo(BeNil(), + "Backup %s/%s has no status.expiredAt; retention premise is vacuous", ns, backupName) + Expect(b.Status.ExpiredAt.Time).To(BeTemporally(">", time.Now()), + "status.expiredAt must be in the future for the retention guard to mean anything") + + snap, err := bkp.WaitForSnapshotForBackup(ctx, c, ns, backupName, + timeouts.For(timeouts.BackupComplete)) + Expect(err).NotTo(HaveOccurred(), + "no ready VolumeSnapshot for completed Backup %s/%s", ns, backupName) + + // Neither the Backup CR nor its snapshot may disappear while + // unexpired. This guards user data against a regression that + // inverts IsExpired() or makes cleanup over-aggressive. We + // never call MarkExpired — both must simply persist across + // several reconcile passes (status writes fire watch events, + // so a broken controller would react well within 45s). + Consistently(func() error { + if _, err := bkp.Get(ctx, c, ns, backupName); err != nil { + return err + } + return c.Get(ctx, client.ObjectKey{Namespace: ns, Name: snap.Name}, &snapshotv1.VolumeSnapshot{}) + }, 45*time.Second, bkp.DefaultPollInterval). + Should(Succeed(), "Backup %s/%s or its VolumeSnapshot was deleted before expiry", ns, backupName) + }) + }) diff --git a/test/e2e/tests/backup/backup_ondemand_test.go b/test/e2e/tests/backup/backup_ondemand_test.go index b4a8680fe..d671f586c 100644 --- a/test/e2e/tests/backup/backup_ondemand_test.go +++ b/test/e2e/tests/backup/backup_ondemand_test.go @@ -7,14 +7,10 @@ import ( . "github.com/onsi/ginkgo/v2" //nolint:revive . "github.com/onsi/gomega" //nolint:revive - "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/documentdb/documentdb-operator/test/e2e" - "github.com/documentdb/documentdb-operator/test/e2e/pkg/e2eutils/assertions" bkp "github.com/documentdb/documentdb-operator/test/e2e/pkg/e2eutils/backup" - shareddb "github.com/documentdb/documentdb-operator/test/shared/documentdb" - "github.com/documentdb/documentdb-operator/test/e2e/pkg/e2eutils/documentdb" "github.com/documentdb/documentdb-operator/test/e2e/pkg/e2eutils/namespaces" "github.com/documentdb/documentdb-operator/test/e2e/pkg/e2eutils/timeouts" ) @@ -43,27 +39,12 @@ var _ = Describe("DocumentDB backup — on-demand CSI snapshot", }) It("takes a CSI volume snapshot and marks the Backup CR Completed", func() { - dd, err := documentdb.Create(ctx, c, ns, clusterName, documentdb.CreateOptions{ - Base: "documentdb", - Vars: baseVars(clusterName, ns, "2Gi"), - ManifestsRoot: manifestsRoot(), - }) - Expect(err).NotTo(HaveOccurred()) - DeferCleanup(func(ctx SpecContext) { - _ = shareddb.Delete(ctx, c, dd, 3*time.Minute) - }) - - // 1. Source cluster healthy before we ask for a backup. - key := types.NamespacedName{Namespace: ns, Name: clusterName} - Eventually(assertions.AssertDocumentDBReady(ctx, c, key), - timeouts.For(timeouts.DocumentDBReady), - timeouts.PollInterval(timeouts.DocumentDBReady), - ).Should(Succeed()) + provisionReadyCluster(ctx, c, ns, clusterName) - // 2. Request an on-demand Backup. Rendering and applying - // this CR is all it takes to trigger the operator path - // that the workflow exercises end-to-end. - _, err = bkp.Create(ctx, c, bkp.BackupVars{ + // Request an on-demand Backup. Rendering and applying this + // CR is all it takes to trigger the operator path that the + // workflow exercises end-to-end. + _, err := bkp.Create(ctx, c, bkp.BackupVars{ Name: backupName, Namespace: ns, ClusterName: clusterName, @@ -74,7 +55,7 @@ var _ = Describe("DocumentDB backup — on-demand CSI snapshot", _ = bkp.Delete(ctx, c, ns, backupName, 1*time.Minute) }) - // 3. Wait for the Backup CR itself to go Completed. This + // 1. Wait for the Backup CR itself to go Completed. This // is the operator-visible signal that the CSI snapshot // finished and the backup metadata was persisted. done, err := bkp.WaitForCompleted(ctx, c, ns, backupName, @@ -83,7 +64,7 @@ var _ = Describe("DocumentDB backup — on-demand CSI snapshot", "Backup %s/%s did not reach Completed", ns, backupName) Expect(string(done.Status.Phase)).To(Equal("completed")) - // 4. Assert a VolumeSnapshot tagged for this Backup + // 2. Assert a VolumeSnapshot tagged for this Backup // reached ReadyToUse. This is what distinguishes the CSI // path from any other backup strategy. snap, err := bkp.WaitForSnapshotForBackup(ctx, c, ns, backupName, diff --git a/test/e2e/tests/backup/helpers_test.go b/test/e2e/tests/backup/helpers_test.go index 936d05c07..afcdb99c5 100644 --- a/test/e2e/tests/backup/helpers_test.go +++ b/test/e2e/tests/backup/helpers_test.go @@ -5,19 +5,25 @@ import ( "os" "path/filepath" "runtime" + "time" . "github.com/onsi/ginkgo/v2" //nolint:revive . "github.com/onsi/gomega" //nolint:revive corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/yaml" previewv1 "github.com/documentdb/documentdb-operator/api/preview" + "github.com/documentdb/documentdb-operator/test/e2e/pkg/e2eutils/assertions" bkp "github.com/documentdb/documentdb-operator/test/e2e/pkg/e2eutils/backup" "github.com/documentdb/documentdb-operator/test/e2e/pkg/e2eutils/clusterprobe" + "github.com/documentdb/documentdb-operator/test/e2e/pkg/e2eutils/documentdb" "github.com/documentdb/documentdb-operator/test/e2e/pkg/e2eutils/fixtures" + "github.com/documentdb/documentdb-operator/test/e2e/pkg/e2eutils/timeouts" + shareddb "github.com/documentdb/documentdb-operator/test/shared/documentdb" ) // credentialSecretName is the secret the backup area seeds in every @@ -116,6 +122,46 @@ func createRecoveryDocumentDB( return dd } +// provisionReadyCluster creates a 1-node DocumentDB in ns, registers +// its cleanup, and blocks until it reports Ready. Shared by every +// backup spec that needs a live source cluster before taking a backup. +func provisionReadyCluster(ctx context.Context, c client.Client, ns, clusterName string) { + dd, err := documentdb.Create(ctx, c, ns, clusterName, documentdb.CreateOptions{ + Base: "documentdb", + Vars: baseVars(clusterName, ns, "2Gi"), + ManifestsRoot: manifestsRoot(), + }) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func(ctx SpecContext) { + _ = shareddb.Delete(ctx, c, dd, 3*time.Minute) + }) + key := types.NamespacedName{Namespace: ns, Name: clusterName} + Eventually(assertions.AssertDocumentDBReady(ctx, c, key), + timeouts.For(timeouts.DocumentDBReady), + timeouts.PollInterval(timeouts.DocumentDBReady), + ).Should(Succeed()) +} + +// createCompletedBackup creates an on-demand Backup (RetentionDays=1) +// against clusterName, registers cleanup, and blocks until the Backup +// CR reports Completed. bkp.Delete swallows IsNotFound, so the +// registered cleanup stays correct even for specs whose Backup is +// garbage-collected by the operator before teardown. +func createCompletedBackup(ctx context.Context, c client.Client, ns, clusterName, backupName string) { + _, err := bkp.Create(ctx, c, bkp.BackupVars{ + Name: backupName, + Namespace: ns, + ClusterName: clusterName, + RetentionDays: 1, + }) + Expect(err).NotTo(HaveOccurred(), "create Backup CR %s/%s", ns, backupName) + DeferCleanup(func(ctx SpecContext) { + _ = bkp.Delete(ctx, c, ns, backupName, 1*time.Minute) + }) + _, err = bkp.WaitForCompleted(ctx, c, ns, backupName, timeouts.For(timeouts.BackupComplete)) + Expect(err).NotTo(HaveOccurred(), "Backup %s/%s did not reach Completed", ns, backupName) +} + // skipUnlessCSISnapshotsUsable is the pre-flight for every backup // spec. The Ginkgo NeedsCSISnapshotsLabel only filters invocations; // this probe additionally verifies that the VolumeSnapshot CRD is