From 57bfa89973455aa2326bb62df727dc4cfebae0fb Mon Sep 17 00:00:00 2001 From: Marcel Boehm Date: Wed, 5 Aug 2026 15:59:20 +0200 Subject: [PATCH 1/4] Skip automatic etcdEncryptionKey rotation for hibernated shoots (#15209) * Skip automatic etcdEncryptionKey rotation for hibernated shoots * Prefactor: hibernationutils * Add HibernationScheduleProblematic constraint * check conflicts only during the next maintenance * Adress PR Feedback * Rename automatic rotation constraint Assisted-by: OpenCode (gpt-5.6-terra) * Report all blocked automatic rotations Assisted-by: OpenCode (gpt-5.6-terra) * Add rotation period helper tests Assisted-by: OpenCode (github-copilot/gpt-5.6-luna) --------- Co-authored-by: Tim Ebert --- dev-setup/skaffold-gardenadm.yaml | 1 + dev-setup/skaffold-operator.yaml | 1 + dev-setup/skaffold-seed.yaml | 1 + .../shoot_credentials_rotation.md | 2 + docs/usage/shoot/shoot_maintenance.md | 1 + docs/usage/shoot/shoot_status.md | 6 + pkg/api/core/v1beta1/helper/shoot.go | 40 ++++ pkg/api/core/v1beta1/helper/shoot_test.go | 68 ++++++ pkg/apis/core/v1beta1/types_shoot.go | 3 + .../shoot/hibernation/reconciler.go | 102 ++------- .../shoot/hibernation/reconciler_test.go | 44 ---- .../shoot/maintenance/reconciler.go | 52 +---- .../shoot/maintenance/reconciler_test.go | 40 ++-- .../controller/shoot/care/constraints.go | 52 ++++- .../controller/shoot/care/constraints_test.go | 211 +++++++++++++++--- .../controller/shoot/care/hibernation.go | 85 +++++++ .../controller/shoot/care/hibernation_test.go | 178 +++++++++++++++ .../controller/shoot/care/reconciler_test.go | 6 +- pkg/utils/hibernation/hibernationschedule.go | 95 ++++++++ .../hibernation/hibernationschedule_test.go | 137 ++++++++++++ pkg/utils/hibernation/suite_test.go | 17 ++ 21 files changed, 908 insertions(+), 234 deletions(-) create mode 100644 pkg/gardenlet/controller/shoot/care/hibernation.go create mode 100644 pkg/gardenlet/controller/shoot/care/hibernation_test.go create mode 100644 pkg/utils/hibernation/hibernationschedule.go create mode 100644 pkg/utils/hibernation/hibernationschedule_test.go create mode 100644 pkg/utils/hibernation/suite_test.go diff --git a/dev-setup/skaffold-gardenadm.yaml b/dev-setup/skaffold-gardenadm.yaml index 4892b57a59c..78573a33a8f 100644 --- a/dev-setup/skaffold-gardenadm.yaml +++ b/dev-setup/skaffold-gardenadm.yaml @@ -775,6 +775,7 @@ build: - pkg/utils/gardener/secretsrotation - pkg/utils/gardener/shootstate - pkg/utils/gardener/tokenrequest + - pkg/utils/hibernation - pkg/utils/imagevector - pkg/utils/istio - pkg/utils/kubernetes diff --git a/dev-setup/skaffold-operator.yaml b/dev-setup/skaffold-operator.yaml index 0ff2f139ccf..46f56e2f4e8 100644 --- a/dev-setup/skaffold-operator.yaml +++ b/dev-setup/skaffold-operator.yaml @@ -700,6 +700,7 @@ build: - pkg/utils/flow - pkg/utils/gardener - pkg/utils/gardener/gardenlet + - pkg/utils/hibernation - pkg/utils/imagevector - pkg/utils/kubernetes - pkg/utils/kubernetes/bootstraptoken diff --git a/dev-setup/skaffold-seed.yaml b/dev-setup/skaffold-seed.yaml index 88246227809..1b3e55edc74 100644 --- a/dev-setup/skaffold-seed.yaml +++ b/dev-setup/skaffold-seed.yaml @@ -292,6 +292,7 @@ build: - pkg/utils/gardener/secretsrotation - pkg/utils/gardener/shootstate - pkg/utils/gardener/tokenrequest + - pkg/utils/hibernation - pkg/utils/imagevector - pkg/utils/istio - pkg/utils/kubernetes diff --git a/docs/usage/shoot-operations/shoot_credentials_rotation.md b/docs/usage/shoot-operations/shoot_credentials_rotation.md index 76b772774a9..8385ae702f9 100644 --- a/docs/usage/shoot-operations/shoot_credentials_rotation.md +++ b/docs/usage/shoot-operations/shoot_credentials_rotation.md @@ -276,6 +276,8 @@ The encryption key has no expiration date. **Unless automatic credentials rotation is enabled, it is the responsibility of the end-user to regularly rotate those credentials.** Refer to [Automatic Credentials Rotation](../shoot/shoot_maintenance.md#automatic-credentials-rotation) for instructions on enabling automatic rotation for etcd encryption key. +Automatic ETCD encryption key rotation requires a running ETCD and `kube-apiserver`, so it is skipped while the Shoot is hibernated. If an overdue automatic rotation cannot run during the next maintenance window because of the hibernation schedule, Gardener reports the `AutomaticCredentialsRotationPossible` constraint in the Shoot status. Adjust the maintenance window or hibernation schedule so that maintenance runs while the Shoot is awake. + The rotation happens in three stages: - In stage one, a new encryption key is created and added to the bundle (together with the old encryption key). diff --git a/docs/usage/shoot/shoot_maintenance.md b/docs/usage/shoot/shoot_maintenance.md index 47e5551bbe1..b3e6ab49e64 100644 --- a/docs/usage/shoot/shoot_maintenance.md +++ b/docs/usage/shoot/shoot_maintenance.md @@ -107,6 +107,7 @@ spec: > See [ETCD Encryption Config](../security/etcd_encryption_config.md) for more details. During the daily maintenance, the `gardener-controller-manager` starts the rotation for specific credentials if the Shoot opted-in for automatic rotation for the given credential and the set period has passed since the last rotation completion. +Automatic ETCD encryption key rotation requires a running ETCD and `kube-apiserver`, so it is skipped while the Shoot is hibernated. If an overdue automatic rotation cannot run during the next maintenance window because of the hibernation schedule, Gardener reports the `AutomaticCredentialsRotationPossible` constraint in the Shoot status. Adjust the maintenance window or hibernation schedule so that maintenance runs while the Shoot is awake. Automatic rotation can be disabled for specific credential by setting the `rotationPeriod` field to `0`. ## Cluster Reconciliation diff --git a/docs/usage/shoot/shoot_status.md b/docs/usage/shoot/shoot_status.md index 457cd8eccb7..7aad94fc062 100644 --- a/docs/usage/shoot/shoot_status.md +++ b/docs/usage/shoot/shoot_status.md @@ -131,6 +131,12 @@ If it's visible, operators should be aware that the annotated resources may dive This constraint indicates that one or more machines in `Failed` phase are currently being preserved (i.e., not terminated) to allow for debugging and analysis. The constraint is not added to `.status.constraints` when no failed machines are currently preserved. See [Machine Preservation](shoot_machine_preservation.md) for more details. +**`AutomaticCredentialsRotationPossible`**: + +This optional constraint indicates whether an overdue automatic ETCD encryption key rotation can run during the next maintenance window. +The constraint is omitted when automatic rotation is not required or can run. It is added with status `False` when the next maintenance window is affected by hibernation, because ETCD encryption key rotation requires a running ETCD and `kube-apiserver`. +If it is present, adjust the maintenance window or hibernation schedule so that maintenance runs while the Shoot is awake. See [ETCD Encryption Key](../shoot-operations/shoot_credentials_rotation.md#etcd-encryption-key) for details. + ### Last Operation diff --git a/pkg/api/core/v1beta1/helper/shoot.go b/pkg/api/core/v1beta1/helper/shoot.go index ffdb89c1a5a..6f3c5e22fcc 100644 --- a/pkg/api/core/v1beta1/helper/shoot.go +++ b/pkg/api/core/v1beta1/helper/shoot.go @@ -8,6 +8,7 @@ import ( "fmt" "slices" "strconv" + "time" "github.com/Masterminds/semver/v3" autoscalingv1 "k8s.io/api/autoscaling/v1" @@ -713,6 +714,45 @@ func IsETCDEncryptionKeyAutoRotationEnabled(shoot *gardencorev1beta1.Shoot) bool shoot.Spec.Maintenance.AutoRotation.Credentials.ETCDEncryptionKey.RotationPeriod.Duration != 0 } +// SSHKeypairRotationPassedRotationPeriod returns true if the SSH keypair rotation period has passed. +// If the credentials have never been rotated, the shoot's creation timestamp is used as the reference point. +func SSHKeypairRotationPassedRotationPeriod(shoot *gardencorev1beta1.Shoot, now time.Time, period metav1.Duration) bool { + latestRotationCompletionTime := shoot.CreationTimestamp.Time + if shoot.Status.Credentials != nil && + shoot.Status.Credentials.Rotation != nil && + shoot.Status.Credentials.Rotation.SSHKeypair != nil && + shoot.Status.Credentials.Rotation.SSHKeypair.LastCompletionTime != nil { + latestRotationCompletionTime = shoot.Status.Credentials.Rotation.SSHKeypair.LastCompletionTime.Time + } + return latestRotationCompletionTime.Before(now.Add(-period.Duration)) +} + +// ObservabilityRotationPassedRotationPeriod returns true if the observability passwords rotation period has passed. +// If the credentials have never been rotated, the shoot's creation timestamp is used as the reference point. +func ObservabilityRotationPassedRotationPeriod(shoot *gardencorev1beta1.Shoot, now time.Time, period metav1.Duration) bool { + latestRotationCompletionTime := shoot.CreationTimestamp.Time + if shoot.Status.Credentials != nil && + shoot.Status.Credentials.Rotation != nil && + shoot.Status.Credentials.Rotation.Observability != nil && + shoot.Status.Credentials.Rotation.Observability.LastCompletionTime != nil { + latestRotationCompletionTime = shoot.Status.Credentials.Rotation.Observability.LastCompletionTime.Time + } + return latestRotationCompletionTime.Before(now.Add(-period.Duration)) +} + +// ETCDEncryptionKeyRotationPassedRotationPeriod returns true if the ETCD encryption key rotation period has passed. +// If the credentials have never been rotated, the shoot's creation timestamp is used as the reference point. +func ETCDEncryptionKeyRotationPassedRotationPeriod(shoot *gardencorev1beta1.Shoot, now time.Time, period metav1.Duration) bool { + latestRotationCompletionTime := shoot.CreationTimestamp.Time + if shoot.Status.Credentials != nil && + shoot.Status.Credentials.Rotation != nil && + shoot.Status.Credentials.Rotation.ETCDEncryptionKey != nil && + shoot.Status.Credentials.Rotation.ETCDEncryptionKey.LastCompletionTime != nil { + latestRotationCompletionTime = shoot.Status.Credentials.Rotation.ETCDEncryptionKey.LastCompletionTime.Time + } + return latestRotationCompletionTime.Before(now.Add(-period.Duration)) +} + // GetEncryptionProviderType returns the encryption provider type. func GetEncryptionProviderType(apiServerConfig *gardencorev1beta1.KubeAPIServerConfig) gardencorev1beta1.EncryptionProviderType { if apiServerConfig != nil && diff --git a/pkg/api/core/v1beta1/helper/shoot_test.go b/pkg/api/core/v1beta1/helper/shoot_test.go index bd297ec1731..f3c94d690f6 100644 --- a/pkg/api/core/v1beta1/helper/shoot_test.go +++ b/pkg/api/core/v1beta1/helper/shoot_test.go @@ -409,6 +409,74 @@ var _ = Describe("Helper", func() { }, true), ) + Describe("rotation period passed", func() { + creationTimestamp := metav1.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + rotationPeriod := 2 * time.Hour + + DescribeTable("#SSHKeypairRotationPassedRotationPeriod", + func(credentials *gardencorev1beta1.ShootCredentials, now time.Time, expectedResult bool) { + shoot := &gardencorev1beta1.Shoot{ + ObjectMeta: metav1.ObjectMeta{CreationTimestamp: creationTimestamp}, + Status: gardencorev1beta1.ShootStatus{Credentials: credentials}, + } + Expect(SSHKeypairRotationPassedRotationPeriod(shoot, now, metav1.Duration{Duration: rotationPeriod})).To(Equal(expectedResult)) + }, + + Entry("should return false when the shoot age is less than rotation period", nil, + creationTimestamp.Add(time.Hour), false), + Entry("should use shoot creation time when rotation status is absent", nil, + creationTimestamp.Add(3*time.Hour), true), + Entry("should return false when the rotation period has not passed since the last completion time", + &gardencorev1beta1.ShootCredentials{Rotation: &gardencorev1beta1.ShootCredentialsRotation{SSHKeypair: &gardencorev1beta1.ShootSSHKeypairRotation{LastCompletionTime: new(metav1.NewTime(creationTimestamp.Add(2 * time.Hour)))}}}, + creationTimestamp.Add(3*time.Hour), false), + Entry("should return true when the rotation period has passed since the last completion time", + &gardencorev1beta1.ShootCredentials{Rotation: &gardencorev1beta1.ShootCredentialsRotation{SSHKeypair: &gardencorev1beta1.ShootSSHKeypairRotation{LastCompletionTime: new(metav1.NewTime(creationTimestamp.Add(2 * time.Hour)))}}}, + creationTimestamp.Add(5*time.Hour), true), + ) + + DescribeTable("#ObservabilityRotationPassedRotationPeriod", + func(credentials *gardencorev1beta1.ShootCredentials, now time.Time, expectedResult bool) { + shoot := &gardencorev1beta1.Shoot{ + ObjectMeta: metav1.ObjectMeta{CreationTimestamp: creationTimestamp}, + Status: gardencorev1beta1.ShootStatus{Credentials: credentials}, + } + Expect(ObservabilityRotationPassedRotationPeriod(shoot, now, metav1.Duration{Duration: rotationPeriod})).To(Equal(expectedResult)) + }, + + Entry("should return false when the shoot age is less than rotation period", nil, + creationTimestamp.Add(time.Hour), false), + Entry("should use shoot creation time when rotation status is absent", nil, + creationTimestamp.Add(3*time.Hour), true), + Entry("should return false when the rotation period has not passed since the last completion time", + &gardencorev1beta1.ShootCredentials{Rotation: &gardencorev1beta1.ShootCredentialsRotation{Observability: &gardencorev1beta1.ObservabilityRotation{LastCompletionTime: new(metav1.NewTime(creationTimestamp.Add(2 * time.Hour)))}}}, + creationTimestamp.Add(3*time.Hour), false), + Entry("should return true when the rotation period has passed since the last completion time", + &gardencorev1beta1.ShootCredentials{Rotation: &gardencorev1beta1.ShootCredentialsRotation{Observability: &gardencorev1beta1.ObservabilityRotation{LastCompletionTime: new(metav1.NewTime(creationTimestamp.Add(2 * time.Hour)))}}}, + creationTimestamp.Add(5*time.Hour), true), + ) + + DescribeTable("#ETCDEncryptionKeyRotationPassedRotationPeriod", + func(credentials *gardencorev1beta1.ShootCredentials, now time.Time, expectedResult bool) { + shoot := &gardencorev1beta1.Shoot{ + ObjectMeta: metav1.ObjectMeta{CreationTimestamp: creationTimestamp}, + Status: gardencorev1beta1.ShootStatus{Credentials: credentials}, + } + Expect(ETCDEncryptionKeyRotationPassedRotationPeriod(shoot, now, metav1.Duration{Duration: rotationPeriod})).To(Equal(expectedResult)) + }, + + Entry("should return false when the shoot age is less than rotation period", nil, + creationTimestamp.Add(time.Hour), false), + Entry("should use shoot creation time when rotation status is absent", nil, + creationTimestamp.Add(3*time.Hour), true), + Entry("should return false when the rotation period has not passed since the last completion time", + &gardencorev1beta1.ShootCredentials{Rotation: &gardencorev1beta1.ShootCredentialsRotation{ETCDEncryptionKey: &gardencorev1beta1.ETCDEncryptionKeyRotation{LastCompletionTime: new(metav1.NewTime(creationTimestamp.Add(2 * time.Hour)))}}}, + creationTimestamp.Add(3*time.Hour), false), + Entry("should return true when the rotation period has passed since the last completion time", + &gardencorev1beta1.ShootCredentials{Rotation: &gardencorev1beta1.ShootCredentialsRotation{ETCDEncryptionKey: &gardencorev1beta1.ETCDEncryptionKeyRotation{LastCompletionTime: new(metav1.NewTime(creationTimestamp.Add(2 * time.Hour)))}}}, + creationTimestamp.Add(5*time.Hour), true), + ) + }) + Describe("#IsMultiZonalShootControlPlane", func() { var shoot *gardencorev1beta1.Shoot diff --git a/pkg/apis/core/v1beta1/types_shoot.go b/pkg/apis/core/v1beta1/types_shoot.go index d956e3108d9..7529460cb40 100644 --- a/pkg/apis/core/v1beta1/types_shoot.go +++ b/pkg/apis/core/v1beta1/types_shoot.go @@ -2183,6 +2183,9 @@ const ( // in the Shoot's control plane namespace in the seed have been annotated with resources.gardener.cloud/ignore=true, // meaning their reconciliation has been disabled. Operators should be aware of such resources as they may diverge from the desired state. ShootHasIgnoredManagedResources ConditionType = "HasIgnoredManagedResources" + // ShootAutomaticCredentialsRotationPossible is a constant for a condition type indicating whether an automatic + // ETCD encryption key rotation can run during the next maintenance window. + ShootAutomaticCredentialsRotationPossible ConditionType = "AutomaticCredentialsRotationPossible" // ShootReadyForMigration is a constant for a condition type indicating whether the Shoot can be migrated. ShootReadyForMigration ConditionType = "ReadyForMigration" // ShootDualStackNodesMigrationReady is a constant for a condition type indicating whether all nodes are migrated to dual-stack . diff --git a/pkg/controllermanager/controller/shoot/hibernation/reconciler.go b/pkg/controllermanager/controller/shoot/hibernation/reconciler.go index cd9b9aed725..8d8537d89b2 100644 --- a/pkg/controllermanager/controller/shoot/hibernation/reconciler.go +++ b/pkg/controllermanager/controller/shoot/hibernation/reconciler.go @@ -10,7 +10,6 @@ import ( "slices" "time" - "github.com/robfig/cron" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -23,6 +22,7 @@ import ( controllermanagerconfigv1alpha1 "github.com/gardener/gardener/pkg/apis/config/controllermanager/v1alpha1" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" + hibernationutils "github.com/gardener/gardener/pkg/utils/hibernation" ) const ( @@ -30,42 +30,6 @@ const ( nextScheduleDelta = 100 * time.Millisecond ) -type operation uint8 - -const ( - hibernate operation = iota - wakeUp -) - -// parsedHibernationSchedule holds the loaded location, parsed cron schedule and information whether -// the cluster should be hibernated or woken up. -type parsedHibernationSchedule struct { - location time.Location - schedule cron.Schedule - operation operation -} - -// next returns the time in UTC from the schedule, that is immediately after the input time 't'. -// The input 't' is converted in the schedule's location before any calculations are done. -func (s *parsedHibernationSchedule) next(t time.Time) time.Time { - return s.schedule.Next(t.In(&s.location)).UTC() -} - -// previous returns the time in UTC from the schedule that is immediately before 'to' and after 'from'. -// Nil is returned if no such time can be found. -// The input times - 'to' and 'from' are converted in the schedule's location before any calculation is done. -func (s *parsedHibernationSchedule) previous(from, to time.Time) *time.Time { - // To get the time that is immediately before `to`, iterate over every activation time in the cron schedule - // that is after "from" until the one that is immediately after `to` is reached. - var previousActivationTime *time.Time - for t := s.schedule.Next(from.In(&s.location)); !t.UTC().After(to.UTC()); t = s.schedule.Next(t) { - inUTC := t.UTC() - previousActivationTime = &inUTC - } - - return previousActivationTime -} - // Reconciler reconciles Shoots and hibernates or wakes them up according to their hibernation schedules. type Reconciler struct { Client client.Client @@ -98,7 +62,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) ( return reconcile.Result{}, nil } - parsedSchedules, err := parseHibernationSchedules(schedules) + parsedSchedules, err := hibernationutils.Parse(schedules) if err != nil { log.Error(err, "Invalid hibernation schedules, stopping reconciliation") return reconcile.Result{}, nil @@ -127,13 +91,13 @@ func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) ( return reconcile.Result{RequeueAfter: requeueAfter}, nil } -func (r *Reconciler) hibernateOrWakeUpShootBasedOnSchedule(ctx context.Context, shoot *gardencorev1beta1.Shoot, schedule *parsedHibernationSchedule, now time.Time) error { +func (r *Reconciler) hibernateOrWakeUpShootBasedOnSchedule(ctx context.Context, shoot *gardencorev1beta1.Shoot, schedule *hibernationutils.ParsedSchedule, now time.Time) error { patch := client.MergeFrom(shoot.DeepCopy()) - switch schedule.operation { - case hibernate: + switch schedule.Operation { + case hibernationutils.Hibernate: shoot.Spec.Hibernation.Enabled = new(true) r.Recorder.Eventf(shoot, nil, corev1.EventTypeNormal, gardencorev1beta1.ShootEventHibernationEnabled, gardencorev1beta1.EventActionReconcile, "Hibernating cluster due to schedule") - case wakeUp: + case hibernationutils.WakeUp: shoot.Spec.Hibernation.Enabled = new(false) r.Recorder.Eventf(shoot, nil, corev1.EventTypeNormal, gardencorev1beta1.ShootEventHibernationDisabled, gardencorev1beta1.EventActionReconcile, "Waking up cluster due to schedule") } @@ -146,54 +110,14 @@ func (r *Reconciler) hibernateOrWakeUpShootBasedOnSchedule(ctx context.Context, return r.Client.Status().Patch(ctx, shoot, patch) } -// parseHibernationSchedules parses the given HibernationSchedules and returns an array of ParsedHibernationSchedules -// If the Location of a HibernationSchedule is `nil`, it is defaulted to UTC. -func parseHibernationSchedules(schedules []gardencorev1beta1.HibernationSchedule) ([]parsedHibernationSchedule, error) { - var parsedHibernationSchedules []parsedHibernationSchedule - - for _, schedule := range schedules { - locationID := time.UTC.String() - if schedule.Location != nil { - locationID = *schedule.Location - } - - location, err := time.LoadLocation(locationID) - if err != nil { - return nil, err - } - - if schedule.Start != nil { - parsed, err := cron.ParseStandard(*schedule.Start) - if err != nil { - return nil, err - } - parsedHibernationSchedules = append(parsedHibernationSchedules, - parsedHibernationSchedule{location: *location, schedule: parsed, operation: hibernate}, - ) - } - - if schedule.End != nil { - parsed, err := cron.ParseStandard(*schedule.End) - if err != nil { - return nil, err - } - parsedHibernationSchedules = append(parsedHibernationSchedules, - parsedHibernationSchedule{location: *location, schedule: parsed, operation: wakeUp}, - ) - } - } - - return parsedHibernationSchedules, nil -} - // nextHibernationTimeDuration returns the time duration after which to requeue the shoot based on the hibernation schedules and current time. // It adds a 100ms padding to the next requeue to account for Network Time Protocol(NTP) time skews. // If the time drifts are adjusted which in most realistic cases would be around 100ms, scheduled hibernation // will still be executed without missing the schedule. -func nextHibernationTimeDuration(schedules []parsedHibernationSchedule, now time.Time) time.Duration { +func nextHibernationTimeDuration(schedules []hibernationutils.ParsedSchedule, now time.Time) time.Duration { timeStamps := make([]time.Time, 0, len(schedules)) for _, schedule := range schedules { - timeStamps = append(timeStamps, schedule.next(now)) + timeStamps = append(timeStamps, schedule.Next(now)) } slices.SortFunc(timeStamps, func(a, b time.Time) int { @@ -203,8 +127,8 @@ func nextHibernationTimeDuration(schedules []parsedHibernationSchedule, now time return timeStamps[0].Add(nextScheduleDelta).Sub(now) } -// getScheduleWithMostRecentTime returns the ParsedHibernationSchedule that contains the schedule with the most recent (previous) execution time. -func getScheduleWithMostRecentTime(schedules []parsedHibernationSchedule, triggerDeadlineDuration *metav1.Duration, shoot *gardencorev1beta1.Shoot, now time.Time) *parsedHibernationSchedule { +// getScheduleWithMostRecentTime returns the ParsedSchedule that contains the schedule with the most recent (previous) execution time. +func getScheduleWithMostRecentTime(schedules []hibernationutils.ParsedSchedule, triggerDeadlineDuration *metav1.Duration, shoot *gardencorev1beta1.Shoot, now time.Time) *hibernationutils.ParsedSchedule { // If the shoot has just been created or has never been hibernated, use the creation timestamp. earliestTime := shoot.CreationTimestamp.Time if shoot.Status.LastHibernationTriggerTime != nil { @@ -225,9 +149,9 @@ func getScheduleWithMostRecentTime(schedules []parsedHibernationSchedule, trigge // Iterate over all schedules that were parsed from the shoot specification until we find one that contains // a time entry between `earliestTime` and `now`` and that time entry is the latest one (most recent) with respect to `now` - var scheduleWithMostRecentTime *parsedHibernationSchedule + var scheduleWithMostRecentTime *hibernationutils.ParsedSchedule for i := range schedules { - cur := schedules[i].previous(earliestTime, now) + cur := schedules[i].Previous(earliestTime, now) if cur == nil { continue } @@ -235,7 +159,7 @@ func getScheduleWithMostRecentTime(schedules []parsedHibernationSchedule, trigge scheduleWithMostRecentTime = &schedules[i] continue } - mostRecentTime := scheduleWithMostRecentTime.previous(earliestTime, now) + mostRecentTime := scheduleWithMostRecentTime.Previous(earliestTime, now) if mostRecentTime == nil { continue } diff --git a/pkg/controllermanager/controller/shoot/hibernation/reconciler_test.go b/pkg/controllermanager/controller/shoot/hibernation/reconciler_test.go index ad9814eea3b..a41b7c5f2e7 100644 --- a/pkg/controllermanager/controller/shoot/hibernation/reconciler_test.go +++ b/pkg/controllermanager/controller/shoot/hibernation/reconciler_test.go @@ -46,7 +46,6 @@ var _ = Describe("Shoot Hibernation", func() { locationEUSofia = "Europe/Sofia" weekDayAt2 = "2022-04-12T02:00:00Z" - weekDayAt0 = "2022-04-12T00:00:00Z" weekDayAt7 = "2022-04-12T07:00:00Z" weekDayAt19 = "2022-04-12T19:00:00Z" @@ -88,49 +87,6 @@ var _ = Describe("Shoot Hibernation", func() { } ) - Context("parsedHibernationSchedule", func() { - Describe("#next", func() { - It("should correctly return the next scheduling time from the parsed schedule", func() { - now := mustParseRFC3339Time(weekDayAt2) - expected := mustParseRFC3339Time(weekDayAt0).Add(24 * time.Hour) - - parsedSchedule := parsedHibernationSchedule{ - location: mustLoadLocation(locationEUBerlin), - schedule: mustParseStandard(everyDayAt2), - } - Expect(parsedSchedule.next(now)).To(Equal(expected)) - }) - }) - - Describe("#previous", func() { - It("should correctly return the previous scheduling time from the parsed schedule if it is within the specified range", func() { - now := mustParseRFC3339Time(weekDayAt2) - from := now.Add(-2 * 24 * time.Hour) - - expected := mustParseRFC3339Time(weekDayAt0) - parsedSchedule := parsedHibernationSchedule{ - location: mustLoadLocation(locationEUBerlin), - schedule: mustParseStandard(everyDayAt2), - } - prev := parsedSchedule.previous(from, now) - Expect(prev).NotTo(BeNil()) - Expect(*prev).To(Equal(expected)) - }) - - It("should return nil if previous scheduling time was not in specified range", func() { - now := mustParseRFC3339Time(weekDayAt2) - from := now.Add(-1 * time.Hour) - - parsedSchedule := parsedHibernationSchedule{ - location: mustLoadLocation(locationEUBerlin), - schedule: mustParseStandard(everyDayAt2), - } - prev := parsedSchedule.previous(from, now) - Expect(prev).To(BeNil()) - }) - }) - }) - Context("Shoot hibernation reconciliation", func() { Describe("#Reconcile", func() { var ( diff --git a/pkg/controllermanager/controller/shoot/maintenance/reconciler.go b/pkg/controllermanager/controller/shoot/maintenance/reconciler.go index b62c5ac90c9..2113a22bd9d 100644 --- a/pkg/controllermanager/controller/shoot/maintenance/reconciler.go +++ b/pkg/controllermanager/controller/shoot/maintenance/reconciler.go @@ -669,7 +669,7 @@ func computeCredentialsToRotationResults(log logr.Logger, shoot *gardencorev1bet ) if sshKeypairRotationEnabled && v1beta1helper.ShootEnablesSSHAccess(shoot) && - sshKeypairRotationPassedRotationPeriod(shoot, now.Time, *shoot.Spec.Maintenance.AutoRotation.Credentials.SSHKeypair.RotationPeriod) { + v1beta1helper.SSHKeypairRotationPassedRotationPeriod(shoot, now.Time, *shoot.Spec.Maintenance.AutoRotation.Credentials.SSHKeypair.RotationPeriod) { reason := "Automatic rotation of SSH keypair configured" log.Info("SSH keypair for workers will be rotated", "reason", reason) maintenanceResults[v1beta1constants.ShootOperationRotateSSHKeypair] = updateResult{ @@ -680,7 +680,7 @@ func computeCredentialsToRotationResults(log logr.Logger, shoot *gardencorev1bet } if observabilityPasswordsRotationEnabled && - observabilityPasswordsRotationPassedRotationPeriod(shoot, now.Time, *shoot.Spec.Maintenance.AutoRotation.Credentials.Observability.RotationPeriod) { + v1beta1helper.ObservabilityRotationPassedRotationPeriod(shoot, now.Time, *shoot.Spec.Maintenance.AutoRotation.Credentials.Observability.RotationPeriod) { reason := "Automatic rotation of observability passwords configured" log.Info("Observability passwords will be rotated", "reason", reason) maintenanceResults[v1beta1constants.OperationRotateObservabilityCredentials] = updateResult{ @@ -691,7 +691,8 @@ func computeCredentialsToRotationResults(log logr.Logger, shoot *gardencorev1bet } if etcdEncryptionKeyRotationEnabled && - etcdEncryptionKeyRotationPassedRotationPeriod(shoot, now.Time, *shoot.Spec.Maintenance.AutoRotation.Credentials.ETCDEncryptionKey.RotationPeriod) { + !v1beta1helper.HibernationIsEnabled(shoot) && // etcd encryption key rotation is not possible for hibernated shoots + v1beta1helper.ETCDEncryptionKeyRotationPassedRotationPeriod(shoot, now.Time, *shoot.Spec.Maintenance.AutoRotation.Credentials.ETCDEncryptionKey.RotationPeriod) { if len(etcdEncryptionKeyRotationPhase) == 0 || etcdEncryptionKeyRotationPhase == gardencorev1beta1.RotationCompleted { reason := "Automatic rotation of etcd encryption key configured" log.Info("ETCD Encryption key will be rotated", "reason", reason) @@ -713,51 +714,6 @@ func computeCredentialsToRotationResults(log logr.Logger, shoot *gardencorev1bet return maintenanceResults } -// sshKeypairRotationPassedRotationPeriod checks if the rotation period for ssh keypair has passed. -func sshKeypairRotationPassedRotationPeriod(shoot *gardencorev1beta1.Shoot, now time.Time, period metav1.Duration) bool { - // If the shoot has just been created or the credentials have never been rotated, use the shoot's creation timestamp to determine whether the rotation period has passed. - latestRotationCompletionTime := shoot.CreationTimestamp.Time - - if shoot.Status.Credentials != nil && - shoot.Status.Credentials.Rotation != nil && - shoot.Status.Credentials.Rotation.SSHKeypair != nil && - shoot.Status.Credentials.Rotation.SSHKeypair.LastCompletionTime != nil { - latestRotationCompletionTime = shoot.Status.Credentials.Rotation.SSHKeypair.LastCompletionTime.Time - } - - return latestRotationCompletionTime.Before(now.Add(-period.Duration)) -} - -// observabilityPasswordsRotationPassedRotationPeriod checks if the rotation period for observability passwords has passed. -func observabilityPasswordsRotationPassedRotationPeriod(shoot *gardencorev1beta1.Shoot, now time.Time, period metav1.Duration) bool { - // If the shoot has just been created or the credentials have never been rotated, use the shoot's creation timestamp to determine whether the rotation period has passed. - latestRotationCompletionTime := shoot.CreationTimestamp.Time - - if shoot.Status.Credentials != nil && - shoot.Status.Credentials.Rotation != nil && - shoot.Status.Credentials.Rotation.Observability != nil && - shoot.Status.Credentials.Rotation.Observability.LastCompletionTime != nil { - latestRotationCompletionTime = shoot.Status.Credentials.Rotation.Observability.LastCompletionTime.Time - } - - return latestRotationCompletionTime.Before(now.Add(-period.Duration)) -} - -// etcdEncryptionKeyRotationPassedRotationPeriod checks if the rotation period for the etcd encryption key has passed. -func etcdEncryptionKeyRotationPassedRotationPeriod(shoot *gardencorev1beta1.Shoot, now time.Time, period metav1.Duration) bool { - // If the shoot has just been created or the credentials have never been rotated, use the shoot's creation timestamp to determine whether the rotation period has passed. - latestRotationCompletionTime := shoot.CreationTimestamp.Time - - if shoot.Status.Credentials != nil && - shoot.Status.Credentials.Rotation != nil && - shoot.Status.Credentials.Rotation.ETCDEncryptionKey != nil && - shoot.Status.Credentials.Rotation.ETCDEncryptionKey.LastCompletionTime != nil { - latestRotationCompletionTime = shoot.Status.Credentials.Rotation.ETCDEncryptionKey.LastCompletionTime.Time - } - - return latestRotationCompletionTime.Before(now.Add(-period.Duration)) -} - func determineKubernetesVersion(kubernetesVersion string, profile *gardencorev1beta1.CloudProfile, isExpired bool) (string, error) { getHigherVersionAutoUpdate := v1beta1helper.GetLatestVersionForPatchAutoUpdate getHigherVersionForceUpdate := v1beta1helper.GetVersionForForcefulUpdateToConsecutiveMinor diff --git a/pkg/controllermanager/controller/shoot/maintenance/reconciler_test.go b/pkg/controllermanager/controller/shoot/maintenance/reconciler_test.go index 479fa33eb6f..a7303c444f2 100644 --- a/pkg/controllermanager/controller/shoot/maintenance/reconciler_test.go +++ b/pkg/controllermanager/controller/shoot/maintenance/reconciler_test.go @@ -133,18 +133,19 @@ var _ = Describe("Shoot Maintenance", func() { MachineImageVersion: new(true), }, }, - Provider: gardencorev1beta1.Provider{Workers: []gardencorev1beta1.Worker{ - { - Name: "cpu-worker", - Machine: gardencorev1beta1.Machine{ - Image: shootCurrentImage, - Architecture: new("amd64"), - Type: "someMachineType", + Provider: gardencorev1beta1.Provider{ + Workers: []gardencorev1beta1.Worker{ + { + Name: "cpu-worker", + Machine: gardencorev1beta1.Machine{ + Image: shootCurrentImage, + Architecture: new("amd64"), + Type: "someMachineType", + }, + UpdateStrategy: new(gardencorev1beta1.AutoRollingUpdate), }, - UpdateStrategy: new(gardencorev1beta1.AutoRollingUpdate), }, }, - }, }, } }) @@ -1015,7 +1016,8 @@ var _ = Describe("Shoot Maintenance", func() { "architecture": []string{v1beta1constants.ArchitectureAMD64}, "someCapability": []string{"value2"}, }, - }, { + }, + { Name: "anotherMachineType", Capabilities: gardencorev1beta1.Capabilities{ "architecture": []string{v1beta1constants.ArchitectureAMD64}, @@ -1274,7 +1276,6 @@ var _ = Describe("Shoot Maintenance", func() { _, err := maintainMachineImages(log, shoot, cloudProfile) Expect(err).To(HaveOccurred()) - }) It("should return an error - cloud profile has no matching (machineImage.type) machine type defined", func() { @@ -1656,6 +1657,17 @@ var _ = Describe("Shoot Maintenance", func() { })) }) + It("should not attempt etcd encryption key rotation when shoot is hibernated", func() { + shoot.Spec.Maintenance.AutoRotation.Credentials.SSHKeypair.RotationPeriod.Duration = 0 + shoot.Spec.Maintenance.AutoRotation.Credentials.Observability.RotationPeriod.Duration = 0 + shoot.Spec.Hibernation = &gardencorev1beta1.Hibernation{ + Enabled: new(true), + } + results := computeCredentialsToRotationResults(log, shoot, metav1.Time{Time: now}) + + Expect(results).To(BeEmpty()) + }) + It("should not return results when the rotation period has not passed", func() { shoot.CreationTimestamp = metav1.Time{Time: now.Add(-48 * time.Hour)} shoot.Status.Credentials = &gardencorev1beta1.ShootCredentials{ @@ -1676,7 +1688,7 @@ var _ = Describe("Shoot Maintenance", func() { Expect(results).To(BeEmpty()) }) - It("should not return results when Shoow is newly created", func() { + It("should not return results when Shoot is newly created", func() { shoot.CreationTimestamp = metav1.Time{Time: now} shoot.Status.Credentials = nil results := computeCredentialsToRotationResults(log, shoot, metav1.Time{Time: now}) @@ -2004,9 +2016,7 @@ var _ = Describe("Shoot Maintenance", func() { }) Describe("#maintainAddons", func() { - var ( - shoot *gardencorev1beta1.Shoot - ) + var shoot *gardencorev1beta1.Shoot BeforeEach(func() { shoot = &gardencorev1beta1.Shoot{ diff --git a/pkg/gardenlet/controller/shoot/care/constraints.go b/pkg/gardenlet/controller/shoot/care/constraints.go index be0b7b42d8d..e0d8cde8847 100644 --- a/pkg/gardenlet/controller/shoot/care/constraints.go +++ b/pkg/gardenlet/controller/shoot/care/constraints.go @@ -63,6 +63,15 @@ func shootHibernatedConstraints(clock clock.Clock, conditions ...gardencorev1bet } continue } + // Optional constraint computed before the hibernation guard. + // Only preserve it if it's non-True (i.e. the configuration is problematic). + // When True, drop it — consistent with filterOptionalConstraints behaviour. + if cond.Type == gardencorev1beta1.ShootAutomaticCredentialsRotationPossible { + if cond.Status != gardencorev1beta1.ConditionTrue { + hibernationConditions = append(hibernationConditions, cond) + } + continue + } hibernationConditions = append(hibernationConditions, v1beta1helper.UpdatedConditionWithClock(clock, cond, gardencorev1beta1.ConditionTrue, "ConstraintNotChecked", "Shoot cluster has been hibernated.")) } return hibernationConditions @@ -135,6 +144,9 @@ func (c *Constraint) constraintsChecks( constraints.preservedFailedMachinesAbsent = v1beta1helper.UpdatedConditionWithClock(c.clock, constraints.preservedFailedMachinesAbsent, status, reason, message) } + status, reason, message = c.checkIfAutomaticCredentialsRotationPossible() + constraints.automaticCredentialsRotationPossible = v1beta1helper.UpdatedConditionWithClock(c.clock, constraints.automaticCredentialsRotationPossible, status, reason, message) + if c.shoot.HibernationEnabled || c.shoot.GetInfo().Status.IsHibernated { return shootHibernatedConstraints(c.clock, constraints.ConvertToSlice()...) } @@ -161,14 +173,14 @@ func (c *Constraint) constraintsChecks( return filterOptionalConstraints( []gardencorev1beta1.Condition{constraints.hibernationPossible, constraints.maintenancePreconditionsSatisfied}, - []gardencorev1beta1.Condition{constraints.caCertificateValiditiesAcceptable, constraints.manualInPlaceWorkersUpdated, constraints.hasIgnoredManagedResources, constraints.preservedFailedMachinesAbsent}, + []gardencorev1beta1.Condition{constraints.caCertificateValiditiesAcceptable, constraints.manualInPlaceWorkersUpdated, constraints.hasIgnoredManagedResources, constraints.preservedFailedMachinesAbsent, constraints.automaticCredentialsRotationPossible}, ) } if !apiServerRunning { // don't check constraints if API server has already been deleted or has not been created yet return filterOptionalConstraints( shootControlPlaneNotRunningConstraints(c.clock, constraints.hibernationPossible, constraints.maintenancePreconditionsSatisfied), - []gardencorev1beta1.Condition{constraints.caCertificateValiditiesAcceptable, constraints.manualInPlaceWorkersUpdated, constraints.hasIgnoredManagedResources, constraints.preservedFailedMachinesAbsent}, + []gardencorev1beta1.Condition{constraints.caCertificateValiditiesAcceptable, constraints.manualInPlaceWorkersUpdated, constraints.hasIgnoredManagedResources, constraints.preservedFailedMachinesAbsent, constraints.automaticCredentialsRotationPossible}, ) } c.shootClient = shootClient.Client() @@ -191,7 +203,7 @@ func (c *Constraint) constraintsChecks( return filterOptionalConstraints( []gardencorev1beta1.Condition{constraints.hibernationPossible, constraints.maintenancePreconditionsSatisfied}, - []gardencorev1beta1.Condition{constraints.caCertificateValiditiesAcceptable, constraints.crdsWithProblematicConversionWebhooks, constraints.manualInPlaceWorkersUpdated, constraints.hasIgnoredManagedResources, constraints.preservedFailedMachinesAbsent}, + []gardencorev1beta1.Condition{constraints.caCertificateValiditiesAcceptable, constraints.crdsWithProblematicConversionWebhooks, constraints.manualInPlaceWorkersUpdated, constraints.hasIgnoredManagedResources, constraints.preservedFailedMachinesAbsent, constraints.automaticCredentialsRotationPossible}, ) } @@ -509,6 +521,36 @@ func wasRemediatedByGardener(annotations map[string]string) bool { return annotations[v1beta1constants.GardenerWarning] != "" } +func (c *Constraint) checkIfAutomaticCredentialsRotationPossible() (gardencorev1beta1.ConditionStatus, string, string) { + shoot := c.shoot.GetInfo() + + if !v1beta1helper.IsETCDEncryptionKeyAutoRotationEnabled(shoot) { + return gardencorev1beta1.ConditionTrue, + "AutomaticCredentialsRotationNotRequired", + "Automatic ETCD encryption key rotation is not enabled." + } + + rotationPeriod := *shoot.Spec.Maintenance.AutoRotation.Credentials.ETCDEncryptionKey.RotationPeriod + if !v1beta1helper.ETCDEncryptionKeyRotationPassedRotationPeriod(shoot, c.clock.Now(), rotationPeriod) { + return gardencorev1beta1.ConditionTrue, + "AutomaticCredentialsRotationNotRequired", + "The ETCD encryption key rotation period has not yet passed." + } + + if IsShootHibernatedDuringNextMaintenanceWindow(shoot, c.clock.Now()) { + return gardencorev1beta1.ConditionFalse, + "MaintenanceWindowDuringHibernation", + "The ETCD encryption key rotation is overdue, " + + "but the next maintenance window falls within a hibernation interval. " + + "The ETCD encryption key auto-rotation cannot be triggered automatically. " + + "Please adjust the maintenance window or the hibernation schedule so that maintenance can run " + + "while the cluster is awake." + } + return gardencorev1beta1.ConditionTrue, + "AutomaticCredentialsRotationPossible", + "The ETCD encryption key rotation is overdue but the next maintenance window is not within a hibernation interval." +} + func filterOptionalConstraints(required, optional []gardencorev1beta1.Condition) []gardencorev1beta1.Condition { var out []gardencorev1beta1.Condition out = append(out, required...) @@ -529,6 +571,7 @@ type ShootConstraints struct { caCertificateValiditiesAcceptable gardencorev1beta1.Condition crdsWithProblematicConversionWebhooks gardencorev1beta1.Condition manualInPlaceWorkersUpdated gardencorev1beta1.Condition + automaticCredentialsRotationPossible gardencorev1beta1.Condition hasIgnoredManagedResources gardencorev1beta1.Condition preservedFailedMachinesAbsent gardencorev1beta1.Condition } @@ -541,6 +584,7 @@ func (g ShootConstraints) ConvertToSlice() []gardencorev1beta1.Condition { g.caCertificateValiditiesAcceptable, g.crdsWithProblematicConversionWebhooks, g.manualInPlaceWorkersUpdated, + g.automaticCredentialsRotationPossible, g.hasIgnoredManagedResources, g.preservedFailedMachinesAbsent, } @@ -554,6 +598,7 @@ func (g ShootConstraints) ConstraintTypes() []gardencorev1beta1.ConditionType { g.caCertificateValiditiesAcceptable.Type, g.crdsWithProblematicConversionWebhooks.Type, g.manualInPlaceWorkersUpdated.Type, + g.automaticCredentialsRotationPossible.Type, g.hasIgnoredManagedResources.Type, g.preservedFailedMachinesAbsent.Type, } @@ -568,6 +613,7 @@ func NewShootConstraints(clock clock.Clock, shoot *gardencorev1beta1.Shoot) Shoo caCertificateValiditiesAcceptable: v1beta1helper.GetOrInitConditionWithClock(clock, shoot.Status.Constraints, gardencorev1beta1.ShootCACertificateValiditiesAcceptable), crdsWithProblematicConversionWebhooks: v1beta1helper.GetOrInitConditionWithClock(clock, shoot.Status.Constraints, gardencorev1beta1.ShootCRDsWithProblematicConversionWebhooks), manualInPlaceWorkersUpdated: v1beta1helper.GetOrInitConditionWithClock(clock, shoot.Status.Constraints, gardencorev1beta1.ShootManualInPlaceWorkersUpdated), + automaticCredentialsRotationPossible: v1beta1helper.GetOrInitConditionWithClock(clock, shoot.Status.Constraints, gardencorev1beta1.ShootAutomaticCredentialsRotationPossible), hasIgnoredManagedResources: v1beta1helper.GetOrInitConditionWithClock(clock, shoot.Status.Constraints, gardencorev1beta1.ShootHasIgnoredManagedResources), preservedFailedMachinesAbsent: v1beta1helper.GetOrInitConditionWithClock(clock, shoot.Status.Constraints, gardencorev1beta1.ShootPreservedFailedMachinesAbsent), } diff --git a/pkg/gardenlet/controller/shoot/care/constraints_test.go b/pkg/gardenlet/controller/shoot/care/constraints_test.go index 992440f7ff1..53d2e77735a 100644 --- a/pkg/gardenlet/controller/shoot/care/constraints_test.go +++ b/pkg/gardenlet/controller/shoot/care/constraints_test.go @@ -11,7 +11,6 @@ import ( "time" machinev1alpha1 "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" - "github.com/go-logr/logr" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" @@ -75,8 +74,8 @@ func (w *webhookTestCase) build() ( APIGroups: []string{w.gvr.Group}, Resources: []string{w.gvr.Resource}, APIVersions: []string{w.gvr.Version}, - }}, - } + }, + }} opType := admissionregistrationv1.OperationAll if w.operationType != nil { @@ -113,15 +112,18 @@ var _ = Describe("Constraints", func() { kubeSystemNamespaceProblematic = []TableEntry{ Entry("namespaceSelector matching no-cleanup", webhookTestCase{ namespaceSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"shoot.gardener.cloud/no-cleanup": "true"}}, + MatchLabels: map[string]string{"shoot.gardener.cloud/no-cleanup": "true"}, + }, }), Entry("namespaceSelector matching purpose", webhookTestCase{ namespaceSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"gardener.cloud/purpose": "kube-system"}}, + MatchLabels: map[string]string{"gardener.cloud/purpose": "kube-system"}, + }, }), Entry("namespaceSelector matching name label", webhookTestCase{ namespaceSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"kubernetes.io/metadata.name": "kube-system"}}, + MatchLabels: map[string]string{"kubernetes.io/metadata.name": "kube-system"}, + }, }), Entry("namespaceSelector matching all gardener labels", webhookTestCase{ namespaceSelector: &metav1.LabelSelector{ @@ -129,14 +131,16 @@ var _ = Describe("Constraints", func() { "shoot.gardener.cloud/no-cleanup": "true", "gardener.cloud/purpose": "kube-system", "kubernetes.io/metadata.name": "kube-system", - }}, + }, + }, }), } kubeSystemNamespaceNotProblematic = []TableEntry{ Entry("not matching namespaceSelector", webhookTestCase{ namespaceSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"foo": "bar"}}, + MatchLabels: map[string]string{"foo": "bar"}, + }, }), Entry("namespaceSelector excluding name label", webhookTestCase{ namespaceSelector: &metav1.LabelSelector{ @@ -160,7 +164,8 @@ var _ = Describe("Constraints", func() { Entry("failurePolicy 'Ignore' and timeoutSeconds ok", webhookTestCase{failurePolicy: &failurePolicyIgnore, timeoutSeconds: &timeoutSecondsNotProblematic, objectSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "app.kubernetes.io/name": "test", - }}}), + }, + }}), Entry("failurePolicy 'Ignore' and timeoutSeconds ok", webhookTestCase{failurePolicy: &failurePolicyIgnore, timeoutSeconds: &timeoutSecondsNotProblematic})) } @@ -204,30 +209,35 @@ var _ = Describe("Constraints", func() { commonTests(gvr, append(kubeSystemNamespaceProblematic, Entry("objectSelector matching no-cleanup", webhookTestCase{ objectSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"shoot.gardener.cloud/no-cleanup": "true"}}, + MatchLabels: map[string]string{"shoot.gardener.cloud/no-cleanup": "true"}, + }, }), Entry("objectSelector matching origin", webhookTestCase{ objectSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"origin": "gardener"}}, + MatchLabels: map[string]string{"origin": "gardener"}, + }, }), Entry("objectSelector matching all gardener labels", webhookTestCase{ objectSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "shoot.gardener.cloud/no-cleanup": "true", "origin": "gardener", - }}, + }, + }, }), Entry("objectSelector and namespaceSelector matching all gardener labels", webhookTestCase{ objectSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "shoot.gardener.cloud/no-cleanup": "true", "origin": "gardener", - }}, + }, + }, namespaceSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "shoot.gardener.cloud/no-cleanup": "true", "gardener.cloud/purpose": "kube-system", - }}, + }, + }, }), ), append(kubeSystemNamespaceNotProblematic, Entry("matching objectSelector, not matching namespaceSelector", webhookTestCase{ @@ -235,22 +245,27 @@ var _ = Describe("Constraints", func() { MatchLabels: map[string]string{ "origin": "gardener", "shoot.gardener.cloud/no-cleanup": "true", - }}, + }, + }, namespaceSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"foo": "bar"}}, + MatchLabels: map[string]string{"foo": "bar"}, + }, }), Entry("not matching objectSelector", webhookTestCase{ objectSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"foo": "bar"}}, + MatchLabels: map[string]string{"foo": "bar"}, + }, }), Entry("matching namespaceSelector, not matching objectSelector", webhookTestCase{ objectSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"foo": "bar"}}, + MatchLabels: map[string]string{"foo": "bar"}, + }, namespaceSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "shoot.gardener.cloud/no-cleanup": "true", "gardener.cloud/purpose": "kube-system", - }}, + }, + }, }), )) } @@ -260,11 +275,13 @@ var _ = Describe("Constraints", func() { problematic = []TableEntry{ Entry("namespaceSelector matching purpose", webhookTestCase{ namespaceSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"gardener.cloud/purpose": "kube-system"}}, + MatchLabels: map[string]string{"gardener.cloud/purpose": "kube-system"}, + }, }), Entry("objectSelector matching purpose", webhookTestCase{ objectSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"gardener.cloud/purpose": "kube-system"}}, + MatchLabels: map[string]string{"gardener.cloud/purpose": "kube-system"}, + }, }), } notProblematic = []TableEntry{ @@ -277,7 +294,8 @@ var _ = Describe("Constraints", func() { }), Entry("not matching namespaceSelector", webhookTestCase{ namespaceSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"foo": "bar"}}, + MatchLabels: map[string]string{"foo": "bar"}, + }, }), Entry("objectSelector not matching purpose", webhookTestCase{ objectSelector: &metav1.LabelSelector{ @@ -288,19 +306,24 @@ var _ = Describe("Constraints", func() { }), Entry("not matching objectSelector", webhookTestCase{ objectSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"foo": "bar"}}, + MatchLabels: map[string]string{"foo": "bar"}, + }, }), Entry("matching objectSelector, not matching namespaceSelector", webhookTestCase{ objectSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"gardener.cloud/purpose": "kube-system"}}, + MatchLabels: map[string]string{"gardener.cloud/purpose": "kube-system"}, + }, namespaceSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"foo": "bar"}}, + MatchLabels: map[string]string{"foo": "bar"}, + }, }), Entry("matching namespaceSelector, not matching objectSelector", webhookTestCase{ objectSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"foo": "bar"}}, + MatchLabels: map[string]string{"foo": "bar"}, + }, namespaceSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{"gardener.cloud/purpose": "kube-system"}}, + MatchLabels: map[string]string{"gardener.cloud/purpose": "kube-system"}, + }, }), } ) @@ -476,7 +499,7 @@ var _ = Describe("Constraints", func() { shoot.SetInfo(&gardencorev1beta1.Shoot{}) constraint = NewConstraint( - logr.Discard(), + GinkgoLogr, shoot, seedClient, func() (kubernetes.Interface, bool, error) { @@ -495,6 +518,7 @@ var _ = Describe("Constraints", func() { {Type: gardencorev1beta1.ShootMaintenancePreconditionsSatisfied}, {Type: gardencorev1beta1.ShootCRDsWithProblematicConversionWebhooks}, {Type: gardencorev1beta1.ShootManualInPlaceWorkersUpdated}, + {Type: gardencorev1beta1.ShootAutomaticCredentialsRotationPossible}, {Type: gardencorev1beta1.ShootHasIgnoredManagedResources}, {Type: gardencorev1beta1.ShootPreservedFailedMachinesAbsent}, }, @@ -583,7 +607,7 @@ var _ = Describe("Constraints", func() { shootPkg.SetInfo(shoot) constraint = NewConstraint( - logr.Discard(), + GinkgoLogr, shootPkg, seedClient, func() (kubernetes.Interface, bool, error) { @@ -602,7 +626,7 @@ var _ = Describe("Constraints", func() { shootPkg.SetInfo(shoot) constraint = NewConstraint( - logr.Discard(), + GinkgoLogr, shootPkg, seedClient, func() (kubernetes.Interface, bool, error) { @@ -625,7 +649,7 @@ var _ = Describe("Constraints", func() { shootPkg.SetInfo(shoot) constraint = NewConstraint( - logr.Discard(), + GinkgoLogr, shootPkg, seedClient, func() (kubernetes.Interface, bool, error) { @@ -660,7 +684,7 @@ var _ = Describe("Constraints", func() { shootPkg.SetInfo(shoot) constraint = NewConstraint( - logr.Discard(), + GinkgoLogr, shootPkg, seedClient, func() (kubernetes.Interface, bool, error) { @@ -678,6 +702,121 @@ var _ = Describe("Constraints", func() { }) }) + Context("#AutomaticCredentialsRotationPossible", func() { + BeforeEach(func() { + shoot = &gardencorev1beta1.Shoot{ + ObjectMeta: metav1.ObjectMeta{ + CreationTimestamp: metav1.NewTime(clock.Now().Add(-48 * time.Hour)), + }, + Spec: gardencorev1beta1.ShootSpec{ + Maintenance: &gardencorev1beta1.Maintenance{ + AutoRotation: &gardencorev1beta1.MaintenanceAutoRotation{ + Credentials: &gardencorev1beta1.MaintenanceCredentialsAutoRotation{ + ETCDEncryptionKey: &gardencorev1beta1.MaintenanceRotationConfig{ + RotationPeriod: &metav1.Duration{Duration: 24 * time.Hour}, + }, + }, + }, + }, + }, + } + }) + + JustBeforeEach(func() { + shootPkg := &shootpkg.Shoot{ControlPlaneNamespace: controlPlaneNamespace} + shootPkg.SetInfo(shoot) + constraint = NewConstraint(GinkgoLogr, shootPkg, seedClient, + func() (kubernetes.Interface, bool, error) { + return fakekubernetes.NewClientSetBuilder().WithClient(shootClient).Build(), true, nil + }, clock) + }) + + It("should remove the constraint when shoot has no hibernation schedule", func() { + Expect(constraint.Check(ctx, constraints)).NotTo(ContainCondition( + OfType(gardencorev1beta1.ShootAutomaticCredentialsRotationPossible), + )) + }) + + It("should keep the constraint when the maintenance window is within the hibernation window", func() { + shoot.Spec.Maintenance.TimeWindow = &gardencorev1beta1.MaintenanceTimeWindow{ + Begin: "010000+0000", + End: "020000+0000", + } + shoot.Spec.Hibernation = &gardencorev1beta1.Hibernation{ + Schedules: []gardencorev1beta1.HibernationSchedule{ + {Start: new("0 0 * * *"), End: new("0 8 * * *")}, + }, + } + + Expect(constraint.Check(ctx, constraints)).To(ContainCondition( + OfType(gardencorev1beta1.ShootAutomaticCredentialsRotationPossible), + WithStatus(gardencorev1beta1.ConditionProgressing), + WithReason("MaintenanceWindowDuringHibernation"), + )) + }) + + It("should remove the constraint when the maintenance window is outside the hibernation window", func() { + shoot.Spec.Maintenance = &gardencorev1beta1.Maintenance{ + TimeWindow: &gardencorev1beta1.MaintenanceTimeWindow{ + Begin: "100000+0000", + End: "110000+0000", + }, + } + shoot.Spec.Hibernation = &gardencorev1beta1.Hibernation{ + Schedules: []gardencorev1beta1.HibernationSchedule{ + {Start: new("0 0 * * *"), End: new("0 8 * * *")}, + }, + } + + Expect(constraint.Check(ctx, constraints)).NotTo(ContainCondition( + OfType(gardencorev1beta1.ShootAutomaticCredentialsRotationPossible), + )) + }) + + Context("when shoot is hibernated", func() { + JustBeforeEach(func() { + shootPkg := &shootpkg.Shoot{ + ControlPlaneNamespace: controlPlaneNamespace, + HibernationEnabled: true, + } + shoot.Status.IsHibernated = true + shootPkg.SetInfo(shoot) + constraint = NewConstraint(GinkgoLogr, shootPkg, seedClient, + func() (kubernetes.Interface, bool, error) { + return fakekubernetes.NewClientSetBuilder().WithClient(shootClient).Build(), true, nil + }, clock) + constraints = NewShootConstraints(testclock.NewFakeClock(time.Time{}), shootPkg.GetInfo()) + }) + + It("should remove the constraint when there is no problematic schedule", func() { + shoot.Status.Constraints = []gardencorev1beta1.Condition{ + {Type: gardencorev1beta1.ShootAutomaticCredentialsRotationPossible, Status: gardencorev1beta1.ConditionTrue}, + } + + Expect(constraint.Check(ctx, constraints)).NotTo(ContainCondition( + OfType(gardencorev1beta1.ShootAutomaticCredentialsRotationPossible), + )) + }) + + It("should preserve the constraint when the configuration is problematic", func() { + shoot.Spec.Maintenance.TimeWindow = &gardencorev1beta1.MaintenanceTimeWindow{ + Begin: "010000+0000", + End: "020000+0000", + } + shoot.Spec.Hibernation = &gardencorev1beta1.Hibernation{ + Schedules: []gardencorev1beta1.HibernationSchedule{ + {Start: new("0 0 * * *"), End: new("0 8 * * *")}, + }, + } + + Expect(constraint.Check(ctx, constraints)).To(ContainCondition( + OfType(gardencorev1beta1.ShootAutomaticCredentialsRotationPossible), + WithReason("MaintenanceWindowDuringHibernation"), + )) + }) + }) + }) + Context("#HasIgnoredManagedResources", func() { It("should remove the constraint when no ManagedResources exist", func() { Expect(constraint.Check(ctx, constraints)).NotTo(ContainCondition( @@ -735,7 +874,7 @@ var _ = Describe("Constraints", func() { shootPkg.SetInfo(shoot) hibernatedConstraint = NewConstraint( - logr.Discard(), + GinkgoLogr, shootPkg, seedClient, func() (kubernetes.Interface, bool, error) { @@ -793,7 +932,7 @@ var _ = Describe("Constraints", func() { }, }) constraint = NewConstraint( - logr.Discard(), + GinkgoLogr, shootPkg, seedClient, func() (kubernetes.Interface, bool, error) { @@ -923,6 +1062,7 @@ var _ = Describe("Constraints", func() { beConditionWithStatusAndMsg("Unknown", "ConditionInitialized", "The condition has been initialized but its semantic check has not been performed yet."), beConditionWithStatusAndMsg("Unknown", "ConditionInitialized", "The condition has been initialized but its semantic check has not been performed yet."), beConditionWithStatusAndMsg("Unknown", "ConditionInitialized", "The condition has been initialized but its semantic check has not been performed yet."), + beConditionWithStatusAndMsg("Unknown", "ConditionInitialized", "The condition has been initialized but its semantic check has not been performed yet."), )) }) @@ -945,6 +1085,7 @@ var _ = Describe("Constraints", func() { beConditionWithStatusAndMsg("Unknown", "ConditionInitialized", "The condition has been initialized but its semantic check has not been performed yet."), beConditionWithStatusAndMsg("Unknown", "ConditionInitialized", "The condition has been initialized but its semantic check has not been performed yet."), beConditionWithStatusAndMsg("Unknown", "ConditionInitialized", "The condition has been initialized but its semantic check has not been performed yet."), + beConditionWithStatusAndMsg("Unknown", "ConditionInitialized", "The condition has been initialized but its semantic check has not been performed yet."), )) }) }) @@ -959,6 +1100,7 @@ var _ = Describe("Constraints", func() { OfType("CACertificateValiditiesAcceptable"), OfType("CRDsWithProblematicConversionWebhooks"), OfType("ManualInPlaceWorkersUpdated"), + OfType("AutomaticCredentialsRotationPossible"), OfType("HasIgnoredManagedResources"), OfType("PreservedFailedMachinesAbsent"), )) @@ -975,6 +1117,7 @@ var _ = Describe("Constraints", func() { gardencorev1beta1.ConditionType("CACertificateValiditiesAcceptable"), gardencorev1beta1.ConditionType("CRDsWithProblematicConversionWebhooks"), gardencorev1beta1.ConditionType("ManualInPlaceWorkersUpdated"), + gardencorev1beta1.ConditionType("AutomaticCredentialsRotationPossible"), gardencorev1beta1.ConditionType("HasIgnoredManagedResources"), gardencorev1beta1.ConditionType("PreservedFailedMachinesAbsent"), )) diff --git a/pkg/gardenlet/controller/shoot/care/hibernation.go b/pkg/gardenlet/controller/shoot/care/hibernation.go new file mode 100644 index 00000000000..ea4bac76d11 --- /dev/null +++ b/pkg/gardenlet/controller/shoot/care/hibernation.go @@ -0,0 +1,85 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package care + +import ( + "time" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + timewindowutils "github.com/gardener/gardener/pkg/apis/utils/timewindow" + hibernationutils "github.com/gardener/gardener/pkg/utils/hibernation" +) + +// IsShootHibernatedDuringNextMaintenanceWindow reports whether the shoot could be in a hibernated state at any point +// during its next maintenance window. Since Gardener only guarantees that maintenance starts at some point within the +// window (not at its beginning), we compare against the end of the window — if a hibernation event falls before the +// window closes, maintenance may never run. It only checks the very next hibernation and wake-up events, so it may +// return false even if the shoot will be hibernated during the next maintenance window. +func IsShootHibernatedDuringNextMaintenanceWindow(shoot *gardencorev1beta1.Shoot, now time.Time) bool { + if shoot.Spec.Maintenance == nil || shoot.Spec.Maintenance.TimeWindow == nil { + return false + } + + maintenanceWindow, err := timewindowutils.ParseMaintenanceTimeWindow(shoot.Spec.Maintenance.TimeWindow.Begin, shoot.Spec.Maintenance.TimeWindow.End) + if err != nil { + return false + } + nextMaintenanceBegin := maintenanceWindow.AdjustedBegin(now) + if !nextMaintenanceBegin.After(now) { + nextMaintenanceBegin = nextMaintenanceBegin.AddDate(0, 0, 1) + } + nextMaintenanceEnd := maintenanceWindow.AdjustedEnd(now) + if !nextMaintenanceEnd.After(nextMaintenanceBegin) { + nextMaintenanceEnd = nextMaintenanceEnd.AddDate(0, 0, 1) + } + + nextHibernateTime, nextWakeUpTime, err := parseNextHibernationEvents(shoot, now) + if err != nil { + return false + } + + if shoot.Status.IsHibernated { + if nextWakeUpTime == nil || nextWakeUpTime.After(nextMaintenanceBegin) { + return true // never wakes up, or wakes up too late + } + + return nextHibernateTime != nil && + nextHibernateTime.After(*nextWakeUpTime) && + nextHibernateTime.Before(nextMaintenanceEnd) + } + + if nextHibernateTime == nil || nextHibernateTime.After(nextMaintenanceEnd) { + return false // stays awake all the way to maintenance + } + + return nextWakeUpTime == nil || + nextWakeUpTime.Before(*nextHibernateTime) || + nextWakeUpTime.After(nextMaintenanceBegin) +} + +func parseNextHibernationEvents(shoot *gardencorev1beta1.Shoot, now time.Time) (nextHibernateTime *time.Time, nextWakeUpTime *time.Time, err error) { + if shoot.Spec.Hibernation == nil || len(shoot.Spec.Hibernation.Schedules) == 0 { + return nil, nil, nil + } + schedules, err := hibernationutils.Parse(shoot.Spec.Hibernation.Schedules) + if err != nil { + return nil, nil, err + } + + for _, s := range schedules { + t := s.Next(now) + switch s.Operation { + case hibernationutils.Hibernate: + if nextHibernateTime == nil || t.Before(*nextHibernateTime) { + nextHibernateTime = &t + } + case hibernationutils.WakeUp: + if nextWakeUpTime == nil || t.Before(*nextWakeUpTime) { + nextWakeUpTime = &t + } + } + } + return nextHibernateTime, nextWakeUpTime, nil +} diff --git a/pkg/gardenlet/controller/shoot/care/hibernation_test.go b/pkg/gardenlet/controller/shoot/care/hibernation_test.go new file mode 100644 index 00000000000..252b9dab705 --- /dev/null +++ b/pkg/gardenlet/controller/shoot/care/hibernation_test.go @@ -0,0 +1,178 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package care_test + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + . "github.com/gardener/gardener/pkg/gardenlet/controller/shoot/care" +) + +var _ = Describe("#IsShootHibernatedDuringNextMaintenanceWindow", func() { + var ( + now = time.Date(2024, time.January, 8, 15, 0, 0, 0, time.UTC) + + // nowHibernated is inside a typical overnight hibernation window: + nowHibernated = time.Date(2024, time.January, 8, 21, 0, 0, 0, time.UTC) + + nightlySchedule = gardencorev1beta1.HibernationSchedule{Start: new("0 20 * * *"), End: new("0 8 * * *")} + weekdaySchedule = gardencorev1beta1.HibernationSchedule{Start: new("0 20 * * 1-5"), End: new("0 8 * * 1-5")} + ) + + const ( + // maintenance window 22:00–23:00 UTC (nightly) + maintBegin = "220000+0000" + maintEnd = "230000+0000" + ) + + makeShoot := func(begin, end string, isHibernated bool, schedules ...gardencorev1beta1.HibernationSchedule) *gardencorev1beta1.Shoot { + shoot := &gardencorev1beta1.Shoot{ + Spec: gardencorev1beta1.ShootSpec{ + Maintenance: &gardencorev1beta1.Maintenance{ + TimeWindow: &gardencorev1beta1.MaintenanceTimeWindow{ + Begin: begin, + End: end, + }, + }, + }, + Status: gardencorev1beta1.ShootStatus{ + IsHibernated: isHibernated, + }, + } + if len(schedules) > 0 { + shoot.Spec.Hibernation = &gardencorev1beta1.Hibernation{ + Schedules: schedules, + } + } + return shoot + } + + makeShootDefault := func(isHibernated bool, schedules ...gardencorev1beta1.HibernationSchedule) *gardencorev1beta1.Shoot { + return makeShoot(maintBegin, maintEnd, isHibernated, schedules...) + } + + DescribeTable("should return the expected result", + func(shoot *gardencorev1beta1.Shoot, t time.Time, expected bool) { + Expect(IsShootHibernatedDuringNextMaintenanceWindow(shoot, t)).To(Equal(expected)) + }, + + Entry("no maintenance time window set", + &gardencorev1beta1.Shoot{}, now, + false, + ), + Entry("no schedules, not hibernated", + makeShootDefault(false), now, + false, + ), + + Entry("hibernated with no schedules — stuck hibernated", + makeShootDefault(true), now, + true, + ), + + // Currently AWAKE + + Entry("awake: maintenance inside nightly hibernation window", + makeShootDefault(false, nightlySchedule), now, + true, + ), + Entry("awake: hibernate after maintenance window end", + makeShootDefault(false, gardencorev1beta1.HibernationSchedule{ + Start: new("30 23 * * *"), End: new("0 8 * * *"), + }), now, + false, + ), + Entry("awake: hibernate inside maintenance window", + makeShootDefault(false, gardencorev1beta1.HibernationSchedule{ + Start: new("30 22 * * *"), End: new("0 8 * * *"), + }), now, + true, + ), + Entry("awake: shoot wakes up before maintenance", + makeShootDefault(false, gardencorev1beta1.HibernationSchedule{ + Start: new("0 20 * * *"), End: new("0 21 * * *"), + }), now, + false, + ), + Entry("awake: hibernate-only schedule, no wake-up", + makeShootDefault(false, gardencorev1beta1.HibernationSchedule{ + Start: new("0 20 * * *"), + }), now, + true, + ), + Entry("awake: wake-only schedule, no hibernate", + makeShootDefault(false, gardencorev1beta1.HibernationSchedule{ + End: new("0 8 * * *"), + }), now, + false, + ), + Entry("awake: weekday schedule, maintenance inside weeknight hibernation window", + makeShootDefault(false, weekdaySchedule), now, + true, + ), + Entry("awake: cross-midnight maintenance inside hibernation window", + makeShoot("230000+0000", "010000+0000", false, + gardencorev1beta1.HibernationSchedule{Start: new("0 20 * * *"), End: new("0 6 * * *")}), now, + true, + ), + Entry("awake: cross-midnight maintenance before next hibernate", + makeShoot("230000+0000", "010000+0000", false, + gardencorev1beta1.HibernationSchedule{Start: new("0 2 * * *"), End: new("0 21 * * *")}), now, + false, + ), + + // Currently HIBERNATED + + Entry("hibernated: manually hibernated, but wakes up in time", + makeShootDefault(true, gardencorev1beta1.HibernationSchedule{ + End: new("0 20 * * *"), + }), now, + false, + ), + + Entry("hibernated: nightly schedule, next wake-up is after tonight's maintenance", + makeShootDefault(true, nightlySchedule), nowHibernated, + true, + ), + Entry("hibernated: wake-up inside maintenance window", + makeShootDefault(true, gardencorev1beta1.HibernationSchedule{ + Start: new("0 20 * * *"), End: new("30 22 * * *"), + }), nowHibernated, + true, + ), + Entry("hibernated: wake-up after maintenance window end", + makeShootDefault(true, gardencorev1beta1.HibernationSchedule{ + Start: new("0 20 * * *"), End: new("30 23 * * *"), + }), nowHibernated, + true, + ), + Entry("hibernated: wake-up before maintenance start", + makeShootDefault(true, gardencorev1beta1.HibernationSchedule{ + Start: new("0 20 * * *"), End: new("30 21 * * *"), + }), nowHibernated, + false, + ), + Entry("hibernated: weekday schedule, no wake-up before tonight's maintenance", + makeShootDefault(true, weekdaySchedule), nowHibernated, + true, + ), + Entry("hibernated: hibernate-only schedule, no wake-up ever", + makeShootDefault(true, gardencorev1beta1.HibernationSchedule{ + Start: new("0 20 * * *"), + }), nowHibernated, + true, + ), + Entry("hibernated: wakes up before maintenance but re-hibernates before it", + makeShootDefault(true, gardencorev1beta1.HibernationSchedule{ + Start: new("45 21 * * *"), End: new("30 21 * * *"), + }), nowHibernated, + true, + ), + ) +}) diff --git a/pkg/gardenlet/controller/shoot/care/reconciler_test.go b/pkg/gardenlet/controller/shoot/care/reconciler_test.go index 7a3c2a8aa5f..fdee87172ce 100644 --- a/pkg/gardenlet/controller/shoot/care/reconciler_test.go +++ b/pkg/gardenlet/controller/shoot/care/reconciler_test.go @@ -653,7 +653,7 @@ func containConditionsInUnknownStatus(message string, isWorkerless bool) types.G } func containConstraintsInUnknownStatus(message string) types.GomegaMatcher { - var expectedLength = 8 + var expectedLength = 9 matcher := And( ContainCondition( OfType(gardencorev1beta1.ShootHibernationPossible), @@ -677,6 +677,10 @@ func containConstraintsInUnknownStatus(message string) types.GomegaMatcher { OfType(gardencorev1beta1.ShootManualInPlaceWorkersUpdated), WithStatus(gardencorev1beta1.ConditionUnknown), WithMessage(message), + ), ContainCondition( + OfType(gardencorev1beta1.ShootAutomaticCredentialsRotationPossible), + WithStatus(gardencorev1beta1.ConditionUnknown), + WithMessage(message), ), ContainCondition( OfType(gardencorev1beta1.ShootHasIgnoredManagedResources), WithStatus(gardencorev1beta1.ConditionUnknown), diff --git a/pkg/utils/hibernation/hibernationschedule.go b/pkg/utils/hibernation/hibernationschedule.go new file mode 100644 index 00000000000..e0996ff582e --- /dev/null +++ b/pkg/utils/hibernation/hibernationschedule.go @@ -0,0 +1,95 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +// Package hibernationschedule provides helpers for parsing and iterating over Shoot +// hibernation schedules expressed as cron expressions. +package hibernation + +import ( + "time" + + "github.com/robfig/cron" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" +) + +// Operation defines the type of operation that is scheduled by a hibernation schedule. +type Operation uint8 + +const ( + // Hibernate indicates that the cluster should be hibernated. + Hibernate Operation = iota + // WakeUp indicates that the cluster should be woken up. + WakeUp +) + +// ParsedSchedule holds the loaded location, parsed cron schedule and information whether +// the cluster should be hibernated or woken up. +type ParsedSchedule struct { + Schedule cron.Schedule + Location time.Location + Operation Operation +} + +// Next returns the time in UTC from the schedule, that is immediately after the input time 't'. +// The input 't' is converted in the schedule's location before any calculations are done. +func (s *ParsedSchedule) Next(t time.Time) time.Time { + return s.Schedule.Next(t.In(&s.Location)).UTC() +} + +// Previous returns the time in UTC from the schedule that is immediately before 'to' and after 'from'. +// Nil is returned if no such time can be found. +// The input times - 'to' and 'from' are converted in the schedule's location before any calculation is done. +func (s *ParsedSchedule) Previous(from, to time.Time) *time.Time { + var last *time.Time + for t := s.Schedule.Next(from.In(&s.Location)); !t.UTC().After(to.UTC()); t = s.Schedule.Next(t) { + inUTC := t.UTC() + last = &inUTC + } + return last +} + +// Parse parses the given HibernationSchedules and returns an array of ParsedSchedules +// If the Location of a HibernationSchedule is `nil`, it is defaulted to UTC. +func Parse(schedules []gardencorev1beta1.HibernationSchedule) ([]ParsedSchedule, error) { + var out []ParsedSchedule + + for _, sched := range schedules { + locationID := time.UTC.String() + if sched.Location != nil && *sched.Location != "" { + locationID = *sched.Location + } + + loc, err := time.LoadLocation(locationID) + if err != nil { + return nil, err + } + + if sched.Start != nil { + parsed, err := cron.ParseStandard(*sched.Start) + if err != nil { + return nil, err + } + out = append(out, ParsedSchedule{ + Schedule: parsed, + Location: *loc, + Operation: Hibernate, + }) + } + + if sched.End != nil { + parsed, err := cron.ParseStandard(*sched.End) + if err != nil { + return nil, err + } + out = append(out, ParsedSchedule{ + Schedule: parsed, + Location: *loc, + Operation: WakeUp, + }) + } + } + + return out, nil +} diff --git a/pkg/utils/hibernation/hibernationschedule_test.go b/pkg/utils/hibernation/hibernationschedule_test.go new file mode 100644 index 00000000000..d856ed22644 --- /dev/null +++ b/pkg/utils/hibernation/hibernationschedule_test.go @@ -0,0 +1,137 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package hibernation_test + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + . "github.com/gardener/gardener/pkg/utils/hibernation" +) + +var _ = Describe("hibernationschedule", func() { + Describe("#Parse", func() { + DescribeTable("should return the expected result", + func(schedules []gardencorev1beta1.HibernationSchedule, expectErr bool, wantOps []Operation) { + out, err := Parse(schedules) + if expectErr { + Expect(err).To(HaveOccurred()) + return + } + Expect(err).NotTo(HaveOccurred()) + Expect(out).To(HaveLen(len(wantOps))) + for i, op := range wantOps { + Expect(out[i].Operation).To(Equal(op)) + } + }, + Entry("nil input → empty slice", nil, false, nil), + Entry("invalid Start → error", + []gardencorev1beta1.HibernationSchedule{{Start: new("not-a-cron")}}, true, nil), + Entry("invalid End → error", + []gardencorev1beta1.HibernationSchedule{{End: new("also-invalid")}}, true, nil), + Entry("unknown location → error", + []gardencorev1beta1.HibernationSchedule{{Start: new("0 20 * * *"), Location: new("Moon/Crater")}}, true, nil), + Entry("Start only → Hibernate", + []gardencorev1beta1.HibernationSchedule{{Start: new("0 20 * * *")}}, false, []Operation{Hibernate}), + Entry("End only → WakeUp", + []gardencorev1beta1.HibernationSchedule{{End: new("0 8 * * *")}}, false, []Operation{WakeUp}), + Entry("Start+End → Hibernate then WakeUp", + []gardencorev1beta1.HibernationSchedule{{Start: new("0 20 * * *"), End: new("0 8 * * *")}}, + false, []Operation{Hibernate, WakeUp}), + Entry("two schedules → four entries", + []gardencorev1beta1.HibernationSchedule{ + {Start: new("0 20 * * 1-5"), End: new("0 8 * * 1-5")}, + {Start: new("0 22 * * 0,6"), End: new("0 10 * * 0,6")}, + }, false, []Operation{Hibernate, WakeUp, Hibernate, WakeUp}), + ) + + It("defaults location to UTC when nil", func() { + out, err := Parse([]gardencorev1beta1.HibernationSchedule{{Start: new("0 20 * * *")}}) + Expect(err).NotTo(HaveOccurred()) + Expect(out[0].Location).To(Equal(*time.UTC)) + }) + + It("loads a non-UTC location", func() { + out, err := Parse([]gardencorev1beta1.HibernationSchedule{{Start: new("0 20 * * *"), Location: new("Europe/Berlin")}}) + Expect(err).NotTo(HaveOccurred()) + berlin, _ := time.LoadLocation("Europe/Berlin") + Expect(out[0].Location).To(Equal(*berlin)) + }) + }) + + // ref is Monday 2006-01-02 00:00:00 UTC — the same anchor used by IsMaintenanceWindowInHibernationWindow. + ref := time.Date(2006, 1, 2, 0, 0, 0, 0, time.UTC) + + // dailyAt20 is a UTC schedule that fires every day at 20:00. + mustDailyAt20 := func() ParsedSchedule { + out, err := Parse([]gardencorev1beta1.HibernationSchedule{{Start: new("0 20 * * *")}}) + Expect(err).NotTo(HaveOccurred()) + return out[0] + } + + Describe("#Next", func() { + DescribeTable("should return the correct next occurrence in UTC", + func(spec, location string, from, want time.Time) { + var loc *string + if location != "" { + loc = new(location) + } + out, err := Parse([]gardencorev1beta1.HibernationSchedule{{Start: new(spec), Location: loc}}) + Expect(err).NotTo(HaveOccurred()) + Expect(out[0].Next(from)).To(Equal(want)) + }, + Entry("daily 20:00 UTC from midnight → same day 20:00", + "0 20 * * *", "", + ref, + time.Date(2006, 1, 2, 20, 0, 0, 0, time.UTC), + ), + Entry("daily 20:00 Europe/Berlin (CET=UTC+1) from midnight UTC → 19:00 UTC", + "0 20 * * *", "Europe/Berlin", + ref, + time.Date(2006, 1, 2, 19, 0, 0, 0, time.UTC), + ), + ) + }) + + Describe("#Previous", func() { + DescribeTable("should return the correct last occurrence in (from, to]", + func(from, to time.Time, want *time.Time) { + s := mustDailyAt20() + prev := s.Previous(from, to) + if want == nil { + Expect(prev).To(BeNil()) + } else { + Expect(prev).NotTo(BeNil()) + Expect(*prev).To(Equal(*want)) + } + }, + // from=Jan2, to=Jan4 → last fire Jan3 20:00 + Entry("last occurrence in two-day window", + ref, ref.Add(2*24*time.Hour), + func() *time.Time { t := time.Date(2006, 1, 3, 20, 0, 0, 0, time.UTC); return &t }(), + ), + // from=Jan3 21:00, to=Jan4 → no fire + Entry("no occurrence in range", + time.Date(2006, 1, 3, 21, 0, 0, 0, time.UTC), ref.Add(2*24*time.Hour), + nil, + ), + // to = Jan3 20:00 exactly → included + Entry("occurrence exactly at to is included", + time.Date(2006, 1, 2, 20, 0, 0, 0, time.UTC), + time.Date(2006, 1, 3, 20, 0, 0, 0, time.UTC), + func() *time.Time { t := time.Date(2006, 1, 3, 20, 0, 0, 0, time.UTC); return &t }(), + ), + // from = Jan3 20:00 exactly → excluded (interval is open on the left) + Entry("occurrence exactly at from is excluded", + time.Date(2006, 1, 3, 20, 0, 0, 0, time.UTC), + ref.Add(2*24*time.Hour), + nil, + ), + ) + }) +}) diff --git a/pkg/utils/hibernation/suite_test.go b/pkg/utils/hibernation/suite_test.go new file mode 100644 index 00000000000..bdb2408174b --- /dev/null +++ b/pkg/utils/hibernation/suite_test.go @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package hibernation_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestHibernation(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Hibernation Suite") +} From 6f8ed687e3577ca7dc3fe9ba9a764ddfd9635ccf Mon Sep 17 00:00:00 2001 From: Rafael Franzke Date: Wed, 5 Aug 2026 15:59:28 +0200 Subject: [PATCH 2/4] [GEP-28] Support shoot `gardenlet` kubeconfig renewal (#15434) * Make `rebootstrap-gardenlet.sh` work for shoot `gardenlet`s as well Previously, this only worked for seed gardenlets, but shoot gardenlets might also need to get rebootstrapped (e.g., in case their client certificate expired for whatever reason). Assisted-by: Claude Signed-off-by: rfranzke * Support `gardener.cloud/operation=renew-kubeconfig` on self-hosted shoot `Gardenlet`s Previously, this didn't work and the `gardenlet` was failing and crash-looping eventually. Assisted-by: Claude Signed-off-by: rfranzke * Renew kubeconfig of shoot `Gardenlet`s when rotating garden cluster CA During a garden cluster CA rotation, the gardener-operator already annotates seed `Gardenlet`s to trigger kubeconfig renewal. Extend this to shoot `Gardenlet`s (those with the `self-hosted-shoot-` name prefix) by annotating them with `gardener.cloud/operation=renew-kubeconfig`. Assisted-by: Claude Signed-off-by: rfranzke * Address PR review feedback Signed-off-by: rfranzke --------- Signed-off-by: rfranzke --- dev-setup/skaffold-operator.yaml | 1 + hack/usage/rebootstrap-gardenlet.sh | 257 ++++++++++++------ .../admission/shootrestriction/handler.go | 7 +- .../shootrestriction/handler_test.go | 45 +++ .../webhook/auth/shoot/authorizer.go | 43 ++- .../webhook/auth/shoot/authorizer_test.go | 44 +++ pkg/controller/gardenletdeployer/actuator.go | 6 +- .../garden/garden/reconciler_reconcile.go | 34 ++- pkg/utils/gardener/gardenlet/gardenlet.go | 28 +- .../gardener/gardenlet/gardenlet_test.go | 19 ++ .../gardener/secretsrotation/gardenaccess.go | 69 ++++- .../secretsrotation/gardenaccess_test.go | 118 ++++++++ pkg/utils/graph/eventhandler_gardenlet.go | 3 - 13 files changed, 556 insertions(+), 118 deletions(-) diff --git a/dev-setup/skaffold-operator.yaml b/dev-setup/skaffold-operator.yaml index 46f56e2f4e8..715f72a3cee 100644 --- a/dev-setup/skaffold-operator.yaml +++ b/dev-setup/skaffold-operator.yaml @@ -689,6 +689,7 @@ build: - pkg/controllerutils/predicate - pkg/controllerutils/routes - pkg/features + - pkg/gardenlet/bootstrap/util - pkg/healthz - pkg/logger - pkg/operator/client diff --git a/hack/usage/rebootstrap-gardenlet.sh b/hack/usage/rebootstrap-gardenlet.sh index 5abd88186d4..cc74a3e8c1a 100755 --- a/hack/usage/rebootstrap-gardenlet.sh +++ b/hack/usage/rebootstrap-gardenlet.sh @@ -16,10 +16,15 @@ NC='\033[0m' # No Color GARDEN_KUBECONFIG="${GARDEN_KUBECONFIG:-}" SEED_KUBECONFIG="${SEED_KUBECONFIG:-}" SEED_NAME="${SEED_NAME:-}" +SHOOT_KUBECONFIG="${SHOOT_KUBECONFIG:-}" +SHOOT_NAMESPACE="${SHOOT_NAMESPACE:-}" +SHOOT_NAME="${SHOOT_NAME:-}" BOOTSTRAP_TOKEN_ID="" BOOTSTRAP_TOKEN_SECRET="" -GARDENLET_NAMESPACE="garden" +GARDENLET_NAMESPACE="" GARDENLET_DEPLOYMENT_NAME="gardenlet" +BOOTSTRAP_KUBECONFIG_SECRET_NAME="" +MODE="" # "seed" or "shoot" function log_info() { echo -e "${GREEN}[INFO]${NC} $1" @@ -44,11 +49,19 @@ Prerequisites: - kubectl - yq (https://github.com/mikefarah/yq) -Required Options: +Required Options (seed gardenlet): --garden-kubeconfig PATH Path to garden cluster kubeconfig --seed-kubeconfig PATH Path to seed cluster kubeconfig --seed-name NAME Name of the seed +Required Options (shoot gardenlet): + --garden-kubeconfig PATH Path to garden cluster kubeconfig + --shoot-kubeconfig PATH Path to shoot cluster kubeconfig + --shoot-namespace NAMESPACE Namespace of the shoot in the garden cluster + --shoot-name NAME Name of the shoot + +Note: --seed-* and --shoot-* options are mutually exclusive. + Optional: --token-id ID Bootstrap token ID (6 characters, random if not provided) --token-secret SECRET Bootstrap token secret (16 characters, random if not provided) @@ -56,7 +69,7 @@ Optional: Examples: $0 --garden-kubeconfig ~/.kube/garden.yaml --seed-kubeconfig ~/.kube/seed.yaml --seed-name my-seed - $0 --garden-kubeconfig garden.yaml --seed-kubeconfig seed.yaml --seed-name my-seed + $0 --garden-kubeconfig ~/.kube/garden.yaml --shoot-kubeconfig ~/.kube/shoot.yaml --shoot-namespace garden --shoot-name my-shoot EOF } @@ -79,19 +92,71 @@ function validate_requirements() { exit 1 fi - if [[ -z "$SEED_KUBECONFIG" ]]; then - log_error "Seed kubeconfig is required. Use --seed-kubeconfig option." - exit 1 + # Determine mode and validate mode-specific options + local has_seed=false + local has_shoot=false + if [[ -n "$SEED_KUBECONFIG" || -n "$SEED_NAME" ]]; then + has_seed=true + fi + if [[ -n "$SHOOT_KUBECONFIG" || -n "$SHOOT_NAMESPACE" || -n "$SHOOT_NAME" ]]; then + has_shoot=true fi - if [[ ! -f "$SEED_KUBECONFIG" ]]; then - log_error "Seed kubeconfig file not found: $SEED_KUBECONFIG" + if [[ "$has_seed" == "true" && "$has_shoot" == "true" ]]; then + log_error "--seed-* and --shoot-* options are mutually exclusive." + exit 1 + elif [[ "$has_seed" == "false" && "$has_shoot" == "false" ]]; then + log_error "Either --seed-kubeconfig/--seed-name or --shoot-kubeconfig/--shoot-namespace/--shoot-name must be provided." exit 1 fi - if [[ -z "$SEED_NAME" ]]; then - log_error "Seed name is required. Use --seed-name option." - exit 1 + if [[ "$has_seed" == "true" ]]; then + MODE="seed" + + if [[ -z "$SEED_KUBECONFIG" ]]; then + log_error "Seed kubeconfig is required. Use --seed-kubeconfig option." + exit 1 + fi + + if [[ ! -f "$SEED_KUBECONFIG" ]]; then + log_error "Seed kubeconfig file not found: $SEED_KUBECONFIG" + exit 1 + fi + + if [[ -z "$SEED_NAME" ]]; then + log_error "Seed name is required. Use --seed-name option." + exit 1 + fi + + GARDENLET_NAMESPACE="garden" + BOOTSTRAP_KUBECONFIG_SECRET_NAME="gardenlet-kubeconfig-bootstrap" + TARGET_KUBECONFIG="$SEED_KUBECONFIG" + else + MODE="shoot" + + if [[ -z "$SHOOT_KUBECONFIG" ]]; then + log_error "Shoot kubeconfig is required. Use --shoot-kubeconfig option." + exit 1 + fi + + if [[ ! -f "$SHOOT_KUBECONFIG" ]]; then + log_error "Shoot kubeconfig file not found: $SHOOT_KUBECONFIG" + exit 1 + fi + + if [[ -z "$SHOOT_NAMESPACE" ]]; then + log_error "Shoot namespace is required. Use --shoot-namespace option." + exit 1 + fi + + if [[ -z "$SHOOT_NAME" ]]; then + log_error "Shoot name is required. Use --shoot-name option." + exit 1 + fi + + GARDENLET_NAMESPACE="kube-system" + BOOTSTRAP_KUBECONFIG_SECRET_NAME="gardenlet-kubeconfig-bootstrap" + TARGET_KUBECONFIG="$SHOOT_KUBECONFIG" fi if ! command -v kubectl &> /dev/null; then @@ -105,13 +170,12 @@ function validate_requirements() { exit 1 fi - log_info "All requirements validated successfully" + log_info "All requirements validated successfully (mode: $MODE)" } function create_bootstrap_token() { log_info "Creating bootstrap token in garden cluster..." - # Generate token ID and secret if not provided if [[ -z "$BOOTSTRAP_TOKEN_ID" ]]; then BOOTSTRAP_TOKEN_ID=$(generate_random_string 6) log_info "Generated token ID: $BOOTSTRAP_TOKEN_ID" @@ -123,6 +187,12 @@ function create_bootstrap_token() { fi local token_name="bootstrap-token-${BOOTSTRAP_TOKEN_ID}" + local description + if [[ "$MODE" == "seed" ]]; then + description="Used for reconnecting the gardenlet for seed ${SEED_NAME} to Gardener" + else + description="Used for connecting the self-hosted Shoot ${SHOOT_NAMESPACE}/${SHOOT_NAME}" + fi # Check if token already exists if kubectl --kubeconfig="$GARDEN_KUBECONFIG" -n kube-system get secret "$token_name" &> /dev/null; then @@ -138,10 +208,9 @@ function create_bootstrap_token() { fi fi - # Create bootstrap token secret kubectl --kubeconfig="$GARDEN_KUBECONFIG" -n kube-system create secret generic "$token_name" \ --type=bootstrap.kubernetes.io/token \ - --from-literal=description="Bootstrap token for gardenlet rebootstrap of seed ${SEED_NAME}" \ + --from-literal=description="$description" \ --from-literal=token-id="$BOOTSTRAP_TOKEN_ID" \ --from-literal=token-secret="$BOOTSTRAP_TOKEN_SECRET" \ --from-literal=usage-bootstrap-authentication=true \ @@ -153,7 +222,6 @@ function create_bootstrap_token() { function get_garden_cluster_info() { log_info "Extracting garden cluster information..." - # Extract server and CA from garden kubeconfig GARDEN_SERVER=$(kubectl --kubeconfig="$GARDEN_KUBECONFIG" config view --minify -o jsonpath='{.clusters[0].cluster.server}') GARDEN_CA=$(kubectl --kubeconfig="$GARDEN_KUBECONFIG" config view --minify --raw -o jsonpath='{.clusters[0].cluster.certificate-authority-data}') @@ -171,12 +239,10 @@ function get_garden_cluster_info() { } function create_bootstrap_kubeconfig_secret() { - log_info "Creating bootstrap kubeconfig secret in seed cluster..." + log_info "Creating bootstrap kubeconfig secret in target cluster..." local bootstrap_token="${BOOTSTRAP_TOKEN_ID}.${BOOTSTRAP_TOKEN_SECRET}" - local secret_name="gardenlet-bootstrap-kubeconfig" - # Create bootstrap kubeconfig local bootstrap_kubeconfig=$(cat < /dev/null; then - log_warn "Bootstrap kubeconfig secret '$secret_name' already exists in seed cluster" - kubectl --kubeconfig="$SEED_KUBECONFIG" -n "$GARDENLET_NAMESPACE" delete secret "$secret_name" + if kubectl --kubeconfig="$TARGET_KUBECONFIG" -n "$GARDENLET_NAMESPACE" get secret "$BOOTSTRAP_KUBECONFIG_SECRET_NAME" &> /dev/null; then + log_warn "Bootstrap kubeconfig secret '$BOOTSTRAP_KUBECONFIG_SECRET_NAME' already exists in target cluster" + kubectl --kubeconfig="$TARGET_KUBECONFIG" -n "$GARDENLET_NAMESPACE" delete secret "$BOOTSTRAP_KUBECONFIG_SECRET_NAME" log_info "Deleted existing bootstrap kubeconfig secret" fi - # Create secret with bootstrap kubeconfig - kubectl --kubeconfig="$SEED_KUBECONFIG" -n "$GARDENLET_NAMESPACE" create secret generic "$secret_name" \ + kubectl --kubeconfig="$TARGET_KUBECONFIG" -n "$GARDENLET_NAMESPACE" create secret generic "$BOOTSTRAP_KUBECONFIG_SECRET_NAME" \ --from-literal=kubeconfig="$bootstrap_kubeconfig" - log_info "Bootstrap kubeconfig secret created successfully: $secret_name" + log_info "Bootstrap kubeconfig secret created successfully: $BOOTSTRAP_KUBECONFIG_SECRET_NAME" } function update_gardenlet_configuration() { log_info "Updating gardenlet configuration..." - # Get the gardenlet deployment - if ! kubectl --kubeconfig="$SEED_KUBECONFIG" -n "$GARDENLET_NAMESPACE" get deployment "$GARDENLET_DEPLOYMENT_NAME" &> /dev/null; then + if ! kubectl --kubeconfig="$TARGET_KUBECONFIG" -n "$GARDENLET_NAMESPACE" get deployment "$GARDENLET_DEPLOYMENT_NAME" &> /dev/null; then log_error "Gardenlet deployment '$GARDENLET_DEPLOYMENT_NAME' not found in namespace '$GARDENLET_NAMESPACE'" exit 1 fi - # Find the ConfigMap used by the gardenlet deployment (volume name: gardenlet-config) - local configmap_name=$(kubectl --kubeconfig="$SEED_KUBECONFIG" -n "$GARDENLET_NAMESPACE" get deployment "$GARDENLET_DEPLOYMENT_NAME" -o jsonpath='{.spec.template.spec.volumes[?(@.name=="gardenlet-config")].configMap.name}') + local configmap_name=$(kubectl --kubeconfig="$TARGET_KUBECONFIG" -n "$GARDENLET_NAMESPACE" get deployment "$GARDENLET_DEPLOYMENT_NAME" -o jsonpath='{.spec.template.spec.volumes[?(@.name=="gardenlet-config")].configMap.name}') if [[ -z "$configmap_name" ]]; then log_error "Could not find ConfigMap referenced by gardenlet deployment volume 'gardenlet-config'" @@ -232,43 +294,35 @@ function update_gardenlet_configuration() { log_info "Found ConfigMap: $configmap_name" - # Get the current ConfigMap content and extract the config.yaml key (this is where the GardenletConfiguration is stored) local config_key="config\.yaml" - local config_content=$(kubectl --kubeconfig="$SEED_KUBECONFIG" -n "$GARDENLET_NAMESPACE" get configmap "$configmap_name" -o jsonpath="{.data['$config_key']}") + local config_content=$(kubectl --kubeconfig="$TARGET_KUBECONFIG" -n "$GARDENLET_NAMESPACE" get configmap "$configmap_name" -o jsonpath="{.data['$config_key']}") if [[ -z "$config_content" ]]; then log_error "Could not find '$config_key' in ConfigMap '$configmap_name'" exit 1 fi - # Create a temporary file with the current config local temp_config=$(mktemp) echo "$config_content" > "$temp_config" - # Update or add bootstrapKubeconfig configuration using yq log_info "Updating gardenlet configuration with bootstrap kubeconfig..." - yq eval -i ".gardenClientConnection.bootstrapKubeconfig.name = \"gardenlet-bootstrap-kubeconfig\"" "$temp_config" + yq eval -i ".gardenClientConnection.bootstrapKubeconfig.name = \"$BOOTSTRAP_KUBECONFIG_SECRET_NAME\"" "$temp_config" yq eval -i ".gardenClientConnection.bootstrapKubeconfig.namespace = \"$GARDENLET_NAMESPACE\"" "$temp_config" - # Generate a new ConfigMap name with timestamp to ensure uniqueness local timestamp=$(date +%s) local new_configmap_name="${configmap_name}-rebootstrap-${timestamp}" log_info "Creating new ConfigMap: $new_configmap_name" - # Create the new ConfigMap with updated configuration - kubectl --kubeconfig="$SEED_KUBECONFIG" -n "$GARDENLET_NAMESPACE" create configmap "$new_configmap_name" \ + kubectl --kubeconfig="$TARGET_KUBECONFIG" -n "$GARDENLET_NAMESPACE" create configmap "$new_configmap_name" \ --from-file="${config_key//\\/}=${temp_config}" - # Clean up temp file rm -f "$temp_config" - # Update the deployment to use the new ConfigMap log_info "Updating gardenlet deployment to use new ConfigMap..." - # Get all volumes and update the gardenlet-config volume to reference the new ConfigMap local volumes_json - volumes_json=$(kubectl --kubeconfig="$SEED_KUBECONFIG" -n "$GARDENLET_NAMESPACE" get deployment "$GARDENLET_DEPLOYMENT_NAME" \ + volumes_json=$(kubectl --kubeconfig="$TARGET_KUBECONFIG" -n "$GARDENLET_NAMESPACE" get deployment "$GARDENLET_DEPLOYMENT_NAME" \ -o yaml | yq eval '(.spec.template.spec.volumes[] | select(.name == "gardenlet-config") | .configMap.name) = "'"$new_configmap_name"'" | .spec.template.spec.volumes' -o json -) if [[ -z "$volumes_json" ]] || [[ "$volumes_json" == "null" ]]; then @@ -276,8 +330,7 @@ function update_gardenlet_configuration() { exit 1 fi - # Patch the deployment to reference the new ConfigMap - kubectl --kubeconfig="$SEED_KUBECONFIG" -n "$GARDENLET_NAMESPACE" patch deployment "$GARDENLET_DEPLOYMENT_NAME" --type=json -p="[ + kubectl --kubeconfig="$TARGET_KUBECONFIG" -n "$GARDENLET_NAMESPACE" patch deployment "$GARDENLET_DEPLOYMENT_NAME" --type=json -p="[ { \"op\": \"replace\", \"path\": \"/spec/template/spec/volumes\", @@ -296,8 +349,8 @@ function delete_expired_kubeconfig() { local kubeconfig_secret_name="gardenlet-kubeconfig" - if kubectl --kubeconfig="$SEED_KUBECONFIG" -n "$GARDENLET_NAMESPACE" get secret "$kubeconfig_secret_name" &> /dev/null; then - kubectl --kubeconfig="$SEED_KUBECONFIG" -n "$GARDENLET_NAMESPACE" delete secret "$kubeconfig_secret_name" + if kubectl --kubeconfig="$TARGET_KUBECONFIG" -n "$GARDENLET_NAMESPACE" get secret "$kubeconfig_secret_name" &> /dev/null; then + kubectl --kubeconfig="$TARGET_KUBECONFIG" -n "$GARDENLET_NAMESPACE" delete secret "$kubeconfig_secret_name" log_info "Deleted expired kubeconfig secret: $kubeconfig_secret_name" else log_warn "Kubeconfig secret '$kubeconfig_secret_name' not found, skipping deletion" @@ -307,10 +360,10 @@ function delete_expired_kubeconfig() { function wait_for_gardenlet_rollout() { log_info "Waiting for gardenlet deployment rollout..." - if kubectl --kubeconfig="$SEED_KUBECONFIG" -n "$GARDENLET_NAMESPACE" get deployment "$GARDENLET_DEPLOYMENT_NAME" &> /dev/null; then + if kubectl --kubeconfig="$TARGET_KUBECONFIG" -n "$GARDENLET_NAMESPACE" get deployment "$GARDENLET_DEPLOYMENT_NAME" &> /dev/null; then log_info "The deployment will restart automatically due to the ConfigMap change" log_info "Waiting for rollout to complete..." - kubectl --kubeconfig="$SEED_KUBECONFIG" -n "$GARDENLET_NAMESPACE" rollout status deployment "$GARDENLET_DEPLOYMENT_NAME" --timeout=5m + kubectl --kubeconfig="$TARGET_KUBECONFIG" -n "$GARDENLET_NAMESPACE" rollout status deployment "$GARDENLET_DEPLOYMENT_NAME" --timeout=5m log_info "Gardenlet deployment rollout completed successfully" else log_error "Gardenlet deployment '$GARDENLET_DEPLOYMENT_NAME' not found in namespace '$GARDENLET_NAMESPACE'" @@ -322,23 +375,20 @@ function verify_bootstrap() { log_info "Verifying bootstrap success..." local kubeconfig_secret_name="gardenlet-kubeconfig" - local bootstrap_secret_name="gardenlet-bootstrap-kubeconfig" local max_wait=300 local elapsed=0 local interval=10 - # Wait for new kubeconfig secret to be created log_info "Waiting for new kubeconfig secret to be created..." while [[ $elapsed -lt $max_wait ]]; do - if kubectl --kubeconfig="$SEED_KUBECONFIG" -n "$GARDENLET_NAMESPACE" get secret "$kubeconfig_secret_name" &> /dev/null; then + if kubectl --kubeconfig="$TARGET_KUBECONFIG" -n "$GARDENLET_NAMESPACE" get secret "$kubeconfig_secret_name" &> /dev/null; then log_info "✓ New kubeconfig secret created" break fi sleep $interval elapsed=$((elapsed + interval)) - # Check if bootstrap secret was deleted - if kubectl --kubeconfig="$SEED_KUBECONFIG" -n "$GARDENLET_NAMESPACE" get secret "$bootstrap_secret_name" &> /dev/null; then + if kubectl --kubeconfig="$TARGET_KUBECONFIG" -n "$GARDENLET_NAMESPACE" get secret "$BOOTSTRAP_KUBECONFIG_SECRET_NAME" &> /dev/null; then log_warn "⚠ Bootstrap secret still exists (it should be deleted automatically)" else log_info "✓ Bootstrap secret was deleted" @@ -349,46 +399,74 @@ function verify_bootstrap() { if [[ $elapsed -ge $max_wait ]]; then log_error "Timeout waiting for new kubeconfig secret to be created" log_error "Check gardenlet logs for errors:" - log_error " kubectl --kubeconfig=$SEED_KUBECONFIG -n $GARDENLET_NAMESPACE logs deployment/$GARDENLET_DEPLOYMENT_NAME" - exit 1 - fi - - # Check seed status in garden cluster with retry logic - log_info "Checking seed status in garden cluster..." - - if ! kubectl --kubeconfig="$GARDEN_KUBECONFIG" get seed "$SEED_NAME" &> /dev/null; then - log_error "Seed resource '$SEED_NAME' not found in garden cluster" + log_error " kubectl --kubeconfig=$TARGET_KUBECONFIG -n $GARDENLET_NAMESPACE logs deployment/$GARDENLET_DEPLOYMENT_NAME" exit 1 fi - # Wait for gardenlet to become ready - log_info "Waiting for gardenlet to report ready status..." - local seed_max_wait=120 # 2 minutes + local seed_max_wait=120 local seed_elapsed=0 local seed_interval=5 - while [[ $seed_elapsed -lt $seed_max_wait ]]; do - local gardenlet_ready - gardenlet_ready=$(kubectl --kubeconfig="$GARDEN_KUBECONFIG" get seed "$SEED_NAME" -o jsonpath='{.status.conditions[?(@.type=="GardenletReady")].status}' 2>/dev/null || echo "") + if [[ "$MODE" == "seed" ]]; then + log_info "Checking seed status in garden cluster..." - if [[ "$gardenlet_ready" == "True" ]]; then - log_info "✓ Seed is healthy and gardenlet is ready" - break + if ! kubectl --kubeconfig="$GARDEN_KUBECONFIG" get seed "$SEED_NAME" &> /dev/null; then + log_error "Seed resource '$SEED_NAME' not found in garden cluster" + exit 1 fi - sleep $seed_interval - seed_elapsed=$((seed_elapsed + seed_interval)) - echo -n "." - done - echo + log_info "Waiting for gardenlet to report ready status..." + while [[ $seed_elapsed -lt $seed_max_wait ]]; do + local gardenlet_ready + gardenlet_ready=$(kubectl --kubeconfig="$GARDEN_KUBECONFIG" get seed "$SEED_NAME" -o jsonpath='{.status.conditions[?(@.type=="GardenletReady")].status}' 2>/dev/null || echo "") + + if [[ "$gardenlet_ready" == "True" ]]; then + log_info "✓ Seed is healthy and gardenlet is ready" + break + fi + + sleep $seed_interval + seed_elapsed=$((seed_elapsed + seed_interval)) + echo -n "." + done + echo + else + log_info "Checking shoot status in garden cluster..." + + if ! kubectl --kubeconfig="$GARDEN_KUBECONFIG" -n "$SHOOT_NAMESPACE" get shoot "$SHOOT_NAME" &> /dev/null; then + log_error "Shoot resource '$SHOOT_NAMESPACE/$SHOOT_NAME' not found in garden cluster" + exit 1 + fi + + log_info "Waiting for gardenlet to report ready status..." + while [[ $seed_elapsed -lt $seed_max_wait ]]; do + local gardenlet_ready + gardenlet_ready=$(kubectl --kubeconfig="$GARDEN_KUBECONFIG" -n "$SHOOT_NAMESPACE" get shoot "$SHOOT_NAME" -o jsonpath='{.status.conditions[?(@.type=="GardenletReady")].status}' 2>/dev/null || echo "") + + if [[ "$gardenlet_ready" == "True" ]]; then + log_info "✓ Shoot gardenlet is ready" + break + fi + + sleep $seed_interval + seed_elapsed=$((seed_elapsed + seed_interval)) + echo -n "." + done + echo + fi log_info "Deleting bootstrap token secret" kubectl --kubeconfig=$GARDEN_KUBECONFIG -n kube-system delete secret bootstrap-token-$BOOTSTRAP_TOKEN_ID --ignore-not-found if [[ $seed_elapsed -ge $seed_max_wait ]]; then log_warn "⚠ Timeout waiting for gardenlet to report ready status" - log_warn " The bootstrap may still be in progress. Check seed status manually:" - log_warn " kubectl --kubeconfig=$GARDEN_KUBECONFIG get seed $SEED_NAME -o yaml | yq eval .status.conditions" + if [[ "$MODE" == "seed" ]]; then + log_warn " The bootstrap may still be in progress. Check seed status manually:" + log_warn " kubectl --kubeconfig=$GARDEN_KUBECONFIG get seed $SEED_NAME -o yaml | yq eval .status.conditions" + else + log_warn " The bootstrap may still be in progress. Check shoot status manually:" + log_warn " kubectl --kubeconfig=$GARDEN_KUBECONFIG -n $SHOOT_NAMESPACE get shoot $SHOOT_NAME -o yaml | yq eval .status.conditions" + fi fi log_info "" @@ -398,10 +476,15 @@ function verify_bootstrap() { log_info "" log_info "Next steps:" log_info "1. Monitor gardenlet logs:" - log_info " kubectl --kubeconfig=$SEED_KUBECONFIG -n $GARDENLET_NAMESPACE logs -f deployment/$GARDENLET_DEPLOYMENT_NAME" + log_info " kubectl --kubeconfig=$TARGET_KUBECONFIG -n $GARDENLET_NAMESPACE logs -f deployment/$GARDENLET_DEPLOYMENT_NAME" log_info "" - log_info "2. Check seed conditions:" - log_info " kubectl --kubeconfig=$GARDEN_KUBECONFIG get seed $SEED_NAME -o yaml | yq eval .status.conditions" + if [[ "$MODE" == "seed" ]]; then + log_info "2. Check seed conditions:" + log_info " kubectl --kubeconfig=$GARDEN_KUBECONFIG get seed $SEED_NAME -o yaml | yq eval .status.conditions" + else + log_info "2. Check shoot conditions:" + log_info " kubectl --kubeconfig=$GARDEN_KUBECONFIG -n $SHOOT_NAMESPACE get shoot $SHOOT_NAME -o yaml | yq eval .status.conditions" + fi } # Parse command line arguments @@ -419,6 +502,18 @@ while [[ $# -gt 0 ]]; do SEED_NAME="$2" shift 2 ;; + --shoot-kubeconfig) + SHOOT_KUBECONFIG="$2" + shift 2 + ;; + --shoot-namespace) + SHOOT_NAMESPACE="$2" + shift 2 + ;; + --shoot-name) + SHOOT_NAME="$2" + shift 2 + ;; --token-id) BOOTSTRAP_TOKEN_ID="$2" shift 2 diff --git a/pkg/admissioncontroller/webhook/admission/shootrestriction/handler.go b/pkg/admissioncontroller/webhook/admission/shootrestriction/handler.go index a9587f570fc..d5fdd8e0082 100644 --- a/pkg/admissioncontroller/webhook/admission/shootrestriction/handler.go +++ b/pkg/admissioncontroller/webhook/admission/shootrestriction/handler.go @@ -186,7 +186,8 @@ func (h *Handler) admitSecret(ctx context.Context, gardenletShootInfo types.Name } kind, namespace, name := gardenletbootstraputil.MetadataFromDescription(string(secret.Data[bootstraptokenapi.BootstrapTokenDescriptionKey])) - if kind == gardenletbootstraputil.KindManagedSeed { + switch kind { + case gardenletbootstraputil.KindManagedSeed: managedSeed := &seedmanagementv1alpha1.ManagedSeed{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}} if err := h.Client.Get(ctx, client.ObjectKeyFromObject(managedSeed), managedSeed); err != nil { if apierrors.IsNotFound(err) { @@ -196,6 +197,10 @@ func (h *Handler) admitSecret(ctx context.Context, gardenletShootInfo types.Name } return h.admit(gardenletShootInfo, types.NamespacedName{Name: managedSeed.Spec.Shoot.Name, Namespace: managedSeed.Namespace}) + + case gardenletbootstraputil.KindGardenlet: + // The Gardenlet resource name carries the `self-hosted-shoot-` prefix; strip it to recover the shoot name. + return h.admit(gardenletShootInfo, types.NamespacedName{Name: strings.TrimPrefix(name, gardenletutils.ResourcePrefixSelfHostedShoot), Namespace: namespace}) } } diff --git a/pkg/admissioncontroller/webhook/admission/shootrestriction/handler_test.go b/pkg/admissioncontroller/webhook/admission/shootrestriction/handler_test.go index c66f0104494..8a50362ae9d 100644 --- a/pkg/admissioncontroller/webhook/admission/shootrestriction/handler_test.go +++ b/pkg/admissioncontroller/webhook/admission/shootrestriction/handler_test.go @@ -706,6 +706,51 @@ Foj/rmOanFj5g6QF3GRDrqaNc1GNEXDU6fW7JsTx6+Anj1M/aDNxOXYqIqUN0s3d })) }) }) + + Context("Gardenlet bootstrap token secret", func() { + BeforeEach(func() { + request.Name = "bootstrap-token-abcdef" + request.Namespace = metav1.NamespaceSystem + }) + + It("should allow because the Gardenlet belongs to gardenlet's shoot", func() { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "bootstrap-token-abcdef", Namespace: metav1.NamespaceSystem}, + Type: corev1.SecretTypeBootstrapToken, + Data: map[string][]byte{ + "description": []byte("A bootstrap token for the Gardenlet for seedmanagement.gardener.cloud/v1alpha1.Gardenlet resource " + shootNamespace + "/self-hosted-shoot-" + shootName + "."), + }, + } + objData, err := runtime.Encode(encoder, secret) + Expect(err).NotTo(HaveOccurred()) + request.Object.Raw = objData + + Expect(handler.Handle(ctx, request)).To(Equal(responseAllowed)) + }) + + It("should forbid because the Gardenlet does not belong to gardenlet's shoot", func() { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "bootstrap-token-abcdef", Namespace: "other-namespace"}, + Type: corev1.SecretTypeBootstrapToken, + Data: map[string][]byte{ + "description": []byte("A bootstrap token for the Gardenlet for seedmanagement.gardener.cloud/v1alpha1.Gardenlet resource other-namespace/self-hosted-shoot-other-shoot."), + }, + } + objData, err := runtime.Encode(encoder, secret) + Expect(err).NotTo(HaveOccurred()) + request.Object.Raw = objData + + Expect(handler.Handle(ctx, request)).To(Equal(admission.Response{ + AdmissionResponse: admissionv1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Code: int32(http.StatusForbidden), + Message: fmt.Sprintf("object does not belong to shoot %s/%s", shootNamespace, shootName), + }, + }, + })) + }) + }) }) }) diff --git a/pkg/admissioncontroller/webhook/auth/shoot/authorizer.go b/pkg/admissioncontroller/webhook/auth/shoot/authorizer.go index d765736de34..f0d3789232a 100644 --- a/pkg/admissioncontroller/webhook/auth/shoot/authorizer.go +++ b/pkg/admissioncontroller/webhook/auth/shoot/authorizer.go @@ -22,6 +22,7 @@ import ( "k8s.io/apiserver/pkg/authentication/user" auth "k8s.io/apiserver/pkg/authorization/authorizer" bootstraptokenapi "k8s.io/cluster-bootstrap/token/api" + bootstraptokenutil "k8s.io/cluster-bootstrap/token/util" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/gardener/gardener/pkg/admissioncontroller/gardenletidentity" @@ -38,6 +39,7 @@ import ( gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" gardenletutils "github.com/gardener/gardener/pkg/utils/gardener/gardenlet" "github.com/gardener/gardener/pkg/utils/graph" + "github.com/gardener/gardener/pkg/utils/kubernetes/bootstraptoken" authorizerwebhook "github.com/gardener/gardener/pkg/webhook/authorizer" ) @@ -359,21 +361,38 @@ func (a *authorizer) authorizeServiceAccount(requestAuthorizer *authwebhook.Requ } func (a *authorizer) authorizeSecret(ctx context.Context, requestAuthorizer *authwebhook.RequestAuthorizer, attrs auth.Attributes) (auth.Decision, string, error) { - // Allow gardenlet to delete bootstrap tokens for its own shoot or for ManagedSeeds referencing its shoot. - if attrs.GetVerb() == "delete" && attrs.GetNamespace() == metav1.NamespaceSystem && strings.HasPrefix(attrs.GetName(), bootstraptokenapi.BootstrapTokenSecretPrefix) { - shootMeta, found, err := gardenletutils.ShootMetaFromBootstrapToken(ctx, a.client, attrs.GetName()) - if err != nil { - if !apierrors.IsNotFound(err) { - return auth.DecisionNoOpinion, "", err - } - } else if found { - if shootMeta.Namespace == requestAuthorizer.ToNamespace && shootMeta.Name == requestAuthorizer.ToName { + if attrs.GetNamespace() == metav1.NamespaceSystem && strings.HasPrefix(attrs.GetName(), bootstraptokenapi.BootstrapTokenSecretPrefix) { + switch attrs.GetVerb() { + case "get", "list", "watch": + // Allow gardenlet to get/list/watch the bootstrap token secret whose name is deterministically derived from + // its shoot identity. This is needed for the renew-kubeconfig flow which calls + // ComputeGardenletKubeconfigWithBootstrapToken to look up the token. The list/watch verbs are issued by the + // SingleObjectCache which primes a watch via a namespaced list with a metadata.name field selector. + expectedSecretName := bootstraptokenutil.BootstrapTokenSecretName(bootstraptoken.TokenID(metav1.ObjectMeta{ + Namespace: requestAuthorizer.ToNamespace, + Name: requestAuthorizer.ToName, + })) + if attrs.GetName() == expectedSecretName { return auth.DecisionAllow, "", nil } - return auth.DecisionNoOpinion, fmt.Sprintf("shoot meta in bootstrap token secret %s does not match with identity of requestor %s/%s", shootMeta, requestAuthorizer.ToNamespace, requestAuthorizer.ToName), nil + return auth.DecisionNoOpinion, fmt.Sprintf("bootstrap token secret name %s does not match expected name %s for shoot %s/%s", attrs.GetName(), expectedSecretName, requestAuthorizer.ToNamespace, requestAuthorizer.ToName), nil + + case "delete": + // Allow gardenlet to delete bootstrap tokens for its own shoot or for ManagedSeeds referencing its shoot. + shootMeta, found, err := gardenletutils.ShootMetaFromBootstrapToken(ctx, a.client, attrs.GetName()) + if err != nil { + if !apierrors.IsNotFound(err) { + return auth.DecisionNoOpinion, "", err + } + } else if found { + if shootMeta.Namespace == requestAuthorizer.ToNamespace && shootMeta.Name == requestAuthorizer.ToName { + return auth.DecisionAllow, "", nil + } + return auth.DecisionNoOpinion, fmt.Sprintf("shoot meta in bootstrap token secret %s does not match with identity of requestor %s/%s", shootMeta, requestAuthorizer.ToNamespace, requestAuthorizer.ToName), nil + } + // No shoot meta found — fall through to graph-based authorization which handles ManagedSeed bootstrap + // tokens via the Secret → ManagedSeed → Shoot edges. } - // No shoot meta found — fall through to graph-based authorization which handles ManagedSeed bootstrap - // tokens via the Secret → ManagedSeed → Shoot edges. } return requestAuthorizer.Check(graph.VertexTypeSecret, attrs, diff --git a/pkg/admissioncontroller/webhook/auth/shoot/authorizer_test.go b/pkg/admissioncontroller/webhook/auth/shoot/authorizer_test.go index edfdbf1da28..471db0e91fd 100644 --- a/pkg/admissioncontroller/webhook/auth/shoot/authorizer_test.go +++ b/pkg/admissioncontroller/webhook/auth/shoot/authorizer_test.go @@ -20,6 +20,7 @@ import ( "k8s.io/apimachinery/pkg/fields" "k8s.io/apiserver/pkg/authentication/user" auth "k8s.io/apiserver/pkg/authorization/authorizer" + bootstraptokenutil "k8s.io/cluster-bootstrap/token/util" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" logzap "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -34,6 +35,7 @@ import ( "github.com/gardener/gardener/pkg/logger" graphutils "github.com/gardener/gardener/pkg/utils/graph" mockgraph "github.com/gardener/gardener/pkg/utils/graph/mock" + "github.com/gardener/gardener/pkg/utils/kubernetes/bootstraptoken" authorizerwebhook "github.com/gardener/gardener/pkg/webhook/authorizer" fakeauthorizerwebhook "github.com/gardener/gardener/pkg/webhook/authorizer/fake" ) @@ -2021,6 +2023,48 @@ var _ = Describe("Shoot", func() { } }) + When("get/list/watch of bootstrap token secret is requested", func() { + var expectedSecretName string + + BeforeEach(func() { + expectedSecretName = bootstraptokenutil.BootstrapTokenSecretName(bootstraptoken.TokenID(metav1.ObjectMeta{ + Namespace: shootNamespace, + Name: shootName, + })) + attrs.Namespace = metav1.NamespaceSystem + attrs.Name = expectedSecretName + }) + + DescribeTable("should allow for the deterministically derived secret name", + func(verb string) { + attrs.Verb = verb + + decision, reason, err := authorizer.Authorize(ctx, attrs) + Expect(err).NotTo(HaveOccurred()) + Expect(decision).To(Equal(auth.DecisionAllow)) + Expect(reason).To(BeEmpty()) + }, + Entry("get", "get"), + Entry("list", "list"), + Entry("watch", "watch"), + ) + + DescribeTable("should not have an opinion for a different bootstrap token secret name", + func(verb string) { + attrs.Verb = verb + attrs.Name = "bootstrap-token-abcdef" + + decision, reason, err := authorizer.Authorize(ctx, attrs) + Expect(err).NotTo(HaveOccurred()) + Expect(decision).To(Equal(auth.DecisionNoOpinion)) + Expect(reason).To(ContainSubstring("does not match expected name")) + }, + Entry("get", "get"), + Entry("list", "list"), + Entry("watch", "watch"), + ) + }) + When("deletion of bootstrap token secret is requested", func() { var bootstrapTokenSecret *corev1.Secret diff --git a/pkg/controller/gardenletdeployer/actuator.go b/pkg/controller/gardenletdeployer/actuator.go index e77843e97e7..00ee85bb7fe 100644 --- a/pkg/controller/gardenletdeployer/actuator.go +++ b/pkg/controller/gardenletdeployer/actuator.go @@ -8,6 +8,7 @@ import ( "context" "fmt" "slices" + "strings" "time" "github.com/go-logr/logr" @@ -920,8 +921,11 @@ func createBootstrapKubeconfig( return "", fmt.Errorf("unknown bootstrap kind for object of type %T", obj) } + // For self-hosted shoot Gardenlet resources the object name carries the `self-hosted-shoot-` prefix, + // but the authorizer computes the expected token name from the shoot name (without prefix). + // Strip the prefix so both sides derive the same deterministic token ID. var ( - tokenID = bootstraptoken.TokenID(metav1.ObjectMeta{Name: obj.GetName(), Namespace: obj.GetNamespace()}) + tokenID = bootstraptoken.TokenID(metav1.ObjectMeta{Name: strings.TrimPrefix(obj.GetName(), gardenletutils.ResourcePrefixSelfHostedShoot), Namespace: obj.GetNamespace()}) tokenDescription = gardenletbootstraputil.Description(kind, obj.GetNamespace(), obj.GetName()) tokenValidity = 24 * time.Hour ) diff --git a/pkg/operator/controller/garden/garden/reconciler_reconcile.go b/pkg/operator/controller/garden/garden/reconciler_reconcile.go index 571b3f2272b..ed81ff2a5af 100644 --- a/pkg/operator/controller/garden/garden/reconciler_reconcile.go +++ b/pkg/operator/controller/garden/garden/reconciler_reconcile.go @@ -508,7 +508,7 @@ func (r *Reconciler) reconcile( // gardenlets will succeed to execute the requested operation. // Therefore the 30s timeout is not sufficient in some cases and longer timeout is needed. renewGardenAccessSecretsInAllSeeds = g.Add(flow.Task{ - Name: "Annotate seeds to trigger renewal of their garden access secrets", + Name: "Annotate Seed resources to trigger renewal of their garden access secrets", Fn: flow.TaskFn(func(ctx context.Context) error { return secretsrotation.RenewGardenSecretsInAllSeeds(ctx, log.WithValues(secretsTypeKey, secretsTypeGardenAccess), virtualClusterClient, v1beta1constants.SeedOperationRenewGardenAccessSecrets) }).RetryUntilTimeout(defaultInterval, 10*time.Minute), @@ -524,7 +524,7 @@ func (r *Reconciler) reconcile( Dependencies: flow.NewTaskIDs(renewGardenAccessSecretsInAllSeeds), }) renewWorkloadIdentityTokensInAllSeeds = g.Add(flow.Task{ - Name: "Annotate seeds to trigger renewal of workload identity tokens", + Name: "Annotate Seed resources to trigger renewal of workload identity tokens", Fn: flow.TaskFn(func(ctx context.Context) error { return secretsrotation.RenewGardenSecretsInAllSeeds(ctx, log.WithValues(secretsTypeKey, secretsTypeWorkloadIdentity), virtualClusterClient, v1beta1constants.SeedOperationRenewWorkloadIdentityTokens) }).RetryUntilTimeout(defaultInterval, 10*time.Minute), @@ -539,29 +539,45 @@ func (r *Reconciler) reconcile( SkipIf: helper.GetWorkloadIdentityKeyRotationPhase(garden.Status.Credentials) != gardencorev1beta1.RotationPreparing, Dependencies: flow.NewTaskIDs(renewWorkloadIdentityTokensInAllSeeds), }) - renewGardenletKubeconfigInAllSeeds = g.Add(flow.Task{ - Name: "Annotate seeds to trigger renewal of their gardenlet kubeconfig", + renewKubeconfigsOfSeedGardenlets = g.Add(flow.Task{ + Name: "Annotate Seed resources to trigger renewal of their gardenlet kubeconfig", Fn: flow.TaskFn(func(ctx context.Context) error { return secretsrotation.RenewGardenSecretsInAllSeeds(ctx, log.WithValues(secretsTypeKey, secretsTypeGardenletKubeconfig), virtualClusterClient, v1beta1constants.GardenerOperationRenewKubeconfig) }).RetryUntilTimeout(defaultInterval, 10*time.Minute), SkipIf: helper.GetCARotationPhase(garden.Status.Credentials) != gardencorev1beta1.RotationPreparing, Dependencies: flow.NewTaskIDs(checkIfWorkloadIdentityTokensRenewalCompletedInAllSeeds), }) - checkIfGardenletKubeconfigRenewalCompletedInAllSeeds = g.Add(flow.Task{ - Name: "Check if all seeds finished the renewal of their gardenlet kubeconfig", + checkIfSeedGardenletKubeconfigRenewalsCompleted = g.Add(flow.Task{ + Name: "Check if all seed gardenlets finished the renewal of their kubeconfig", Fn: flow.TaskFn(func(ctx context.Context) error { return secretsrotation.CheckIfGardenSecretsRenewalCompletedInAllSeeds(ctx, virtualClusterClient, v1beta1constants.GardenerOperationRenewKubeconfig, secretsTypeGardenletKubeconfig) }).RetryUntilTimeout(defaultInterval, 10*time.Minute), SkipIf: helper.GetCARotationPhase(garden.Status.Credentials) != gardencorev1beta1.RotationPreparing, - Dependencies: flow.NewTaskIDs(renewGardenletKubeconfigInAllSeeds), + Dependencies: flow.NewTaskIDs(renewKubeconfigsOfSeedGardenlets), + }) + renewKubeconfigsOfShootGardenlets = g.Add(flow.Task{ + Name: "Annotate Gardenlet resources of self-hosted shoots to trigger renewal of their kubeconfig", + Fn: flow.TaskFn(func(ctx context.Context) error { + return secretsrotation.RenewKubeconfigInAllShootGardenlets(ctx, log.WithValues(secretsTypeKey, secretsTypeGardenletKubeconfig), virtualClusterClient) + }).RetryUntilTimeout(defaultInterval, 10*time.Minute), + SkipIf: helper.GetCARotationPhase(garden.Status.Credentials) != gardencorev1beta1.RotationPreparing, + Dependencies: flow.NewTaskIDs(initializeVirtualClusterClient), + }) + _ = g.Add(flow.Task{ + Name: "Check if all shoot gardenlets finished the renewal of their kubeconfig", + Fn: flow.TaskFn(func(ctx context.Context) error { + return secretsrotation.CheckIfKubeconfigRenewalCompletedInAllShootGardenlets(ctx, virtualClusterClient) + }).RetryUntilTimeout(defaultInterval, 10*time.Minute), + SkipIf: helper.GetCARotationPhase(garden.Status.Credentials) != gardencorev1beta1.RotationPreparing, + Dependencies: flow.NewTaskIDs(renewKubeconfigsOfShootGardenlets), }) _ = g.Add(flow.Task{ - Name: "Annotate seeds to trigger reconciliation after observability credentials rotation", + Name: "Annotate Seed resources to trigger reconciliation after observability credentials rotation", Fn: flow.TaskFn(func(ctx context.Context) error { return secretsrotation.RenewGardenSecretsInAllSeeds(ctx, log, virtualClusterClient, v1beta1constants.GardenerOperationReconcile) }).RetryUntilTimeout(defaultInterval, 10*time.Minute), SkipIf: !helper.IsObservabilityRotationInitiationTimeAfterLastCompletionTime(garden.Status.Credentials), - Dependencies: flow.NewTaskIDs(generateAndReplicateGlobalObservabilityIngressPassword, checkIfGardenletKubeconfigRenewalCompletedInAllSeeds), + Dependencies: flow.NewTaskIDs(generateAndReplicateGlobalObservabilityIngressPassword, checkIfSeedGardenletKubeconfigRenewalsCompleted), }) rewriteResourcesAddLabel = g.Add(flow.Task{ diff --git a/pkg/utils/gardener/gardenlet/gardenlet.go b/pkg/utils/gardener/gardenlet/gardenlet.go index 597c7643914..c406c84bb06 100644 --- a/pkg/utils/gardener/gardenlet/gardenlet.go +++ b/pkg/utils/gardener/gardenlet/gardenlet.go @@ -22,6 +22,7 @@ import ( gardenletconfigv1alpha1 "github.com/gardener/gardener/pkg/apis/config/gardenlet/v1alpha1" operatorv1alpha1 "github.com/gardener/gardener/pkg/apis/operator/v1alpha1" "github.com/gardener/gardener/pkg/apis/seedmanagement/encoding" + gardenletbootstraputil "github.com/gardener/gardener/pkg/gardenlet/bootstrap/util" operatorclient "github.com/gardener/gardener/pkg/operator/client" kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" "github.com/gardener/gardener/pkg/utils/kubernetes/bootstraptoken" @@ -101,19 +102,26 @@ func ShootMetaFromBootstrapToken(ctx context.Context, reader client.Reader, boot func extractShootMetaFromBootstrapToken(bootstrapTokenSecret *corev1.Secret) (types.NamespacedName, bool, error) { description := string(bootstrapTokenSecret.Data[bootstraptokenapi.BootstrapTokenDescriptionKey]) - if !strings.HasPrefix(description, bootstraptoken.SelfHostedShootBootstrapTokenSecretDescriptionPrefix) { - return types.NamespacedName{}, false, nil - } - parts := strings.Fields(strings.TrimPrefix(description, bootstraptoken.SelfHostedShootBootstrapTokenSecretDescriptionPrefix)) - if len(parts) == 0 { - return types.NamespacedName{}, false, fmt.Errorf("could not extract shoot meta from bootstrap token description: %s", description) + if strings.HasPrefix(description, bootstraptoken.SelfHostedShootBootstrapTokenSecretDescriptionPrefix) { + parts := strings.Fields(strings.TrimPrefix(description, bootstraptoken.SelfHostedShootBootstrapTokenSecretDescriptionPrefix)) + if len(parts) == 0 { + return types.NamespacedName{}, false, fmt.Errorf("could not extract shoot meta from bootstrap token description: %s", description) + } + + split := strings.Split(parts[0], "/") + if len(split) != 2 { + return types.NamespacedName{}, false, fmt.Errorf("could not extract shoot namespace and name from bootstrap token description: %s", description) + } + + return types.NamespacedName{Namespace: split[0], Name: split[1]}, true, nil } - split := strings.Split(parts[0], "/") - if len(split) != 2 { - return types.NamespacedName{}, false, fmt.Errorf("could not extract shoot namespace and name from bootstrap token description: %s", description) + // Bootstrap tokens for self-hosted shoot Gardenlets created via the deployer use the standard gardenlet + // description format. Extract the shoot namespace and name by stripping the `self-hosted-shoot-` prefix. + if kind, namespace, name := gardenletbootstraputil.MetadataFromDescription(description); kind == gardenletbootstraputil.KindGardenlet { + return types.NamespacedName{Namespace: namespace, Name: strings.TrimPrefix(name, ResourcePrefixSelfHostedShoot)}, true, nil } - return types.NamespacedName{Namespace: split[0], Name: split[1]}, true, nil + return types.NamespacedName{}, false, nil } diff --git a/pkg/utils/gardener/gardenlet/gardenlet_test.go b/pkg/utils/gardener/gardenlet/gardenlet_test.go index fe626ccd4f3..761fbce019d 100644 --- a/pkg/utils/gardener/gardenlet/gardenlet_test.go +++ b/pkg/utils/gardener/gardenlet/gardenlet_test.go @@ -385,5 +385,24 @@ var _ = Describe("Gardenlet", func() { Expect(err).NotTo(HaveOccurred()) Expect(found).To(BeFalse()) }) + + It("should successfully extract shoot meta from a Gardenlet bootstrap token (deployer format)", func() { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: bootstrapTokenSecretName, + Namespace: "kube-system", + }, + Data: map[string][]byte{ + "description": []byte("A bootstrap token for the Gardenlet for seedmanagement.gardener.cloud/v1alpha1.Gardenlet resource " + expectedShootNamespace + "/self-hosted-shoot-" + expectedShootName + "."), + }, + } + + Expect(fakeClient.Create(ctx, secret)).To(Succeed()) + + result, found, err := ShootMetaFromBootstrapToken(ctx, fakeClient, bootstrapTokenSecretName) + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(result).To(Equal(expectedNamespacedName)) + }) }) }) diff --git a/pkg/utils/gardener/secretsrotation/gardenaccess.go b/pkg/utils/gardener/secretsrotation/gardenaccess.go index b36fa572ba5..ac2dc7aae44 100644 --- a/pkg/utils/gardener/secretsrotation/gardenaccess.go +++ b/pkg/utils/gardener/secretsrotation/gardenaccess.go @@ -7,6 +7,8 @@ package secretsrotation import ( "context" "fmt" + "slices" + "strings" "github.com/go-logr/logr" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -14,7 +16,9 @@ import ( gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + seedmanagementv1alpha1 "github.com/gardener/gardener/pkg/apis/seedmanagement/v1alpha1" "github.com/gardener/gardener/pkg/utils/flow" + gardenletutils "github.com/gardener/gardener/pkg/utils/gardener/gardenlet" kubernetesutils "github.com/gardener/gardener/pkg/utils/kubernetes" ) @@ -38,7 +42,6 @@ func RenewGardenSecretsInAllSeeds(ctx context.Context, log logr.Logger, c client return fmt.Errorf("error annotating seed %s: already annotated with \"%s: %s\"", seed.Name, v1beta1constants.GardenerOperation, seed.Annotations[v1beta1constants.GardenerOperation]) } - seed := seed tasks = append(tasks, func(ctx context.Context) error { log := log.WithValues("seed", seed.Name) @@ -72,3 +75,67 @@ func CheckIfGardenSecretsRenewalCompletedInAllSeeds(ctx context.Context, c clien return nil } + +// RenewKubeconfigInAllShootGardenlets annotates all Gardenlet objects for self-hosted shoots to trigger renewal of +// their garden cluster kubeconfig. +func RenewKubeconfigInAllShootGardenlets(ctx context.Context, log logr.Logger, c client.Client) error { + gardenletList := &metav1.PartialObjectMetadataList{} + gardenletList.SetGroupVersionKind(seedmanagementv1alpha1.SchemeGroupVersion.WithKind("GardenletList")) + if err := c.List(ctx, gardenletList, client.InNamespace(v1beta1constants.GardenNamespace)); err != nil { + return err + } + + gardenletList.Items = slices.DeleteFunc(gardenletList.Items, func(objectMeta metav1.PartialObjectMetadata) bool { + return !strings.HasPrefix(objectMeta.Name, gardenletutils.ResourcePrefixSelfHostedShoot) + }) + + log.Info("Gardenlets requiring renewal of their kubeconfig", "number", len(gardenletList.Items)) + + var tasks []flow.TaskFn + for _, gardenlet := range gardenletList.Items { + if gardenlet.Annotations[v1beta1constants.GardenerOperation] == v1beta1constants.GardenerOperationRenewKubeconfig { + continue + } + + if gardenlet.Annotations[v1beta1constants.GardenerOperation] != "" { + return fmt.Errorf("error annotating gardenlet %s: already annotated with \"%s: %s\"", client.ObjectKeyFromObject(&gardenlet), v1beta1constants.GardenerOperation, gardenlet.Annotations[v1beta1constants.GardenerOperation]) + } + + tasks = append(tasks, func(ctx context.Context) error { + log := log.WithValues("gardenlet", client.ObjectKeyFromObject(&gardenlet)) + + gardenlet.SetGroupVersionKind(seedmanagementv1alpha1.SchemeGroupVersion.WithKind("Gardenlet")) + patch := client.MergeFrom(gardenlet.DeepCopy()) + kubernetesutils.SetMetaDataAnnotation(&gardenlet.ObjectMeta, v1beta1constants.GardenerOperation, v1beta1constants.GardenerOperationRenewKubeconfig) + if err := c.Patch(ctx, &gardenlet, patch); err != nil { + return fmt.Errorf("error annotating Gardenlet %s: %w", client.ObjectKeyFromObject(&gardenlet), err) + } + log.Info("Successfully annotated gardenlet to renew its kubeconfig") + return nil + }) + } + + return flow.ParallelN(5, tasks...)(ctx) +} + +// CheckIfKubeconfigRenewalCompletedInAllShootGardenlets checks if renewal of the garden cluster kubeconfig is +// completed for all Gardenlet objects for self-hosted shoots. +func CheckIfKubeconfigRenewalCompletedInAllShootGardenlets(ctx context.Context, c client.Client) error { + gardenletList := &metav1.PartialObjectMetadataList{} + gardenletList.SetGroupVersionKind(seedmanagementv1alpha1.SchemeGroupVersion.WithKind("GardenletList")) + if err := c.List(ctx, gardenletList, client.InNamespace(v1beta1constants.GardenNamespace)); err != nil { + return err + } + + gardenletList.Items = slices.DeleteFunc(gardenletList.Items, func(objectMeta metav1.PartialObjectMetadata) bool { + return !strings.HasPrefix(objectMeta.Name, gardenletutils.ResourcePrefixSelfHostedShoot) + }) + + for _, gardenlet := range gardenletList.Items { + if gardenlet.Annotations[v1beta1constants.GardenerOperation] == v1beta1constants.GardenerOperationRenewKubeconfig { + return fmt.Errorf("renewing kubeconfig for Gardenlet %s is not yet completed", client.ObjectKeyFromObject(&gardenlet)) + } + } + + return nil +} diff --git a/pkg/utils/gardener/secretsrotation/gardenaccess_test.go b/pkg/utils/gardener/secretsrotation/gardenaccess_test.go index df4d584424f..79c2bda1ced 100644 --- a/pkg/utils/gardener/secretsrotation/gardenaccess_test.go +++ b/pkg/utils/gardener/secretsrotation/gardenaccess_test.go @@ -15,6 +15,7 @@ import ( fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + seedmanagementv1alpha1 "github.com/gardener/gardener/pkg/apis/seedmanagement/v1alpha1" "github.com/gardener/gardener/pkg/client/kubernetes" . "github.com/gardener/gardener/pkg/utils/gardener/secretsrotation" ) @@ -151,4 +152,121 @@ var _ = Describe("RenewGardenAccess", func() { Expect(RenewGardenSecretsInAllSeeds(ctx, logger.WithValues(secretType, gardenAccess), gardenClient, renewGardenAccessSecrets)).To(MatchError(ContainSubstring("error annotating seed seed1: already annotated with \"gardener.cloud/operation: reconcile\""))) }) }) + + Context("#RenewKubeconfigInAllShootGardenlets", func() { + var gardenlets []seedmanagementv1alpha1.Gardenlet + + BeforeEach(func() { + gardenlets = []seedmanagementv1alpha1.Gardenlet{ + {ObjectMeta: metav1.ObjectMeta{Name: "self-hosted-shoot-g1", Namespace: "garden"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "self-hosted-shoot-g2", Namespace: "garden"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "self-hosted-shoot-g3", Namespace: "garden"}}, + } + }) + + createGardenlets := func() error { + for _, gardenlet := range gardenlets { + if err := gardenClient.Create(ctx, &gardenlet); err != nil { + return err + } + } + return nil + } + + It("should succeed and annotate all self-hosted-shoot gardenlets", func() { + Expect(createGardenlets()).To(Succeed()) + + Expect(RenewKubeconfigInAllShootGardenlets(ctx, logger.WithValues(secretType, gardenletKubeconfig), gardenClient)).To(Succeed()) + + gardenletList := seedmanagementv1alpha1.GardenletList{} + Expect(gardenClient.List(ctx, &gardenletList)).To(Succeed()) + for _, gardenlet := range gardenletList.Items { + Expect(gardenlet.Annotations["gardener.cloud/operation"]).To(Equal(renewKubeconfig)) + } + }) + + It("should succeed if some gardenlets are already annotated with `renew-kubeconfig`", func() { + gardenlets[0].SetAnnotations(map[string]string{"gardener.cloud/operation": renewKubeconfig}) + Expect(createGardenlets()).To(Succeed()) + + Expect(RenewKubeconfigInAllShootGardenlets(ctx, logger.WithValues(secretType, gardenletKubeconfig), gardenClient)).To(Succeed()) + + gardenletList := seedmanagementv1alpha1.GardenletList{} + Expect(gardenClient.List(ctx, &gardenletList)).To(Succeed()) + for _, gardenlet := range gardenletList.Items { + Expect(gardenlet.Annotations["gardener.cloud/operation"]).To(Equal(renewKubeconfig)) + } + }) + + It("should fail if some gardenlets have a different `gardener.cloud/operation` annotation", func() { + gardenlets[0].SetAnnotations(map[string]string{"gardener.cloud/operation": "reconcile"}) + Expect(createGardenlets()).To(Succeed()) + + Expect(RenewKubeconfigInAllShootGardenlets(ctx, logger.WithValues(secretType, gardenletKubeconfig), gardenClient)).To(MatchError(ContainSubstring("error annotating gardenlet garden/self-hosted-shoot-g1: already annotated with \"gardener.cloud/operation: reconcile\""))) + }) + + It("should skip gardenlets not related to self-hosted shoots", func() { + nonSelfHosted := seedmanagementv1alpha1.Gardenlet{ObjectMeta: metav1.ObjectMeta{Name: "managed-seed-gardenlet", Namespace: "garden"}} + Expect(gardenClient.Create(ctx, &nonSelfHosted)).To(Succeed()) + Expect(createGardenlets()).To(Succeed()) + + Expect(RenewKubeconfigInAllShootGardenlets(ctx, logger.WithValues(secretType, gardenletKubeconfig), gardenClient)).To(Succeed()) + + updated := &seedmanagementv1alpha1.Gardenlet{} + Expect(gardenClient.Get(ctx, client.ObjectKeyFromObject(&nonSelfHosted), updated)).To(Succeed()) + Expect(updated.Annotations["gardener.cloud/operation"]).To(BeEmpty()) + }) + }) + + Context("#CheckIfKubeconfigRenewalCompletedInAllShootGardenlets", func() { + var gardenlets []seedmanagementv1alpha1.Gardenlet + + BeforeEach(func() { + gardenlets = []seedmanagementv1alpha1.Gardenlet{ + {ObjectMeta: metav1.ObjectMeta{Name: "self-hosted-shoot-g1", Namespace: "garden"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "self-hosted-shoot-g2", Namespace: "garden"}}, + } + }) + + createGardenlets := func() error { + for _, gardenlet := range gardenlets { + if err := gardenClient.Create(ctx, &gardenlet); err != nil { + return err + } + } + return nil + } + + It("should succeed if no gardenlet is annotated anymore", func() { + Expect(createGardenlets()).To(Succeed()) + + Expect(CheckIfKubeconfigRenewalCompletedInAllShootGardenlets(ctx, gardenClient)).To(Succeed()) + }) + + It("should succeed if some gardenlets have a different `gardener.cloud/operation` annotation", func() { + gardenlets[0].SetAnnotations(map[string]string{"gardener.cloud/operation": "reconcile"}) + Expect(createGardenlets()).To(Succeed()) + + Expect(CheckIfKubeconfigRenewalCompletedInAllShootGardenlets(ctx, gardenClient)).To(Succeed()) + }) + + It("should fail if some gardenlets are still annotated with `renew-kubeconfig`", func() { + gardenlets[1].SetAnnotations(map[string]string{"gardener.cloud/operation": renewKubeconfig}) + Expect(createGardenlets()).To(Succeed()) + + Expect(CheckIfKubeconfigRenewalCompletedInAllShootGardenlets(ctx, gardenClient)).To(MatchError(ContainSubstring("renewing kubeconfig for Gardenlet garden/self-hosted-shoot-g2 is not yet completed"))) + }) + + It("should succeed if only non-self-hosted-shoot gardenlets are still annotated with `renew-kubeconfig`", func() { + nonSelfHosted := seedmanagementv1alpha1.Gardenlet{ObjectMeta: metav1.ObjectMeta{ + Name: "managed-seed-gardenlet", + Namespace: "garden", + Annotations: map[string]string{"gardener.cloud/operation": renewKubeconfig}, + }} + Expect(gardenClient.Create(ctx, &nonSelfHosted)).To(Succeed()) + Expect(createGardenlets()).To(Succeed()) + + Expect(CheckIfKubeconfigRenewalCompletedInAllShootGardenlets(ctx, gardenClient)).To(Succeed()) + }) + }) }) diff --git a/pkg/utils/graph/eventhandler_gardenlet.go b/pkg/utils/graph/eventhandler_gardenlet.go index e583660a46c..51e6ba03425 100644 --- a/pkg/utils/graph/eventhandler_gardenlet.go +++ b/pkg/utils/graph/eventhandler_gardenlet.go @@ -164,7 +164,4 @@ func (g *graph) handleGardenletCreateOrUpdateForShoots(gardenlet *seedmanagement ) g.addEdge(gardenletVertex, shootVertex) - - // TODO(rfranzke): Check if we need to support the 'allowBootstrap' logic for self-hosted shoots as well (see - // handling for seeds). } From f309580150327e9e0da8b5b52af768d4b728ebe4 Mon Sep 17 00:00:00 2001 From: Shafeeque E S Date: Wed, 5 Aug 2026 19:29:37 +0530 Subject: [PATCH 3/4] Increase timeout to 10m for VerifyInPlaceUpdateCompletion (#15443) --- test/utils/shoots/update/inplace/shoot.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/utils/shoots/update/inplace/shoot.go b/test/utils/shoots/update/inplace/shoot.go index 90e80cacb91..a2eb9dad05d 100644 --- a/test/utils/shoots/update/inplace/shoot.go +++ b/test/utils/shoots/update/inplace/shoot.go @@ -62,7 +62,7 @@ func ItShouldVerifyInPlaceUpdateCompletion(s *ShootContext) { It("Verify in-place update completion", func(ctx SpecContext) { VerifyInPlaceUpdateCompletion(ctx, s.Log, s.GardenClient, s.Shoot) - }, SpecTimeout(5*time.Minute)) + }, SpecTimeout(10*time.Minute)) } // VerifyInPlaceUpdateCompletion verifies that the in-place update was completed successfully by checking the From 06a3e2ff6f4d2abb3801667e12ce583e1b810f95 Mon Sep 17 00:00:00 2001 From: Victor Herrero Otal Date: Wed, 5 Aug 2026 18:23:12 +0200 Subject: [PATCH 4/4] Ensure global observability secret is synced during rotation before triggering seed reconciliation (#15437) * Ensure global observability secret is synced before seed reconciliation Commit `d393a96` added rotation for the global observability ingress secret. However, it missed a race condition that is only visible in real Gardener landscapes. The rotation assumes the following order: t0: The secrets manager generates a new global observability ingress secret in the runtime cluster and copies it to the `garden` namespace in the virtual garden cluster. t1: The gardener-controller-manager (GCM) sees the new secret in the `garden` namespace and propagates it to each seed namespace. t2: The gardener-operator annotates the seeds for reconciliation. t3: The gardenlet picks up and deploys the new global observability ingress secret in the seed. This path works, but t1 and t2 swap if the gardener-operator triggers seed reconciliation before the GCM copies the new secret to the corresponding seed namespaces. This commit propagates the `last-rotation-initiation-time` label to the secret replicas, and adds a step to the gardener-operator rotation flow between t1 and t2. The step blocks until every seed namespace in the virtual garden holds a replica whose `last-rotation-initiation-time` is greater than or equal to the one on the original secret in the runtime cluster. This enforces the t1-before-t2 ordering: the operator does not annotate the seeds until the GCM has finished propagating the new secret. * Deduplicate secret creation in observability rotation tests The tests that exercise empty and non-numeric `last-rotation-initiation-time` label values inlined the full secret creation. This adds a string-based helper that carries the label value as a string. --- dev-setup/skaffold-gardenadm.yaml | 2 + dev-setup/skaffold-operator.yaml | 5 + dev-setup/skaffold-seed.yaml | 2 + .../garden/garden/reconciler_reconcile.go | 22 +++- pkg/utils/gardener/secrets.go | 5 + pkg/utils/gardener/secrets_test.go | 4 +- .../gardener/secretsrotation/observability.go | 65 ++++++++++ .../secretsrotation/observability_test.go | 114 ++++++++++++++++++ 8 files changed, 217 insertions(+), 2 deletions(-) create mode 100644 pkg/utils/gardener/secretsrotation/observability.go create mode 100644 pkg/utils/gardener/secretsrotation/observability_test.go diff --git a/dev-setup/skaffold-gardenadm.yaml b/dev-setup/skaffold-gardenadm.yaml index 78573a33a8f..0d0ba5ec193 100644 --- a/dev-setup/skaffold-gardenadm.yaml +++ b/dev-setup/skaffold-gardenadm.yaml @@ -389,6 +389,7 @@ build: - pkg/utils/managedresources/builder - pkg/utils/retry - pkg/utils/secrets + - pkg/utils/secrets/manager - pkg/utils/signals - pkg/utils/structuredmap - pkg/utils/validation @@ -510,6 +511,7 @@ build: - pkg/utils/kubernetes/unstructured - pkg/utils/retry - pkg/utils/secrets + - pkg/utils/secrets/manager - pkg/utils/signals - pkg/utils/validation - pkg/utils/validation/admissionplugins diff --git a/dev-setup/skaffold-operator.yaml b/dev-setup/skaffold-operator.yaml index 715f72a3cee..07eb12a7339 100644 --- a/dev-setup/skaffold-operator.yaml +++ b/dev-setup/skaffold-operator.yaml @@ -359,6 +359,7 @@ build: - pkg/utils/kubernetes/unstructured - pkg/utils/retry - pkg/utils/secrets + - pkg/utils/secrets/manager - pkg/utils/signals - pkg/utils/validation - pkg/utils/validation/admissionplugins @@ -532,6 +533,7 @@ build: - pkg/utils/kubernetes/unstructured - pkg/utils/retry - pkg/utils/secrets + - pkg/utils/secrets/manager - pkg/utils/signals - pkg/utils/validation - pkg/utils/validation/admissionplugins @@ -711,6 +713,7 @@ build: - pkg/utils/managedresources/builder - pkg/utils/retry - pkg/utils/secrets + - pkg/utils/secrets/manager - pkg/utils/signals - pkg/utils/validation - pkg/utils/validation/admissionplugins @@ -791,6 +794,7 @@ build: - pkg/utils/kubernetes/unstructured - pkg/utils/retry - pkg/utils/secrets + - pkg/utils/secrets/manager - pkg/utils/signals - pkg/utils/validation - pkg/utils/validation/cidr @@ -891,6 +895,7 @@ build: - pkg/utils/kubernetes/unstructured - pkg/utils/retry - pkg/utils/secrets + - pkg/utils/secrets/manager - pkg/utils/signals - pkg/utils/validation - pkg/utils/validation/kubernetesversion diff --git a/dev-setup/skaffold-seed.yaml b/dev-setup/skaffold-seed.yaml index 1b3e55edc74..36da979adce 100644 --- a/dev-setup/skaffold-seed.yaml +++ b/dev-setup/skaffold-seed.yaml @@ -454,6 +454,7 @@ build: - pkg/utils/kubernetes/unstructured - pkg/utils/retry - pkg/utils/secrets + - pkg/utils/secrets/manager - pkg/utils/signals - pkg/utils/validation - pkg/utils/validation/admissionplugins @@ -579,6 +580,7 @@ build: - pkg/utils/managedresources/builder - pkg/utils/retry - pkg/utils/secrets + - pkg/utils/secrets/manager - pkg/utils/signals - pkg/utils/structuredmap - pkg/utils/validation diff --git a/pkg/operator/controller/garden/garden/reconciler_reconcile.go b/pkg/operator/controller/garden/garden/reconciler_reconcile.go index ed81ff2a5af..5f2b873908a 100644 --- a/pkg/operator/controller/garden/garden/reconciler_reconcile.go +++ b/pkg/operator/controller/garden/garden/reconciler_reconcile.go @@ -11,6 +11,7 @@ import ( "net" "net/url" "slices" + "strconv" "strings" "time" @@ -175,6 +176,8 @@ func (r *Reconciler) reconcile( encryptionProviderToUse = v1beta1helper.GetEncryptionProviderType(garden.Spec.VirtualCluster.Kubernetes.KubeAPIServer.KubeAPIServerConfig) encryptionProvider = helper.GetEncryptionProviderTypeInStatus(garden.Status) + globalObservabilitySecretLastRotationInitiationTimestamp int64 + g = flow.NewGraph("Garden reconciliation") generateGenericTokenKubeconfig = g.Add(flow.Task{ Name: "Generating generic token kubeconfig", @@ -483,6 +486,15 @@ func (r *Reconciler) reconcile( return fmt.Errorf("failed to generate global observability ingress secret: %w", err) } + // accept missing and empty last-rotation-initiation-time label values for human-generated or + // brand new generated secrets, but fail for any other content that cannot be parsed as an integer. + if lastRotationInitiationTime := secret.Labels[secretsmanager.LabelKeyLastRotationInitiationTime]; lastRotationInitiationTime != "" { + globalObservabilitySecretLastRotationInitiationTimestamp, err = strconv.ParseInt(lastRotationInitiationTime, 10, 64) + if err != nil { + return fmt.Errorf("error parsing last rotation initiation time of global observability secret in namespace %q: %w", r.GardenNamespace, err) + } + } + _, err = gardenerutils.ReplicateGlobalMonitoringSecret(ctx, virtualClusterClient, secret, r.GardenNamespace, func(name string) string { return strings.TrimPrefix(name, "global-") }) @@ -563,6 +575,14 @@ func (r *Reconciler) reconcile( SkipIf: helper.GetCARotationPhase(garden.Status.Credentials) != gardencorev1beta1.RotationPreparing, Dependencies: flow.NewTaskIDs(initializeVirtualClusterClient), }) + waitUntilGlobalObservabilitySecretPropagatedToAllSeeds = g.Add(flow.Task{ + Name: "Wait until global observability secret is propagated to all seed namespaces", + Fn: flow.TaskFn(func(ctx context.Context) error { + return secretsrotation.CheckIfGlobalObservabilitySecretPropagatedToAllSeeds(ctx, virtualClusterClient, globalObservabilitySecretLastRotationInitiationTimestamp) + }).RetryUntilTimeout(defaultInterval, 10*time.Minute), + SkipIf: !helper.IsObservabilityRotationInitiationTimeAfterLastCompletionTime(garden.Status.Credentials), + Dependencies: flow.NewTaskIDs(generateAndReplicateGlobalObservabilityIngressPassword), + }) _ = g.Add(flow.Task{ Name: "Check if all shoot gardenlets finished the renewal of their kubeconfig", Fn: flow.TaskFn(func(ctx context.Context) error { @@ -577,7 +597,7 @@ func (r *Reconciler) reconcile( return secretsrotation.RenewGardenSecretsInAllSeeds(ctx, log, virtualClusterClient, v1beta1constants.GardenerOperationReconcile) }).RetryUntilTimeout(defaultInterval, 10*time.Minute), SkipIf: !helper.IsObservabilityRotationInitiationTimeAfterLastCompletionTime(garden.Status.Credentials), - Dependencies: flow.NewTaskIDs(generateAndReplicateGlobalObservabilityIngressPassword, checkIfSeedGardenletKubeconfigRenewalsCompleted), + Dependencies: flow.NewTaskIDs(waitUntilGlobalObservabilitySecretPropagatedToAllSeeds, checkIfSeedGardenletKubeconfigRenewalsCompleted), }) rewriteResourcesAddLabel = g.Add(flow.Task{ diff --git a/pkg/utils/gardener/secrets.go b/pkg/utils/gardener/secrets.go index 30b1701b099..36850c638e0 100644 --- a/pkg/utils/gardener/secrets.go +++ b/pkg/utils/gardener/secrets.go @@ -26,6 +26,7 @@ import ( "github.com/gardener/gardener/pkg/controllerutils" "github.com/gardener/gardener/pkg/utils" secretsutils "github.com/gardener/gardener/pkg/utils/secrets" + secretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager" ) var ( @@ -68,6 +69,10 @@ func ReplicateGlobalMonitoringSecret(ctx context.Context, c client.Client, globa metav1.SetMetaDataLabel(&globalMonitoringSecretReplica.ObjectMeta, v1beta1constants.GardenRole, v1beta1constants.GardenRoleGlobalMonitoring) metav1.SetMetaDataLabel(&globalMonitoringSecretReplica.ObjectMeta, v1beta1constants.GardenerPurpose, LabelPurposeGlobalMonitoringSecret) + if rotationInitiationTime, ok := globalMonitoringSecret.Labels[secretsmanager.LabelKeyLastRotationInitiationTime]; ok { + metav1.SetMetaDataLabel(&globalMonitoringSecretReplica.ObjectMeta, secretsmanager.LabelKeyLastRotationInitiationTime, rotationInitiationTime) + } + globalMonitoringSecretReplica.Type = globalMonitoringSecret.Type globalMonitoringSecretReplica.Data = globalMonitoringSecret.Data globalMonitoringSecretReplica.Immutable = globalMonitoringSecret.Immutable diff --git a/pkg/utils/gardener/secrets_test.go b/pkg/utils/gardener/secrets_test.go index 07fe4b50f83..34743da0b83 100644 --- a/pkg/utils/gardener/secrets_test.go +++ b/pkg/utils/gardener/secrets_test.go @@ -87,7 +87,7 @@ var _ = Describe("Secrets", func() { ObjectMeta: metav1.ObjectMeta{ Name: "global-monitoring-secret", Namespace: "foo", - Labels: map[string]string{"bar": "baz"}, + Labels: map[string]string{"bar": "baz", "last-rotation-initiation-time": "1700000000"}, Annotations: map[string]string{"baz": "foo"}, }, Type: corev1.SecretTypeOpaque, @@ -104,6 +104,8 @@ var _ = Describe("Secrets", func() { assertions := func(secret *corev1.Secret) { Expect(secret.Labels).To(HaveKeyWithValue("gardener.cloud/role", "global-monitoring")) Expect(secret.Labels).To(HaveKeyWithValue("gardener.cloud/purpose", "global-monitoring-secret-replica")) + Expect(secret.Labels).To(HaveKeyWithValue("last-rotation-initiation-time", "1700000000")) + Expect(secret.Labels).NotTo(HaveKey("bar")) Expect(secret.Type).To(Equal(globalMonitoringSecret.Type)) Expect(secret.Immutable).To(Equal(globalMonitoringSecret.Immutable)) for k, v := range globalMonitoringSecret.Data { diff --git a/pkg/utils/gardener/secretsrotation/observability.go b/pkg/utils/gardener/secretsrotation/observability.go new file mode 100644 index 00000000000..76dbd31c4da --- /dev/null +++ b/pkg/utils/gardener/secretsrotation/observability.go @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package secretsrotation + +import ( + "context" + "fmt" + "strconv" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" + secretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager" +) + +// CheckIfGlobalObservabilitySecretPropagatedToAllSeeds waits until the global observability secret has been synced by the +// gardener-controller-manager from the garden namespace into every seed namespace of the virtual cluster. +func CheckIfGlobalObservabilitySecretPropagatedToAllSeeds(ctx context.Context, c client.Client, lastRotationInitiationTimestamp int64) error { + seedList := &metav1.PartialObjectMetadataList{} + seedList.SetGroupVersionKind(gardencorev1beta1.SchemeGroupVersion.WithKind("SeedList")) + if err := c.List(ctx, seedList); err != nil { + return fmt.Errorf("failed to list seeds: %w", err) + } + + for _, seed := range seedList.Items { + seedNamespace := gardenerutils.ComputeGardenNamespace(seed.Name) + + secretList := &metav1.PartialObjectMetadataList{} + secretList.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("SecretList")) + secretSelector := client.MatchingLabels{v1beta1constants.GardenRole: v1beta1constants.GardenRoleGlobalMonitoring} + if err := c.List(ctx, secretList, client.InNamespace(seedNamespace), secretSelector); err != nil { + return fmt.Errorf("failed to list global observability secrets in namespace %q of seed %q: %w", seedNamespace, seed.Name, err) + } + + for _, secret := range secretList.Items { + secretLastRotationInitiationTime, ok := secret.Labels[secretsmanager.LabelKeyLastRotationInitiationTime] + + // accept missing last-rotation-initiation-time label values, e.g., human-managed secrets. + if !ok { + continue + } + + // fail empty last-rotation-initiation-time label values, e.g., the secret is being rotated for the first time. + if secretLastRotationInitiationTime == "" { + return fmt.Errorf("global observability secret in namespace %q of seed %q does not yet carry a last rotation initiation time", seedNamespace, seed.Name) + } + + secretLastRotationInitiationTimestamp, err := strconv.ParseInt(secretLastRotationInitiationTime, 10, 64) + if err != nil { + return fmt.Errorf("error parsing last rotation initiation time of global observability secret in namespace %q of seed %q: %w", seedNamespace, seed.Name, err) + } + if secretLastRotationInitiationTimestamp < lastRotationInitiationTimestamp { + return fmt.Errorf("global observability secret is not yet propagated to namespace %q of seed %q", seedNamespace, seed.Name) + } + } + } + + return nil +} diff --git a/pkg/utils/gardener/secretsrotation/observability_test.go b/pkg/utils/gardener/secretsrotation/observability_test.go new file mode 100644 index 00000000000..f32ceac257e --- /dev/null +++ b/pkg/utils/gardener/secretsrotation/observability_test.go @@ -0,0 +1,114 @@ +// SPDX-FileCopyrightText: SAP SE or an SAP affiliate company and Gardener contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package secretsrotation_test + +import ( + "context" + "strconv" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + + gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1" + v1beta1constants "github.com/gardener/gardener/pkg/apis/core/v1beta1/constants" + "github.com/gardener/gardener/pkg/client/kubernetes" + gardenerutils "github.com/gardener/gardener/pkg/utils/gardener" + . "github.com/gardener/gardener/pkg/utils/gardener/secretsrotation" + secretsmanager "github.com/gardener/gardener/pkg/utils/secrets/manager" +) + +var _ = Describe("Observability", func() { + Context("#CheckIfGlobalObservabilitySecretPropagatedToAllSeeds", func() { + const lastRotationInitiationTimestamp = 1700000000 + + var ( + ctx context.Context + gardenClient client.Client + seeds []gardencorev1beta1.Seed + ) + + BeforeEach(func() { + ctx = context.TODO() + gardenClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.GardenScheme).Build() + + seeds = []gardencorev1beta1.Seed{ + {ObjectMeta: metav1.ObjectMeta{Name: "seed1"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "seed2"}}, + } + for _, seed := range seeds { + Expect(gardenClient.Create(ctx, &seed)).To(Succeed()) + } + }) + + createGlobalMonitoringSecretWithRawTimestamp := func(seedName, timestamp string) { + Expect(gardenClient.Create(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ + Name: "observability-ingress", + Namespace: gardenerutils.ComputeGardenNamespace(seedName), + Labels: map[string]string{ + v1beta1constants.GardenRole: v1beta1constants.GardenRoleGlobalMonitoring, + secretsmanager.LabelKeyLastRotationInitiationTime: timestamp, + }, + }})).To(Succeed()) + } + + createGlobalMonitoringSecret := func(seedName string, timestamp int) { + createGlobalMonitoringSecretWithRawTimestamp(seedName, strconv.Itoa(timestamp)) + } + + createGlobalMonitoringSecretWithoutTimestamp := func(seedName string) { + Expect(gardenClient.Create(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ + Name: "observability-ingress", + Namespace: gardenerutils.ComputeGardenNamespace(seedName), + Labels: map[string]string{v1beta1constants.GardenRole: v1beta1constants.GardenRoleGlobalMonitoring}, + }})).To(Succeed()) + } + + It("should succeed when the secret propagated to all seeds with the expected timestamp", func() { + createGlobalMonitoringSecret("seed1", lastRotationInitiationTimestamp) + createGlobalMonitoringSecret("seed2", lastRotationInitiationTimestamp) + + Expect(CheckIfGlobalObservabilitySecretPropagatedToAllSeeds(ctx, gardenClient, lastRotationInitiationTimestamp)).To(Succeed()) + }) + + It("should succeed when a seed carries a newer timestamp than expected", func() { + createGlobalMonitoringSecret("seed1", lastRotationInitiationTimestamp) + createGlobalMonitoringSecret("seed2", lastRotationInitiationTimestamp+1) + + Expect(CheckIfGlobalObservabilitySecretPropagatedToAllSeeds(ctx, gardenClient, lastRotationInitiationTimestamp)).To(Succeed()) + }) + + It("should fail when a seed still carries an older timestamp", func() { + createGlobalMonitoringSecret("seed1", lastRotationInitiationTimestamp) + createGlobalMonitoringSecret("seed2", lastRotationInitiationTimestamp-1) + + Expect(CheckIfGlobalObservabilitySecretPropagatedToAllSeeds(ctx, gardenClient, lastRotationInitiationTimestamp)).To(MatchError(ContainSubstring("not yet propagated to namespace \"seed-seed2\" of seed \"seed2\""))) + }) + + It("should ignore a secret without the rotation initiation time label", func() { + createGlobalMonitoringSecret("seed1", lastRotationInitiationTimestamp) + createGlobalMonitoringSecretWithoutTimestamp("seed2") + + Expect(CheckIfGlobalObservabilitySecretPropagatedToAllSeeds(ctx, gardenClient, lastRotationInitiationTimestamp)).To(Succeed()) + }) + + It("should fail when a secret carries an empty rotation initiation time label", func() { + createGlobalMonitoringSecret("seed1", lastRotationInitiationTimestamp) + createGlobalMonitoringSecretWithRawTimestamp("seed2", "") + + Expect(CheckIfGlobalObservabilitySecretPropagatedToAllSeeds(ctx, gardenClient, lastRotationInitiationTimestamp)).To(MatchError(ContainSubstring("does not yet carry a last rotation initiation time"))) + }) + + It("should fail when a secret carries a non-numeric rotation initiation time label", func() { + createGlobalMonitoringSecret("seed1", lastRotationInitiationTimestamp) + createGlobalMonitoringSecretWithRawTimestamp("seed2", "not-a-number") + + Expect(CheckIfGlobalObservabilitySecretPropagatedToAllSeeds(ctx, gardenClient, lastRotationInitiationTimestamp)).To(MatchError(ContainSubstring("error parsing last rotation initiation time"))) + }) + }) +})