From 30491114ca517c47c1acf116d57a1a7b45562eb2 Mon Sep 17 00:00:00 2001 From: Nont <9658731+nwnt@users.noreply.github.com> Date: Wed, 17 Sep 2025 10:32:13 -0500 Subject: [PATCH 1/8] fix: add validation for PickFixed cluster names (#252) --- pkg/utils/validator/clusterresourceplacement.go | 7 +++++++ .../validator/clusterresourceplacement_test.go | 16 ++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/pkg/utils/validator/clusterresourceplacement.go b/pkg/utils/validator/clusterresourceplacement.go index 1b5fc2629..e07e9b56a 100644 --- a/pkg/utils/validator/clusterresourceplacement.go +++ b/pkg/utils/validator/clusterresourceplacement.go @@ -194,6 +194,13 @@ func validatePolicyForPickFixedPlacementType(policy *placementv1beta1.PlacementP } uniqueClusterNames := make(map[string]bool) for _, name := range policy.ClusterNames { + nameErr := validation.IsDNS1123Subdomain(name) + if nameErr != nil { + allErr = append(allErr, fmt.Errorf("PickFixed cluster name %s is not a valid member name: %s", name, strings.Join(nameErr, "; "))) + } + if len(name) > validation.DNS1035LabelMaxLength { + allErr = append(allErr, fmt.Errorf("PickFixed cluster name %s cannot have length exceeding %d", name, validation.DNS1035LabelMaxLength)) + } if _, ok := uniqueClusterNames[name]; ok { allErr = append(allErr, fmt.Errorf("cluster names must be unique for policy type %s", placementv1beta1.PickFixedPlacementType)) break diff --git a/pkg/utils/validator/clusterresourceplacement_test.go b/pkg/utils/validator/clusterresourceplacement_test.go index a3cbab952..dbec2699d 100644 --- a/pkg/utils/validator/clusterresourceplacement_test.go +++ b/pkg/utils/validator/clusterresourceplacement_test.go @@ -514,6 +514,22 @@ func TestValidateClusterResourcePlacement_PickFixedPlacementPolicy(t *testing.T) wantErr: true, wantErrMsg: "cluster names must be unique for policy type PickFixed", }, + "invalid placement policy - PickFixed with invalid cluster names": { + policy: &placementv1beta1.PlacementPolicy{ + PlacementType: placementv1beta1.PickFixedPlacementType, + ClusterNames: []string{"test@,cluster1"}, + }, + wantErr: true, + wantErrMsg: "PickFixed cluster name test@,cluster1 is not a valid member name: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*')", + }, + "invalid placement policy - PickFixed with too long cluster name": { + policy: &placementv1beta1.PlacementPolicy{ + PlacementType: placementv1beta1.PickFixedPlacementType, + ClusterNames: []string{"this-is-a-very-long-cluster-name-that-exceeds-the-maximum-allowed-length-for-dns-labels"}, + }, + wantErr: true, + wantErrMsg: "PickFixed cluster name this-is-a-very-long-cluster-name-that-exceeds-the-maximum-allowed-length-for-dns-labels cannot have length exceeding 63", + }, "invalid placement policy - PickFixed with non nil number of clusters": { policy: &placementv1beta1.PlacementPolicy{ PlacementType: placementv1beta1.PickFixedPlacementType, From 8bd12e04bcd59052c8d1c9b96d3c247b5558f71a Mon Sep 17 00:00:00 2001 From: Arvind Thirumurugan Date: Thu, 18 Sep 2025 17:52:49 -0700 Subject: [PATCH 2/8] feat: handle namespace resource selector change CRPS (#240) --- .../v1beta1/clusterresourceplacement_types.go | 11 + pkg/controllers/placement/controller.go | 38 +- .../placement/controller_integration_test.go | 56 +- .../placement/namespace_status_sync.go | 256 ++++++- .../placement/namespace_status_sync_test.go | 636 +++++++++++------- pkg/utils/condition/reason.go | 6 + test/e2e/actuals_test.go | 154 ++++- test/e2e/placement_status_sync_test.go | 213 +++++- test/utils/crpstatussync/crp_status_sync.go | 12 +- 9 files changed, 1018 insertions(+), 364 deletions(-) diff --git a/apis/placement/v1beta1/clusterresourceplacement_types.go b/apis/placement/v1beta1/clusterresourceplacement_types.go index a76a73758..5a6678d19 100644 --- a/apis/placement/v1beta1/clusterresourceplacement_types.go +++ b/apis/placement/v1beta1/clusterresourceplacement_types.go @@ -1287,6 +1287,17 @@ const ( // clusters, or an error has occurred. // * Unknown: Fleet has not finished processing the diff reporting yet. ClusterResourcePlacementDiffReportedConditionType ClusterResourcePlacementConditionType = "ClusterResourcePlacementDiffReported" + + // ClusterResourcePlacementStatusSyncedConditionType indicates whether Fleet has successfully + // created or updated the ClusterResourcePlacementStatus object in the target namespace when + // StatusReportingScope is NamespaceAccessible. + // + // It can have the following condition statuses: + // * True: Fleet has successfully created or updated the ClusterResourcePlacementStatus object + // in the target namespace. + // * False: Fleet has failed to create or update the ClusterResourcePlacementStatus object + // in the target namespace. + ClusterResourcePlacementStatusSyncedConditionType ClusterResourcePlacementConditionType = "ClusterResourcePlacementStatusSynced" ) // ResourcePlacementConditionType defines a specific condition of a resource placement object. diff --git a/pkg/controllers/placement/controller.go b/pkg/controllers/placement/controller.go index aa615c2f7..ff7c10daa 100644 --- a/pkg/controllers/placement/controller.go +++ b/pkg/controllers/placement/controller.go @@ -142,6 +142,19 @@ func (r *Reconciler) handleUpdate(ctx context.Context, placementObj fleetv1beta1 } } + // Validate namespace selector consistency for NamespaceAccessible CRPs. + if isNamespaceAccessibleCRP(placementObj) { + isValid, err := r.validateNamespaceSelectorConsistency(ctx, placementObj) + if err != nil { + klog.V(2).ErrorS(err, "Namespace resource selector validation failed for NamespaceAccessible CRP", "placement", placementKObj) + return ctrl.Result{}, err + } + if !isValid { + klog.V(2).InfoS("Invalid Namespace resource selector specified for NamespaceAccessible CRP, stopping reconciliation", "placement", placementKObj) + return ctrl.Result{}, nil + } + } + // validate the resource selectors first before creating any snapshot envelopeObjCount, selectedResources, selectedResourceIDs, err := r.selectResourcesForPlacement(placementObj) if err != nil { @@ -160,16 +173,17 @@ func (r *Reconciler) handleUpdate(ctx context.Context, placementObj fleetv1beta1 } placementObj.SetConditions(scheduleCondition) - // Sync ClusterResourcePlacementStatus object if StatusReportingScope is NamespaceAccessible - if syncErr := r.syncClusterResourcePlacementStatus(ctx, placementObj); syncErr != nil { - klog.ErrorS(syncErr, "Failed to sync ClusterResourcePlacementStatus", "placement", placementKObj) - return ctrl.Result{}, syncErr - } - if updateErr := r.Client.Status().Update(ctx, placementObj); updateErr != nil { klog.ErrorS(updateErr, "Failed to update the status", "placement", placementKObj) return ctrl.Result{}, controller.NewUpdateIgnoreConflictError(updateErr) } + klog.V(2).InfoS("Updated the placement status with scheduled condition", "placement", placementKObj) + + if isNamespaceAccessibleCRP(placementObj) { + if err := r.handleNamespaceAccessibleCRP(ctx, placementObj); err != nil { + return ctrl.Result{}, err + } + } // no need to retry faster, the user needs to fix the resource selectors return ctrl.Result{RequeueAfter: controllerResyncPeriod}, nil @@ -212,18 +226,18 @@ func (r *Reconciler) handleUpdate(ctx context.Context, placementObj fleetv1beta1 return ctrl.Result{}, err } - // Sync ClusterResourcePlacementStatus object if StatusReportingScope is NamespaceAccessible - if err := r.syncClusterResourcePlacementStatus(ctx, placementObj); err != nil { - klog.ErrorS(err, "Failed to sync ClusterResourcePlacementStatus", "placement", placementKObj) - return ctrl.Result{}, err - } - if err := r.Client.Status().Update(ctx, placementObj); err != nil { klog.ErrorS(err, "Failed to update the status", "placement", placementKObj) return ctrl.Result{}, err } klog.V(2).InfoS("Updated the placement status", "placement", placementKObj) + if isNamespaceAccessibleCRP(placementObj) { + if err := r.handleNamespaceAccessibleCRP(ctx, placementObj); err != nil { + return ctrl.Result{}, err + } + } + // We skip checking the last resource condition (available) because it will be covered by checking isRolloutCompleted func. for i := condition.RolloutStartedCondition; i < condition.TotalCondition-1; i++ { var oldCond, newCond *metav1.Condition diff --git a/pkg/controllers/placement/controller_integration_test.go b/pkg/controllers/placement/controller_integration_test.go index 77d17a976..107029846 100644 --- a/pkg/controllers/placement/controller_integration_test.go +++ b/pkg/controllers/placement/controller_integration_test.go @@ -39,7 +39,7 @@ import ( "github.com/kubefleet-dev/kubefleet/pkg/utils" "github.com/kubefleet-dev/kubefleet/pkg/utils/condition" "github.com/kubefleet-dev/kubefleet/pkg/utils/resource" - statussyncutils "github.com/kubefleet-dev/kubefleet/test/utils/crpstatussync" + "github.com/kubefleet-dev/kubefleet/test/utils/crpstatussync" metricsUtils "github.com/kubefleet-dev/kubefleet/test/utils/metrics" ) @@ -2189,26 +2189,31 @@ var _ = Describe("Test ClusterResourcePlacement Controller", func() { }) It("Should handle missing target namespace", func() { - By("Validate CRP keeps retrying due to missing namespace (status remain nil)") + By("Validate CRP status includes StatusSynced condition") wantCRP := &placementv1beta1.ClusterResourcePlacement{ ObjectMeta: metav1.ObjectMeta{ Name: testCRPName, Finalizers: []string{placementv1beta1.PlacementCleanupFinalizer}, }, Spec: crp.Spec, + Status: placementv1beta1.PlacementStatus{ + ObservedResourceIndex: "0", + Conditions: []metav1.Condition{ + { + Status: metav1.ConditionUnknown, + Type: string(placementv1beta1.ClusterResourcePlacementScheduledConditionType), + Reason: condition.SchedulingUnknownReason, + }, + { + Status: metav1.ConditionFalse, + Type: string(placementv1beta1.ClusterResourcePlacementStatusSyncedConditionType), + Reason: condition.StatusSyncFailedReason, + }, + }, + }, } - retrieveAndValidateClusterResourcePlacement(testCRPName, wantCRP) - // Ensure that CRP status remains nil as the controller keeps retrying. - Consistently(func() error { - gotCRP := &placementv1beta1.ClusterResourcePlacement{} - if err := k8sClient.Get(ctx, types.NamespacedName{Name: testCRPName}, gotCRP); err != nil { - return err - } - if diff := cmp.Diff(wantCRP, gotCRP, cmpCRPOptions); diff != "" { - return fmt.Errorf("clusterResourcePlacement mismatch (-want, +got) :\n%s", diff) - } - return nil - }, consistentlyTimeout, interval).Should(Succeed(), "ClusterResourcePlacement status must be nil") + retrieveAndValidateClusterResourcePlacement(crp.Name, wantCRP) + // Namespace doesn't exist, so no CRPS should be created - sanity check. By("Ensure no ClusterResourcePlacementStatus is created in the nonexistent namespace") Consistently(func() bool { @@ -2250,7 +2255,7 @@ var _ = Describe("Test ClusterResourcePlacement Controller", func() { { Group: corev1.GroupName, Version: "v1", - Kind: "InvalidKind", // Invalid kind to trigger user error + Kind: "InvalidKind", // Invalid kind to trigger user error. Name: "invalid-resource", }, }, @@ -2268,7 +2273,7 @@ var _ = Describe("Test ClusterResourcePlacement Controller", func() { }, } Expect(k8sClient.Delete(ctx, crp)).Should(Succeed()) - retrieveAndValidateCRPDeletion(gotCRP) + retrieveAndValidateCRPDeletion(crp) // Need to manually clean up CRPS in test environment https://book.kubebuilder.io/reference/envtest#testing-considerations. By("Deleting crps") @@ -2293,8 +2298,8 @@ var _ = Describe("Test ClusterResourcePlacement Controller", func() { Expect(k8sClient.Delete(ctx, namespace)).Should(Succeed()) }) - It("Should handle invalid resource selector", func() { - By("Validate CRP status") + It("Should handle invalid resource selector and create CRPS", func() { + By("Validate CRP status includes both Scheduled and StatusSynced conditions") wantCRP := &placementv1beta1.ClusterResourcePlacement{ ObjectMeta: metav1.ObjectMeta{ Name: testCRPName, @@ -2303,20 +2308,25 @@ var _ = Describe("Test ClusterResourcePlacement Controller", func() { Spec: crp.Spec, Status: placementv1beta1.PlacementStatus{ Conditions: []metav1.Condition{ + // ObservedResourceIndex is not set because resource selector is invalid. { Status: metav1.ConditionFalse, Type: string(placementv1beta1.ClusterResourcePlacementScheduledConditionType), Reason: condition.InvalidResourceSelectorsReason, }, + { + Status: metav1.ConditionTrue, + Type: string(placementv1beta1.ClusterResourcePlacementStatusSyncedConditionType), + Reason: condition.StatusSyncSucceededReason, + }, }, }, } - gotCRP = retrieveAndValidateClusterResourcePlacement(crp.Name, wantCRP) + retrieveAndValidateClusterResourcePlacement(crp.Name, wantCRP) - By("Validate CRPS status", func() { - crpsMatchesActual := statussyncutils.CRPSStatusMatchesCRPActual(ctx, k8sClient, testCRPName, namespaceName) - Eventually(crpsMatchesActual, eventuallyTimeout, interval).Should(Succeed(), "ClusterResourcePlacementStatus should match expected structure and CRP status") - }) + By("Validate CRPS is created and matches CRP status") + crpsMatchesActual := crpstatussync.CRPSStatusMatchesCRPActual(ctx, k8sClient, testCRPName, namespaceName) + Eventually(crpsMatchesActual, eventuallyTimeout, interval).Should(Succeed(), "ClusterResourcePlacementStatus should be created and match expected structure") }) }) }) diff --git a/pkg/controllers/placement/namespace_status_sync.go b/pkg/controllers/placement/namespace_status_sync.go index 726132d14..cd1a226d3 100644 --- a/pkg/controllers/placement/namespace_status_sync.go +++ b/pkg/controllers/placement/namespace_status_sync.go @@ -27,70 +27,150 @@ import ( placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1" "github.com/kubefleet-dev/kubefleet/pkg/utils" + "github.com/kubefleet-dev/kubefleet/pkg/utils/condition" "github.com/kubefleet-dev/kubefleet/pkg/utils/controller" ) -// extractNamespaceFromResourceSelectors extracts the namespace name from ResourceSelectors -// when StatusReportingScope is NamespaceAccessible. Returns empty string if not applicable. -func extractNamespaceFromResourceSelectors(placementObj placementv1beta1.PlacementObj) string { - spec := placementObj.GetPlacementSpec() +const ( + noNamespaceResourceSelectorMsg = "NamespaceAccessible ClusterResourcePlacement doesn't specify a resource selector which selects a namespace" - // Only process if StatusReportingScope is NamespaceAccessible. - if spec.StatusReportingScope != placementv1beta1.NamespaceAccessible { - klog.V(2).InfoS("StatusReportingScope is not NamespaceAccessible", "placement", klog.KObj(placementObj)) - return "" - } + failedCRPSMessageFmt = "Failed to create or update ClusterResourcePlacementStatus: %v" + successfulCRPSMessageFmt = "Successfully created or updated ClusterResourcePlacementStatus in namespace '%s'" + namespaceConsistencyMessageFmt = "namespace resource selector is choosing a different namespace '%s' from what was originally picked '%s'. This is not allowed for NamespaceAccessible ClusterResourcePlacement" +) +// extractNamespaceFromResourceSelectors extracts the target namespace name from the +// ClusterResourcePlacement's ResourceSelectors. This function looks for a Namespace +// resource selector and returns its name. Returns empty string if no Namespace +// selector is found. +func extractNamespaceFromResourceSelectors(crp *placementv1beta1.ClusterResourcePlacement) string { // CEL validation ensures exactly one Namespace selector exists when NamespaceAccessible. - for _, selector := range spec.ResourceSelectors { + for _, selector := range crp.Spec.ResourceSelectors { selectorGVK := schema.GroupVersionKind{ Group: selector.Group, Version: selector.Version, Kind: selector.Kind, } - // Check if this is a namespace selector by comparing with the standard namespace GVK + // Check if this is a namespace selector by comparing with the standard namespace GVK. if selectorGVK == utils.NamespaceGVK { return selector.Name } } - // This should never happen due to CEL validation, but defensive programming. - klog.ErrorS(controller.NewUnexpectedBehaviorError(fmt.Errorf("no namespace selector found despite NamespaceAccessible scope")), - "Failed to find valid Namespace selector for NamespaceAccessible scope", - "placement", klog.KObj(placementObj)) return "" } -// TODO: Need to handle the case where namespace selector changes on the CRP. -// syncClusterResourcePlacementStatus creates or updates ClusterResourcePlacementStatus -// object in the target namespace when StatusReportingScope is NamespaceAccessible. -func (r *Reconciler) syncClusterResourcePlacementStatus(ctx context.Context, placementObj placementv1beta1.PlacementObj) error { - // Only sync for ClusterResourcePlacement objects (not ResourcePlacement). +// isNamespaceAccessibleCRP checks if the given placement object is a ClusterResourcePlacement +// with StatusReportingScope set to NamespaceAccessible. Returns true if the placement is +// a CRP with NamespaceAccessible scope, false otherwise. +func isNamespaceAccessibleCRP(placementObj placementv1beta1.PlacementObj) bool { crp, ok := placementObj.(*placementv1beta1.ClusterResourcePlacement) if !ok { - // This is a ResourcePlacement, not a ClusterResourcePlacement - skip sync. - klog.V(2).InfoS("Skipped processing RP to create/update ClusterResourcePlacementStatus") - return nil + klog.V(2).InfoS("Skipped processing RP to create/update ClusterResourcePlacementStatus", "placement", klog.KObj(placementObj)) + return false + } + + // Only process if StatusReportingScope is NamespaceAccessible. + if crp.Spec.StatusReportingScope != placementv1beta1.NamespaceAccessible { + klog.V(2).InfoS("StatusReportingScope is not NamespaceAccessible", "crp", klog.KObj(placementObj)) + return false + } + + return true +} + +// filterStatusSyncedCondition creates a copy of the placement status with the +// ClusterResourcePlacementStatusSynced condition removed. This is used when +// creating ClusterResourcePlacementStatus objects because the StatusSynced +// condition is only relevant for the main CRP object, not the CRPS object. +// Returns a filtered copy of the status with all conditions except StatusSynced. +func filterStatusSyncedCondition(status *placementv1beta1.PlacementStatus) *placementv1beta1.PlacementStatus { + filteredStatus := status.DeepCopy() + + // Filter out the ClusterResourcePlacementStatusSynced condition. + filteredConditions := make([]metav1.Condition, 0, len(filteredStatus.Conditions)) + for _, condition := range filteredStatus.Conditions { + if condition.Type != string(placementv1beta1.ClusterResourcePlacementStatusSyncedConditionType) { + filteredConditions = append(filteredConditions, condition) + } + } + filteredStatus.Conditions = filteredConditions + + return filteredStatus +} + +// buildStatusSyncedCondition creates a StatusSynced condition based on the sync result. +func buildNamepsaceAccessibleStatusSyncedCondition(syncErr error, targetNamespace string, generation int64) metav1.Condition { + if syncErr != nil { + return metav1.Condition{ + Type: string(placementv1beta1.ClusterResourcePlacementStatusSyncedConditionType), + Status: metav1.ConditionFalse, + Reason: condition.StatusSyncFailedReason, + Message: fmt.Sprintf(failedCRPSMessageFmt, syncErr), + ObservedGeneration: generation, + } + } + + return metav1.Condition{ + Type: string(placementv1beta1.ClusterResourcePlacementStatusSyncedConditionType), + Status: metav1.ConditionTrue, + Reason: condition.StatusSyncSucceededReason, + Message: fmt.Sprintf(successfulCRPSMessageFmt, targetNamespace), + ObservedGeneration: generation, + } +} + +// buildNamespaceAccessibleScheduledCondition creates a Scheduled condition with invalid resource selector reason. +func buildNamespaceAccessibleScheduledCondition(generation int64, msg string) metav1.Condition { + return metav1.Condition{ + Type: string(placementv1beta1.ClusterResourcePlacementScheduledConditionType), + Status: metav1.ConditionFalse, + Reason: condition.InvalidResourceSelectorsReason, + Message: msg, + ObservedGeneration: generation, + } +} + +// setConditionAndUpdateStatus is a helper function that sets the argued condition on the placement +// object and updates its status. +func (r *Reconciler) setConditionAndUpdateStatus(ctx context.Context, placementObj placementv1beta1.PlacementObj, condition metav1.Condition) error { + placementKObj := klog.KObj(placementObj) + + // Set the condition on the placement object. + placementObj.SetConditions(condition) + + // Update the placement status. + if err := r.Client.Status().Update(ctx, placementObj); err != nil { + klog.ErrorS(err, "Failed to update the placement status with StatusSynced condition", "crp", placementKObj) + return controller.NewUpdateIgnoreConflictError(err) } - // Extract target namespace. - targetNamespace := extractNamespaceFromResourceSelectors(placementObj) + klog.V(2).InfoS("Updated the placement status with StatusSynced condition", "crp", placementKObj) + + return nil +} + +// syncClusterResourcePlacementStatus creates or updates a ClusterResourcePlacementStatus +// object in the target namespace for ClusterResourcePlacements with NamespaceAccessible scope. +// It extracts the target namespace from the CRP's resource selectors, creates/updates a CRPS object +// with filtered status (excluding StatusSynced condition), and sets the CRP as owner for +// automatic cleanup. Returns an error if the operation fails. +func (r *Reconciler) syncClusterResourcePlacementStatus(ctx context.Context, crp *placementv1beta1.ClusterResourcePlacement, targetNamespace string) error { if targetNamespace == "" { - // Not NamespaceAccessible or no namespace found - skip sync. - klog.V(2).InfoS("Skipped processing CRP to create/update ClusterResourcePlacementStatus", "crp", klog.KObj(crp)) + // No valid target namespace - skip sync. return nil } - + crpKObj := klog.KObj(crp) crpStatus := &placementv1beta1.ClusterResourcePlacementStatus{ ObjectMeta: metav1.ObjectMeta{ - Name: crp.Name, // Same name as CRP. + Name: crp.Name, Namespace: targetNamespace, }, } // Use CreateOrUpdate to handle both creation and update cases. op, err := controllerutil.CreateOrUpdate(ctx, r.Client, crpStatus, func() error { - // Set the placement status and update time. - crpStatus.PlacementStatus = *crp.Status.DeepCopy() + // Set the placement status (excluding StatusSynced condition) and update time. + crpStatus.PlacementStatus = *filterStatusSyncedCondition(&crp.Status) crpStatus.LastUpdatedTime = metav1.Now() // Set CRP as owner - this ensures automatic cleanup when CRP is deleted. @@ -98,10 +178,120 @@ func (r *Reconciler) syncClusterResourcePlacementStatus(ctx context.Context, pla }) if err != nil { - klog.ErrorS(err, "Failed to create or update ClusterResourcePlacementStatus", "crp", klog.KObj(crp), "namespace", targetNamespace) + klog.ErrorS(err, "Failed to create or update ClusterResourcePlacementStatus", "crp", crpKObj, "namespace", targetNamespace) return controller.NewAPIServerError(false, fmt.Errorf("failed to create or update ClusterResourcePlacementStatus: %w", err)) } - klog.V(2).InfoS("Successfully handled ClusterResourcePlacementStatus", "crp", klog.KObj(crp), "namespace", targetNamespace, "operation", op) + klog.V(2).InfoS("Successfully handled ClusterResourcePlacementStatus", "crp", crpKObj, "namespace", targetNamespace, "operation", op) + return nil +} + +// handleNamespaceAccessibleCRP handles the complete workflow for ClusterResourcePlacements +// with NamespaceAccessible scope. It syncs the ClusterResourcePlacementStatus object in +// the target namespace, builds a StatusSynced/Scheduled condition based on the sync result, +// target namespace adds the condition to the CRP, and updates the CRP status. +func (r *Reconciler) handleNamespaceAccessibleCRP(ctx context.Context, placementObj placementv1beta1.PlacementObj) error { + placementKObj := klog.KObj(placementObj) + crp, _ := placementObj.(*placementv1beta1.ClusterResourcePlacement) + // Extract target namespace from resource selectors. + targetNamespace := extractNamespaceFromResourceSelectors(crp) + if targetNamespace == "" { + klog.ErrorS(controller.NewUnexpectedBehaviorError(fmt.Errorf("no namespace selector found despite NamespaceAccessible scope")), + "Failed to find valid Namespace selector for NamespaceAccessible scope", + "crp", placementKObj) + } + syncErr := r.syncClusterResourcePlacementStatus(ctx, crp, targetNamespace) + + var namespaceAccessibleCondition metav1.Condition + if syncErr != nil { + // status synced condition with error. + namespaceAccessibleCondition = buildNamepsaceAccessibleStatusSyncedCondition(syncErr, targetNamespace, placementObj.GetGeneration()) + } else if targetNamespace == "" { + // scheduled condition with invalid resource selector reason. + namespaceAccessibleCondition = buildNamespaceAccessibleScheduledCondition(placementObj.GetGeneration(), noNamespaceResourceSelectorMsg) + } else { + // successful status synced condition. + namespaceAccessibleCondition = buildNamepsaceAccessibleStatusSyncedCondition(syncErr, targetNamespace, placementObj.GetGeneration()) + } + + if err := r.setConditionAndUpdateStatus(ctx, placementObj, namespaceAccessibleCondition); err != nil { + return err + } + + if syncErr != nil { + klog.ErrorS(syncErr, "Failed to sync ClusterResourcePlacementStatus", "placement", klog.KObj(placementObj)) + return syncErr + } + return nil } + +// validateNamespaceSelectorConsistency validates that ClusterResourcePlacement with +// NamespaceAccessible scope have consistent namespace selectors. This function assumes +// the placement is already confirmed to be NamespaceAccessible. It checks if the +// namespace selector has changed from what was originally selected and sets the +// Scheduled condition accordingly. Returns an error if validation fails or if +// there are issues updating the placement status. +func (r *Reconciler) validateNamespaceSelectorConsistency(ctx context.Context, placementObj placementv1beta1.PlacementObj) (bool, error) { + placementKObj := klog.KObj(placementObj) + crp, _ := placementObj.(*placementv1beta1.ClusterResourcePlacement) + + placementStatus := placementObj.GetPlacementStatus() + currentTargetNamespace := "" + + // Extract namespace from selected resources in status. + for _, resource := range placementStatus.SelectedResources { + resourceGVK := schema.GroupVersionKind{ + Group: resource.Group, + Version: resource.Version, + Kind: resource.Kind, + } + // Check if this is a namespace resource by comparing with the standard namespace GVK. + if resourceGVK == utils.NamespaceGVK { + currentTargetNamespace = resource.Name + break + } + } + + // CRP status has not been populated or no namespace selected by CRP. + if currentTargetNamespace == "" { + klog.V(2).InfoS("No namespace selected yet in status", "crp", placementKObj) + // No namespace selected yet - skip validation for now. + return true, nil + } + + // Extract target namespace from resource selectors in spec. + targetNamespaceFromSelector := extractNamespaceFromResourceSelectors(crp) + if targetNamespaceFromSelector == "" { + klog.ErrorS(controller.NewUnexpectedBehaviorError(fmt.Errorf("no namespace selector found despite NamespaceAccessible scope")), + "Failed to find valid Namespace selector for NamespaceAccessible scope", + "crp", placementKObj) + scheduledCondition := buildNamespaceAccessibleScheduledCondition(placementObj.GetGeneration(), noNamespaceResourceSelectorMsg) + if err := r.setConditionAndUpdateStatus(ctx, placementObj, scheduledCondition); err != nil { + return false, err + } + return false, nil + } + + // If both namespaces exist and they don't match, it means the namespace selector changed. + if currentTargetNamespace != targetNamespaceFromSelector { + klog.V(2).InfoS("Namespace selector has changed for NamespaceAccessible CRP", + "crp", placementKObj, + "currentTargetNamespace", currentTargetNamespace, + "newTargetNamespace", targetNamespaceFromSelector) + + scheduledCondition := buildNamespaceAccessibleScheduledCondition(placementObj.GetGeneration(), fmt.Sprintf(namespaceConsistencyMessageFmt, targetNamespaceFromSelector, currentTargetNamespace)) + + // Set condition and update placement status. + if err := r.setConditionAndUpdateStatus(ctx, placementObj, scheduledCondition); err != nil { + return false, err + } + klog.ErrorS(controller.NewUserError(fmt.Errorf(namespaceConsistencyMessageFmt, currentTargetNamespace, targetNamespaceFromSelector)), + "Namespace selector inconsistency detected", + "crp", placementKObj) + return false, nil + } + + // Validation passed. + return true, nil +} diff --git a/pkg/controllers/placement/namespace_status_sync_test.go b/pkg/controllers/placement/namespace_status_sync_test.go index d9953c140..e3d2c3256 100644 --- a/pkg/controllers/placement/namespace_status_sync_test.go +++ b/pkg/controllers/placement/namespace_status_sync_test.go @@ -18,13 +18,13 @@ package placement import ( "context" + "fmt" "testing" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" corev1 "k8s.io/api/core/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -34,6 +34,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1" + "github.com/kubefleet-dev/kubefleet/pkg/utils/condition" ) var ( @@ -44,263 +45,437 @@ var ( } ) -func TestExtractNamespaceFromResourceSelectors(t *testing.T) { +func TestHandleNamespaceAccessibleCRP(t *testing.T) { testCases := []struct { - name string - placement placementv1beta1.ClusterResourcePlacement - want string + name string + placementObjName string + existingObjects []client.Object + targetNamespace string + wantCRPSOperation bool + wantCRPCondition *metav1.Condition + wantError bool }{ { - name: "NamespaceAccessible with namespace selector", - placement: placementv1beta1.ClusterResourcePlacement{ - Spec: placementv1beta1.PlacementSpec{ - StatusReportingScope: placementv1beta1.NamespaceAccessible, - ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ - { - Group: "", - Version: "v1", - Kind: "Namespace", - Name: "test-namespace", + name: "successful sync - create new CRPS and set StatusSynced to True", + placementObjName: "test-crp", + existingObjects: []client.Object{ + &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-crp", + Generation: 1, + }, + Spec: placementv1beta1.PlacementSpec{ + StatusReportingScope: placementv1beta1.NamespaceAccessible, + ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "test-namespace", + }, }, - { - Group: "rbac.authorization.k8s.io", - Version: "v1", - Kind: "ClusterRole", - Name: "test-cluster-role", + }, + Status: placementv1beta1.PlacementStatus{ + ObservedResourceIndex: "0", + Conditions: []metav1.Condition{ + { + Type: "TestCondition", + Status: metav1.ConditionTrue, + Reason: "TestReason", + }, }, }, }, + &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-namespace", + }, + }, }, - want: "test-namespace", + targetNamespace: "test-namespace", + wantCRPSOperation: true, + wantCRPCondition: &metav1.Condition{ + Type: string(placementv1beta1.ClusterResourcePlacementStatusSyncedConditionType), + Status: metav1.ConditionTrue, + Reason: "StatusSyncSucceeded", + Message: fmt.Sprintf(successfulCRPSMessageFmt, "test-namespace"), + ObservedGeneration: 1, + }, + wantError: false, }, { - name: "ClusterScopeOnly should return empty", - placement: placementv1beta1.ClusterResourcePlacement{ - Spec: placementv1beta1.PlacementSpec{ - StatusReportingScope: placementv1beta1.ClusterScopeOnly, - ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ - { - Group: "", - Version: "v1", - Kind: "Namespace", - Name: "test-namespace", + name: "successful sync - update existing CRPS and set StatusSynced to True", + placementObjName: "test-crp", + existingObjects: []client.Object{ + &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-namespace", + }, + }, + &placementv1beta1.ClusterResourcePlacementStatus{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-crp", + Namespace: "test-namespace", + }, + PlacementStatus: placementv1beta1.PlacementStatus{ + ObservedResourceIndex: "0", + Conditions: []metav1.Condition{ + { + Type: string(placementv1beta1.ClusterResourcePlacementScheduledConditionType), + Status: metav1.ConditionFalse, + Reason: "OldReason", + }, }, }, }, - }, - want: "", - }, - { - name: "StatusReportingScope is not specified, should return empty", - placement: placementv1beta1.ClusterResourcePlacement{ - Spec: placementv1beta1.PlacementSpec{ - ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ - { - Group: "", - Version: "v1", - Kind: "Namespace", - Name: "test-namespace", + &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-crp", + Generation: 1, + }, + Spec: placementv1beta1.PlacementSpec{ + StatusReportingScope: placementv1beta1.NamespaceAccessible, + ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "test-namespace", + }, + }, + }, + Status: placementv1beta1.PlacementStatus{ + ObservedResourceIndex: "0", + Conditions: []metav1.Condition{ + { + Type: string(placementv1beta1.ClusterResourcePlacementScheduledConditionType), + Status: metav1.ConditionTrue, + Reason: "UpdatedReason", + }, }, }, }, }, - want: "", + targetNamespace: "test-namespace", + wantCRPSOperation: true, + wantCRPCondition: &metav1.Condition{ + Type: string(placementv1beta1.ClusterResourcePlacementStatusSyncedConditionType), + Status: metav1.ConditionTrue, + Reason: "StatusSyncSucceeded", + Message: fmt.Sprintf(successfulCRPSMessageFmt, "test-namespace"), + ObservedGeneration: 1, + }, + wantError: false, }, { - name: "NamespaceAccessible without namespace selector", // CEL validation should prevent this case. - placement: placementv1beta1.ClusterResourcePlacement{ - Spec: placementv1beta1.PlacementSpec{ - StatusReportingScope: placementv1beta1.NamespaceAccessible, - ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ - { - Group: "apps", - Version: "v1", - Kind: "Deployment", - Name: "test-deployment", + name: "sync failure - missing namespace selector and set Scheduled condition to false", + placementObjName: "test-crp-no-ns", + existingObjects: []client.Object{ + &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-crp-no-ns", + Generation: 1, + }, + Spec: placementv1beta1.PlacementSpec{ + StatusReportingScope: placementv1beta1.NamespaceAccessible, + ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "ClusterRole", + Name: "test-cluster-role", + }, }, }, + Status: placementv1beta1.PlacementStatus{ + ObservedResourceIndex: "0", + }, }, }, - want: "", + targetNamespace: "", + wantCRPSOperation: false, + wantCRPCondition: &metav1.Condition{ + Type: string(placementv1beta1.ClusterResourcePlacementScheduledConditionType), + Status: metav1.ConditionFalse, + Reason: condition.InvalidResourceSelectorsReason, + Message: noNamespaceResourceSelectorMsg, + ObservedGeneration: 1, + }, + wantError: false, }, } for _, tc := range testCases { + scheme := statusSyncServiceScheme(t) t.Run(tc.name, func(t *testing.T) { - got := extractNamespaceFromResourceSelectors(&tc.placement) - if got != tc.want { - t.Errorf("extractNamespaceFromResourceSelectors() = %v, want %v", got, tc.want) + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(tc.existingObjects...). + WithStatusSubresource(&placementv1beta1.ClusterResourcePlacement{}). + Build() + + reconciler := &Reconciler{ + Client: fakeClient, + Scheme: scheme, + } + + // Get the CRP from the fake client to ensure it has proper metadata and when handleNamespaceAccessibleCRP is called CRP must already exist. + gotCRP := &placementv1beta1.ClusterResourcePlacement{} + if err := fakeClient.Get(context.Background(), types.NamespacedName{Name: tc.placementObjName}, gotCRP); err != nil { + t.Fatalf("Failed to get CRP from fake client: %v", err) } + + // Call handleNamespaceAccessibleCRP with the CRP from the client. + err := reconciler.handleNamespaceAccessibleCRP(context.Background(), gotCRP) + + // Check if error expectation matches. + if tc.wantError && err == nil { + t.Fatal("Expected error but got none") + } + if !tc.wantError && err != nil { + t.Fatalf("handleNamespaceAccessibleCRP() failed: %v", err) + } + + // Check ClusterResourcePlacementStatus creation/update using cmp.Diff. + if tc.wantCRPSOperation && tc.targetNamespace != "" { + gotCRPS := &placementv1beta1.ClusterResourcePlacementStatus{} + getErr := fakeClient.Get(context.Background(), types.NamespacedName{ + Name: tc.placementObjName, + Namespace: tc.targetNamespace, + }, gotCRPS) + + if getErr != nil { + if !tc.wantError { + t.Fatalf("Expected ClusterResourcePlacementStatus to be created/updated but got error: %v", getErr) + } + } else { + // Construct expected CRPS based on the CRP. + wantCRPS := &placementv1beta1.ClusterResourcePlacementStatus{ + ObjectMeta: metav1.ObjectMeta{ + Name: tc.placementObjName, + Namespace: tc.targetNamespace, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "placement.kubernetes-fleet.io/v1beta1", + Kind: "ClusterResourcePlacement", + Name: tc.placementObjName, + Controller: ptr.To(true), + BlockOwnerDeletion: ptr.To(true), + }, + }, + }, + PlacementStatus: gotCRP.Status, + } + + // Filter out StatusSynced condition from the expected CRPS. + filteredConditions := make([]metav1.Condition, 0) + for _, cond := range gotCRP.Status.Conditions { + if cond.Type != string(placementv1beta1.ClusterResourcePlacementStatusSyncedConditionType) { + filteredConditions = append(filteredConditions, cond) + } + } + wantCRPS.PlacementStatus.Conditions = filteredConditions + + // Use cmp.Diff to compare CRPS objects. + if diff := cmp.Diff(wantCRPS, gotCRPS, crpsCmpOpts...); diff != "" { + t.Fatalf("ClusterResourcePlacementStatus mismatch (-want +got):\n%s", diff) + } + + // Additional validation that LastUpdatedTime is set (ignored in comparison). + if gotCRPS.LastUpdatedTime.IsZero() { + t.Fatal("Expected LastUpdatedTime to be set on CRPS") + } + } + } else { + // Ensure no CRPS exists in any namespace. + crpsList := &placementv1beta1.ClusterResourcePlacementStatusList{} + if err := fakeClient.List(context.Background(), crpsList); err != nil { + t.Fatalf("Failed to list ClusterResourcePlacementStatus: %v", err) + } + if len(crpsList.Items) != 0 { + t.Fatalf("Expected no ClusterResourcePlacementStatus to exist, but found %d", len(crpsList.Items)) + } + } + + verifyCRPCondition(t, fakeClient, tc.placementObjName, tc.wantCRPCondition) }) } } -func TestSyncClusterResourcePlacementStatus(t *testing.T) { - scheme := runtime.NewScheme() - if err := clientgoscheme.AddToScheme(scheme); err != nil { - t.Fatalf("failed to add client go scheme: %v", err) - } - if err := placementv1beta1.AddToScheme(scheme); err != nil { - t.Fatalf("failed to add fleet scheme: %v", err) - } - +func TestValidateNamespaceSelectorConsistency(t *testing.T) { testCases := []struct { - name string - placementObj placementv1beta1.PlacementObj - existingObjects []client.Object - expectOperation bool - targetNamespace string + name string + placementObjName string + existingObjects []client.Object + wantCRPCondition *metav1.Condition + wantIsValid bool + wantError bool }{ { - name: "Create new ClusterResourcePlacementStatus", - placementObj: &placementv1beta1.ClusterResourcePlacement{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-crp", - }, - Spec: placementv1beta1.PlacementSpec{ - StatusReportingScope: placementv1beta1.NamespaceAccessible, - ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ - { - Group: "", - Version: "v1", - Kind: "Namespace", - Name: "test-namespace", + name: "no namespace selector - should set Scheduled to false and return false", + placementObjName: "test-crp-no-selector", + existingObjects: []client.Object{ + &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-crp-no-selector", + Generation: 1, + }, + Spec: placementv1beta1.PlacementSpec{ + StatusReportingScope: placementv1beta1.NamespaceAccessible, + ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "rbac.authorization.k8s.io", + Version: "v1", + Kind: "ClusterRole", + Name: "test-cluster-role", + }, }, }, - }, - Status: placementv1beta1.PlacementStatus{ - ObservedResourceIndex: "test-index", - Conditions: []metav1.Condition{ - { - Type: "TestCondition", - Status: metav1.ConditionTrue, - Reason: "TestReason", + Status: placementv1beta1.PlacementStatus{ + ObservedResourceIndex: "0", + SelectedResources: []placementv1beta1.ResourceIdentifier{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "test-namespace", + Namespace: "", + }, }, }, }, }, - existingObjects: []client.Object{ - &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-namespace", - }, - }, + wantCRPCondition: &metav1.Condition{ + Type: string(placementv1beta1.ClusterResourcePlacementScheduledConditionType), + Status: metav1.ConditionFalse, + Reason: condition.InvalidResourceSelectorsReason, + Message: noNamespaceResourceSelectorMsg, + ObservedGeneration: 1, }, - expectOperation: true, - targetNamespace: "test-namespace", + wantIsValid: false, + wantError: false, }, { - name: "Update existing ClusterResourcePlacementStatus", - placementObj: &placementv1beta1.ClusterResourcePlacement{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-crp", - }, - Spec: placementv1beta1.PlacementSpec{ - StatusReportingScope: placementv1beta1.NamespaceAccessible, - ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ - { - Group: "", - Version: "v1", - Kind: "Namespace", - Name: "test-namespace", - }, + name: "empty selected resources status - should pass validation and return true", + placementObjName: "test-crp-empty-selected", + existingObjects: []client.Object{ + &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-crp-empty-selected", + Generation: 1, }, - }, - Status: placementv1beta1.PlacementStatus{ - ObservedResourceIndex: "updated-index", - Conditions: []metav1.Condition{ - { - Type: "UpdatedCondition", - Status: metav1.ConditionTrue, - Reason: "UpdatedReason", + Spec: placementv1beta1.PlacementSpec{ + StatusReportingScope: placementv1beta1.NamespaceAccessible, + ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "test-namespace", + }, }, }, + Status: placementv1beta1.PlacementStatus{ + ObservedResourceIndex: "0", + SelectedResources: []placementv1beta1.ResourceIdentifier{}, // Empty selected resources. + }, }, }, + wantCRPCondition: nil, // No condition should be set + wantIsValid: true, + wantError: false, + }, + { + name: "consistent namespace selector - should pass validation and return true", + placementObjName: "test-crp-consistent", existingObjects: []client.Object{ - &corev1.Namespace{ + &placementv1beta1.ClusterResourcePlacement{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-namespace", + Name: "test-crp-consistent", + Generation: 1, }, - }, - &placementv1beta1.ClusterResourcePlacementStatus{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-crp", - Namespace: "test-namespace", + Spec: placementv1beta1.PlacementSpec{ + StatusReportingScope: placementv1beta1.NamespaceAccessible, + ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "test-namespace", + }, + }, }, - PlacementStatus: placementv1beta1.PlacementStatus{ - ObservedResourceIndex: "old-index", + Status: placementv1beta1.PlacementStatus{ + ObservedResourceIndex: "0", + SelectedResources: []placementv1beta1.ResourceIdentifier{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "test-namespace", + Namespace: "", + }, + }, }, - LastUpdatedTime: metav1.Now(), }, }, - expectOperation: true, - targetNamespace: "test-namespace", + wantCRPCondition: nil, // No condition should be set. + wantIsValid: true, + wantError: false, }, { - name: "ClusterScopeOnly should not sync", - placementObj: &placementv1beta1.ClusterResourcePlacement{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-crp", - }, - Spec: placementv1beta1.PlacementSpec{ - StatusReportingScope: placementv1beta1.ClusterScopeOnly, - ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ - { - Group: "", - Version: "v1", - Kind: "Namespace", - Name: "test-namespace", - }, - }, - }, - }, + name: "inconsistent namespace selector - should set Scheduled to false and return false", + placementObjName: "test-crp-inconsistent", existingObjects: []client.Object{ - &corev1.Namespace{ + &placementv1beta1.ClusterResourcePlacement{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-namespace", + Name: "test-crp-inconsistent", + Generation: 2, }, - }, - }, - expectOperation: false, - targetNamespace: "test-namespace", - }, - { - name: "ResourcePlacement should not sync", - placementObj: &placementv1beta1.ResourcePlacement{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-rp", - Namespace: "test-namespace", - }, - Spec: placementv1beta1.PlacementSpec{ - StatusReportingScope: placementv1beta1.NamespaceAccessible, - ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ - { - Group: "", - Version: "v1", - Kind: "Namespace", - Name: "test-namespace", + Spec: placementv1beta1.PlacementSpec{ + StatusReportingScope: placementv1beta1.NamespaceAccessible, + ResourceSelectors: []placementv1beta1.ResourceSelectorTerm{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "new-namespace", // Changed from original. + }, }, }, - }, - }, - existingObjects: []client.Object{ - &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-namespace", + Status: placementv1beta1.PlacementStatus{ + ObservedResourceIndex: "0", + SelectedResources: []placementv1beta1.ResourceIdentifier{ + { + Group: "", + Version: "v1", + Kind: "Namespace", + Name: "original-namespace", // Original namespace in status. + Namespace: "", + }, + }, }, }, }, - expectOperation: false, - targetNamespace: "test-namespace", + wantCRPCondition: &metav1.Condition{ + Type: string(placementv1beta1.ClusterResourcePlacementScheduledConditionType), + Status: metav1.ConditionFalse, + Reason: condition.InvalidResourceSelectorsReason, + Message: fmt.Sprintf(namespaceConsistencyMessageFmt, "new-namespace", "original-namespace"), + ObservedGeneration: 2, + }, + wantIsValid: false, + wantError: false, }, } for _, tc := range testCases { + scheme := statusSyncServiceScheme(t) t.Run(tc.name, func(t *testing.T) { fakeClient := fake.NewClientBuilder(). WithScheme(scheme). WithObjects(tc.existingObjects...). + WithStatusSubresource(&placementv1beta1.ClusterResourcePlacement{}). Build() reconciler := &Reconciler{ @@ -308,62 +483,63 @@ func TestSyncClusterResourcePlacementStatus(t *testing.T) { Scheme: scheme, } - err := reconciler.syncClusterResourcePlacementStatus(context.Background(), tc.placementObj) - if err != nil { - t.Fatalf("syncClusterResourcePlacementStatus() failed: %v", err) + // Get the CRP from the fake client to ensure it has proper metadata and when validateNamespaceSelectorConsistency is called CRP must already exist. + gotCRP := &placementv1beta1.ClusterResourcePlacement{} + if err := fakeClient.Get(context.Background(), types.NamespacedName{Name: tc.placementObjName}, gotCRP); err != nil { + t.Fatalf("Failed to get CRP from fake client: %v", err) } - crpStatus := &placementv1beta1.ClusterResourcePlacementStatus{} - err = fakeClient.Get(context.Background(), types.NamespacedName{Name: tc.placementObj.GetName(), Namespace: tc.targetNamespace}, crpStatus) + // Call validateNamespaceSelectorConsistency. + gotIsValid, err := reconciler.validateNamespaceSelectorConsistency(context.Background(), gotCRP) - if !tc.expectOperation { - if err == nil { - t.Fatal("Expected no ClusterResourcePlacementStatus to be present, but one exists") - } - // CRPS should not exist. - if k8serrors.IsNotFound(err) { - return - } + // Check if error expectation matches. + if tc.wantError && err == nil { + t.Fatal("Expected error but got none") } - - if err != nil { - t.Fatalf("expected ClusterResourcePlacementStatus to exist but got error: %v", err) + if !tc.wantError && err != nil { + t.Fatalf("validateNamespaceSelectorConsistency() failed: %v", err) } - // Verify LastUpdatedTime is set. - if crpStatus.LastUpdatedTime.IsZero() { - t.Fatal("Expected LastUpdatedTime to be set, but it was zero") + // Check if validity expectation matches. + if tc.wantIsValid != gotIsValid { + t.Fatalf("validateNamespaceSelectorConsistency() validity mismatch: want %v, got %v", tc.wantIsValid, gotIsValid) } - // Verify the ClusterResourcePlacementStatus exists. - crp, ok := tc.placementObj.(*placementv1beta1.ClusterResourcePlacement) - if !ok { - return // ResourcePlacement case. - } + verifyCRPCondition(t, fakeClient, tc.placementObjName, tc.wantCRPCondition) + }) + } +} - // Use cmp.Diff to compare the key fields. - wantStatus := placementv1beta1.ClusterResourcePlacementStatus{ - ObjectMeta: metav1.ObjectMeta{ - Name: crp.Name, - Namespace: tc.targetNamespace, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "placement.kubernetes-fleet.io/v1beta1", - Kind: "ClusterResourcePlacement", - Name: crp.Name, - UID: crp.UID, - Controller: ptr.To(true), - BlockOwnerDeletion: ptr.To(true), - }, - }, - }, - PlacementStatus: crp.Status, - } +func statusSyncServiceScheme(t *testing.T) *runtime.Scheme { + scheme := runtime.NewScheme() + if err := clientgoscheme.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add client go scheme: %v", err) + } + if err := placementv1beta1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add scheme: %v", err) + } + return scheme +} - // Ignore metadata fields that Kubernetes sets automatically and LastUpdatedTime since it's time-dependent. - if diff := cmp.Diff(wantStatus, *crpStatus, crpsCmpOpts...); diff != "" { - t.Fatalf("ClusterResourcePlacementStatus mismatch (-want +got):\n%s", diff) - } - }) +func verifyCRPCondition(t *testing.T, fakeClient client.WithWatch, placementName string, wantCondition *metav1.Condition) { + updatedCRP := &placementv1beta1.ClusterResourcePlacement{} + if err := fakeClient.Get(context.Background(), types.NamespacedName{Name: placementName}, updatedCRP); err != nil { + t.Fatalf("Failed to get updated CRP: %v", err) + } + + var gotCondition *metav1.Condition + if wantCondition == nil { + gotCondition = nil + } else { + switch wantCondition.Type { + case string(placementv1beta1.ClusterResourcePlacementStatusSyncedConditionType): + gotCondition = updatedCRP.GetCondition(string(placementv1beta1.ClusterResourcePlacementStatusSyncedConditionType)) + case string(placementv1beta1.ClusterResourcePlacementScheduledConditionType): + gotCondition = updatedCRP.GetCondition(string(placementv1beta1.ClusterResourcePlacementScheduledConditionType)) + } + } + + if diff := cmp.Diff(wantCondition, gotCondition, cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime")); diff != "" { + t.Fatalf("Namespace Accessible condition mismatch (-want +got):\n%s", diff) } } diff --git a/pkg/utils/condition/reason.go b/pkg/utils/condition/reason.go index 08d4e2dce..b1d963e58 100644 --- a/pkg/utils/condition/reason.go +++ b/pkg/utils/condition/reason.go @@ -102,6 +102,12 @@ const ( // diff reporting has been fully completed. DiffReportedStatusTrueReason = "DiffReportingCompleted" + // StatusSyncFailedReason is the reason string of placement condition when the status sync failed. + StatusSyncFailedReason = "StatusSyncFailed" + + // StatusSyncSucceededReason is the reason string of placement condition when the status sync succeeded. + StatusSyncSucceededReason = "StatusSyncSucceeded" + // TODO: Add a user error reason ) diff --git a/test/e2e/actuals_test.go b/test/e2e/actuals_test.go index 5403d3639..8afcdc5e8 100644 --- a/test/e2e/actuals_test.go +++ b/test/e2e/actuals_test.go @@ -1269,6 +1269,11 @@ func placementStatusWithOverrideUpdatedActual( } } +func namespaceAccessibleCRPStatusUpdatedActual(wantSelectedResourceIdentifiers []placementv1beta1.ResourceIdentifier, wantSelectedClusters, wantUnselectedClusters []string, wantObservedResourceIndex string, statusSyncedConditionStatus metav1.ConditionStatus, isResourceSelectorInvalid bool) func() error { + crpKey := types.NamespacedName{Name: fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess())} + return customizedCRPStatusUpdatedActualWithStatusSynced(crpKey, wantSelectedResourceIdentifiers, wantSelectedClusters, wantUnselectedClusters, wantObservedResourceIndex, true, statusSyncedConditionStatus, isResourceSelectorInvalid) +} + func crpStatusUpdatedActual(wantSelectedResourceIdentifiers []placementv1beta1.ResourceIdentifier, wantSelectedClusters, wantUnselectedClusters []string, wantObservedResourceIndex string) func() error { crpKey := types.NamespacedName{Name: fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess())} return customizedPlacementStatusUpdatedActual(crpKey, wantSelectedResourceIdentifiers, wantSelectedClusters, wantUnselectedClusters, wantObservedResourceIndex, true) @@ -1520,47 +1525,82 @@ func customizedPlacementStatusUpdatedActual( return fmt.Errorf("failed to get placement %s: %w", placementKey, err) } - wantPlacementStatus := []placementv1beta1.PerClusterPlacementStatus{} - for _, name := range wantSelectedClusters { - wantPlacementStatus = append(wantPlacementStatus, placementv1beta1.PerClusterPlacementStatus{ - ClusterName: name, - ObservedResourceIndex: wantObservedResourceIndex, - Conditions: perClusterRolloutCompletedConditions(placement.GetGeneration(), resourceIsTrackable, false), - }) - } - for i := 0; i < len(wantUnselectedClusters); i++ { - wantPlacementStatus = append(wantPlacementStatus, placementv1beta1.PerClusterPlacementStatus{ - Conditions: perClusterScheduleFailedConditions(placement.GetGeneration()), - }) + wantStatus := buildWantPlacementStatus(placementKey, placement.GetGeneration(), wantSelectedResourceIdentifiers, wantSelectedClusters, wantUnselectedClusters, wantObservedResourceIndex, resourceIsTrackable) + if diff := cmp.Diff(placement.GetPlacementStatus(), wantStatus, placementStatusCmpOptions...); diff != "" { + return fmt.Errorf("Placement status diff (-got, +want): %s", diff) } + return nil + } +} - var wantPlacementConditions []metav1.Condition +func buildWantPlacementStatus( + placementKey types.NamespacedName, + placementGeneration int64, + wantSelectedResourceIdentifiers []placementv1beta1.ResourceIdentifier, + wantSelectedClusters, wantUnselectedClusters []string, + wantObservedResourceIndex string, + resourceIsTrackable bool, +) *placementv1beta1.PlacementStatus { + wantPlacementStatus := []placementv1beta1.PerClusterPlacementStatus{} + for _, name := range wantSelectedClusters { + wantPlacementStatus = append(wantPlacementStatus, placementv1beta1.PerClusterPlacementStatus{ + ClusterName: name, + ObservedResourceIndex: wantObservedResourceIndex, + Conditions: perClusterRolloutCompletedConditions(placementGeneration, resourceIsTrackable, false), + }) + } + for i := 0; i < len(wantUnselectedClusters); i++ { + wantPlacementStatus = append(wantPlacementStatus, placementv1beta1.PerClusterPlacementStatus{ + Conditions: perClusterScheduleFailedConditions(placementGeneration), + }) + } + + var wantPlacementConditions []metav1.Condition + if len(wantSelectedClusters) > 0 { + wantPlacementConditions = placementRolloutCompletedConditions(placementKey, placementGeneration, false) + } else { + // We don't set the remaining resource conditions. + wantPlacementConditions = placementScheduledConditions(placementKey, placementGeneration) + } + + if len(wantUnselectedClusters) > 0 { if len(wantSelectedClusters) > 0 { - wantPlacementConditions = placementRolloutCompletedConditions(placementKey, placement.GetGeneration(), false) + wantPlacementConditions = placementSchedulePartiallyFailedConditions(placementKey, placementGeneration) } else { - // We don't set the remaining resource conditions. - wantPlacementConditions = placementScheduledConditions(placementKey, placement.GetGeneration()) + // we don't set the remaining resource conditions if there is no clusters to select + wantPlacementConditions = placementScheduleFailedConditions(placementKey, placementGeneration) } + } - if len(wantUnselectedClusters) > 0 { - if len(wantSelectedClusters) > 0 { - wantPlacementConditions = placementSchedulePartiallyFailedConditions(placementKey, placement.GetGeneration()) - } else { - // we don't set the remaining resource conditions if there is no clusters to select - wantPlacementConditions = placementScheduleFailedConditions(placementKey, placement.GetGeneration()) - } - } + // Note that the placement controller will only keep decisions regarding unselected clusters for a placement if: + // + // * The placement is of the PickN placement type and the required N count cannot be fulfilled; or + // * The placement is of the PickFixed placement type and the list of target clusters specified cannot be fulfilled. + wantStatus := &placementv1beta1.PlacementStatus{ + Conditions: wantPlacementConditions, + PerClusterPlacementStatuses: wantPlacementStatus, + SelectedResources: wantSelectedResourceIdentifiers, + ObservedResourceIndex: wantObservedResourceIndex, + } + return wantStatus +} - // Note that the placement controller will only keep decisions regarding unselected clusters for a placement if: - // - // * The placement is of the PickN placement type and the required N count cannot be fulfilled; or - // * The placement is of the PickFixed placement type and the list of target clusters specified cannot be fulfilled. - wantStatus := &placementv1beta1.PlacementStatus{ - Conditions: wantPlacementConditions, - PerClusterPlacementStatuses: wantPlacementStatus, - SelectedResources: wantSelectedResourceIdentifiers, - ObservedResourceIndex: wantObservedResourceIndex, +func customizedCRPStatusUpdatedActualWithStatusSynced( + placementKey types.NamespacedName, + wantSelectedResourceIdentifiers []placementv1beta1.ResourceIdentifier, + wantSelectedClusters, wantUnselectedClusters []string, + wantObservedResourceIndex string, + resourceIsTrackable bool, + statusSyncedConditionStatus metav1.ConditionStatus, + isResourceSelectorInvalid bool, +) func() error { + return func() error { + placement, err := retrievePlacement(placementKey) + if err != nil { + return fmt.Errorf("failed to get placement %s: %w", placementKey, err) } + + wantStatus := buildWantCRPStatusWithStatusSynced(placementKey, placement.GetGeneration(), wantSelectedResourceIdentifiers, wantSelectedClusters, wantUnselectedClusters, wantObservedResourceIndex, resourceIsTrackable, statusSyncedConditionStatus, isResourceSelectorInvalid) if diff := cmp.Diff(placement.GetPlacementStatus(), wantStatus, placementStatusCmpOptions...); diff != "" { return fmt.Errorf("Placement status diff (-got, +want): %s", diff) } @@ -1568,6 +1608,54 @@ func customizedPlacementStatusUpdatedActual( } } +func buildWantCRPStatusWithStatusSynced( + placementKey types.NamespacedName, + placementGeneration int64, + wantSelectedResourceIdentifiers []placementv1beta1.ResourceIdentifier, + wantSelectedClusters, wantUnselectedClusters []string, + wantObservedResourceIndex string, + resourceIsTrackable bool, + statusSyncedConditionStatus metav1.ConditionStatus, + isResourceSelectorInvalid bool, +) *placementv1beta1.PlacementStatus { + if isResourceSelectorInvalid { + // when namespace resource selector is invalid, the observed generation doesn't change for all condition expect Scheduled. + placementGeneration = placementGeneration - 1 + } + wantStatus := buildWantPlacementStatus(placementKey, placementGeneration, wantSelectedResourceIdentifiers, wantSelectedClusters, wantUnselectedClusters, wantObservedResourceIndex, resourceIsTrackable) + + var statusSyncedCondition metav1.Condition + switch statusSyncedConditionStatus { + case metav1.ConditionTrue: + statusSyncedCondition = metav1.Condition{ + Type: string(placementv1beta1.ClusterResourcePlacementStatusSyncedConditionType), + Status: metav1.ConditionTrue, + Reason: condition.StatusSyncSucceededReason, + ObservedGeneration: placementGeneration, + } + case metav1.ConditionFalse: + statusSyncedCondition = metav1.Condition{ + Type: string(placementv1beta1.ClusterResourcePlacementStatusSyncedConditionType), + Status: metav1.ConditionFalse, + Reason: condition.StatusSyncFailedReason, + ObservedGeneration: placementGeneration, + } + } + wantStatus.Conditions = append(wantStatus.Conditions, statusSyncedCondition) + + if isResourceSelectorInvalid { + for i := range wantStatus.Conditions { + if wantStatus.Conditions[i].Type == string(placementv1beta1.ClusterResourcePlacementScheduledConditionType) { + wantStatus.Conditions[i].Status = metav1.ConditionFalse + wantStatus.Conditions[i].Reason = condition.InvalidResourceSelectorsReason + // when namespace resource selector is invalid, the observed generation is only updated for Scheduled condition. + wantStatus.Conditions[i].ObservedGeneration = placementGeneration + 1 + } + } + } + return wantStatus +} + func safeRolloutWorkloadCRPStatusUpdatedActual(wantSelectedResourceIdentifiers []placementv1beta1.ResourceIdentifier, failedWorkloadResourceIdentifier placementv1beta1.ResourceIdentifier, wantSelectedClusters []string, wantObservedResourceIndex string, failedResourceObservedGeneration int64) func() error { crpKey := types.NamespacedName{Name: fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess())} return safeRolloutWorkloadPlacementStatusUpdatedActual(crpKey, wantSelectedResourceIdentifiers, failedWorkloadResourceIdentifier, wantSelectedClusters, wantObservedResourceIndex, failedResourceObservedGeneration) diff --git a/test/e2e/placement_status_sync_test.go b/test/e2e/placement_status_sync_test.go index 1ef25e72c..bef9b518a 100644 --- a/test/e2e/placement_status_sync_test.go +++ b/test/e2e/placement_status_sync_test.go @@ -20,7 +20,6 @@ import ( "fmt" "time" - "github.com/google/go-cmp/cmp" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" @@ -66,9 +65,9 @@ var _ = Describe("ClusterResourcePlacementStatus E2E Tests", Ordered, func() { ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) - It("should update CRP status with 2 clusters as expected", func() { + It("should update CRP status with 2 clusters with synced condition set to true as expected", func() { expectedClusters := []string{memberCluster2EastCanaryName, memberCluster3WestProdName} - statusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), expectedClusters, nil, "0") + statusUpdatedActual := namespaceAccessibleCRPStatusUpdatedActual(workResourceIdentifiers(), expectedClusters, nil, "0", metav1.ConditionTrue, false) Eventually(statusUpdatedActual, crpsEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status with 2 clusters") }) @@ -89,8 +88,8 @@ var _ = Describe("ClusterResourcePlacementStatus E2E Tests", Ordered, func() { }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP to select 3 clusters") }) - It("should update CRP status with 3 clusters as expected", func() { - statusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") + It("should update CRP status with 3 clusters with synced condition set to true as expected", func() { + statusUpdatedActual := namespaceAccessibleCRPStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0", metav1.ConditionTrue, false) Eventually(statusUpdatedActual, crpsEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status with 3 clusters") }) @@ -115,7 +114,7 @@ var _ = Describe("ClusterResourcePlacementStatus E2E Tests", Ordered, func() { Namespace: appNamespace().Name, } - // Wait for CRPS to be recreated by the controller + // Wait for CRPS to be recreated by the controller. Eventually(func() bool { crpStatus := &placementv1beta1.ClusterResourcePlacementStatus{} if err := hubClient.Get(ctx, crpStatusKey, crpStatus); err != nil { @@ -124,11 +123,16 @@ var _ = Describe("ClusterResourcePlacementStatus E2E Tests", Ordered, func() { return crpStatus.DeletionTimestamp == nil }, eventuallyDuration, eventuallyInterval).Should(BeTrue(), "ClusterResourcePlacementStatus should be recreated after manual deletion") - // Verify the recreated CRPS matches the current CRP status + // Verify the recreated CRPS matches the current CRP status. crpsMatchesActual := statussyncutils.CRPSStatusMatchesCRPActual(ctx, hubClient, crpName, appNamespace().Name) Eventually(crpsMatchesActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Recreated ClusterResourcePlacementStatus should match current CRP status") }) + It("CRP status should remain unchanged", func() { + statusUpdatedActual := namespaceAccessibleCRPStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0", metav1.ConditionTrue, false) + Consistently(statusUpdatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to update CRP status with 3 clusters") + }) + It("delete CRP", func() { // Delete the CRP. crp := &placementv1beta1.ClusterResourcePlacement{ @@ -207,7 +211,6 @@ var _ = Describe("ClusterResourcePlacementStatus E2E Tests", Ordered, func() { }) Context("Namespace deletion with ClusterResourcePlacementStatus, StatusReportingScope is NamespaceAccessible", Ordered, func() { - var crpStatusBeforeWait placementv1beta1.PlacementStatus crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) BeforeAll(func() { @@ -222,8 +225,7 @@ var _ = Describe("ClusterResourcePlacementStatus E2E Tests", Ordered, func() { Spec: placementv1beta1.PlacementSpec{ ResourceSelectors: workResourceSelector(), Policy: &placementv1beta1.PlacementPolicy{ - PlacementType: placementv1beta1.PickNPlacementType, - NumberOfClusters: ptr.To(int32(2)), + PlacementType: placementv1beta1.PickAllPlacementType, }, StatusReportingScope: placementv1beta1.NamespaceAccessible, }, @@ -236,12 +238,13 @@ var _ = Describe("ClusterResourcePlacementStatus E2E Tests", Ordered, func() { ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) - It("should create ClusterResourcePlacementStatus in the work resources namespace", func() { + It("should update CRP status with synced condition set to true", func() { // Wait for CRP status to be updated. - expectedClusters := []string{memberCluster2EastCanaryName, memberCluster3WestProdName} - statusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), expectedClusters, nil, "0") + statusUpdatedActual := namespaceAccessibleCRPStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0", metav1.ConditionTrue, false) Eventually(statusUpdatedActual, crpsEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status") + }) + It("should create ClusterResourcePlacementStatus in the work resources namespace", func() { // Verify CRPS is created in the work resources namespace. crpsMatchesActual := statussyncutils.CRPSStatusMatchesCRPActual(ctx, hubClient, crpName, appNamespace().Name) Eventually(crpsMatchesActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "ClusterResourcePlacementStatus should be created in work resources namespace") @@ -266,46 +269,192 @@ var _ = Describe("ClusterResourcePlacementStatus E2E Tests", Ordered, func() { }, consistentlyDuration, consistentlyInterval).Should(BeTrue(), "Namespace should remain deleted") }) - It("should copy CRP status after namespace deletion", func() { - // Get the CRP status immediately after namespace deletion. + It("should update CRP status with Status synced condition set to false", func() { + // Wait for CRP status to be updated. + statusUpdatedActual := namespaceAccessibleCRPStatusUpdatedActual([]placementv1beta1.ResourceIdentifier{}, allMemberClusterNames, nil, "1", metav1.ConditionFalse, false) + Eventually(statusUpdatedActual, crpsEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status") + }) + + It("should recreate namespace manually", func() { + ns := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: appNamespace().Name, + }, + } + Expect(hubClient.Create(ctx, &ns)).To(Succeed(), "Failed to recreate work resources namespace") + + // Wait for namespace creation + Eventually(func() error { + return hubClient.Get(ctx, types.NamespacedName{Name: appNamespace().Name}, &ns) + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), fmt.Sprintf("Failed to get %s", appNamespace().Name)) + }) + + It("should update CRP status with Status synced condition set to true", func() { + // Wait for CRP status to be updated. + statusUpdatedActual := namespaceAccessibleCRPStatusUpdatedActual(workNamespaceIdentifiers(), allMemberClusterNames, nil, "2", metav1.ConditionTrue, false) + Eventually(statusUpdatedActual, crpsEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status") + }) + }) + + Context("NamespaceAccessible CRP with namespace selector change for consistency validation - namespace exists", func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + newNamespaceName := "new-namespace" + BeforeAll(func() { + // Create work resources that will be selected by the CRP. + createWorkResources() + + // Create another namespace. + ns := corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: newNamespaceName, + }, + } + Expect(hubClient.Create(ctx, &ns)).To(Succeed(), "Failed to create new-namespace") + // Wait for namespace creation + Eventually(func() error { + return hubClient.Get(ctx, types.NamespacedName{Name: newNamespaceName}, &ns) + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), fmt.Sprintf("Failed to get %s", newNamespaceName)) + + // Create CRP with NamespaceAccessible scope. + crp := &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: workResourceSelector(), + Policy: &placementv1beta1.PlacementPolicy{ + PlacementType: placementv1beta1.PickAllPlacementType, + }, + StatusReportingScope: placementv1beta1.NamespaceAccessible, + }, + } + Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP") + }) + + AfterAll(func() { + // Clean up resources. + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + + // Delete the additional namespace. + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: newNamespaceName, + }, + } + Expect(hubClient.Delete(ctx, ns)).To(Succeed(), "Failed to delete new-namespace") + }) + + It("should update CRP status with initial namespace selection", func() { + statusUpdatedActual := namespaceAccessibleCRPStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0", metav1.ConditionTrue, false) + Eventually(statusUpdatedActual, crpsEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status") + }) + + It("should create CRPS in the initial namespace and verify its status", func() { + crpsMatchesActual := statussyncutils.CRPSStatusMatchesCRPActual(ctx, hubClient, crpName, appNamespace().Name) + Eventually(crpsMatchesActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "CRPS should be created in initial namespace and match CRP status") + }) + + It("should update CRP to select a different namespace", func() { + // Update CRP to select the new-namespace instead. Eventually(func() error { crp := &placementv1beta1.ClusterResourcePlacement{} if err := hubClient.Get(ctx, types.NamespacedName{Name: crpName}, crp); err != nil { return err } - crpStatusBeforeWait = *crp.Status.DeepCopy() - return nil - }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to get CRP status after namespace deletion") + crp.Spec.ResourceSelectors = []placementv1beta1.ResourceSelectorTerm{ + { + Group: corev1.GroupName, + Version: "v1", + Kind: "Namespace", + Name: newNamespaceName, + }, + } + return hubClient.Update(ctx, crp) + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP to select different namespace") + }) + + It("should update CRP status with scheduled condition set to false", func() { + statusUpdatedActual := namespaceAccessibleCRPStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0", metav1.ConditionTrue, true) + Eventually(statusUpdatedActual, crpsEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status") }) - It("update CRP spec to trigger status change", func() { - // Update CRP spec to trigger a status change. + It("should update CRP to select original namespace", func() { Eventually(func() error { crp := &placementv1beta1.ClusterResourcePlacement{} if err := hubClient.Get(ctx, types.NamespacedName{Name: crpName}, crp); err != nil { return err } - // Change number of clusters to 1 to trigger status update. - crp.Spec.Policy.NumberOfClusters = ptr.To(int32(3)) + crp.Spec.ResourceSelectors = workResourceSelector() return hubClient.Update(ctx, crp) - }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP spec to trigger status change") + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP to select different namespace") + }) + + It("should update CRP status with scheduled condition set to true", func() { + statusUpdatedActual := namespaceAccessibleCRPStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0", metav1.ConditionTrue, false) + Eventually(statusUpdatedActual, crpsEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status") }) + }) + + Context("NamespaceAccessible CRP with namespace selector change for consistency validation - namespace doesn't exist", func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) - It("should not update CRP status after namespace deletion", func() { - // Since CRPS can't be updated (namespace doesn't exist), CRP status should remain unchanged. - Consistently(func() error { + BeforeAll(func() { + // Create work resources that will be selected by the CRP. + createWorkResources() + + // Create CRP with NamespaceAccessible scope. + crp := &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: workResourceSelector(), + Policy: &placementv1beta1.PlacementPolicy{ + PlacementType: placementv1beta1.PickAllPlacementType, + }, + StatusReportingScope: placementv1beta1.NamespaceAccessible, + }, + } + Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP") + }) + + AfterAll(func() { + // Clean up resources. + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + }) + + It("should update CRP status with initial namespace selection", func() { + statusUpdatedActual := namespaceAccessibleCRPStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0", metav1.ConditionTrue, false) + Eventually(statusUpdatedActual, crpsEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status") + }) + + It("should create CRPS in the initial namespace and verify its status", func() { + crpsMatchesActual := statussyncutils.CRPSStatusMatchesCRPActual(ctx, hubClient, crpName, appNamespace().Name) + Eventually(crpsMatchesActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "CRPS should be created in initial namespace and match CRP status") + }) + + It("should update CRP to select a non-existent namespace", func() { + // Update CRP to select the non-existent-namespace instead. + Eventually(func() error { crp := &placementv1beta1.ClusterResourcePlacement{} if err := hubClient.Get(ctx, types.NamespacedName{Name: crpName}, crp); err != nil { return err } - currentStatus := crp.Status - // Compare the entire status using cmp.Diff for better error reporting. - if diff := cmp.Diff(crpStatusBeforeWait, currentStatus); diff != "" { - return fmt.Errorf("CRP status changed after namespace deletion (-expected +actual):\n%s", diff) + crp.Spec.ResourceSelectors = []placementv1beta1.ResourceSelectorTerm{ + { + Group: corev1.GroupName, + Version: "v1", + Kind: "Namespace", + Name: "non-existent-namespace", + }, } + return hubClient.Update(ctx, crp) + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP to select different namespace") + }) - return nil - }, consistentlyDuration, consistentlyInterval).Should(Succeed(), "CRP status should remain unchanged after namespace deletion") + It("should update CRP status with scheduled condition set to false", func() { + statusUpdatedActual := namespaceAccessibleCRPStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0", metav1.ConditionTrue, true) + Eventually(statusUpdatedActual, crpsEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status") }) }) }) diff --git a/test/utils/crpstatussync/crp_status_sync.go b/test/utils/crpstatussync/crp_status_sync.go index 6880585b1..0257d5226 100644 --- a/test/utils/crpstatussync/crp_status_sync.go +++ b/test/utils/crpstatussync/crp_status_sync.go @@ -42,6 +42,16 @@ func CRPSStatusMatchesCRPActual(ctx context.Context, client client.Client, crpNa } // Construct expected CRPS. + // Filter out StatusSynced condition as it's specific to CRP status sync process + expectedStatus := crp.Status.DeepCopy() + var filteredConditions []metav1.Condition + for _, condition := range expectedStatus.Conditions { + if condition.Type != string(placementv1beta1.ClusterResourcePlacementStatusSyncedConditionType) { + filteredConditions = append(filteredConditions, condition) + } + } + expectedStatus.Conditions = filteredConditions + wantCRPS := &placementv1beta1.ClusterResourcePlacementStatus{ ObjectMeta: metav1.ObjectMeta{ Name: crpName, @@ -57,7 +67,7 @@ func CRPSStatusMatchesCRPActual(ctx context.Context, client client.Client, crpNa }, }, }, - PlacementStatus: crp.Status, + PlacementStatus: *expectedStatus, } // Compare CRPS with expected, ignoring fields that vary. From a2c35f2952b34b5b342bff06b6f59dc47260599a Mon Sep 17 00:00:00 2001 From: Zhiying Lin <54013513+zhiying-lin@users.noreply.github.com> Date: Sat, 20 Sep 2025 03:45:14 +0800 Subject: [PATCH 3/8] interface: support skipping validation when deleting member cluster (#251) interface: support force delete when deleting member cluster Signed-off-by: Zhiying Lin --- apis/cluster/v1beta1/membercluster_types.go | 24 ++ apis/cluster/v1beta1/zz_generated.deepcopy.go | 22 +- .../v1alpha1/zz_generated.deepcopy.go | 2 +- .../v1beta1/zz_generated.deepcopy.go | 2 +- apis/v1alpha1/zz_generated.deepcopy.go | 2 +- ...er.kubernetes-fleet.io_memberclusters.yaml | 12 + ...t.io_clusterresourceplacementstatuses.yaml | 345 ++++++++++++++++++ test/apis/v1alpha1/zz_generated.deepcopy.go | 2 +- 8 files changed, 406 insertions(+), 5 deletions(-) diff --git a/apis/cluster/v1beta1/membercluster_types.go b/apis/cluster/v1beta1/membercluster_types.go index 8c9601ea8..3680c681a 100644 --- a/apis/cluster/v1beta1/membercluster_types.go +++ b/apis/cluster/v1beta1/membercluster_types.go @@ -74,6 +74,30 @@ type MemberClusterSpec struct { // +kubebuilder:validation:MaxItems=100 // +optional Taints []Taint `json:"taints,omitempty"` + + // DeleteOptions for deleting the MemberCluster. + // +optional + DeleteOptions *DeleteOptions `json:"deleteOptions,omitempty"` +} + +// DeleteValidationMode identifies the type of validation when deleting a MemberCluster. +// +enum +type DeleteValidationMode string + +const ( + // DeleteValidationModeSkip skips the validation when deleting a MemberCluster. + DeleteValidationModeSkip = "Skip" + + // DeleteValidationModeStrict performs strict validation when deleting a MemberCluster. + DeleteValidationModeStrict = "Strict" +) + +type DeleteOptions struct { + // Mode of validation. Can be "Skip", or "Strict". Default is Strict. + // +kubebuilder:validation:Enum=Skip;Strict + // +kubebuilder:default=Strict + // +kubebuilder:validation:Optional + ValidationMode DeleteValidationMode `json:"validationMode,omitempty"` } // PropertyName is the name of a cluster property; it should be a Kubernetes label name. diff --git a/apis/cluster/v1beta1/zz_generated.deepcopy.go b/apis/cluster/v1beta1/zz_generated.deepcopy.go index 17e71a1a2..d0e850ca0 100644 --- a/apis/cluster/v1beta1/zz_generated.deepcopy.go +++ b/apis/cluster/v1beta1/zz_generated.deepcopy.go @@ -21,7 +21,7 @@ limitations under the License. package v1beta1 import ( - v1 "k8s.io/api/core/v1" + "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -49,6 +49,21 @@ func (in *AgentStatus) DeepCopy() *AgentStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeleteOptions) DeepCopyInto(out *DeleteOptions) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeleteOptions. +func (in *DeleteOptions) DeepCopy() *DeleteOptions { + if in == nil { + return nil + } + out := new(DeleteOptions) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *InternalMemberCluster) DeepCopyInto(out *InternalMemberCluster) { *out = *in @@ -228,6 +243,11 @@ func (in *MemberClusterSpec) DeepCopyInto(out *MemberClusterSpec) { *out = make([]Taint, len(*in)) copy(*out, *in) } + if in.DeleteOptions != nil { + in, out := &in.DeleteOptions, &out.DeleteOptions + *out = new(DeleteOptions) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemberClusterSpec. diff --git a/apis/placement/v1alpha1/zz_generated.deepcopy.go b/apis/placement/v1alpha1/zz_generated.deepcopy.go index df9f5e6d7..6d1656d18 100644 --- a/apis/placement/v1alpha1/zz_generated.deepcopy.go +++ b/apis/placement/v1alpha1/zz_generated.deepcopy.go @@ -22,7 +22,7 @@ package v1alpha1 import ( "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" ) diff --git a/apis/placement/v1beta1/zz_generated.deepcopy.go b/apis/placement/v1beta1/zz_generated.deepcopy.go index 05fef4c3d..0e755b56e 100644 --- a/apis/placement/v1beta1/zz_generated.deepcopy.go +++ b/apis/placement/v1beta1/zz_generated.deepcopy.go @@ -21,7 +21,7 @@ limitations under the License. package v1beta1 import ( - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" ) diff --git a/apis/v1alpha1/zz_generated.deepcopy.go b/apis/v1alpha1/zz_generated.deepcopy.go index 85550ca19..27a862c43 100644 --- a/apis/v1alpha1/zz_generated.deepcopy.go +++ b/apis/v1alpha1/zz_generated.deepcopy.go @@ -22,7 +22,7 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/config/crd/bases/cluster.kubernetes-fleet.io_memberclusters.yaml b/config/crd/bases/cluster.kubernetes-fleet.io_memberclusters.yaml index dae3675ee..fdc6093e7 100644 --- a/config/crd/bases/cluster.kubernetes-fleet.io_memberclusters.yaml +++ b/config/crd/bases/cluster.kubernetes-fleet.io_memberclusters.yaml @@ -432,6 +432,18 @@ spec: spec: description: The desired state of MemberCluster. properties: + deleteOptions: + description: DeleteOptions for deleting the MemberCluster. + properties: + validationMode: + default: Strict + description: Mode of validation. Can be "Skip", or "Strict". Default + is Strict. + enum: + - Skip + - Strict + type: string + type: object heartbeatPeriodSeconds: default: 60 description: 'How often (in seconds) for the member cluster to send diff --git a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacementstatuses.yaml b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacementstatuses.yaml index b49d38120..a474e0244 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacementstatuses.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacementstatuses.yaml @@ -19,6 +19,351 @@ spec: singular: clusterresourceplacementstatus scope: Namespaced versions: + - name: v1 + schema: + openAPIV3Schema: + description: ClusterResourcePlacementStatus defines the observed state of + the ClusterResourcePlacement object. + properties: + conditions: + description: Conditions is an array of current observed conditions for + ClusterResourcePlacement. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + observedResourceIndex: + description: |- + Resource index logically represents the generation of the selected resources. + We take a new snapshot of the selected resources whenever the selection or their content change. + Each snapshot has a different resource index. + One resource snapshot can contain multiple clusterResourceSnapshots CRs in order to store large amount of resources. + To get clusterResourceSnapshot of a given resource index, use the following command: + `kubectl get ClusterResourceSnapshot --selector=kubernetes-fleet.io/resource-index=$ObservedResourceIndex ` + ObservedResourceIndex is the resource index that the conditions in the ClusterResourcePlacementStatus observe. + For example, a condition of `ClusterResourcePlacementWorkSynchronized` type + is observing the synchronization status of the resource snapshot with the resource index $ObservedResourceIndex. + type: string + placementStatuses: + description: |- + PlacementStatuses contains a list of placement status on the clusters that are selected by PlacementPolicy. + Each selected cluster according to the latest resource placement is guaranteed to have a corresponding placementStatuses. + In the pickN case, there are N placement statuses where N = NumberOfClusters; Or in the pickFixed case, there are + N placement statuses where N = ClusterNames. + In these cases, some of them may not have assigned clusters when we cannot fill the required number of clusters. + items: + description: ResourcePlacementStatus represents the placement status + of selected resources for one target cluster. + properties: + applicableClusterResourceOverrides: + description: |- + ApplicableClusterResourceOverrides contains a list of applicable ClusterResourceOverride snapshots associated with + the selected resources. + + This field is alpha-level and is for the override policy feature. + items: + type: string + type: array + applicableResourceOverrides: + description: |- + ApplicableResourceOverrides contains a list of applicable ResourceOverride snapshots associated with the selected + resources. + + This field is alpha-level and is for the override policy feature. + items: + description: NamespacedName comprises a resource name, with a + mandatory namespace. + properties: + name: + description: Name is the name of the namespaced scope resource. + type: string + namespace: + description: Namespace is namespace of the namespaced scope + resource. + type: string + required: + - name + - namespace + type: object + type: array + clusterName: + description: |- + ClusterName is the name of the cluster this resource is assigned to. + If it is not empty, its value should be unique cross all placement decisions for the Placement. + type: string + conditions: + description: Conditions is an array of current observed conditions + for ResourcePlacementStatus. + items: + description: Condition contains details for one aspect of the + current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + failedPlacements: + description: |- + FailedPlacements is a list of all the resources failed to be placed to the given cluster or the resource is unavailable. + Note that we only include 100 failed resource placements even if there are more than 100. + This field is only meaningful if the `ClusterName` is not empty. + items: + description: FailedResourcePlacement contains the failure details + of a failed resource placement. + properties: + condition: + description: The failed condition status. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + envelope: + description: Envelope identifies the envelope object that + contains this resource. + properties: + name: + description: Name of the envelope object. + type: string + namespace: + description: Namespace is the namespace of the envelope + object. Empty if the envelope object is cluster scoped. + type: string + type: + default: ConfigMap + description: Type of the envelope object. + enum: + - ConfigMap + type: string + required: + - name + type: object + group: + description: Group is the group name of the selected resource. + type: string + kind: + description: Kind represents the Kind of the selected resources. + type: string + name: + description: Name of the target resource. + type: string + namespace: + description: Namespace is the namespace of the resource. Empty + if the resource is cluster scoped. + type: string + version: + description: Version is the version of the selected resource. + type: string + required: + - condition + - kind + - name + - version + type: object + maxItems: 100 + type: array + type: object + type: array + selectedResources: + description: SelectedResources contains a list of resources selected by + ResourceSelectors. + items: + description: ResourceIdentifier identifies one Kubernetes resource. + properties: + envelope: + description: Envelope identifies the envelope object that contains + this resource. + properties: + name: + description: Name of the envelope object. + type: string + namespace: + description: Namespace is the namespace of the envelope object. + Empty if the envelope object is cluster scoped. + type: string + type: + default: ConfigMap + description: Type of the envelope object. + enum: + - ConfigMap + type: string + required: + - name + type: object + group: + description: Group is the group name of the selected resource. + type: string + kind: + description: Kind represents the Kind of the selected resources. + type: string + name: + description: Name of the target resource. + type: string + namespace: + description: Namespace is the namespace of the resource. Empty if + the resource is cluster scoped. + type: string + version: + description: Version is the version of the selected resource. + type: string + required: + - kind + - name + - version + type: object + type: array + type: object + served: true + storage: false - additionalPrinterColumns: - jsonPath: .sourceStatus.observedResourceIndex name: Resource-Index diff --git a/test/apis/v1alpha1/zz_generated.deepcopy.go b/test/apis/v1alpha1/zz_generated.deepcopy.go index 143bdee7b..081bec913 100644 --- a/test/apis/v1alpha1/zz_generated.deepcopy.go +++ b/test/apis/v1alpha1/zz_generated.deepcopy.go @@ -21,7 +21,7 @@ limitations under the License. package v1alpha1 import ( - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) From 93905a0fb6d4ba85b4aa99d29d85d6872bcd12b0 Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Mon, 22 Sep 2025 17:10:36 +1000 Subject: [PATCH 4/8] test: add integration tests for work applier exponential backoff behavior (#191) --- Makefile | 2 + .../workapplier/availability_tracker_test.go | 2 +- .../workapplier/backoff_integration_test.go | 524 ++++++++++++++++++ pkg/controllers/workapplier/backoff_test.go | 68 +-- ...roller_integration_migrated_helper_test.go | 8 +- .../controller_integration_migrated_test.go | 70 +-- .../controller_integration_test.go | 462 +++++++-------- .../drift_detection_takeover_test.go | 8 +- .../workapplier/preprocess_test.go | 2 +- pkg/controllers/workapplier/status_test.go | 22 +- pkg/controllers/workapplier/suite_test.go | 179 ++++-- 11 files changed, 993 insertions(+), 354 deletions(-) create mode 100644 pkg/controllers/workapplier/backoff_integration_test.go diff --git a/Makefile b/Makefile index 8c7a8e190..8815749f1 100644 --- a/Makefile +++ b/Makefile @@ -137,6 +137,8 @@ test: manifests generate fmt vet local-unit-test integration-test## Run tests. ## workaround to bypass the pkg/controllers/workv1alpha1 tests failure ## rollout controller tests need a bit longer to complete, so we increase the timeout ## +# Set up the timeout parameters as some of the test lengths have exceeded the default 10 minute mark. +# TO-DO (chenyu1): enable parallelization for single package integration tests. .PHONY: local-unit-test local-unit-test: $(ENVTEST) ## Run tests. export CGO_ENABLED=1 && \ diff --git a/pkg/controllers/workapplier/availability_tracker_test.go b/pkg/controllers/workapplier/availability_tracker_test.go index 1127cf612..03a212f83 100644 --- a/pkg/controllers/workapplier/availability_tracker_test.go +++ b/pkg/controllers/workapplier/availability_tracker_test.go @@ -1013,7 +1013,7 @@ func TestTrackInMemberClusterObjAvailabilityByGVR(t *testing.T) { // TestTrackInMemberClusterObjAvailability tests the trackInMemberClusterObjAvailability method. func TestTrackInMemberClusterObjAvailability(t *testing.T) { ctx := context.Background() - workRef := klog.KRef(memberReservedNSName, workName) + workRef := klog.KRef(memberReservedNSName1, workName) availableDeploy := deploy.DeepCopy() availableDeploy.Status = appsv1.DeploymentStatus{ diff --git a/pkg/controllers/workapplier/backoff_integration_test.go b/pkg/controllers/workapplier/backoff_integration_test.go new file mode 100644 index 000000000..cdc857298 --- /dev/null +++ b/pkg/controllers/workapplier/backoff_integration_test.go @@ -0,0 +1,524 @@ +/* +Copyright 2025 The KubeFleet Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package workapplier + +import ( + "fmt" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + fleetv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1" + "github.com/kubefleet-dev/kubefleet/pkg/utils" + "github.com/kubefleet-dev/kubefleet/pkg/utils/condition" +) + +// Note (chenyu1): all test cases in this file use a separate test environment +// (same hub cluster, different fleet member reserved namespace, different +// work applier instance) from the other integration tests. This is needed +// to (relatively speaking) reliably verify the exponential backoff behavior +// in the work applier. + +var ( + diffObservationTimeChangedActual = func( + workName string, + wantWorkStatus *fleetv1beta1.WorkStatus, + curDiffObservedTime, lastDiffObservedTime *metav1.Time, + ) func() error { + return func() error { + // Retrieve the Work object. + work := &fleetv1beta1.Work{} + if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName2}, work); err != nil { + return fmt.Errorf("failed to retrieve the Work object: %w", err) + } + + // Verify that the status never changed (except for timestamps). + if diff := cmp.Diff( + &work.Status, wantWorkStatus, + ignoreFieldConditionLTTMsg, + ignoreDiffDetailsObsTime, ignoreDriftDetailsObsTime, + cmpopts.SortSlices(lessFuncPatchDetail), + ); diff != "" { + StopTrying("the work object status has changed unexpectedly").Wrap(fmt.Errorf("work status diff (-got, +want):\n%s", diff)).Now() + } + + curDiffObservedTime.Time = work.Status.ManifestConditions[0].DiffDetails.ObservationTime.Time + if !curDiffObservedTime.Equal(lastDiffObservedTime) { + return nil + } + return fmt.Errorf("the diff observation time remains unchanged") + } + } + + driftObservationTimeChangedActual = func( + workName string, + wantWorkStatus *fleetv1beta1.WorkStatus, + curDriftObservedTime, lastDriftObservedTime *metav1.Time, + ) func() error { + return func() error { + // Retrieve the Work object. + work := &fleetv1beta1.Work{} + if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName2}, work); err != nil { + return fmt.Errorf("failed to retrieve the Work object: %w", err) + } + + // Verify that the status never changed (except for timestamps). + if diff := cmp.Diff( + &work.Status, wantWorkStatus, + ignoreFieldConditionLTTMsg, + ignoreDiffDetailsObsTime, ignoreDriftDetailsObsTime, + cmpopts.SortSlices(lessFuncPatchDetail), + ); diff != "" { + StopTrying("the work object status has changed unexpectedly").Wrap(fmt.Errorf("work status diff (-got, +want):\n%s", diff)).Now() + } + + curDriftObservedTime.Time = work.Status.ManifestConditions[0].DriftDetails.ObservationTime.Time + if !curDriftObservedTime.Equal(lastDriftObservedTime) { + return nil + } + return fmt.Errorf("the drift observation time remains unchanged") + } + } +) + +var _ = Describe("exponential backoff", func() { + Context("slow backoff and fast backoff", Ordered, func() { + workName := fmt.Sprintf(workNameTemplate, utils.RandStr()) + // The environment prepared by the envtest package does not support namespace + // deletion; each test case would use a new namespace. + nsName := fmt.Sprintf(nsNameTemplate, utils.RandStr()) + + var regularNS *corev1.Namespace + var lastDiffObservedTime *metav1.Time + + wantWorkStatus := &fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: condition.WorkNotAllManifestsAppliedReason, + ObservedGeneration: 1, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ApplyOrReportDiffResTypeFailedToTakeOver), + ObservedGeneration: 0, + }, + }, + DiffDetails: &fleetv1beta1.DiffDetails{ + ObservedInMemberClusterGeneration: ptr.To(int64(0)), + ObservedDiffs: []fleetv1beta1.PatchDetail{ + { + Path: fmt.Sprintf("/metadata/labels/%s", dummyLabelKey), + ValueInMember: dummyLabelValue2, + ValueInHub: dummyLabelValue1, + }, + }, + }, + }, + }, + } + + BeforeAll(func() { + // Prepare a NS object. + regularNS = ns.DeepCopy() + regularNS.Name = nsName + regularNS.Labels = map[string]string{ + dummyLabelKey: dummyLabelValue1, + } + regularNSJSON := marshalK8sObjJSON(regularNS) + + // Create the NS object on the member cluster, with a different label value. + preExistingNS := ns.DeepCopy() + preExistingNS.Name = nsName + preExistingNS.Labels = map[string]string{ + dummyLabelKey: dummyLabelValue2, + } + Expect(memberClient2.Create(ctx, preExistingNS)).To(Succeed(), "Failed to create pre-existing NS") + + // Create a new Work object with all the manifest JSONs. + // + // This Work object uses an apply strategy that allows takeover but will check for diffs (partial + // comparison). Due to the presence of diffs, the Work object will be considered to be of a state + // of apply op failure. + applyStrategy := &fleetv1beta1.ApplyStrategy{ + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeIfNoDiff, + } + createWorkObject(workName, memberReservedNSName2, applyStrategy, regularNSJSON) + }) + + // For simplicity reasons, this test case will skip some of the regular apply op result verification + // (finalizer check, AppliedWork object check, etc.). + + It("should update the Work object status", func() { + // Prepare the status information. + + // Use custom check logic so that the test case can track timestamps across steps. + Eventually(func() error { + // Retrieve the Work object. + work := &fleetv1beta1.Work{} + if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName2}, work); err != nil { + return fmt.Errorf("failed to retrieve the Work object: %w", err) + } + + if diff := cmp.Diff( + &work.Status, wantWorkStatus, + ignoreFieldConditionLTTMsg, + ignoreDiffDetailsObsTime, ignoreDriftDetailsObsTime, + cmpopts.SortSlices(lessFuncPatchDetail), + ); diff != "" { + return fmt.Errorf("work status diff (-got, +want):\n%s", diff) + } + + // Track the observation timestamp of the diff details. + lastDiffObservedTime = &work.Status.ManifestConditions[0].DiffDetails.ObservationTime + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update Work object status") + }) + + It("should wait for the fixed delay period of time before next reconciliation", func() { + curDiffObservedTime := &metav1.Time{} + + // Need to poll a bit longer to avoid flakiness. + Eventually(diffObservationTimeChangedActual(workName, wantWorkStatus, curDiffObservedTime, lastDiffObservedTime), eventuallyDuration*2, eventuallyInterval).Should(Or(Succeed())) + + // Fixed delay is set to 10 seconds. Give a one-sec leeway to avoid flakiness. + Expect(curDiffObservedTime.Sub(lastDiffObservedTime.Time)).To(And( + BeNumerically(">=", time.Second*9), + BeNumerically("<=", time.Second*11), + ), "the interval between two observations is not as expected") + + // Update the tracked observation time. + lastDiffObservedTime = curDiffObservedTime + }) + + It("should start to back off slowly (attempt 1)", func() { + curDiffObservedTime := &metav1.Time{} + + // Need to poll a bit longer to avoid flakiness. + Eventually(diffObservationTimeChangedActual(workName, wantWorkStatus, curDiffObservedTime, lastDiffObservedTime), eventuallyDuration*3, eventuallyInterval).Should(Or(Succeed())) + + // The first slow backoff delay is 20 seconds. Give a one-sec leeway to avoid flakiness. + Expect(curDiffObservedTime.Sub(lastDiffObservedTime.Time)).To(And( + BeNumerically(">=", time.Second*19), + BeNumerically("<=", time.Second*21), + ), "the interval between two observations is not as expected") + + // Update the tracked observation time. + lastDiffObservedTime = curDiffObservedTime + }) + + It("should start to back off slowly (attempt 2)", func() { + curDiffObservedTime := &metav1.Time{} + + // Need to poll a bit longer to avoid flakiness. + Eventually(diffObservationTimeChangedActual(workName, wantWorkStatus, curDiffObservedTime, lastDiffObservedTime), eventuallyDuration*4, eventuallyInterval).Should(Or(Succeed())) + + // The second slow backoff delay is 30 seconds. Give a one-sec leeway to avoid flakiness. + Expect(curDiffObservedTime.Sub(lastDiffObservedTime.Time)).To(And( + BeNumerically(">=", time.Second*29), + BeNumerically("<=", time.Second*31), + ), "the interval between two observations is not as expected") + + // Update the tracked observation time. + lastDiffObservedTime = curDiffObservedTime + }) + + It("should start to back off fastly (attempt #1)", func() { + curDiffObservedTime := &metav1.Time{} + + // Need to poll a bit longer to avoid flakiness. + Eventually(diffObservationTimeChangedActual(workName, wantWorkStatus, curDiffObservedTime, lastDiffObservedTime), eventuallyDuration*7, eventuallyInterval).Should(Or(Succeed())) + + // The first fast backoff delay is 60 seconds. Give a one-sec leeway to avoid flakiness. + Expect(curDiffObservedTime.Sub(lastDiffObservedTime.Time)).To(And( + BeNumerically(">=", time.Second*59), + BeNumerically("<=", time.Second*61), + ), "the interval between two observations is not as expected") + + // Update the tracked observation time. + lastDiffObservedTime = curDiffObservedTime + }) + + It("should reach maximum backoff", func() { + curDiffObservedTime := &metav1.Time{} + + // Need to poll a bit longer to avoid flakiness. + Eventually(diffObservationTimeChangedActual(workName, wantWorkStatus, curDiffObservedTime, lastDiffObservedTime), eventuallyDuration*10, eventuallyInterval).Should(Or(Succeed())) + + // The maximum backoff delay is 60 seconds. Give a one-sec leeway to avoid flakiness. + Expect(curDiffObservedTime.Sub(lastDiffObservedTime.Time)).To(And( + BeNumerically(">=", time.Second*89), + BeNumerically("<=", time.Second*91), + ), "the interval between two observations is not as expected") + + // Update the tracked observation time. + lastDiffObservedTime = curDiffObservedTime + }) + + AfterAll(func() { + // Delete the Work object and related resources. + deleteWorkObject(workName, memberReservedNSName2) + + // Ensure that the AppliedWork object has been removed. + appliedWorkRemovedActual := appliedWorkRemovedActual(workName, nsName) + Eventually(appliedWorkRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the AppliedWork object") + + workRemovedActual := workRemovedActual(workName) + Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") + // The environment prepared by the envtest package does not support namespace + // deletion; consequently this test suite would not attempt to verify its deletion. + }) + }) + + Context("skip to fast backoff (available)", Ordered, func() { + workName := fmt.Sprintf(workNameTemplate, utils.RandStr()) + // The environment prepared by the envtest package does not support namespace + // deletion; each test case would use a new namespace. + nsName := fmt.Sprintf(nsNameTemplate, utils.RandStr()) + + var regularNS *corev1.Namespace + var lastDriftObservedTime *metav1.Time + + wantWorkStatus := &fleetv1beta1.WorkStatus{ + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAppliedReason, + ObservedGeneration: 1, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAvailableReason, + ObservedGeneration: 1, + }, + }, + ManifestConditions: []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeApplied), + ObservedGeneration: 0, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(AvailabilityResultTypeAvailable), + ObservedGeneration: 0, + }, + }, + DriftDetails: &fleetv1beta1.DriftDetails{ + ObservedInMemberClusterGeneration: 0, + ObservedDrifts: []fleetv1beta1.PatchDetail{ + { + Path: fmt.Sprintf("/metadata/labels/%s", dummyLabelKey), + ValueInMember: dummyLabelValue2, + }, + { + Path: "/spec/finalizers", + ValueInMember: "[kubernetes]", + }, + }, + }, + }, + }, + } + + BeforeAll(func() { + // Prepare a NS object. + regularNS = ns.DeepCopy() + regularNS.Name = nsName + regularNSJSON := marshalK8sObjJSON(regularNS) + + // Create the NS object on the member cluster, with a different label value. + preExistingNS := ns.DeepCopy() + preExistingNS.Name = nsName + preExistingNS.Labels = map[string]string{ + dummyLabelKey: dummyLabelValue2, + } + Expect(memberClient2.Create(ctx, preExistingNS)).To(Succeed(), "Failed to create pre-existing NS") + + // Create a new Work object with all the manifest JSONs. + // + // This Work object uses an apply strategy that always take over and use full + // comparison for drift detection. Apply op will be successful, and namespaces are + // considered to be immediately available after creation; Fleet will report the label + // differences as drifts without blocking the apply ops. + applyStrategy := &fleetv1beta1.ApplyStrategy{ + ComparisonOption: fleetv1beta1.ComparisonOptionTypeFullComparison, + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeAlways, + WhenToApply: fleetv1beta1.WhenToApplyTypeAlways, + } + createWorkObject(workName, memberReservedNSName2, applyStrategy, regularNSJSON) + }) + + // For simplicity reasons, this test case will skip some of the regular apply op result verification + // (finalizer check, AppliedWork object check, etc.). + + It("should update the Work object status", func() { + // Prepare the status information. + + // Use custom check logic so that the test case can track timestamps across steps. + Eventually(func() error { + // Retrieve the Work object. + work := &fleetv1beta1.Work{} + if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName2}, work); err != nil { + return fmt.Errorf("failed to retrieve the Work object: %w", err) + } + + if diff := cmp.Diff( + &work.Status, wantWorkStatus, + ignoreFieldConditionLTTMsg, + ignoreDiffDetailsObsTime, ignoreDriftDetailsObsTime, + cmpopts.SortSlices(lessFuncPatchDetail), + ); diff != "" { + return fmt.Errorf("work status diff (-got, +want):\n%s", diff) + } + + // Track the observation timestamp of the diff details. + lastDriftObservedTime = &work.Status.ManifestConditions[0].DriftDetails.ObservationTime + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update Work object status") + }) + + It("should wait for the fixed delay period of time before next reconciliation", func() { + curDriftObservedTime := &metav1.Time{} + + // Need to poll a bit longer to avoid flakiness. + Eventually(driftObservationTimeChangedActual(workName, wantWorkStatus, curDriftObservedTime, lastDriftObservedTime), eventuallyDuration*2, eventuallyInterval).Should(Or(Succeed())) + + // Fixed delay is set to 10 seconds. Give a one-sec leeway to avoid flakiness. + Expect(curDriftObservedTime.Sub(lastDriftObservedTime.Time)).To(And( + BeNumerically(">=", time.Second*9), + BeNumerically("<=", time.Second*11), + ), "the interval between two observations is not as expected") + + // Update the tracked observation time. + lastDriftObservedTime = curDriftObservedTime + }) + + It("should start to back off slowly (attempt #1)", func() { + curDriftObservedTime := &metav1.Time{} + + // Need to poll a bit longer to avoid flakiness. + Eventually(driftObservationTimeChangedActual(workName, wantWorkStatus, curDriftObservedTime, lastDriftObservedTime), eventuallyDuration*3, eventuallyInterval).Should(Or(Succeed())) + + // The first fast backoff delay is 60 seconds. Give a one-sec leeway to avoid flakiness. + Expect(curDriftObservedTime.Sub(lastDriftObservedTime.Time)).To(And( + BeNumerically(">=", time.Second*19), + BeNumerically("<=", time.Second*21), + ), "the interval between two observations is not as expected") + + // Update the tracked observation time. + lastDriftObservedTime = curDriftObservedTime + }) + + It("should skip to backing off fastly (attempt #1)", func() { + curDriftObservedTime := &metav1.Time{} + + // Need to poll a bit longer to avoid flakiness. + Eventually(driftObservationTimeChangedActual(workName, wantWorkStatus, curDriftObservedTime, lastDriftObservedTime), eventuallyDuration*5, eventuallyInterval).Should(Or(Succeed())) + + // The first fast backoff delay is 40 seconds. Give a one-sec leeway to avoid flakiness. + Expect(curDriftObservedTime.Sub(lastDriftObservedTime.Time)).To(And( + BeNumerically(">=", time.Second*39), + BeNumerically("<=", time.Second*41), + ), "the interval between two observations is not as expected") + + // Update the tracked observation time. + lastDriftObservedTime = curDriftObservedTime + }) + + It("should skip to backing off fastly (attempt #2)", func() { + curDriftObservedTime := &metav1.Time{} + + // Need to poll a bit longer to avoid flakiness. + Eventually(driftObservationTimeChangedActual(workName, wantWorkStatus, curDriftObservedTime, lastDriftObservedTime), eventuallyDuration*9, eventuallyInterval).Should(Or(Succeed())) + + // The second fast backoff delay is 80 seconds. Give a one-sec leeway to avoid flakiness. + Expect(curDriftObservedTime.Sub(lastDriftObservedTime.Time)).To(And( + BeNumerically(">=", time.Second*79), + BeNumerically("<=", time.Second*81), + ), "the interval between two observations is not as expected") + + // Update the tracked observation time. + lastDriftObservedTime = curDriftObservedTime + }) + + It("should reach maximum backoff", func() { + curDriftObservedTime := &metav1.Time{} + + // Need to poll a bit longer to avoid flakiness. + Eventually(driftObservationTimeChangedActual(workName, wantWorkStatus, curDriftObservedTime, lastDriftObservedTime), eventuallyDuration*10, eventuallyInterval).Should(Or(Succeed())) + + // The maximum backoff delay is 60 seconds. Give a one-sec leeway to avoid flakiness. + Expect(curDriftObservedTime.Sub(lastDriftObservedTime.Time)).To(And( + BeNumerically(">=", time.Second*89), + BeNumerically("<=", time.Second*91), + ), "the interval between two observations is not as expected") + + // Update the tracked observation time. + lastDriftObservedTime = curDriftObservedTime + }) + + AfterAll(func() { + // Delete the Work object and related resources. + deleteWorkObject(workName, memberReservedNSName2) + + // Ensure that the AppliedWork object has been removed. + appliedWorkRemovedActual := appliedWorkRemovedActual(workName, nsName) + Eventually(appliedWorkRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the AppliedWork object") + + workRemovedActual := workRemovedActual(workName) + Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") + // The environment prepared by the envtest package does not support namespace + // deletion; consequently this test suite would not attempt to verify its deletion. + }) + }) +}) diff --git a/pkg/controllers/workapplier/backoff_test.go b/pkg/controllers/workapplier/backoff_test.go index fac61b8e6..feecaa768 100644 --- a/pkg/controllers/workapplier/backoff_test.go +++ b/pkg/controllers/workapplier/backoff_test.go @@ -33,7 +33,7 @@ import ( func TestWhenWithFullNormalSequence(t *testing.T) { work := &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, }, } @@ -150,7 +150,7 @@ func TestWhenWithFullNormalSequence(t *testing.T) { func TestWhenWithFullNoSlowBackoffSequence(t *testing.T) { work := &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, }, } @@ -211,7 +211,7 @@ func TestWhenWithFullNoSlowBackoffSequence(t *testing.T) { func TestWhenWithFullNoFastBackoffSequeuce(t *testing.T) { work := &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, }, } @@ -276,7 +276,7 @@ func TestWhenWithFullNoFastBackoffSequeuce(t *testing.T) { func TestWhenWithNoBackoffSequence(t *testing.T) { work := &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, }, } @@ -768,7 +768,7 @@ func TestWhenWithGenerationAndProcessingResultChange(t *testing.T) { name: "first requeue", work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, }, }, @@ -779,7 +779,7 @@ func TestWhenWithGenerationAndProcessingResultChange(t *testing.T) { name: "second requeue", work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, }, }, @@ -790,7 +790,7 @@ func TestWhenWithGenerationAndProcessingResultChange(t *testing.T) { name: "requeue (#3) w/ gen change", work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, Generation: 2, }, @@ -802,7 +802,7 @@ func TestWhenWithGenerationAndProcessingResultChange(t *testing.T) { name: "requeue #4", work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, Generation: 2, }, @@ -815,7 +815,7 @@ func TestWhenWithGenerationAndProcessingResultChange(t *testing.T) { name: "requeue #5", work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, Generation: 2, }, @@ -827,7 +827,7 @@ func TestWhenWithGenerationAndProcessingResultChange(t *testing.T) { name: "requeue #6", work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, Generation: 2, }, @@ -839,7 +839,7 @@ func TestWhenWithGenerationAndProcessingResultChange(t *testing.T) { name: "requeue #7 w/ processing result change", work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, Generation: 2, }, @@ -855,7 +855,7 @@ func TestWhenWithGenerationAndProcessingResultChange(t *testing.T) { name: "requeue #8", work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, Generation: 2, }, @@ -871,7 +871,7 @@ func TestWhenWithGenerationAndProcessingResultChange(t *testing.T) { name: "requeue #9", work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, Generation: 2, }, @@ -887,7 +887,7 @@ func TestWhenWithGenerationAndProcessingResultChange(t *testing.T) { name: "requeue #10", work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, Generation: 2, }, @@ -903,7 +903,7 @@ func TestWhenWithGenerationAndProcessingResultChange(t *testing.T) { name: "requeue #11", work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, Generation: 2, }, @@ -919,7 +919,7 @@ func TestWhenWithGenerationAndProcessingResultChange(t *testing.T) { name: "requeue #12 w/ both gen and processing result change", work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, Generation: 3, }, @@ -970,7 +970,7 @@ func TestWhenWithSkipToFastBackoff(t *testing.T) { name: "first requeue", work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, }, }, @@ -986,7 +986,7 @@ func TestWhenWithSkipToFastBackoff(t *testing.T) { name: "requeue #2, work becomes available", work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, }, Status: fleetv1beta1.WorkStatus{ @@ -1010,7 +1010,7 @@ func TestWhenWithSkipToFastBackoff(t *testing.T) { name: "requeue #3, work stays available", work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, }, Status: fleetv1beta1.WorkStatus{ @@ -1034,7 +1034,7 @@ func TestWhenWithSkipToFastBackoff(t *testing.T) { name: "requeue #4, work stays available", work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, }, Status: fleetv1beta1.WorkStatus{ @@ -1058,7 +1058,7 @@ func TestWhenWithSkipToFastBackoff(t *testing.T) { name: "requeue #5, work stays available", work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, }, Status: fleetv1beta1.WorkStatus{ @@ -1082,7 +1082,7 @@ func TestWhenWithSkipToFastBackoff(t *testing.T) { name: "requeue #6, work changed to ReportDiff mode", work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, Generation: 2, }, @@ -1107,7 +1107,7 @@ func TestWhenWithSkipToFastBackoff(t *testing.T) { name: "requeue #7, no diff found", work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, Generation: 2, }, @@ -1132,7 +1132,7 @@ func TestWhenWithSkipToFastBackoff(t *testing.T) { name: "requeue #8, no diff found", work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, Generation: 2, }, @@ -1157,7 +1157,7 @@ func TestWhenWithSkipToFastBackoff(t *testing.T) { name: "requeue #9, no diff found", work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, Generation: 2, }, @@ -1182,7 +1182,7 @@ func TestWhenWithSkipToFastBackoff(t *testing.T) { name: "requeue #9, diff found", work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, Generation: 2, }, @@ -1207,7 +1207,7 @@ func TestWhenWithSkipToFastBackoff(t *testing.T) { name: "requeue #10, diff found", work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, Generation: 2, }, @@ -1232,7 +1232,7 @@ func TestWhenWithSkipToFastBackoff(t *testing.T) { name: "requeue #11, diff found", work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, Generation: 2, }, @@ -1272,15 +1272,15 @@ func TestWhenWithSkipToFastBackoff(t *testing.T) { // TestForget tests the Forget method. func TestForget(t *testing.T) { workNamespacedName1 := types.NamespacedName{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: fmt.Sprintf(workNameTemplate, "1"), } workNamespacedName2 := types.NamespacedName{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: fmt.Sprintf(workNameTemplate, "2"), } workNamespacedName3 := types.NamespacedName{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: fmt.Sprintf(workNameTemplate, "3"), } @@ -1333,7 +1333,7 @@ func TestForget(t *testing.T) { }, work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workNamespacedName2.Name, }, }, @@ -1392,7 +1392,7 @@ func TestForget(t *testing.T) { }, work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workNamespacedName3.Name, }, }, @@ -1452,7 +1452,7 @@ func TestForget(t *testing.T) { func TestComputeProcessingResultHash(t *testing.T) { work := &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Name: workName, }, } diff --git a/pkg/controllers/workapplier/controller_integration_migrated_helper_test.go b/pkg/controllers/workapplier/controller_integration_migrated_helper_test.go index 5e494080d..0833bd963 100644 --- a/pkg/controllers/workapplier/controller_integration_migrated_helper_test.go +++ b/pkg/controllers/workapplier/controller_integration_migrated_helper_test.go @@ -41,7 +41,7 @@ func createWorkWithManifest(manifest runtime.Object) *fleetv1beta1.Work { newWork := fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ Name: "work-" + utilrand.String(5), - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, }, Spec: fleetv1beta1.WorkSpec{ Workload: fleetv1beta1.WorkloadTemplate{ @@ -59,7 +59,7 @@ func createWorkWithManifest(manifest runtime.Object) *fleetv1beta1.Work { // verifyAppliedConfigMap verifies that the applied CM is the same as the CM we want to apply func verifyAppliedConfigMap(cm *corev1.ConfigMap) *corev1.ConfigMap { var appliedCM corev1.ConfigMap - Expect(memberClient.Get(context.Background(), types.NamespacedName{Name: cm.GetName(), Namespace: cm.GetNamespace()}, &appliedCM)).Should(Succeed()) + Expect(memberClient1.Get(context.Background(), types.NamespacedName{Name: cm.GetName(), Namespace: cm.GetNamespace()}, &appliedCM)).Should(Succeed()) By("Check the config map label") Expect(cmp.Diff(appliedCM.Labels, cm.Labels)).Should(BeEmpty()) @@ -81,7 +81,7 @@ func verifyAppliedConfigMap(cm *corev1.ConfigMap) *corev1.ConfigMap { func waitForWorkToApply(workName string) *fleetv1beta1.Work { var resultWork fleetv1beta1.Work Eventually(func() bool { - err := hubClient.Get(context.Background(), types.NamespacedName{Name: workName, Namespace: memberReservedNSName}, &resultWork) + err := hubClient.Get(context.Background(), types.NamespacedName{Name: workName, Namespace: memberReservedNSName1}, &resultWork) if err != nil { return false } @@ -105,7 +105,7 @@ func waitForWorkToApply(workName string) *fleetv1beta1.Work { func waitForWorkToBeAvailable(workName string) *fleetv1beta1.Work { var resultWork fleetv1beta1.Work Eventually(func() bool { - err := hubClient.Get(context.Background(), types.NamespacedName{Name: workName, Namespace: memberReservedNSName}, &resultWork) + err := hubClient.Get(context.Background(), types.NamespacedName{Name: workName, Namespace: memberReservedNSName1}, &resultWork) if err != nil { return false } diff --git a/pkg/controllers/workapplier/controller_integration_migrated_test.go b/pkg/controllers/workapplier/controller_integration_migrated_test.go index 1f25be53c..ebc6a047e 100644 --- a/pkg/controllers/workapplier/controller_integration_migrated_test.go +++ b/pkg/controllers/workapplier/controller_integration_migrated_test.go @@ -108,7 +108,7 @@ var _ = Describe("Work Controller", func() { By("Check applied config map") var configMap corev1.ConfigMap - Expect(memberClient.Get(context.Background(), types.NamespacedName{Name: cmName, Namespace: cmNamespace}, &configMap)).Should(Succeed()) + Expect(memberClient1.Get(context.Background(), types.NamespacedName{Name: cmName, Namespace: cmNamespace}, &configMap)).Should(Succeed()) Expect(cmp.Diff(configMap.Labels, cm.Labels)).Should(BeEmpty()) Expect(cmp.Diff(configMap.Data, cm.Data)).Should(BeEmpty()) @@ -223,11 +223,11 @@ var _ = Describe("Work Controller", func() { // remove label key2 and key3 delete(cm.Labels, "labelKey2") delete(cm.Labels, "labelKey3") - Expect(memberClient.Update(context.Background(), appliedCM)).Should(Succeed()) + Expect(memberClient1.Update(context.Background(), appliedCM)).Should(Succeed()) By("Get the last applied config map and verify it's updated") var modifiedCM corev1.ConfigMap - Expect(memberClient.Get(context.Background(), types.NamespacedName{Name: cm.GetName(), Namespace: cm.GetNamespace()}, &modifiedCM)).Should(Succeed()) + Expect(memberClient1.Get(context.Background(), types.NamespacedName{Name: cm.GetName(), Namespace: cm.GetNamespace()}, &modifiedCM)).Should(Succeed()) Expect(cmp.Diff(appliedCM.Labels, modifiedCM.Labels)).Should(BeEmpty()) Expect(cmp.Diff(appliedCM.Data, modifiedCM.Data)).Should(BeEmpty()) @@ -250,7 +250,7 @@ var _ = Describe("Work Controller", func() { waitForWorkToApply(work.GetName()) By("Get the last applied config map") - Expect(memberClient.Get(context.Background(), types.NamespacedName{Name: cmName, Namespace: cmNamespace}, appliedCM)).Should(Succeed()) + Expect(memberClient1.Get(context.Background(), types.NamespacedName{Name: cmName, Namespace: cmNamespace}, appliedCM)).Should(Succeed()) By("Check the config map data") // data1's value picks up our change @@ -317,7 +317,7 @@ var _ = Describe("Work Controller", func() { By("Check applied TestResource") var appliedTestResource testv1alpha1.TestResource - Expect(memberClient.Get(context.Background(), types.NamespacedName{Name: testResourceName, Namespace: testResourceNamespace}, &appliedTestResource)).Should(Succeed()) + Expect(memberClient1.Get(context.Background(), types.NamespacedName{Name: testResourceName, Namespace: testResourceNamespace}, &appliedTestResource)).Should(Succeed()) By("verify the TestResource spec") Expect(cmp.Diff(appliedTestResource.Spec, testResource.Spec)).Should(BeEmpty()) @@ -338,11 +338,11 @@ var _ = Describe("Work Controller", func() { appliedTestResource.Spec.Items = []string{"a", "b"} appliedTestResource.Spec.Foo = "foo1" appliedTestResource.Spec.Bar = "bar1" - Expect(memberClient.Update(context.Background(), &appliedTestResource)).Should(Succeed()) + Expect(memberClient1.Update(context.Background(), &appliedTestResource)).Should(Succeed()) By("Verify applied TestResource modified") var modifiedTestResource testv1alpha1.TestResource - Expect(memberClient.Get(context.Background(), types.NamespacedName{Name: testResourceName, Namespace: testResourceNamespace}, &modifiedTestResource)).Should(Succeed()) + Expect(memberClient1.Get(context.Background(), types.NamespacedName{Name: testResourceName, Namespace: testResourceNamespace}, &modifiedTestResource)).Should(Succeed()) Expect(cmp.Diff(appliedTestResource.Spec, modifiedTestResource.Spec)).Should(BeEmpty()) By("Modify the TestResource") @@ -364,7 +364,7 @@ var _ = Describe("Work Controller", func() { waitForWorkToApply(work.GetName()) By("Get the last applied TestResource") - Expect(memberClient.Get(context.Background(), types.NamespacedName{Name: testResourceName, Namespace: testResourceNamespace}, &appliedTestResource)).Should(Succeed()) + Expect(memberClient1.Get(context.Background(), types.NamespacedName{Name: testResourceName, Namespace: testResourceNamespace}, &appliedTestResource)).Should(Succeed()) By("Check the TestResource spec, its an override for arrays") expectedItems := []string{"a", "b"} @@ -412,11 +412,11 @@ var _ = Describe("Work Controller", func() { By("Delete the last applied annotation from the current resource") delete(appliedCM.Annotations, fleetv1beta1.LastAppliedConfigAnnotation) - Expect(memberClient.Update(ctx, appliedCM)).Should(Succeed()) + Expect(memberClient1.Update(ctx, appliedCM)).Should(Succeed()) By("Get the last applied config map and verify it does not have the last applied annotation") var modifiedCM corev1.ConfigMap - Expect(memberClient.Get(ctx, types.NamespacedName{Name: cm.GetName(), Namespace: cm.GetNamespace()}, &modifiedCM)).Should(Succeed()) + Expect(memberClient1.Get(ctx, types.NamespacedName{Name: cm.GetName(), Namespace: cm.GetNamespace()}, &modifiedCM)).Should(Succeed()) Expect(modifiedCM.Annotations[fleetv1beta1.LastAppliedConfigAnnotation]).Should(BeEmpty()) By("Modify the manifest") @@ -438,7 +438,7 @@ var _ = Describe("Work Controller", func() { waitForWorkToApply(work.GetName()) By("Check applied configMap is modified even without the last applied annotation") - Expect(memberClient.Get(ctx, types.NamespacedName{Name: cmName, Namespace: cmNamespace}, appliedCM)).Should(Succeed()) + Expect(memberClient1.Get(ctx, types.NamespacedName{Name: cmName, Namespace: cmNamespace}, appliedCM)).Should(Succeed()) verifyAppliedConfigMap(cm) Expect(hubClient.Delete(ctx, work)).Should(Succeed(), "Failed to deleted the work") @@ -533,7 +533,7 @@ var _ = Describe("Work Controller", func() { Data: data, } // make sure we can call join as many as possible - Expect(workApplier.Join(ctx)).Should(Succeed()) + Expect(workApplier1.Join(ctx)).Should(Succeed()) work = createWorkWithManifest(cm) err := hubClient.Create(ctx, work) Expect(err).ToNot(HaveOccurred()) @@ -548,7 +548,7 @@ var _ = Describe("Work Controller", func() { By("mark the work controller as leave") Eventually(func() error { - return workApplier.Leave(ctx) + return workApplier1.Leave(ctx) }, eventuallyDuration, eventuallyInterval).Should(Succeed()) By("make sure the manifests have no finalizer and its status match the member cluster") @@ -561,12 +561,12 @@ var _ = Describe("Work Controller", func() { } for i := 0; i < numWork; i++ { var resultWork fleetv1beta1.Work - Expect(hubClient.Get(ctx, types.NamespacedName{Name: works[i].GetName(), Namespace: memberReservedNSName}, &resultWork)).Should(Succeed()) + Expect(hubClient.Get(ctx, types.NamespacedName{Name: works[i].GetName(), Namespace: memberReservedNSName1}, &resultWork)).Should(Succeed()) Expect(controllerutil.ContainsFinalizer(&resultWork, fleetv1beta1.WorkFinalizer)).Should(BeFalse()) // make sure that leave can be called as many times as possible // The work may be updated and may hit 409 error. Eventually(func() error { - return workApplier.Leave(ctx) + return workApplier1.Leave(ctx) }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to set the work controller to leave") By(fmt.Sprintf("change the work = %s", work.GetName())) cm = &corev1.ConfigMap{ @@ -591,7 +591,7 @@ var _ = Describe("Work Controller", func() { for i := 0; i < numWork; i++ { By(fmt.Sprintf("updated the work = %s", works[i].GetName())) var resultWork fleetv1beta1.Work - err := hubClient.Get(context.Background(), types.NamespacedName{Name: works[i].GetName(), Namespace: memberReservedNSName}, &resultWork) + err := hubClient.Get(context.Background(), types.NamespacedName{Name: works[i].GetName(), Namespace: memberReservedNSName1}, &resultWork) Expect(err).Should(Succeed()) Expect(controllerutil.ContainsFinalizer(&resultWork, fleetv1beta1.WorkFinalizer)).Should(BeFalse()) applyCond := meta.FindStatusCondition(resultWork.Status.Conditions, fleetv1beta1.WorkConditionTypeApplied) @@ -599,14 +599,14 @@ var _ = Describe("Work Controller", func() { return false } By("check if the config map is not changed") - Expect(memberClient.Get(ctx, types.NamespacedName{Name: cmNames[i], Namespace: cmNamespace}, &configMap)).Should(Succeed()) + Expect(memberClient1.Get(ctx, types.NamespacedName{Name: cmNames[i], Namespace: cmNamespace}, &configMap)).Should(Succeed()) Expect(cmp.Diff(configMap.Data, data)).Should(BeEmpty()) } return true }, eventuallyDuration, eventuallyInterval).Should(BeTrue()) By("enable the work controller again") - Expect(workApplier.Join(ctx)).Should(Succeed()) + Expect(workApplier1.Join(ctx)).Should(Succeed()) By("make sure the work change get picked up") for i := 0; i < numWork; i++ { @@ -614,7 +614,7 @@ var _ = Describe("Work Controller", func() { Expect(len(resultWork.Status.ManifestConditions)).Should(Equal(1)) Expect(meta.IsStatusConditionTrue(resultWork.Status.ManifestConditions[0].Conditions, fleetv1beta1.WorkConditionTypeApplied)).Should(BeTrue()) By("the work is applied, check if the applied config map is updated") - Expect(memberClient.Get(ctx, types.NamespacedName{Name: cmNames[i], Namespace: cmNamespace}, &configMap)).Should(Succeed()) + Expect(memberClient1.Get(ctx, types.NamespacedName{Name: cmNames[i], Namespace: cmNamespace}, &configMap)).Should(Succeed()) Expect(cmp.Diff(configMap.Data, newData)).Should(BeEmpty()) } }) @@ -634,7 +634,7 @@ var _ = Describe("Work Status Reconciler", func() { Name: resourceNamespace, }, } - Expect(memberClient.Create(context.Background(), &rns)).Should(Succeed(), "Failed to create the resource namespace") + Expect(memberClient1.Create(context.Background(), &rns)).Should(Succeed(), "Failed to create the resource namespace") // Create the Work object with some type of Manifest resource. cm = &corev1.ConfigMap{ @@ -668,7 +668,7 @@ var _ = Describe("Work Status Reconciler", func() { work = &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ Name: "work-" + utilrand.String(5), - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, }, Spec: fleetv1beta1.WorkSpec{ Workload: fleetv1beta1.WorkloadTemplate{ @@ -688,7 +688,7 @@ var _ = Describe("Work Status Reconciler", func() { AfterEach(func() { // TODO: Ensure that all resources are being deleted. Expect(hubClient.Delete(context.Background(), work)).Should(Succeed()) - Expect(memberClient.Delete(context.Background(), &rns)).Should(Succeed()) + Expect(memberClient1.Delete(context.Background(), &rns)).Should(Succeed()) }) It("Should delete the manifest from the member cluster after it is removed from work", func() { @@ -698,7 +698,7 @@ var _ = Describe("Work Status Reconciler", func() { By("Make sure that the work is applied") currentWork := waitForWorkToApply(work.Name) var appliedWork fleetv1beta1.AppliedWork - Expect(memberClient.Get(context.Background(), types.NamespacedName{Name: work.Name}, &appliedWork)).Should(Succeed()) + Expect(memberClient1.Get(context.Background(), types.NamespacedName{Name: work.Name}, &appliedWork)).Should(Succeed()) Expect(len(appliedWork.Status.AppliedResources)).Should(Equal(2)) By("Remove configMap 2 from the work") @@ -712,12 +712,12 @@ var _ = Describe("Work Status Reconciler", func() { By("Verify that the resource is removed from the cluster") Eventually(func() bool { var configMap corev1.ConfigMap - return apierrors.IsNotFound(memberClient.Get(context.Background(), types.NamespacedName{Name: cm2.Name, Namespace: resourceNamespace}, &configMap)) + return apierrors.IsNotFound(memberClient1.Get(context.Background(), types.NamespacedName{Name: cm2.Name, Namespace: resourceNamespace}, &configMap)) }, eventuallyDuration, eventuallyInterval).Should(BeTrue()) By("Verify that the appliedWork status is correct") Eventually(func() bool { - Expect(memberClient.Get(context.Background(), types.NamespacedName{Name: work.Name}, &appliedWork)).Should(Succeed()) + Expect(memberClient1.Get(context.Background(), types.NamespacedName{Name: work.Name}, &appliedWork)).Should(Succeed()) return len(appliedWork.Status.AppliedResources) == 1 }, eventuallyDuration, eventuallyInterval).Should(BeTrue()) Expect(appliedWork.Status.AppliedResources[0].Name).Should(Equal(cm.GetName())) @@ -734,7 +734,7 @@ var _ = Describe("Work Status Reconciler", func() { By("Make sure that the work is applied") currentWork := waitForWorkToApply(work.Name) var appliedWork fleetv1beta1.AppliedWork - Expect(memberClient.Get(context.Background(), types.NamespacedName{Name: work.Name}, &appliedWork)).Should(Succeed()) + Expect(memberClient1.Get(context.Background(), types.NamespacedName{Name: work.Name}, &appliedWork)).Should(Succeed()) Expect(len(appliedWork.Status.AppliedResources)).Should(Equal(2)) By("replace configMap with a bad object from the work") @@ -759,17 +759,17 @@ var _ = Describe("Work Status Reconciler", func() { By("Verify that the configMaps are removed from the cluster even if the new resource didn't apply") Eventually(func() bool { var configMap corev1.ConfigMap - return apierrors.IsNotFound(memberClient.Get(context.Background(), types.NamespacedName{Name: cm.Name, Namespace: resourceNamespace}, &configMap)) + return apierrors.IsNotFound(memberClient1.Get(context.Background(), types.NamespacedName{Name: cm.Name, Namespace: resourceNamespace}, &configMap)) }, eventuallyDuration, eventuallyInterval).Should(BeTrue()) Eventually(func() bool { var configMap corev1.ConfigMap - return apierrors.IsNotFound(memberClient.Get(context.Background(), types.NamespacedName{Name: cm2.Name, Namespace: resourceNamespace}, &configMap)) + return apierrors.IsNotFound(memberClient1.Get(context.Background(), types.NamespacedName{Name: cm2.Name, Namespace: resourceNamespace}, &configMap)) }, eventuallyDuration, eventuallyInterval).Should(BeTrue()) By("Verify that the appliedWork status is correct") Eventually(func() bool { - Expect(memberClient.Get(context.Background(), types.NamespacedName{Name: work.Name}, &appliedWork)).Should(Succeed()) + Expect(memberClient1.Get(context.Background(), types.NamespacedName{Name: work.Name}, &appliedWork)).Should(Succeed()) return len(appliedWork.Status.AppliedResources) == 0 }, eventuallyDuration, eventuallyInterval).Should(BeTrue()) }) @@ -781,14 +781,14 @@ var _ = Describe("Work Status Reconciler", func() { By("Make sure that the work is applied") currentWork := waitForWorkToApply(work.Name) var appliedWork fleetv1beta1.AppliedWork - Expect(memberClient.Get(context.Background(), types.NamespacedName{Name: work.Name}, &appliedWork)).Should(Succeed()) + Expect(memberClient1.Get(context.Background(), types.NamespacedName{Name: work.Name}, &appliedWork)).Should(Succeed()) Expect(len(appliedWork.Status.AppliedResources)).Should(Equal(2)) By("Make sure that the manifests exist on the member cluster") Eventually(func() bool { var configMap corev1.ConfigMap - return memberClient.Get(context.Background(), types.NamespacedName{Name: cm2.Name, Namespace: resourceNamespace}, &configMap) == nil && - memberClient.Get(context.Background(), types.NamespacedName{Name: cm.Name, Namespace: resourceNamespace}, &configMap) == nil + return memberClient1.Get(context.Background(), types.NamespacedName{Name: cm2.Name, Namespace: resourceNamespace}, &configMap) == nil && + memberClient1.Get(context.Background(), types.NamespacedName{Name: cm.Name, Namespace: resourceNamespace}, &configMap) == nil }, eventuallyDuration, eventuallyInterval).Should(BeTrue()) By("Change the order of the two configs in the work") @@ -805,13 +805,13 @@ var _ = Describe("Work Status Reconciler", func() { By("Verify that nothing is removed from the cluster") Consistently(func() bool { var configMap corev1.ConfigMap - return memberClient.Get(context.Background(), types.NamespacedName{Name: cm2.Name, Namespace: resourceNamespace}, &configMap) == nil && - memberClient.Get(context.Background(), types.NamespacedName{Name: cm.Name, Namespace: resourceNamespace}, &configMap) == nil + return memberClient1.Get(context.Background(), types.NamespacedName{Name: cm2.Name, Namespace: resourceNamespace}, &configMap) == nil && + memberClient1.Get(context.Background(), types.NamespacedName{Name: cm.Name, Namespace: resourceNamespace}, &configMap) == nil }, consistentlyDuration, consistentlyInterval).Should(BeTrue()) By("Verify that the appliedWork status is correct") Eventually(func() bool { - Expect(memberClient.Get(context.Background(), types.NamespacedName{Name: work.Name}, &appliedWork)).Should(Succeed()) + Expect(memberClient1.Get(context.Background(), types.NamespacedName{Name: work.Name}, &appliedWork)).Should(Succeed()) return len(appliedWork.Status.AppliedResources) == 2 }, eventuallyDuration, eventuallyInterval).Should(BeTrue()) Expect(appliedWork.Status.AppliedResources[0].Name).Should(Equal(cm2.GetName())) diff --git a/pkg/controllers/workapplier/controller_integration_test.go b/pkg/controllers/workapplier/controller_integration_test.go index a6073d4ef..e76537c39 100644 --- a/pkg/controllers/workapplier/controller_integration_test.go +++ b/pkg/controllers/workapplier/controller_integration_test.go @@ -75,7 +75,7 @@ var ( ) // createWorkObject creates a new Work object with the given name, manifests, and apply strategy. -func createWorkObject(workName string, applyStrategy *fleetv1beta1.ApplyStrategy, rawManifestJSON ...[]byte) { +func createWorkObject(workName, memberClusterReservedNSName string, applyStrategy *fleetv1beta1.ApplyStrategy, rawManifestJSON ...[]byte) { manifests := make([]fleetv1beta1.Manifest, len(rawManifestJSON)) for idx := range rawManifestJSON { manifests[idx] = fleetv1beta1.Manifest{ @@ -88,7 +88,7 @@ func createWorkObject(workName string, applyStrategy *fleetv1beta1.ApplyStrategy work := &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ Name: workName, - Namespace: memberReservedNSName, + Namespace: memberClusterReservedNSName, }, Spec: fleetv1beta1.WorkSpec{ Workload: fleetv1beta1.WorkloadTemplate{ @@ -111,7 +111,7 @@ func updateWorkObject(workName string, applyStrategy *fleetv1beta1.ApplyStrategy } work := &fleetv1beta1.Work{} - Expect(hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName}, work)).To(Succeed()) + Expect(hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName1}, work)).To(Succeed()) work.Spec.Workload.Manifests = manifests work.Spec.ApplyStrategy = applyStrategy @@ -131,7 +131,7 @@ func workFinalizerAddedActual(workName string) func() error { return func() error { // Retrieve the Work object. work := &fleetv1beta1.Work{} - if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName}, work); err != nil { + if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName1}, work); err != nil { return fmt.Errorf("failed to retrieve the Work object: %w", err) } @@ -147,7 +147,7 @@ func appliedWorkCreatedActual(workName string) func() error { return func() error { // Retrieve the AppliedWork object. appliedWork := &fleetv1beta1.AppliedWork{} - if err := memberClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName}, appliedWork); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName1}, appliedWork); err != nil { return fmt.Errorf("failed to retrieve the AppliedWork object: %w", err) } @@ -157,7 +157,7 @@ func appliedWorkCreatedActual(workName string) func() error { }, Spec: fleetv1beta1.AppliedWorkSpec{ WorkName: workName, - WorkNamespace: memberReservedNSName, + WorkNamespace: memberReservedNSName1, }, } if diff := cmp.Diff( @@ -174,7 +174,7 @@ func appliedWorkCreatedActual(workName string) func() error { func prepareAppliedWorkOwnerRef(workName string) *metav1.OwnerReference { // Retrieve the AppliedWork object. appliedWork := &fleetv1beta1.AppliedWork{} - Expect(memberClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName}, appliedWork)).To(Succeed(), "Failed to retrieve the AppliedWork object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName1}, appliedWork)).To(Succeed(), "Failed to retrieve the AppliedWork object") // Prepare the expected OwnerReference. return &metav1.OwnerReference{ @@ -190,7 +190,7 @@ func regularNSObjectAppliedActual(nsName string, appliedWorkOwnerRef *metav1.Own return func() error { // Retrieve the NS object. gotNS := &corev1.Namespace{} - if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, gotNS); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, gotNS); err != nil { return fmt.Errorf("failed to retrieve the NS object: %w", err) } @@ -222,7 +222,7 @@ func regularDeploymentObjectAppliedActual(nsName, deployName string, appliedWork return func() error { // Retrieve the Deployment object. gotDeploy := &appsv1.Deployment{} - if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, gotDeploy); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, gotDeploy); err != nil { return fmt.Errorf("failed to retrieve the Deployment object: %w", err) } @@ -285,7 +285,7 @@ func regularClusterRoleObjectAppliedActual(clusterRoleName string, appliedWorkOw return func() error { // Retrieve the ClusterRole object. gotClusterRole := &rbacv1.ClusterRole{} - if err := memberClient.Get(ctx, client.ObjectKey{Name: clusterRoleName}, gotClusterRole); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: clusterRoleName}, gotClusterRole); err != nil { return fmt.Errorf("failed to retrieve the ClusterRole object: %w", err) } @@ -318,7 +318,7 @@ func regularConfigMapObjectAppliedActual(nsName, configMapName string, appliedWo return func() error { // Retrieve the ConfigMap object. gotConfigMap := &corev1.ConfigMap{} - if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: configMapName}, gotConfigMap); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: configMapName}, gotConfigMap); err != nil { return fmt.Errorf("failed to retrieve the ConfigMap object: %w", err) } @@ -351,7 +351,7 @@ func regularConfigMapObjectAppliedActual(nsName, configMapName string, appliedWo func markDeploymentAsAvailable(nsName, deployName string) { // Retrieve the Deployment object. gotDeploy := &appsv1.Deployment{} - Expect(memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, gotDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, gotDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") // Mark the Deployment object as available. now := metav1.Now() @@ -377,7 +377,7 @@ func markDeploymentAsAvailable(nsName, deployName string) { }, }, } - Expect(memberClient.Status().Update(ctx, gotDeploy)).To(Succeed(), "Failed to mark the Deployment object as available") + Expect(memberClient1.Status().Update(ctx, gotDeploy)).To(Succeed(), "Failed to mark the Deployment object as available") } func workStatusUpdated( @@ -390,7 +390,7 @@ func workStatusUpdated( return func() error { // Retrieve the Work object. work := &fleetv1beta1.Work{} - if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName}, work); err != nil { + if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName1}, work); err != nil { return fmt.Errorf("failed to retrieve the Work object: %w", err) } @@ -459,7 +459,7 @@ func appliedWorkStatusUpdated(workName string, appliedResourceMeta []fleetv1beta return func() error { // Retrieve the AppliedWork object. appliedWork := &fleetv1beta1.AppliedWork{} - if err := memberClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName}, appliedWork); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName1}, appliedWork); err != nil { return fmt.Errorf("failed to retrieve the AppliedWork object: %w", err) } @@ -478,7 +478,7 @@ func workRemovedActual(workName string) func() error { // Wait for the removal of the Work object. return func() error { work := &fleetv1beta1.Work{} - if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName}, work); !errors.IsNotFound(err) && err != nil { + if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName1}, work); !errors.IsNotFound(err) && err != nil { return fmt.Errorf("work object still exists or an unexpected error occurred: %w", err) } if controllerutil.ContainsFinalizer(work, fleetv1beta1.WorkFinalizer) { @@ -489,12 +489,12 @@ func workRemovedActual(workName string) func() error { } } -func deleteWorkObject(workName string) { +func deleteWorkObject(workName, memberClusterReservedNSName string) { // Retrieve the Work object. work := &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ Name: workName, - Namespace: memberReservedNSName, + Namespace: memberClusterReservedNSName, }, } Expect(hubClient.Delete(ctx, work)).To(Succeed(), "Failed to delete the Work object") @@ -503,7 +503,7 @@ func deleteWorkObject(workName string) { func checkNSOwnerReferences(workName, nsName string) { // Retrieve the AppliedWork object. appliedWork := &fleetv1beta1.AppliedWork{} - Expect(memberClient.Get(ctx, client.ObjectKey{Name: workName}, appliedWork)).To(Succeed(), "Failed to retrieve the AppliedWork object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: workName}, appliedWork)).To(Succeed(), "Failed to retrieve the AppliedWork object") // Check that the Namespace object has the AppliedWork as an owner reference. ns := &corev1.Namespace{ @@ -511,7 +511,7 @@ func checkNSOwnerReferences(workName, nsName string) { Name: nsName, }, } - Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, ns)).To(Succeed(), "Failed to retrieve the Namespace object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, ns)).To(Succeed(), "Failed to retrieve the Namespace object") Expect(ns.OwnerReferences).To(ContainElement(metav1.OwnerReference{ APIVersion: fleetv1beta1.GroupVersion.String(), Kind: "AppliedWork", @@ -525,7 +525,7 @@ func appliedWorkRemovedActual(workName, nsName string) func() error { return func() error { // Retrieve the AppliedWork object. appliedWork := &fleetv1beta1.AppliedWork{} - if err := memberClient.Get(ctx, client.ObjectKey{Name: workName}, appliedWork); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: workName}, appliedWork); err != nil { if errors.IsNotFound(err) { // The AppliedWork object has been deleted, which is expected. return nil @@ -536,7 +536,7 @@ func appliedWorkRemovedActual(workName, nsName string) func() error { // The AppliedWork object is being deleted, but the finalizer is still present. Remove the finalizer as there // are no real built-in controllers in this test environment to handle garbage collection. controllerutil.RemoveFinalizer(appliedWork, metav1.FinalizerDeleteDependents) - Expect(memberClient.Update(ctx, appliedWork)).To(Succeed(), "Failed to remove the finalizer from the AppliedWork object") + Expect(memberClient1.Update(ctx, appliedWork)).To(Succeed(), "Failed to remove the finalizer from the AppliedWork object") } return fmt.Errorf("appliedWork object still exists") } @@ -551,11 +551,11 @@ func regularDeployRemovedActual(nsName, deployName string) func() error { Name: deployName, }, } - if err := memberClient.Delete(ctx, deploy); err != nil && !errors.IsNotFound(err) { + if err := memberClient1.Delete(ctx, deploy); err != nil && !errors.IsNotFound(err) { return fmt.Errorf("failed to delete the Deployment object: %w", err) } - if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, deploy); !errors.IsNotFound(err) { + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, deploy); !errors.IsNotFound(err) { return fmt.Errorf("deployment object still exists or an unexpected error occurred: %w", err) } return nil @@ -570,11 +570,11 @@ func regularClusterRoleRemovedActual(clusterRoleName string) func() error { Name: clusterRoleName, }, } - if err := memberClient.Delete(ctx, clusterRole); err != nil && !errors.IsNotFound(err) { + if err := memberClient1.Delete(ctx, clusterRole); err != nil && !errors.IsNotFound(err) { return fmt.Errorf("failed to delete the ClusterRole object: %w", err) } - if err := memberClient.Get(ctx, client.ObjectKey{Name: clusterRoleName}, clusterRole); !errors.IsNotFound(err) { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: clusterRoleName}, clusterRole); !errors.IsNotFound(err) { return fmt.Errorf("clusterRole object still exists or an unexpected error occurred: %w", err) } return nil @@ -590,12 +590,12 @@ func regularConfigMapRemovedActual(nsName, configMapName string) func() error { Name: configMapName, }, } - if err := memberClient.Delete(ctx, configMap); err != nil && !errors.IsNotFound(err) { + if err := memberClient1.Delete(ctx, configMap); err != nil && !errors.IsNotFound(err) { return fmt.Errorf("failed to delete the ConfigMap object: %w", err) } // Check that the ConfigMap object has been deleted. - if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: configMapName}, configMap); !errors.IsNotFound(err) { + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: configMapName}, configMap); !errors.IsNotFound(err) { return fmt.Errorf("configMap object still exists or an unexpected error occurred: %w", err) } return nil @@ -606,7 +606,7 @@ func regularNSObjectNotAppliedActual(nsName string) func() error { return func() error { // Retrieve the NS object. ns := &corev1.Namespace{} - if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, ns); !errors.IsNotFound(err) { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, ns); !errors.IsNotFound(err) { return fmt.Errorf("namespace object exists or an unexpected error occurred: %w", err) } return nil @@ -622,7 +622,7 @@ func regularDeployNotRemovedActual(nsName, deployName string) func() error { Name: deployName, }, } - if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, deploy); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, deploy); err != nil { return fmt.Errorf("failed to retrieve the Deployment object: %w", err) } return nil @@ -653,7 +653,7 @@ var _ = Describe("applying manifests", func() { regularDeployJSON := marshalK8sObjJSON(regularDeploy) // Create a new Work object with all the manifest JSONs. - createWorkObject(workName, nil, regularNSJSON, regularDeployJSON) + createWorkObject(workName, memberReservedNSName1, nil, regularNSJSON, regularDeployJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -673,13 +673,13 @@ var _ = Describe("applying manifests", func() { regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the namespace object") - Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") // Ensure that the Deployment object has been applied as expected. regularDeploymentObjectAppliedActual := regularDeploymentObjectAppliedActual(nsName, deployName, appliedWorkOwnerRef) Eventually(regularDeploymentObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the deployment object") - Expect(memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") }) It("can mark the deployment as available", func() { @@ -790,7 +790,7 @@ var _ = Describe("applying manifests", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Ensure applied manifest has been removed. regularDeployRemovedActual := regularDeployRemovedActual(nsName, deployName) @@ -808,7 +808,7 @@ var _ = Describe("applying manifests", func() { Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") // The environment prepared by the envtest package does not support namespace - // deletion; consequently this test suite would not attempt so verify its deletion. + // deletion; consequently this test suite would not attempt to verify its deletion. }) }) @@ -835,7 +835,7 @@ var _ = Describe("applying manifests", func() { regularDeployJSON := marshalK8sObjJSON(regularDeploy) // Create a new Work object with all the manifest JSONs. - createWorkObject(workName, nil, regularNSJSON, regularDeployJSON) + createWorkObject(workName, memberReservedNSName1, nil, regularNSJSON, regularDeployJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -855,13 +855,13 @@ var _ = Describe("applying manifests", func() { regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the namespace object") - Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") // Ensure that the Deployment object has been applied as expected. regularDeploymentObjectAppliedActual := regularDeploymentObjectAppliedActual(nsName, deployName, appliedWorkOwnerRef) Eventually(regularDeploymentObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the deployment object") - Expect(memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") }) It("can mark the deployment as available", func() { @@ -1053,7 +1053,7 @@ var _ = Describe("applying manifests", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Kubebuilder suggests that in a testing environment like this, to check for the existence of the AppliedWork object // OwnerReference in the Namespace object (https://book.kubebuilder.io/reference/envtest.html#testing-considerations). @@ -1066,7 +1066,7 @@ var _ = Describe("applying manifests", func() { workRemovedActual := workRemovedActual(workName) Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") // The environment prepared by the envtest package does not support namespace - // deletion; consequently this test suite would not attempt so verify its deletion. + // deletion; consequently this test suite would not attempt to verify its deletion. }) }) @@ -1101,7 +1101,7 @@ var _ = Describe("applying manifests", func() { regularDeployJSON := marshalK8sObjJSON(regularDeploy) // Create a new Work object with all the manifest JSONs. - createWorkObject(workName, nil, regularNSJSON, regularDeployJSON) + createWorkObject(workName, memberReservedNSName1, nil, regularNSJSON, regularDeployJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -1121,7 +1121,7 @@ var _ = Describe("applying manifests", func() { Eventually(func() error { // Retrieve the NS object. gotNS := &corev1.Namespace{} - if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, gotNS); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, gotNS); err != nil { return fmt.Errorf("failed to retrieve the NS object: %w", err) } @@ -1150,14 +1150,14 @@ var _ = Describe("applying manifests", func() { return nil }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the namespace object") - Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") }) It("should not apply the Deployment object", func() { Consistently(func() error { // List all Deployments. gotDeployList := &appsv1.DeploymentList{} - if err := memberClient.List(ctx, gotDeployList, client.InNamespace(nsName)); err != nil { + if err := memberClient1.List(ctx, gotDeployList, client.InNamespace(nsName)); err != nil { return fmt.Errorf("failed to list Deployment objects: %w", err) } @@ -1250,7 +1250,7 @@ var _ = Describe("applying manifests", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Kubebuilder suggests that in a testing environment like this, to check for the existence of the AppliedWork object // OwnerReference in the Namespace object (https://book.kubebuilder.io/reference/envtest.html#testing-considerations). @@ -1264,7 +1264,7 @@ var _ = Describe("applying manifests", func() { Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") // The environment prepared by the envtest package does not support namespace - // deletion; consequently this test suite would not attempt so verify its deletion. + // deletion; consequently this test suite would not attempt to verify its deletion. }) }) @@ -1301,7 +1301,7 @@ var _ = Describe("applying manifests", func() { regularConfigMapJSON := marshalK8sObjJSON(regularConfigMap) // Create a new Work object with all the manifest JSONs. - createWorkObject(workName, nil, regularNSJSON, decodingErredDeployJSON, regularConfigMapJSON) + createWorkObject(workName, memberReservedNSName1, nil, regularNSJSON, decodingErredDeployJSON, regularConfigMapJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -1321,12 +1321,12 @@ var _ = Describe("applying manifests", func() { regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the namespace object") - Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") // Ensure that the ConfigMap object has been applied as expected. regularConfigMapObjectAppliedActual := regularConfigMapObjectAppliedActual(nsName, configMapName, appliedWorkOwnerRef) Eventually(regularConfigMapObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the ConfigMap object") - Expect(memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: configMapName}, regularConfigMap)).To(Succeed(), "Failed to retrieve the ConfigMap object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: configMapName}, regularConfigMap)).To(Succeed(), "Failed to retrieve the ConfigMap object") }) It("should update the Work object status", func() { @@ -1442,7 +1442,7 @@ var _ = Describe("applying manifests", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Ensure applied manifest has been removed. regularConfigMapRemovedActual := regularConfigMapRemovedActual(nsName, configMapName) @@ -1460,7 +1460,7 @@ var _ = Describe("applying manifests", func() { Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") // The environment prepared by the envtest package does not support namespace - // deletion; consequently this test suite would not attempt so verify its deletion. + // deletion; consequently this test suite would not attempt to verify its deletion. }) }) @@ -1489,7 +1489,7 @@ var _ = Describe("applying manifests", func() { malformedConfigMapJSON := marshalK8sObjJSON(malformedConfigMap) // Create a new Work object with all the manifest JSONs and proper apply strategy. - createWorkObject(workName, nil, regularNSJSON, malformedConfigMapJSON) + createWorkObject(workName, memberReservedNSName1, nil, regularNSJSON, malformedConfigMapJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -1508,7 +1508,7 @@ var _ = Describe("applying manifests", func() { Consistently(func() error { configMap := &corev1.ConfigMap{} objKey := client.ObjectKey{Namespace: nsName, Name: malformedConfigMap.Name} - if err := memberClient.Get(ctx, objKey, configMap); !errors.IsNotFound(err) { + if err := memberClient1.Get(ctx, objKey, configMap); !errors.IsNotFound(err) { return fmt.Errorf("the config map exists, or an unexpected error has occurred: %w", err) } return nil @@ -1520,7 +1520,7 @@ var _ = Describe("applying manifests", func() { regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the namespace object") - Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") }) It("should update the Work object status", func() { @@ -1605,7 +1605,7 @@ var _ = Describe("applying manifests", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Ensure that the AppliedWork object has been removed. appliedWorkRemovedActual := appliedWorkRemovedActual(workName, nsName) @@ -1650,7 +1650,7 @@ var _ = Describe("work applier garbage collection", func() { regularDeployJSON := marshalK8sObjJSON(regularDeploy) // Create a new Work object with all the manifest JSONs. - createWorkObject(workName, &fleetv1beta1.ApplyStrategy{AllowCoOwnership: true}, regularNSJSON, regularDeployJSON) + createWorkObject(workName, memberReservedNSName1, &fleetv1beta1.ApplyStrategy{AllowCoOwnership: true}, regularNSJSON, regularDeployJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -1670,13 +1670,13 @@ var _ = Describe("work applier garbage collection", func() { regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the namespace object") - Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") // Ensure that the Deployment object has been applied as expected. regularDeploymentObjectAppliedActual := regularDeploymentObjectAppliedActual(nsName, deployName, appliedWorkOwnerRef) Eventually(regularDeploymentObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the deployment object") - Expect(memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") }) It("can mark the deployment as available", func() { @@ -1788,16 +1788,16 @@ var _ = Describe("work applier garbage collection", func() { It("can update Deployment object to add another owner reference", func() { // Retrieve the Deployment object. gotDeploy := &appsv1.Deployment{} - Expect(memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, gotDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, gotDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") // Add another owner reference to the Deployment object. gotDeploy.OwnerReferences = append(gotDeploy.OwnerReferences, anotherOwnerReference) - Expect(memberClient.Update(ctx, gotDeploy)).To(Succeed(), "Failed to update the Deployment object with another owner reference") + Expect(memberClient1.Update(ctx, gotDeploy)).To(Succeed(), "Failed to update the Deployment object with another owner reference") // Ensure that the Deployment object has been updated as expected. Eventually(func() error { // Retrieve the Deployment object again. - if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, gotDeploy); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, gotDeploy); err != nil { return fmt.Errorf("failed to retrieve the Deployment object: %w", err) } @@ -1819,14 +1819,14 @@ var _ = Describe("work applier garbage collection", func() { It("should start deleting the Work object", func() { // Start deleting the Work object. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) }) It("should start deleting the AppliedWork object", func() { // Ensure that the Work object is being deleted. Eventually(func() error { appliedWork := &fleetv1beta1.AppliedWork{} - if err := memberClient.Get(ctx, client.ObjectKey{Name: workName}, appliedWork); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: workName}, appliedWork); err != nil { return err } if !appliedWork.DeletionTimestamp.IsZero() && controllerutil.ContainsFinalizer(appliedWork, metav1.FinalizerDeleteDependents) { @@ -1844,7 +1844,7 @@ var _ = Describe("work applier garbage collection", func() { Eventually(func() error { // Retrieve the Deployment object again. gotDeploy := &appsv1.Deployment{} - if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, gotDeploy); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, gotDeploy); err != nil { return fmt.Errorf("failed to retrieve the Deployment object: %w", err) } // Check that the Deployment object has been updated as expected. @@ -1876,13 +1876,13 @@ var _ = Describe("work applier garbage collection", func() { // Ensure that the Deployment object still exists. Consistently(func() error { - return memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy) + return memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy) }, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Deployment object has been removed unexpectedly") // Delete objects created by the test suite so that the next test case can run without issues. - Expect(memberClient.Delete(ctx, regularDeploy)).To(Succeed(), "Failed to delete the Deployment object") + Expect(memberClient1.Delete(ctx, regularDeploy)).To(Succeed(), "Failed to delete the Deployment object") // The environment prepared by the envtest package does not support namespace - // deletion; consequently this test suite would not attempt so verify its deletion. + // deletion; consequently this test suite would not attempt to verify its deletion. }) }) @@ -1915,7 +1915,7 @@ var _ = Describe("work applier garbage collection", func() { regularClusterRoleJSON := marshalK8sObjJSON(regularClusterRole) // Create a new Work object with all the manifest JSONs. - createWorkObject(workName, &fleetv1beta1.ApplyStrategy{AllowCoOwnership: true}, regularNSJSON, regularDeployJSON, regularClusterRoleJSON) + createWorkObject(workName, memberReservedNSName1, &fleetv1beta1.ApplyStrategy{AllowCoOwnership: true}, regularNSJSON, regularDeployJSON, regularClusterRoleJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -1935,19 +1935,19 @@ var _ = Describe("work applier garbage collection", func() { regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the namespace object") - Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") // Ensure that the Deployment object has been applied as expected. regularDeploymentObjectAppliedActual := regularDeploymentObjectAppliedActual(nsName, deployName, appliedWorkOwnerRef) Eventually(regularDeploymentObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the deployment object") - Expect(memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") // Ensure that the ClusterRole object has been applied as expected. regularClusterRoleObjectAppliedActual := regularClusterRoleObjectAppliedActual(clusterRoleName, appliedWorkOwnerRef) Eventually(regularClusterRoleObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the clusterRole object") - Expect(memberClient.Get(ctx, client.ObjectKey{Name: clusterRoleName}, regularClusterRole)).To(Succeed(), "Failed to retrieve the clusterRole object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: clusterRoleName}, regularClusterRole)).To(Succeed(), "Failed to retrieve the clusterRole object") }) It("can mark the deployment as available", func() { @@ -2094,11 +2094,11 @@ var _ = Describe("work applier garbage collection", func() { It("can update ClusterRole object to add another owner reference", func() { // Retrieve the ClusterRole object. gotClusterRole := &rbacv1.ClusterRole{} - Expect(memberClient.Get(ctx, client.ObjectKey{Name: clusterRoleName}, gotClusterRole)).To(Succeed(), "Failed to retrieve the ClusterRole object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: clusterRoleName}, gotClusterRole)).To(Succeed(), "Failed to retrieve the ClusterRole object") // Retrieve the Deployment object. gotDeploy := &appsv1.Deployment{} - Expect(memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, gotDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, gotDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") // Add another owner reference to the ClusterRole object. // Note: This is an invalid owner reference, as it adds a namespace-scoped object as an owner of a cluster-scoped object. @@ -2108,12 +2108,12 @@ var _ = Describe("work applier garbage collection", func() { Name: gotDeploy.Name, UID: gotDeploy.UID, }) - Expect(memberClient.Update(ctx, gotClusterRole)).To(Succeed(), "Failed to update the ClusterRole object with another owner reference") + Expect(memberClient1.Update(ctx, gotClusterRole)).To(Succeed(), "Failed to update the ClusterRole object with another owner reference") // Ensure that the ClusterRole object has been updated as expected. Eventually(func() error { // Retrieve the ClusterRole object again. - if err := memberClient.Get(ctx, client.ObjectKey{Name: clusterRoleName}, gotClusterRole); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: clusterRoleName}, gotClusterRole); err != nil { return fmt.Errorf("failed to retrieve the ClusterRole object: %w", err) } @@ -2135,14 +2135,14 @@ var _ = Describe("work applier garbage collection", func() { It("should start deleting the Work object", func() { // Start deleting the Work object. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) }) It("should start deleting the AppliedWork object", func() { // Ensure that the Work object is being deleted. Eventually(func() error { appliedWork := &fleetv1beta1.AppliedWork{} - if err := memberClient.Get(ctx, client.ObjectKey{Name: workName}, appliedWork); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: workName}, appliedWork); err != nil { return err } if !appliedWork.DeletionTimestamp.IsZero() && controllerutil.ContainsFinalizer(appliedWork, metav1.FinalizerDeleteDependents) { @@ -2160,7 +2160,7 @@ var _ = Describe("work applier garbage collection", func() { Eventually(func() error { // Retrieve the ClusterRole object again. gotClusterRole := &rbacv1.ClusterRole{} - if err := memberClient.Get(ctx, client.ObjectKey{Name: clusterRoleName}, gotClusterRole); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: clusterRoleName}, gotClusterRole); err != nil { return fmt.Errorf("failed to retrieve the ClusterRole object: %w", err) } @@ -2196,12 +2196,12 @@ var _ = Describe("work applier garbage collection", func() { // Ensure that the ClusterRole object still exists. Consistently(func() error { - return memberClient.Get(ctx, client.ObjectKey{Name: clusterRoleName}, regularClusterRole) + return memberClient1.Get(ctx, client.ObjectKey{Name: clusterRoleName}, regularClusterRole) }, consistentlyDuration, consistentlyInterval).Should(BeNil(), "ClusterRole object has been removed unexpectedly") // Delete objects created by the test suite so that the next test case can run without issues. - Expect(memberClient.Delete(ctx, regularClusterRole)).To(Succeed(), "Failed to delete the clusterRole object") + Expect(memberClient1.Delete(ctx, regularClusterRole)).To(Succeed(), "Failed to delete the clusterRole object") // The environment prepared by the envtest package does not support namespace - // deletion; consequently this test suite would not attempt so verify its deletion. + // deletion; consequently this test suite would not attempt to verify its deletion. }) }) @@ -2234,7 +2234,7 @@ var _ = Describe("work applier garbage collection", func() { regularClusterRoleJSON := marshalK8sObjJSON(regularClusterRole) // Create a new Work object with all the manifest JSONs. - createWorkObject(workName, &fleetv1beta1.ApplyStrategy{AllowCoOwnership: true}, regularNSJSON, regularDeployJSON, regularClusterRoleJSON) + createWorkObject(workName, memberReservedNSName1, &fleetv1beta1.ApplyStrategy{AllowCoOwnership: true}, regularNSJSON, regularDeployJSON, regularClusterRoleJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -2254,19 +2254,19 @@ var _ = Describe("work applier garbage collection", func() { regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the namespace object") - Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") // Ensure that the Deployment object has been applied as expected. regularDeploymentObjectAppliedActual := regularDeploymentObjectAppliedActual(nsName, deployName, appliedWorkOwnerRef) Eventually(regularDeploymentObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the deployment object") - Expect(memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") // Ensure that the ClusterRole object has been applied as expected. regularClusterRoleObjectAppliedActual := regularClusterRoleObjectAppliedActual(clusterRoleName, appliedWorkOwnerRef) Eventually(regularClusterRoleObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the clusterRole object") - Expect(memberClient.Get(ctx, client.ObjectKey{Name: clusterRoleName}, regularClusterRole)).To(Succeed(), "Failed to retrieve the clusterRole object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: clusterRoleName}, regularClusterRole)).To(Succeed(), "Failed to retrieve the clusterRole object") }) It("can mark the deployment as available", func() { @@ -2413,11 +2413,11 @@ var _ = Describe("work applier garbage collection", func() { It("can update Deployment object to add another owner reference", func() { // Retrieve the ClusterRole object. gotClusterRole := &rbacv1.ClusterRole{} - Expect(memberClient.Get(ctx, client.ObjectKey{Name: clusterRoleName}, gotClusterRole)).To(Succeed(), "Failed to retrieve the ClusterRole object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: clusterRoleName}, gotClusterRole)).To(Succeed(), "Failed to retrieve the ClusterRole object") // Retrieve the Deployment object. gotDeploy := &appsv1.Deployment{} - Expect(memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, gotDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, gotDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") // Add another owner reference to the Deployment object. gotDeploy.OwnerReferences = append(gotDeploy.OwnerReferences, metav1.OwnerReference{ @@ -2426,12 +2426,12 @@ var _ = Describe("work applier garbage collection", func() { Name: gotClusterRole.Name, UID: gotClusterRole.UID, }) - Expect(memberClient.Update(ctx, gotDeploy)).To(Succeed(), "Failed to update the Deployment object with another owner reference") + Expect(memberClient1.Update(ctx, gotDeploy)).To(Succeed(), "Failed to update the Deployment object with another owner reference") // Ensure that the Deployment object has been updated as expected. Eventually(func() error { // Retrieve the Deployment object again. - if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, gotDeploy); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, gotDeploy); err != nil { return fmt.Errorf("failed to retrieve the Deployment object: %w", err) } @@ -2453,14 +2453,14 @@ var _ = Describe("work applier garbage collection", func() { It("should start deleting the Work object", func() { // Start deleting the Work object. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) }) It("should start deleting the AppliedWork object", func() { // Ensure that the Work object is being deleted. Eventually(func() error { appliedWork := &fleetv1beta1.AppliedWork{} - if err := memberClient.Get(ctx, client.ObjectKey{Name: workName}, appliedWork); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: workName}, appliedWork); err != nil { return err } if !appliedWork.DeletionTimestamp.IsZero() && controllerutil.ContainsFinalizer(appliedWork, metav1.FinalizerDeleteDependents) { @@ -2478,7 +2478,7 @@ var _ = Describe("work applier garbage collection", func() { Eventually(func() error { // Retrieve the Deployment object. gotDeploy := &appsv1.Deployment{} - if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, gotDeploy); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, gotDeploy); err != nil { return fmt.Errorf("failed to retrieve the ClusterRole object: %w", err) } @@ -2513,12 +2513,12 @@ var _ = Describe("work applier garbage collection", func() { // Ensure that the Deployment object still exists. Consistently(func() error { - return memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy) + return memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy) }, consistentlyDuration, consistentlyInterval).Should(BeNil(), "Deployment object has been removed unexpectedly") // Delete objects created by the test suite so that the next test case can run without issues. - Expect(memberClient.Delete(ctx, regularDeploy)).To(Succeed(), "Failed to delete the Deployment object") + Expect(memberClient1.Delete(ctx, regularDeploy)).To(Succeed(), "Failed to delete the Deployment object") // The environment prepared by the envtest package does not support namespace - // deletion; consequently this test suite would not attempt so verify its deletion. + // deletion; consequently this test suite would not attempt to verify its deletion. }) }) }) @@ -2547,8 +2547,8 @@ var _ = Describe("drift detection and takeover", func() { regularDeployJSON := marshalK8sObjJSON(regularDeploy) // Create the resources on the member cluster side. - Expect(memberClient.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") - Expect(memberClient.Create(ctx, regularDeploy)).To(Succeed(), "Failed to create the Deployment object") + Expect(memberClient1.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") + Expect(memberClient1.Create(ctx, regularDeploy)).To(Succeed(), "Failed to create the Deployment object") markDeploymentAsAvailable(nsName, deployName) @@ -2557,7 +2557,7 @@ var _ = Describe("drift detection and takeover", func() { ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeIfNoDiff, } - createWorkObject(workName, applyStrategy, regularNSJSON, regularDeployJSON) + createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON, regularDeployJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -2577,13 +2577,13 @@ var _ = Describe("drift detection and takeover", func() { regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the namespace object") - Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") // Ensure that the Deployment object has been applied as expected. regularDeploymentObjectAppliedActual := regularDeploymentObjectAppliedActual(nsName, deployName, appliedWorkOwnerRef) Eventually(regularDeploymentObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the deployment object") - Expect(memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") }) It("can mark the deployment as available", func() { @@ -2692,7 +2692,7 @@ var _ = Describe("drift detection and takeover", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Ensure applied manifest has been removed. regularDeployRemovedActual := regularDeployRemovedActual(nsName, deployName) @@ -2710,7 +2710,7 @@ var _ = Describe("drift detection and takeover", func() { Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") // The environment prepared by the envtest package does not support namespace - // deletion; consequently this test suite would not attempt so verify its deletion. + // deletion; consequently this test suite would not attempt to verify its deletion. }) }) @@ -2747,8 +2747,8 @@ var _ = Describe("drift detection and takeover", func() { regularDeploy.Spec.Replicas = ptr.To(int32(2)) // Create the resources on the member cluster side. - Expect(memberClient.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") - Expect(memberClient.Create(ctx, regularDeploy)).To(Succeed(), "Failed to create the Deployment object") + Expect(memberClient1.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") + Expect(memberClient1.Create(ctx, regularDeploy)).To(Succeed(), "Failed to create the Deployment object") markDeploymentAsAvailable(nsName, deployName) @@ -2757,7 +2757,7 @@ var _ = Describe("drift detection and takeover", func() { ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeIfNoDiff, } - createWorkObject(workName, applyStrategy, regularNSJSON, regularDeployJSON) + createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON, regularDeployJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -2789,7 +2789,7 @@ var _ = Describe("drift detection and takeover", func() { Eventually(func() error { // Retrieve the NS object. - if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { return fmt.Errorf("failed to retrieve the NS object: %w", err) } @@ -2818,7 +2818,7 @@ var _ = Describe("drift detection and takeover", func() { wantDeploy.Spec.Replicas = ptr.To(int32(2)) Consistently(func() error { - if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy); err != nil { return fmt.Errorf("failed to retrieve the Deployment object: %w", err) } @@ -2962,7 +2962,7 @@ var _ = Describe("drift detection and takeover", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Ensure that the Deployment object has been left alone. regularDeployNotRemovedActual := regularDeployNotRemovedActual(nsName, deployName) @@ -2976,7 +2976,7 @@ var _ = Describe("drift detection and takeover", func() { Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") // The environment prepared by the envtest package does not support namespace - // deletion; consequently this test suite would not attempt so verify its deletion. + // deletion; consequently this test suite would not attempt to verify its deletion. }) }) @@ -3012,8 +3012,8 @@ var _ = Describe("drift detection and takeover", func() { regularDeploy.Spec.Replicas = ptr.To(int32(2)) // Create the resources on the member cluster side. - Expect(memberClient.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") - Expect(memberClient.Create(ctx, regularDeploy)).To(Succeed(), "Failed to create the Deployment object") + Expect(memberClient1.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") + Expect(memberClient1.Create(ctx, regularDeploy)).To(Succeed(), "Failed to create the Deployment object") markDeploymentAsAvailable(nsName, deployName) @@ -3022,7 +3022,7 @@ var _ = Describe("drift detection and takeover", func() { ComparisonOption: fleetv1beta1.ComparisonOptionTypeFullComparison, WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeIfNoDiff, } - createWorkObject(workName, applyStrategy, regularNSJSON, regularDeployJSON) + createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON, regularDeployJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -3048,7 +3048,7 @@ var _ = Describe("drift detection and takeover", func() { Consistently(func() error { // Retrieve the NS object. - if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { return fmt.Errorf("failed to retrieve the NS object: %w", err) } @@ -3075,7 +3075,7 @@ var _ = Describe("drift detection and takeover", func() { wantDeploy.Spec.Replicas = ptr.To(int32(2)) Consistently(func() error { - if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy); err != nil { return fmt.Errorf("failed to retrieve the Deployment object: %w", err) } @@ -3240,7 +3240,7 @@ var _ = Describe("drift detection and takeover", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Ensure that the Deployment object has been left alone. regularDeployNotRemovedActual := regularDeployNotRemovedActual(nsName, deployName) @@ -3254,7 +3254,7 @@ var _ = Describe("drift detection and takeover", func() { Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") // The environment prepared by the envtest package does not support namespace - // deletion; consequently this test suite would not attempt so verify its deletion. + // deletion; consequently this test suite would not attempt to verify its deletion. }) }) @@ -3285,7 +3285,7 @@ var _ = Describe("drift detection and takeover", func() { ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, WhenToApply: fleetv1beta1.WhenToApplyTypeIfNotDrifted, } - createWorkObject(workName, applyStrategy, regularNSJSON, regularDeployJSON) + createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON, regularDeployJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -3305,13 +3305,13 @@ var _ = Describe("drift detection and takeover", func() { regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the namespace object") - Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") // Ensure that the Deployment object has been applied as expected. regularDeploymentObjectAppliedActual := regularDeploymentObjectAppliedActual(nsName, deployName, appliedWorkOwnerRef) Eventually(regularDeploymentObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the deployment object") - Expect(memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") }) It("can mark the deployment as available", func() { @@ -3425,7 +3425,7 @@ var _ = Describe("drift detection and takeover", func() { Eventually(func() error { // Retrieve the Deployment object. updatedDeploy := &appsv1.Deployment{} - if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, updatedDeploy); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, updatedDeploy); err != nil { return fmt.Errorf("failed to retrieve the Deployment object: %w", err) } @@ -3433,7 +3433,7 @@ var _ = Describe("drift detection and takeover", func() { updatedDeploy.Spec.Replicas = ptr.To(int32(2)) // Update the Deployment object. - if err := memberClient.Update(ctx, updatedDeploy); err != nil { + if err := memberClient1.Update(ctx, updatedDeploy); err != nil { return fmt.Errorf("failed to update the Deployment object: %w", err) } return nil @@ -3442,7 +3442,7 @@ var _ = Describe("drift detection and takeover", func() { Eventually(func() error { // Retrieve the NS object. updatedNS := &corev1.Namespace{} - if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { return fmt.Errorf("failed to retrieve the NS object: %w", err) } @@ -3453,7 +3453,7 @@ var _ = Describe("drift detection and takeover", func() { updatedNS.Labels[dummyLabelKey] = dummyLabelValue1 // Update the NS object. - if err := memberClient.Update(ctx, updatedNS); err != nil { + if err := memberClient1.Update(ctx, updatedNS); err != nil { return fmt.Errorf("failed to update the NS object: %w", err) } return nil @@ -3477,7 +3477,7 @@ var _ = Describe("drift detection and takeover", func() { Consistently(func() error { // Retrieve the NS object. - if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { return fmt.Errorf("failed to retrieve the NS object: %w", err) } @@ -3509,7 +3509,7 @@ var _ = Describe("drift detection and takeover", func() { wantDeploy.Spec.Replicas = ptr.To(int32(2)) Consistently(func() error { - if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy); err != nil { return fmt.Errorf("failed to retrieve the Deployment object: %w", err) } @@ -3654,7 +3654,7 @@ var _ = Describe("drift detection and takeover", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Ensure that the Deployment object has been left alone. regularDeployNotRemovedActual := regularDeployNotRemovedActual(nsName, deployName) @@ -3672,7 +3672,7 @@ var _ = Describe("drift detection and takeover", func() { Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") // The environment prepared by the envtest package does not support namespace - // deletion; consequently this test suite would not attempt so verify its deletion. + // deletion; consequently this test suite would not attempt to verify its deletion. }) }) @@ -3698,7 +3698,7 @@ var _ = Describe("drift detection and takeover", func() { ComparisonOption: fleetv1beta1.ComparisonOptionTypeFullComparison, WhenToApply: fleetv1beta1.WhenToApplyTypeIfNotDrifted, } - createWorkObject(workName, applyStrategy, regularNSJSON) + createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -3718,7 +3718,7 @@ var _ = Describe("drift detection and takeover", func() { regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the namespace object") - Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") }) It("should update the Work object status", func() { @@ -3790,7 +3790,7 @@ var _ = Describe("drift detection and takeover", func() { Eventually(func() error { // Retrieve the NS object. updatedNS := &corev1.Namespace{} - if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { return fmt.Errorf("failed to retrieve the NS object: %w", err) } @@ -3801,7 +3801,7 @@ var _ = Describe("drift detection and takeover", func() { updatedNS.Labels[dummyLabelKey] = dummyLabelValue1 // Update the NS object. - if err := memberClient.Update(ctx, updatedNS); err != nil { + if err := memberClient1.Update(ctx, updatedNS); err != nil { return fmt.Errorf("failed to update the NS object: %w", err) } return nil @@ -3824,7 +3824,7 @@ var _ = Describe("drift detection and takeover", func() { Consistently(func() error { // Retrieve the NS object. - if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { return fmt.Errorf("failed to retrieve the NS object: %w", err) } @@ -3900,7 +3900,7 @@ var _ = Describe("drift detection and takeover", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Ensure that the AppliedWork object has been removed. appliedWorkRemovedActual := appliedWorkRemovedActual(workName, nsName) @@ -3910,7 +3910,7 @@ var _ = Describe("drift detection and takeover", func() { Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") // The environment prepared by the envtest package does not support namespace - // deletion; consequently this test suite would not attempt so verify its deletion. + // deletion; consequently this test suite would not attempt to verify its deletion. }) }) @@ -3938,7 +3938,7 @@ var _ = Describe("drift detection and takeover", func() { ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, WhenToApply: fleetv1beta1.WhenToApplyTypeAlways, } - createWorkObject(workName, applyStrategy, regularNSJSON) + createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -3958,7 +3958,7 @@ var _ = Describe("drift detection and takeover", func() { regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the namespace object") - Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") }) It("should update the Work object status", func() { @@ -4030,7 +4030,7 @@ var _ = Describe("drift detection and takeover", func() { Eventually(func() error { // Retrieve the NS object. updatedNS := &corev1.Namespace{} - if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { return fmt.Errorf("failed to retrieve the NS object: %w", err) } @@ -4041,7 +4041,7 @@ var _ = Describe("drift detection and takeover", func() { updatedNS.Labels[dummyLabelKey] = dummyLabelValue2 // Update the NS object. - if err := memberClient.Update(ctx, updatedNS); err != nil { + if err := memberClient1.Update(ctx, updatedNS); err != nil { return fmt.Errorf("failed to update the NS object: %w", err) } return nil @@ -4065,7 +4065,7 @@ var _ = Describe("drift detection and takeover", func() { nsOverwrittenActual := func() error { // Retrieve the NS object. - if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { return fmt.Errorf("failed to retrieve the NS object: %w", err) } @@ -4154,7 +4154,7 @@ var _ = Describe("drift detection and takeover", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Kubebuilder suggests that in a testing environment like this, to check for the existence of the AppliedWork object // OwnerReference in the Namespace object (https://book.kubebuilder.io/reference/envtest.html#testing-considerations). @@ -4168,7 +4168,7 @@ var _ = Describe("drift detection and takeover", func() { Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") // The environment prepared by the envtest package does not support namespace - // deletion; consequently this test suite would not attempt so verify its deletion. + // deletion; consequently this test suite would not attempt to verify its deletion. }) }) @@ -4195,7 +4195,7 @@ var _ = Describe("drift detection and takeover", func() { ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, WhenToApply: fleetv1beta1.WhenToApplyTypeIfNotDrifted, } - createWorkObject(workName, applyStrategy, marshalK8sObjJSON(regularNS)) + createWorkObject(workName, memberReservedNSName1, applyStrategy, marshalK8sObjJSON(regularNS)) }) It("should add cleanup finalizer to the Work object", func() { @@ -4215,7 +4215,7 @@ var _ = Describe("drift detection and takeover", func() { regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the namespace object") - Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") }) It("should update the Work object status", func() { @@ -4287,7 +4287,7 @@ var _ = Describe("drift detection and takeover", func() { Eventually(func() error { // Retrieve the NS object. updatedNS := &corev1.Namespace{} - if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { return fmt.Errorf("failed to retrieve the NS object: %w", err) } @@ -4298,7 +4298,7 @@ var _ = Describe("drift detection and takeover", func() { updatedNS.Labels[dummyLabelKey] = dummyLabelValue2 // Update the NS object. - if err := memberClient.Update(ctx, updatedNS); err != nil { + if err := memberClient1.Update(ctx, updatedNS); err != nil { return fmt.Errorf("failed to update the NS object: %w", err) } return nil @@ -4321,7 +4321,7 @@ var _ = Describe("drift detection and takeover", func() { Consistently(func() error { // Retrieve the NS object. - if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { return fmt.Errorf("failed to retrieve the NS object: %w", err) } @@ -4428,7 +4428,7 @@ var _ = Describe("drift detection and takeover", func() { Eventually(func() error { // Retrieve the NS object. - if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { return fmt.Errorf("failed to retrieve the NS object: %w", err) } @@ -4515,7 +4515,7 @@ var _ = Describe("drift detection and takeover", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Kubebuilder suggests that in a testing environment like this, to check for the existence of the AppliedWork object // OwnerReference in the Namespace object (https://book.kubebuilder.io/reference/envtest.html#testing-considerations). @@ -4529,7 +4529,7 @@ var _ = Describe("drift detection and takeover", func() { Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") // The environment prepared by the envtest package does not support namespace - // deletion; consequently this test suite would not attempt so verify its deletion. + // deletion; consequently this test suite would not attempt to verify its deletion. }) }) @@ -4556,7 +4556,7 @@ var _ = Describe("drift detection and takeover", func() { ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, WhenToApply: fleetv1beta1.WhenToApplyTypeIfNotDrifted, } - createWorkObject(workName, applyStrategy, marshalK8sObjJSON(regularNS)) + createWorkObject(workName, memberReservedNSName1, applyStrategy, marshalK8sObjJSON(regularNS)) }) It("should add cleanup finalizer to the Work object", func() { @@ -4576,7 +4576,7 @@ var _ = Describe("drift detection and takeover", func() { regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the namespace object") - Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") }) It("should update the Work object status", func() { @@ -4648,7 +4648,7 @@ var _ = Describe("drift detection and takeover", func() { Eventually(func() error { // Retrieve the NS object. updatedNS := &corev1.Namespace{} - if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { return fmt.Errorf("failed to retrieve the NS object: %w", err) } @@ -4659,7 +4659,7 @@ var _ = Describe("drift detection and takeover", func() { updatedNS.Labels[dummyLabelKey] = dummyLabelValue2 // Update the NS object. - if err := memberClient.Update(ctx, updatedNS); err != nil { + if err := memberClient1.Update(ctx, updatedNS); err != nil { return fmt.Errorf("failed to update the NS object: %w", err) } return nil @@ -4724,7 +4724,7 @@ var _ = Describe("drift detection and takeover", func() { Eventually(func() error { // Retrieve the NS object. updatedNS := &corev1.Namespace{} - if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { return fmt.Errorf("failed to retrieve the NS object: %w", err) } @@ -4735,7 +4735,7 @@ var _ = Describe("drift detection and takeover", func() { updatedNS.Labels[dummyLabelKey] = dummyLabelValue4 // Update the NS object. - if err := memberClient.Update(ctx, updatedNS); err != nil { + if err := memberClient1.Update(ctx, updatedNS); err != nil { return fmt.Errorf("failed to update the NS object: %w", err) } return nil @@ -4793,7 +4793,7 @@ var _ = Describe("drift detection and takeover", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Kubebuilder suggests that in a testing environment like this, to check for the existence of the AppliedWork object // OwnerReference in the Namespace object (https://book.kubebuilder.io/reference/envtest.html#testing-considerations). @@ -4807,7 +4807,7 @@ var _ = Describe("drift detection and takeover", func() { Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") // The environment prepared by the envtest package does not support namespace - // deletion; consequently this test suite would not attempt so verify its deletion. + // deletion; consequently this test suite would not attempt to verify its deletion. }) }) @@ -4835,13 +4835,13 @@ var _ = Describe("drift detection and takeover", func() { regularNSJSON := marshalK8sObjJSON(regularNS) // Create the resources on the member cluster side. - Expect(memberClient.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") + Expect(memberClient1.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") // Create a new Work object with all the manifest JSONs and proper apply strategy. applyStrategy := &fleetv1beta1.ApplyStrategy{ WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeNever, } - createWorkObject(workName, applyStrategy, regularNSJSON, marshalK8sObjJSON(regularDeploy)) + createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON, marshalK8sObjJSON(regularDeploy)) }) It("should add cleanup finalizer to the Work object", func() { @@ -4861,14 +4861,14 @@ var _ = Describe("drift detection and takeover", func() { regularDeploymentObjectAppliedActual := regularDeploymentObjectAppliedActual(nsName, deployName, appliedWorkOwnerRef) Eventually(regularDeploymentObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the deployment object") - Expect(memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") }) It("should not apply the manifests that have corresponding resources", func() { Eventually(func() error { // Retrieve the NS object. updatedNS := &corev1.Namespace{} - if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { return fmt.Errorf("failed to retrieve the NS object: %w", err) } @@ -4975,7 +4975,7 @@ var _ = Describe("drift detection and takeover", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Ensure applied manifest has been removed. regularDeployRemovedActual := regularDeployRemovedActual(nsName, deployName) @@ -4989,7 +4989,7 @@ var _ = Describe("drift detection and takeover", func() { Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") // The environment prepared by the envtest package does not support namespace - // deletion; consequently this test suite would not attempt so verify its deletion. + // deletion; consequently this test suite would not attempt to verify its deletion. }) }) }) @@ -5014,7 +5014,7 @@ var _ = Describe("report diff", func() { applyStrategy := &fleetv1beta1.ApplyStrategy{ Type: fleetv1beta1.ApplyStrategyTypeReportDiff, } - createWorkObject(workName, applyStrategy, regularNSJSON) + createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -5085,7 +5085,7 @@ var _ = Describe("report diff", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Ensure that the AppliedWork object has been removed. appliedWorkRemovedActual := appliedWorkRemovedActual(workName, nsName) @@ -5095,7 +5095,7 @@ var _ = Describe("report diff", func() { Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") // The environment prepared by the envtest package does not support namespace - // deletion; consequently this test suite would not attempt so verify its deletion. + // deletion; consequently this test suite would not attempt to verify its deletion. }) }) @@ -5122,18 +5122,18 @@ var _ = Describe("report diff", func() { regularDeployJSON := marshalK8sObjJSON(regularDeploy) // Create the objects first in the member cluster. - Expect(memberClient.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") + Expect(memberClient1.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") // Create a diff in the replica count field. regularDeploy.Spec.Replicas = ptr.To(int32(2)) - Expect(memberClient.Create(ctx, regularDeploy)).To(Succeed(), "Failed to create the Deployment object") + Expect(memberClient1.Create(ctx, regularDeploy)).To(Succeed(), "Failed to create the Deployment object") // Create a new Work object with all the manifest JSONs and proper apply strategy. applyStrategy := &fleetv1beta1.ApplyStrategy{ ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, Type: fleetv1beta1.ApplyStrategyTypeReportDiff, } - createWorkObject(workName, applyStrategy, regularNSJSON, regularDeployJSON) + createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON, regularDeployJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -5161,7 +5161,7 @@ var _ = Describe("report diff", func() { wantDeploy.Spec.Replicas = ptr.To(int32(2)) deployOwnedButNotApplied := func() error { - if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy); err != nil { return fmt.Errorf("failed to retrieve the Deployment object: %w", err) } @@ -5221,7 +5221,7 @@ var _ = Describe("report diff", func() { } nsOwnedButNotApplied := func() error { - if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { return fmt.Errorf("failed to retrieve the NS object: %w", err) } @@ -5322,7 +5322,7 @@ var _ = Describe("report diff", func() { Eventually(func() error { // Retrieve the Deployment object. updatedDeploy := &appsv1.Deployment{} - if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, updatedDeploy); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, updatedDeploy); err != nil { return fmt.Errorf("failed to retrieve the Deployment object: %w", err) } @@ -5330,7 +5330,7 @@ var _ = Describe("report diff", func() { updatedDeploy.Spec.Replicas = ptr.To(int32(1)) // Update the Deployment object. - if err := memberClient.Update(ctx, updatedDeploy); err != nil { + if err := memberClient1.Update(ctx, updatedDeploy); err != nil { return fmt.Errorf("failed to update the Deployment object: %w", err) } return nil @@ -5409,7 +5409,7 @@ var _ = Describe("report diff", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Ensure that the Deployment object has been left alone. regularDeployNotRemovedActual := regularDeployNotRemovedActual(nsName, deployName) @@ -5423,7 +5423,7 @@ var _ = Describe("report diff", func() { Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") // The environment prepared by the envtest package does not support namespace - // deletion; consequently this test suite would not attempt so verify its deletion. + // deletion; consequently this test suite would not attempt to verify its deletion. }) }) @@ -5449,11 +5449,11 @@ var _ = Describe("report diff", func() { regularDeployJSON := marshalK8sObjJSON(regularDeploy) // Create the objects first in the member cluster. - Expect(memberClient.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") + Expect(memberClient1.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") // Create a diff in the replica count field. regularDeploy.Spec.Replicas = ptr.To(int32(2)) - Expect(memberClient.Create(ctx, regularDeploy)).To(Succeed(), "Failed to create the Deployment object") + Expect(memberClient1.Create(ctx, regularDeploy)).To(Succeed(), "Failed to create the Deployment object") // Create a new Work object with all the manifest JSONs and proper apply strategy. applyStrategy := &fleetv1beta1.ApplyStrategy{ @@ -5461,7 +5461,7 @@ var _ = Describe("report diff", func() { Type: fleetv1beta1.ApplyStrategyTypeReportDiff, WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeNever, } - createWorkObject(workName, applyStrategy, regularNSJSON, regularDeployJSON) + createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON, regularDeployJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -5479,7 +5479,7 @@ var _ = Describe("report diff", func() { Consistently(func() error { // Retrieve the NS object. updatedNS := &corev1.Namespace{} - if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { return fmt.Errorf("failed to retrieve the NS object: %w", err) } @@ -5503,7 +5503,7 @@ var _ = Describe("report diff", func() { Consistently(func() error { // Retrieve the Deployment object. updatedDeploy := &appsv1.Deployment{} - if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, updatedDeploy); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, updatedDeploy); err != nil { return fmt.Errorf("failed to retrieve the Deployment object: %w", err) } @@ -5624,7 +5624,7 @@ var _ = Describe("report diff", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Ensure applied manifest has been removed. regularDeployRemovedActual := regularDeployRemovedActual(nsName, deployName) @@ -5638,7 +5638,7 @@ var _ = Describe("report diff", func() { Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") // The environment prepared by the envtest package does not support namespace - // deletion; consequently this test suite would not attempt so verify its deletion. + // deletion; consequently this test suite would not attempt to verify its deletion. }) }) @@ -5670,7 +5670,7 @@ var _ = Describe("report diff", func() { applyStrategy := &fleetv1beta1.ApplyStrategy{ Type: fleetv1beta1.ApplyStrategyTypeReportDiff, } - createWorkObject(workName, applyStrategy, regularNSJSON, malformedConfigMapJSON) + createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON, malformedConfigMapJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -5689,7 +5689,7 @@ var _ = Describe("report diff", func() { Consistently(func() error { configMap := &corev1.ConfigMap{} objKey := client.ObjectKey{Namespace: nsName, Name: malformedConfigMap.Name} - if err := memberClient.Get(ctx, objKey, configMap); !errors.IsNotFound(err) { + if err := memberClient1.Get(ctx, objKey, configMap); !errors.IsNotFound(err) { return fmt.Errorf("the config map exists, or an unexpected error has occurred: %w", err) } return nil @@ -5768,7 +5768,7 @@ var _ = Describe("report diff", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Ensure that the AppliedWork object has been removed. appliedWorkRemovedActual := appliedWorkRemovedActual(workName, nsName) @@ -5807,18 +5807,18 @@ var _ = Describe("handling different apply strategies", func() { regularDeployJSON := marshalK8sObjJSON(regularDeploy) // Create the objects first in the member cluster. - Expect(memberClient.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") + Expect(memberClient1.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") // Create a diff in the replica count field. regularDeploy.Spec.Replicas = ptr.To(int32(2)) - Expect(memberClient.Create(ctx, regularDeploy)).To(Succeed(), "Failed to create the Deployment object") + Expect(memberClient1.Create(ctx, regularDeploy)).To(Succeed(), "Failed to create the Deployment object") // Create a new Work object with all the manifest JSONs and proper apply strategy. applyStrategy := &fleetv1beta1.ApplyStrategy{ ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, Type: fleetv1beta1.ApplyStrategyTypeReportDiff, } - createWorkObject(workName, applyStrategy, regularNSJSON, regularDeployJSON) + createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON, regularDeployJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -5846,7 +5846,7 @@ var _ = Describe("handling different apply strategies", func() { wantDeploy.Spec.Replicas = ptr.To(int32(2)) deployOwnedButNotApplied := func() error { - if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy); err != nil { return fmt.Errorf("failed to retrieve the Deployment object: %w", err) } @@ -5906,7 +5906,7 @@ var _ = Describe("handling different apply strategies", func() { } nsOwnedButNotApplied := func() error { - if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { return fmt.Errorf("failed to retrieve the NS object: %w", err) } @@ -6005,7 +6005,7 @@ var _ = Describe("handling different apply strategies", func() { It("can update the apply strategy", func() { Eventually(func() error { work := &fleetv1beta1.Work{} - if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName}, work); err != nil { + if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName1}, work); err != nil { return fmt.Errorf("failed to retrieve the Work object: %w", err) } @@ -6126,7 +6126,7 @@ var _ = Describe("handling different apply strategies", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Ensure applied manifest has been removed. regularDeployRemovedActual := regularDeployRemovedActual(nsName, deployName) @@ -6144,7 +6144,7 @@ var _ = Describe("handling different apply strategies", func() { Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") // The environment prepared by the envtest package does not support namespace - // deletion; consequently this test suite would not attempt so verify its deletion. + // deletion; consequently this test suite would not attempt to verify its deletion. }) }) @@ -6175,7 +6175,7 @@ var _ = Describe("handling different apply strategies", func() { ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, Type: fleetv1beta1.ApplyStrategyTypeServerSideApply, } - createWorkObject(workName, applyStrategy, regularNSJSON, regularDeployJSON) + createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON, regularDeployJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -6195,13 +6195,13 @@ var _ = Describe("handling different apply strategies", func() { regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the namespace object") - Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") // Ensure that the Deployment object has been applied as expected. regularDeploymentObjectAppliedActual := regularDeploymentObjectAppliedActual(nsName, deployName, appliedWorkOwnerRef) Eventually(regularDeploymentObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the deployment object") - Expect(memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy)).To(Succeed(), "Failed to retrieve the Deployment object") }) It("should update the AppliedWork object status", func() { @@ -6309,7 +6309,7 @@ var _ = Describe("handling different apply strategies", func() { It("can update the apply strategy", func() { Eventually(func() error { work := &fleetv1beta1.Work{} - if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName}, work); err != nil { + if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName1}, work); err != nil { return fmt.Errorf("failed to retrieve the Work object: %w", err) } @@ -6387,7 +6387,7 @@ var _ = Describe("handling different apply strategies", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Ensure applied manifest has been removed. regularDeployRemovedActual := regularDeployRemovedActual(nsName, deployName) @@ -6405,7 +6405,7 @@ var _ = Describe("handling different apply strategies", func() { Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") // The environment prepared by the envtest package does not support namespace - // deletion; consequently this test suite would not attempt so verify its deletion. + // deletion; consequently this test suite would not attempt to verify its deletion. }) }) @@ -6433,10 +6433,10 @@ var _ = Describe("handling different apply strategies", func() { // Create objects in the member cluster. preExistingNS := regularNS.DeepCopy() - Expect(memberClient.Create(ctx, preExistingNS)).To(Succeed(), "Failed to create the NS object") + Expect(memberClient1.Create(ctx, preExistingNS)).To(Succeed(), "Failed to create the NS object") preExistingDeploy := regularDeploy.DeepCopy() preExistingDeploy.Spec.Replicas = ptr.To(int32(2)) - Expect(memberClient.Create(ctx, preExistingDeploy)).To(Succeed(), "Failed to create the Deployment object") + Expect(memberClient1.Create(ctx, preExistingDeploy)).To(Succeed(), "Failed to create the Deployment object") // Create a new Work object with all the manifest JSONs and proper apply strategy. applyStrategy := &fleetv1beta1.ApplyStrategy{ @@ -6444,7 +6444,7 @@ var _ = Describe("handling different apply strategies", func() { Type: fleetv1beta1.ApplyStrategyTypeClientSideApply, WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeNever, } - createWorkObject(workName, applyStrategy, regularNSJSON, regularDeployJSON) + createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON, regularDeployJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -6467,7 +6467,7 @@ var _ = Describe("handling different apply strategies", func() { Consistently(func() error { preExistingNS := &corev1.Namespace{} - if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, preExistingNS); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, preExistingNS); err != nil { return fmt.Errorf("failed to retrieve the NS object: %w", err) } @@ -6493,7 +6493,7 @@ var _ = Describe("handling different apply strategies", func() { Consistently(func() error { preExistingDeploy := &appsv1.Deployment{} - if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, preExistingDeploy); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, preExistingDeploy); err != nil { return fmt.Errorf("failed to retrieve the Deployment object: %w", err) } @@ -6598,7 +6598,7 @@ var _ = Describe("handling different apply strategies", func() { It("can update the apply strategy", func() { Eventually(func() error { work := &fleetv1beta1.Work{} - if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName}, work); err != nil { + if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName1}, work); err != nil { return fmt.Errorf("failed to retrieve the Work object: %w", err) } @@ -6619,7 +6619,7 @@ var _ = Describe("handling different apply strategies", func() { regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the namespace object") - Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") }) It("should not take over some objects", func() { @@ -6632,7 +6632,7 @@ var _ = Describe("handling different apply strategies", func() { Consistently(func() error { preExistingDeploy := &appsv1.Deployment{} - if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, preExistingDeploy); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, preExistingDeploy); err != nil { return fmt.Errorf("failed to retrieve the Deployment object: %w", err) } @@ -6772,7 +6772,7 @@ var _ = Describe("handling different apply strategies", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Ensure applied manifest has been removed. regularDeployRemovedActual := regularDeployRemovedActual(nsName, deployName) @@ -6790,7 +6790,7 @@ var _ = Describe("handling different apply strategies", func() { Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") // The environment prepared by the envtest package does not support namespace - // deletion; consequently this test suite would not attempt so verify its deletion. + // deletion; consequently this test suite would not attempt to verify its deletion. }) }) @@ -6843,7 +6843,7 @@ var _ = Describe("handling different apply strategies", func() { oversizedCMJSON := marshalK8sObjJSON(oversizedCM) // Create a new Work object with all the manifest JSONs and proper apply strategy. - createWorkObject(workName, nil, regularNSJSON, oversizedCMJSON) + createWorkObject(workName, memberReservedNSName1, nil, regularNSJSON, oversizedCMJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -6863,12 +6863,12 @@ var _ = Describe("handling different apply strategies", func() { regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the namespace object") - Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") // Ensure that the oversized ConfigMap object has been applied as expected via SSA. Eventually(func() error { gotConfigMap := &corev1.ConfigMap{} - if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: configMapName}, gotConfigMap); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: configMapName}, gotConfigMap); err != nil { return fmt.Errorf("failed to retrieve the ConfigMap object: %w", err) } @@ -6913,7 +6913,7 @@ var _ = Describe("handling different apply strategies", func() { return nil }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the oversized configMap object") - Expect(memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: configMapName}, oversizedCM)).To(Succeed(), "Failed to retrieve the ConfigMap object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: configMapName}, oversizedCM)).To(Succeed(), "Failed to retrieve the ConfigMap object") }) It("should update the Work object status", func() { @@ -6988,7 +6988,7 @@ var _ = Describe("handling different apply strategies", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Ensure that all applied manifests have been removed. appliedWorkRemovedActual := appliedWorkRemovedActual(workName, nsName) @@ -7002,18 +7002,18 @@ var _ = Describe("handling different apply strategies", func() { Name: configMapName, }, } - if err := memberClient.Delete(ctx, cm); err != nil && !errors.IsNotFound(err) { + if err := memberClient1.Delete(ctx, cm); err != nil && !errors.IsNotFound(err) { return fmt.Errorf("failed to delete the ConfigMap object: %w", err) } - if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: configMapName}, cm); !errors.IsNotFound(err) { + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: configMapName}, cm); !errors.IsNotFound(err) { return fmt.Errorf("the ConfigMap object still exists or an unexpected error occurred: %w", err) } return nil }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the oversized configMap object") // The environment prepared by the envtest package does not support namespace - // deletion; consequently this test suite would not attempt so verify its deletion. + // deletion; consequently this test suite would not attempt to verify its deletion. }) }) }) @@ -7051,7 +7051,7 @@ var _ = Describe("negative cases", func() { malformedConfigMapJSON := marshalK8sObjJSON(malformedConfigMap) // Create a Work object with all the manifest JSONs. - createWorkObject(workName, nil, regularNSJSON, malformedConfigMapJSON, regularConfigMapJson) + createWorkObject(workName, memberReservedNSName1, nil, regularNSJSON, malformedConfigMapJSON, regularConfigMapJson) }) It("should add cleanup finalizer to the Work object", func() { @@ -7071,13 +7071,13 @@ var _ = Describe("negative cases", func() { regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the namespace object") - Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") // Ensure that the ConfigMap object has been applied as expected. regularConfigMapAppliedActual := regularConfigMapObjectAppliedActual(nsName, configMapName, appliedWorkOwnerRef) Eventually(regularConfigMapAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the ConfigMap object") - Expect(memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: configMapName}, regularConfigMap)).To(Succeed(), "Failed to retrieve the ConfigMap object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: configMapName}, regularConfigMap)).To(Succeed(), "Failed to retrieve the ConfigMap object") }) It("should update the Work object status", func() { @@ -7198,7 +7198,7 @@ var _ = Describe("negative cases", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Ensure applied manifest has been removed. regularConfigMapRemovedActual := regularConfigMapRemovedActual(nsName, configMapName) @@ -7244,7 +7244,7 @@ var _ = Describe("negative cases", func() { regularConfigMapJSON := marshalK8sObjJSON(regularConfigMap) // Create a Work object with all the manifest JSONs. - createWorkObject(workName, nil, regularNSJSON, regularConfigMapJSON) + createWorkObject(workName, memberReservedNSName1, nil, regularNSJSON, regularConfigMapJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -7264,7 +7264,7 @@ var _ = Describe("negative cases", func() { regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the namespace object") - Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") }) It("should update the Work object status", func() { @@ -7349,7 +7349,7 @@ var _ = Describe("negative cases", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Kubebuilder suggests that in a testing environment like this, to check for the existence of the AppliedWork object // OwnerReference in the Namespace object (https://book.kubebuilder.io/reference/envtest.html#testing-considerations). @@ -7395,7 +7395,7 @@ var _ = Describe("negative cases", func() { duplicatedConfigMap.Data[dummyLabelKey] = dummyLabelValue2 // Create a Work object with all the manifest JSONs. - createWorkObject(workName, nil, regularNSJSON, regularConfigMapJSON, duplicatedConfigMapJSON) + createWorkObject(workName, memberReservedNSName1, nil, regularNSJSON, regularConfigMapJSON, duplicatedConfigMapJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -7415,13 +7415,13 @@ var _ = Describe("negative cases", func() { regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the namespace object") - Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") // Ensure that the ConfigMap object has been applied as expected. regularConfigMapAppliedActual := regularConfigMapObjectAppliedActual(nsName, configMapName, appliedWorkOwnerRef) Eventually(regularConfigMapAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the ConfigMap object") - Expect(memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: configMapName}, regularConfigMap)).To(Succeed(), "Failed to retrieve the ConfigMap object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: configMapName}, regularConfigMap)).To(Succeed(), "Failed to retrieve the ConfigMap object") }) It("should update the Work object status", func() { @@ -7590,7 +7590,7 @@ var _ = Describe("negative cases", func() { duplicatedConfigMapJSON := marshalK8sObjJSON(duplicatedConfigMap) // Create a Work object with all the manifest JSONs. - createWorkObject(workName, nil, regularNSJSON, regularConfigMapJSON, malformedConfigMapJSON, configMapWithGenerateNameJSON, duplicatedConfigMapJSON) + createWorkObject(workName, memberReservedNSName1, nil, regularNSJSON, regularConfigMapJSON, malformedConfigMapJSON, configMapWithGenerateNameJSON, duplicatedConfigMapJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -7610,13 +7610,13 @@ var _ = Describe("negative cases", func() { regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the namespace object") - Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") // Ensure that the ConfigMap object has been applied as expected. regularConfigMapAppliedActual := regularConfigMapObjectAppliedActual(nsName, configMapName, appliedWorkOwnerRef) Eventually(regularConfigMapAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the ConfigMap object") - Expect(memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: configMapName}, regularConfigMap)).To(Succeed(), "Failed to retrieve the ConfigMap object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: configMapName}, regularConfigMap)).To(Succeed(), "Failed to retrieve the ConfigMap object") }) It("should update the Work object status", func() { @@ -7774,7 +7774,7 @@ var _ = Describe("negative cases", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Ensure applied manifest has been removed. regularConfigMapRemovedActual := regularConfigMapRemovedActual(nsName, configMapName) diff --git a/pkg/controllers/workapplier/drift_detection_takeover_test.go b/pkg/controllers/workapplier/drift_detection_takeover_test.go index 017633d65..4cdc4398d 100644 --- a/pkg/controllers/workapplier/drift_detection_takeover_test.go +++ b/pkg/controllers/workapplier/drift_detection_takeover_test.go @@ -129,7 +129,7 @@ func TestTakeOverPreExistingObject(t *testing.T) { workObj: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ Name: "dummy-work", - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, }, }, applyStrategy: &fleetv1beta1.ApplyStrategy{ @@ -206,7 +206,7 @@ func TestTakeOverPreExistingObject(t *testing.T) { r := &Reconciler{ hubClient: fakeHubClient, spokeDynamicClient: fakeMemberClient, - workNameSpace: memberReservedNSName, + workNameSpace: memberReservedNSName1, } takenOverObj, patchDetails, err := r.takeOverPreExistingObject( @@ -496,7 +496,7 @@ func TestRemoveLeftBehindAppliedWorkOwnerRefs(t *testing.T) { workObj: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ Name: workName, - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, }, }, }, @@ -512,7 +512,7 @@ func TestRemoveLeftBehindAppliedWorkOwnerRefs(t *testing.T) { r := &Reconciler{ hubClient: fakeHubClient, - workNameSpace: memberReservedNSName, + workNameSpace: memberReservedNSName1, } gotOwnerRefs, err := r.removeLeftBehindAppliedWorkOwnerRefs(ctx, tc.ownerRefs) diff --git a/pkg/controllers/workapplier/preprocess_test.go b/pkg/controllers/workapplier/preprocess_test.go index 305acc42e..3e453d642 100644 --- a/pkg/controllers/workapplier/preprocess_test.go +++ b/pkg/controllers/workapplier/preprocess_test.go @@ -98,7 +98,7 @@ func TestCheckForDuplicatedManifests(t *testing.T) { work := &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ Name: workName, - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, }, } diff --git a/pkg/controllers/workapplier/status_test.go b/pkg/controllers/workapplier/status_test.go index fd4f837a4..a8d9f3a96 100644 --- a/pkg/controllers/workapplier/status_test.go +++ b/pkg/controllers/workapplier/status_test.go @@ -52,7 +52,7 @@ func TestRefreshWorkStatus(t *testing.T) { workNS := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: memberReservedNSName, + Name: memberReservedNSName1, }, } @@ -79,7 +79,7 @@ func TestRefreshWorkStatus(t *testing.T) { work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ Name: workName, - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Generation: 1, }, }, @@ -148,7 +148,7 @@ func TestRefreshWorkStatus(t *testing.T) { work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ Name: workName, - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Generation: 2, }, }, @@ -236,7 +236,7 @@ func TestRefreshWorkStatus(t *testing.T) { work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ Name: workName, - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, }, }, bundles: []*manifestProcessingBundle{ @@ -374,7 +374,7 @@ func TestRefreshWorkStatus(t *testing.T) { work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ Name: workName, - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Generation: 2, }, Status: fleetv1beta1.WorkStatus{ @@ -552,7 +552,7 @@ func TestRefreshWorkStatus(t *testing.T) { work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ Name: workName, - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Generation: 2, }, Spec: fleetv1beta1.WorkSpec{ @@ -662,7 +662,7 @@ func TestRefreshWorkStatus(t *testing.T) { work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ Name: workName, - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Generation: 2, }, Spec: fleetv1beta1.WorkSpec{ @@ -760,7 +760,7 @@ func TestRefreshWorkStatus(t *testing.T) { work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ Name: workName, - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Generation: 2, }, Spec: fleetv1beta1.WorkSpec{ @@ -923,7 +923,7 @@ func TestRefreshWorkStatus(t *testing.T) { work: &fleetv1beta1.Work{ ObjectMeta: metav1.ObjectMeta{ Name: workName, - Namespace: memberReservedNSName, + Namespace: memberReservedNSName1, Generation: 2, }, Spec: fleetv1beta1.WorkSpec{ @@ -1076,7 +1076,7 @@ func TestRefreshWorkStatus(t *testing.T) { Build() r := &Reconciler{ hubClient: fakeClient, - workNameSpace: memberReservedNSName, + workNameSpace: memberReservedNSName1, } err := r.refreshWorkStatus(ctx, tc.work, tc.bundles) @@ -1085,7 +1085,7 @@ func TestRefreshWorkStatus(t *testing.T) { } updatedWork := &fleetv1beta1.Work{} - if err := fakeClient.Get(ctx, types.NamespacedName{Namespace: memberReservedNSName, Name: workName}, updatedWork); err != nil { + if err := fakeClient.Get(ctx, types.NamespacedName{Namespace: memberReservedNSName1, Name: workName}, updatedWork); err != nil { t.Fatalf("Work Get() = %v, want no error", err) } opts := []cmp.Option{ diff --git a/pkg/controllers/workapplier/suite_test.go b/pkg/controllers/workapplier/suite_test.go index 40be7a437..c61e0d82a 100644 --- a/pkg/controllers/workapplier/suite_test.go +++ b/pkg/controllers/workapplier/suite_test.go @@ -20,6 +20,7 @@ import ( "context" "flag" "path/filepath" + "sync" "testing" "time" @@ -33,12 +34,15 @@ import ( "k8s.io/klog/v2" "k8s.io/klog/v2/textlogger" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + ctrloption "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/predicate" fleetv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1" testv1alpha1 "github.com/kubefleet-dev/kubefleet/test/apis/v1alpha1" @@ -47,18 +51,27 @@ import ( // These tests use Ginkgo (BDD-style Go testing framework). Refer to // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. var ( - hubCfg *rest.Config - memberCfg *rest.Config - hubEnv *envtest.Environment - memberEnv *envtest.Environment - hubMgr manager.Manager - hubClient client.Client - memberClient client.Client - memberDynamicClient dynamic.Interface - workApplier *Reconciler + hubCfg *rest.Config + hubEnv *envtest.Environment + hubClient client.Client + + memberCfg1 *rest.Config + memberEnv1 *envtest.Environment + hubMgr1 manager.Manager + memberClient1 client.Client + memberDynamicClient1 dynamic.Interface + workApplier1 *Reconciler + + memberCfg2 *rest.Config + memberEnv2 *envtest.Environment + hubMgr2 manager.Manager + memberClient2 client.Client + memberDynamicClient2 dynamic.Interface + workApplier2 *Reconciler ctx context.Context cancel context.CancelFunc + wg sync.WaitGroup ) const ( @@ -67,7 +80,8 @@ const ( // The count of workers for the work applier controller. workerCount = 4 - memberReservedNSName = "fleet-member-experimental" + memberReservedNSName1 = "fleet-member-experimental-1" + memberReservedNSName2 = "fleet-member-experimental-2" ) func TestAPIs(t *testing.T) { @@ -77,12 +91,19 @@ func TestAPIs(t *testing.T) { } func setupResources() { - ns := &corev1.Namespace{ + ns1 := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: memberReservedNSName1, + }, + } + Expect(hubClient.Create(ctx, ns1)).To(Succeed()) + + ns2 := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: memberReservedNSName, + Name: memberReservedNSName2, }, } - Expect(hubClient.Create(ctx, ns)).To(Succeed()) + Expect(hubClient.Create(ctx, ns2)).To(Succeed()) } var _ = BeforeSuite(func() { @@ -102,7 +123,20 @@ var _ = BeforeSuite(func() { filepath.Join("../../../", "test", "manifests"), }, } - memberEnv = &envtest.Environment{ + // memberEnv1 is the test environment for verifying most work applier behaviors. + memberEnv1 = &envtest.Environment{ + CRDDirectoryPaths: []string{ + filepath.Join("../../../", "config", "crd", "bases"), + filepath.Join("../../../", "test", "manifests"), + }, + } + // memberEnv2 is the test environment for verifying the correctness of exponential backoff as + // enabled in the work applier. + // + // Note (chenyu1): to avoid flakiness, a separate test environment with a work applier of special + // exponential backoff setting is used so that the backoffs can be identified in a more + // apparent and controllable manner. + memberEnv2 = &envtest.Environment{ CRDDirectoryPaths: []string{ filepath.Join("../../../", "config", "crd", "bases"), filepath.Join("../../../", "test", "manifests"), @@ -114,9 +148,13 @@ var _ = BeforeSuite(func() { Expect(err).ToNot(HaveOccurred()) Expect(hubCfg).ToNot(BeNil()) - memberCfg, err = memberEnv.Start() + memberCfg1, err = memberEnv1.Start() Expect(err).ToNot(HaveOccurred()) - Expect(memberCfg).ToNot(BeNil()) + Expect(memberCfg1).ToNot(BeNil()) + + memberCfg2, err = memberEnv2.Start() + Expect(err).ToNot(HaveOccurred()) + Expect(memberCfg2).ToNot(BeNil()) err = fleetv1beta1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) @@ -128,52 +166,125 @@ var _ = BeforeSuite(func() { Expect(err).ToNot(HaveOccurred()) Expect(hubClient).ToNot(BeNil()) - memberClient, err = client.New(memberCfg, client.Options{Scheme: scheme.Scheme}) + memberClient1, err = client.New(memberCfg1, client.Options{Scheme: scheme.Scheme}) + Expect(err).ToNot(HaveOccurred()) + Expect(memberClient1).ToNot(BeNil()) + + memberClient2, err = client.New(memberCfg2, client.Options{Scheme: scheme.Scheme}) Expect(err).ToNot(HaveOccurred()) - Expect(memberClient).ToNot(BeNil()) + Expect(memberClient2).ToNot(BeNil()) // This setup also requires a client-go dynamic client for the member cluster. - memberDynamicClient, err = dynamic.NewForConfig(memberCfg) + memberDynamicClient1, err = dynamic.NewForConfig(memberCfg1) + Expect(err).ToNot(HaveOccurred()) + + memberDynamicClient2, err = dynamic.NewForConfig(memberCfg2) Expect(err).ToNot(HaveOccurred()) By("Setting up the resources") setupResources() - By("Setting up the controller and the controller manager") - hubMgr, err = ctrl.NewManager(hubCfg, ctrl.Options{ + By("Setting up the controller and the controller manager for member cluster 1") + hubMgr1, err = ctrl.NewManager(hubCfg, ctrl.Options{ + Scheme: scheme.Scheme, + Metrics: server.Options{ + BindAddress: "0", + }, + Cache: cache.Options{ + DefaultNamespaces: map[string]cache.Config{ + memberReservedNSName1: {}, + }, + }, + Logger: textlogger.NewLogger(textlogger.NewConfig(textlogger.Verbosity(4))), + }) + Expect(err).ToNot(HaveOccurred()) + + workApplier1 = NewReconciler( + hubClient, + memberReservedNSName1, + memberDynamicClient1, + memberClient1, + memberClient1.RESTMapper(), + hubMgr1.GetEventRecorderFor("work-applier"), + maxConcurrentReconciles, + workerCount, + 30*time.Second, + true, + 60, + nil, // Use the default backoff rate limiter. + ) + Expect(workApplier1.SetupWithManager(hubMgr1)).To(Succeed()) + + By("Setting up the controller and the controller manager for member cluster 2") + hubMgr2, err = ctrl.NewManager(hubCfg, ctrl.Options{ Scheme: scheme.Scheme, Metrics: server.Options{ BindAddress: "0", }, Cache: cache.Options{ DefaultNamespaces: map[string]cache.Config{ - memberReservedNSName: {}, + memberReservedNSName2: {}, }, }, Logger: textlogger.NewLogger(textlogger.NewConfig(textlogger.Verbosity(4))), }) Expect(err).ToNot(HaveOccurred()) - workApplier = NewReconciler( + superLongExponentialBackoffRateLimiter := NewRequeueMultiStageWithExponentialBackoffRateLimiter( + // Allow one attempt of backoff with fixed delay. + 1, + // Use a fixed delay of 10 seconds. + 10, + // Set the exponential backoff base factor to 1.5 for the slow backoff stage. + 1.5, + // Set the initial slow backoff delay to 20 seconds. + 20, + // Set the maximum slow backoff delay to 30 seconds (allow 2 slow backoffs). + 30, + // Set the exponential backoff base factor to 2 for the fast backoff stage. + 2, + // Set the maximum fast backoff delay to 90 seconds (allow 1 fast backoff). + 90, + // Allow skipping to fast backoff stage. + true, + ) + workApplier2 = NewReconciler( hubClient, - memberReservedNSName, - memberDynamicClient, - memberClient, - memberClient.RESTMapper(), - hubMgr.GetEventRecorderFor("work-applier"), + memberReservedNSName2, + memberDynamicClient2, + memberClient2, + memberClient2.RESTMapper(), + hubMgr2.GetEventRecorderFor("work-applier"), maxConcurrentReconciles, workerCount, 30*time.Second, true, 60, - defaultRequeueRateLimiter, + superLongExponentialBackoffRateLimiter, ) - Expect(workApplier.SetupWithManager(hubMgr)).To(Succeed()) + // Due to name conflicts, the second work applier must be set up manually. + err = ctrl.NewControllerManagedBy(hubMgr2).Named("work-applier-controller-duplicate"). + WithOptions(ctrloption.Options{ + MaxConcurrentReconciles: workApplier2.concurrentReconciles, + }). + For(&fleetv1beta1.Work{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). + Complete(workApplier2) + Expect(err).NotTo(HaveOccurred()) + + wg = sync.WaitGroup{} + wg.Add(2) + go func() { + defer GinkgoRecover() + defer wg.Done() + Expect(workApplier1.Join(ctx)).To(Succeed()) + Expect(hubMgr1.Start(ctx)).To(Succeed()) + }() go func() { defer GinkgoRecover() - Expect(workApplier.Join(ctx)).To(Succeed()) - Expect(hubMgr.Start(ctx)).To(Succeed()) + defer wg.Done() + Expect(workApplier2.Join(ctx)).To(Succeed()) + Expect(hubMgr2.Start(ctx)).To(Succeed()) }() }) @@ -181,7 +292,9 @@ var _ = AfterSuite(func() { defer klog.Flush() cancel() + wg.Wait() By("Tearing down the test environment") Expect(hubEnv.Stop()).To(Succeed()) - Expect(memberEnv.Stop()).To(Succeed()) + Expect(memberEnv1.Stop()).To(Succeed()) + Expect(memberEnv2.Stop()).To(Succeed()) }) From 8af6fcc3e728da0afdb9d91c43bc97c7c8f4fac6 Mon Sep 17 00:00:00 2001 From: Zhiying Lin <54013513+zhiying-lin@users.noreply.github.com> Date: Tue, 23 Sep 2025 10:33:21 +0800 Subject: [PATCH 5/8] feat: update mc webhook to support skipping validation (#253) Signed-off-by: Zhiying Lin --- .../membercluster_validating_webhook.go | 12 +++- test/e2e/join_and_leave_test.go | 58 ++++++++++++++++++- test/e2e/utils_test.go | 12 ++++ 3 files changed, 80 insertions(+), 2 deletions(-) diff --git a/pkg/webhook/membercluster/membercluster_validating_webhook.go b/pkg/webhook/membercluster/membercluster_validating_webhook.go index e4dd958dd..24df38cdd 100644 --- a/pkg/webhook/membercluster/membercluster_validating_webhook.go +++ b/pkg/webhook/membercluster/membercluster_validating_webhook.go @@ -42,7 +42,18 @@ func (v *memberClusterValidator) Handle(ctx context.Context, req admission.Reque mcObjectName := types.NamespacedName{Name: req.Name, Namespace: req.Namespace} klog.V(2).InfoS("Validating webhook handling member cluster", "operation", req.Operation, "memberCluster", mcObjectName) + var mc clusterv1beta1.MemberCluster if req.Operation == admissionv1.Delete { // Will reject the requests whenever the serviceExport is not deleted + if err := v.decoder.DecodeRaw(req.OldObject, &mc); err != nil { + klog.ErrorS(err, "Failed to decode member cluster object for validating fields", "userName", req.UserInfo.Username, "groups", req.UserInfo.Groups) + return admission.Errored(http.StatusBadRequest, err) + } + + if mc.Spec.DeleteOptions != nil && mc.Spec.DeleteOptions.ValidationMode == clusterv1beta1.DeleteValidationModeSkip { + klog.V(2).InfoS("Skipping validation for member cluster DELETE", "memberCluster", mcObjectName) + return admission.Allowed("Skipping validation for member cluster DELETE") + } + klog.V(2).InfoS("Validating webhook member cluster DELETE", "memberCluster", mcObjectName) namespaceName := fmt.Sprintf(utils.NamespaceNameFormat, mcObjectName.Name) internalServiceExportList := &fleetnetworkingv1alpha1.InternalServiceExportList{} @@ -59,7 +70,6 @@ func (v *memberClusterValidator) Handle(ctx context.Context, req admission.Reque return admission.Allowed("Member cluster is ready to leave") } - var mc clusterv1beta1.MemberCluster if err := v.decoder.Decode(req, &mc); err != nil { klog.ErrorS(err, "Failed to decode member cluster object for validating fields", "userName", req.UserInfo.Username, "groups", req.UserInfo.Groups) return admission.Errored(http.StatusBadRequest, err) diff --git a/test/e2e/join_and_leave_test.go b/test/e2e/join_and_leave_test.go index 7cb4fab88..c7f00259b 100644 --- a/test/e2e/join_and_leave_test.go +++ b/test/e2e/join_and_leave_test.go @@ -330,7 +330,63 @@ var _ = Describe("Test member cluster join and leave flow", Label("joinleave"), } }) - // Skip the serviceExport check + It("create a dummy internalServiceExport in the reserved member namespace", func() { + for idx := range allMemberClusterNames { + memberCluster := allMemberClusters[idx] + namespaceName := fmt.Sprintf(utils.NamespaceNameFormat, memberCluster.ClusterName) + internalServiceExport := &fleetnetworkingv1alpha1.InternalServiceExport{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespaceName, + Name: internalServiceExportName, + }, + Spec: fleetnetworkingv1alpha1.InternalServiceExportSpec{ + ServiceReference: fleetnetworkingv1alpha1.ExportedObjectReference{ + NamespacedName: "test-namespace/test-svc", + ClusterID: memberCluster.ClusterName, + Kind: "Service", + Namespace: "test-namespace", + Name: "test-svc", + ResourceVersion: "0", + UID: "00000000-0000-0000-0000-000000000000", + }, + Ports: []fleetnetworkingv1alpha1.ServicePort{ + { + Protocol: corev1.ProtocolTCP, + Port: 4848, + }, + }, + }, + } + Expect(hubClient.Create(ctx, internalServiceExport)).To(Succeed(), "Failed to create internalServiceExport") + } + }) + + It("Should fail the unjoin requests", func() { + for idx := range allMemberClusters { + memberCluster := allMemberClusters[idx] + mcObj := &clusterv1beta1.MemberCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: memberCluster.ClusterName, + }, + } + err := hubClient.Delete(ctx, mcObj) + Expect(err).ShouldNot(Succeed(), "Want the deletion to be denied") + var statusErr *apierrors.StatusError + Expect(errors.As(err, &statusErr)).To(BeTrue(), fmt.Sprintf("Delete memberCluster call produced error %s. Error type wanted is %s.", reflect.TypeOf(err), reflect.TypeOf(&apierrors.StatusError{}))) + Expect(statusErr.ErrStatus.Message).Should(MatchRegexp("Please delete serviceExport test-namespace/test-svc in the member cluster before leaving, request is denied")) + } + }) + + It("Updating the member cluster to skip validation", func() { + for idx := range allMemberClusterNames { + memberCluster := allMemberClusters[idx] + deleteOptions := &clusterv1beta1.DeleteOptions{ + ValidationMode: clusterv1beta1.DeleteValidationModeSkip, + } + updateMemberClusterDeleteOptions(memberCluster.ClusterName, deleteOptions) + } + }) + It("Should be able to trigger the member cluster DELETE", func() { setAllMemberClustersToLeave() }) diff --git a/test/e2e/utils_test.go b/test/e2e/utils_test.go index 685af43a6..c0791e805 100644 --- a/test/e2e/utils_test.go +++ b/test/e2e/utils_test.go @@ -86,6 +86,18 @@ func createMemberCluster(name, svcAccountName string, labels, annotations map[st Expect(hubClient.Create(ctx, mcObj)).To(Succeed(), "Failed to create member cluster object %s", name) } +func updateMemberClusterDeleteOptions(name string, deleteOptions *clusterv1beta1.DeleteOptions) { + Eventually(func() error { + mcObj := &clusterv1beta1.MemberCluster{} + if err := hubClient.Get(ctx, types.NamespacedName{Name: name}, mcObj); err != nil { + return err + } + + mcObj.Spec.DeleteOptions = deleteOptions + return hubClient.Update(ctx, mcObj) + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update member cluster delete options") +} + // markMemberClusterAsHealthy marks the specified member cluster as healthy. func markMemberClusterAsHealthy(name string) { Eventually(func() error { From 0ead4d43f753c94b24c29f299a5185002647c55b Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Tue, 23 Sep 2025 22:03:29 +1000 Subject: [PATCH 6/8] feat: add support for degraded diff drift calculation (#209) --- pkg/controllers/workapplier/apply.go | 8 +- pkg/controllers/workapplier/controller.go | 14 +- .../controller_integration_test.go | 734 ++++++++++++++++++ .../workapplier/controller_test.go | 38 + .../workapplier/drift_detection_takeover.go | 61 +- .../drift_detection_takeover_test.go | 26 +- pkg/controllers/workapplier/process.go | 52 +- pkg/controllers/workapplier/status.go | 11 + pkg/controllers/workapplier/suite_test.go | 3 + 9 files changed, 907 insertions(+), 40 deletions(-) diff --git a/pkg/controllers/workapplier/apply.go b/pkg/controllers/workapplier/apply.go index 4e447e4bf..d889cbe8f 100644 --- a/pkg/controllers/workapplier/apply.go +++ b/pkg/controllers/workapplier/apply.go @@ -320,8 +320,12 @@ func (r *Reconciler) serverSideApply( Resource(*gvr).Namespace(manifestObj.GetNamespace()). Apply(ctx, manifestObj.GetName(), manifestObj, applyOpts) if err != nil { - wrappedErr := controller.NewAPIServerError(false, err) - return nil, fmt.Errorf("failed to apply the manifest object: %w", wrappedErr) + // Note (chenyu1): the current implementation of NewAPIServerError flattens the passed-in error, + // effectively dropping the error hierarchy. This can be an issue if the caller of the function + // needs to identify a specific error in the hierarchy; to avoid this, this method will not return + // any error wrapped by NewAPIServerError. + _ = controller.NewAPIServerError(false, err) + return nil, fmt.Errorf("failed to apply the manifest object: an error is returned by the API server: %w", err) } return appliedObj, nil } diff --git a/pkg/controllers/workapplier/controller.go b/pkg/controllers/workapplier/controller.go index 2aa30bd21..95bda4311 100644 --- a/pkg/controllers/workapplier/controller.go +++ b/pkg/controllers/workapplier/controller.go @@ -276,6 +276,7 @@ const ( ApplyOrReportDiffResTypeNotTakenOver ManifestProcessingApplyOrReportDiffResultType = "NotTakenOver" ApplyOrReportDiffResTypeFailedToRunDriftDetection ManifestProcessingApplyOrReportDiffResultType = "FailedToRunDriftDetection" ApplyOrReportDiffResTypeFoundDrifts ManifestProcessingApplyOrReportDiffResultType = "FoundDrifts" + ApplyOrReportDiffResTypeFoundDriftsInDegradedMode ManifestProcessingApplyOrReportDiffResultType = "FoundDriftsInDegradedMode" // Note that the reason string below uses the same value as kept in the old work applier. ApplyOrReportDiffResTypeFailedToApply ManifestProcessingApplyOrReportDiffResultType = "ManifestApplyFailed" @@ -297,15 +298,17 @@ const ( ApplyOrReportDiffResTypeFailedToReportDiff ManifestProcessingApplyOrReportDiffResultType = "FailedToReportDiff" // The result type for successful diff reportings. - ApplyOrReportDiffResTypeFoundDiff ManifestProcessingApplyOrReportDiffResultType = "FoundDiff" - ApplyOrReportDiffResTypeNoDiffFound ManifestProcessingApplyOrReportDiffResultType = "NoDiffFound" + ApplyOrReportDiffResTypeFoundDiff ManifestProcessingApplyOrReportDiffResultType = "FoundDiff" + ApplyOrReportDiffResTypeFoundDiffInDegradedMode ManifestProcessingApplyOrReportDiffResultType = "FoundDiffInDegradedMode" + ApplyOrReportDiffResTypeNoDiffFound ManifestProcessingApplyOrReportDiffResultType = "NoDiffFound" ) const ( // The descriptions for different diff reporting result types. - ApplyOrReportDiffResTypeFailedToReportDiffDescription = "Failed to report the diff between the hub cluster and the member cluster (error = %s)" - ApplyOrReportDiffResTypeNoDiffFoundDescription = "No diff has been found between the hub cluster and the member cluster" - ApplyOrReportDiffResTypeFoundDiffDescription = "Diff has been found between the hub cluster and the member cluster" + ApplyOrReportDiffResTypeFailedToReportDiffDescription = "Failed to report the diff between the hub cluster and the member cluster (error = %s)" + ApplyOrReportDiffResTypeNoDiffFoundDescription = "No diff has been found between the hub cluster and the member cluster" + ApplyOrReportDiffResTypeFoundDiffDescription = "Diff has been found between the hub cluster and the member cluster" + ApplyOrReportDiffResTypeFoundDiffInDegradedModeDescription = "Diff has been found in degraded mode: cannot perform partial comparison as the member cluster API server rejected the manifest object (object is invalid)" ) var ( @@ -319,6 +322,7 @@ var ( ApplyOrReportDiffResTypeNotTakenOver, ApplyOrReportDiffResTypeFailedToRunDriftDetection, ApplyOrReportDiffResTypeFoundDrifts, + ApplyOrReportDiffResTypeFoundDriftsInDegradedMode, ApplyOrReportDiffResTypeFailedToApply, ApplyOrReportDiffResTypeAppliedWithFailedDriftDetection, ApplyOrReportDiffResTypeApplied, diff --git a/pkg/controllers/workapplier/controller_integration_test.go b/pkg/controllers/workapplier/controller_integration_test.go index e76537c39..838c84786 100644 --- a/pkg/controllers/workapplier/controller_integration_test.go +++ b/pkg/controllers/workapplier/controller_integration_test.go @@ -27,6 +27,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -281,6 +282,60 @@ func regularDeploymentObjectAppliedActual(nsName, deployName string, appliedWork } } +func regularJobObjectAppliedActual(nsName, jobName string, appliedWorkOwnerRef *metav1.OwnerReference) func() error { + return func() error { + // Retrieve the Job object. + gotJob := &batchv1.Job{} + if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: jobName}, gotJob); err != nil { + return fmt.Errorf("failed to retrieve the Job object: %w", err) + } + + // Check that the Job object has been created as expected. + + // To ignore default values automatically, here the test suite rebuilds the objects. + wantJob := job.DeepCopy() + wantJob.TypeMeta = metav1.TypeMeta{} + wantJob.Namespace = nsName + wantJob.Name = jobName + wantJob.OwnerReferences = []metav1.OwnerReference{ + *appliedWorkOwnerRef, + } + + rebuiltGotJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: gotJob.Namespace, + Name: gotJob.Name, + OwnerReferences: gotJob.OwnerReferences, + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": gotJob.Spec.Template.ObjectMeta.Labels["app"], + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: gotJob.Spec.Template.Spec.Containers[0].Name, + Image: gotJob.Spec.Template.Spec.Containers[0].Image, + Command: gotJob.Spec.Template.Spec.Containers[0].Command, + }, + }, + RestartPolicy: gotJob.Spec.Template.Spec.RestartPolicy, + }, + }, + Parallelism: gotJob.Spec.Parallelism, + Completions: gotJob.Spec.Completions, + }, + } + if diff := cmp.Diff(rebuiltGotJob, wantJob); diff != "" { + return fmt.Errorf("job diff (-got +want):\n%s", diff) + } + return nil + } +} + func regularClusterRoleObjectAppliedActual(clusterRoleName string, appliedWorkOwnerRef *metav1.OwnerReference) func() error { return func() error { // Retrieve the ClusterRole object. @@ -629,6 +684,22 @@ func regularDeployNotRemovedActual(nsName, deployName string) func() error { } } +func regularJobNotRemovedActual(nsName, jobName string) func() error { + return func() error { + // Retrieve the Job object. + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: nsName, + Name: jobName, + }, + } + if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: jobName}, job); err != nil { + return fmt.Errorf("failed to retrieve the Job object: %w", err) + } + return nil + } +} + var _ = Describe("applying manifests", func() { Context("apply new manifests (regular)", Ordered, func() { workName := fmt.Sprintf(workNameTemplate, utils.RandStr()) @@ -3676,6 +3747,403 @@ var _ = Describe("drift detection and takeover", func() { }) }) + // Note (chenyu1): this test case is built upon the mutable scheduling directives + // feature that is enabled in Kubernetes since version 1.27. This feature is designed + // specifically for cloud-native queue implementations, which allow them to + // fine-tune how Job pods are scheduled by modifying certain scheduling-related + // fields when the Job is just created in the suspended state. Without this feature + // we wouldn't be able to introduce drifts to Job objects once they are created. + Context("detect drifts (apply if no drift, drift occurred, partial comparison, degraded mode)", Ordered, func() { + workName := fmt.Sprintf(workNameTemplate, utils.RandStr()) + // The environment prepared by the envtest package does not support namespace + // deletion; each test case would use a new namespace. + nsName := fmt.Sprintf(nsNameTemplate, utils.RandStr()) + + var appliedWorkOwnerRef *metav1.OwnerReference + var regularNS *corev1.Namespace + var regularJob *batchv1.Job + + BeforeAll(func() { + // Prepare a NS object. + regularNS = ns.DeepCopy() + regularNS.Name = nsName + regularNSJSON := marshalK8sObjJSON(regularNS) + + // Prepare a Job object. + regularJob = job.DeepCopy() + regularJob.Namespace = nsName + regularJob.Name = jobName + regularJob.Spec.Suspend = ptr.To(true) + regularJob.Spec.Template.Labels[dummyLabelKey] = dummyLabelValue1 + regularJobJSON := marshalK8sObjJSON(regularJob) + + // Create a new Work object with all the manifest JSONs and proper apply strategy. + applyStrategy := &fleetv1beta1.ApplyStrategy{ + ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, + WhenToApply: fleetv1beta1.WhenToApplyTypeIfNotDrifted, + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeAlways, + } + createWorkObject(workName, applyStrategy, regularNSJSON, regularJobJSON) + }) + + It("should add cleanup finalizer to the Work object", func() { + finalizerAddedActual := workFinalizerAddedActual(workName) + Eventually(finalizerAddedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to add cleanup finalizer to the Work object") + }) + + It("should prepare an AppliedWork object", func() { + appliedWorkCreatedActual := appliedWorkCreatedActual(workName) + Eventually(appliedWorkCreatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to prepare an AppliedWork object") + + appliedWorkOwnerRef = prepareAppliedWorkOwnerRef(workName) + }) + + It("should update the Work object status", func() { + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAppliedReason, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: condition.WorkNotTrackableReason, + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeApplied), + ObservedGeneration: 0, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(AvailabilityResultTypeAvailable), + ObservedGeneration: 0, + }, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "batch", + Version: "v1", + Kind: "Job", + Resource: "jobs", + Name: jobName, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeApplied), + ObservedGeneration: 1, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(AvailabilityResultTypeNotTrackable), + ObservedGeneration: 1, + }, + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, nil, nil) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update work status") + }) + + It("should apply all manifests", func() { + // Ensure that the NS object has been applied as expected. + regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) + Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the namespace object") + + Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + + // Ensure that the Job object has been applied as expected. + regularJobObjectAppliedActual := regularJobObjectAppliedActual(nsName, jobName, appliedWorkOwnerRef) + Eventually(regularJobObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the job object") + + Expect(memberClient.Get(ctx, client.ObjectKey{Name: jobName, Namespace: nsName}, regularJob)).To(Succeed(), "Failed to retrieve the Job object") + }) + + It("should update the AppliedWork object status", func() { + // Prepare the status information. + appliedResourceMeta := []fleetv1beta1.AppliedResourceMeta{ + { + WorkResourceIdentifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + UID: regularNS.UID, + }, + { + WorkResourceIdentifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "batch", + Version: "v1", + Kind: "Job", + Resource: "jobs", + Name: jobName, + Namespace: nsName, + }, + UID: regularJob.UID, + }, + } + + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update appliedWork status") + }) + + It("can update the job object directly on the member cluster side", func() { + // Update the labels in the pod template. + // + // This is only possible when the Job is just created in the suspended state. + Expect(memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: jobName}, regularJob)).To(Succeed(), "Failed to retrieve the Job object") + + // Use an Eventually block to guard transient errors. + Eventually(func() error { + regularJob.Spec.Template.Labels[dummyLabelKey] = dummyLabelValue2 + return memberClient.Update(ctx, regularJob) + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update the Job object") + + // Unsuspend the Job object. This would make the pod template immutable. + Eventually(func() error { + regularJob.Spec.Suspend = ptr.To(false) + return memberClient.Update(ctx, regularJob) + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to unsuspend the Job object") + }) + + It("should update the Work object status", func() { + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: condition.WorkNotAllManifestsAppliedReason, + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeApplied), + ObservedGeneration: 0, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(AvailabilityResultTypeAvailable), + ObservedGeneration: 0, + }, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "batch", + Version: "v1", + Kind: "Job", + Resource: "jobs", + Name: jobName, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ApplyOrReportDiffResTypeFoundDriftsInDegradedMode), + ObservedGeneration: regularJob.Generation, + }, + }, + DriftDetails: &fleetv1beta1.DriftDetails{ + ObservedInMemberClusterGeneration: regularJob.Generation, + ObservedDrifts: []fleetv1beta1.PatchDetail{ + { + Path: "/spec/template/metadata/labels/foo", + ValueInMember: dummyLabelValue2, + ValueInHub: dummyLabelValue1, + }, + }, + }, + }, + } + + // Use custom status comparison logic as in this test case drift calculation is expected + // to run in degraded mode, which includes additional dynamic output that need to be + // filtered out. + Eventually(func() error { + // Retrieve the Work object. + work := &fleetv1beta1.Work{} + if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName}, work); err != nil { + return fmt.Errorf("failed to retrieve the Work object: %w", err) + } + + // Prepare the expected Work object status. + + // Update the conditions with the observed generation. + // + // Note that the observed generation of a manifest condition is that of an applied + // resource, not that of the Work object. + for idx := range workConds { + workConds[idx].ObservedGeneration = work.Generation + } + wantWorkStatus := fleetv1beta1.WorkStatus{ + Conditions: workConds, + ManifestConditions: manifestConds, + } + + if len(work.Status.ManifestConditions) == 2 && work.Status.ManifestConditions[1].DriftDetails != nil { + println(fmt.Sprintf("see me:\n%+v", work.Status.ManifestConditions[1].DriftDetails.ObservedDrifts)) + } + // Check that the Work object status has been updated as expected. + if diff := cmp.Diff( + work.Status, wantWorkStatus, + ignoreFieldConditionLTTMsg, + ignoreDiffDetailsObsTime, ignoreDriftDetailsObsTime, + cmpopts.SortSlices(lessFuncPatchDetail), + cmpopts.IgnoreSliceElements(func(d fleetv1beta1.PatchDetail) bool { + return d.Path != "/spec/template/metadata/labels/foo" + }), + ); diff != "" { + return fmt.Errorf("work status diff (-got, +want):\n%s", diff) + } + + // For simplicity reasons, the diff timestamps are not checked. + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update work status") + }) + + It("should not apply the job manifest", func() { + // Ensure that the changes made before have not been overwritten. + Consistently(func() error { + // Retrieve the Job object. + gotJob := &batchv1.Job{} + if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: jobName}, gotJob); err != nil { + return fmt.Errorf("failed to retrieve the Job object: %w", err) + } + + // Check that the Job object has been created as expected. + + // To ignore default values automatically, here the test suite rebuilds the objects. + wantJob := job.DeepCopy() + wantJob.TypeMeta = metav1.TypeMeta{} + wantJob.Namespace = nsName + wantJob.Name = jobName + wantJob.OwnerReferences = []metav1.OwnerReference{ + *appliedWorkOwnerRef, + } + wantJob.Spec.Template.Labels[dummyLabelKey] = dummyLabelValue2 + wantJob.Spec.Suspend = ptr.To(false) + + rebuiltGotJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: gotJob.Namespace, + Name: gotJob.Name, + OwnerReferences: gotJob.OwnerReferences, + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": gotJob.Spec.Template.ObjectMeta.Labels["app"], + dummyLabelKey: gotJob.Spec.Template.ObjectMeta.Labels[dummyLabelKey], + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: gotJob.Spec.Template.Spec.Containers[0].Name, + Image: gotJob.Spec.Template.Spec.Containers[0].Image, + Command: gotJob.Spec.Template.Spec.Containers[0].Command, + }, + }, + RestartPolicy: gotJob.Spec.Template.Spec.RestartPolicy, + }, + }, + Suspend: gotJob.Spec.Suspend, + Parallelism: gotJob.Spec.Parallelism, + Completions: gotJob.Spec.Completions, + }, + } + if diff := cmp.Diff(rebuiltGotJob, wantJob); diff != "" { + return fmt.Errorf("job diff (-got +want):\n%s", diff) + } + return nil + }, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to leave the Job object alone") + + Expect(memberClient.Get(ctx, client.ObjectKey{Name: jobName, Namespace: nsName}, regularJob)).To(Succeed(), "Failed to retrieve the Job object") + }) + + It("should update the AppliedWork object status", func() { + // Prepare the status information. + appliedResourceMeta := []fleetv1beta1.AppliedResourceMeta{ + { + WorkResourceIdentifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + UID: regularNS.UID, + }, + } + + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update appliedWork status") + }) + + AfterAll(func() { + // Delete the Work object and related resources. + deleteWorkObject(workName) + + // Ensure that the Job object has been left alone. + jobNotRemovedActual := regularJobNotRemovedActual(nsName, jobName) + Consistently(jobNotRemovedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to remove the job object") + + // Ensure that the AppliedWork object has been removed. + appliedWorkRemovedActual := appliedWorkRemovedActual(workName, nsName) + Eventually(appliedWorkRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the AppliedWork object") + + workRemovedActual := workRemovedActual(workName) + Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") + + // The environment prepared by the envtest package does not support namespace + // deletion; consequently this test suite would not attempt so verify its deletion. + }) + }) + // For simplicity reasons, this test case will only involve a NS object. Context("detect drifts (apply if no drift, drift occurred, full comparison)", Ordered, func() { workName := fmt.Sprintf(workNameTemplate, utils.RandStr()) @@ -5781,6 +6249,272 @@ var _ = Describe("report diff", func() { // deletion; consequently this test suite would not attempt so verify its deletion. }) }) + + Context("report diff only (partial comparison, degraded)", Ordered, func() { + workName := fmt.Sprintf(workNameTemplate, utils.RandStr()) + // The environment prepared by the envtest package does not support namespace + // deletion; each test case would use a new namespace. + nsName := fmt.Sprintf(nsNameTemplate, utils.RandStr()) + + //var appliedWorkOwnerRef *metav1.OwnerReference + var regularNS *corev1.Namespace + var regularJob *batchv1.Job + + BeforeAll(func() { + // Prepare a NS object. + regularNS = ns.DeepCopy() + regularNS.Name = nsName + regularNSJSON := marshalK8sObjJSON(regularNS) + + // Prepare a Job object. + regularJob = job.DeepCopy() + regularJob.Namespace = nsName + regularJob.Name = jobName + + // Create the objects first in the member cluster. + Expect(memberClient.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") + Expect(memberClient.Create(ctx, regularJob)).To(Succeed(), "Failed to create the Job object") + + // Update the values on the hub cluster side so that diffs will be found. + updatedJob := job.DeepCopy() + updatedJob.Namespace = nsName + updatedJob.Name = jobName + // `.spec.completions` is an immutable field in Job objects. + updatedJob.Spec.Completions = ptr.To(int32(3)) + // `.spec.template` is an immutable field in Job objects. + updatedJob.Spec.Template.Spec.Containers[0].Image = "busybox:v0.0.1" + updatedJSONJSON := marshalK8sObjJSON(updatedJob) + + // Create a new Work object with all the manifest JSONs and proper apply strategy. + applyStrategy := &fleetv1beta1.ApplyStrategy{ + ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, + Type: fleetv1beta1.ApplyStrategyTypeReportDiff, + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeNever, + } + createWorkObject(workName, applyStrategy, regularNSJSON, updatedJSONJSON) + }) + + It("should add cleanup finalizer to the Work object", func() { + finalizerAddedActual := workFinalizerAddedActual(workName) + Eventually(finalizerAddedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to add cleanup finalizer to the Work object") + }) + + It("should prepare an AppliedWork object", func() { + appliedWorkCreatedActual := appliedWorkCreatedActual(workName) + Eventually(appliedWorkCreatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to prepare an AppliedWork object") + + appliedWorkOwnerRef = prepareAppliedWorkOwnerRef(workName) + }) + + It("should update the Work object status", func() { + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionFalse, + Reason: condition.WorkNotAllManifestsDiffReportedReason, + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeNoDiffFound), + ObservedGeneration: 0, + }, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "batch", + Version: "v1", + Kind: "Job", + Resource: "jobs", + Name: jobName, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeFoundDiffInDegradedMode), + ObservedGeneration: 1, + }, + }, + DiffDetails: &fleetv1beta1.DiffDetails{ + ObservedInMemberClusterGeneration: ®ularJob.Generation, + ObservedDiffs: []fleetv1beta1.PatchDetail{ + { + Path: "/spec/completions", + ValueInMember: "2", + ValueInHub: "3", + }, + { + Path: "/spec/template/spec/containers/0/image", + ValueInMember: "busybox", + ValueInHub: "busybox:v0.0.1", + }, + }, + }, + }, + } + + // Use custom status comparison logic as in this test case diff calculation is expected + // to run in degraded mode, which includes additional dynamic output that need to be + // filtered out. + Eventually(func() error { + // Retrieve the Work object. + work := &fleetv1beta1.Work{} + if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName}, work); err != nil { + return fmt.Errorf("failed to retrieve the Work object: %w", err) + } + + // Prepare the expected Work object status. + + // Update the conditions with the observed generation. + // + // Note that the observed generation of a manifest condition is that of an applied + // resource, not that of the Work object. + for idx := range workConds { + workConds[idx].ObservedGeneration = work.Generation + } + wantWorkStatus := fleetv1beta1.WorkStatus{ + Conditions: workConds, + ManifestConditions: manifestConds, + } + + // Check that the Work object status has been updated as expected. + if diff := cmp.Diff( + work.Status, wantWorkStatus, + ignoreFieldConditionLTTMsg, + ignoreDiffDetailsObsTime, ignoreDriftDetailsObsTime, + cmpopts.SortSlices(lessFuncPatchDetail), + cmpopts.IgnoreSliceElements(func(d fleetv1beta1.PatchDetail) bool { + return d.Path != "/spec/completions" && d.Path != "/spec/template/spec/containers/0/image" + }), + ); diff != "" { + return fmt.Errorf("work status diff (-got, +want):\n%s", diff) + } + + // For simplicity reasons, the diff timestamps are not checked. + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update work status") + }) + + It("should not own the objects or apply the manifests to them", func() { + Consistently(func() error { + // Retrieve the NS object. + updatedNS := &corev1.Namespace{} + if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { + return fmt.Errorf("failed to retrieve the NS object: %w", err) + } + + // Rebuild the NS object to ignore default values automatically. + rebuiltGotNS := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: updatedNS.Name, + OwnerReferences: updatedNS.OwnerReferences, + }, + } + + wantNS := ns.DeepCopy() + wantNS.Name = nsName + if diff := cmp.Diff(rebuiltGotNS, wantNS, ignoreFieldTypeMetaInNamespace); diff != "" { + return fmt.Errorf("namespace diff (-got +want):\n%s", diff) + } + return nil + }, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to leave the NS object alone") + + Consistently(func() error { + // Retrieve the Job object. + updatedJob := &batchv1.Job{} + if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: jobName}, updatedJob); err != nil { + return fmt.Errorf("failed to retrieve the Job object: %w", err) + } + + // Rebuild the Job object to ignore default values automatically. + if len(updatedJob.Spec.Template.Spec.Containers) != 1 { + return fmt.Errorf("unexpected number of containers in the Job pod template spec: %d, want 1", len(updatedJob.Spec.Template.Spec.Containers)) + } + rebuiltGotJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: updatedJob.Namespace, + Name: updatedJob.Name, + OwnerReferences: updatedJob.OwnerReferences, + }, + Spec: batchv1.JobSpec{ + Parallelism: updatedJob.Spec.Parallelism, + Completions: updatedJob.Spec.Completions, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": updatedJob.Spec.Template.ObjectMeta.Labels["app"], + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: updatedJob.Spec.Template.Spec.Containers[0].Name, + Image: updatedJob.Spec.Template.Spec.Containers[0].Image, + Command: updatedJob.Spec.Template.Spec.Containers[0].Command, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + }, + }, + } + + wantJob := job.DeepCopy() + wantJob.TypeMeta = metav1.TypeMeta{} + wantJob.Namespace = nsName + wantJob.Name = jobName + + if diff := cmp.Diff(rebuiltGotJob, wantJob); diff != "" { + return fmt.Errorf("job diff (-got +want):\n%s", diff) + } + return nil + }, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to leave the Job object alone") + }) + + It("should have no applied object reportings in the AppliedWork status", func() { + // Prepare the status information. + var appliedResourceMeta []fleetv1beta1.AppliedResourceMeta + + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update appliedWork status") + }) + + AfterAll(func() { + // Delete the Work object and related resources. + deleteWorkObject(workName) + + // Ensure that the Job object has been left alone. + jobNotRemovedActual := regularJobNotRemovedActual(nsName, jobName) + Consistently(jobNotRemovedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to remove the job object") + + // Ensure that the AppliedWork object has been removed. + appliedWorkRemovedActual := appliedWorkRemovedActual(workName, nsName) + Eventually(appliedWorkRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the AppliedWork object") + + workRemovedActual := workRemovedActual(workName) + Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") + + // The environment prepared by the envtest package does not support namespace + // deletion; consequently this test suite would not attempt so verify its deletion. + }) + }) }) var _ = Describe("handling different apply strategies", func() { diff --git a/pkg/controllers/workapplier/controller_test.go b/pkg/controllers/workapplier/controller_test.go index 4cb8c8903..c94463ccf 100644 --- a/pkg/controllers/workapplier/controller_test.go +++ b/pkg/controllers/workapplier/controller_test.go @@ -25,6 +25,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -43,6 +44,7 @@ const ( workName = "work-1" deployName = "deploy-1" + jobName = "job-1" configMapName = "configmap-1" nsName = "ns-1" clusterRoleName = "clusterrole-1" @@ -96,6 +98,42 @@ var ( deployUnstructured *unstructured.Unstructured deployJSON []byte + job = &batchv1.Job{ + TypeMeta: metav1.TypeMeta{ + Kind: "Job", + APIVersion: "batch/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: jobName, + Namespace: nsName, + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "busybox", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "busybox", + Image: "busybox", + Command: []string{ + "sh", + "-c", + "sleep 60", + }, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + }, + Parallelism: ptr.To(int32(1)), + Completions: ptr.To(int32(2)), + }, + } + ns = &corev1.Namespace{ TypeMeta: metav1.TypeMeta{ Kind: "Namespace", diff --git a/pkg/controllers/workapplier/drift_detection_takeover.go b/pkg/controllers/workapplier/drift_detection_takeover.go index 38f088a50..530f9de4d 100644 --- a/pkg/controllers/workapplier/drift_detection_takeover.go +++ b/pkg/controllers/workapplier/drift_detection_takeover.go @@ -46,7 +46,7 @@ func (r *Reconciler) takeOverPreExistingObject( manifestObj, inMemberClusterObj *unstructured.Unstructured, applyStrategy *fleetv1beta1.ApplyStrategy, expectedAppliedWorkOwnerRef *metav1.OwnerReference, -) (*unstructured.Unstructured, []fleetv1beta1.PatchDetail, error) { +) (*unstructured.Unstructured, []fleetv1beta1.PatchDetail, bool, error) { inMemberClusterObjCopy := inMemberClusterObj.DeepCopy() existingOwnerRefs := inMemberClusterObjCopy.GetOwnerReferences() @@ -59,7 +59,7 @@ func (r *Reconciler) takeOverPreExistingObject( // removing any owner reference that points to an orphaned AppliedWork object. existingOwnerRefs, err := r.removeLeftBehindAppliedWorkOwnerRefs(ctx, existingOwnerRefs) if err != nil { - return nil, nil, fmt.Errorf("failed to remove left-behind AppliedWork owner references: %w", err) + return nil, nil, false, fmt.Errorf("failed to remove left-behind AppliedWork owner references: %w", err) } // Check this object is already owned by another object (or controller); if so, Fleet will only @@ -69,7 +69,7 @@ func (r *Reconciler) takeOverPreExistingObject( // No takeover will be performed. // // Note that This will be registered as an (apply) error. - return nil, nil, fmt.Errorf("the object is already owned by some other sources(s) and co-ownership is disallowed") + return nil, nil, false, fmt.Errorf("the object is already owned by some other sources(s) and co-ownership is disallowed") } // Check if the object is already owned by Fleet, but the owner is a different AppliedWork @@ -83,19 +83,19 @@ func (r *Reconciler) takeOverPreExistingObject( // user confusion. To address this corner case, Fleet would now deny placing the same object // twice. if isPlacedByFleetInDuplicate(existingOwnerRefs, expectedAppliedWorkOwnerRef) { - return nil, nil, fmt.Errorf("the object is already owned by another Fleet AppliedWork object") + return nil, nil, false, fmt.Errorf("the object is already owned by another Fleet AppliedWork object") } // Check if the takeover action requires additional steps (configuration difference inspection). // // Note that the default takeover action is AlwaysApply. if applyStrategy.WhenToTakeOver == fleetv1beta1.WhenToTakeOverTypeIfNoDiff { - configDiffs, err := r.diffBetweenManifestAndInMemberClusterObjects(ctx, gvr, manifestObj, inMemberClusterObjCopy, applyStrategy.ComparisonOption) + configDiffs, diffCalculatedInDegradedMode, err := r.diffBetweenManifestAndInMemberClusterObjects(ctx, gvr, manifestObj, inMemberClusterObjCopy, applyStrategy.ComparisonOption) switch { case err != nil: - return nil, nil, fmt.Errorf("failed to calculate configuration diffs between the manifest object and the object from the member cluster: %w", err) + return nil, nil, false, fmt.Errorf("failed to calculate configuration diffs between the manifest object and the object from the member cluster: %w", err) case len(configDiffs) > 0: - return nil, configDiffs, nil + return nil, configDiffs, diffCalculatedInDegradedMode, nil } } @@ -107,10 +107,10 @@ func (r *Reconciler) takeOverPreExistingObject( Update(ctx, inMemberClusterObjCopy, metav1.UpdateOptions{}) if err != nil { wrappedErr := controller.NewAPIServerError(false, err) - return nil, nil, fmt.Errorf("failed to take over the object: %w", wrappedErr) + return nil, nil, false, fmt.Errorf("failed to take over the object: %w", wrappedErr) } - return takenOverInMemberClusterObj, nil, nil + return takenOverInMemberClusterObj, nil, false, nil } // diffBetweenManifestAndInMemberClusterObjects calculates the differences between the manifest object @@ -120,16 +120,17 @@ func (r *Reconciler) diffBetweenManifestAndInMemberClusterObjects( gvr *schema.GroupVersionResource, manifestObj, inMemberClusterObj *unstructured.Unstructured, cmpOption fleetv1beta1.ComparisonOptionType, -) ([]fleetv1beta1.PatchDetail, error) { +) ([]fleetv1beta1.PatchDetail, bool, error) { switch cmpOption { case fleetv1beta1.ComparisonOptionTypePartialComparison: return r.partialDiffBetweenManifestAndInMemberClusterObjects(ctx, gvr, manifestObj, inMemberClusterObj) case fleetv1beta1.ComparisonOptionTypeFullComparison: // For the full comparison, Fleet compares directly the JSON representations of the // manifest object and the object in the member cluster. - return preparePatchDetails(manifestObj, inMemberClusterObj) + patchDetails, err := preparePatchDetails(manifestObj, inMemberClusterObj) + return patchDetails, false, err default: - return nil, fmt.Errorf("an invalid comparison option is specified") + return nil, false, fmt.Errorf("an invalid comparison option is specified") } } @@ -141,13 +142,10 @@ func (r *Reconciler) partialDiffBetweenManifestAndInMemberClusterObjects( ctx context.Context, gvr *schema.GroupVersionResource, manifestObj, inMemberClusterObj *unstructured.Unstructured, -) ([]fleetv1beta1.PatchDetail, error) { +) ([]fleetv1beta1.PatchDetail, bool, error) { // Fleet calculates the partial diff between two objects by running apply ops in the dry-run // mode. appliedObj, err := r.applyInDryRunMode(ctx, gvr, manifestObj, inMemberClusterObj) - if err != nil { - return nil, fmt.Errorf("failed to apply the manifest in dry-run mode: %w", err) - } // After the dry-run apply op, all the managed fields should have been overwritten using the // values from the manifest object, while leaving all the unmanaged fields untouched. This @@ -157,8 +155,35 @@ func (r *Reconciler) partialDiffBetweenManifestAndInMemberClusterObjects( // imply that running an actual apply op would lead to unexpected changes, which signifies // the presence of partial drifts (drifts in managed fields). - // Prepare the patch details. - return preparePatchDetails(appliedObj, inMemberClusterObj) + switch { + case err == nil: + // The dry-run apply op has succeeded. All managed fields should have been overwritten using the + // values from the manifest object, while leaving all the unmanaged fields untouched. This + // would allow Fleet to compare the object returned by the dry-run apply op with the object + // that is currently in the member cluster; if all the fields are consistent, it is safe + // for us to assume that there are no drifts, otherwise, any fields that are different + // imply that running an actual apply op would lead to unexpected changes, which signifies + // the presence of partial drifts (drifts in managed fields). + patchDetails, err := preparePatchDetails(appliedObj, inMemberClusterObj) + return patchDetails, false, err + case errors.IsInvalid(err): + // The dry-run apply op has failed as the manifest object provided is not valid. This could + // happen when the apply op involves fields that are immutable or the apply op attempts to + // set invalid values. This error implies that the in-cluster object has already been modified, + // and any change that the user supplies right now cannot be accepted. + // + // In this case, fall back to full comparison. Report that the diff is being calculated in a + // degraded manner. + // + // This is not considered as a diff calculation error. + klog.V(2).InfoS("Calculate diffs in degraded mode as the manifest object cannot be server-side applied in dry-run mode", + "gvr", gvr, "manifestObj", klog.KObj(manifestObj), "serverErr", err) + patchDetails, err := preparePatchDetails(manifestObj, inMemberClusterObj) + return patchDetails, true, err + default: + // An unexpected error has occurred. + return nil, false, fmt.Errorf("failed to apply the manifest in dry-run mode: %w", err) + } } // organizeJSONPatchIntoFleetPatchDetails organizes the JSON patch operations into Fleet patch details. diff --git a/pkg/controllers/workapplier/drift_detection_takeover_test.go b/pkg/controllers/workapplier/drift_detection_takeover_test.go index 4cdc4398d..161b6f685 100644 --- a/pkg/controllers/workapplier/drift_detection_takeover_test.go +++ b/pkg/controllers/workapplier/drift_detection_takeover_test.go @@ -98,16 +98,17 @@ func TestTakeOverPreExistingObject(t *testing.T) { }) testCases := []struct { - name string - gvr *schema.GroupVersionResource - manifestObj *unstructured.Unstructured - inMemberClusterObj *unstructured.Unstructured - workObj *fleetv1beta1.Work - applyStrategy *fleetv1beta1.ApplyStrategy - expectedAppliedWorkOwnerRef *metav1.OwnerReference - wantErred bool - wantTakeOverObj *unstructured.Unstructured - wantPatchDetails []fleetv1beta1.PatchDetail + name string + gvr *schema.GroupVersionResource + manifestObj *unstructured.Unstructured + inMemberClusterObj *unstructured.Unstructured + workObj *fleetv1beta1.Work + applyStrategy *fleetv1beta1.ApplyStrategy + expectedAppliedWorkOwnerRef *metav1.OwnerReference + wantErred bool + wantTakeOverObj *unstructured.Unstructured + wantPatchDetails []fleetv1beta1.PatchDetail + wantDiffCalculatedInDegradedMode bool }{ { name: "existing non-Fleet owner, co-ownership not allowed", @@ -209,7 +210,7 @@ func TestTakeOverPreExistingObject(t *testing.T) { workNameSpace: memberReservedNSName1, } - takenOverObj, patchDetails, err := r.takeOverPreExistingObject( + takenOverObj, patchDetails, diffCalculatedInDegradedMode, err := r.takeOverPreExistingObject( ctx, tc.gvr, tc.manifestObj, tc.inMemberClusterObj, @@ -231,6 +232,9 @@ func TestTakeOverPreExistingObject(t *testing.T) { if diff := cmp.Diff(patchDetails, tc.wantPatchDetails); diff != "" { t.Errorf("patchDetails mismatches (-got, +want):\n%s", diff) } + if diffCalculatedInDegradedMode != tc.wantDiffCalculatedInDegradedMode { + t.Errorf("diffCalculatedInDegradedMode = %v, want %v", diffCalculatedInDegradedMode, tc.wantDiffCalculatedInDegradedMode) + } }) } } diff --git a/pkg/controllers/workapplier/process.go b/pkg/controllers/workapplier/process.go index d58c183c2..1df00762a 100644 --- a/pkg/controllers/workapplier/process.go +++ b/pkg/controllers/workapplier/process.go @@ -235,7 +235,7 @@ func (r *Reconciler) takeOverInMemberClusterObjectIfApplicable( // Take over the object. Note that this steps adds only the owner reference; no other // fields are modified (on the object from the member cluster). - takenOverInMemberClusterObj, configDiffs, err := r.takeOverPreExistingObject(ctx, + takenOverInMemberClusterObj, configDiffs, diffCalculatedInDegradedMode, err := r.takeOverPreExistingObject(ctx, bundle.gvr, bundle.manifestObj, bundle.inMemberClusterObj, work.Spec.ApplyStrategy, expectedAppliedWorkOwnerRef) switch { @@ -247,6 +247,19 @@ func (r *Reconciler) takeOverInMemberClusterObjectIfApplicable( "work", klog.KObj(work), "GVR", *bundle.gvr, "manifestObj", klog.KObj(bundle.manifestObj), "inMemberClusterObj", klog.KObj(bundle.inMemberClusterObj), "expectedAppliedWorkOwnerRef", *expectedAppliedWorkOwnerRef) return true + case len(configDiffs) > 0 && diffCalculatedInDegradedMode: + // Takeover cannot be performed as configuration differences are found between the manifest + // object and the object in the member cluster in degraded mode. + // + // Note that though degraded diff calculation itself is not considered as an error, the + // presence of diffs is indeed an error. + bundle.diffs = configDiffs + bundle.applyOrReportDiffErr = fmt.Errorf("cannot take over object: configuration differences are found between the manifest object and the corresponding object in the member cluster in degraded mode (full comparison is performed instead of partial comparison, as the manifest object is considered to be invalid by the member cluster API server)") + bundle.applyOrReportDiffResTyp = ApplyOrReportDiffResTypeFailedToTakeOver + klog.V(2).InfoS("Cannot take over object as configuration differences are found between the manifest object and the corresponding object in the member cluster in degraded mode", + "work", klog.KObj(work), "GVR", *bundle.gvr, "manifestObj", klog.KObj(bundle.manifestObj), + "expectedAppliedWorkOwnerRef", *expectedAppliedWorkOwnerRef) + return true case len(configDiffs) > 0: // Takeover cannot be performed as configuration differences are found between the manifest // object and the object in the member cluster. @@ -337,7 +350,7 @@ func (r *Reconciler) reportDiffOnlyIfApplicable( // The object has been created in the member cluster; Fleet will calculate the configuration // diffs between the manifest object and the object from the member cluster. - configDiffs, err := r.diffBetweenManifestAndInMemberClusterObjects(ctx, + configDiffs, diffCalculatedInDegradedMode, err := r.diffBetweenManifestAndInMemberClusterObjects(ctx, bundle.gvr, bundle.manifestObj, bundle.inMemberClusterObj, work.Spec.ApplyStrategy.ComparisonOption) @@ -350,6 +363,13 @@ func (r *Reconciler) reportDiffOnlyIfApplicable( "Failed to calculate configuration diffs between the manifest object and the object from the member cluster", "work", klog.KObj(work), "GVR", *bundle.gvr, "manifestObj", klog.KObj(bundle.manifestObj), "inMemberClusterObj", klog.KObj(bundle.inMemberClusterObj), "expectedAppliedWorkOwnerRef", *expectedAppliedWorkOwnerRef) + case len(configDiffs) > 0 && diffCalculatedInDegradedMode: + // Configuration diffs are found in degraded mode. + bundle.diffs = configDiffs + bundle.applyOrReportDiffResTyp = ApplyOrReportDiffResTypeFoundDiffInDegradedMode + klog.V(2).InfoS("Diff report completed; configuration diffs are found in degraded mode", + "GVR", *bundle.gvr, "manifestObj", klog.KObj(bundle.manifestObj), + "work", klog.KObj(work)) case len(configDiffs) > 0: // Configuration diffs are found. bundle.diffs = configDiffs @@ -422,7 +442,7 @@ func (r *Reconciler) performPreApplyDriftDetectionIfApplicable( return false default: // Run the drift detection process. - drifts, err := r.diffBetweenManifestAndInMemberClusterObjects(ctx, + drifts, driftsCalculatedInDegradedMode, err := r.diffBetweenManifestAndInMemberClusterObjects(ctx, bundle.gvr, bundle.manifestObj, bundle.inMemberClusterObj, work.Spec.ApplyStrategy.ComparisonOption) @@ -436,6 +456,18 @@ func (r *Reconciler) performPreApplyDriftDetectionIfApplicable( "work", klog.KObj(work), "GVR", *bundle.gvr, "manifestObj", klog.KObj(bundle.manifestObj), "inMemberClusterObj", klog.KObj(bundle.inMemberClusterObj), "expectedAppliedWorkOwnerRef", *expectedAppliedWorkOwnerRef) return true + case len(drifts) > 0 && driftsCalculatedInDegradedMode: + // Configuration drifts are found in degraded mode. + // + // Note that though degraded drift calculation itself is not considered as an error, the + // presence of drifts in the pre-apply drift detection step is indeed an error. + bundle.drifts = drifts + bundle.applyOrReportDiffErr = fmt.Errorf("cannot apply manifest: drifts are found between the manifest and the object from the member cluster in degraded mode (full comparison is performed instead of partial comparison, as the manifest object is considered to be invalid by the member cluster API server)") + bundle.applyOrReportDiffResTyp = ApplyOrReportDiffResTypeFoundDriftsInDegradedMode + klog.V(2).InfoS("Cannot apply manifest: drifts are found between the manifest and the object from the member cluster in degraded mode", + "work", klog.KObj(work), "GVR", *bundle.gvr, "manifestObj", klog.KObj(bundle.manifestObj), + "inMemberClusterObj", klog.KObj(bundle.inMemberClusterObj), "expectedAppliedWorkOwnerRef", *expectedAppliedWorkOwnerRef) + return true case len(drifts) > 0: // Drifts are found in the pre-apply drift detection process. bundle.drifts = drifts @@ -493,7 +525,7 @@ func (r *Reconciler) performPostApplyDriftDetectionIfApplicable( return false } - drifts, err := r.diffBetweenManifestAndInMemberClusterObjects(ctx, + drifts, driftsCalculatedInDegradedMode, err := r.diffBetweenManifestAndInMemberClusterObjects(ctx, bundle.gvr, bundle.manifestObj, bundle.inMemberClusterObj, work.Spec.ApplyStrategy.ComparisonOption) @@ -509,6 +541,18 @@ func (r *Reconciler) performPostApplyDriftDetectionIfApplicable( "work", klog.KObj(work), "GVR", *bundle.gvr, "manifestObj", klog.KObj(bundle.manifestObj), "inMemberClusterObj", klog.KObj(bundle.inMemberClusterObj), "expectedAppliedWorkOwnerRef", *expectedAppliedWorkOwnerRef) return true + case len(drifts) > 0 && driftsCalculatedInDegradedMode: + // Configuration drifts are found, but they were calculated in degraded mode. + // + // Normally this should never happen, as post apply drift detection will only run if the full + // comparison mode is enabled, but degraded mode only applies to partial comparison mode. + + // Surface the drifts as normal, but raise an unexpected behavior flag. + bundle.drifts = drifts + klog.V(2).InfoS("Post-apply drift detection completed; drifts are found in degraded mode", + "manifestObj", klog.KObj(bundle.manifestObj), "GVR", *bundle.gvr, "work", klog.KObj(work)) + // The presence of such drifts are not considered as an error. + return false case len(drifts) > 0: // Drifts are found in the post-apply drift detection process. bundle.drifts = drifts diff --git a/pkg/controllers/workapplier/status.go b/pkg/controllers/workapplier/status.go index 53145f11b..1a17ba150 100644 --- a/pkg/controllers/workapplier/status.go +++ b/pkg/controllers/workapplier/status.go @@ -429,6 +429,17 @@ func setManifestDiffReportedCondition( Message: ApplyOrReportDiffResTypeFoundDiffDescription, ObservedGeneration: inMemberClusterObjGeneration, } + case applyOrReportDiffResTyp == ApplyOrReportDiffResTypeFoundDiffInDegradedMode: + // Found diffs in degraded mode. + // + // This is not considered as a system error. + diffReportedCond = &metav1.Condition{ + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeFoundDiffInDegradedMode), + Message: ApplyOrReportDiffResTypeFoundDiffInDegradedModeDescription, + ObservedGeneration: inMemberClusterObjGeneration, + } default: // There are cases where the work applier might not be able to complete the diff reporting // due to failures in the pre-processing or processing stage (e.g., the manifest cannot be decoded, diff --git a/pkg/controllers/workapplier/suite_test.go b/pkg/controllers/workapplier/suite_test.go index c61e0d82a..afb9b2ed2 100644 --- a/pkg/controllers/workapplier/suite_test.go +++ b/pkg/controllers/workapplier/suite_test.go @@ -26,6 +26,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/dynamic" @@ -156,6 +157,8 @@ var _ = BeforeSuite(func() { Expect(err).ToNot(HaveOccurred()) Expect(memberCfg2).ToNot(BeNil()) + err = batchv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) err = fleetv1beta1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) err = testv1alpha1.AddToScheme(scheme.Scheme) From deeab22cd2ab604755b54c485633265648540639 Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Fri, 26 Sep 2025 13:24:41 +1000 Subject: [PATCH 7/8] fix: address a minor stale merge inconsistency issue (#256) Minor fixes Signed-off-by: michaelawyu --- apis/cluster/v1beta1/zz_generated.deepcopy.go | 2 +- .../v1alpha1/zz_generated.deepcopy.go | 2 +- .../v1beta1/zz_generated.deepcopy.go | 2 +- apis/v1alpha1/zz_generated.deepcopy.go | 2 +- .../controller_integration_test.go | 38 +++++++++---------- test/apis/v1alpha1/zz_generated.deepcopy.go | 2 +- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/apis/cluster/v1beta1/zz_generated.deepcopy.go b/apis/cluster/v1beta1/zz_generated.deepcopy.go index d0e850ca0..5ce0e1367 100644 --- a/apis/cluster/v1beta1/zz_generated.deepcopy.go +++ b/apis/cluster/v1beta1/zz_generated.deepcopy.go @@ -21,7 +21,7 @@ limitations under the License. package v1beta1 import ( - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/apis/placement/v1alpha1/zz_generated.deepcopy.go b/apis/placement/v1alpha1/zz_generated.deepcopy.go index 6d1656d18..df9f5e6d7 100644 --- a/apis/placement/v1alpha1/zz_generated.deepcopy.go +++ b/apis/placement/v1alpha1/zz_generated.deepcopy.go @@ -22,7 +22,7 @@ package v1alpha1 import ( "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1" - "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" ) diff --git a/apis/placement/v1beta1/zz_generated.deepcopy.go b/apis/placement/v1beta1/zz_generated.deepcopy.go index 0e755b56e..05fef4c3d 100644 --- a/apis/placement/v1beta1/zz_generated.deepcopy.go +++ b/apis/placement/v1beta1/zz_generated.deepcopy.go @@ -21,7 +21,7 @@ limitations under the License. package v1beta1 import ( - "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" ) diff --git a/apis/v1alpha1/zz_generated.deepcopy.go b/apis/v1alpha1/zz_generated.deepcopy.go index 27a862c43..85550ca19 100644 --- a/apis/v1alpha1/zz_generated.deepcopy.go +++ b/apis/v1alpha1/zz_generated.deepcopy.go @@ -22,7 +22,7 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) diff --git a/pkg/controllers/workapplier/controller_integration_test.go b/pkg/controllers/workapplier/controller_integration_test.go index 838c84786..c925b6e95 100644 --- a/pkg/controllers/workapplier/controller_integration_test.go +++ b/pkg/controllers/workapplier/controller_integration_test.go @@ -286,7 +286,7 @@ func regularJobObjectAppliedActual(nsName, jobName string, appliedWorkOwnerRef * return func() error { // Retrieve the Job object. gotJob := &batchv1.Job{} - if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: jobName}, gotJob); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: jobName}, gotJob); err != nil { return fmt.Errorf("failed to retrieve the Job object: %w", err) } @@ -693,7 +693,7 @@ func regularJobNotRemovedActual(nsName, jobName string) func() error { Name: jobName, }, } - if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: jobName}, job); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: jobName}, job); err != nil { return fmt.Errorf("failed to retrieve the Job object: %w", err) } return nil @@ -3783,7 +3783,7 @@ var _ = Describe("drift detection and takeover", func() { WhenToApply: fleetv1beta1.WhenToApplyTypeIfNotDrifted, WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeAlways, } - createWorkObject(workName, applyStrategy, regularNSJSON, regularJobJSON) + createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON, regularJobJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -3873,13 +3873,13 @@ var _ = Describe("drift detection and takeover", func() { regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the namespace object") - Expect(memberClient.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") // Ensure that the Job object has been applied as expected. regularJobObjectAppliedActual := regularJobObjectAppliedActual(nsName, jobName, appliedWorkOwnerRef) Eventually(regularJobObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the job object") - Expect(memberClient.Get(ctx, client.ObjectKey{Name: jobName, Namespace: nsName}, regularJob)).To(Succeed(), "Failed to retrieve the Job object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: jobName, Namespace: nsName}, regularJob)).To(Succeed(), "Failed to retrieve the Job object") }) It("should update the AppliedWork object status", func() { @@ -3918,18 +3918,18 @@ var _ = Describe("drift detection and takeover", func() { // Update the labels in the pod template. // // This is only possible when the Job is just created in the suspended state. - Expect(memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: jobName}, regularJob)).To(Succeed(), "Failed to retrieve the Job object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: jobName}, regularJob)).To(Succeed(), "Failed to retrieve the Job object") // Use an Eventually block to guard transient errors. Eventually(func() error { regularJob.Spec.Template.Labels[dummyLabelKey] = dummyLabelValue2 - return memberClient.Update(ctx, regularJob) + return memberClient1.Update(ctx, regularJob) }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update the Job object") // Unsuspend the Job object. This would make the pod template immutable. Eventually(func() error { regularJob.Spec.Suspend = ptr.To(false) - return memberClient.Update(ctx, regularJob) + return memberClient1.Update(ctx, regularJob) }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to unsuspend the Job object") }) @@ -4004,7 +4004,7 @@ var _ = Describe("drift detection and takeover", func() { Eventually(func() error { // Retrieve the Work object. work := &fleetv1beta1.Work{} - if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName}, work); err != nil { + if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName1}, work); err != nil { return fmt.Errorf("failed to retrieve the Work object: %w", err) } @@ -4048,7 +4048,7 @@ var _ = Describe("drift detection and takeover", func() { Consistently(func() error { // Retrieve the Job object. gotJob := &batchv1.Job{} - if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: jobName}, gotJob); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: jobName}, gotJob); err != nil { return fmt.Errorf("failed to retrieve the Job object: %w", err) } @@ -4101,7 +4101,7 @@ var _ = Describe("drift detection and takeover", func() { return nil }, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to leave the Job object alone") - Expect(memberClient.Get(ctx, client.ObjectKey{Name: jobName, Namespace: nsName}, regularJob)).To(Succeed(), "Failed to retrieve the Job object") + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: jobName, Namespace: nsName}, regularJob)).To(Succeed(), "Failed to retrieve the Job object") }) It("should update the AppliedWork object status", func() { @@ -4126,7 +4126,7 @@ var _ = Describe("drift detection and takeover", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Ensure that the Job object has been left alone. jobNotRemovedActual := regularJobNotRemovedActual(nsName, jobName) @@ -6272,8 +6272,8 @@ var _ = Describe("report diff", func() { regularJob.Name = jobName // Create the objects first in the member cluster. - Expect(memberClient.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") - Expect(memberClient.Create(ctx, regularJob)).To(Succeed(), "Failed to create the Job object") + Expect(memberClient1.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") + Expect(memberClient1.Create(ctx, regularJob)).To(Succeed(), "Failed to create the Job object") // Update the values on the hub cluster side so that diffs will be found. updatedJob := job.DeepCopy() @@ -6291,7 +6291,7 @@ var _ = Describe("report diff", func() { Type: fleetv1beta1.ApplyStrategyTypeReportDiff, WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeNever, } - createWorkObject(workName, applyStrategy, regularNSJSON, updatedJSONJSON) + createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON, updatedJSONJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -6376,7 +6376,7 @@ var _ = Describe("report diff", func() { Eventually(func() error { // Retrieve the Work object. work := &fleetv1beta1.Work{} - if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName}, work); err != nil { + if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName1}, work); err != nil { return fmt.Errorf("failed to retrieve the Work object: %w", err) } @@ -6416,7 +6416,7 @@ var _ = Describe("report diff", func() { Consistently(func() error { // Retrieve the NS object. updatedNS := &corev1.Namespace{} - if err := memberClient.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { return fmt.Errorf("failed to retrieve the NS object: %w", err) } @@ -6439,7 +6439,7 @@ var _ = Describe("report diff", func() { Consistently(func() error { // Retrieve the Job object. updatedJob := &batchv1.Job{} - if err := memberClient.Get(ctx, client.ObjectKey{Namespace: nsName, Name: jobName}, updatedJob); err != nil { + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: jobName}, updatedJob); err != nil { return fmt.Errorf("failed to retrieve the Job object: %w", err) } @@ -6498,7 +6498,7 @@ var _ = Describe("report diff", func() { AfterAll(func() { // Delete the Work object and related resources. - deleteWorkObject(workName) + deleteWorkObject(workName, memberReservedNSName1) // Ensure that the Job object has been left alone. jobNotRemovedActual := regularJobNotRemovedActual(nsName, jobName) diff --git a/test/apis/v1alpha1/zz_generated.deepcopy.go b/test/apis/v1alpha1/zz_generated.deepcopy.go index 081bec913..143bdee7b 100644 --- a/test/apis/v1alpha1/zz_generated.deepcopy.go +++ b/test/apis/v1alpha1/zz_generated.deepcopy.go @@ -21,7 +21,7 @@ limitations under the License. package v1alpha1 import ( - "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) From fafe73e085c5af09792eb37d3961b368a6a2a512 Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Wed, 1 Oct 2025 15:48:03 +1000 Subject: [PATCH 8/8] feat: security: obscure drift/diff reportings on secret object data fields (re-submission) (#258) --- .../controller_integration_test.go | 2255 ++++++++++++----- .../workapplier/controller_test.go | 15 + .../workapplier/drift_detection_takeover.go | 47 + .../drift_detection_takeover_test.go | 148 ++ 4 files changed, 1887 insertions(+), 578 deletions(-) diff --git a/pkg/controllers/workapplier/controller_integration_test.go b/pkg/controllers/workapplier/controller_integration_test.go index c925b6e95..7841ad72a 100644 --- a/pkg/controllers/workapplier/controller_integration_test.go +++ b/pkg/controllers/workapplier/controller_integration_test.go @@ -403,6 +403,40 @@ func regularConfigMapObjectAppliedActual(nsName, configMapName string, appliedWo } } +func regularSecretObjectAppliedActual(nsName, secretName string, appliedWorkOwnerRef *metav1.OwnerReference) func() error { + return func() error { + // Retrieve the Secret object. + gotSecret := &corev1.Secret{} + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: secretName}, gotSecret); err != nil { + return fmt.Errorf("failed to retrieve the Secret object: %w", err) + } + + // Check that the Secret object has been created as expected. + + // To ignore default values automatically, here the test suite rebuilds the objects. + wantSecret := secret.DeepCopy() + wantSecret.TypeMeta = metav1.TypeMeta{} + wantSecret.Namespace = nsName + wantSecret.Name = secretName + wantSecret.OwnerReferences = []metav1.OwnerReference{ + *appliedWorkOwnerRef, + } + + rebuiltGotSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: gotSecret.Name, + Namespace: gotSecret.Namespace, + OwnerReferences: gotSecret.OwnerReferences, + }, + Data: gotSecret.Data, + } + if diff := cmp.Diff(rebuiltGotSecret, wantSecret); diff != "" { + return fmt.Errorf("secret diff (-got +want):\n%s", diff) + } + return nil + } +} + func markDeploymentAsAvailable(nsName, deployName string) { // Retrieve the Deployment object. gotDeploy := &appsv1.Deployment{} @@ -657,6 +691,27 @@ func regularConfigMapRemovedActual(nsName, configMapName string) func() error { } } +func regularSecretRemovedActual(nsName, secretName string) func() error { + return func() error { + // Retrieve the Secret object. + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: nsName, + Name: secretName, + }, + } + if err := memberClient1.Delete(ctx, secret); err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete the Secret object: %w", err) + } + + // Check that the Secret object has been deleted. + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: secretName}, secret); !errors.IsNotFound(err) { + return fmt.Errorf("secret object still exists or an unexpected error occurred: %w", err) + } + return nil + } +} + func regularNSObjectNotAppliedActual(nsName string) func() error { return func() error { // Retrieve the NS object. @@ -5460,17 +5515,17 @@ var _ = Describe("drift detection and takeover", func() { // deletion; consequently this test suite would not attempt to verify its deletion. }) }) -}) -var _ = Describe("report diff", func() { - // For simplicity reasons, this test case will only involve a NS object. - Context("report diff only (new object)", Ordered, func() { + Context("obscure sensitive data (drift detection)", Ordered, func() { workName := fmt.Sprintf(workNameTemplate, utils.RandStr()) // The environment prepared by the envtest package does not support namespace // deletion; each test case would use a new namespace. nsName := fmt.Sprintf(nsNameTemplate, utils.RandStr()) + var appliedWorkOwnerRef *metav1.OwnerReference var regularNS *corev1.Namespace + var regularCM *corev1.ConfigMap + var regularSecret *corev1.Secret BeforeAll(func() { // Prepare a NS object. @@ -5478,11 +5533,24 @@ var _ = Describe("report diff", func() { regularNS.Name = nsName regularNSJSON := marshalK8sObjJSON(regularNS) + // Prepare a ConfigMap object. + regularCM = configMap.DeepCopy() + regularCM.Namespace = nsName + regularCMJSON := marshalK8sObjJSON(regularCM) + + // Prepare a Secret object. + regularSecret = secret.DeepCopy() + regularSecret.Namespace = nsName + regularSecretJSON := marshalK8sObjJSON(regularSecret) + // Create a new Work object with all the manifest JSONs and proper apply strategy. applyStrategy := &fleetv1beta1.ApplyStrategy{ - Type: fleetv1beta1.ApplyStrategyTypeReportDiff, + Type: fleetv1beta1.ApplyStrategyTypeClientSideApply, + WhenToApply: fleetv1beta1.WhenToApplyTypeIfNotDrifted, + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeNever, + ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, } - createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON) + createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON, regularCMJSON, regularSecretJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -5497,19 +5565,38 @@ var _ = Describe("report diff", func() { appliedWorkOwnerRef = prepareAppliedWorkOwnerRef(workName) }) - It("should not apply the manifests", func() { - // Ensure that the NS object has not been applied. - regularNSObjectNotAppliedActual := regularNSObjectNotAppliedActual(nsName) - Consistently(regularNSObjectNotAppliedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to avoid applying the namespace object") + It("should apply the manifests", func() { + // Ensure that the NS object has been applied as expected. + regularNSObjectAppliedActual := regularNSObjectAppliedActual(nsName, appliedWorkOwnerRef) + Eventually(regularNSObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the namespace object") + + Expect(memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS)).To(Succeed(), "Failed to retrieve the NS object") + + // Ensure that the ConfigMap object has been applied as expected. + regularCMObjectAppliedActual := regularConfigMapObjectAppliedActual(nsName, configMapName, appliedWorkOwnerRef) + Eventually(regularCMObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the configMap object") + + Expect(memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: configMapName}, regularCM)).To(Succeed(), "Failed to retrieve the ConfigMap object") + + // Ensure that the Secret object has been applied as expected. + regularSecretObjectAppliedActual := regularSecretObjectAppliedActual(nsName, secretName, appliedWorkOwnerRef) + Eventually(regularSecretObjectAppliedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to apply the secret object") + + Expect(memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: secretName}, regularSecret)).To(Succeed(), "Failed to retrieve the Secret object") }) It("should update the Work object status", func() { // Prepare the status information. workConds := []metav1.Condition{ { - Type: fleetv1beta1.WorkConditionTypeDiffReported, + Type: fleetv1beta1.WorkConditionTypeApplied, Status: metav1.ConditionTrue, - Reason: condition.WorkAllManifestsDiffReportedReason, + Reason: condition.WorkAllManifestsAppliedReason, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsAvailableReason, }, } manifestConds := []fleetv1beta1.ManifestCondition{ @@ -5524,18 +5611,66 @@ var _ = Describe("report diff", func() { }, Conditions: []metav1.Condition{ { - Type: fleetv1beta1.WorkConditionTypeDiffReported, + Type: fleetv1beta1.WorkConditionTypeApplied, Status: metav1.ConditionTrue, - Reason: string(ApplyOrReportDiffResTypeFoundDiff), + Reason: string(ApplyOrReportDiffResTypeApplied), + ObservedGeneration: 0, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(AvailabilityResultTypeAvailable), ObservedGeneration: 0, }, }, - DiffDetails: &fleetv1beta1.DiffDetails{ - ObservedDiffs: []fleetv1beta1.PatchDetail{ - { - Path: "/", - ValueInHub: "(the whole object)", - }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "", + Version: "v1", + Kind: "ConfigMap", + Resource: "configmaps", + Name: configMapName, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeApplied), + ObservedGeneration: 0, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(AvailabilityResultTypeAvailable), + ObservedGeneration: 0, + }, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 2, + Group: "", + Version: "v1", + Kind: "Secret", + Resource: "secrets", + Name: secretName, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeApplied), + ObservedGeneration: 0, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(AvailabilityResultTypeAvailable), + ObservedGeneration: 0, }, }, }, @@ -5547,180 +5682,177 @@ var _ = Describe("report diff", func() { It("should update the AppliedWork object status", func() { // Prepare the status information. - appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, nil) + appliedResourceMeta := []fleetv1beta1.AppliedResourceMeta{ + { + WorkResourceIdentifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + UID: regularNS.UID, + }, + { + WorkResourceIdentifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "", + Version: "v1", + Kind: "ConfigMap", + Resource: "configmaps", + Name: configMapName, + Namespace: nsName, + }, + UID: regularCM.UID, + }, + { + WorkResourceIdentifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 2, + Group: "", + Version: "v1", + Kind: "Secret", + Resource: "secrets", + Name: secretName, + Namespace: nsName, + }, + UID: regularSecret.UID, + }, + } + + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update appliedWork status") }) - AfterAll(func() { - // Delete the Work object and related resources. - deleteWorkObject(workName, memberReservedNSName1) + It("can make changes to the objects", func() { + // Use Eventually blocks to avoid conflicts. + Eventually(func() error { + // Retrieve the ConfigMap object. + updatedCM := &corev1.ConfigMap{} + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: configMapName}, updatedCM); err != nil { + return fmt.Errorf("failed to retrieve the ConfigMap object: %w", err) + } - // Ensure that the AppliedWork object has been removed. - appliedWorkRemovedActual := appliedWorkRemovedActual(workName, nsName) - Eventually(appliedWorkRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the AppliedWork object") + // Update the ConfigMap object. + if updatedCM.Labels == nil { + updatedCM.Labels = map[string]string{} + } + // Add a label and modify the data entry. + updatedCM.Labels[dummyLabelKey] = dummyLabelValue1 + updatedCM.Data[dummyLabelKey] = dummyLabelValue2 + if err := memberClient1.Update(ctx, updatedCM); err != nil { + return fmt.Errorf("failed to update the ConfigMap object: %w", err) + } - workRemovedActual := workRemovedActual(workName) - Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") + return nil + }).Should(Succeed(), "Failed to make changes to the ConfigMap object") - // The environment prepared by the envtest package does not support namespace - // deletion; consequently this test suite would not attempt to verify its deletion. - }) - }) + Eventually(func() error { + // Retrieve the Secret object. + updatedSecret := &corev1.Secret{} + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: secretName}, updatedSecret); err != nil { + return fmt.Errorf("failed to retrieve the Secret object: %w", err) + } - Context("report diff only (with diff present, diff disappears later, partial comparison)", Ordered, func() { - workName := fmt.Sprintf(workNameTemplate, utils.RandStr()) - // The environment prepared by the envtest package does not support namespace - // deletion; each test case would use a new namespace. - nsName := fmt.Sprintf(nsNameTemplate, utils.RandStr()) + // Update the Secret object; modify the data entry and add a new string data entry. + dummyLabelValue2Bytes := []byte(dummyLabelValue2) + b64encoded := make([]byte, base64.StdEncoding.EncodedLen(len(dummyLabelValue2Bytes))) + base64.StdEncoding.Encode(b64encoded, dummyLabelValue2Bytes) + updatedSecret.Data[dummyLabelKey] = b64encoded - var appliedWorkOwnerRef *metav1.OwnerReference - var regularNS *corev1.Namespace - var regularDeploy *appsv1.Deployment + updatedSecret.StringData = map[string]string{ + "fooStr": dummyLabelValue3, + } - BeforeAll(func() { - // Prepare a NS object. - regularNS = ns.DeepCopy() - regularNS.Name = nsName - regularNSJSON := marshalK8sObjJSON(regularNS) + if err := memberClient1.Update(ctx, updatedSecret); err != nil { + return fmt.Errorf("failed to update the Secret object: %w", err) + } - // Prepare a Deployment object. - regularDeploy = deploy.DeepCopy() - regularDeploy.Namespace = nsName - regularDeploy.Name = deployName - regularDeployJSON := marshalK8sObjJSON(regularDeploy) + return nil + }).Should(Succeed(), "Failed to make changes to the Secret object") + }) - // Create the objects first in the member cluster. - Expect(memberClient1.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") + It("should stop applying some objects", func() { + // Verify that the changes in managed fields are not overwritten. + Consistently(func() error { + gotCM := &corev1.ConfigMap{} + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: configMapName}, gotCM); err != nil { + return fmt.Errorf("failed to retrieve the ConfigMap object: %w", err) + } - // Create a diff in the replica count field. - regularDeploy.Spec.Replicas = ptr.To(int32(2)) - Expect(memberClient1.Create(ctx, regularDeploy)).To(Succeed(), "Failed to create the Deployment object") + // Rebuild the CM to ignore default/populated values in fields. + rebuiltCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: nsName, + Name: configMapName, + Labels: gotCM.Labels, + }, + Data: gotCM.Data, + } - // Create a new Work object with all the manifest JSONs and proper apply strategy. - applyStrategy := &fleetv1beta1.ApplyStrategy{ - ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, - Type: fleetv1beta1.ApplyStrategyTypeReportDiff, - } - createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON, regularDeployJSON) - }) + wantCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: nsName, + Name: configMapName, + Labels: map[string]string{ + dummyLabelKey: dummyLabelValue1, + }, + }, + Data: map[string]string{ + dummyLabelKey: dummyLabelValue2, + }, + } - It("should add cleanup finalizer to the Work object", func() { - finalizerAddedActual := workFinalizerAddedActual(workName) - Eventually(finalizerAddedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to add cleanup finalizer to the Work object") - }) + if diff := cmp.Diff(rebuiltCM, wantCM); diff != "" { + return fmt.Errorf("configMap object diffs (-got, +want):\n%s", diff) + } + return nil + }, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to leave the ConfigMap object alone") - It("should prepare an AppliedWork object", func() { - appliedWorkCreatedActual := appliedWorkCreatedActual(workName) - Eventually(appliedWorkCreatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to prepare an AppliedWork object") + Consistently(func() error { + gotSecret := &corev1.Secret{} + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: secretName}, gotSecret); err != nil { + return fmt.Errorf("failed to retrieve the Secret object: %w", err) + } - appliedWorkOwnerRef = prepareAppliedWorkOwnerRef(workName) - }) + // Rebuild the Secret to ignore default/populated values in fields. + rebuiltSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: nsName, + Name: secretName, + }, + Data: gotSecret.Data, + StringData: gotSecret.StringData, + } - It("should own the objects, but not apply any manifests", func() { - // Verify that the Deployment manifest has not been applied, yet Fleet has assumed - // its ownership. - wantDeploy := deploy.DeepCopy() - wantDeploy.TypeMeta = metav1.TypeMeta{} - wantDeploy.Namespace = nsName - wantDeploy.Name = deployName - wantDeploy.OwnerReferences = []metav1.OwnerReference{ - *appliedWorkOwnerRef, - } - wantDeploy.Spec.Replicas = ptr.To(int32(2)) - - deployOwnedButNotApplied := func() error { - if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy); err != nil { - return fmt.Errorf("failed to retrieve the Deployment object: %w", err) - } - - if len(regularDeploy.Spec.Template.Spec.Containers) != 1 { - return fmt.Errorf("number of containers in the Deployment object, got %d, want %d", len(regularDeploy.Spec.Template.Spec.Containers), 1) - } - if len(regularDeploy.Spec.Template.Spec.Containers[0].Ports) != 1 { - return fmt.Errorf("number of ports in the first container, got %d, want %d", len(regularDeploy.Spec.Template.Spec.Containers[0].Ports), 1) - } - - // To ignore default values automatically, here the test suite rebuilds the objects. - rebuiltGotDeploy := &appsv1.Deployment{ + dummyLabelValue2Bytes := []byte(dummyLabelValue2) + b64encoded := make([]byte, base64.StdEncoding.EncodedLen(len(dummyLabelValue2Bytes))) + base64.StdEncoding.Encode(b64encoded, dummyLabelValue2Bytes) + wantSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Namespace: regularDeploy.Namespace, - Name: regularDeploy.Name, - OwnerReferences: regularDeploy.OwnerReferences, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: regularDeploy.Spec.Replicas, - Selector: regularDeploy.Spec.Selector, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: regularDeploy.Spec.Template.ObjectMeta.Labels, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: regularDeploy.Spec.Template.Spec.Containers[0].Name, - Image: regularDeploy.Spec.Template.Spec.Containers[0].Image, - Ports: []corev1.ContainerPort{ - { - ContainerPort: regularDeploy.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort, - }, - }, - }, - }, - }, - }, + Namespace: nsName, + Name: secretName, }, - } - - if diff := cmp.Diff(rebuiltGotDeploy, wantDeploy); diff != "" { - return fmt.Errorf("deployment diff (-got +want):\n%s", diff) - } - return nil - } - - Eventually(deployOwnedButNotApplied, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to own the Deployment object without applying the manifest") - Consistently(deployOwnedButNotApplied, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to own the Deployment object without applying the manifest") - - // Verify that Fleet has assumed ownership of the NS object. - wantNS := ns.DeepCopy() - wantNS.TypeMeta = metav1.TypeMeta{} - wantNS.Name = nsName - wantNS.OwnerReferences = []metav1.OwnerReference{ - *appliedWorkOwnerRef, - } - - nsOwnedButNotApplied := func() error { - if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { - return fmt.Errorf("failed to retrieve the NS object: %w", err) - } - - // To ignore default values automatically, here the test suite rebuilds the objects. - rebuiltGotNS := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: regularNS.Name, - OwnerReferences: regularNS.OwnerReferences, + Data: map[string][]byte{ + dummyLabelKey: b64encoded, + "fooStr": []byte(dummyLabelValue3), }, } - if diff := cmp.Diff(rebuiltGotNS, wantNS); diff != "" { - return fmt.Errorf("namespace diff (-got +want):\n%s", diff) + if diff := cmp.Diff(rebuiltSecret, wantSecret); diff != "" { + return fmt.Errorf("secret object diffs (-got, +want):\n%s", diff) } return nil - } - Eventually(nsOwnedButNotApplied, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to own the NS object without applying the manifest") - Consistently(nsOwnedButNotApplied, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to own the NS object without applying the manifest") + }, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to leave the Secret object alone") }) It("should update the Work object status", func() { - noLaterThanTimestamp := metav1.Time{ - Time: time.Now().Add(time.Second * 30), - } - // Prepare the status information. workConds := []metav1.Condition{ { - Type: fleetv1beta1.WorkConditionTypeDiffReported, - Status: metav1.ConditionTrue, - Reason: condition.WorkAllManifestsDiffReportedReason, + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: condition.WorkNotAllManifestsAppliedReason, }, } manifestConds := []fleetv1beta1.ManifestCondition{ @@ -5735,9 +5867,15 @@ var _ = Describe("report diff", func() { }, Conditions: []metav1.Condition{ { - Type: fleetv1beta1.WorkConditionTypeDiffReported, + Type: fleetv1beta1.WorkConditionTypeApplied, Status: metav1.ConditionTrue, - Reason: string(ApplyOrReportDiffResTypeNoDiffFound), + Reason: string(ApplyOrReportDiffResTypeApplied), + ObservedGeneration: 0, + }, + { + Type: fleetv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionTrue, + Reason: string(AvailabilityResultTypeAvailable), ObservedGeneration: 0, }, }, @@ -5745,82 +5883,92 @@ var _ = Describe("report diff", func() { { Identifier: fleetv1beta1.WorkResourceIdentifier{ Ordinal: 1, - Group: "apps", + Group: "", Version: "v1", - Kind: "Deployment", - Resource: "deployments", - Name: deployName, + Kind: "ConfigMap", + Resource: "configmaps", + Name: configMapName, Namespace: nsName, }, Conditions: []metav1.Condition{ { - Type: fleetv1beta1.WorkConditionTypeDiffReported, - Status: metav1.ConditionTrue, - Reason: string(ApplyOrReportDiffResTypeFoundDiff), - ObservedGeneration: 1, + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ApplyOrReportDiffResTypeFoundDrifts), + ObservedGeneration: 0, }, }, - DiffDetails: &fleetv1beta1.DiffDetails{ - ObservedInMemberClusterGeneration: ®ularDeploy.Generation, - ObservedDiffs: []fleetv1beta1.PatchDetail{ + DriftDetails: &fleetv1beta1.DriftDetails{ + ObservedInMemberClusterGeneration: regularCM.Generation, + ObservedDrifts: []fleetv1beta1.PatchDetail{ { - Path: "/spec/replicas", - ValueInMember: "2", - ValueInHub: "1", + Path: fmt.Sprintf("/data/%s", dummyLabelKey), + ValueInMember: dummyLabelValue2, + ValueInHub: dummyLabelValue1, + }, + }, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 2, + Group: "", + Version: "v1", + Kind: "Secret", + Resource: "secrets", + Name: secretName, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ApplyOrReportDiffResTypeFoundDrifts), + ObservedGeneration: 0, + }, + }, + DriftDetails: &fleetv1beta1.DriftDetails{ + ObservedInMemberClusterGeneration: regularSecret.Generation, + ObservedDrifts: []fleetv1beta1.PatchDetail{ + { + Path: fmt.Sprintf("/data/%s", dummyLabelKey), + ValueInHub: "(redacted for security reasons)", + ValueInMember: "(redacted for security reasons)", }, }, }, }, } - workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, &noLaterThanTimestamp, &noLaterThanTimestamp) + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, nil, nil) Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update work status") }) - It("should have no applied object reportings in the AppliedWork status", func() { - // Prepare the status information. - var appliedResourceMeta []fleetv1beta1.AppliedResourceMeta - - appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) - Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update appliedWork status") - }) - - It("can make changes to the objects", func() { - // Use Eventually blocks to avoid conflicts. + It("can switch to full comparison mode", func() { + // Use Eventually block to avoid potential conflicts. Eventually(func() error { - // Retrieve the Deployment object. - updatedDeploy := &appsv1.Deployment{} - if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, updatedDeploy); err != nil { - return fmt.Errorf("failed to retrieve the Deployment object: %w", err) + work := &fleetv1beta1.Work{} + if err := hubClient.Get(ctx, client.ObjectKey{Namespace: memberReservedNSName1, Name: workName}, work); err != nil { + return fmt.Errorf("failed to retrieve the Work object: %w", err) } - // Make changes to the Deployment object. - updatedDeploy.Spec.Replicas = ptr.To(int32(1)) - - // Update the Deployment object. - if err := memberClient1.Update(ctx, updatedDeploy); err != nil { - return fmt.Errorf("failed to update the Deployment object: %w", err) + if work.Spec.ApplyStrategy == nil { + work.Spec.ApplyStrategy = &fleetv1beta1.ApplyStrategy{} } - return nil - }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update the Deployment object") - }) - - It("can mark the deployment as available", func() { - markDeploymentAsAvailable(nsName, deployName) + work.Spec.ApplyStrategy.ComparisonOption = fleetv1beta1.ComparisonOptionTypeFullComparison + return hubClient.Update(ctx, work) + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to switch to full comparison mode") }) + // For simplicity reasons this test will not verify again that the objects are not modified, + // as another test spec has covered the case. It("should update the Work object status", func() { - // Shift the timestamp to account for drift/diff detection delays. - noLaterThanTimestamp := metav1.Time{ - Time: time.Now().Add(time.Second * 30), - } - // Prepare the status information. workConds := []metav1.Condition{ { - Type: fleetv1beta1.WorkConditionTypeDiffReported, - Status: metav1.ConditionTrue, - Reason: condition.WorkAllManifestsDiffReportedReason, + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: condition.WorkNotAllManifestsAppliedReason, }, } manifestConds := []fleetv1beta1.ManifestCondition{ @@ -5835,53 +5983,113 @@ var _ = Describe("report diff", func() { }, Conditions: []metav1.Condition{ { - Type: fleetv1beta1.WorkConditionTypeDiffReported, - Status: metav1.ConditionTrue, - Reason: string(ApplyOrReportDiffResTypeNoDiffFound), + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ApplyOrReportDiffResTypeFoundDrifts), ObservedGeneration: 0, }, }, + DriftDetails: &fleetv1beta1.DriftDetails{ + ObservedInMemberClusterGeneration: regularNS.Generation, + ObservedDrifts: []fleetv1beta1.PatchDetail{ + { + Path: "/spec/finalizers", + ValueInMember: "[kubernetes]", + }, + }, + }, }, { Identifier: fleetv1beta1.WorkResourceIdentifier{ Ordinal: 1, - Group: "apps", + Group: "", Version: "v1", - Kind: "Deployment", - Resource: "deployments", - Name: deployName, + Kind: "ConfigMap", + Resource: "configmaps", + Name: configMapName, Namespace: nsName, }, Conditions: []metav1.Condition{ { - Type: fleetv1beta1.WorkConditionTypeDiffReported, - Status: metav1.ConditionTrue, - Reason: string(ApplyOrReportDiffResTypeNoDiffFound), - ObservedGeneration: 2, + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ApplyOrReportDiffResTypeFoundDrifts), + ObservedGeneration: 0, + }, + }, + DriftDetails: &fleetv1beta1.DriftDetails{ + ObservedInMemberClusterGeneration: regularCM.Generation, + ObservedDrifts: []fleetv1beta1.PatchDetail{ + { + Path: fmt.Sprintf("/data/%s", dummyLabelKey), + ValueInMember: dummyLabelValue2, + ValueInHub: dummyLabelValue1, + }, + { + Path: fmt.Sprintf("/metadata/labels/%s", dummyLabelKey), + ValueInMember: dummyLabelValue1, + }, }, }, }, - } - - workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, &noLaterThanTimestamp, &noLaterThanTimestamp) - Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update work status") - }) - - It("should have no applied object reportings in the AppliedWork status", func() { - // Prepare the status information. - var appliedResourceMeta []fleetv1beta1.AppliedResourceMeta - - appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) - Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update appliedWork status") - }) - - AfterAll(func() { - // Delete the Work object and related resources. - deleteWorkObject(workName, memberReservedNSName1) - - // Ensure that the Deployment object has been left alone. - regularDeployNotRemovedActual := regularDeployNotRemovedActual(nsName, deployName) - Consistently(regularDeployNotRemovedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to remove the deployment object") + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 2, + Group: "", + Version: "v1", + Kind: "Secret", + Resource: "secrets", + Name: secretName, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeApplied, + Status: metav1.ConditionFalse, + Reason: string(ApplyOrReportDiffResTypeFoundDrifts), + ObservedGeneration: 0, + }, + }, + DriftDetails: &fleetv1beta1.DriftDetails{ + ObservedInMemberClusterGeneration: regularSecret.Generation, + ObservedDrifts: []fleetv1beta1.PatchDetail{ + { + Path: fmt.Sprintf("/data/%s", dummyLabelKey), + ValueInHub: "(redacted for security reasons)", + ValueInMember: "(redacted for security reasons)", + }, + { + Path: fmt.Sprintf("/data/%s", "fooStr"), + ValueInMember: "(redacted for security reasons)", + }, + { + Path: "/type", + ValueInMember: "Opaque", + }, + }, + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, nil, nil) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update work status") + }) + + AfterAll(func() { + // Delete the Work object and related resources. + deleteWorkObject(workName, memberReservedNSName1) + + // Ensure that the ConfigMap object has been removed. + regularCMRemovedActual := regularConfigMapRemovedActual(nsName, configMapName) + Eventually(regularCMRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the ConfigMap object") + + // Ensure that the secret object has been removed. + regularSecretRemovedActual := regularSecretRemovedActual(nsName, secretName) + Eventually(regularSecretRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Secret object") + + // Kubebuilder suggests that in a testing environment like this, to check for the existence of the AppliedWork object + // OwnerReference in the Namespace object (https://book.kubebuilder.io/reference/envtest.html#testing-considerations). + checkNSOwnerReferences(workName, nsName) // Ensure that the AppliedWork object has been removed. appliedWorkRemovedActual := appliedWorkRemovedActual(workName, nsName) @@ -5891,18 +6099,20 @@ var _ = Describe("report diff", func() { Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") // The environment prepared by the envtest package does not support namespace - // deletion; consequently this test suite would not attempt to verify its deletion. + // deletion; consequently this test suite would not attempt so verify its deletion. }) }) +}) - Context("report diff only (w/ not taken over resources, partial comparison, a.k.a. do not touch anything and just report diff)", Ordered, func() { +var _ = Describe("report diff", func() { + // For simplicity reasons, this test case will only involve a NS object. + Context("report diff only (new object)", Ordered, func() { workName := fmt.Sprintf(workNameTemplate, utils.RandStr()) // The environment prepared by the envtest package does not support namespace // deletion; each test case would use a new namespace. nsName := fmt.Sprintf(nsNameTemplate, utils.RandStr()) var regularNS *corev1.Namespace - var regularDeploy *appsv1.Deployment BeforeAll(func() { // Prepare a NS object. @@ -5910,26 +6120,11 @@ var _ = Describe("report diff", func() { regularNS.Name = nsName regularNSJSON := marshalK8sObjJSON(regularNS) - // Prepare a Deployment object. - regularDeploy = deploy.DeepCopy() - regularDeploy.Namespace = nsName - regularDeploy.Name = deployName - regularDeployJSON := marshalK8sObjJSON(regularDeploy) - - // Create the objects first in the member cluster. - Expect(memberClient1.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") - - // Create a diff in the replica count field. - regularDeploy.Spec.Replicas = ptr.To(int32(2)) - Expect(memberClient1.Create(ctx, regularDeploy)).To(Succeed(), "Failed to create the Deployment object") - // Create a new Work object with all the manifest JSONs and proper apply strategy. applyStrategy := &fleetv1beta1.ApplyStrategy{ - ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, - Type: fleetv1beta1.ApplyStrategyTypeReportDiff, - WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeNever, + Type: fleetv1beta1.ApplyStrategyTypeReportDiff, } - createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON, regularDeployJSON) + createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -5940,83 +6135,14 @@ var _ = Describe("report diff", func() { It("should prepare an AppliedWork object", func() { appliedWorkCreatedActual := appliedWorkCreatedActual(workName) Eventually(appliedWorkCreatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to prepare an AppliedWork object") - }) - - It("should not apply any manifest", func() { - // Verify that the NS manifest has not been applied. - Consistently(func() error { - // Retrieve the NS object. - updatedNS := &corev1.Namespace{} - if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { - return fmt.Errorf("failed to retrieve the NS object: %w", err) - } - - // Rebuild the NS object to ignore default values automatically. - rebuiltGotNS := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: updatedNS.Name, - OwnerReferences: updatedNS.OwnerReferences, - }, - } - wantNS := ns.DeepCopy() - wantNS.Name = nsName - if diff := cmp.Diff(rebuiltGotNS, wantNS, ignoreFieldTypeMetaInNamespace); diff != "" { - return fmt.Errorf("namespace diff (-got +want):\n%s", diff) - } - - return nil - }, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to leave the NS object alone") - - // Verify that the Deployment manifest has not been applied. - Consistently(func() error { - // Retrieve the Deployment object. - updatedDeploy := &appsv1.Deployment{} - if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, updatedDeploy); err != nil { - return fmt.Errorf("failed to retrieve the Deployment object: %w", err) - } - - // Rebuild the Deployment object to ignore default values automatically. - rebuiltGotDeploy := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: updatedDeploy.Namespace, - Name: updatedDeploy.Name, - OwnerReferences: updatedDeploy.OwnerReferences, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: updatedDeploy.Spec.Replicas, - Selector: updatedDeploy.Spec.Selector, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: updatedDeploy.Spec.Template.ObjectMeta.Labels, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: updatedDeploy.Spec.Template.Spec.Containers[0].Name, - Image: updatedDeploy.Spec.Template.Spec.Containers[0].Image, - Ports: []corev1.ContainerPort{ - { - ContainerPort: updatedDeploy.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort, - }, - }, - }, - }, - }, - }, - }, - } - wantDeploy := deploy.DeepCopy() - wantDeploy.TypeMeta = metav1.TypeMeta{} - wantDeploy.Namespace = nsName - wantDeploy.Name = deployName - wantDeploy.Spec.Replicas = ptr.To(int32(2)) + appliedWorkOwnerRef = prepareAppliedWorkOwnerRef(workName) + }) - if diff := cmp.Diff(rebuiltGotDeploy, wantDeploy); diff != "" { - return fmt.Errorf("deployment diff (-got +want):\n%s", diff) - } - return nil - }, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to leave the Deployment object alone") + It("should not apply the manifests", func() { + // Ensure that the NS object has not been applied. + regularNSObjectNotAppliedActual := regularNSObjectNotAppliedActual(nsName) + Consistently(regularNSObjectNotAppliedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to avoid applying the namespace object") }) It("should update the Work object status", func() { @@ -6038,42 +6164,21 @@ var _ = Describe("report diff", func() { Resource: "namespaces", Name: nsName, }, - Conditions: []metav1.Condition{ - { - Type: fleetv1beta1.WorkConditionTypeDiffReported, - Status: metav1.ConditionTrue, - Reason: string(ApplyOrReportDiffResTypeNoDiffFound), - ObservedGeneration: 0, - }, - }, - }, - { - Identifier: fleetv1beta1.WorkResourceIdentifier{ - Ordinal: 1, - Group: "apps", - Version: "v1", - Kind: "Deployment", - Resource: "deployments", - Name: deployName, - Namespace: nsName, - }, Conditions: []metav1.Condition{ { Type: fleetv1beta1.WorkConditionTypeDiffReported, Status: metav1.ConditionTrue, Reason: string(ApplyOrReportDiffResTypeFoundDiff), - ObservedGeneration: 1, + ObservedGeneration: 0, }, }, DiffDetails: &fleetv1beta1.DiffDetails{ ObservedDiffs: []fleetv1beta1.PatchDetail{ { - Path: "/spec/replicas", - ValueInHub: "1", - ValueInMember: "2", + Path: "/", + ValueInHub: "(the whole object)", }, }, - ObservedInMemberClusterGeneration: ptr.To(int64(1)), }, }, } @@ -6082,11 +6187,9 @@ var _ = Describe("report diff", func() { Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update work status") }) - It("should have no applied object reportings in the AppliedWork status", func() { + It("should update the AppliedWork object status", func() { // Prepare the status information. - var appliedResourceMeta []fleetv1beta1.AppliedResourceMeta - - appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, nil) Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update appliedWork status") }) @@ -6094,10 +6197,6 @@ var _ = Describe("report diff", func() { // Delete the Work object and related resources. deleteWorkObject(workName, memberReservedNSName1) - // Ensure applied manifest has been removed. - regularDeployRemovedActual := regularDeployRemovedActual(nsName, deployName) - Eventually(regularDeployRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the deployment object") - // Ensure that the AppliedWork object has been removed. appliedWorkRemovedActual := appliedWorkRemovedActual(workName, nsName) Eventually(appliedWorkRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the AppliedWork object") @@ -6110,14 +6209,15 @@ var _ = Describe("report diff", func() { }) }) - Context("report diff failure (decoding error)", Ordered, func() { + Context("report diff only (with diff present, diff disappears later, partial comparison)", Ordered, func() { workName := fmt.Sprintf(workNameTemplate, utils.RandStr()) // The environment prepared by the envtest package does not support namespace // deletion; each test case would use a new namespace. nsName := fmt.Sprintf(nsNameTemplate, utils.RandStr()) + var appliedWorkOwnerRef *metav1.OwnerReference var regularNS *corev1.Namespace - var malformedConfigMap *corev1.ConfigMap + var regularDeploy *appsv1.Deployment BeforeAll(func() { // Prepare a NS object. @@ -6125,20 +6225,25 @@ var _ = Describe("report diff", func() { regularNS.Name = nsName regularNSJSON := marshalK8sObjJSON(regularNS) - malformedConfigMap = configMap.DeepCopy() - malformedConfigMap.Namespace = nsName - // This will trigger a decoding error on the work applier side as this API is not registered. - malformedConfigMap.TypeMeta = metav1.TypeMeta{ - APIVersion: "malformed/v10", - Kind: "Unknown", - } - malformedConfigMapJSON := marshalK8sObjJSON(malformedConfigMap) + // Prepare a Deployment object. + regularDeploy = deploy.DeepCopy() + regularDeploy.Namespace = nsName + regularDeploy.Name = deployName + regularDeployJSON := marshalK8sObjJSON(regularDeploy) + + // Create the objects first in the member cluster. + Expect(memberClient1.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") + + // Create a diff in the replica count field. + regularDeploy.Spec.Replicas = ptr.To(int32(2)) + Expect(memberClient1.Create(ctx, regularDeploy)).To(Succeed(), "Failed to create the Deployment object") // Create a new Work object with all the manifest JSONs and proper apply strategy. applyStrategy := &fleetv1beta1.ApplyStrategy{ - Type: fleetv1beta1.ApplyStrategyTypeReportDiff, + ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, + Type: fleetv1beta1.ApplyStrategyTypeReportDiff, } - createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON, malformedConfigMapJSON) + createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON, regularDeployJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -6153,28 +6258,113 @@ var _ = Describe("report diff", func() { appliedWorkOwnerRef = prepareAppliedWorkOwnerRef(workName) }) - It("should not apply any manifest", func() { - Consistently(func() error { - configMap := &corev1.ConfigMap{} - objKey := client.ObjectKey{Namespace: nsName, Name: malformedConfigMap.Name} - if err := memberClient1.Get(ctx, objKey, configMap); !errors.IsNotFound(err) { - return fmt.Errorf("the config map exists, or an unexpected error has occurred: %w", err) + It("should own the objects, but not apply any manifests", func() { + // Verify that the Deployment manifest has not been applied, yet Fleet has assumed + // its ownership. + wantDeploy := deploy.DeepCopy() + wantDeploy.TypeMeta = metav1.TypeMeta{} + wantDeploy.Namespace = nsName + wantDeploy.Name = deployName + wantDeploy.OwnerReferences = []metav1.OwnerReference{ + *appliedWorkOwnerRef, + } + wantDeploy.Spec.Replicas = ptr.To(int32(2)) + + deployOwnedButNotApplied := func() error { + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, regularDeploy); err != nil { + return fmt.Errorf("failed to retrieve the Deployment object: %w", err) } - return nil - }, consistentlyDuration, consistentlyInterval).Should(Succeed(), "The config map has been applied unexpectedly") - Consistently(regularNSObjectNotAppliedActual(nsName), consistentlyDuration, consistentlyInterval).Should(Succeed(), "The namespace object has been applied unexpectedly") - }) + if len(regularDeploy.Spec.Template.Spec.Containers) != 1 { + return fmt.Errorf("number of containers in the Deployment object, got %d, want %d", len(regularDeploy.Spec.Template.Spec.Containers), 1) + } + if len(regularDeploy.Spec.Template.Spec.Containers[0].Ports) != 1 { + return fmt.Errorf("number of ports in the first container, got %d, want %d", len(regularDeploy.Spec.Template.Spec.Containers[0].Ports), 1) + } - It("should update the Work object status", func() { - // Prepare the status information. - workConds := []metav1.Condition{ - { - Type: fleetv1beta1.WorkConditionTypeDiffReported, - Status: metav1.ConditionFalse, - Reason: condition.WorkNotAllManifestsDiffReportedReason, - }, - } + // To ignore default values automatically, here the test suite rebuilds the objects. + rebuiltGotDeploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: regularDeploy.Namespace, + Name: regularDeploy.Name, + OwnerReferences: regularDeploy.OwnerReferences, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: regularDeploy.Spec.Replicas, + Selector: regularDeploy.Spec.Selector, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: regularDeploy.Spec.Template.ObjectMeta.Labels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: regularDeploy.Spec.Template.Spec.Containers[0].Name, + Image: regularDeploy.Spec.Template.Spec.Containers[0].Image, + Ports: []corev1.ContainerPort{ + { + ContainerPort: regularDeploy.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort, + }, + }, + }, + }, + }, + }, + }, + } + + if diff := cmp.Diff(rebuiltGotDeploy, wantDeploy); diff != "" { + return fmt.Errorf("deployment diff (-got +want):\n%s", diff) + } + return nil + } + + Eventually(deployOwnedButNotApplied, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to own the Deployment object without applying the manifest") + Consistently(deployOwnedButNotApplied, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to own the Deployment object without applying the manifest") + + // Verify that Fleet has assumed ownership of the NS object. + wantNS := ns.DeepCopy() + wantNS.TypeMeta = metav1.TypeMeta{} + wantNS.Name = nsName + wantNS.OwnerReferences = []metav1.OwnerReference{ + *appliedWorkOwnerRef, + } + + nsOwnedButNotApplied := func() error { + if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, regularNS); err != nil { + return fmt.Errorf("failed to retrieve the NS object: %w", err) + } + + // To ignore default values automatically, here the test suite rebuilds the objects. + rebuiltGotNS := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: regularNS.Name, + OwnerReferences: regularNS.OwnerReferences, + }, + } + + if diff := cmp.Diff(rebuiltGotNS, wantNS); diff != "" { + return fmt.Errorf("namespace diff (-got +want):\n%s", diff) + } + return nil + } + Eventually(nsOwnedButNotApplied, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to own the NS object without applying the manifest") + Consistently(nsOwnedButNotApplied, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to own the NS object without applying the manifest") + }) + + It("should update the Work object status", func() { + noLaterThanTimestamp := metav1.Time{ + Time: time.Now().Add(time.Second * 30), + } + + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsDiffReportedReason, + }, + } manifestConds := []fleetv1beta1.ManifestCondition{ { Identifier: fleetv1beta1.WorkResourceIdentifier{ @@ -6189,48 +6379,141 @@ var _ = Describe("report diff", func() { { Type: fleetv1beta1.WorkConditionTypeDiffReported, Status: metav1.ConditionTrue, - Reason: string(ApplyOrReportDiffResTypeFoundDiff), + Reason: string(ApplyOrReportDiffResTypeNoDiffFound), ObservedGeneration: 0, }, }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Resource: "deployments", + Name: deployName, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeFoundDiff), + ObservedGeneration: 1, + }, + }, DiffDetails: &fleetv1beta1.DiffDetails{ + ObservedInMemberClusterGeneration: ®ularDeploy.Generation, ObservedDiffs: []fleetv1beta1.PatchDetail{ { - Path: "/", - ValueInHub: "(the whole object)", + Path: "/spec/replicas", + ValueInMember: "2", + ValueInHub: "1", }, }, }, }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, &noLaterThanTimestamp, &noLaterThanTimestamp) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update work status") + }) + + It("should have no applied object reportings in the AppliedWork status", func() { + // Prepare the status information. + var appliedResourceMeta []fleetv1beta1.AppliedResourceMeta + + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update appliedWork status") + }) + + It("can make changes to the objects", func() { + // Use Eventually blocks to avoid conflicts. + Eventually(func() error { + // Retrieve the Deployment object. + updatedDeploy := &appsv1.Deployment{} + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, updatedDeploy); err != nil { + return fmt.Errorf("failed to retrieve the Deployment object: %w", err) + } + + // Make changes to the Deployment object. + updatedDeploy.Spec.Replicas = ptr.To(int32(1)) + + // Update the Deployment object. + if err := memberClient1.Update(ctx, updatedDeploy); err != nil { + return fmt.Errorf("failed to update the Deployment object: %w", err) + } + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update the Deployment object") + }) + + It("can mark the deployment as available", func() { + markDeploymentAsAvailable(nsName, deployName) + }) + + It("should update the Work object status", func() { + // Shift the timestamp to account for drift/diff detection delays. + noLaterThanTimestamp := metav1.Time{ + Time: time.Now().Add(time.Second * 30), + } + + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsDiffReportedReason, + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeNoDiffFound), + ObservedGeneration: 0, + }, + }, + }, { - // Note that this specific decoding error will not block the work applier from extracting - // the GVR, hence the populated API group, version and kind information. Identifier: fleetv1beta1.WorkResourceIdentifier{ Ordinal: 1, - Group: "malformed", - Version: "v10", - Kind: "Unknown", - Resource: "", - Name: malformedConfigMap.Name, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Resource: "deployments", + Name: deployName, Namespace: nsName, }, Conditions: []metav1.Condition{ { - Type: fleetv1beta1.WorkConditionTypeDiffReported, - Status: metav1.ConditionFalse, - Reason: string(ApplyOrReportDiffResTypeFailedToReportDiff), + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeNoDiffFound), + ObservedGeneration: 2, }, }, }, } - workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, nil, nil) + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, &noLaterThanTimestamp, &noLaterThanTimestamp) Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update work status") }) - It("should update the AppliedWork object status", func() { + It("should have no applied object reportings in the AppliedWork status", func() { // Prepare the status information. - appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, nil) + var appliedResourceMeta []fleetv1beta1.AppliedResourceMeta + + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update appliedWork status") }) @@ -6238,6 +6521,10 @@ var _ = Describe("report diff", func() { // Delete the Work object and related resources. deleteWorkObject(workName, memberReservedNSName1) + // Ensure that the Deployment object has been left alone. + regularDeployNotRemovedActual := regularDeployNotRemovedActual(nsName, deployName) + Consistently(regularDeployNotRemovedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to remove the deployment object") + // Ensure that the AppliedWork object has been removed. appliedWorkRemovedActual := appliedWorkRemovedActual(workName, nsName) Eventually(appliedWorkRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the AppliedWork object") @@ -6246,19 +6533,18 @@ var _ = Describe("report diff", func() { Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") // The environment prepared by the envtest package does not support namespace - // deletion; consequently this test suite would not attempt so verify its deletion. + // deletion; consequently this test suite would not attempt to verify its deletion. }) }) - Context("report diff only (partial comparison, degraded)", Ordered, func() { + Context("report diff only (w/ not taken over resources, partial comparison, a.k.a. do not touch anything and just report diff)", Ordered, func() { workName := fmt.Sprintf(workNameTemplate, utils.RandStr()) // The environment prepared by the envtest package does not support namespace // deletion; each test case would use a new namespace. nsName := fmt.Sprintf(nsNameTemplate, utils.RandStr()) - //var appliedWorkOwnerRef *metav1.OwnerReference var regularNS *corev1.Namespace - var regularJob *batchv1.Job + var regularDeploy *appsv1.Deployment BeforeAll(func() { // Prepare a NS object. @@ -6266,24 +6552,18 @@ var _ = Describe("report diff", func() { regularNS.Name = nsName regularNSJSON := marshalK8sObjJSON(regularNS) - // Prepare a Job object. - regularJob = job.DeepCopy() - regularJob.Namespace = nsName - regularJob.Name = jobName + // Prepare a Deployment object. + regularDeploy = deploy.DeepCopy() + regularDeploy.Namespace = nsName + regularDeploy.Name = deployName + regularDeployJSON := marshalK8sObjJSON(regularDeploy) // Create the objects first in the member cluster. Expect(memberClient1.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") - Expect(memberClient1.Create(ctx, regularJob)).To(Succeed(), "Failed to create the Job object") - // Update the values on the hub cluster side so that diffs will be found. - updatedJob := job.DeepCopy() - updatedJob.Namespace = nsName - updatedJob.Name = jobName - // `.spec.completions` is an immutable field in Job objects. - updatedJob.Spec.Completions = ptr.To(int32(3)) - // `.spec.template` is an immutable field in Job objects. - updatedJob.Spec.Template.Spec.Containers[0].Image = "busybox:v0.0.1" - updatedJSONJSON := marshalK8sObjJSON(updatedJob) + // Create a diff in the replica count field. + regularDeploy.Spec.Replicas = ptr.To(int32(2)) + Expect(memberClient1.Create(ctx, regularDeploy)).To(Succeed(), "Failed to create the Deployment object") // Create a new Work object with all the manifest JSONs and proper apply strategy. applyStrategy := &fleetv1beta1.ApplyStrategy{ @@ -6291,7 +6571,7 @@ var _ = Describe("report diff", func() { Type: fleetv1beta1.ApplyStrategyTypeReportDiff, WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeNever, } - createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON, updatedJSONJSON) + createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON, regularDeployJSON) }) It("should add cleanup finalizer to the Work object", func() { @@ -6302,21 +6582,805 @@ var _ = Describe("report diff", func() { It("should prepare an AppliedWork object", func() { appliedWorkCreatedActual := appliedWorkCreatedActual(workName) Eventually(appliedWorkCreatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to prepare an AppliedWork object") - - appliedWorkOwnerRef = prepareAppliedWorkOwnerRef(workName) }) - It("should update the Work object status", func() { - // Prepare the status information. - workConds := []metav1.Condition{ - { - Type: fleetv1beta1.WorkConditionTypeDiffReported, - Status: metav1.ConditionFalse, - Reason: condition.WorkNotAllManifestsDiffReportedReason, - }, - } - manifestConds := []fleetv1beta1.ManifestCondition{ - { + It("should not apply any manifest", func() { + // Verify that the NS manifest has not been applied. + Consistently(func() error { + // Retrieve the NS object. + updatedNS := &corev1.Namespace{} + if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { + return fmt.Errorf("failed to retrieve the NS object: %w", err) + } + + // Rebuild the NS object to ignore default values automatically. + rebuiltGotNS := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: updatedNS.Name, + OwnerReferences: updatedNS.OwnerReferences, + }, + } + wantNS := ns.DeepCopy() + wantNS.Name = nsName + if diff := cmp.Diff(rebuiltGotNS, wantNS, ignoreFieldTypeMetaInNamespace); diff != "" { + return fmt.Errorf("namespace diff (-got +want):\n%s", diff) + } + + return nil + }, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to leave the NS object alone") + + // Verify that the Deployment manifest has not been applied. + Consistently(func() error { + // Retrieve the Deployment object. + updatedDeploy := &appsv1.Deployment{} + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: deployName}, updatedDeploy); err != nil { + return fmt.Errorf("failed to retrieve the Deployment object: %w", err) + } + + // Rebuild the Deployment object to ignore default values automatically. + rebuiltGotDeploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: updatedDeploy.Namespace, + Name: updatedDeploy.Name, + OwnerReferences: updatedDeploy.OwnerReferences, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: updatedDeploy.Spec.Replicas, + Selector: updatedDeploy.Spec.Selector, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: updatedDeploy.Spec.Template.ObjectMeta.Labels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: updatedDeploy.Spec.Template.Spec.Containers[0].Name, + Image: updatedDeploy.Spec.Template.Spec.Containers[0].Image, + Ports: []corev1.ContainerPort{ + { + ContainerPort: updatedDeploy.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort, + }, + }, + }, + }, + }, + }, + }, + } + + wantDeploy := deploy.DeepCopy() + wantDeploy.TypeMeta = metav1.TypeMeta{} + wantDeploy.Namespace = nsName + wantDeploy.Name = deployName + wantDeploy.Spec.Replicas = ptr.To(int32(2)) + + if diff := cmp.Diff(rebuiltGotDeploy, wantDeploy); diff != "" { + return fmt.Errorf("deployment diff (-got +want):\n%s", diff) + } + return nil + }, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to leave the Deployment object alone") + }) + + It("should update the Work object status", func() { + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsDiffReportedReason, + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeNoDiffFound), + ObservedGeneration: 0, + }, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "apps", + Version: "v1", + Kind: "Deployment", + Resource: "deployments", + Name: deployName, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeFoundDiff), + ObservedGeneration: 1, + }, + }, + DiffDetails: &fleetv1beta1.DiffDetails{ + ObservedDiffs: []fleetv1beta1.PatchDetail{ + { + Path: "/spec/replicas", + ValueInHub: "1", + ValueInMember: "2", + }, + }, + ObservedInMemberClusterGeneration: ptr.To(int64(1)), + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, nil, nil) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update work status") + }) + + It("should have no applied object reportings in the AppliedWork status", func() { + // Prepare the status information. + var appliedResourceMeta []fleetv1beta1.AppliedResourceMeta + + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update appliedWork status") + }) + + AfterAll(func() { + // Delete the Work object and related resources. + deleteWorkObject(workName, memberReservedNSName1) + + // Ensure applied manifest has been removed. + regularDeployRemovedActual := regularDeployRemovedActual(nsName, deployName) + Eventually(regularDeployRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the deployment object") + + // Ensure that the AppliedWork object has been removed. + appliedWorkRemovedActual := appliedWorkRemovedActual(workName, nsName) + Eventually(appliedWorkRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the AppliedWork object") + + workRemovedActual := workRemovedActual(workName) + Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") + + // The environment prepared by the envtest package does not support namespace + // deletion; consequently this test suite would not attempt to verify its deletion. + }) + }) + + Context("report diff failure (decoding error)", Ordered, func() { + workName := fmt.Sprintf(workNameTemplate, utils.RandStr()) + // The environment prepared by the envtest package does not support namespace + // deletion; each test case would use a new namespace. + nsName := fmt.Sprintf(nsNameTemplate, utils.RandStr()) + + var regularNS *corev1.Namespace + var malformedConfigMap *corev1.ConfigMap + + BeforeAll(func() { + // Prepare a NS object. + regularNS = ns.DeepCopy() + regularNS.Name = nsName + regularNSJSON := marshalK8sObjJSON(regularNS) + + malformedConfigMap = configMap.DeepCopy() + malformedConfigMap.Namespace = nsName + // This will trigger a decoding error on the work applier side as this API is not registered. + malformedConfigMap.TypeMeta = metav1.TypeMeta{ + APIVersion: "malformed/v10", + Kind: "Unknown", + } + malformedConfigMapJSON := marshalK8sObjJSON(malformedConfigMap) + + // Create a new Work object with all the manifest JSONs and proper apply strategy. + applyStrategy := &fleetv1beta1.ApplyStrategy{ + Type: fleetv1beta1.ApplyStrategyTypeReportDiff, + } + createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON, malformedConfigMapJSON) + }) + + It("should add cleanup finalizer to the Work object", func() { + finalizerAddedActual := workFinalizerAddedActual(workName) + Eventually(finalizerAddedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to add cleanup finalizer to the Work object") + }) + + It("should prepare an AppliedWork object", func() { + appliedWorkCreatedActual := appliedWorkCreatedActual(workName) + Eventually(appliedWorkCreatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to prepare an AppliedWork object") + + appliedWorkOwnerRef = prepareAppliedWorkOwnerRef(workName) + }) + + It("should not apply any manifest", func() { + Consistently(func() error { + configMap := &corev1.ConfigMap{} + objKey := client.ObjectKey{Namespace: nsName, Name: malformedConfigMap.Name} + if err := memberClient1.Get(ctx, objKey, configMap); !errors.IsNotFound(err) { + return fmt.Errorf("the config map exists, or an unexpected error has occurred: %w", err) + } + return nil + }, consistentlyDuration, consistentlyInterval).Should(Succeed(), "The config map has been applied unexpectedly") + + Consistently(regularNSObjectNotAppliedActual(nsName), consistentlyDuration, consistentlyInterval).Should(Succeed(), "The namespace object has been applied unexpectedly") + }) + + It("should update the Work object status", func() { + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionFalse, + Reason: condition.WorkNotAllManifestsDiffReportedReason, + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeFoundDiff), + ObservedGeneration: 0, + }, + }, + DiffDetails: &fleetv1beta1.DiffDetails{ + ObservedDiffs: []fleetv1beta1.PatchDetail{ + { + Path: "/", + ValueInHub: "(the whole object)", + }, + }, + }, + }, + { + // Note that this specific decoding error will not block the work applier from extracting + // the GVR, hence the populated API group, version and kind information. + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "malformed", + Version: "v10", + Kind: "Unknown", + Resource: "", + Name: malformedConfigMap.Name, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionFalse, + Reason: string(ApplyOrReportDiffResTypeFailedToReportDiff), + }, + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, nil, nil) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update work status") + }) + + It("should update the AppliedWork object status", func() { + // Prepare the status information. + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, nil) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update appliedWork status") + }) + + AfterAll(func() { + // Delete the Work object and related resources. + deleteWorkObject(workName, memberReservedNSName1) + + // Ensure that the AppliedWork object has been removed. + appliedWorkRemovedActual := appliedWorkRemovedActual(workName, nsName) + Eventually(appliedWorkRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the AppliedWork object") + + workRemovedActual := workRemovedActual(workName) + Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") + + // The environment prepared by the envtest package does not support namespace + // deletion; consequently this test suite would not attempt so verify its deletion. + }) + }) + + Context("report diff only (partial comparison, degraded)", Ordered, func() { + workName := fmt.Sprintf(workNameTemplate, utils.RandStr()) + // The environment prepared by the envtest package does not support namespace + // deletion; each test case would use a new namespace. + nsName := fmt.Sprintf(nsNameTemplate, utils.RandStr()) + + //var appliedWorkOwnerRef *metav1.OwnerReference + var regularNS *corev1.Namespace + var regularJob *batchv1.Job + + BeforeAll(func() { + // Prepare a NS object. + regularNS = ns.DeepCopy() + regularNS.Name = nsName + regularNSJSON := marshalK8sObjJSON(regularNS) + + // Prepare a Job object. + regularJob = job.DeepCopy() + regularJob.Namespace = nsName + regularJob.Name = jobName + + // Create the objects first in the member cluster. + Expect(memberClient1.Create(ctx, regularNS)).To(Succeed(), "Failed to create the NS object") + Expect(memberClient1.Create(ctx, regularJob)).To(Succeed(), "Failed to create the Job object") + + // Update the values on the hub cluster side so that diffs will be found. + updatedJob := job.DeepCopy() + updatedJob.Namespace = nsName + updatedJob.Name = jobName + // `.spec.completions` is an immutable field in Job objects. + updatedJob.Spec.Completions = ptr.To(int32(3)) + // `.spec.template` is an immutable field in Job objects. + updatedJob.Spec.Template.Spec.Containers[0].Image = "busybox:v0.0.1" + updatedJSONJSON := marshalK8sObjJSON(updatedJob) + + // Create a new Work object with all the manifest JSONs and proper apply strategy. + applyStrategy := &fleetv1beta1.ApplyStrategy{ + ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, + Type: fleetv1beta1.ApplyStrategyTypeReportDiff, + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeNever, + } + createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON, updatedJSONJSON) + }) + + It("should add cleanup finalizer to the Work object", func() { + finalizerAddedActual := workFinalizerAddedActual(workName) + Eventually(finalizerAddedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to add cleanup finalizer to the Work object") + }) + + It("should prepare an AppliedWork object", func() { + appliedWorkCreatedActual := appliedWorkCreatedActual(workName) + Eventually(appliedWorkCreatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to prepare an AppliedWork object") + + appliedWorkOwnerRef = prepareAppliedWorkOwnerRef(workName) + }) + + It("should update the Work object status", func() { + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionFalse, + Reason: condition.WorkNotAllManifestsDiffReportedReason, + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeNoDiffFound), + ObservedGeneration: 0, + }, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "batch", + Version: "v1", + Kind: "Job", + Resource: "jobs", + Name: jobName, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeFoundDiffInDegradedMode), + ObservedGeneration: 1, + }, + }, + DiffDetails: &fleetv1beta1.DiffDetails{ + ObservedInMemberClusterGeneration: ®ularJob.Generation, + ObservedDiffs: []fleetv1beta1.PatchDetail{ + { + Path: "/spec/completions", + ValueInMember: "2", + ValueInHub: "3", + }, + { + Path: "/spec/template/spec/containers/0/image", + ValueInMember: "busybox", + ValueInHub: "busybox:v0.0.1", + }, + }, + }, + }, + } + + // Use custom status comparison logic as in this test case diff calculation is expected + // to run in degraded mode, which includes additional dynamic output that need to be + // filtered out. + Eventually(func() error { + // Retrieve the Work object. + work := &fleetv1beta1.Work{} + if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName1}, work); err != nil { + return fmt.Errorf("failed to retrieve the Work object: %w", err) + } + + // Prepare the expected Work object status. + + // Update the conditions with the observed generation. + // + // Note that the observed generation of a manifest condition is that of an applied + // resource, not that of the Work object. + for idx := range workConds { + workConds[idx].ObservedGeneration = work.Generation + } + wantWorkStatus := fleetv1beta1.WorkStatus{ + Conditions: workConds, + ManifestConditions: manifestConds, + } + + // Check that the Work object status has been updated as expected. + if diff := cmp.Diff( + work.Status, wantWorkStatus, + ignoreFieldConditionLTTMsg, + ignoreDiffDetailsObsTime, ignoreDriftDetailsObsTime, + cmpopts.SortSlices(lessFuncPatchDetail), + cmpopts.IgnoreSliceElements(func(d fleetv1beta1.PatchDetail) bool { + return d.Path != "/spec/completions" && d.Path != "/spec/template/spec/containers/0/image" + }), + ); diff != "" { + return fmt.Errorf("work status diff (-got, +want):\n%s", diff) + } + + // For simplicity reasons, the diff timestamps are not checked. + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update work status") + }) + + It("should not own the objects or apply the manifests to them", func() { + Consistently(func() error { + // Retrieve the NS object. + updatedNS := &corev1.Namespace{} + if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { + return fmt.Errorf("failed to retrieve the NS object: %w", err) + } + + // Rebuild the NS object to ignore default values automatically. + rebuiltGotNS := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: updatedNS.Name, + OwnerReferences: updatedNS.OwnerReferences, + }, + } + + wantNS := ns.DeepCopy() + wantNS.Name = nsName + if diff := cmp.Diff(rebuiltGotNS, wantNS, ignoreFieldTypeMetaInNamespace); diff != "" { + return fmt.Errorf("namespace diff (-got +want):\n%s", diff) + } + return nil + }, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to leave the NS object alone") + + Consistently(func() error { + // Retrieve the Job object. + updatedJob := &batchv1.Job{} + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: jobName}, updatedJob); err != nil { + return fmt.Errorf("failed to retrieve the Job object: %w", err) + } + + // Rebuild the Job object to ignore default values automatically. + if len(updatedJob.Spec.Template.Spec.Containers) != 1 { + return fmt.Errorf("unexpected number of containers in the Job pod template spec: %d, want 1", len(updatedJob.Spec.Template.Spec.Containers)) + } + rebuiltGotJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: updatedJob.Namespace, + Name: updatedJob.Name, + OwnerReferences: updatedJob.OwnerReferences, + }, + Spec: batchv1.JobSpec{ + Parallelism: updatedJob.Spec.Parallelism, + Completions: updatedJob.Spec.Completions, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": updatedJob.Spec.Template.ObjectMeta.Labels["app"], + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: updatedJob.Spec.Template.Spec.Containers[0].Name, + Image: updatedJob.Spec.Template.Spec.Containers[0].Image, + Command: updatedJob.Spec.Template.Spec.Containers[0].Command, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + }, + }, + } + + wantJob := job.DeepCopy() + wantJob.TypeMeta = metav1.TypeMeta{} + wantJob.Namespace = nsName + wantJob.Name = jobName + + if diff := cmp.Diff(rebuiltGotJob, wantJob); diff != "" { + return fmt.Errorf("job diff (-got +want):\n%s", diff) + } + return nil + }, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to leave the Job object alone") + }) + + It("should have no applied object reportings in the AppliedWork status", func() { + // Prepare the status information. + var appliedResourceMeta []fleetv1beta1.AppliedResourceMeta + + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update appliedWork status") + }) + + AfterAll(func() { + // Delete the Work object and related resources. + deleteWorkObject(workName, memberReservedNSName1) + + // Ensure that the Job object has been left alone. + jobNotRemovedActual := regularJobNotRemovedActual(nsName, jobName) + Consistently(jobNotRemovedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to remove the job object") + + // Ensure that the AppliedWork object has been removed. + appliedWorkRemovedActual := appliedWorkRemovedActual(workName, nsName) + Eventually(appliedWorkRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the AppliedWork object") + + workRemovedActual := workRemovedActual(workName) + Eventually(workRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Work object") + + // The environment prepared by the envtest package does not support namespace + // deletion; consequently this test suite would not attempt so verify its deletion. + }) + }) + + Context("obscure sensitive data (report diff mode)", Ordered, func() { + workName := fmt.Sprintf(workNameTemplate, utils.RandStr()) + // The environment prepared by the envtest package does not support namespace + // deletion; each test case would use a new namespace. + nsName := fmt.Sprintf(nsNameTemplate, utils.RandStr()) + + var regularNS *corev1.Namespace + var regularCM *corev1.ConfigMap + var regularSecret *corev1.Secret + + BeforeAll(func() { + // Prepare a NS object. + regularNS = ns.DeepCopy() + regularNS.Name = nsName + regularNSJSON := marshalK8sObjJSON(regularNS) + + // Create the namespace on the member cluster side. + Expect(memberClient1.Create(ctx, regularNS.DeepCopy())).To(Succeed(), "Failed to create the NS object") + + // Prepare a ConfigMap object. + regularCM = configMap.DeepCopy() + regularCM.Namespace = nsName + regularCMJSON := marshalK8sObjJSON(regularCM) + + // Create the ConfigMap object on the member cluster side. + Expect(memberClient1.Create(ctx, regularCM.DeepCopy())).To(Succeed(), "Failed to create the ConfigMap object") + + // Prepare a Secret object. + regularSecret = secret.DeepCopy() + regularSecret.Namespace = nsName + regularSecretJSON := marshalK8sObjJSON(regularSecret) + + // Create the Secret object on the member cluster side. + Expect(memberClient1.Create(ctx, regularSecret.DeepCopy())).To(Succeed(), "Failed to create the Secret object") + + // Create a new Work object with all the manifest JSONs and proper apply strategy. + applyStrategy := &fleetv1beta1.ApplyStrategy{ + Type: fleetv1beta1.ApplyStrategyTypeReportDiff, + WhenToTakeOver: fleetv1beta1.WhenToTakeOverTypeNever, + ComparisonOption: fleetv1beta1.ComparisonOptionTypePartialComparison, + } + createWorkObject(workName, memberReservedNSName1, applyStrategy, regularNSJSON, regularCMJSON, regularSecretJSON) + }) + + It("should add cleanup finalizer to the Work object", func() { + finalizerAddedActual := workFinalizerAddedActual(workName) + Eventually(finalizerAddedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to add cleanup finalizer to the Work object") + }) + + It("should prepare an AppliedWork object", func() { + appliedWorkCreatedActual := appliedWorkCreatedActual(workName) + Eventually(appliedWorkCreatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to prepare an AppliedWork object") + }) + + // For simplicity reasons, this test spec will not verify individual fields on the objects, + // but only the owner references (no owner should be present). + It("should not own (take over) any object", func() { + Consistently(func() error { + // Verify that the regular NS, CM, and Secret objects do not have the appliedWorkOwnerRef as their owner. + ns := &corev1.Namespace{} + if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, ns); err != nil { + return fmt.Errorf("Failed to retrieve the namespace object: %w", err) + } + if len(ns.OwnerReferences) > 0 { + return fmt.Errorf("Namespace %s has owner references in presence: %v", nsName, ns.OwnerReferences) + } + + cm := &corev1.ConfigMap{} + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: configMapName}, cm); err != nil { + return fmt.Errorf("Failed to retrieve the ConfigMap object: %w", err) + } + if len(cm.OwnerReferences) > 0 { + return fmt.Errorf("ConfigMap %s has owner references in presence: %v", configMapName, cm.OwnerReferences) + } + + sec := &corev1.Secret{} + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: secretName}, sec); err != nil { + return fmt.Errorf("Failed to retrieve the Secret object: %w", err) + } + if len(sec.OwnerReferences) > 0 { + return fmt.Errorf("Secret %s has owner references in presence: %v", secretName, sec.OwnerReferences) + } + return nil + }, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to leave the objects alone") + }) + + It("should update the Work object status", func() { + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsDiffReportedReason, + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeNoDiffFound), + ObservedGeneration: 0, + }, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "", + Version: "v1", + Kind: "ConfigMap", + Resource: "configmaps", + Name: configMapName, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeNoDiffFound), + ObservedGeneration: 0, + }, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 2, + Group: "", + Version: "v1", + Kind: "Secret", + Resource: "secrets", + Name: secretName, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeNoDiffFound), + ObservedGeneration: 0, + }, + }, + }, + } + + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, nil, nil) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update work status") + }) + + It("should update the AppliedWork object status", func() { + // Prepare the status information. + appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, nil) + Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update appliedWork status") + }) + + It("can make changes to the objects", func() { + // Use Eventually blocks to avoid conflicts. + Eventually(func() error { + // Retrieve the ConfigMap object. + updatedCM := &corev1.ConfigMap{} + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: configMapName}, updatedCM); err != nil { + return fmt.Errorf("failed to retrieve the ConfigMap object: %w", err) + } + + // Update the ConfigMap object. + if updatedCM.Labels == nil { + updatedCM.Labels = map[string]string{} + } + // Add a label and modify the data entry. + updatedCM.Labels[dummyLabelKey] = dummyLabelValue1 + updatedCM.Data[dummyLabelKey] = dummyLabelValue2 + if err := memberClient1.Update(ctx, updatedCM); err != nil { + return fmt.Errorf("failed to update the ConfigMap object: %w", err) + } + + return nil + }).Should(Succeed(), "Failed to make changes to the ConfigMap object") + + Eventually(func() error { + // Retrieve the Secret object. + updatedSecret := &corev1.Secret{} + if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: secretName}, updatedSecret); err != nil { + return fmt.Errorf("failed to retrieve the Secret object: %w", err) + } + + // Update the Secret object; modify the data entry and add a new string data entry. + dummyLabelValue2Bytes := []byte(dummyLabelValue2) + b64encoded := make([]byte, base64.StdEncoding.EncodedLen(len(dummyLabelValue2Bytes))) + base64.StdEncoding.Encode(b64encoded, dummyLabelValue2Bytes) + updatedSecret.Data[dummyLabelKey] = b64encoded + + updatedSecret.StringData = map[string]string{ + "fooStr": dummyLabelValue3, + } + + if err := memberClient1.Update(ctx, updatedSecret); err != nil { + return fmt.Errorf("failed to update the Secret object: %w", err) + } + + return nil + }).Should(Succeed(), "Failed to make changes to the Secret object") + }) + + It("should update the Work object status", func() { + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsDiffReportedReason, + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { Identifier: fleetv1beta1.WorkResourceIdentifier{ Ordinal: 0, Group: "", @@ -6337,172 +7401,207 @@ var _ = Describe("report diff", func() { { Identifier: fleetv1beta1.WorkResourceIdentifier{ Ordinal: 1, - Group: "batch", + Group: "", Version: "v1", - Kind: "Job", - Resource: "jobs", - Name: jobName, + Kind: "ConfigMap", + Resource: "configmaps", + Name: configMapName, Namespace: nsName, }, Conditions: []metav1.Condition{ { Type: fleetv1beta1.WorkConditionTypeDiffReported, Status: metav1.ConditionTrue, - Reason: string(ApplyOrReportDiffResTypeFoundDiffInDegradedMode), - ObservedGeneration: 1, + Reason: string(ApplyOrReportDiffResTypeFoundDiff), + ObservedGeneration: 0, }, }, DiffDetails: &fleetv1beta1.DiffDetails{ - ObservedInMemberClusterGeneration: ®ularJob.Generation, + ObservedInMemberClusterGeneration: ®ularCM.Generation, ObservedDiffs: []fleetv1beta1.PatchDetail{ { - Path: "/spec/completions", - ValueInMember: "2", - ValueInHub: "3", + Path: fmt.Sprintf("/data/%s", dummyLabelKey), + ValueInMember: dummyLabelValue2, + ValueInHub: dummyLabelValue1, }, + }, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 2, + Group: "", + Version: "v1", + Kind: "Secret", + Resource: "secrets", + Name: secretName, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeFoundDiff), + ObservedGeneration: 0, + }, + }, + DiffDetails: &fleetv1beta1.DiffDetails{ + ObservedInMemberClusterGeneration: ®ularSecret.Generation, + ObservedDiffs: []fleetv1beta1.PatchDetail{ { - Path: "/spec/template/spec/containers/0/image", - ValueInMember: "busybox", - ValueInHub: "busybox:v0.0.1", + Path: fmt.Sprintf("/data/%s", dummyLabelKey), + ValueInHub: "(redacted for security reasons)", + ValueInMember: "(redacted for security reasons)", }, }, }, }, } - // Use custom status comparison logic as in this test case diff calculation is expected - // to run in degraded mode, which includes additional dynamic output that need to be - // filtered out. + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, nil, nil) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update work status") + }) + + It("can switch to full comparison mode", func() { + // Use Eventually block to avoid potential conflicts. Eventually(func() error { - // Retrieve the Work object. work := &fleetv1beta1.Work{} - if err := hubClient.Get(ctx, client.ObjectKey{Name: workName, Namespace: memberReservedNSName1}, work); err != nil { + if err := hubClient.Get(ctx, client.ObjectKey{Namespace: memberReservedNSName1, Name: workName}, work); err != nil { return fmt.Errorf("failed to retrieve the Work object: %w", err) } - // Prepare the expected Work object status. - - // Update the conditions with the observed generation. - // - // Note that the observed generation of a manifest condition is that of an applied - // resource, not that of the Work object. - for idx := range workConds { - workConds[idx].ObservedGeneration = work.Generation - } - wantWorkStatus := fleetv1beta1.WorkStatus{ - Conditions: workConds, - ManifestConditions: manifestConds, - } - - // Check that the Work object status has been updated as expected. - if diff := cmp.Diff( - work.Status, wantWorkStatus, - ignoreFieldConditionLTTMsg, - ignoreDiffDetailsObsTime, ignoreDriftDetailsObsTime, - cmpopts.SortSlices(lessFuncPatchDetail), - cmpopts.IgnoreSliceElements(func(d fleetv1beta1.PatchDetail) bool { - return d.Path != "/spec/completions" && d.Path != "/spec/template/spec/containers/0/image" - }), - ); diff != "" { - return fmt.Errorf("work status diff (-got, +want):\n%s", diff) + if work.Spec.ApplyStrategy == nil { + work.Spec.ApplyStrategy = &fleetv1beta1.ApplyStrategy{} } - - // For simplicity reasons, the diff timestamps are not checked. - return nil - }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update work status") + work.Spec.ApplyStrategy.ComparisonOption = fleetv1beta1.ComparisonOptionTypeFullComparison + return hubClient.Update(ctx, work) + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to switch to full comparison mode") }) - It("should not own the objects or apply the manifests to them", func() { - Consistently(func() error { - // Retrieve the NS object. - updatedNS := &corev1.Namespace{} - if err := memberClient1.Get(ctx, client.ObjectKey{Name: nsName}, updatedNS); err != nil { - return fmt.Errorf("failed to retrieve the NS object: %w", err) - } - - // Rebuild the NS object to ignore default values automatically. - rebuiltGotNS := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: updatedNS.Name, - OwnerReferences: updatedNS.OwnerReferences, + It("should update the Work object status", func() { + // Prepare the status information. + workConds := []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: condition.WorkAllManifestsDiffReportedReason, + }, + } + manifestConds := []fleetv1beta1.ManifestCondition{ + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 0, + Group: "", + Version: "v1", + Kind: "Namespace", + Resource: "namespaces", + Name: nsName, }, - } - - wantNS := ns.DeepCopy() - wantNS.Name = nsName - if diff := cmp.Diff(rebuiltGotNS, wantNS, ignoreFieldTypeMetaInNamespace); diff != "" { - return fmt.Errorf("namespace diff (-got +want):\n%s", diff) - } - return nil - }, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to leave the NS object alone") - - Consistently(func() error { - // Retrieve the Job object. - updatedJob := &batchv1.Job{} - if err := memberClient1.Get(ctx, client.ObjectKey{Namespace: nsName, Name: jobName}, updatedJob); err != nil { - return fmt.Errorf("failed to retrieve the Job object: %w", err) - } - - // Rebuild the Job object to ignore default values automatically. - if len(updatedJob.Spec.Template.Spec.Containers) != 1 { - return fmt.Errorf("unexpected number of containers in the Job pod template spec: %d, want 1", len(updatedJob.Spec.Template.Spec.Containers)) - } - rebuiltGotJob := &batchv1.Job{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: updatedJob.Namespace, - Name: updatedJob.Name, - OwnerReferences: updatedJob.OwnerReferences, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeFoundDiff), + ObservedGeneration: 0, + }, }, - Spec: batchv1.JobSpec{ - Parallelism: updatedJob.Spec.Parallelism, - Completions: updatedJob.Spec.Completions, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "app": updatedJob.Spec.Template.ObjectMeta.Labels["app"], - }, + DiffDetails: &fleetv1beta1.DiffDetails{ + ObservedInMemberClusterGeneration: ®ularNS.Generation, + ObservedDiffs: []fleetv1beta1.PatchDetail{ + { + Path: "/spec/finalizers", + ValueInMember: "[kubernetes]", }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: updatedJob.Spec.Template.Spec.Containers[0].Name, - Image: updatedJob.Spec.Template.Spec.Containers[0].Image, - Command: updatedJob.Spec.Template.Spec.Containers[0].Command, - }, - }, - RestartPolicy: corev1.RestartPolicyNever, + }, + }, + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 1, + Group: "", + Version: "v1", + Kind: "ConfigMap", + Resource: "configmaps", + Name: configMapName, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeFoundDiff), + ObservedGeneration: 0, + }, + }, + DiffDetails: &fleetv1beta1.DiffDetails{ + ObservedInMemberClusterGeneration: ®ularCM.Generation, + ObservedDiffs: []fleetv1beta1.PatchDetail{ + { + Path: fmt.Sprintf("/data/%s", dummyLabelKey), + ValueInMember: dummyLabelValue2, + ValueInHub: dummyLabelValue1, + }, + { + Path: fmt.Sprintf("/metadata/labels/%s", dummyLabelKey), + ValueInMember: dummyLabelValue1, }, }, }, - } - - wantJob := job.DeepCopy() - wantJob.TypeMeta = metav1.TypeMeta{} - wantJob.Namespace = nsName - wantJob.Name = jobName - - if diff := cmp.Diff(rebuiltGotJob, wantJob); diff != "" { - return fmt.Errorf("job diff (-got +want):\n%s", diff) - } - return nil - }, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to leave the Job object alone") - }) - - It("should have no applied object reportings in the AppliedWork status", func() { - // Prepare the status information. - var appliedResourceMeta []fleetv1beta1.AppliedResourceMeta + }, + { + Identifier: fleetv1beta1.WorkResourceIdentifier{ + Ordinal: 2, + Group: "", + Version: "v1", + Kind: "Secret", + Resource: "secrets", + Name: secretName, + Namespace: nsName, + }, + Conditions: []metav1.Condition{ + { + Type: fleetv1beta1.WorkConditionTypeDiffReported, + Status: metav1.ConditionTrue, + Reason: string(ApplyOrReportDiffResTypeFoundDiff), + ObservedGeneration: 0, + }, + }, + DiffDetails: &fleetv1beta1.DiffDetails{ + ObservedInMemberClusterGeneration: ®ularSecret.Generation, + ObservedDiffs: []fleetv1beta1.PatchDetail{ + { + Path: fmt.Sprintf("/data/%s", dummyLabelKey), + ValueInHub: "(redacted for security reasons)", + ValueInMember: "(redacted for security reasons)", + }, + { + Path: fmt.Sprintf("/data/%s", "fooStr"), + ValueInMember: "(redacted for security reasons)", + }, + { + Path: "/type", + ValueInMember: "Opaque", + }, + }, + }, + }, + } - appliedWorkStatusUpdatedActual := appliedWorkStatusUpdated(workName, appliedResourceMeta) - Eventually(appliedWorkStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update appliedWork status") + workStatusUpdatedActual := workStatusUpdated(workName, workConds, manifestConds, nil, nil) + Eventually(workStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update work status") }) AfterAll(func() { // Delete the Work object and related resources. deleteWorkObject(workName, memberReservedNSName1) - // Ensure that the Job object has been left alone. - jobNotRemovedActual := regularJobNotRemovedActual(nsName, jobName) - Consistently(jobNotRemovedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to remove the job object") + // Ensure that the ConfigMap object has been removed. + regularCMRemovedActual := regularConfigMapRemovedActual(nsName, configMapName) + Eventually(regularCMRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the ConfigMap object") + + // Ensure that the secret object has been removed. + regularSecretRemovedActual := regularSecretRemovedActual(nsName, secretName) + Eventually(regularSecretRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove the Secret object") // Ensure that the AppliedWork object has been removed. appliedWorkRemovedActual := appliedWorkRemovedActual(workName, nsName) diff --git a/pkg/controllers/workapplier/controller_test.go b/pkg/controllers/workapplier/controller_test.go index c94463ccf..5527b3573 100644 --- a/pkg/controllers/workapplier/controller_test.go +++ b/pkg/controllers/workapplier/controller_test.go @@ -46,6 +46,7 @@ const ( deployName = "deploy-1" jobName = "job-1" configMapName = "configmap-1" + secretName = "secret-1" nsName = "ns-1" clusterRoleName = "clusterrole-1" ) @@ -160,6 +161,20 @@ var ( }, } + secret = &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: nsName, + Name: secretName, + }, + Data: map[string][]byte{ + dummyLabelKey: []byte(dummyLabelValue1), + }, + } + clusterRole = &rbacv1.ClusterRole{ TypeMeta: metav1.TypeMeta{ APIVersion: "rbac.authorization.k8s.io/v1", diff --git a/pkg/controllers/workapplier/drift_detection_takeover.go b/pkg/controllers/workapplier/drift_detection_takeover.go index 530f9de4d..457fe2d9e 100644 --- a/pkg/controllers/workapplier/drift_detection_takeover.go +++ b/pkg/controllers/workapplier/drift_detection_takeover.go @@ -19,6 +19,7 @@ package workapplier import ( "context" "fmt" + "strings" "github.com/qri-io/jsonpointer" "github.com/wI2L/jsondiff" @@ -324,10 +325,17 @@ func preparePatchDetails(srcObj, destObj *unstructured.Unstructured) ([]fleetv1b return nil, wrappedErr } + // Prepare Fleet patch details from the JSON patches. details, err := organizeJSONPatchIntoFleetPatchDetails(patch, srcObjCopy.Object) if err != nil { return nil, fmt.Errorf("failed to organize JSON patch operations into Fleet patch details: %w", err) } + + // Obscure sensitive fields in the patch details. + // + // This currently only concerns Secret objects (core API group); all the drift/diff outputs regarding + // a Secret object's data (`.data` or `.stringData` fields) are obscured. + details = obscureSensitiveFieldsInPatchDetails(srcObj, details) return details, nil } @@ -369,3 +377,42 @@ func (r *Reconciler) removeLeftBehindAppliedWorkOwnerRefs(ctx context.Context, o return updatedOwnerRefs, nil } + +// obscureSensitiveFieldsInPatchDetails obscures sensitive fields from the patch details so that +// such information will not be included in the drift/diff outputs. +// +// At this moment fields in the following API objects are discarded: +// +// * `.data` and `.stringData` field (and their children) in all Secret objects (`core` API group). +// +// Note (chenyu1): there are other Kubernetes API objects that also feature sensitive data, +// such as TokenRequest and CertificateSigningRequest; these objects are not included in the list +// as they have the sensitive information in the status, which are not accounted for in the +// drift/diff calculation in the very beginning. +func obscureSensitiveFieldsInPatchDetails( + srcObj *unstructured.Unstructured, details []fleetv1beta1.PatchDetail, +) (sanitizedPatchDetails []fleetv1beta1.PatchDetail) { + // Verify if the object is a Secret. + if srcObj.GetAPIVersion() != "v1" || srcObj.GetKind() != "Secret" { + return details + } + + for idx := range details { + pd := &details[idx] + // Note (chenyu1): the string data field in the Secret object is provided by Kubernetes + // as a write-only field for convenience reasons; all entries shall be merged into the + // data field. Here the code still processes the field just for completeness reasons; + // in practice it will never be included as part of the patch details. + if strings.HasPrefix(pd.Path, "/data") || strings.HasPrefix(pd.Path, "/stringData") { + // Obscure all patch details that concerns the Secret object's data. + + if len(pd.ValueInHub) > 0 { + pd.ValueInHub = "(redacted for security reasons)" + } + if len(pd.ValueInMember) > 0 { + pd.ValueInMember = "(redacted for security reasons)" + } + } + } + return details +} diff --git a/pkg/controllers/workapplier/drift_detection_takeover_test.go b/pkg/controllers/workapplier/drift_detection_takeover_test.go index 161b6f685..40c9484e8 100644 --- a/pkg/controllers/workapplier/drift_detection_takeover_test.go +++ b/pkg/controllers/workapplier/drift_detection_takeover_test.go @@ -729,3 +729,151 @@ func TestOrganizeJSONPatchIntoFleetPatchDetails(t *testing.T) { }) } } + +// TestObscureSensitiveFieldsInPatchDetails tets the obscureSensitiveFieldsInPatchDetails function. +func TestObscureSensitiveFieldsInPatchDetails(t *testing.T) { + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "app", + }, + } + unstructuredSecret := toUnstructured(t, secret) + + testCases := []struct { + name string + // This object is only needed for API version/kind comparison reasons; its data will not be + // used in the test spec. + srcObj *unstructured.Unstructured + patchDetails []fleetv1beta1.PatchDetail + wantPatchDetails []fleetv1beta1.PatchDetail + }{ + { + name: "not a secret object (same API group, namespace)", + srcObj: toUnstructured(t, ns.DeepCopy()), + patchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/labels/foo", + ValueInMember: "bar", + }, + }, + wantPatchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/labels/foo", + ValueInMember: "bar", + }, + }, + }, + { + name: "not a secret object (same API group, config map)", + srcObj: toUnstructured(t, configMap.DeepCopy()), + patchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/data/foo", + ValueInMember: "bar", + }, + }, + wantPatchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/data/foo", + ValueInMember: "bar", + }, + }, + }, + { + name: "not a secret object (different API group)", + srcObj: toUnstructured(t, deploy.DeepCopy()), + patchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/labels/foo", + ValueInMember: "bar", + }, + }, + wantPatchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/labels/foo", + ValueInMember: "bar", + }, + }, + }, + { + name: "secret object (no data patches)", + srcObj: unstructuredSecret, + patchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/labels/foo", + ValueInMember: "bar", + }, + }, + wantPatchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/labels/foo", + ValueInMember: "bar", + }, + }, + }, + { + name: "secret object (data patches, hub diff only)", + srcObj: unstructuredSecret, + patchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/data/foo", + ValueInHub: "bar", + }, + }, + wantPatchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/data/foo", + ValueInHub: "(redacted for security reasons)", + }, + }, + }, + { + name: "secret object (data patches, member diff only)", + srcObj: unstructuredSecret, + patchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/data/foo", + ValueInMember: "bar", + }, + }, + wantPatchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/data/foo", + ValueInMember: "(redacted for security reasons)", + }, + }, + }, + { + name: "secret object (string data patches, both end diffs)", + srcObj: unstructuredSecret, + patchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/stringData/foo", + ValueInHub: "bar", + ValueInMember: "baz", + }, + }, + wantPatchDetails: []fleetv1beta1.PatchDetail{ + { + Path: "/stringData/foo", + ValueInHub: "(redacted for security reasons)", + ValueInMember: "(redacted for security reasons)", + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + updatedPD := obscureSensitiveFieldsInPatchDetails(tc.srcObj, tc.patchDetails) + if diff := cmp.Diff(updatedPD, tc.wantPatchDetails); diff != "" { + t.Errorf("patchDetails mismatches (-got, +want):\n%s", diff) + } + }) + } +}