diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a2266687..da8a87a46 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,34 +57,6 @@ jobs: ## Comma-separated list of files to upload files: ./it-coverage.xml;./ut-coverage.xml - e2e-tests-v1alpha1: - runs-on: ubuntu-latest - needs: [ - detect-noop, - ] - if: needs.detect-noop.outputs.noop != 'true' - steps: - - name: Set up Go - uses: actions/setup-go@v6 - with: - go-version: ${{ env.GO_VERSION }} - - - name: Check out code into the Go module directory - uses: actions/checkout@v5 - - - name: Install Kind - # Before updating the kind version to use, verify that the current kind image - # is still supported by the version. - run: | - go install sigs.k8s.io/kind@v0.20.0 - - - name: Run e2e tests - run: | - OUTPUT_TYPE=type=docker make docker-build-member-agent docker-build-hub-agent docker-build-refresh-token e2e-tests-v1alpha1 - env: - KUBECONFIG: '/home/runner/.kube/config' - HUB_SERVER_URL: 'https://172.19.0.2:6443' - e2e-tests: strategy: fail-fast: false @@ -129,13 +101,13 @@ jobs: - name: Install Ginkgo CLI run: | - go install github.com/onsi/ginkgo/v2/ginkgo@v2.19.1 + go install github.com/onsi/ginkgo/v2/ginkgo@v2.23.4 - name: Install Kind # Before updating the kind version to use, verify that the current kind image # is still supported by the version. run: | - go install sigs.k8s.io/kind@v0.22.0 + go install sigs.k8s.io/kind@v0.30.0 - name: Run e2e tests run: | diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index fb98847ad..67f30593d 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -42,7 +42,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -56,7 +56,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v3 + uses: github/codeql-action/autobuild@v4 # ℹī¸ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun @@ -69,4 +69,4 @@ jobs: # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index 2b4848318..ca21beb6e 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -47,7 +47,7 @@ jobs: uses: actions/checkout@v5 - name: Login to ${{ env.REGISTRY }} - uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} diff --git a/Makefile b/Makefile index 14e17fd17..b65b6aab0 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ REGISTRY ?= ghcr.io -KIND_IMAGE ?= kindest/node:v1.31.0 +KIND_IMAGE ?= kindest/node:v1.33.4 ifndef TAG TAG ?= $(shell git rev-parse --short=7 HEAD) endif @@ -20,6 +20,30 @@ TARGET_ARCH ?= amd64 # progress. BUILDKIT_PROGRESS_TYPE ?= auto +TARGET_OS ?= linux +TARGET_ARCH ?= amd64 +AUTO_DETECT_ARCH ?= TRUE + +# Auto-detect system architecture if it is allowed and the necessary commands are available on the system. +ifeq ($(AUTO_DETECT_ARCH), TRUE) +ARCH_CMD_INSTALLED := $(shell command -v arch 2>/dev/null) +ifdef ARCH_CMD_INSTALLED +TARGET_ARCH := $(shell arch) +# The arch command may return arch strings that are aliases of expected TARGET_ARCH values; +# do the mapping here. +ifeq ($(TARGET_ARCH),$(filter $(TARGET_ARCH),x86_64)) + TARGET_ARCH := amd64 +else ifeq ($(TARGET_ARCH),$(filter $(TARGET_ARCH),aarch64 arm)) + TARGET_ARCH := arm64 +endif +$(info Auto-detected system architecture: $(TARGET_ARCH)) +endif +endif + +# Note (chenyu1): switch to the `plain` progress type to see the full outputs in the docker build +# progress. +BUILDKIT_PROGRESS_TYPE ?= auto + KUBECONFIG ?= $(HOME)/.kube/config HUB_SERVER_URL ?= https://172.19.0.2:6443 @@ -311,6 +335,13 @@ push: # On some systems the emulation setup might not work at all (e.g., macOS on Apple Silicon -> Rosetta 2 will be used # by Docker Desktop as the default emulation option for AMD64 on ARM64 container compatibility). .PHONY: docker-buildx-builder +# Note (chenyu1): the step below sets up emulation for building/running non-native binaries on the host. The original +# setup assumes that the Makefile is always run on an x86_64 platform, and adds support for non-x86_64 hosts. Here +# we keep the original setup if the build target is x86_64 platforms (default) for compatibility reasons, but will switch to +# a more general setup for non-x86_64 hosts. +# +# On some systems the emulation setup might not work at all (e.g., macOS on Apple Silicon -> Rosetta 2 will be used +# by Docker Desktop as the default emulation option for AMD64 on ARM64 container compatibility). docker-buildx-builder: @if ! docker buildx ls | grep $(BUILDX_BUILDER_NAME); then \ if [ "$(TARGET_ARCH)" = "amd64" ] ; then \ diff --git a/apis/placement/v1beta1/clusterresourceplacement_types.go b/apis/placement/v1beta1/clusterresourceplacement_types.go index 3ffde5098..9809e5b47 100644 --- a/apis/placement/v1beta1/clusterresourceplacement_types.go +++ b/apis/placement/v1beta1/clusterresourceplacement_types.go @@ -539,6 +539,11 @@ type RolloutStrategy struct { // DeleteStrategy configures the deletion behavior when the ClusterResourcePlacement is deleted. // +kubebuilder:validation:Optional DeleteStrategy *DeleteStrategy `json:"deleteStrategy,omitempty"` + + // ReportBackStrategy describes how to report back the status of applied resources on the member cluster. + // +kubebuilder:validation:Optional + // +kubebuilder:validation:XValidation:rule="(self == null) || (self.type == 'Mirror' ? size(self.destination) != 0 : true)",message="when reportBackStrategy.type is 'Mirror', a destination must be specified" + ReportBackStrategy *ReportBackStrategy `json:"reportBackStrategy,omitempty"` } // ApplyStrategy describes when and how to apply the selected resource to the target cluster. @@ -1480,6 +1485,66 @@ const ( DeletePropagationPolicyDelete DeletePropagationPolicy = "Delete" ) +type ReportBackStrategyType string + +const ( + // ReportBackStrategyTypeDisabled disables status back-reporting from the member clusters. + ReportBackStrategyTypeDisabled ReportBackStrategyType = "Disabled" + + // ReportBackStrategyTypeMirror enables status back-reporting by + // copying the status fields verbatim to some destination on the hub cluster side. + ReportBackStrategyTypeMirror ReportBackStrategyType = "Mirror" +) + +type ReportBackDestination string + +const ( + // ReportBackDestinationOriginalResource implies the status fields will be copied verbatim to the + // the original resource on the hub cluster side. This is only performed when the placement object has a + // scheduling policy that selects exactly one member cluster (i.e., a pickFixed scheduling policy with + // exactly one cluster name, or a pickN scheduling policy with the numberOfClusters field set to 1). + ReportBackDestinationOriginalResource ReportBackDestination = "OriginalResource" + + // ReportBackDestinationWorkAPI implies the status fields will be copied verbatim via the Work API + // on the hub cluster side. Users may look up the status of a specific resource applied to a specific + // member cluster by inspecting the corresponding Work object on the hub cluster side. + ReportBackDestinationWorkAPI ReportBackDestination = "WorkAPI" +) + +// ReportBackStrategy describes how to report back the resource status from member clusters. +type ReportBackStrategy struct { + // Type dictates the type of the report back strategy to use. + // + // Available options include: + // + // * Disabled: status back-reporting is disabled. This is the default behavior. + // + // * Mirror: status back-reporting is enabled by copying the status fields verbatim to + // a destination on the hub cluster side; see the Destination field for more information. + // + // +kubebuilder:default=Disabled + // +kubebuilder:validation:Enum=Disabled;Mirror + // +kubebuilder:validation:Required + Type ReportBackStrategyType `json:"type"` + + // Destination dictates where to copy the status fields to when the report back strategy type is Mirror. + // + // Available options include: + // + // * OriginalResource: the status fields will be copied verbatim to the original resource on the hub cluster side. + // This is only performed when the placement object has a scheduling policy that selects exactly one member cluster + // (i.e., a pickFixed scheduling policy with exactly one cluster name, or a pickN scheduling policy with the numberOfClusters + // field set to 1). + // + // * WorkAPI: the status fields will be copied verbatim via the Work API on the hub cluster side. Users may look up + // the status of a specific resource applied to a specific member cluster by inspecting the corresponding Work object + // on the hub cluster side. This is the default behavior. + // + // +kubebuilder:validation:Enum=OriginalResource;WorkAPI + // +kubebuilder:validation:Optional + Destination *ReportBackDestination `json:"destination,omitempty"` +} + // ClusterResourcePlacementList contains a list of ClusterResourcePlacement. // +kubebuilder:resource:scope="Cluster" // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/apis/placement/v1beta1/commons.go b/apis/placement/v1beta1/commons.go index 5ac8cb495..03e8e9dc0 100644 --- a/apis/placement/v1beta1/commons.go +++ b/apis/placement/v1beta1/commons.go @@ -47,6 +47,12 @@ const ( ClusterStagedUpdateStrategyKind = "ClusterStagedUpdateStrategy" // ClusterApprovalRequestKind is the kind of the ClusterApprovalRequest. ClusterApprovalRequestKind = "ClusterApprovalRequest" + // StagedUpdateRunKind is the kind of the StagedUpdateRun. + StagedUpdateRunKind = "StagedUpdateRun" + // StagedUpdateStrategyKind is the kind of the StagedUpdateStrategy. + StagedUpdateStrategyKind = "StagedUpdateStrategy" + // ApprovalRequestKind is the kind of the ApprovalRequest. + ApprovalRequestKind = "ApprovalRequest" // ClusterResourcePlacementEvictionKind is the kind of the ClusterResourcePlacementEviction. ClusterResourcePlacementEvictionKind = "ClusterResourcePlacementEviction" // ClusterResourcePlacementDisruptionBudgetKind is the kind of the ClusterResourcePlacementDisruptionBudget. @@ -145,9 +151,9 @@ const ( // This is used to remember if an "unscheduled" binding was moved from a "bound" state or a "scheduled" state. PreviousBindingStateAnnotation = FleetPrefix + "previous-binding-state" - // ClusterStagedUpdateRunFinalizer is used by the ClusterStagedUpdateRun controller to make sure that the ClusterStagedUpdateRun + // UpdateRunFinalizer is used by the UpdateRun controller to make sure that the UpdateRun // object is not deleted until all its dependent resources are deleted. - ClusterStagedUpdateRunFinalizer = FleetPrefix + "stagedupdaterun-finalizer" + UpdateRunFinalizer = FleetPrefix + "stagedupdaterun-finalizer" // TargetUpdateRunLabel indicates the target update run on a staged run related object. TargetUpdateRunLabel = FleetPrefix + "targetupdaterun" diff --git a/apis/placement/v1beta1/stageupdate_types.go b/apis/placement/v1beta1/stageupdate_types.go index ac216c4ef..544c9b77a 100644 --- a/apis/placement/v1beta1/stageupdate_types.go +++ b/apis/placement/v1beta1/stageupdate_types.go @@ -17,9 +17,69 @@ limitations under the License. package v1beta1 import ( + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "go.goms.io/fleet/apis" ) +// make sure the UpdateRunObj and UpdateRunObjList interfaces are implemented by the +// ClusterStagedUpdateRun and StagedUpdateRun types. +var _ UpdateRunObj = &ClusterStagedUpdateRun{} +var _ UpdateRunObj = &StagedUpdateRun{} +var _ UpdateRunObjList = &ClusterStagedUpdateRunList{} +var _ UpdateRunObjList = &StagedUpdateRunList{} + +// make sure the UpdateStrategyObj and UpdateStrategyObjList interfaces are implemented by the +// ClusterStagedUpdateStrategy and StagedUpdateStrategy types. +var _ UpdateStrategyObj = &ClusterStagedUpdateStrategy{} +var _ UpdateStrategyObj = &StagedUpdateStrategy{} +var _ UpdateStrategyObjList = &ClusterStagedUpdateStrategyList{} +var _ UpdateStrategyObjList = &StagedUpdateStrategyList{} + +// make sure the ApprovalRequestObj and ApprovalRequestObjList interfaces are implemented by the +// ClusterApprovalRequest and ApprovalRequest types. +var _ ApprovalRequestObj = &ClusterApprovalRequest{} +var _ ApprovalRequestObj = &ApprovalRequest{} +var _ ApprovalRequestObjList = &ClusterApprovalRequestList{} +var _ ApprovalRequestObjList = &ApprovalRequestList{} + +// UpdateRunSpecGetterSetter offers the functionality to work with UpdateRunSpec. +// +kubebuilder:object:generate=false +type UpdateRunSpecGetterSetter interface { + GetUpdateRunSpec() *UpdateRunSpec + SetUpdateRunSpec(UpdateRunSpec) +} + +// UpdateRunStatusGetterSetter offers the functionality to work with UpdateRunStatus. +// +kubebuilder:object:generate=false +type UpdateRunStatusGetterSetter interface { + GetUpdateRunStatus() *UpdateRunStatus + SetUpdateRunStatus(UpdateRunStatus) +} + +// UpdateRunObj offers the functionality to work with staged update run objects, including ClusterStagedUpdateRuns and StagedUpdateRuns. +// +kubebuilder:object:generate=false +type UpdateRunObj interface { + apis.ConditionedObj + UpdateRunSpecGetterSetter + UpdateRunStatusGetterSetter +} + +// UpdateRunListItemGetter offers the functionality to get a list of UpdateRunObj items. +// +kubebuilder:object:generate=false +type UpdateRunListItemGetter interface { + GetUpdateRunObjs() []UpdateRunObj +} + +// UpdateRunObjList offers the functionality to work with staged update run object list. +// +kubebuilder:object:generate=false +type UpdateRunObjList interface { + client.ObjectList + UpdateRunListItemGetter +} + // +genclient // +genclient:Cluster // +kubebuilder:object:root=true @@ -49,16 +109,46 @@ type ClusterStagedUpdateRun struct { // The desired state of ClusterStagedUpdateRun. The spec is immutable. // +kubebuilder:validation:Required // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="The spec field is immutable" - Spec StagedUpdateRunSpec `json:"spec"` + Spec UpdateRunSpec `json:"spec"` // The observed status of ClusterStagedUpdateRun. // +kubebuilder:validation:Optional - Status StagedUpdateRunStatus `json:"status,omitempty"` + Status UpdateRunStatus `json:"status,omitempty"` +} + +// GetCondition returns the condition of the ClusterStagedUpdateRun. +func (c *ClusterStagedUpdateRun) GetCondition(conditionType string) *metav1.Condition { + return meta.FindStatusCondition(c.Status.Conditions, conditionType) +} + +// SetConditions sets the conditions of the ClusterStagedUpdateRun. +func (c *ClusterStagedUpdateRun) SetConditions(conditions ...metav1.Condition) { + c.Status.Conditions = conditions +} + +// GetUpdateRunSpec returns the staged update run spec. +func (c *ClusterStagedUpdateRun) GetUpdateRunSpec() *UpdateRunSpec { + return &c.Spec +} + +// SetUpdateRunSpec sets the staged update run spec. +func (c *ClusterStagedUpdateRun) SetUpdateRunSpec(spec UpdateRunSpec) { + c.Spec = spec +} + +// GetUpdateRunStatus returns the staged update run status. +func (c *ClusterStagedUpdateRun) GetUpdateRunStatus() *UpdateRunStatus { + return &c.Status +} + +// SetUpdateRunStatus sets the staged update run status. +func (c *ClusterStagedUpdateRun) SetUpdateRunStatus(status UpdateRunStatus) { + c.Status = status } -// StagedUpdateRunSpec defines the desired rollout strategy and the snapshot indices of the resources to be updated. +// UpdateRunSpec defines the desired rollout strategy and the snapshot indices of the resources to be updated. // It specifies a stage-by-stage update process across selected clusters for the given ResourcePlacement object. -type StagedUpdateRunSpec struct { +type UpdateRunSpec struct { // PlacementName is the name of placement that this update run is applied to. // There can be multiple active update runs for each placement, but // it's up to the DevOps team to ensure they don't conflict with each other. @@ -79,6 +169,33 @@ type StagedUpdateRunSpec struct { StagedUpdateStrategyName string `json:"stagedRolloutStrategyName"` } +// UpdateStrategySpecGetterSetter offers the functionality to work with UpdateStrategySpec. +// +kubebuilder:object:generate=false +type UpdateStrategySpecGetterSetter interface { + GetUpdateStrategySpec() *UpdateStrategySpec + SetUpdateStrategySpec(UpdateStrategySpec) +} + +// UpdateStrategyObj offers the functionality to work with staged update strategy objects, including ClusterStagedUpdateStrategies and StagedUpdateStrategies. +// +kubebuilder:object:generate=false +type UpdateStrategyObj interface { + client.Object + UpdateStrategySpecGetterSetter +} + +// UpdateStrategyListItemGetter offers the functionality to get a list of UpdateStrategyObj items. +// +kubebuilder:object:generate=false +type UpdateStrategyListItemGetter interface { + GetUpdateStrategyObjs() []UpdateStrategyObj +} + +// UpdateStrategyObjList offers the functionality to work with staged update strategy object list. +// +kubebuilder:object:generate=false +type UpdateStrategyObjList interface { + client.ObjectList + UpdateStrategyListItemGetter +} + // +genclient // +genclient:cluster // +kubebuilder:object:root=true @@ -95,11 +212,21 @@ type ClusterStagedUpdateStrategy struct { // The desired state of ClusterStagedUpdateStrategy. // +kubebuilder:validation:Required - Spec StagedUpdateStrategySpec `json:"spec"` + Spec UpdateStrategySpec `json:"spec"` +} + +// GetUpdateStrategySpec returns the staged update strategy spec. +func (c *ClusterStagedUpdateStrategy) GetUpdateStrategySpec() *UpdateStrategySpec { + return &c.Spec } -// StagedUpdateStrategySpec defines the desired state of the StagedUpdateStrategy. -type StagedUpdateStrategySpec struct { +// SetUpdateStrategySpec sets the staged update strategy spec. +func (c *ClusterStagedUpdateStrategy) SetUpdateStrategySpec(spec UpdateStrategySpec) { + c.Spec = spec +} + +// UpdateStrategySpec defines the desired state of the StagedUpdateStrategy. +type UpdateStrategySpec struct { // Stage specifies the configuration for each update stage. // +kubebuilder:validation:MaxItems=31 // +kubebuilder:validation:Required @@ -115,6 +242,15 @@ type ClusterStagedUpdateStrategyList struct { Items []ClusterStagedUpdateStrategy `json:"items"` } +// GetUpdateStrategyObjs returns the update strategy objects in the list. +func (c *ClusterStagedUpdateStrategyList) GetUpdateStrategyObjs() []UpdateStrategyObj { + objs := make([]UpdateStrategyObj, len(c.Items)) + for i := range c.Items { + objs[i] = &c.Items[i] + } + return objs +} + // StageConfig describes a single update stage. // The clusters in each stage are updated sequentially. // The update stops if any of the updates fail. @@ -162,8 +298,8 @@ type AfterStageTask struct { WaitTime *metav1.Duration `json:"waitTime,omitempty"` } -// StagedUpdateRunStatus defines the observed state of the ClusterStagedUpdateRun. -type StagedUpdateRunStatus struct { +// UpdateRunStatus defines the observed state of the ClusterStagedUpdateRun. +type UpdateRunStatus struct { // PolicySnapShotIndexUsed records the policy snapshot index of the ClusterResourcePlacement (CRP) that // the update run is based on. The index represents the latest policy snapshot at the start of the update run. // If a newer policy snapshot is detected after the run starts, the staged update run is abandoned. @@ -185,13 +321,13 @@ type StagedUpdateRunStatus struct { // +kubebuilder:validation:Optional ApplyStrategy *ApplyStrategy `json:"appliedStrategy,omitempty"` - // StagedUpdateStrategySnapshot is the snapshot of the StagedUpdateStrategy used for the update run. + // UpdateStrategySnapshot is the snapshot of the UpdateStrategy used for the update run. // The snapshot is immutable during the update run. // The strategy is applied to the list of clusters scheduled by the CRP according to the current policy. // The update run fails to initialize if the strategy fails to produce a valid list of stages where each selected // cluster is included in exactly one stage. // +kubebuilder:validation:Optional - StagedUpdateStrategySnapshot *StagedUpdateStrategySpec `json:"stagedUpdateStrategySnapshot,omitempty"` + UpdateStrategySnapshot *UpdateStrategySpec `json:"stagedUpdateStrategySnapshot,omitempty"` // StagesStatus lists the current updating status of each stage. // The list is empty if the update run is not started or failed to initialize. @@ -412,6 +548,50 @@ type ClusterStagedUpdateRunList struct { Items []ClusterStagedUpdateRun `json:"items"` } +// GetUpdateRunObjs returns the update run objects in the list. +func (c *ClusterStagedUpdateRunList) GetUpdateRunObjs() []UpdateRunObj { + objs := make([]UpdateRunObj, len(c.Items)) + for i := range c.Items { + objs[i] = &c.Items[i] + } + return objs +} + +// ApprovalRequestSpecGetterSetter offers the functionality to work with ApprovalRequestSpec. +// +kubebuilder:object:generate=false +type ApprovalRequestSpecGetterSetter interface { + GetApprovalRequestSpec() *ApprovalRequestSpec + SetApprovalRequestSpec(ApprovalRequestSpec) +} + +// ApprovalRequestStatusGetterSetter offers the functionality to work with ApprovalRequestStatus. +// +kubebuilder:object:generate=false +type ApprovalRequestStatusGetterSetter interface { + GetApprovalRequestStatus() *ApprovalRequestStatus + SetApprovalRequestStatus(ApprovalRequestStatus) +} + +// ApprovalRequestObj offers the functionality to work with approval request objects, including ClusterApprovalRequests and ApprovalRequests. +// +kubebuilder:object:generate=false +type ApprovalRequestObj interface { + apis.ConditionedObj + ApprovalRequestSpecGetterSetter + ApprovalRequestStatusGetterSetter +} + +// ApprovalRequestListItemGetter offers the functionality to get a list of ApprovalRequestObj items. +// +kubebuilder:object:generate=false +type ApprovalRequestListItemGetter interface { + GetApprovalRequestObjs() []ApprovalRequestObj +} + +// ApprovalRequestObjList offers the functionality to work with approval request object list. +// +kubebuilder:object:generate=false +type ApprovalRequestObjList interface { + client.ObjectList + ApprovalRequestListItemGetter +} + // +genclient // +genclient:Cluster // +kubebuilder:object:root=true @@ -443,6 +623,36 @@ type ClusterApprovalRequest struct { Status ApprovalRequestStatus `json:"status,omitempty"` } +// GetCondition returns the condition of the ClusterApprovalRequest. +func (c *ClusterApprovalRequest) GetCondition(conditionType string) *metav1.Condition { + return meta.FindStatusCondition(c.Status.Conditions, conditionType) +} + +// SetConditions sets the conditions of the ClusterApprovalRequest. +func (c *ClusterApprovalRequest) SetConditions(conditions ...metav1.Condition) { + c.Status.Conditions = conditions +} + +// GetApprovalRequestSpec returns the approval request spec. +func (c *ClusterApprovalRequest) GetApprovalRequestSpec() *ApprovalRequestSpec { + return &c.Spec +} + +// SetApprovalRequestSpec sets the approval request spec. +func (c *ClusterApprovalRequest) SetApprovalRequestSpec(spec ApprovalRequestSpec) { + c.Spec = spec +} + +// GetApprovalRequestStatus returns the approval request status. +func (c *ClusterApprovalRequest) GetApprovalRequestStatus() *ApprovalRequestStatus { + return &c.Status +} + +// SetApprovalRequestStatus sets the approval request status. +func (c *ClusterApprovalRequest) SetApprovalRequestStatus(status ApprovalRequestStatus) { + c.Status = status +} + // ApprovalRequestSpec defines the desired state of the update run approval request. // The entire spec is immutable. type ApprovalRequestSpec struct { @@ -492,8 +702,224 @@ type ClusterApprovalRequestList struct { Items []ClusterApprovalRequest `json:"items"` } +// GetApprovalRequestObjs returns the approval request objects in the list. +func (c *ClusterApprovalRequestList) GetApprovalRequestObjs() []ApprovalRequestObj { + objs := make([]ApprovalRequestObj, len(c.Items)) + for i := range c.Items { + objs[i] = &c.Items[i] + } + return objs +} + +// +genclient +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Namespaced,categories={fleet,fleet-placement},shortName=sur +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:storageversion +// +kubebuilder:printcolumn:JSONPath=`.spec.placementName`,name="Placement",type=string +// +kubebuilder:printcolumn:JSONPath=`.spec.resourceSnapshotIndex`,name="Resource-Snapshot-Index",type=string +// +kubebuilder:printcolumn:JSONPath=`.status.policySnapshotIndexUsed`,name="Policy-Snapshot-Index",type=string +// +kubebuilder:printcolumn:JSONPath=`.status.conditions[?(@.type=="Initialized")].status`,name="Initialized",type=string +// +kubebuilder:printcolumn:JSONPath=`.status.conditions[?(@.type=="Succeeded")].status`,name="Succeeded",type=string +// +kubebuilder:printcolumn:JSONPath=`.metadata.creationTimestamp`,name="Age",type=date +// +kubebuilder:printcolumn:JSONPath=`.spec.stagedRolloutStrategyName`,name="Strategy",priority=1,type=string +// +kubebuilder:validation:XValidation:rule="size(self.metadata.name) < 128",message="metadata.name max length is 127" + +// StagedUpdateRun represents a stage by stage update process that applies ResourcePlacement +// selected resources to specified clusters. +// Resources from unselected clusters are removed after all stages in the update strategy are completed. +// Each StagedUpdateRun object corresponds to a single release of a specific resource version. +// The release is abandoned if the StagedUpdateRun object is deleted or the scheduling decision changes. +// The name of the StagedUpdateRun must conform to RFC 1123. +type StagedUpdateRun struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // The desired state of StagedUpdateRun. The spec is immutable. + // +kubebuilder:validation:Required + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="The spec field is immutable" + Spec UpdateRunSpec `json:"spec"` + + // The observed status of StagedUpdateRun. + // +kubebuilder:validation:Optional + Status UpdateRunStatus `json:"status,omitempty"` +} + +// GetCondition returns the condition of the StagedUpdateRun. +func (s *StagedUpdateRun) GetCondition(conditionType string) *metav1.Condition { + return meta.FindStatusCondition(s.Status.Conditions, conditionType) +} + +// SetConditions sets the conditions of the StagedUpdateRun. +func (s *StagedUpdateRun) SetConditions(conditions ...metav1.Condition) { + s.Status.Conditions = conditions +} + +// GetUpdateRunSpec returns the staged update run spec. +func (s *StagedUpdateRun) GetUpdateRunSpec() *UpdateRunSpec { + return &s.Spec +} + +// SetUpdateRunSpec sets the staged update run spec. +func (s *StagedUpdateRun) SetUpdateRunSpec(spec UpdateRunSpec) { + s.Spec = spec +} + +// GetUpdateRunStatus returns the staged update run status. +func (s *StagedUpdateRun) GetUpdateRunStatus() *UpdateRunStatus { + return &s.Status +} + +// SetUpdateRunStatus sets the staged update run status. +func (s *StagedUpdateRun) SetUpdateRunStatus(status UpdateRunStatus) { + s.Status = status +} + +// StagedUpdateRunList contains a list of StagedUpdateRun. +// +kubebuilder:resource:scope=Namespaced +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type StagedUpdateRunList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []StagedUpdateRun `json:"items"` +} + +// GetUpdateRunObjs returns the update run objects in the list. +func (s *StagedUpdateRunList) GetUpdateRunObjs() []UpdateRunObj { + objs := make([]UpdateRunObj, len(s.Items)) + for i := range s.Items { + objs[i] = &s.Items[i] + } + return objs +} + +// +genclient +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Namespaced,categories={fleet,fleet-placement},shortName=sus +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:storageversion + +// StagedUpdateStrategy defines a reusable strategy that specifies the stages and the sequence +// in which the selected cluster resources will be updated on the member clusters. +type StagedUpdateStrategy struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // The desired state of StagedUpdateStrategy. + // +kubebuilder:validation:Required + Spec UpdateStrategySpec `json:"spec"` +} + +// GetUpdateStrategySpec returns the staged update strategy spec. +func (s *StagedUpdateStrategy) GetUpdateStrategySpec() *UpdateStrategySpec { + return &s.Spec +} + +// SetUpdateStrategySpec sets the staged update strategy spec. +func (s *StagedUpdateStrategy) SetUpdateStrategySpec(spec UpdateStrategySpec) { + s.Spec = spec +} + +// StagedUpdateStrategyList contains a list of StagedUpdateStrategy. +// +kubebuilder:resource:scope=Namespaced +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type StagedUpdateStrategyList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []StagedUpdateStrategy `json:"items"` +} + +// GetUpdateStrategyObjs returns the update strategy objects in the list. +func (s *StagedUpdateStrategyList) GetUpdateStrategyObjs() []UpdateStrategyObj { + objs := make([]UpdateStrategyObj, len(s.Items)) + for i := range s.Items { + objs[i] = &s.Items[i] + } + return objs +} + +// +genclient +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Namespaced,categories={fleet,fleet-placement},shortName=areq +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:storageversion +// +kubebuilder:printcolumn:JSONPath=`.spec.parentStageRollout`,name="Update-Run",type=string +// +kubebuilder:printcolumn:JSONPath=`.spec.targetStage`,name="Stage",type=string +// +kubebuilder:printcolumn:JSONPath=`.status.conditions[?(@.type=="Approved")].status`,name="Approved",type=string +// +kubebuilder:printcolumn:JSONPath=`.metadata.creationTimestamp`,name="Age",type=date + +// ApprovalRequest defines a request for user approval for staged update run. +// The request object MUST have the following labels: +// - `TargetUpdateRun`: Points to the staged update run that this approval request is for. +// - `TargetStage`: The name of the stage that this approval request is for. +// - `IsLatestUpdateRunApproval`: Indicates whether this approval request is the latest one related to this update run. +type ApprovalRequest struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + // The desired state of ApprovalRequest. + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="The spec field is immutable" + // +kubebuilder:validation:Required + Spec ApprovalRequestSpec `json:"spec"` + + // The observed state of ApprovalRequest. + // +kubebuilder:validation:Optional + Status ApprovalRequestStatus `json:"status,omitempty"` +} + +// GetCondition returns the condition of the ApprovalRequest. +func (a *ApprovalRequest) GetCondition(conditionType string) *metav1.Condition { + return meta.FindStatusCondition(a.Status.Conditions, conditionType) +} + +// SetConditions sets the conditions of the ApprovalRequest. +func (a *ApprovalRequest) SetConditions(conditions ...metav1.Condition) { + a.Status.Conditions = conditions +} + +// GetApprovalRequestSpec returns the approval request spec. +func (a *ApprovalRequest) GetApprovalRequestSpec() *ApprovalRequestSpec { + return &a.Spec +} + +// SetApprovalRequestSpec sets the approval request spec. +func (a *ApprovalRequest) SetApprovalRequestSpec(spec ApprovalRequestSpec) { + a.Spec = spec +} + +// GetApprovalRequestStatus returns the approval request status. +func (a *ApprovalRequest) GetApprovalRequestStatus() *ApprovalRequestStatus { + return &a.Status +} + +// SetApprovalRequestStatus sets the approval request status. +func (a *ApprovalRequest) SetApprovalRequestStatus(status ApprovalRequestStatus) { + a.Status = status +} + +// ApprovalRequestList contains a list of ApprovalRequest. +// +kubebuilder:resource:scope=Namespaced +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type ApprovalRequestList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ApprovalRequest `json:"items"` +} + +// GetApprovalRequestObjs returns the approval request objects in the list. +func (a *ApprovalRequestList) GetApprovalRequestObjs() []ApprovalRequestObj { + objs := make([]ApprovalRequestObj, len(a.Items)) + for i := range a.Items { + objs[i] = &a.Items[i] + } + return objs +} + func init() { SchemeBuilder.Register( &ClusterStagedUpdateRun{}, &ClusterStagedUpdateRunList{}, &ClusterStagedUpdateStrategy{}, &ClusterStagedUpdateStrategyList{}, &ClusterApprovalRequest{}, &ClusterApprovalRequestList{}, + &StagedUpdateRun{}, &StagedUpdateRunList{}, &StagedUpdateStrategy{}, &StagedUpdateStrategyList{}, &ApprovalRequest{}, &ApprovalRequestList{}, ) } diff --git a/apis/placement/v1beta1/work_types.go b/apis/placement/v1beta1/work_types.go index 4781e72ff..f4592f3d7 100644 --- a/apis/placement/v1beta1/work_types.go +++ b/apis/placement/v1beta1/work_types.go @@ -68,6 +68,10 @@ type WorkSpec struct { // and is owned by other appliers. // +optional ApplyStrategy *ApplyStrategy `json:"applyStrategy,omitempty"` + + // ReportBackStrategy describes how to report back the status of applied resources on the member cluster. + // +optional + ReportBackStrategy *ReportBackStrategy `json:"reportBackStrategy,omitempty"` } // WorkloadTemplate represents the manifest workload to be deployed on spoke cluster @@ -142,7 +146,7 @@ type DriftDetails struct { // +kubebuilder:validation:Required ObservedInMemberClusterGeneration int64 `json:"observedInMemberClusterGeneration"` - // FirsftDriftedObservedTime is the timestamp when the drift was first detected. + // FirstDriftedObservedTime is the timestamp when the drift was first detected. // // +kubebuilder:validation:Required // +kubebuilder:validation:Type=string @@ -176,7 +180,7 @@ type DiffDetails struct { // +kubebuilder:validation:Optional ObservedInMemberClusterGeneration *int64 `json:"observedInMemberClusterGeneration"` - // FirsftDiffedObservedTime is the timestamp when the configuration difference + // FirstDiffedObservedTime is the timestamp when the configuration difference // was first detected. // // +kubebuilder:validation:Required @@ -196,6 +200,19 @@ type DiffDetails struct { ObservedDiffs []PatchDetail `json:"observedDiffs,omitempty"` } +type BackReportedStatus struct { + // +kubebuilder:validation:EmbeddedResource + // +kubebuilder:pruning:PreserveUnknownFields + ObservedStatus runtime.RawExtension `json:"observedStatus,omitempty"` + + // ObservationTime is the timestamp when the status was last back reported. + // + // +kubebuilder:validation:Required + // +kubebuilder:validation:Type=string + // +kubebuilder:validation:Format=date-time + ObservationTime metav1.Time `json:"observationTime"` +} + // ManifestCondition represents the conditions of the resources deployed on // spoke cluster. type ManifestCondition struct { @@ -228,6 +245,11 @@ type ManifestCondition struct { // // +kubebuilder:validation:Optional DiffDetails *DiffDetails `json:"diffDetails,omitempty"` + + // BackReportedStatus is the status reported back from the member cluster (if applicable). + // + // +kubebuilder:validation:Optional + BackReportedStatus *BackReportedStatus `json:"backReportedStatus,omitempty"` } // +genclient diff --git a/apis/placement/v1beta1/zz_generated.deepcopy.go b/apis/placement/v1beta1/zz_generated.deepcopy.go index 05fef4c3d..487ead5f1 100644 --- a/apis/placement/v1beta1/zz_generated.deepcopy.go +++ b/apis/placement/v1beta1/zz_generated.deepcopy.go @@ -218,6 +218,65 @@ func (in *ApplyStrategy) DeepCopy() *ApplyStrategy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ApprovalRequest) DeepCopyInto(out *ApprovalRequest) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ApprovalRequest. +func (in *ApprovalRequest) DeepCopy() *ApprovalRequest { + if in == nil { + return nil + } + out := new(ApprovalRequest) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ApprovalRequest) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ApprovalRequestList) DeepCopyInto(out *ApprovalRequestList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ApprovalRequest, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ApprovalRequestList. +func (in *ApprovalRequestList) DeepCopy() *ApprovalRequestList { + if in == nil { + return nil + } + out := new(ApprovalRequestList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ApprovalRequestList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ApprovalRequestSpec) DeepCopyInto(out *ApprovalRequestSpec) { *out = *in @@ -255,6 +314,23 @@ func (in *ApprovalRequestStatus) DeepCopy() *ApprovalRequestStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackReportedStatus) DeepCopyInto(out *BackReportedStatus) { + *out = *in + in.ObservedStatus.DeepCopyInto(&out.ObservedStatus) + in.ObservationTime.DeepCopyInto(&out.ObservationTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackReportedStatus. +func (in *BackReportedStatus) DeepCopy() *BackReportedStatus { + if in == nil { + return nil + } + out := new(BackReportedStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterAffinity) DeepCopyInto(out *ClusterAffinity) { *out = *in @@ -1432,6 +1508,11 @@ func (in *ManifestCondition) DeepCopyInto(out *ManifestCondition) { *out = new(DiffDetails) (*in).DeepCopyInto(*out) } + if in.BackReportedStatus != nil { + in, out := &in.BackReportedStatus, &out.BackReportedStatus + *out = new(BackReportedStatus) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ManifestCondition. @@ -1837,6 +1918,26 @@ func (in *PropertySorter) DeepCopy() *PropertySorter { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReportBackStrategy) DeepCopyInto(out *ReportBackStrategy) { + *out = *in + if in.Destination != nil { + in, out := &in.Destination, &out.Destination + *out = new(ReportBackDestination) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReportBackStrategy. +func (in *ReportBackStrategy) DeepCopy() *ReportBackStrategy { + if in == nil { + return nil + } + out := new(ReportBackStrategy) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResourceBinding) DeepCopyInto(out *ResourceBinding) { *out = *in @@ -2482,6 +2583,11 @@ func (in *RolloutStrategy) DeepCopyInto(out *RolloutStrategy) { *out = new(DeleteStrategy) **out = **in } + if in.ReportBackStrategy != nil { + in, out := &in.ReportBackStrategy, &out.ReportBackStrategy + *out = new(ReportBackStrategy) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RolloutStrategy. @@ -2699,86 +2805,122 @@ func (in *StageUpdatingStatus) DeepCopy() *StageUpdatingStatus { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *StagedUpdateRunSpec) DeepCopyInto(out *StagedUpdateRunSpec) { +func (in *StagedUpdateRun) DeepCopyInto(out *StagedUpdateRun) { *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StagedUpdateRunSpec. -func (in *StagedUpdateRunSpec) DeepCopy() *StagedUpdateRunSpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StagedUpdateRun. +func (in *StagedUpdateRun) DeepCopy() *StagedUpdateRun { if in == nil { return nil } - out := new(StagedUpdateRunSpec) + out := new(StagedUpdateRun) in.DeepCopyInto(out) return out } +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *StagedUpdateRun) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *StagedUpdateRunStatus) DeepCopyInto(out *StagedUpdateRunStatus) { +func (in *StagedUpdateRunList) DeepCopyInto(out *StagedUpdateRunList) { *out = *in - if in.ApplyStrategy != nil { - in, out := &in.ApplyStrategy, &out.ApplyStrategy - *out = new(ApplyStrategy) - (*in).DeepCopyInto(*out) - } - if in.StagedUpdateStrategySnapshot != nil { - in, out := &in.StagedUpdateStrategySnapshot, &out.StagedUpdateStrategySnapshot - *out = new(StagedUpdateStrategySpec) - (*in).DeepCopyInto(*out) - } - if in.StagesStatus != nil { - in, out := &in.StagesStatus, &out.StagesStatus - *out = make([]StageUpdatingStatus, len(*in)) + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]StagedUpdateRun, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.DeletionStageStatus != nil { - in, out := &in.DeletionStageStatus, &out.DeletionStageStatus - *out = new(StageUpdatingStatus) - (*in).DeepCopyInto(*out) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StagedUpdateRunList. +func (in *StagedUpdateRunList) DeepCopy() *StagedUpdateRunList { + if in == nil { + return nil } - if in.Conditions != nil { - in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } + out := new(StagedUpdateRunList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *StagedUpdateRunList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c } + return nil } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StagedUpdateRunStatus. -func (in *StagedUpdateRunStatus) DeepCopy() *StagedUpdateRunStatus { +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StagedUpdateStrategy) DeepCopyInto(out *StagedUpdateStrategy) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StagedUpdateStrategy. +func (in *StagedUpdateStrategy) DeepCopy() *StagedUpdateStrategy { if in == nil { return nil } - out := new(StagedUpdateRunStatus) + out := new(StagedUpdateStrategy) in.DeepCopyInto(out) return out } +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *StagedUpdateStrategy) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *StagedUpdateStrategySpec) DeepCopyInto(out *StagedUpdateStrategySpec) { +func (in *StagedUpdateStrategyList) DeepCopyInto(out *StagedUpdateStrategyList) { *out = *in - if in.Stages != nil { - in, out := &in.Stages, &out.Stages - *out = make([]StageConfig, len(*in)) + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]StagedUpdateStrategy, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StagedUpdateStrategySpec. -func (in *StagedUpdateStrategySpec) DeepCopy() *StagedUpdateStrategySpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StagedUpdateStrategyList. +func (in *StagedUpdateStrategyList) DeepCopy() *StagedUpdateStrategyList { if in == nil { return nil } - out := new(StagedUpdateStrategySpec) + out := new(StagedUpdateStrategyList) in.DeepCopyInto(out) return out } +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *StagedUpdateStrategyList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Toleration) DeepCopyInto(out *Toleration) { *out = *in @@ -2814,6 +2956,87 @@ func (in *TopologySpreadConstraint) DeepCopy() *TopologySpreadConstraint { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UpdateRunSpec) DeepCopyInto(out *UpdateRunSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpdateRunSpec. +func (in *UpdateRunSpec) DeepCopy() *UpdateRunSpec { + if in == nil { + return nil + } + out := new(UpdateRunSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UpdateRunStatus) DeepCopyInto(out *UpdateRunStatus) { + *out = *in + if in.ApplyStrategy != nil { + in, out := &in.ApplyStrategy, &out.ApplyStrategy + *out = new(ApplyStrategy) + (*in).DeepCopyInto(*out) + } + if in.UpdateStrategySnapshot != nil { + in, out := &in.UpdateStrategySnapshot, &out.UpdateStrategySnapshot + *out = new(UpdateStrategySpec) + (*in).DeepCopyInto(*out) + } + if in.StagesStatus != nil { + in, out := &in.StagesStatus, &out.StagesStatus + *out = make([]StageUpdatingStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.DeletionStageStatus != nil { + in, out := &in.DeletionStageStatus, &out.DeletionStageStatus + *out = new(StageUpdatingStatus) + (*in).DeepCopyInto(*out) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpdateRunStatus. +func (in *UpdateRunStatus) DeepCopy() *UpdateRunStatus { + if in == nil { + return nil + } + out := new(UpdateRunStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UpdateStrategySpec) DeepCopyInto(out *UpdateStrategySpec) { + *out = *in + if in.Stages != nil { + in, out := &in.Stages, &out.Stages + *out = make([]StageConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpdateStrategySpec. +func (in *UpdateStrategySpec) DeepCopy() *UpdateStrategySpec { + if in == nil { + return nil + } + out := new(UpdateStrategySpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Work) DeepCopyInto(out *Work) { *out = *in @@ -2897,6 +3120,11 @@ func (in *WorkSpec) DeepCopyInto(out *WorkSpec) { *out = new(ApplyStrategy) (*in).DeepCopyInto(*out) } + if in.ReportBackStrategy != nil { + in, out := &in.ReportBackStrategy, &out.ReportBackStrategy + *out = new(ReportBackStrategy) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkSpec. diff --git a/charts/hub-agent/templates/crds/placement.kubernetes-fleet.io_approvalrequests.yaml b/charts/hub-agent/templates/crds/placement.kubernetes-fleet.io_approvalrequests.yaml new file mode 120000 index 000000000..044efe183 --- /dev/null +++ b/charts/hub-agent/templates/crds/placement.kubernetes-fleet.io_approvalrequests.yaml @@ -0,0 +1 @@ +../../../../config/crd/bases/placement.kubernetes-fleet.io_approvalrequests.yaml \ No newline at end of file diff --git a/charts/hub-agent/templates/crds/placement.kubernetes-fleet.io_stagedupdateruns.yaml b/charts/hub-agent/templates/crds/placement.kubernetes-fleet.io_stagedupdateruns.yaml new file mode 120000 index 000000000..1d44518b1 --- /dev/null +++ b/charts/hub-agent/templates/crds/placement.kubernetes-fleet.io_stagedupdateruns.yaml @@ -0,0 +1 @@ +../../../../config/crd/bases/placement.kubernetes-fleet.io_stagedupdateruns.yaml \ No newline at end of file diff --git a/charts/hub-agent/templates/crds/placement.kubernetes-fleet.io_stagedupdatestrategies.yaml b/charts/hub-agent/templates/crds/placement.kubernetes-fleet.io_stagedupdatestrategies.yaml new file mode 120000 index 000000000..211d5ddda --- /dev/null +++ b/charts/hub-agent/templates/crds/placement.kubernetes-fleet.io_stagedupdatestrategies.yaml @@ -0,0 +1 @@ +../../../../config/crd/bases/placement.kubernetes-fleet.io_stagedupdatestrategies.yaml \ No newline at end of file diff --git a/charts/hub-agent/values.yaml b/charts/hub-agent/values.yaml index 58b5f2011..9e1fa9e7c 100644 --- a/charts/hub-agent/values.yaml +++ b/charts/hub-agent/values.yaml @@ -58,5 +58,4 @@ hubAPIQPS: 250 hubAPIBurst: 1000 MaxConcurrentClusterPlacement: 100 ConcurrentResourceChangeSyncs: 20 -logFileMaxSize: "10000000" MaxFleetSizeSupported: 100 diff --git a/cmd/crdinstaller/utils/util_test.go b/cmd/crdinstaller/utils/util_test.go index fbcaa8e24..22f474b7d 100644 --- a/cmd/crdinstaller/utils/util_test.go +++ b/cmd/crdinstaller/utils/util_test.go @@ -60,6 +60,7 @@ func runTest(t *testing.T, crdPath string) { wantedCRDNames: []string{ "memberclusters.cluster.kubernetes-fleet.io", "internalmemberclusters.cluster.kubernetes-fleet.io", + "approvalrequests.placement.kubernetes-fleet.io", "clusterapprovalrequests.placement.kubernetes-fleet.io", "clusterresourcebindings.placement.kubernetes-fleet.io", "clusterresourceenvelopes.placement.kubernetes-fleet.io", @@ -80,6 +81,8 @@ func runTest(t *testing.T, crdPath string) { "resourceplacements.placement.kubernetes-fleet.io", "resourcesnapshots.placement.kubernetes-fleet.io", "schedulingpolicysnapshots.placement.kubernetes-fleet.io", + "stagedupdateruns.placement.kubernetes-fleet.io", + "stagedupdatestrategies.placement.kubernetes-fleet.io", "works.placement.kubernetes-fleet.io", "clusterprofiles.multicluster.x-k8s.io", }, diff --git a/cmd/hubagent/main.go b/cmd/hubagent/main.go index 1fa081ee2..1e1d2071d 100644 --- a/cmd/hubagent/main.go +++ b/cmd/hubagent/main.go @@ -35,7 +35,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/metrics" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" ctrlwebhook "sigs.k8s.io/controller-runtime/pkg/webhook" workv1alpha1 "sigs.k8s.io/work-api/pkg/apis/v1alpha1" @@ -50,7 +49,6 @@ import ( "go.goms.io/fleet/cmd/hubagent/workload" mcv1alpha1 "go.goms.io/fleet/pkg/controllers/membercluster/v1alpha1" mcv1beta1 "go.goms.io/fleet/pkg/controllers/membercluster/v1beta1" - fleetmetrics "go.goms.io/fleet/pkg/metrics" "go.goms.io/fleet/pkg/webhook" "go.goms.io/fleet/pkg/webhook/managedresource" // +kubebuilder:scaffold:imports @@ -85,17 +83,6 @@ func init() { utilruntime.Must(clusterinventory.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme klog.InitFlags(nil) - - metrics.Registry.MustRegister( - fleetmetrics.JoinResultMetrics, - fleetmetrics.LeaveResultMetrics, - fleetmetrics.PlacementApplyFailedCount, - fleetmetrics.PlacementApplySucceedCount, - fleetmetrics.SchedulingCycleDurationMilliseconds, - fleetmetrics.SchedulerActiveWorkers, - fleetmetrics.FleetPlacementStatusLastTimeStampSeconds, - fleetmetrics.FleetEvictionStatus, - ) } func main() { @@ -125,7 +112,8 @@ func main() { mgrOpts := ctrl.Options{ Scheme: scheme, Cache: cache.Options{ - SyncPeriod: &opts.ResyncPeriod.Duration, + SyncPeriod: &opts.ResyncPeriod.Duration, + DefaultTransform: cache.TransformStripManagedFields(), }, LeaderElection: opts.LeaderElection.LeaderElect, LeaderElectionID: opts.LeaderElection.ResourceName, diff --git a/cmd/hubagent/workload/setup.go b/cmd/hubagent/workload/setup.go index 0209c70db..5ea91a81a 100644 --- a/cmd/hubagent/workload/setup.go +++ b/cmd/hubagent/workload/setup.go @@ -114,6 +114,12 @@ var ( placementv1beta1.GroupVersion.WithKind(placementv1beta1.ClusterApprovalRequestKind), } + stagedUpdateRunGVKs = []schema.GroupVersionKind{ + placementv1beta1.GroupVersion.WithKind(placementv1beta1.StagedUpdateRunKind), + placementv1beta1.GroupVersion.WithKind(placementv1beta1.StagedUpdateStrategyKind), + placementv1beta1.GroupVersion.WithKind(placementv1beta1.ApprovalRequestKind), + } + clusterInventoryGVKs = []schema.GroupVersionKind{ clusterinventory.GroupVersion.WithKind("ClusterProfile"), } @@ -331,10 +337,27 @@ func SetupControllers(ctx context.Context, wg *sync.WaitGroup, mgr ctrl.Manager, if err = (&updaterun.Reconciler{ Client: mgr.GetClient(), InformerManager: dynamicInformerManager, - }).SetupWithManager(mgr); err != nil { + }).SetupWithManagerForClusterStagedUpdateRun(mgr); err != nil { klog.ErrorS(err, "Unable to set up clusterStagedUpdateRun controller") return err } + + if opts.EnableResourcePlacement { + for _, gvk := range stagedUpdateRunGVKs { + if err = utils.CheckCRDInstalled(discoverClient, gvk); err != nil { + klog.ErrorS(err, "Unable to find the required CRD", "GVK", gvk) + return err + } + } + klog.Info("Setting up stagedUpdateRun controller") + if err = (&updaterun.Reconciler{ + Client: mgr.GetClient(), + InformerManager: dynamicInformerManager, + }).SetupWithManagerForStagedUpdateRun(mgr); err != nil { + klog.ErrorS(err, "Unable to set up stagedUpdateRun controller") + return err + } + } } // Set up the work generator diff --git a/cmd/memberagent/main.go b/cmd/memberagent/main.go index 9f5f04aea..9fe6456bc 100644 --- a/cmd/memberagent/main.go +++ b/cmd/memberagent/main.go @@ -45,7 +45,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" - "sigs.k8s.io/controller-runtime/pkg/metrics" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" workv1alpha1 "sigs.k8s.io/work-api/pkg/apis/v1alpha1" @@ -57,7 +56,6 @@ import ( imcv1beta1 "go.goms.io/fleet/pkg/controllers/internalmembercluster/v1beta1" "go.goms.io/fleet/pkg/controllers/workapplier" workv1alpha1controller "go.goms.io/fleet/pkg/controllers/workv1alpha1" - fleetmetrics "go.goms.io/fleet/pkg/metrics" "go.goms.io/fleet/pkg/propertyprovider" "go.goms.io/fleet/pkg/propertyprovider/azure" "go.goms.io/fleet/pkg/utils" @@ -93,6 +91,11 @@ var ( enablePprof = flag.Bool("enable-pprof", false, "enable pprof profiling") pprofPort = flag.Int("pprof-port", 6065, "port for pprof profiling") hubPprofPort = flag.Int("hub-pprof-port", 6066, "port for hub pprof profiling") + hubQPS = flag.Float64("hub-api-qps", 50, "QPS to use while talking with fleet-apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.") + hubBurst = flag.Int("hub-api-burst", 500, "Burst to use while talking with fleet-apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.") + memberQPS = flag.Float64("member-api-qps", 250, "QPS to use while talking with fleet-apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.") + memberBurst = flag.Int("member-api-burst", 1000, "Burst to use while talking with fleet-apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.") + // Work applier requeue rate limiter settings. workApplierRequeueRateLimiterAttemptsWithFixedDelay = flag.Int("work-applier-requeue-rate-limiter-attempts-with-fixed-delay", 1, "If set, the work applier will requeue work objects with a fixed delay for the specified number of attempts before switching to exponential backoff.") workApplierRequeueRateLimiterFixedDelaySeconds = flag.Float64("work-applier-requeue-rate-limiter-fixed-delay-seconds", 5.0, "If set, the work applier will requeue work objects with this fixed delay in seconds for the specified number of attempts before switching to exponential backoff.") @@ -116,13 +119,6 @@ func init() { utilruntime.Must(clusterv1beta1.AddToScheme(scheme)) utilruntime.Must(placementv1beta1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme - - metrics.Registry.MustRegister( - fleetmetrics.JoinResultMetrics, - fleetmetrics.LeaveResultMetrics, - fleetmetrics.FleetWorkProcessingRequestsTotal, - fleetmetrics.FleetManifestProcessingRequestsTotal, - ) } func main() { @@ -150,6 +146,8 @@ func main() { klog.FlushAndExit(klog.ExitFlushTimeout, 1) } hubConfig, err := buildHubConfig(hubURL, *useCertificateAuth, *tlsClientInsecure) + hubConfig.QPS = float32(*hubQPS) + hubConfig.Burst = *hubBurst if err != nil { klog.ErrorS(err, "Failed to build Kubernetes client configuration for the hub cluster") klog.FlushAndExit(klog.ExitFlushTimeout, 1) @@ -164,6 +162,8 @@ func main() { mcNamespace := fmt.Sprintf(utils.NamespaceNameFormat, mcName) memberConfig := ctrl.GetConfigOrDie() + memberConfig.QPS = float32(*memberQPS) + memberConfig.Burst = *memberBurst // we place the leader election lease on the member cluster to avoid adding load to the hub hubOpts := ctrl.Options{ Scheme: scheme, diff --git a/config/crd/bases/placement.kubernetes-fleet.io_approvalrequests.yaml b/config/crd/bases/placement.kubernetes-fleet.io_approvalrequests.yaml new file mode 100644 index 000000000..3835fd87c --- /dev/null +++ b/config/crd/bases/placement.kubernetes-fleet.io_approvalrequests.yaml @@ -0,0 +1,152 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.0 + name: approvalrequests.placement.kubernetes-fleet.io +spec: + group: placement.kubernetes-fleet.io + names: + categories: + - fleet + - fleet-placement + kind: ApprovalRequest + listKind: ApprovalRequestList + plural: approvalrequests + shortNames: + - areq + singular: approvalrequest + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.parentStageRollout + name: Update-Run + type: string + - jsonPath: .spec.targetStage + name: Stage + type: string + - jsonPath: .status.conditions[?(@.type=="Approved")].status + name: Approved + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: |- + ApprovalRequest defines a request for user approval for staged update run. + The request object MUST have the following labels: + - `TargetUpdateRun`: Points to the staged update run that this approval request is for. + - `TargetStage`: The name of the stage that this approval request is for. + - `IsLatestUpdateRunApproval`: Indicates whether this approval request is the latest one related to this update run. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: The desired state of ApprovalRequest. + properties: + parentStageRollout: + description: The name of the staged update run that this approval + request is for. + type: string + targetStage: + description: The name of the update stage that this approval request + is for. + type: string + required: + - parentStageRollout + - targetStage + type: object + x-kubernetes-validations: + - message: The spec field is immutable + rule: self == oldSelf + status: + description: The observed state of ApprovalRequest. + properties: + conditions: + description: |- + Conditions is an array of current observed conditions for the specific type of post-update task. + Known conditions are "Approved" and "ApprovalAccepted". + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml index 186898038..ec0caf947 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml @@ -1947,6 +1947,51 @@ spec: - Delete type: string type: object + reportBackStrategy: + description: ReportBackStrategy describes how to report back the + status of applied resources on the member cluster. + properties: + destination: + description: |- + Destination dictates where to copy the status fields to when the report back strategy type is Mirror. + + Available options include: + + * OriginalResource: the status fields will be copied verbatim to the original resource on the hub cluster side. + This is only performed when the placement object has a scheduling policy that selects exactly one member cluster + (i.e., a pickFixed scheduling policy with exactly one cluster name, or a pickN scheduling policy with the numberOfClusters + field set to 1). + + * WorkAPI: the status fields will be copied verbatim via the Work API on the hub cluster side. Users may look up + the status of a specific resource applied to a specific member cluster by inspecting the corresponding Work object + on the hub cluster side. This is the default behavior. + enum: + - OriginalResource + - WorkAPI + type: string + type: + default: Disabled + description: |- + Type dictates the type of the report back strategy to use. + + Available options include: + + * Disabled: status back-reporting is disabled. This is the default behavior. + + * Mirror: status back-reporting is enabled by copying the status fields verbatim to + a destination on the hub cluster side; see the Destination field for more information. + enum: + - Disabled + - Mirror + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: when reportBackStrategy.type is 'Mirror', a destination + must be specified + rule: '(self == null) || (self.type == ''Mirror'' ? size(self.destination) + != 0 : true)' rollingUpdate: description: Rolling update config params. Present only if RolloutStrategyType = RollingUpdate. diff --git a/config/crd/bases/placement.kubernetes-fleet.io_clusterstagedupdateruns.yaml b/config/crd/bases/placement.kubernetes-fleet.io_clusterstagedupdateruns.yaml index 251a37ff5..4b83c14ae 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_clusterstagedupdateruns.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_clusterstagedupdateruns.yaml @@ -1773,7 +1773,7 @@ spec: type: string stagedUpdateStrategySnapshot: description: |- - StagedUpdateStrategySnapshot is the snapshot of the StagedUpdateStrategy used for the update run. + UpdateStrategySnapshot is the snapshot of the UpdateStrategy used for the update run. The snapshot is immutable during the update run. The strategy is applied to the list of clusters scheduled by the CRP according to the current policy. The update run fails to initialize if the strategy fails to produce a valid list of stages where each selected diff --git a/config/crd/bases/placement.kubernetes-fleet.io_resourceplacements.yaml b/config/crd/bases/placement.kubernetes-fleet.io_resourceplacements.yaml index 743ad2355..92488b409 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_resourceplacements.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_resourceplacements.yaml @@ -882,6 +882,51 @@ spec: - Delete type: string type: object + reportBackStrategy: + description: ReportBackStrategy describes how to report back the + status of applied resources on the member cluster. + properties: + destination: + description: |- + Destination dictates where to copy the status fields to when the report back strategy type is Mirror. + + Available options include: + + * OriginalResource: the status fields will be copied verbatim to the original resource on the hub cluster side. + This is only performed when the placement object has a scheduling policy that selects exactly one member cluster + (i.e., a pickFixed scheduling policy with exactly one cluster name, or a pickN scheduling policy with the numberOfClusters + field set to 1). + + * WorkAPI: the status fields will be copied verbatim via the Work API on the hub cluster side. Users may look up + the status of a specific resource applied to a specific member cluster by inspecting the corresponding Work object + on the hub cluster side. This is the default behavior. + enum: + - OriginalResource + - WorkAPI + type: string + type: + default: Disabled + description: |- + Type dictates the type of the report back strategy to use. + + Available options include: + + * Disabled: status back-reporting is disabled. This is the default behavior. + + * Mirror: status back-reporting is enabled by copying the status fields verbatim to + a destination on the hub cluster side; see the Destination field for more information. + enum: + - Disabled + - Mirror + type: string + required: + - type + type: object + x-kubernetes-validations: + - message: when reportBackStrategy.type is 'Mirror', a destination + must be specified + rule: '(self == null) || (self.type == ''Mirror'' ? size(self.destination) + != 0 : true)' rollingUpdate: description: Rolling update config params. Present only if RolloutStrategyType = RollingUpdate. diff --git a/config/crd/bases/placement.kubernetes-fleet.io_stagedupdateruns.yaml b/config/crd/bases/placement.kubernetes-fleet.io_stagedupdateruns.yaml new file mode 100644 index 000000000..32462ceaa --- /dev/null +++ b/config/crd/bases/placement.kubernetes-fleet.io_stagedupdateruns.yaml @@ -0,0 +1,1108 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.0 + name: stagedupdateruns.placement.kubernetes-fleet.io +spec: + group: placement.kubernetes-fleet.io + names: + categories: + - fleet + - fleet-placement + kind: StagedUpdateRun + listKind: StagedUpdateRunList + plural: stagedupdateruns + shortNames: + - sur + singular: stagedupdaterun + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.placementName + name: Placement + type: string + - jsonPath: .spec.resourceSnapshotIndex + name: Resource-Snapshot-Index + type: string + - jsonPath: .status.policySnapshotIndexUsed + name: Policy-Snapshot-Index + type: string + - jsonPath: .status.conditions[?(@.type=="Initialized")].status + name: Initialized + type: string + - jsonPath: .status.conditions[?(@.type=="Succeeded")].status + name: Succeeded + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.stagedRolloutStrategyName + name: Strategy + priority: 1 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: |- + StagedUpdateRun represents a stage by stage update process that applies ResourcePlacement + selected resources to specified clusters. + Resources from unselected clusters are removed after all stages in the update strategy are completed. + Each StagedUpdateRun object corresponds to a single release of a specific resource version. + The release is abandoned if the StagedUpdateRun object is deleted or the scheduling decision changes. + The name of the StagedUpdateRun must conform to RFC 1123. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: The desired state of StagedUpdateRun. The spec is immutable. + properties: + placementName: + description: |- + PlacementName is the name of placement that this update run is applied to. + There can be multiple active update runs for each placement, but + it's up to the DevOps team to ensure they don't conflict with each other. + maxLength: 255 + type: string + resourceSnapshotIndex: + description: |- + The resource snapshot index of the selected resources to be updated across clusters. + The index represents a group of resource snapshots that includes all the resources a ResourcePlacement selected. + type: string + stagedRolloutStrategyName: + description: |- + The name of the update strategy that specifies the stages and the sequence + in which the selected resources will be updated on the member clusters. The stages + are computed according to the referenced strategy when the update run starts + and recorded in the status field. + type: string + required: + - placementName + - resourceSnapshotIndex + - stagedRolloutStrategyName + type: object + x-kubernetes-validations: + - message: The spec field is immutable + rule: self == oldSelf + status: + description: The observed status of StagedUpdateRun. + properties: + appliedStrategy: + description: |- + ApplyStrategy is the apply strategy that the stagedUpdateRun is using. + It is the same as the apply strategy in the CRP when the staged update run starts. + The apply strategy is not updated during the update run even if it changes in the CRP. + properties: + allowCoOwnership: + description: |- + AllowCoOwnership controls whether co-ownership between Fleet and other agents are allowed + on a Fleet-managed resource. If set to false, Fleet will refuse to apply manifests to + a resource that has been owned by one or more non-Fleet agents. + + Note that Fleet does not support the case where one resource is being placed multiple + times by different CRPs on the same member cluster. An apply error will be returned if + Fleet finds that a resource has been owned by another placement attempt by Fleet, even + with the AllowCoOwnership setting set to true. + type: boolean + comparisonOption: + default: PartialComparison + description: |- + ComparisonOption controls how Fleet compares the desired state of a resource, as kept in + a hub cluster manifest, with the current state of the resource (if applicable) in the + member cluster. + + Available options are: + + * PartialComparison: with this option, Fleet will compare only fields that are managed by + Fleet, i.e., the fields that are specified explicitly in the hub cluster manifest. + Unmanaged fields are ignored. This is the default option. + + * FullComparison: with this option, Fleet will compare all fields of the resource, + even if the fields are absent from the hub cluster manifest. + + Consider using the PartialComparison option if you would like to: + + * use the default values for certain fields; or + * let another agent, e.g., HPAs, VPAs, etc., on the member cluster side manage some fields; or + * allow ad-hoc or cluster-specific settings on the member cluster side. + + To use the FullComparison option, it is recommended that you: + + * specify all fields as appropriate in the hub cluster, even if you are OK with using default + values; + * make sure that no fields are managed by agents other than Fleet on the member cluster + side, such as HPAs, VPAs, or other controllers. + + See the Fleet documentation for further explanations and usage examples. + enum: + - PartialComparison + - FullComparison + type: string + serverSideApplyConfig: + description: ServerSideApplyConfig defines the configuration for + server side apply. It is honored only when type is ServerSideApply. + properties: + force: + description: |- + Force represents to force apply to succeed when resolving the conflicts + For any conflicting fields, + - If true, use the values from the resource to be applied to overwrite the values of the existing resource in the + target cluster, as well as take over ownership of such fields. + - If false, apply will fail with the reason ApplyConflictWithOtherApplier. + + For non-conflicting fields, values stay unchanged and ownership are shared between appliers. + type: boolean + type: object + type: + default: ClientSideApply + description: |- + Type is the apply strategy to use; it determines how Fleet applies manifests from the + hub cluster to a member cluster. + + Available options are: + + * ClientSideApply: Fleet uses three-way merge to apply manifests, similar to how kubectl + performs a client-side apply. This is the default option. + + Note that this strategy requires that Fleet keep the last applied configuration in the + annotation of an applied resource. If the object gets so large that apply ops can no longer + be executed, Fleet will switch to server-side apply. + + Use ComparisonOption and WhenToApply settings to control when an apply op can be executed. + + * ServerSideApply: Fleet uses server-side apply to apply manifests; Fleet itself will + become the field manager for specified fields in the manifests. Specify + ServerSideApplyConfig as appropriate if you would like Fleet to take over field + ownership upon conflicts. This is the recommended option for most scenarios; it might + help reduce object size and safely resolve conflicts between field values. For more + information, please refer to the Kubernetes documentation + (https://kubernetes.io/docs/reference/using-api/server-side-apply/#comparison-with-client-side-apply). + + Use ComparisonOption and WhenToApply settings to control when an apply op can be executed. + + * ReportDiff: Fleet will compare the desired state of a resource as kept in the hub cluster + with its current state (if applicable) on the member cluster side, and report any + differences. No actual apply ops would be executed, and resources will be left alone as they + are on the member clusters. + + If configuration differences are found on a resource, Fleet will consider this as an apply + error, which might block rollout depending on the specified rollout strategy. + + Use ComparisonOption setting to control how the difference is calculated. + + ClientSideApply and ServerSideApply apply strategies only work when Fleet can assume + ownership of a resource (e.g., the resource is created by Fleet, or Fleet has taken over + the resource). See the comments on the WhenToTakeOver field for more information. + ReportDiff apply strategy, however, will function regardless of Fleet's ownership + status. One may set up a CRP with the ReportDiff strategy and the Never takeover option, + and this will turn Fleet into a detection tool that reports only configuration differences + but do not touch any resources on the member cluster side. + + For a comparison between the different strategies and usage examples, refer to the + Fleet documentation. + enum: + - ClientSideApply + - ServerSideApply + - ReportDiff + type: string + whenToApply: + default: Always + description: |- + WhenToApply controls when Fleet would apply the manifests on the hub cluster to the member + clusters. + + Available options are: + + * Always: with this option, Fleet will periodically apply hub cluster manifests + on the member cluster side; this will effectively overwrite any change in the fields + managed by Fleet (i.e., specified in the hub cluster manifest). This is the default + option. + + Note that this option would revert any ad-hoc changes made on the member cluster side in the + managed fields; if you would like to make temporary edits on the member cluster side + in the managed fields, switch to IfNotDrifted option. Note that changes in unmanaged + fields will be left alone; if you use the FullDiff compare option, such changes will + be reported as drifts. + + * IfNotDrifted: with this option, Fleet will stop applying hub cluster manifests on + clusters that have drifted from the desired state; apply ops would still continue on + the rest of the clusters. Drifts are calculated using the ComparisonOption, + as explained in the corresponding field. + + Use this option if you would like Fleet to detect drifts in your multi-cluster setup. + A drift occurs when an agent makes an ad-hoc change on the member cluster side that + makes affected resources deviate from its desired state as kept in the hub cluster; + and this option grants you an opportunity to view the drift details and take actions + accordingly. The drift details will be reported in the CRP status. + + To fix a drift, you may: + + * revert the changes manually on the member cluster side + * update the hub cluster manifest; this will trigger Fleet to apply the latest revision + of the manifests, which will overwrite the drifted fields + (if they are managed by Fleet) + * switch to the Always option; this will trigger Fleet to apply the current revision + of the manifests, which will overwrite the drifted fields (if they are managed by Fleet). + * if applicable and necessary, delete the drifted resources on the member cluster side; Fleet + will attempt to re-create them using the hub cluster manifests + enum: + - Always + - IfNotDrifted + type: string + whenToTakeOver: + default: Always + description: |- + WhenToTakeOver determines the action to take when Fleet applies resources to a member + cluster for the first time and finds out that the resource already exists in the cluster. + + This setting is most relevant in cases where you would like Fleet to manage pre-existing + resources on a member cluster. + + Available options include: + + * Always: with this action, Fleet will apply the hub cluster manifests to the member + clusters even if the affected resources already exist. This is the default action. + + Note that this might lead to fields being overwritten on the member clusters, if they + are specified in the hub cluster manifests. + + * IfNoDiff: with this action, Fleet will apply the hub cluster manifests to the member + clusters if (and only if) pre-existing resources look the same as the hub cluster manifests. + + This is a safer option as pre-existing resources that are inconsistent with the hub cluster + manifests will not be overwritten; Fleet will ignore them until the inconsistencies + are resolved properly: any change you make to the hub cluster manifests would not be + applied, and if you delete the manifests or even the ClusterResourcePlacement itself + from the hub cluster, these pre-existing resources would not be taken away. + + Fleet will check for inconsistencies in accordance with the ComparisonOption setting. See also + the comments on the ComparisonOption field for more information. + + If a diff has been found in a field that is **managed** by Fleet (i.e., the field + **is specified ** in the hub cluster manifest), consider one of the following actions: + * set the field in the member cluster to be of the same value as that in the hub cluster + manifest. + * update the hub cluster manifest so that its field value matches with that in the member + cluster. + * switch to the Always action, which will allow Fleet to overwrite the field with the + value in the hub cluster manifest. + + If a diff has been found in a field that is **not managed** by Fleet (i.e., the field + **is not specified** in the hub cluster manifest), consider one of the following actions: + * remove the field from the member cluster. + * update the hub cluster manifest so that the field is included in the hub cluster manifest. + + If appropriate, you may also delete the object from the member cluster; Fleet will recreate + it using the hub cluster manifest. + + * Never: with this action, Fleet will not apply a hub cluster manifest to the member + clusters if there is a corresponding pre-existing resource. However, if a manifest + has never been applied yet; or it has a corresponding resource which Fleet has assumed + ownership, apply op will still be executed. + + This is the safest option; one will have to remove the pre-existing resources (so that + Fleet can re-create them) or switch to a different + WhenToTakeOver option before Fleet starts processing the corresponding hub cluster + manifests. + + If you prefer Fleet stop processing all manifests, use this option along with the + ReportDiff apply strategy type. This setup would instruct Fleet to touch nothing + on the member cluster side but still report configuration differences between the + hub cluster and member clusters. Fleet will not give up ownership + that it has already assumed though. + enum: + - Always + - IfNoDiff + - Never + type: string + type: object + conditions: + description: |- + Conditions is an array of current observed conditions for StagedUpdateRun. + Known conditions are "Initialized", "Progressing", "Succeeded". + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + deletionStageStatus: + description: |- + DeletionStageStatus lists the current status of the deletion stage. The deletion stage + removes all the resources from the clusters that are not selected by the + current policy after all the update stages are completed. + properties: + afterStageTaskStatus: + description: |- + The status of the post-update tasks associated with the current stage. + Empty if the stage has not finished updating all the clusters. + items: + properties: + approvalRequestName: + description: |- + The name of the approval request object that is created for this stage. + Only valid if the AfterStageTaskType is Approval. + type: string + conditions: + description: |- + Conditions is an array of current observed conditions for the specific type of post-update task. + Known conditions are "ApprovalRequestCreated", "WaitTimeElapsed", and "ApprovalRequestApproved". + items: + description: Condition contains details for one aspect + of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, + False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in + foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: + description: The type of the post-update task. + enum: + - TimedWait + - Approval + type: string + required: + - type + type: object + maxItems: 2 + type: array + clusters: + description: The list of each cluster's updating status in this + stage. + items: + description: ClusterUpdatingStatus defines the status of the + update run on a cluster. + properties: + clusterName: + description: The name of the cluster. + type: string + clusterResourceOverrideSnapshots: + description: |- + ClusterResourceOverrides contains a list of applicable ClusterResourceOverride snapshot names + associated with the cluster. + The list is computed at the beginning of the update run and not updated during the update run. + The list is empty if there are no cluster overrides associated with the cluster. + items: + type: string + type: array + conditions: + description: |- + Conditions is an array of current observed conditions for clusters. Empty if the cluster has not started updating. + Known conditions are "Started", "Succeeded". + items: + description: Condition contains details for one aspect + of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, + False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in + foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + resourceOverrideSnapshots: + description: |- + ResourceOverrideSnapshots is a list of ResourceOverride snapshots associated with the cluster. + The list is computed at the beginning of the update run and not updated during the update run. + The list is empty if there are no resource overrides associated with the cluster. + items: + description: NamespacedName comprises a resource name, + with a mandatory namespace. + properties: + name: + description: Name is the name of the namespaced scope + resource. + type: string + namespace: + description: Namespace is namespace of the namespaced + scope resource. + type: string + required: + - name + - namespace + type: object + type: array + required: + - clusterName + type: object + type: array + conditions: + description: |- + Conditions is an array of current observed updating conditions for the stage. Empty if the stage has not started updating. + Known conditions are "Progressing", "Succeeded". + items: + description: Condition contains details for one aspect of the + current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + endTime: + description: The time when the update finished on the stage. Empty + if the stage has not started updating. + format: date-time + type: string + stageName: + description: The name of the stage. + type: string + startTime: + description: The time when the update started on the stage. Empty + if the stage has not started updating. + format: date-time + type: string + required: + - clusters + - stageName + type: object + policyObservedClusterCount: + description: |- + PolicyObservedClusterCount records the number of observed clusters in the policy snapshot. + It is recorded at the beginning of the update run from the policy snapshot object. + If the `ObservedClusterCount` value is updated during the update run, the update run is abandoned. + type: integer + policySnapshotIndexUsed: + description: |- + PolicySnapShotIndexUsed records the policy snapshot index of the ClusterResourcePlacement (CRP) that + the update run is based on. The index represents the latest policy snapshot at the start of the update run. + If a newer policy snapshot is detected after the run starts, the staged update run is abandoned. + The scheduler must identify all clusters that meet the current policy before the update run begins. + All clusters involved in the update run are selected from the list of clusters scheduled by the CRP according + to the current policy. + type: string + stagedUpdateStrategySnapshot: + description: |- + UpdateStrategySnapshot is the snapshot of the UpdateStrategy used for the update run. + The snapshot is immutable during the update run. + The strategy is applied to the list of clusters scheduled by the CRP according to the current policy. + The update run fails to initialize if the strategy fails to produce a valid list of stages where each selected + cluster is included in exactly one stage. + properties: + stages: + description: Stage specifies the configuration for each update + stage. + items: + description: |- + StageConfig describes a single update stage. + The clusters in each stage are updated sequentially. + The update stops if any of the updates fail. + properties: + afterStageTasks: + description: |- + The collection of tasks that each stage needs to complete successfully before moving to the next stage. + Each task is executed in parallel and there cannot be more than one task of the same type. + items: + description: AfterStageTask is the collection of post-stage + tasks that ALL need to be completed before moving to + the next stage. + properties: + type: + description: The type of the after-stage task. + enum: + - TimedWait + - Approval + type: string + waitTime: + description: The time to wait after all the clusters + in the current stage complete the update before + moving to the next stage. + pattern: ^0|([0-9]+(\.[0-9]+)?(s|m|h))+$ + type: string + required: + - type + type: object + maxItems: 2 + type: array + x-kubernetes-validations: + - message: AfterStageTaskType is Approval, waitTime is not + allowed + rule: '!self.exists(e, e.type == ''Approval'' && has(e.waitTime))' + - message: AfterStageTaskType is TimedWait, waitTime is + required + rule: '!self.exists(e, e.type == ''TimedWait'' && !has(e.waitTime))' + labelSelector: + description: |- + LabelSelector is a label query over all the joined member clusters. Clusters matching the query are selected + for this stage. There cannot be overlapping clusters between stages when the stagedUpdateRun is created. + If the label selector is empty, the stage includes all the selected clusters. + If the label selector is nil, the stage does not include any selected clusters. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: The name of the stage. This MUST be unique + within the same StagedUpdateStrategy. + maxLength: 63 + pattern: ^[a-z0-9]+$ + type: string + sortingLabelKey: + description: |- + The label key used to sort the selected clusters. + The clusters within the stage are updated sequentially following the rule below: + - primary: Ascending order based on the value of the label key, interpreted as integers if present. + - secondary: Ascending order based on the name of the cluster if the label key is absent or the label value is the same. + type: string + required: + - name + type: object + maxItems: 31 + type: array + required: + - stages + type: object + stagesStatus: + description: |- + StagesStatus lists the current updating status of each stage. + The list is empty if the update run is not started or failed to initialize. + items: + description: StageUpdatingStatus defines the status of the update + run in a stage. + properties: + afterStageTaskStatus: + description: |- + The status of the post-update tasks associated with the current stage. + Empty if the stage has not finished updating all the clusters. + items: + properties: + approvalRequestName: + description: |- + The name of the approval request object that is created for this stage. + Only valid if the AfterStageTaskType is Approval. + type: string + conditions: + description: |- + Conditions is an array of current observed conditions for the specific type of post-update task. + Known conditions are "ApprovalRequestCreated", "WaitTimeElapsed", and "ApprovalRequestApproved". + items: + description: Condition contains details for one aspect + of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, + False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in + foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: + description: The type of the post-update task. + enum: + - TimedWait + - Approval + type: string + required: + - type + type: object + maxItems: 2 + type: array + clusters: + description: The list of each cluster's updating status in this + stage. + items: + description: ClusterUpdatingStatus defines the status of the + update run on a cluster. + properties: + clusterName: + description: The name of the cluster. + type: string + clusterResourceOverrideSnapshots: + description: |- + ClusterResourceOverrides contains a list of applicable ClusterResourceOverride snapshot names + associated with the cluster. + The list is computed at the beginning of the update run and not updated during the update run. + The list is empty if there are no cluster overrides associated with the cluster. + items: + type: string + type: array + conditions: + description: |- + Conditions is an array of current observed conditions for clusters. Empty if the cluster has not started updating. + Known conditions are "Started", "Succeeded". + items: + description: Condition contains details for one aspect + of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, + False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in + foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + resourceOverrideSnapshots: + description: |- + ResourceOverrideSnapshots is a list of ResourceOverride snapshots associated with the cluster. + The list is computed at the beginning of the update run and not updated during the update run. + The list is empty if there are no resource overrides associated with the cluster. + items: + description: NamespacedName comprises a resource name, + with a mandatory namespace. + properties: + name: + description: Name is the name of the namespaced + scope resource. + type: string + namespace: + description: Namespace is namespace of the namespaced + scope resource. + type: string + required: + - name + - namespace + type: object + type: array + required: + - clusterName + type: object + type: array + conditions: + description: |- + Conditions is an array of current observed updating conditions for the stage. Empty if the stage has not started updating. + Known conditions are "Progressing", "Succeeded". + items: + description: Condition contains details for one aspect of + the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, + Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + endTime: + description: The time when the update finished on the stage. + Empty if the stage has not started updating. + format: date-time + type: string + stageName: + description: The name of the stage. + type: string + startTime: + description: The time when the update started on the stage. + Empty if the stage has not started updating. + format: date-time + type: string + required: + - clusters + - stageName + type: object + type: array + type: object + required: + - spec + type: object + x-kubernetes-validations: + - message: metadata.name max length is 127 + rule: size(self.metadata.name) < 128 + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/placement.kubernetes-fleet.io_stagedupdatestrategies.yaml b/config/crd/bases/placement.kubernetes-fleet.io_stagedupdatestrategies.yaml new file mode 100644 index 000000000..4e6d5fe02 --- /dev/null +++ b/config/crd/bases/placement.kubernetes-fleet.io_stagedupdatestrategies.yaml @@ -0,0 +1,163 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.0 + name: stagedupdatestrategies.placement.kubernetes-fleet.io +spec: + group: placement.kubernetes-fleet.io + names: + categories: + - fleet + - fleet-placement + kind: StagedUpdateStrategy + listKind: StagedUpdateStrategyList + plural: stagedupdatestrategies + shortNames: + - sus + singular: stagedupdatestrategy + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: |- + StagedUpdateStrategy defines a reusable strategy that specifies the stages and the sequence + in which the selected cluster resources will be updated on the member clusters. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: The desired state of StagedUpdateStrategy. + properties: + stages: + description: Stage specifies the configuration for each update stage. + items: + description: |- + StageConfig describes a single update stage. + The clusters in each stage are updated sequentially. + The update stops if any of the updates fail. + properties: + afterStageTasks: + description: |- + The collection of tasks that each stage needs to complete successfully before moving to the next stage. + Each task is executed in parallel and there cannot be more than one task of the same type. + items: + description: AfterStageTask is the collection of post-stage + tasks that ALL need to be completed before moving to the + next stage. + properties: + type: + description: The type of the after-stage task. + enum: + - TimedWait + - Approval + type: string + waitTime: + description: The time to wait after all the clusters in + the current stage complete the update before moving + to the next stage. + pattern: ^0|([0-9]+(\.[0-9]+)?(s|m|h))+$ + type: string + required: + - type + type: object + maxItems: 2 + type: array + x-kubernetes-validations: + - message: AfterStageTaskType is Approval, waitTime is not allowed + rule: '!self.exists(e, e.type == ''Approval'' && has(e.waitTime))' + - message: AfterStageTaskType is TimedWait, waitTime is required + rule: '!self.exists(e, e.type == ''TimedWait'' && !has(e.waitTime))' + labelSelector: + description: |- + LabelSelector is a label query over all the joined member clusters. Clusters matching the query are selected + for this stage. There cannot be overlapping clusters between stages when the stagedUpdateRun is created. + If the label selector is empty, the stage includes all the selected clusters. + If the label selector is nil, the stage does not include any selected clusters. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: The name of the stage. This MUST be unique within + the same StagedUpdateStrategy. + maxLength: 63 + pattern: ^[a-z0-9]+$ + type: string + sortingLabelKey: + description: |- + The label key used to sort the selected clusters. + The clusters within the stage are updated sequentially following the rule below: + - primary: Ascending order based on the value of the label key, interpreted as integers if present. + - secondary: Ascending order based on the name of the cluster if the label key is absent or the label value is the same. + type: string + required: + - name + type: object + maxItems: 31 + type: array + required: + - stages + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/config/crd/bases/placement.kubernetes-fleet.io_works.yaml b/config/crd/bases/placement.kubernetes-fleet.io_works.yaml index 66cec7ac2..e2220df73 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_works.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_works.yaml @@ -531,6 +531,46 @@ spec: - Never type: string type: object + reportBackStrategy: + description: ReportBackStrategy describes how to report back the status + of applied resources on the member cluster. + properties: + destination: + description: |- + Destination dictates where to copy the status fields to when the report back strategy type is Mirror. + + Available options include: + + * OriginalResource: the status fields will be copied verbatim to the original resource on the hub cluster side. + This is only performed when the placement object has a scheduling policy that selects exactly one member cluster + (i.e., a pickFixed scheduling policy with exactly one cluster name, or a pickN scheduling policy with the numberOfClusters + field set to 1). + + * WorkAPI: the status fields will be copied verbatim via the Work API on the hub cluster side. Users may look up + the status of a specific resource applied to a specific member cluster by inspecting the corresponding Work object + on the hub cluster side. This is the default behavior. + enum: + - OriginalResource + - WorkAPI + type: string + type: + default: Disabled + description: |- + Type dictates the type of the report back strategy to use. + + Available options include: + + * Disabled: status back-reporting is disabled. This is the default behavior. + + * Mirror: status back-reporting is enabled by copying the status fields verbatim to + a destination on the hub cluster side; see the Destination field for more information. + enum: + - Disabled + - Mirror + type: string + required: + - type + type: object workload: description: Workload represents the manifest workload to be deployed on spoke cluster @@ -624,6 +664,22 @@ spec: ManifestCondition represents the conditions of the resources deployed on spoke cluster. properties: + backReportedStatus: + description: BackReportedStatus is the status reported back + from the member cluster (if applicable). + properties: + observationTime: + description: ObservationTime is the timestamp when the status + was last back reported. + format: date-time + type: string + observedStatus: + type: object + x-kubernetes-embedded-resource: true + x-kubernetes-preserve-unknown-fields: true + required: + - observationTime + type: object conditions: description: Conditions represents the conditions of this resource on spoke cluster @@ -695,7 +751,7 @@ spec: properties: firstDiffedObservedTime: description: |- - FirsftDiffedObservedTime is the timestamp when the configuration difference + FirstDiffedObservedTime is the timestamp when the configuration difference was first detected. format: date-time type: string @@ -765,8 +821,8 @@ spec: at the same time. properties: firstDriftedObservedTime: - description: FirsftDriftedObservedTime is the timestamp - when the drift was first detected. + description: FirstDriftedObservedTime is the timestamp when + the drift was first detected. format: date-time type: string observationTime: diff --git a/hack/Azure/setup/createHubCluster.sh b/hack/Azure/setup/createHubCluster.sh index 4c55ae355..6dd4d2092 100755 --- a/hack/Azure/setup/createHubCluster.sh +++ b/hack/Azure/setup/createHubCluster.sh @@ -40,7 +40,7 @@ helm install hub-agent charts/hub-agent/ \ --set ConcurrentRolloutSyncs=20 \ --set hubAPIQPS=100 \ --set hubAPIBurst=1000 \ - --set logFileMaxSize=100000000 \ + --set logFileMaxSize=5000 \ --set MaxFleetSizeSupported=100 # Check the status of the hub agent diff --git a/pkg/controllers/clusterresourceplacementeviction/controller.go b/pkg/controllers/clusterresourceplacementeviction/controller.go index cd31a3c68..db6265c5c 100644 --- a/pkg/controllers/clusterresourceplacementeviction/controller.go +++ b/pkg/controllers/clusterresourceplacementeviction/controller.go @@ -35,7 +35,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" - "go.goms.io/fleet/pkg/metrics" + hubmetrics "go.goms.io/fleet/pkg/metrics/hub" bindingutils "go.goms.io/fleet/pkg/utils/binding" "go.goms.io/fleet/pkg/utils/condition" "go.goms.io/fleet/pkg/utils/controller" @@ -58,7 +58,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim var internalError bool defer func() { if internalError { - metrics.FleetEvictionStatus.WithLabelValues(evictionName, "false", "unknown").SetToCurrentTime() + hubmetrics.FleetEvictionStatus.WithLabelValues(evictionName, "false", "unknown").SetToCurrentTime() } latency := time.Since(startTime).Milliseconds() klog.V(2).InfoS("ClusterResourcePlacementEviction reconciliation ends", "clusterResourcePlacementEviction", evictionName, "latency", latency) @@ -346,12 +346,12 @@ func markEvictionNotExecuted(eviction *placementv1beta1.ClusterResourcePlacement } func emitEvictionCompleteMetric(eviction *placementv1beta1.ClusterResourcePlacementEviction) { - metrics.FleetEvictionStatus.DeletePartialMatch(prometheus.Labels{"name": eviction.GetName(), "isCompleted": "false"}) + hubmetrics.FleetEvictionStatus.DeletePartialMatch(prometheus.Labels{"name": eviction.GetName(), "isCompleted": "false"}) // check to see if eviction is valid. if condition.IsConditionStatusTrue(eviction.GetCondition(string(placementv1beta1.PlacementEvictionConditionTypeValid)), eviction.GetGeneration()) { - metrics.FleetEvictionStatus.WithLabelValues(eviction.Name, "true", "true").SetToCurrentTime() + hubmetrics.FleetEvictionStatus.WithLabelValues(eviction.Name, "true", "true").SetToCurrentTime() } else { - metrics.FleetEvictionStatus.WithLabelValues(eviction.Name, "true", "false").SetToCurrentTime() + hubmetrics.FleetEvictionStatus.WithLabelValues(eviction.Name, "true", "false").SetToCurrentTime() } } @@ -363,7 +363,7 @@ func (r *Reconciler) SetupWithManager(mgr runtime.Manager) error { WithEventFilter(predicate.Funcs{ DeleteFunc: func(e event.DeleteEvent) bool { // delete complete status metric for eviction and skip reconciliation. - count := metrics.FleetEvictionStatus.DeletePartialMatch(prometheus.Labels{"name": e.Object.GetName()}) + count := hubmetrics.FleetEvictionStatus.DeletePartialMatch(prometheus.Labels{"name": e.Object.GetName()}) klog.V(2).InfoS("ClusterResourcePlacementEviction is being deleted", "clusterResourcePlacementEviction", e.Object.GetName(), "metricCount", count) return false }, diff --git a/pkg/controllers/clusterresourceplacementeviction/controller_intergration_test.go b/pkg/controllers/clusterresourceplacementeviction/controller_intergration_test.go index 9583bb512..dbb138dab 100644 --- a/pkg/controllers/clusterresourceplacementeviction/controller_intergration_test.go +++ b/pkg/controllers/clusterresourceplacementeviction/controller_intergration_test.go @@ -33,7 +33,7 @@ import ( ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" - "go.goms.io/fleet/pkg/metrics" + hubmetrics "go.goms.io/fleet/pkg/metrics/hub" "go.goms.io/fleet/pkg/utils/condition" testutilseviction "go.goms.io/fleet/test/utils/eviction" ) @@ -58,9 +58,9 @@ var _ = Describe("Test ClusterResourcePlacementEviction Controller", func() { BeforeEach(func() { // Reset metrics before each test - metrics.FleetEvictionStatus.Reset() + hubmetrics.FleetEvictionStatus.Reset() // emit incomplete eviction metric to simulate eviction failed once. - metrics.FleetEvictionStatus.WithLabelValues(evictionName, "false", "unknown").SetToCurrentTime() + hubmetrics.FleetEvictionStatus.WithLabelValues(evictionName, "false", "unknown").SetToCurrentTime() }) AfterEach(func() { @@ -68,7 +68,7 @@ var _ = Describe("Test ClusterResourcePlacementEviction Controller", func() { ensureAllBindingsAreRemoved(crpName) ensureEvictionRemoved(evictionName) ensureCRPRemoved(crpName) - metrics.FleetEvictionStatus.Reset() + hubmetrics.FleetEvictionStatus.Reset() }) It("Invalid Eviction Blocked - emit complete metric with isValid=false, isComplete=true", func() { diff --git a/pkg/controllers/clusterresourceplacementeviction/controller_test.go b/pkg/controllers/clusterresourceplacementeviction/controller_test.go index 95fed8601..e1229251d 100644 --- a/pkg/controllers/clusterresourceplacementeviction/controller_test.go +++ b/pkg/controllers/clusterresourceplacementeviction/controller_test.go @@ -38,7 +38,7 @@ import ( ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" - "go.goms.io/fleet/pkg/metrics" + hubmetrics "go.goms.io/fleet/pkg/metrics/hub" "go.goms.io/fleet/pkg/utils/condition" "go.goms.io/fleet/pkg/utils/defaulter" ) @@ -1493,7 +1493,7 @@ func TestReconcileForIncompleteEvictionMetric(t *testing.T) { isComplete := "false" // Reset metrics before each test - metrics.FleetEvictionStatus.Reset() + hubmetrics.FleetEvictionStatus.Reset() scheme := serviceScheme(t) fakeClient := fake.NewClientBuilder(). diff --git a/pkg/controllers/clusterresourceplacementeviction/suite_test.go b/pkg/controllers/clusterresourceplacementeviction/suite_test.go index 13576f72d..dc34eaa86 100644 --- a/pkg/controllers/clusterresourceplacementeviction/suite_test.go +++ b/pkg/controllers/clusterresourceplacementeviction/suite_test.go @@ -19,7 +19,6 @@ package clusterresourceplacementeviction import ( "context" "flag" - "os" "path/filepath" "testing" @@ -34,11 +33,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" - ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" - "go.goms.io/fleet/pkg/metrics" ) var ( @@ -50,13 +47,6 @@ var ( cancel context.CancelFunc ) -func TestMain(m *testing.M) { - // Register here as the metric is both tested in ginkgo tests and go unit tests. - ctrlmetrics.Registry.MustRegister(metrics.FleetEvictionStatus) - - os.Exit(m.Run()) -} - func TestAPIs(t *testing.T) { RegisterFailHandler(Fail) diff --git a/pkg/controllers/internalmembercluster/v1alpha1/member_controller.go b/pkg/controllers/internalmembercluster/v1alpha1/member_controller.go index f15f8ec6e..87397e3b2 100644 --- a/pkg/controllers/internalmembercluster/v1alpha1/member_controller.go +++ b/pkg/controllers/internalmembercluster/v1alpha1/member_controller.go @@ -36,7 +36,7 @@ import ( fleetv1alpha1 "go.goms.io/fleet/apis/v1alpha1" "go.goms.io/fleet/pkg/controllers/workv1alpha1" - "go.goms.io/fleet/pkg/metrics" + sharedmetrics "go.goms.io/fleet/pkg/metrics/shared" ) // Reconciler reconciles a InternalMemberCluster object in the member cluster. @@ -273,7 +273,7 @@ func (r *Reconciler) markInternalMemberClusterJoined(imc fleetv1alpha1.Condition if existingCondition == nil || existingCondition.ObservedGeneration != imc.GetGeneration() || existingCondition.Status != newCondition.Status { r.recorder.Event(imc, corev1.EventTypeNormal, eventReasonInternalMemberClusterJoined, "internal member cluster joined") klog.V(2).InfoS("joined", "InternalMemberCluster", klog.KObj(imc)) - metrics.ReportJoinResultMetric() + sharedmetrics.ReportJoinResultMetric() } imc.SetConditionsWithType(fleetv1alpha1.MemberAgent, newCondition) @@ -313,7 +313,7 @@ func (r *Reconciler) markInternalMemberClusterLeft(imc fleetv1alpha1.Conditioned if existingCondition == nil || existingCondition.ObservedGeneration != imc.GetGeneration() || existingCondition.Status != newCondition.Status { r.recorder.Event(imc, corev1.EventTypeNormal, eventReasonInternalMemberClusterLeft, "internal member cluster left") klog.V(2).InfoS("left", "InternalMemberCluster", klog.KObj(imc)) - metrics.ReportLeaveResultMetric() + sharedmetrics.ReportLeaveResultMetric() } imc.SetConditionsWithType(fleetv1alpha1.MemberAgent, newCondition) diff --git a/pkg/controllers/internalmembercluster/v1beta1/member_controller.go b/pkg/controllers/internalmembercluster/v1beta1/member_controller.go index c05af5086..4419477ea 100644 --- a/pkg/controllers/internalmembercluster/v1beta1/member_controller.go +++ b/pkg/controllers/internalmembercluster/v1beta1/member_controller.go @@ -40,7 +40,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" clusterv1beta1 "go.goms.io/fleet/apis/cluster/v1beta1" - "go.goms.io/fleet/pkg/metrics" + sharedmetrics "go.goms.io/fleet/pkg/metrics/shared" "go.goms.io/fleet/pkg/propertyprovider" "go.goms.io/fleet/pkg/utils/condition" "go.goms.io/fleet/pkg/utils/controller" @@ -636,7 +636,7 @@ func (r *Reconciler) markInternalMemberClusterJoined(imc clusterv1beta1.Conditio if existingCondition == nil || existingCondition.ObservedGeneration != imc.GetGeneration() || existingCondition.Status != newCondition.Status { r.recorder.Event(imc, corev1.EventTypeNormal, EventReasonInternalMemberClusterJoined, "internal member cluster joined") klog.V(2).InfoS("InternalMemberCluster has joined", "internalMemberCluster", klog.KObj(imc)) - metrics.ReportJoinResultMetric() + sharedmetrics.ReportJoinResultMetric() } imc.SetConditionsWithType(clusterv1beta1.MemberAgent, newCondition) @@ -676,7 +676,7 @@ func (r *Reconciler) markInternalMemberClusterLeft(imc clusterv1beta1.Conditione if existingCondition == nil || existingCondition.ObservedGeneration != imc.GetGeneration() || existingCondition.Status != newCondition.Status { r.recorder.Event(imc, corev1.EventTypeNormal, EventReasonInternalMemberClusterLeft, "internal member cluster left") klog.V(2).InfoS("InternalMemberCluster has left", "internalMemberCluster", klog.KObj(imc)) - metrics.ReportLeaveResultMetric() + sharedmetrics.ReportLeaveResultMetric() } imc.SetConditionsWithType(clusterv1beta1.MemberAgent, newCondition) diff --git a/pkg/controllers/membercluster/v1alpha1/membercluster_controller.go b/pkg/controllers/membercluster/v1alpha1/membercluster_controller.go index 4547d1a5d..871ed4b33 100644 --- a/pkg/controllers/membercluster/v1alpha1/membercluster_controller.go +++ b/pkg/controllers/membercluster/v1alpha1/membercluster_controller.go @@ -42,7 +42,7 @@ import ( "go.goms.io/fleet/apis" fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" fleetv1alpha1 "go.goms.io/fleet/apis/v1alpha1" - "go.goms.io/fleet/pkg/metrics" + sharedmetrics "go.goms.io/fleet/pkg/metrics/shared" "go.goms.io/fleet/pkg/utils" ) @@ -523,7 +523,7 @@ func markMemberClusterJoined(recorder record.EventRecorder, mc apis.ConditionedO if existingCondition == nil || existingCondition.Status != newCondition.Status { recorder.Event(mc, corev1.EventTypeNormal, reasonMemberClusterJoined, "member cluster joined") klog.V(2).InfoS("memberCluster joined", "memberCluster", klog.KObj(mc)) - metrics.ReportJoinResultMetric() + sharedmetrics.ReportJoinResultMetric() } mc.SetConditions(newCondition) @@ -550,7 +550,7 @@ func markMemberClusterLeft(recorder record.EventRecorder, mc apis.ConditionedObj if existingCondition == nil || existingCondition.Status != newCondition.Status { recorder.Event(mc, corev1.EventTypeNormal, reasonMemberClusterJoined, "member cluster left") klog.V(2).InfoS("memberCluster left", "memberCluster", klog.KObj(mc)) - metrics.ReportLeaveResultMetric() + sharedmetrics.ReportLeaveResultMetric() } mc.SetConditions(newCondition, notReadyCondition) diff --git a/pkg/controllers/membercluster/v1beta1/membercluster_controller.go b/pkg/controllers/membercluster/v1beta1/membercluster_controller.go index 69477378d..e96243a64 100644 --- a/pkg/controllers/membercluster/v1beta1/membercluster_controller.go +++ b/pkg/controllers/membercluster/v1beta1/membercluster_controller.go @@ -43,7 +43,7 @@ import ( "go.goms.io/fleet/apis" clusterv1beta1 "go.goms.io/fleet/apis/cluster/v1beta1" placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" - "go.goms.io/fleet/pkg/metrics" + sharedmetrics "go.goms.io/fleet/pkg/metrics/shared" "go.goms.io/fleet/pkg/utils" "go.goms.io/fleet/pkg/utils/condition" "go.goms.io/fleet/pkg/utils/controller" @@ -635,7 +635,7 @@ func markMemberClusterJoined(recorder record.EventRecorder, mc apis.ConditionedO if existingCondition == nil || existingCondition.Status != newCondition.Status { recorder.Event(mc, corev1.EventTypeNormal, reasonMemberClusterJoined, "member cluster joined") klog.V(2).InfoS("memberCluster joined", "memberCluster", klog.KObj(mc)) - metrics.ReportJoinResultMetric() + sharedmetrics.ReportJoinResultMetric() } mc.SetConditions(newCondition) @@ -664,7 +664,7 @@ func markMemberClusterLeft(recorder record.EventRecorder, mc apis.ConditionedObj if existingCondition == nil || existingCondition.Status != newCondition.Status { recorder.Event(mc, corev1.EventTypeNormal, reasonMemberClusterJoined, "member cluster left") klog.V(2).InfoS("memberCluster left", "memberCluster", klog.KObj(mc)) - metrics.ReportLeaveResultMetric() + sharedmetrics.ReportLeaveResultMetric() } mc.SetConditions(newCondition, notReadyCondition) diff --git a/pkg/controllers/placement/controller.go b/pkg/controllers/placement/controller.go index 232cd1776..eba1bb7bb 100644 --- a/pkg/controllers/placement/controller.go +++ b/pkg/controllers/placement/controller.go @@ -37,7 +37,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" - "go.goms.io/fleet/pkg/metrics" + hubmetrics "go.goms.io/fleet/pkg/metrics/hub" "go.goms.io/fleet/pkg/scheduler/queue" "go.goms.io/fleet/pkg/utils/annotations" "go.goms.io/fleet/pkg/utils/condition" @@ -112,7 +112,7 @@ func (r *Reconciler) handleDelete(ctx context.Context, placementObj fleetv1beta1 return ctrl.Result{}, err } // change the metrics to add nameplace of namespace - metrics.FleetPlacementStatusLastTimeStampSeconds.DeletePartialMatch(prometheus.Labels{"namespace": placementObj.GetNamespace(), "name": placementObj.GetName()}) + hubmetrics.FleetPlacementStatusLastTimeStampSeconds.DeletePartialMatch(prometheus.Labels{"namespace": placementObj.GetNamespace(), "name": placementObj.GetName()}) controllerutil.RemoveFinalizer(placementObj, fleetv1beta1.PlacementCleanupFinalizer) if err := r.Client.Update(ctx, placementObj); err != nil { klog.ErrorS(err, "Failed to remove placement finalizer", "placement", placementKObj) @@ -1266,7 +1266,7 @@ func emitPlacementStatusMetric(placementObj fleetv1beta1.PlacementObj) { status = string(cond.Status) reason = cond.Reason } - metrics.FleetPlacementStatusLastTimeStampSeconds.WithLabelValues(placementObj.GetNamespace(), placementObj.GetName(), strconv.FormatInt(placementObj.GetGeneration(), 10), scheduledConditionType, status, reason).SetToCurrentTime() + hubmetrics.FleetPlacementStatusLastTimeStampSeconds.WithLabelValues(placementObj.GetNamespace(), placementObj.GetName(), strconv.FormatInt(placementObj.GetGeneration(), 10), scheduledConditionType, status, reason).SetToCurrentTime() return } @@ -1280,12 +1280,12 @@ func emitPlacementStatusMetric(placementObj fleetv1beta1.PlacementObj) { status = string(cond.Status) reason = cond.Reason } - metrics.FleetPlacementStatusLastTimeStampSeconds.WithLabelValues(placementObj.GetNamespace(), placementObj.GetName(), strconv.FormatInt(placementObj.GetGeneration(), 10), conditionType, status, reason).SetToCurrentTime() + hubmetrics.FleetPlacementStatusLastTimeStampSeconds.WithLabelValues(placementObj.GetNamespace(), placementObj.GetName(), strconv.FormatInt(placementObj.GetGeneration(), 10), conditionType, status, reason).SetToCurrentTime() return } } // Emit the "Completed" condition metric to indicate that the placement has completed. // This condition is used solely for metric reporting purposes. - metrics.FleetPlacementStatusLastTimeStampSeconds.WithLabelValues(placementObj.GetNamespace(), placementObj.GetName(), strconv.FormatInt(placementObj.GetGeneration(), 10), "Completed", string(metav1.ConditionTrue), "Completed").SetToCurrentTime() + hubmetrics.FleetPlacementStatusLastTimeStampSeconds.WithLabelValues(placementObj.GetNamespace(), placementObj.GetName(), strconv.FormatInt(placementObj.GetGeneration(), 10), "Completed", string(metav1.ConditionTrue), "Completed").SetToCurrentTime() } diff --git a/pkg/controllers/placement/controller_integration_test.go b/pkg/controllers/placement/controller_integration_test.go index ddb54f157..7c1ab0a6c 100644 --- a/pkg/controllers/placement/controller_integration_test.go +++ b/pkg/controllers/placement/controller_integration_test.go @@ -35,7 +35,7 @@ import ( ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" - "go.goms.io/fleet/pkg/metrics" + hubmetrics "go.goms.io/fleet/pkg/metrics/hub" "go.goms.io/fleet/pkg/utils" "go.goms.io/fleet/pkg/utils/condition" "go.goms.io/fleet/pkg/utils/resource" @@ -382,7 +382,7 @@ var _ = Describe("Test ClusterResourcePlacement Controller", func() { Context("When creating new pickAll ClusterResourcePlacement", func() { BeforeEach(func() { // Reset metric before each test - metrics.FleetPlacementStatusLastTimeStampSeconds.Reset() + hubmetrics.FleetPlacementStatusLastTimeStampSeconds.Reset() By("Create a new crp") crp = &placementv1beta1.ClusterResourcePlacement{ @@ -1411,7 +1411,7 @@ var _ = Describe("Test ClusterResourcePlacement Controller", func() { Context("When creating a ReportDiff ClusterResourcePlacement", func() { BeforeEach(func() { // Reset metric before each test - metrics.FleetPlacementStatusLastTimeStampSeconds.Reset() + hubmetrics.FleetPlacementStatusLastTimeStampSeconds.Reset() By("Create a new crp") crp = &placementv1beta1.ClusterResourcePlacement{ @@ -1929,7 +1929,7 @@ var _ = Describe("Test ClusterResourcePlacement Controller", func() { Context("When creating an ClusterResourcePlacement with user error", func() { BeforeEach(func() { // Reset metric before each test - metrics.FleetPlacementStatusLastTimeStampSeconds.Reset() + hubmetrics.FleetPlacementStatusLastTimeStampSeconds.Reset() }) AfterEach(func() { @@ -2000,7 +2000,7 @@ var _ = Describe("Test ClusterResourcePlacement Controller", func() { Context("When creating an ClusterResourcePlacement with External RolloutStrategy", func() { BeforeEach(func() { // Reset metric before each test - metrics.FleetPlacementStatusLastTimeStampSeconds.Reset() + hubmetrics.FleetPlacementStatusLastTimeStampSeconds.Reset() By("Create a new crp with external rollout strategy") crp = &placementv1beta1.ClusterResourcePlacement{ diff --git a/pkg/controllers/placement/placement_controllerv1alpha1.go b/pkg/controllers/placement/placement_controllerv1alpha1.go index fe2995e96..d78ab41b4 100644 --- a/pkg/controllers/placement/placement_controllerv1alpha1.go +++ b/pkg/controllers/placement/placement_controllerv1alpha1.go @@ -33,7 +33,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" fleetv1alpha1 "go.goms.io/fleet/apis/v1alpha1" - "go.goms.io/fleet/pkg/metrics" + hubmetrics "go.goms.io/fleet/pkg/metrics/hub" "go.goms.io/fleet/pkg/utils" "go.goms.io/fleet/pkg/utils/condition" "go.goms.io/fleet/pkg/utils/controller" @@ -323,7 +323,7 @@ func (r *Reconciler) updatePlacementAppliedCondition(placement *fleetv1alpha1.Cl }) if preAppliedCond == nil || preAppliedCond.Status != metav1.ConditionTrue { klog.V(2).InfoS("successfully applied all selected resources", "placement", placementRef) - metrics.PlacementApplySucceedCount.WithLabelValues(placement.GetName()).Inc() + hubmetrics.PlacementApplySucceedCount.WithLabelValues(placement.GetName()).Inc() r.Recorder.Event(placement, corev1.EventTypeNormal, "ResourceApplied", "successfully applied all selected resources") } case errors.Is(applyErr, ErrStillPendingManifest): @@ -349,7 +349,7 @@ func (r *Reconciler) updatePlacementAppliedCondition(placement *fleetv1alpha1.Cl }) if preAppliedCond == nil || preAppliedCond.Status != metav1.ConditionFalse { klog.V(2).InfoS("failed to apply some selected resources", "placement", placementRef) - metrics.PlacementApplyFailedCount.WithLabelValues(placement.GetName()).Inc() + hubmetrics.PlacementApplyFailedCount.WithLabelValues(placement.GetName()).Inc() r.Recorder.Event(placement, corev1.EventTypeWarning, "ResourceApplyFailed", "failed to apply some selected resources") } } diff --git a/pkg/controllers/placement/suite_test.go b/pkg/controllers/placement/suite_test.go index fca00ca98..c8381a4f4 100644 --- a/pkg/controllers/placement/suite_test.go +++ b/pkg/controllers/placement/suite_test.go @@ -35,7 +35,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/manager" - ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" @@ -43,7 +42,6 @@ import ( "go.goms.io/fleet/pkg/controllers/bindingwatcher" "go.goms.io/fleet/pkg/controllers/placementwatcher" "go.goms.io/fleet/pkg/controllers/schedulingpolicysnapshot" - "go.goms.io/fleet/pkg/metrics" "go.goms.io/fleet/pkg/utils" "go.goms.io/fleet/pkg/utils/controller" "go.goms.io/fleet/pkg/utils/informer" @@ -152,9 +150,6 @@ var _ = BeforeSuite(func() { }).SetupWithManagerForClusterResourceBinding(mgr) Expect(err).Should(Succeed(), "failed to create clusterResourceBinding watcher") - // Register metrics. - ctrlmetrics.Registry.MustRegister(metrics.FleetPlacementStatusLastTimeStampSeconds) - ctx, cancel = context.WithCancel(context.TODO()) // Run the controller manager go func() { diff --git a/pkg/controllers/updaterun/controller.go b/pkg/controllers/updaterun/controller.go index ca50c2d0d..7eb36e042 100644 --- a/pkg/controllers/updaterun/controller.go +++ b/pkg/controllers/updaterun/controller.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package updaterun features a controller to reconcile the clusterStagedUpdateRun objects. +// Package updaterun features a controller to reconcile the updateRun objects. package updaterun import ( @@ -41,7 +41,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" - "go.goms.io/fleet/pkg/metrics" + hubmetrics "go.goms.io/fleet/pkg/metrics/hub" "go.goms.io/fleet/pkg/utils" "go.goms.io/fleet/pkg/utils/condition" "go.goms.io/fleet/pkg/utils/controller" @@ -49,14 +49,14 @@ import ( ) var ( - // errStagedUpdatedAborted is the error when the ClusterStagedUpdateRun is aborted. - errStagedUpdatedAborted = fmt.Errorf("cannot continue the ClusterStagedUpdateRun") - // errInitializedFailed is the error when the ClusterStagedUpdateRun fails to initialize. + // errStagedUpdatedAborted is the error when the updateRun is aborted. + errStagedUpdatedAborted = fmt.Errorf("cannot continue the updateRun") + // errInitializedFailed is the error when the updateRun fails to initialize. // It is a wrapped error of errStagedUpdatedAborted, because some initialization functions are reused in the validation step. - errInitializedFailed = fmt.Errorf("%w: failed to initialize the clusterStagedUpdateRun", errStagedUpdatedAborted) + errInitializedFailed = fmt.Errorf("%w: failed to initialize the updateRun", errStagedUpdatedAborted) ) -// Reconciler reconciles a ClusterStagedUpdateRun object. +// Reconciler reconciles an updateRun object. type Reconciler struct { client.Client recorder record.EventRecorder @@ -66,28 +66,28 @@ type Reconciler struct { func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtime.Result, error) { startTime := time.Now() - klog.V(2).InfoS("ClusterStagedUpdateRun reconciliation starts", "clusterStagedUpdateRun", req.NamespacedName) + klog.V(2).InfoS("UpdateRun reconciliation starts", "updateRun", req.NamespacedName) defer func() { latency := time.Since(startTime).Milliseconds() - klog.V(2).InfoS("ClusterStagedUpdateRun reconciliation ends", "clusterStagedUpdateRun", req.NamespacedName, "latency", latency) + klog.V(2).InfoS("UpdateRun reconciliation ends", "updateRun", req.NamespacedName, "latency", latency) }() - var updateRun placementv1beta1.ClusterStagedUpdateRun - if err := r.Client.Get(ctx, req.NamespacedName, &updateRun); err != nil { - klog.ErrorS(err, "Failed to get clusterStagedUpdateRun object", "clusterStagedUpdateRun", req.Name) + updateRun, err := controller.FetchUpdateRunFromRequest(ctx, r.Client, req) + if err != nil { + klog.ErrorS(err, "Failed to get updateRun object", "updateRun", req.NamespacedName) return runtime.Result{}, client.IgnoreNotFound(err) } - runObjRef := klog.KObj(&updateRun) + runObjRef := klog.KObj(updateRun) // Remove waitTime from the updateRun status for AfterStageTask for type Approval. - removeWaitTimeFromUpdateRunStatus(&updateRun) - - // Handle the deletion of the clusterStagedUpdateRun. - if !updateRun.DeletionTimestamp.IsZero() { - klog.V(2).InfoS("The clusterStagedUpdateRun is being deleted", "clusterStagedUpdateRun", runObjRef) - deleted, waitTime, err := r.handleDelete(ctx, updateRun.DeepCopy()) - if err != nil { - return runtime.Result{}, err + removeWaitTimeFromUpdateRunStatus(updateRun) + + // Handle the deletion of the updateRun. + if !updateRun.GetDeletionTimestamp().IsZero() { + klog.V(2).InfoS("The updateRun is being deleted", "updateRun", runObjRef) + deleted, waitTime, deleteErr := r.handleDelete(ctx, updateRun.DeepCopyObject().(placementv1beta1.UpdateRunObj)) + if deleteErr != nil { + return runtime.Result{}, deleteErr } if deleted { return runtime.Result{}, nil @@ -95,281 +95,357 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim return runtime.Result{RequeueAfter: waitTime}, nil } - // Add the finalizer to the clusterStagedUpdateRun. - if err := r.ensureFinalizer(ctx, &updateRun); err != nil { - klog.ErrorS(err, "Failed to add the finalizer to the clusterStagedUpdateRun", "clusterStagedUpdateRun", runObjRef) + // Add the finalizer to the updateRun. + if err := r.ensureFinalizer(ctx, updateRun); err != nil { + klog.ErrorS(err, "Failed to add the finalizer to the updateRun", "updateRun", runObjRef) return runtime.Result{}, err } // Emit the update run status metric based on status conditions in the updateRun. - defer emitUpdateRunStatusMetric(&updateRun) + defer emitUpdateRunStatusMetric(updateRun) var updatingStageIndex int - var toBeUpdatedBindings, toBeDeletedBindings []*placementv1beta1.ClusterResourceBinding - var err error - initCond := meta.FindStatusCondition(updateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionInitialized)) - if !condition.IsConditionStatusTrue(initCond, updateRun.Generation) { - if condition.IsConditionStatusFalse(initCond, updateRun.Generation) { - klog.V(2).InfoS("The clusterStagedUpdateRun has failed to initialize", "errorMsg", initCond.Message, "clusterStagedUpdateRun", runObjRef) + var toBeUpdatedBindings, toBeDeletedBindings []placementv1beta1.BindingObj + updateRunStatus := updateRun.GetUpdateRunStatus() + initCond := meta.FindStatusCondition(updateRunStatus.Conditions, string(placementv1beta1.StagedUpdateRunConditionInitialized)) + if !condition.IsConditionStatusTrue(initCond, updateRun.GetGeneration()) { + if condition.IsConditionStatusFalse(initCond, updateRun.GetGeneration()) { + klog.V(2).InfoS("The updateRun has failed to initialize", "errorMsg", initCond.Message, "updateRun", runObjRef) return runtime.Result{}, nil } - if toBeUpdatedBindings, toBeDeletedBindings, err = r.initialize(ctx, &updateRun); err != nil { - klog.ErrorS(err, "Failed to initialize the clusterStagedUpdateRun", "clusterStagedUpdateRun", runObjRef) + var initErr error + if toBeUpdatedBindings, toBeDeletedBindings, initErr = r.initialize(ctx, updateRun); initErr != nil { + klog.ErrorS(initErr, "Failed to initialize the updateRun", "updateRun", runObjRef) // errInitializedFailed cannot be retried. - if errors.Is(err, errInitializedFailed) { - return runtime.Result{}, r.recordInitializationFailed(ctx, &updateRun, err.Error()) + if errors.Is(initErr, errInitializedFailed) { + return runtime.Result{}, r.recordInitializationFailed(ctx, updateRun, initErr.Error()) } - return runtime.Result{}, err + return runtime.Result{}, initErr } updatingStageIndex = 0 // start from the first stage. - klog.V(2).InfoS("Initialized the clusterStagedUpdateRun", "clusterStagedUpdateRun", runObjRef) + klog.V(2).InfoS("Initialized the updateRun", "updateRun", runObjRef) } else { - klog.V(2).InfoS("The clusterStagedUpdateRun is initialized", "clusterStagedUpdateRun", runObjRef) - // Check if the clusterStagedUpdateRun is finished. - finishedCond := meta.FindStatusCondition(updateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionSucceeded)) - if condition.IsConditionStatusTrue(finishedCond, updateRun.Generation) || condition.IsConditionStatusFalse(finishedCond, updateRun.Generation) { - klog.V(2).InfoS("The clusterStagedUpdateRun is finished", "finishedSuccessfully", finishedCond.Status, "clusterStagedUpdateRun", runObjRef) + klog.V(2).InfoS("The updateRun is initialized", "updateRun", runObjRef) + // Check if the updateRun is finished. + finishedCond := meta.FindStatusCondition(updateRunStatus.Conditions, string(placementv1beta1.StagedUpdateRunConditionSucceeded)) + if condition.IsConditionStatusTrue(finishedCond, updateRun.GetGeneration()) || condition.IsConditionStatusFalse(finishedCond, updateRun.GetGeneration()) { + klog.V(2).InfoS("The updateRun is finished", "finishedSuccessfully", finishedCond.Status, "updateRun", runObjRef) return runtime.Result{}, nil } - - // Validate the clusterStagedUpdateRun status to ensure the update can be continued and get the updating stage index and cluster indices. - if updatingStageIndex, toBeUpdatedBindings, toBeDeletedBindings, err = r.validate(ctx, &updateRun); err != nil { + var validateErr error + // Validate the updateRun status to ensure the update can be continued and get the updating stage index and cluster indices. + if updatingStageIndex, toBeUpdatedBindings, toBeDeletedBindings, validateErr = r.validate(ctx, updateRun); validateErr != nil { // errStagedUpdatedAborted cannot be retried. - if errors.Is(err, errStagedUpdatedAborted) { - return runtime.Result{}, r.recordUpdateRunFailed(ctx, &updateRun, err.Error()) + if errors.Is(validateErr, errStagedUpdatedAborted) { + return runtime.Result{}, r.recordUpdateRunFailed(ctx, updateRun, validateErr.Error()) } - return runtime.Result{}, err + return runtime.Result{}, validateErr } - klog.V(2).InfoS("The clusterStagedUpdateRun is validated", "clusterStagedUpdateRun", runObjRef) + klog.V(2).InfoS("The updateRun is validated", "updateRun", runObjRef) } // The previous run is completed but the update to the status failed. if updatingStageIndex == -1 { - klog.V(2).InfoS("The clusterStagedUpdateRun is completed", "clusterStagedUpdateRun", runObjRef) - return runtime.Result{}, r.recordUpdateRunSucceeded(ctx, &updateRun) + klog.V(2).InfoS("The updateRun is completed", "updateRun", runObjRef) + return runtime.Result{}, r.recordUpdateRunSucceeded(ctx, updateRun) } // Execute the updateRun. - klog.V(2).InfoS("Continue to execute the clusterStagedUpdateRun", "updatingStageIndex", updatingStageIndex, "clusterStagedUpdateRun", runObjRef) - finished, waitTime, execErr := r.execute(ctx, &updateRun, updatingStageIndex, toBeUpdatedBindings, toBeDeletedBindings) + klog.V(2).InfoS("Continue to execute the updateRun", "updatingStageIndex", updatingStageIndex, "updateRun", runObjRef) + finished, waitTime, execErr := r.execute(ctx, updateRun, updatingStageIndex, toBeUpdatedBindings, toBeDeletedBindings) if errors.Is(execErr, errStagedUpdatedAborted) { // errStagedUpdatedAborted cannot be retried. - return runtime.Result{}, r.recordUpdateRunFailed(ctx, &updateRun, execErr.Error()) + return runtime.Result{}, r.recordUpdateRunFailed(ctx, updateRun, execErr.Error()) } if finished { - klog.V(2).InfoS("The clusterStagedUpdateRun is completed", "clusterStagedUpdateRun", runObjRef) - return runtime.Result{}, r.recordUpdateRunSucceeded(ctx, &updateRun) + klog.V(2).InfoS("The updateRun is completed", "updateRun", runObjRef) + return runtime.Result{}, r.recordUpdateRunSucceeded(ctx, updateRun) } // The execution is not finished yet or it encounters a retriable error. // We need to record the status and requeue. - if updateErr := r.recordUpdateRunStatus(ctx, &updateRun); updateErr != nil { + if updateErr := r.recordUpdateRunStatus(ctx, updateRun); updateErr != nil { return runtime.Result{}, updateErr } - klog.V(2).InfoS("The clusterStagedUpdateRun is not finished yet", "requeueWaitTime", waitTime, "execErr", execErr, "clusterStagedUpdateRun", runObjRef) + klog.V(2).InfoS("The updateRun is not finished yet", "requeueWaitTime", waitTime, "execErr", execErr, "updateRun", runObjRef) if execErr != nil { return runtime.Result{}, execErr } return runtime.Result{Requeue: true, RequeueAfter: waitTime}, nil } -// handleDelete handles the deletion of the clusterStagedUpdateRun object. -// We delete all the dependent resources, including approvalRequest objects, of the clusterStagedUpdateRun object. -func (r *Reconciler) handleDelete(ctx context.Context, updateRun *placementv1beta1.ClusterStagedUpdateRun) (bool, time.Duration, error) { +// handleDelete handles the deletion of the updateRun object. +// We delete all the dependent resources, including approvalRequest objects, of the updateRun object. +func (r *Reconciler) handleDelete(ctx context.Context, updateRun placementv1beta1.UpdateRunObj) (bool, time.Duration, error) { runObjRef := klog.KObj(updateRun) // Delete all the associated approvalRequests. - approvalRequest := &placementv1beta1.ClusterApprovalRequest{} - if err := r.Client.DeleteAllOf(ctx, approvalRequest, client.MatchingLabels{placementv1beta1.TargetUpdateRunLabel: updateRun.GetName()}); err != nil { - klog.ErrorS(err, "Failed to delete all associated approvalRequests", "clusterStagedUpdateRun", runObjRef) + var approvalRequest placementv1beta1.ApprovalRequestObj + deleteOptions := []client.DeleteAllOfOption{client.MatchingLabels{placementv1beta1.TargetUpdateRunLabel: updateRun.GetName()}} + if updateRun.GetNamespace() == "" { + approvalRequest = &placementv1beta1.ClusterApprovalRequest{} + } else { + approvalRequest = &placementv1beta1.ApprovalRequest{} + deleteOptions = append(deleteOptions, client.InNamespace(updateRun.GetNamespace())) + } + if err := r.Client.DeleteAllOf(ctx, approvalRequest, deleteOptions...); err != nil { + klog.ErrorS(err, "Failed to delete all associated approvalRequests", "updateRun", runObjRef) return false, 0, controller.NewAPIServerError(false, err) } - klog.V(2).InfoS("Deleted all approvalRequests associated with the clusterStagedUpdateRun", "clusterStagedUpdateRun", runObjRef) + klog.V(2).InfoS("Deleted all approvalRequests associated with the updateRun", "updateRun", runObjRef) // Delete the update run status metric. - metrics.FleetUpdateRunStatusLastTimestampSeconds.DeletePartialMatch(prometheus.Labels{"name": updateRun.GetName()}) + hubmetrics.FleetUpdateRunStatusLastTimestampSeconds.DeletePartialMatch(prometheus.Labels{"namespace": updateRun.GetNamespace(), "name": updateRun.GetName()}) - controllerutil.RemoveFinalizer(updateRun, placementv1beta1.ClusterStagedUpdateRunFinalizer) + controllerutil.RemoveFinalizer(updateRun, placementv1beta1.UpdateRunFinalizer) if err := r.Client.Update(ctx, updateRun); err != nil { - klog.ErrorS(err, "Failed to remove updateRun finalizer", "clusterStagedUpdateRun", runObjRef) + klog.ErrorS(err, "Failed to remove updateRun finalizer", "updateRun", runObjRef) return false, 0, controller.NewUpdateIgnoreConflictError(err) } return true, 0, nil } -// ensureFinalizer makes sure that the ClusterStagedUpdateRun CR has a finalizer on it. -func (r *Reconciler) ensureFinalizer(ctx context.Context, updateRun *placementv1beta1.ClusterStagedUpdateRun) error { - if controllerutil.ContainsFinalizer(updateRun, placementv1beta1.ClusterStagedUpdateRunFinalizer) { +// ensureFinalizer makes sure that the updateRun CR has a finalizer on it. +func (r *Reconciler) ensureFinalizer(ctx context.Context, updateRun placementv1beta1.UpdateRunObj) error { + if controllerutil.ContainsFinalizer(updateRun, placementv1beta1.UpdateRunFinalizer) { return nil } - klog.InfoS("Added the staged update run finalizer", "stagedUpdateRun", klog.KObj(updateRun)) - controllerutil.AddFinalizer(updateRun, placementv1beta1.ClusterStagedUpdateRunFinalizer) + klog.InfoS("Added the updateRun finalizer", "updateRun", klog.KObj(updateRun)) + controllerutil.AddFinalizer(updateRun, placementv1beta1.UpdateRunFinalizer) return r.Update(ctx, updateRun, client.FieldOwner(utils.UpdateRunControllerFieldManagerName)) } -// recordUpdateRunSucceeded records the succeeded condition in the ClusterStagedUpdateRun status. -func (r *Reconciler) recordUpdateRunSucceeded(ctx context.Context, updateRun *placementv1beta1.ClusterStagedUpdateRun) error { - meta.SetStatusCondition(&updateRun.Status.Conditions, metav1.Condition{ +// recordUpdateRunSucceeded records the succeeded condition in the updateRun status. +func (r *Reconciler) recordUpdateRunSucceeded(ctx context.Context, updateRun placementv1beta1.UpdateRunObj) error { + updateRunStatus := updateRun.GetUpdateRunStatus() + meta.SetStatusCondition(&updateRunStatus.Conditions, metav1.Condition{ Type: string(placementv1beta1.StagedUpdateRunConditionProgressing), Status: metav1.ConditionFalse, - ObservedGeneration: updateRun.Generation, + ObservedGeneration: updateRun.GetGeneration(), Reason: condition.UpdateRunSucceededReason, Message: "All stages are completed", }) - meta.SetStatusCondition(&updateRun.Status.Conditions, metav1.Condition{ + meta.SetStatusCondition(&updateRunStatus.Conditions, metav1.Condition{ Type: string(placementv1beta1.StagedUpdateRunConditionSucceeded), Status: metav1.ConditionTrue, - ObservedGeneration: updateRun.Generation, + ObservedGeneration: updateRun.GetGeneration(), Reason: condition.UpdateRunSucceededReason, Message: "All stages are completed successfully", }) if updateErr := r.Client.Status().Update(ctx, updateRun); updateErr != nil { - klog.ErrorS(updateErr, "Failed to update the ClusterStagedUpdateRun status as succeeded", "clusterStagedUpdateRun", klog.KObj(updateRun)) + klog.ErrorS(updateErr, "Failed to update the updateRun status as succeeded", "updateRun", klog.KObj(updateRun)) // updateErr can be retried. return controller.NewUpdateIgnoreConflictError(updateErr) } return nil } -// recordUpdateRunFailed records the failed condition in the ClusterStagedUpdateRun status. -func (r *Reconciler) recordUpdateRunFailed(ctx context.Context, updateRun *placementv1beta1.ClusterStagedUpdateRun, message string) error { - meta.SetStatusCondition(&updateRun.Status.Conditions, metav1.Condition{ +// recordUpdateRunFailed records the failed condition in the updateRun status. +func (r *Reconciler) recordUpdateRunFailed(ctx context.Context, updateRun placementv1beta1.UpdateRunObj, message string) error { + updateRunStatus := updateRun.GetUpdateRunStatus() + meta.SetStatusCondition(&updateRunStatus.Conditions, metav1.Condition{ Type: string(placementv1beta1.StagedUpdateRunConditionProgressing), Status: metav1.ConditionFalse, - ObservedGeneration: updateRun.Generation, + ObservedGeneration: updateRun.GetGeneration(), Reason: condition.UpdateRunFailedReason, Message: "The stages are aborted due to a non-recoverable error", }) - meta.SetStatusCondition(&updateRun.Status.Conditions, metav1.Condition{ + meta.SetStatusCondition(&updateRunStatus.Conditions, metav1.Condition{ Type: string(placementv1beta1.StagedUpdateRunConditionSucceeded), Status: metav1.ConditionFalse, - ObservedGeneration: updateRun.Generation, + ObservedGeneration: updateRun.GetGeneration(), Reason: condition.UpdateRunFailedReason, Message: message, }) if updateErr := r.Client.Status().Update(ctx, updateRun); updateErr != nil { - klog.ErrorS(updateErr, "Failed to update the ClusterStagedUpdateRun status as failed", "clusterStagedUpdateRun", klog.KObj(updateRun)) + klog.ErrorS(updateErr, "Failed to update the updateRun status as failed", "updateRun", klog.KObj(updateRun)) // updateErr can be retried. return controller.NewUpdateIgnoreConflictError(updateErr) } return nil } -// recordUpdateRunStatus records the ClusterStagedUpdateRun status. -func (r *Reconciler) recordUpdateRunStatus(ctx context.Context, updateRun *placementv1beta1.ClusterStagedUpdateRun) error { +// recordUpdateRunStatus records the updateRun status. +func (r *Reconciler) recordUpdateRunStatus(ctx context.Context, updateRun placementv1beta1.UpdateRunObj) error { if updateErr := r.Client.Status().Update(ctx, updateRun); updateErr != nil { - klog.ErrorS(updateErr, "Failed to update the ClusterStagedUpdateRun status", "clusterStagedUpdateRun", klog.KObj(updateRun)) + klog.ErrorS(updateErr, "Failed to update the updateRun status", "updateRun", klog.KObj(updateRun)) return controller.NewUpdateIgnoreConflictError(updateErr) } return nil } -// SetupWithManager sets up the controller with the Manager. -func (r *Reconciler) SetupWithManager(mgr runtime.Manager) error { - r.recorder = mgr.GetEventRecorderFor("clusterresource-stagedupdaterun-controller") +// SetupWithManagerForClusterStagedUpdateRun sets up the controller with the Manager for ClusterStagedUpdateRun resources. +func (r *Reconciler) SetupWithManagerForClusterStagedUpdateRun(mgr runtime.Manager) error { + r.recorder = mgr.GetEventRecorderFor("clusterstagedupdaterun-controller") return runtime.NewControllerManagedBy(mgr). - Named("clusterresource-stagedupdaterun-controller"). + Named("clusterstagedupdaterun-controller"). For(&placementv1beta1.ClusterStagedUpdateRun{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). Watches(&placementv1beta1.ClusterApprovalRequest{}, &handler.Funcs{ // We watch for ClusterApprovalRequest to be approved. UpdateFunc: func(ctx context.Context, e event.UpdateEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { klog.V(2).InfoS("Handling a clusterApprovalRequest update event", "clusterApprovalRequest", klog.KObj(e.ObjectNew)) - handleClusterApprovalRequestUpdate(e.ObjectOld, e.ObjectNew, q) + handleApprovalRequestUpdate(e.ObjectOld, e.ObjectNew, q, true) }, // We watch for ClusterApprovalRequest deletion events to recreate it ASAP. DeleteFunc: func(ctx context.Context, e event.DeleteEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { klog.V(2).InfoS("Handling a clusterApprovalRequest delete event", "clusterApprovalRequest", klog.KObj(e.Object)) - handleClusterApprovalRequestDelete(e.Object, q) + handleApprovalRequestDelete(e.Object, q, true) }, }).Complete(r) } -// handleClusterApprovalRequestUpdate finds the ClusterStagedUpdateRun creating the ClusterApprovalRequest, -// and enqueues it to the ClusterStagedUpdateRun controller queue only when the approved condition is changed. -func handleClusterApprovalRequestUpdate(oldObj, newObj client.Object, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { - oldAppReq, ok := oldObj.(*placementv1beta1.ClusterApprovalRequest) - if !ok { - klog.V(2).ErrorS(controller.NewUnexpectedBehaviorError(fmt.Errorf("cannot cast runtime object to ClusterApprovalRequest")), - "Invalid object type", "object", klog.KObj(oldObj)) - return - } - newAppReq, ok := newObj.(*placementv1beta1.ClusterApprovalRequest) - if !ok { - klog.V(2).ErrorS(controller.NewUnexpectedBehaviorError(fmt.Errorf("cannot cast runtime object to ClusterApprovalRequest")), - "Invalid object type", "object", klog.KObj(newObj)) - return +// SetupWithManagerForStagedUpdateRun sets up the controller with the Manager for StagedUpdateRun resources. +func (r *Reconciler) SetupWithManagerForStagedUpdateRun(mgr runtime.Manager) error { + r.recorder = mgr.GetEventRecorderFor("stagedupdaterun-controller") + return runtime.NewControllerManagedBy(mgr). + Named("stagedupdaterun-controller"). + For(&placementv1beta1.StagedUpdateRun{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). + Watches(&placementv1beta1.ApprovalRequest{}, &handler.Funcs{ + // We watch for ApprovalRequest to be approved. + UpdateFunc: func(ctx context.Context, e event.UpdateEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + klog.V(2).InfoS("Handling an approvalRequest update event", "approvalRequest", klog.KObj(e.ObjectNew)) + handleApprovalRequestUpdate(e.ObjectOld, e.ObjectNew, q, false) + }, + // We watch for ApprovalRequest deletion events to recreate it ASAP. + DeleteFunc: func(ctx context.Context, e event.DeleteEvent, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + klog.V(2).InfoS("Handling an approvalRequest delete event", "approvalRequest", klog.KObj(e.Object)) + handleApprovalRequestDelete(e.Object, q, false) + }, + }).Complete(r) +} + +// handleApprovalRequestUpdate finds the UpdateRun creating the ApprovalRequest, +// and enqueues it to the UpdateRun controller queue only when the approved condition is changed. +// The isClusterScoped parameter determines whether to handle ClusterApprovalRequest (true) or ApprovalRequest (false). +func handleApprovalRequestUpdate(oldObj, newObj client.Object, q workqueue.TypedRateLimitingInterface[reconcile.Request], isClusterScoped bool) { + var oldAppReq, newAppReq placementv1beta1.ApprovalRequestObj + + if isClusterScoped { + oldClusterAppReq, ok := oldObj.(*placementv1beta1.ClusterApprovalRequest) + if !ok { + klog.V(2).ErrorS(controller.NewUnexpectedBehaviorError(fmt.Errorf("cannot cast runtime object to ClusterApprovalRequest")), + "Invalid object type", "object", klog.KObj(oldObj)) + return + } + newClusterAppReq, ok := newObj.(*placementv1beta1.ClusterApprovalRequest) + if !ok { + klog.V(2).ErrorS(controller.NewUnexpectedBehaviorError(fmt.Errorf("cannot cast runtime object to ClusterApprovalRequest")), + "Invalid object type", "object", klog.KObj(newObj)) + return + } + oldAppReq = oldClusterAppReq + newAppReq = newClusterAppReq + } else { + oldNamespacedAppReq, ok := oldObj.(*placementv1beta1.ApprovalRequest) + if !ok { + klog.V(2).ErrorS(controller.NewUnexpectedBehaviorError(fmt.Errorf("cannot cast runtime object to ApprovalRequest")), + "Invalid object type", "object", klog.KObj(oldObj)) + return + } + newNamespacedAppReq, ok := newObj.(*placementv1beta1.ApprovalRequest) + if !ok { + klog.V(2).ErrorS(controller.NewUnexpectedBehaviorError(fmt.Errorf("cannot cast runtime object to ApprovalRequest")), + "Invalid object type", "object", klog.KObj(newObj)) + return + } + oldAppReq = oldNamespacedAppReq + newAppReq = newNamespacedAppReq } - approvedInOld := condition.IsConditionStatusTrue(meta.FindStatusCondition(oldAppReq.Status.Conditions, string(placementv1beta1.ApprovalRequestConditionApproved)), oldAppReq.Generation) - approvedInNew := condition.IsConditionStatusTrue(meta.FindStatusCondition(newAppReq.Status.Conditions, string(placementv1beta1.ApprovalRequestConditionApproved)), newAppReq.Generation) + approvedInOld := condition.IsConditionStatusTrue(meta.FindStatusCondition(oldAppReq.GetApprovalRequestStatus().Conditions, string(placementv1beta1.ApprovalRequestConditionApproved)), oldAppReq.GetGeneration()) + approvedInNew := condition.IsConditionStatusTrue(meta.FindStatusCondition(newAppReq.GetApprovalRequestStatus().Conditions, string(placementv1beta1.ApprovalRequestConditionApproved)), newAppReq.GetGeneration()) if approvedInOld == approvedInNew { - klog.V(2).InfoS("The approval status is not changed, ignore queueing", "clusterApprovalRequest", klog.KObj(newAppReq)) + klog.V(2).InfoS("The approval status is not changed, ignore queueing", "approvalRequestObj", klog.KObj(newAppReq)) return } - updateRun := newAppReq.Spec.TargetUpdateRun + updateRun := newAppReq.GetApprovalRequestSpec().TargetUpdateRun if len(updateRun) == 0 { - klog.V(2).ErrorS(controller.NewUnexpectedBehaviorError(fmt.Errorf("TargetUpdateRun field in ClusterApprovalRequest is empty")), - "Invalid clusterApprovalRequest", "clusterApprovalRequest", klog.KObj(newAppReq)) + klog.V(2).ErrorS(controller.NewUnexpectedBehaviorError(fmt.Errorf("TargetUpdateRun field in ApprovalRequest is empty")), + "Invalid approval request", "approvalRequestObj", klog.KObj(newAppReq)) return } + // enqueue to the updaterun controller queue. q.Add(reconcile.Request{ - NamespacedName: types.NamespacedName{Name: updateRun}, + NamespacedName: types.NamespacedName{ + Namespace: newAppReq.GetNamespace(), + Name: updateRun, + }, }) } -// handleClusterApprovalRequestDelete finds the ClusterStagedUpdateRun creating the ClusterApprovalRequest, -// and enqueues it to the ClusterStagedUpdateRun controller queue when the ClusterApprovalRequest is deleted. -func handleClusterApprovalRequestDelete(obj client.Object, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { - appReq, ok := obj.(*placementv1beta1.ClusterApprovalRequest) - if !ok { - klog.V(2).ErrorS(controller.NewUnexpectedBehaviorError(fmt.Errorf("cannot cast runtime object to ClusterApprovalRequest")), - "Invalid object type", "object", klog.KObj(obj)) - return +// handleApprovalRequestDelete finds the UpdateRun creating the ApprovalRequest, +// and enqueues it to the UpdateRun controller queue when the ApprovalRequest is deleted. +// The isClusterScoped parameter determines whether to handle ClusterApprovalRequest (true) or ApprovalRequest (false). +func handleApprovalRequestDelete(obj client.Object, q workqueue.TypedRateLimitingInterface[reconcile.Request], isClusterScoped bool) { + var appReq placementv1beta1.ApprovalRequestObj + + if isClusterScoped { + clusterAppReq, ok := obj.(*placementv1beta1.ClusterApprovalRequest) + if !ok { + klog.V(2).ErrorS(controller.NewUnexpectedBehaviorError(fmt.Errorf("cannot cast runtime object to ClusterApprovalRequest")), + "Invalid object type", "object", klog.KObj(obj)) + return + } + appReq = clusterAppReq + } else { + namespacedAppReq, ok := obj.(*placementv1beta1.ApprovalRequest) + if !ok { + klog.V(2).ErrorS(controller.NewUnexpectedBehaviorError(fmt.Errorf("cannot cast runtime object to ApprovalRequest")), + "Invalid object type", "object", klog.KObj(obj)) + return + } + appReq = namespacedAppReq } - isApproved := meta.FindStatusCondition(appReq.Status.Conditions, string(placementv1beta1.ApprovalRequestConditionApproved)) - approvalAccepted := condition.IsConditionStatusTrue(meta.FindStatusCondition(appReq.Status.Conditions, string(placementv1beta1.ApprovalRequestConditionApprovalAccepted)), appReq.Generation) + + isApproved := meta.FindStatusCondition(appReq.GetApprovalRequestStatus().Conditions, string(placementv1beta1.ApprovalRequestConditionApproved)) + approvalAccepted := condition.IsConditionStatusTrue(meta.FindStatusCondition(appReq.GetApprovalRequestStatus().Conditions, string(placementv1beta1.ApprovalRequestConditionApprovalAccepted)), appReq.GetGeneration()) if isApproved != nil && approvalAccepted { - klog.V(2).InfoS("The approval request has been approved and accepted, ignore queueing for delete event", "clusterApprovalRequest", klog.KObj(appReq)) + klog.V(2).InfoS("The approval request has been approved and accepted, ignore queueing for delete event", "approvalRequestObj", klog.KObj(appReq)) return } - updateRun := appReq.Spec.TargetUpdateRun + updateRun := appReq.GetApprovalRequestSpec().TargetUpdateRun if len(updateRun) == 0 { - klog.V(2).ErrorS(controller.NewUnexpectedBehaviorError(fmt.Errorf("TargetUpdateRun field in ClusterApprovalRequest is empty")), - "Invalid clusterApprovalRequest", "clusterApprovalRequest", klog.KObj(appReq)) + klog.V(2).ErrorS(controller.NewUnexpectedBehaviorError(fmt.Errorf("TargetUpdateRun field in ApprovalRequest is empty")), + "Invalid approval request", "approvalRequestObj", klog.KObj(appReq)) return } + // enqueue to the updaterun controller queue. q.Add(reconcile.Request{ - NamespacedName: types.NamespacedName{Name: appReq.Spec.TargetUpdateRun}, + NamespacedName: types.NamespacedName{ + Namespace: appReq.GetNamespace(), + Name: updateRun, + }, }) } // emitUpdateRunStatusMetric emits the update run status metric based on status conditions in the updateRun. -func emitUpdateRunStatusMetric(updateRun *placementv1beta1.ClusterStagedUpdateRun) { - generation := updateRun.Generation +func emitUpdateRunStatusMetric(updateRun placementv1beta1.UpdateRunObj) { + generation := updateRun.GetGeneration() genStr := strconv.FormatInt(generation, 10) - succeedCond := meta.FindStatusCondition(updateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionSucceeded)) + updateRunStatus := updateRun.GetUpdateRunStatus() + succeedCond := meta.FindStatusCondition(updateRunStatus.Conditions, string(placementv1beta1.StagedUpdateRunConditionSucceeded)) if succeedCond != nil && succeedCond.ObservedGeneration == generation { - metrics.FleetUpdateRunStatusLastTimestampSeconds.WithLabelValues(updateRun.Name, genStr, + hubmetrics.FleetUpdateRunStatusLastTimestampSeconds.WithLabelValues(updateRun.GetNamespace(), updateRun.GetName(), genStr, string(placementv1beta1.StagedUpdateRunConditionSucceeded), string(succeedCond.Status), succeedCond.Reason).SetToCurrentTime() return } - progressingCond := meta.FindStatusCondition(updateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionProgressing)) + progressingCond := meta.FindStatusCondition(updateRunStatus.Conditions, string(placementv1beta1.StagedUpdateRunConditionProgressing)) if progressingCond != nil && progressingCond.ObservedGeneration == generation { - metrics.FleetUpdateRunStatusLastTimestampSeconds.WithLabelValues(updateRun.Name, genStr, + hubmetrics.FleetUpdateRunStatusLastTimestampSeconds.WithLabelValues(updateRun.GetNamespace(), updateRun.GetName(), genStr, string(placementv1beta1.StagedUpdateRunConditionProgressing), string(progressingCond.Status), progressingCond.Reason).SetToCurrentTime() return } - initializedCond := meta.FindStatusCondition(updateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionInitialized)) + initializedCond := meta.FindStatusCondition(updateRunStatus.Conditions, string(placementv1beta1.StagedUpdateRunConditionInitialized)) if initializedCond != nil && initializedCond.ObservedGeneration == generation { - metrics.FleetUpdateRunStatusLastTimestampSeconds.WithLabelValues(updateRun.Name, genStr, + hubmetrics.FleetUpdateRunStatusLastTimestampSeconds.WithLabelValues(updateRun.GetNamespace(), updateRun.GetName(), genStr, string(placementv1beta1.StagedUpdateRunConditionInitialized), string(initializedCond.Status), initializedCond.Reason).SetToCurrentTime() return } @@ -378,13 +454,14 @@ func emitUpdateRunStatusMetric(updateRun *placementv1beta1.ClusterStagedUpdateRu klog.V(2).InfoS("There's no valid status condition on updateRun, status updating failed possibly", "updateRun", klog.KObj(updateRun)) } -func removeWaitTimeFromUpdateRunStatus(updateRun *placementv1beta1.ClusterStagedUpdateRun) { +func removeWaitTimeFromUpdateRunStatus(updateRun placementv1beta1.UpdateRunObj) { // Remove waitTime from the updateRun status for AfterStageTask for type Approval. - if updateRun.Status.StagedUpdateStrategySnapshot != nil { - for i := range updateRun.Status.StagedUpdateStrategySnapshot.Stages { - for j := range updateRun.Status.StagedUpdateStrategySnapshot.Stages[i].AfterStageTasks { - if updateRun.Status.StagedUpdateStrategySnapshot.Stages[i].AfterStageTasks[j].Type == placementv1beta1.AfterStageTaskTypeApproval { - updateRun.Status.StagedUpdateStrategySnapshot.Stages[i].AfterStageTasks[j].WaitTime = nil + updateRunStatus := updateRun.GetUpdateRunStatus() + if updateRunStatus.UpdateStrategySnapshot != nil { + for i := range updateRunStatus.UpdateStrategySnapshot.Stages { + for j := range updateRunStatus.UpdateStrategySnapshot.Stages[i].AfterStageTasks { + if updateRunStatus.UpdateStrategySnapshot.Stages[i].AfterStageTasks[j].Type == placementv1beta1.AfterStageTaskTypeApproval { + updateRunStatus.UpdateStrategySnapshot.Stages[i].AfterStageTasks[j].WaitTime = nil } } } diff --git a/pkg/controllers/updaterun/controller_integration_test.go b/pkg/controllers/updaterun/controller_integration_test.go index e829d48fe..bd31b3b6e 100644 --- a/pkg/controllers/updaterun/controller_integration_test.go +++ b/pkg/controllers/updaterun/controller_integration_test.go @@ -43,7 +43,7 @@ import ( clusterv1beta1 "go.goms.io/fleet/apis/cluster/v1beta1" placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" - "go.goms.io/fleet/pkg/metrics" + hubmetrics "go.goms.io/fleet/pkg/metrics/hub" "go.goms.io/fleet/pkg/utils" "go.goms.io/fleet/pkg/utils/condition" metricsutils "go.goms.io/fleet/test/utils/metrics" @@ -233,7 +233,7 @@ var _ = Describe("Test the clusterStagedUpdateRun controller", func() { }) func resetUpdateRunMetrics() { - metrics.FleetUpdateRunStatusLastTimestampSeconds.Reset() + hubmetrics.FleetUpdateRunStatusLastTimestampSeconds.Reset() } // validateUpdateRunMetricsEmitted validates the update run status metrics are emitted and are emitted in the correct order. @@ -263,6 +263,7 @@ func generateMetricsLabels( condition, status, reason string, ) []*prometheusclientmodel.LabelPair { return []*prometheusclientmodel.LabelPair{ + {Name: ptr.To("namespace"), Value: &updateRun.Namespace}, {Name: ptr.To("name"), Value: &updateRun.Name}, {Name: ptr.To("generation"), Value: ptr.To(strconv.FormatInt(updateRun.Generation, 10))}, {Name: ptr.To("condition"), Value: ptr.To(condition)}, @@ -336,7 +337,7 @@ func generateTestClusterStagedUpdateRun() *placementv1beta1.ClusterStagedUpdateR ObjectMeta: metav1.ObjectMeta{ Name: testUpdateRunName, }, - Spec: placementv1beta1.StagedUpdateRunSpec{ + Spec: placementv1beta1.UpdateRunSpec{ PlacementName: testCRPName, ResourceSnapshotIndex: testResourceSnapshotIndex, StagedUpdateStrategyName: testUpdateStrategyName, @@ -493,7 +494,7 @@ func generateTestClusterStagedUpdateStrategy() *placementv1beta1.ClusterStagedUp ObjectMeta: metav1.ObjectMeta{ Name: testUpdateStrategyName, }, - Spec: placementv1beta1.StagedUpdateStrategySpec{ + Spec: placementv1beta1.UpdateStrategySpec{ Stages: []placementv1beta1.StageConfig{ { Name: "stage1", @@ -547,7 +548,7 @@ func generateTestClusterStagedUpdateStrategyWithSingleStage(afterStageTasks []pl ObjectMeta: metav1.ObjectMeta{ Name: testUpdateStrategyName, }, - Spec: placementv1beta1.StagedUpdateStrategySpec{ + Spec: placementv1beta1.UpdateStrategySpec{ Stages: []placementv1beta1.StageConfig{ { Name: "stage1", @@ -662,7 +663,7 @@ func validateUpdateRunHasFinalizer(ctx context.Context, updateRun *placementv1be if err := k8sClient.Get(ctx, namespacedName, updateRun); err != nil { return fmt.Errorf("failed to get clusterStagedUpdateRun %s: %w", namespacedName, err) } - if !controllerutil.ContainsFinalizer(updateRun, placementv1beta1.ClusterStagedUpdateRunFinalizer) { + if !controllerutil.ContainsFinalizer(updateRun, placementv1beta1.UpdateRunFinalizer) { return fmt.Errorf("finalizer not added to clusterStagedUpdateRun %s", namespacedName) } return nil diff --git a/pkg/controllers/updaterun/controller_test.go b/pkg/controllers/updaterun/controller_test.go index 251871741..2d9cb35ef 100644 --- a/pkg/controllers/updaterun/controller_test.go +++ b/pkg/controllers/updaterun/controller_test.go @@ -30,18 +30,37 @@ import ( placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" ) -func TestHandleClusterApprovalRequestUpdate(t *testing.T) { +func TestHandleApprovalRequestUpdate(t *testing.T) { tests := map[string]struct { - oldObj client.Object - newObj client.Object - shouldEnqueue bool - queuedName string + oldObj client.Object + newObj client.Object + shouldEnqueue bool + queuedName string + isClusterScoped bool }{ - "it should not enqueue anything if the obj is not a ClusterApprovalRequest": { - oldObj: &placementv1beta1.ClusterStagedUpdateRun{}, - shouldEnqueue: false, + "cluster-scoped: it should not enqueue anything if the obj is not a ClusterApprovalRequest": { + oldObj: &placementv1beta1.ClusterStagedUpdateRun{}, + shouldEnqueue: false, + isClusterScoped: true, }, - "it should not enqueue anything if targetUpdateRun in spec is empty": { + "namespaced: it should not enqueue anything if the obj is not an ApprovalRequest": { + oldObj: &placementv1beta1.StagedUpdateRun{}, + shouldEnqueue: false, + isClusterScoped: false, + }, + "cluster-scoped: it should not enqueue anything if newObj is not a ClusterApprovalRequest": { + oldObj: &placementv1beta1.ClusterApprovalRequest{}, + newObj: &placementv1beta1.ClusterStagedUpdateRun{}, + shouldEnqueue: false, + isClusterScoped: true, + }, + "namespaced: it should not enqueue anything if newObj is not an ApprovalRequest": { + oldObj: &placementv1beta1.ApprovalRequest{}, + newObj: &placementv1beta1.StagedUpdateRun{}, + shouldEnqueue: false, + isClusterScoped: false, + }, + "cluster-scoped: it should not enqueue anything if targetUpdateRun in spec is empty": { oldObj: &placementv1beta1.ClusterApprovalRequest{ ObjectMeta: metav1.ObjectMeta{ Generation: 1, @@ -67,9 +86,39 @@ func TestHandleClusterApprovalRequestUpdate(t *testing.T) { }, }, }, - shouldEnqueue: false, + shouldEnqueue: false, + isClusterScoped: true, + }, + "namespaced: it should not enqueue anything if targetUpdateRun in spec is empty": { + oldObj: &placementv1beta1.ApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 1, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: "", + }, + }, + newObj: &placementv1beta1.ApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 1, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: "", + }, + Status: placementv1beta1.ApprovalRequestStatus{ + Conditions: []metav1.Condition{ + { + Status: metav1.ConditionTrue, + Type: string(placementv1beta1.ApprovalRequestConditionApproved), + ObservedGeneration: 1, + }, + }, + }, + }, + shouldEnqueue: false, + isClusterScoped: false, }, - "it should enqueue the targetUpdateRun if oldObj is not approved while newobj is approved": { + "cluster-scoped: it should enqueue the targetUpdateRun if oldObj is not approved while newobj is approved": { oldObj: &placementv1beta1.ClusterApprovalRequest{ ObjectMeta: metav1.ObjectMeta{ Generation: 1, @@ -95,10 +144,41 @@ func TestHandleClusterApprovalRequestUpdate(t *testing.T) { }, }, }, - shouldEnqueue: true, - queuedName: "test", + shouldEnqueue: true, + queuedName: "test", + isClusterScoped: true, }, - "it should enqueue the targetUpdateRun if oldObj is not declined while newobj is approved": { + "namespaced: it should enqueue the targetUpdateRun if oldObj is not approved while newobj is approved": { + oldObj: &placementv1beta1.ApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 1, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: "test", + }, + }, + newObj: &placementv1beta1.ApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 1, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: "test", + }, + Status: placementv1beta1.ApprovalRequestStatus{ + Conditions: []metav1.Condition{ + { + Status: metav1.ConditionTrue, + Type: string(placementv1beta1.ApprovalRequestConditionApproved), + ObservedGeneration: 1, + }, + }, + }, + }, + shouldEnqueue: true, + queuedName: "test", + isClusterScoped: false, + }, + "cluster-scoped: it should enqueue the targetUpdateRun if oldObj is not declined while newobj is approved": { oldObj: &placementv1beta1.ClusterApprovalRequest{ ObjectMeta: metav1.ObjectMeta{ Generation: 1, @@ -133,10 +213,50 @@ func TestHandleClusterApprovalRequestUpdate(t *testing.T) { }, }, }, - shouldEnqueue: true, - queuedName: "test", + shouldEnqueue: true, + queuedName: "test", + isClusterScoped: true, }, - "it should enqueue the targetUpdateRun if oldObj is approved while newobj is not approved": { + "namespaced: it should enqueue the targetUpdateRun if oldObj is not declined while newobj is approved": { + oldObj: &placementv1beta1.ApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 1, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: "test", + }, + Status: placementv1beta1.ApprovalRequestStatus{ + Conditions: []metav1.Condition{ + { + Status: metav1.ConditionFalse, + Type: string(placementv1beta1.ApprovalRequestConditionApproved), + ObservedGeneration: 1, + }, + }, + }, + }, + newObj: &placementv1beta1.ApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 1, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: "test", + }, + Status: placementv1beta1.ApprovalRequestStatus{ + Conditions: []metav1.Condition{ + { + Status: metav1.ConditionTrue, + Type: string(placementv1beta1.ApprovalRequestConditionApproved), + ObservedGeneration: 1, + }, + }, + }, + }, + shouldEnqueue: true, + queuedName: "test", + isClusterScoped: false, + }, + "cluster-scoped: it should enqueue the targetUpdateRun if oldObj is approved while newobj is not approved": { oldObj: &placementv1beta1.ClusterApprovalRequest{ ObjectMeta: metav1.ObjectMeta{ Generation: 1, @@ -162,10 +282,41 @@ func TestHandleClusterApprovalRequestUpdate(t *testing.T) { TargetUpdateRun: "test", }, }, - shouldEnqueue: true, - queuedName: "test", + shouldEnqueue: true, + queuedName: "test", + isClusterScoped: true, }, - "it should enqueue the targetUpdateRun if oldObj is approved while newobj is declined": { + "namespaced: it should enqueue the targetUpdateRun if oldObj is approved while newobj is not approved": { + oldObj: &placementv1beta1.ApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 1, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: "test", + }, + Status: placementv1beta1.ApprovalRequestStatus{ + Conditions: []metav1.Condition{ + { + Status: metav1.ConditionTrue, + Type: string(placementv1beta1.ApprovalRequestConditionApproved), + ObservedGeneration: 1, + }, + }, + }, + }, + newObj: &placementv1beta1.ApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 1, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: "test", + }, + }, + shouldEnqueue: true, + queuedName: "test", + isClusterScoped: false, + }, + "cluster-scoped: it should enqueue the targetUpdateRun if oldObj is approved while newobj is declined": { oldObj: &placementv1beta1.ClusterApprovalRequest{ ObjectMeta: metav1.ObjectMeta{ Generation: 1, @@ -200,10 +351,50 @@ func TestHandleClusterApprovalRequestUpdate(t *testing.T) { }, }, }, - shouldEnqueue: true, - queuedName: "test", + shouldEnqueue: true, + queuedName: "test", + isClusterScoped: true, + }, + "namespaced: it should enqueue the targetUpdateRun if oldObj is approved while newobj is declined": { + oldObj: &placementv1beta1.ApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 1, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: "test", + }, + Status: placementv1beta1.ApprovalRequestStatus{ + Conditions: []metav1.Condition{ + { + Status: metav1.ConditionTrue, + Type: string(placementv1beta1.ApprovalRequestConditionApproved), + ObservedGeneration: 1, + }, + }, + }, + }, + newObj: &placementv1beta1.ApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 1, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: "test", + }, + Status: placementv1beta1.ApprovalRequestStatus{ + Conditions: []metav1.Condition{ + { + Status: metav1.ConditionFalse, + Type: string(placementv1beta1.ApprovalRequestConditionApproved), + ObservedGeneration: 1, + }, + }, + }, + }, + shouldEnqueue: true, + queuedName: "test", + isClusterScoped: false, }, - "it should not enqueue the targetUpdateRun if neither oldObj nor newobj is approved": { + "cluster-scoped: it should not enqueue the targetUpdateRun if neither oldObj nor newobj is approved": { oldObj: &placementv1beta1.ClusterApprovalRequest{ ObjectMeta: metav1.ObjectMeta{ Generation: 1, @@ -220,9 +411,30 @@ func TestHandleClusterApprovalRequestUpdate(t *testing.T) { TargetUpdateRun: "test", }, }, - shouldEnqueue: false, + shouldEnqueue: false, + isClusterScoped: true, }, - "it should not enqueue the targetUpdateRun if both oldObj and newobj are approved": { + "namespaced: it should not enqueue the targetUpdateRun if neither oldObj nor newobj is approved": { + oldObj: &placementv1beta1.ApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 1, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: "test", + }, + }, + newObj: &placementv1beta1.ApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 1, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: "test", + }, + }, + shouldEnqueue: false, + isClusterScoped: false, + }, + "cluster-scoped: it should not enqueue the targetUpdateRun if both oldObj and newobj are approved": { oldObj: &placementv1beta1.ClusterApprovalRequest{ ObjectMeta: metav1.ObjectMeta{ Generation: 1, @@ -257,9 +469,48 @@ func TestHandleClusterApprovalRequestUpdate(t *testing.T) { }, }, }, - shouldEnqueue: false, + shouldEnqueue: false, + isClusterScoped: true, + }, + "namespaced: it should not enqueue the targetUpdateRun if both oldObj and newobj are approved": { + oldObj: &placementv1beta1.ApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 1, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: "test", + }, + Status: placementv1beta1.ApprovalRequestStatus{ + Conditions: []metav1.Condition{ + { + Status: metav1.ConditionTrue, + Type: string(placementv1beta1.ApprovalRequestConditionApproved), + ObservedGeneration: 1, + }, + }, + }, + }, + newObj: &placementv1beta1.ApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 1, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: "test", + }, + Status: placementv1beta1.ApprovalRequestStatus{ + Conditions: []metav1.Condition{ + { + Status: metav1.ConditionTrue, + Type: string(placementv1beta1.ApprovalRequestConditionApproved), + ObservedGeneration: 1, + }, + }, + }, + }, + shouldEnqueue: false, + isClusterScoped: false, }, - "it should not enqueue the targetUpdateRun if both oldObj and newobj are declined": { + "cluster-scoped: it should not enqueue the targetUpdateRun if both oldObj and newobj are declined": { oldObj: &placementv1beta1.ClusterApprovalRequest{ ObjectMeta: metav1.ObjectMeta{ Generation: 1, @@ -294,14 +545,53 @@ func TestHandleClusterApprovalRequestUpdate(t *testing.T) { }, }, }, - shouldEnqueue: false, + shouldEnqueue: false, + isClusterScoped: true, + }, + "namespaced: it should not enqueue the targetUpdateRun if both oldObj and newobj are declined": { + oldObj: &placementv1beta1.ApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 1, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: "test", + }, + Status: placementv1beta1.ApprovalRequestStatus{ + Conditions: []metav1.Condition{ + { + Status: metav1.ConditionFalse, + Type: string(placementv1beta1.ApprovalRequestConditionApproved), + ObservedGeneration: 1, + }, + }, + }, + }, + newObj: &placementv1beta1.ApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 1, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: "test", + }, + Status: placementv1beta1.ApprovalRequestStatus{ + Conditions: []metav1.Condition{ + { + Status: metav1.ConditionFalse, + Type: string(placementv1beta1.ApprovalRequestConditionApproved), + ObservedGeneration: 1, + }, + }, + }, + }, + shouldEnqueue: false, + isClusterScoped: false, }, } for name, tt := range tests { t.Run(name, func(t *testing.T) { queue := &controllertest.Queue{TypedInterface: workqueue.NewTypedRateLimitingQueue[reconcile.Request]( workqueue.DefaultTypedItemBasedRateLimiter[reconcile.Request]())} - handleClusterApprovalRequestUpdate(tt.oldObj, tt.newObj, queue) + handleApprovalRequestUpdate(tt.oldObj, tt.newObj, queue, tt.isClusterScoped) if got := queue.Len() != 0; got != tt.shouldEnqueue { t.Fatalf("handleClusterApprovalRequest() shouldEnqueue got %t, want %t", got, tt.shouldEnqueue) } @@ -315,25 +605,42 @@ func TestHandleClusterApprovalRequestUpdate(t *testing.T) { } } -func TestHandleClusterApprovalRequestDelete(t *testing.T) { +func TestHandleApprovalRequestDelete(t *testing.T) { tests := map[string]struct { - obj client.Object - shouldEnqueue bool - queuedName string + obj client.Object + shouldEnqueue bool + queuedName string + isClusterScoped bool }{ - "it should not enqueue anything if the obj is not a ClusterApprovalRequest": { - obj: &placementv1beta1.ClusterStagedUpdateRun{}, - shouldEnqueue: false, + "cluster-scoped: it should not enqueue anything if the obj is not a ClusterApprovalRequest": { + obj: &placementv1beta1.ClusterStagedUpdateRun{}, + shouldEnqueue: false, + isClusterScoped: true, }, - "it should not enqueue anything if targetUpdateRun in spec is empty": { + "namespaced: it should not enqueue anything if the obj is not an ApprovalRequest": { + obj: &placementv1beta1.StagedUpdateRun{}, + shouldEnqueue: false, + isClusterScoped: false, + }, + "cluster-scoped: it should not enqueue anything if targetUpdateRun in spec is empty": { obj: &placementv1beta1.ClusterApprovalRequest{ Spec: placementv1beta1.ApprovalRequestSpec{ TargetUpdateRun: "", }, }, - shouldEnqueue: false, + shouldEnqueue: false, + isClusterScoped: true, }, - "it should enqueue the targetUpdateRun, if ClusterApprovalRequest has neither Approved/ApprovalAccepted status set": { + "namespaced: it should not enqueue anything if targetUpdateRun in spec is empty": { + obj: &placementv1beta1.ApprovalRequest{ + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: "", + }, + }, + shouldEnqueue: false, + isClusterScoped: false, + }, + "cluster-scoped: it should enqueue the targetUpdateRun, if ClusterApprovalRequest has neither Approved/ApprovalAccepted status set": { obj: &placementv1beta1.ClusterApprovalRequest{ ObjectMeta: metav1.ObjectMeta{ Generation: 1, @@ -342,10 +649,24 @@ func TestHandleClusterApprovalRequestDelete(t *testing.T) { TargetUpdateRun: "test-update-run", }, }, - shouldEnqueue: true, - queuedName: "test-update-run", + shouldEnqueue: true, + queuedName: "test-update-run", + isClusterScoped: true, + }, + "namespaced: it should enqueue the targetUpdateRun, if ApprovalRequest has neither Approved/ApprovalAccepted status set": { + obj: &placementv1beta1.ApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 1, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: "test-update-run", + }, + }, + shouldEnqueue: true, + queuedName: "test-update-run", + isClusterScoped: false, }, - "it should enqueue the targetUpdateRun, if ClusterApprovalRequest has only Approved status set to true": { + "cluster-scoped: it should enqueue the targetUpdateRun, if ClusterApprovalRequest has only Approved status set to true": { obj: &placementv1beta1.ClusterApprovalRequest{ ObjectMeta: metav1.ObjectMeta{ Generation: 1, @@ -363,10 +684,33 @@ func TestHandleClusterApprovalRequestDelete(t *testing.T) { }, }, }, - shouldEnqueue: true, - queuedName: "test-update-run", + shouldEnqueue: true, + queuedName: "test-update-run", + isClusterScoped: true, }, - "it should enqueue the targetUpdateRun, if ClusterApprovalRequest has only Approved status set to false": { + "namespaced: it should enqueue the targetUpdateRun, if ApprovalRequest has only Approved status set to true": { + obj: &placementv1beta1.ApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 1, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: "test-update-run", + }, + Status: placementv1beta1.ApprovalRequestStatus{ + Conditions: []metav1.Condition{ + { + Status: metav1.ConditionTrue, + Type: string(placementv1beta1.ApprovalRequestConditionApproved), + ObservedGeneration: 1, + }, + }, + }, + }, + shouldEnqueue: true, + queuedName: "test-update-run", + isClusterScoped: false, + }, + "cluster-scoped: it should enqueue the targetUpdateRun, if ClusterApprovalRequest has only Approved status set to false": { obj: &placementv1beta1.ClusterApprovalRequest{ ObjectMeta: metav1.ObjectMeta{ Generation: 1, @@ -384,10 +728,33 @@ func TestHandleClusterApprovalRequestDelete(t *testing.T) { }, }, }, - shouldEnqueue: true, - queuedName: "test-update-run", + shouldEnqueue: true, + queuedName: "test-update-run", + isClusterScoped: true, }, - "it should not enqueue updateRun, if ClusterApprovalRequest has Approved set to false, ApprovalAccepted status set to true": { + "namespaced: it should enqueue the targetUpdateRun, if ApprovalRequest has only Approved status set to false": { + obj: &placementv1beta1.ApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 1, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: "test-update-run", + }, + Status: placementv1beta1.ApprovalRequestStatus{ + Conditions: []metav1.Condition{ + { + Status: metav1.ConditionTrue, + Type: string(placementv1beta1.ApprovalRequestConditionApproved), + ObservedGeneration: 1, + }, + }, + }, + }, + shouldEnqueue: true, + queuedName: "test-update-run", + isClusterScoped: false, + }, + "cluster-scoped: it should not enqueue updateRun, if ClusterApprovalRequest has Approved set to false, ApprovalAccepted status set to true": { obj: &placementv1beta1.ClusterApprovalRequest{ ObjectMeta: metav1.ObjectMeta{ Generation: 1, @@ -410,9 +777,36 @@ func TestHandleClusterApprovalRequestDelete(t *testing.T) { }, }, }, - shouldEnqueue: false, + shouldEnqueue: false, + isClusterScoped: true, + }, + "namespaced: it should not enqueue updateRun, if ApprovalRequest has Approved set to false, ApprovalAccepted status set to true": { + obj: &placementv1beta1.ApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 1, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: "test-update-run", + }, + Status: placementv1beta1.ApprovalRequestStatus{ + Conditions: []metav1.Condition{ + { + Status: metav1.ConditionFalse, + Type: string(placementv1beta1.ApprovalRequestConditionApproved), + ObservedGeneration: 1, + }, + { + Status: metav1.ConditionTrue, + Type: string(placementv1beta1.ApprovalRequestConditionApprovalAccepted), + ObservedGeneration: 1, + }, + }, + }, + }, + shouldEnqueue: false, + isClusterScoped: false, }, - "it should not enqueue updateRun, if ClusterApprovalRequest has Approved, ApprovalAccepted status set to true": { + "cluster-scoped: it should not enqueue updateRun, if ClusterApprovalRequest has Approved, ApprovalAccepted status set to true": { obj: &placementv1beta1.ClusterApprovalRequest{ ObjectMeta: metav1.ObjectMeta{ Generation: 1, @@ -435,7 +829,34 @@ func TestHandleClusterApprovalRequestDelete(t *testing.T) { }, }, }, - shouldEnqueue: false, + shouldEnqueue: false, + isClusterScoped: true, + }, + "namespaced: it should not enqueue updateRun, if ApprovalRequest has Approved, ApprovalAccepted status set to true": { + obj: &placementv1beta1.ApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Generation: 1, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: "test-update-run", + }, + Status: placementv1beta1.ApprovalRequestStatus{ + Conditions: []metav1.Condition{ + { + Status: metav1.ConditionTrue, + Type: string(placementv1beta1.ApprovalRequestConditionApproved), + ObservedGeneration: 1, + }, + { + Status: metav1.ConditionTrue, + Type: string(placementv1beta1.ApprovalRequestConditionApprovalAccepted), + ObservedGeneration: 1, + }, + }, + }, + }, + shouldEnqueue: false, + isClusterScoped: false, }, } @@ -443,7 +864,7 @@ func TestHandleClusterApprovalRequestDelete(t *testing.T) { t.Run(name, func(t *testing.T) { queue := &controllertest.Queue{TypedInterface: workqueue.NewTypedRateLimitingQueue[reconcile.Request]( workqueue.DefaultTypedItemBasedRateLimiter[reconcile.Request]())} - handleClusterApprovalRequestDelete(tt.obj, queue) + handleApprovalRequestDelete(tt.obj, queue, tt.isClusterScoped) if got := queue.Len() != 0; got != tt.shouldEnqueue { t.Fatalf("handleClusterApprovalRequestDelete() shouldEnqueue got %t, want %t", got, tt.shouldEnqueue) } @@ -465,36 +886,36 @@ func TestRemoveWaitTimeFromUpdateRunStatus(t *testing.T) { }{ "should handle empty stages": { inputUpdateRun: &placementv1beta1.ClusterStagedUpdateRun{ - Status: placementv1beta1.StagedUpdateRunStatus{ - StagedUpdateStrategySnapshot: &placementv1beta1.StagedUpdateStrategySpec{ + Status: placementv1beta1.UpdateRunStatus{ + UpdateStrategySnapshot: &placementv1beta1.UpdateStrategySpec{ Stages: []placementv1beta1.StageConfig{}, }, }, }, wantUpdateRun: &placementv1beta1.ClusterStagedUpdateRun{ - Status: placementv1beta1.StagedUpdateRunStatus{ - StagedUpdateStrategySnapshot: &placementv1beta1.StagedUpdateStrategySpec{ + Status: placementv1beta1.UpdateRunStatus{ + UpdateStrategySnapshot: &placementv1beta1.UpdateStrategySpec{ Stages: []placementv1beta1.StageConfig{}, }, }, }, }, - "should handle nil StagedUpdateStrategySnapshot": { + "should handle nil UpdateStrategySnapshot": { inputUpdateRun: &placementv1beta1.ClusterStagedUpdateRun{ - Status: placementv1beta1.StagedUpdateRunStatus{ - StagedUpdateStrategySnapshot: nil, + Status: placementv1beta1.UpdateRunStatus{ + UpdateStrategySnapshot: nil, }, }, wantUpdateRun: &placementv1beta1.ClusterStagedUpdateRun{ - Status: placementv1beta1.StagedUpdateRunStatus{ - StagedUpdateStrategySnapshot: nil, + Status: placementv1beta1.UpdateRunStatus{ + UpdateStrategySnapshot: nil, }, }, }, "should remove waitTime from Approval tasks only": { inputUpdateRun: &placementv1beta1.ClusterStagedUpdateRun{ - Status: placementv1beta1.StagedUpdateRunStatus{ - StagedUpdateStrategySnapshot: &placementv1beta1.StagedUpdateStrategySpec{ + Status: placementv1beta1.UpdateRunStatus{ + UpdateStrategySnapshot: &placementv1beta1.UpdateStrategySpec{ Stages: []placementv1beta1.StageConfig{ { AfterStageTasks: []placementv1beta1.AfterStageTask{ @@ -513,8 +934,8 @@ func TestRemoveWaitTimeFromUpdateRunStatus(t *testing.T) { }, }, wantUpdateRun: &placementv1beta1.ClusterStagedUpdateRun{ - Status: placementv1beta1.StagedUpdateRunStatus{ - StagedUpdateStrategySnapshot: &placementv1beta1.StagedUpdateStrategySpec{ + Status: placementv1beta1.UpdateRunStatus{ + UpdateStrategySnapshot: &placementv1beta1.UpdateStrategySpec{ Stages: []placementv1beta1.StageConfig{ { AfterStageTasks: []placementv1beta1.AfterStageTask{ @@ -534,8 +955,8 @@ func TestRemoveWaitTimeFromUpdateRunStatus(t *testing.T) { }, "should handle multiple stages": { inputUpdateRun: &placementv1beta1.ClusterStagedUpdateRun{ - Status: placementv1beta1.StagedUpdateRunStatus{ - StagedUpdateStrategySnapshot: &placementv1beta1.StagedUpdateStrategySpec{ + Status: placementv1beta1.UpdateRunStatus{ + UpdateStrategySnapshot: &placementv1beta1.UpdateStrategySpec{ Stages: []placementv1beta1.StageConfig{ { AfterStageTasks: []placementv1beta1.AfterStageTask{ @@ -562,8 +983,8 @@ func TestRemoveWaitTimeFromUpdateRunStatus(t *testing.T) { }, }, wantUpdateRun: &placementv1beta1.ClusterStagedUpdateRun{ - Status: placementv1beta1.StagedUpdateRunStatus{ - StagedUpdateStrategySnapshot: &placementv1beta1.StagedUpdateStrategySpec{ + Status: placementv1beta1.UpdateRunStatus{ + UpdateStrategySnapshot: &placementv1beta1.UpdateStrategySpec{ Stages: []placementv1beta1.StageConfig{ { AfterStageTasks: []placementv1beta1.AfterStageTask{ diff --git a/pkg/controllers/updaterun/execution.go b/pkg/controllers/updaterun/execution.go index e2aa96d56..af65339a4 100644 --- a/pkg/controllers/updaterun/execution.go +++ b/pkg/controllers/updaterun/execution.go @@ -27,6 +27,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" @@ -51,24 +52,25 @@ var ( ) // execute executes the update run by updating the clusters in the updating stage specified by updatingStageIndex. -// It returns a boolean indicating if the clusterStageUpdateRun execution is completed, +// It returns a boolean indicating if the updateRun execution is completed, // the time to wait before rechecking the cluster update status, and any error encountered. func (r *Reconciler) execute( ctx context.Context, - updateRun *placementv1beta1.ClusterStagedUpdateRun, + updateRun placementv1beta1.UpdateRunObj, updatingStageIndex int, - toBeUpdatedBindings, toBeDeletedBindings []*placementv1beta1.ClusterResourceBinding, + toBeUpdatedBindings, toBeDeletedBindings []placementv1beta1.BindingObj, ) (bool, time.Duration, error) { // Mark updateRun as progressing if it's not already marked as waiting or stuck. // This avoids triggering an unnecessary in-memory transition from stuck (waiting) -> progressing -> stuck (waiting), // which would update the lastTransitionTime even though the status hasn't effectively changed. markUpdateRunProgressingIfNotWaitingOrStuck(updateRun) - if updatingStageIndex < len(updateRun.Status.StagesStatus) { - updatingStage := &updateRun.Status.StagesStatus[updatingStageIndex] + updateRunStatus := updateRun.GetUpdateRunStatus() + if updatingStageIndex < len(updateRunStatus.StagesStatus) { + updatingStage := &updateRunStatus.StagesStatus[updatingStageIndex] waitTime, execErr := r.executeUpdatingStage(ctx, updateRun, updatingStageIndex, toBeUpdatedBindings) if errors.Is(execErr, errStagedUpdatedAborted) { - markStageUpdatingFailed(updatingStage, updateRun.Generation, execErr.Error()) + markStageUpdatingFailed(updatingStage, updateRun.GetGeneration(), execErr.Error()) return true, waitTime, execErr } // The execution has not finished yet. @@ -77,28 +79,31 @@ func (r *Reconciler) execute( // All the stages have finished, now start the delete stage. finished, execErr := r.executeDeleteStage(ctx, updateRun, toBeDeletedBindings) if errors.Is(execErr, errStagedUpdatedAborted) { - markStageUpdatingFailed(updateRun.Status.DeletionStageStatus, updateRun.Generation, execErr.Error()) + markStageUpdatingFailed(updateRunStatus.DeletionStageStatus, updateRun.GetGeneration(), execErr.Error()) return true, 0, execErr } return finished, clusterUpdatingWaitTime, execErr } -// executeUpdatingStage executes a single updating stage by updating the clusterResourceBindings. +// executeUpdatingStage executes a single updating stage by updating the bindings. func (r *Reconciler) executeUpdatingStage( ctx context.Context, - updateRun *placementv1beta1.ClusterStagedUpdateRun, + updateRun placementv1beta1.UpdateRunObj, updatingStageIndex int, - toBeUpdatedBindings []*placementv1beta1.ClusterResourceBinding, + toBeUpdatedBindings []placementv1beta1.BindingObj, ) (time.Duration, error) { - updatingStageStatus := &updateRun.Status.StagesStatus[updatingStageIndex] + updateRunStatus := updateRun.GetUpdateRunStatus() + updateRunSpec := updateRun.GetUpdateRunSpec() + updatingStageStatus := &updateRunStatus.StagesStatus[updatingStageIndex] // The parse error is ignored because the initialization should have caught it. - resourceIndex, _ := strconv.Atoi(updateRun.Spec.ResourceSnapshotIndex) - resourceSnapshotName := fmt.Sprintf(placementv1beta1.ResourceSnapshotNameFmt, updateRun.Spec.PlacementName, resourceIndex) + resourceIndex, _ := strconv.Atoi(updateRunSpec.ResourceSnapshotIndex) + resourceSnapshotName := fmt.Sprintf(placementv1beta1.ResourceSnapshotNameFmt, updateRunSpec.PlacementName, resourceIndex) updateRunRef := klog.KObj(updateRun) // Create the map of the toBeUpdatedBindings. - toBeUpdatedBindingsMap := make(map[string]*placementv1beta1.ClusterResourceBinding, len(toBeUpdatedBindings)) + toBeUpdatedBindingsMap := make(map[string]placementv1beta1.BindingObj, len(toBeUpdatedBindings)) for _, binding := range toBeUpdatedBindings { - toBeUpdatedBindingsMap[binding.Spec.TargetCluster] = binding + bindingSpec := binding.GetBindingSpec() + toBeUpdatedBindingsMap[bindingSpec.TargetCluster] = binding } finishedClusterCount := 0 @@ -107,51 +112,53 @@ func (r *Reconciler) executeUpdatingStage( clusterStatus := &updatingStageStatus.Clusters[i] clusterStartedCond := meta.FindStatusCondition(clusterStatus.Conditions, string(placementv1beta1.ClusterUpdatingConditionStarted)) clusterUpdateSucceededCond := meta.FindStatusCondition(clusterStatus.Conditions, string(placementv1beta1.ClusterUpdatingConditionSucceeded)) - if condition.IsConditionStatusFalse(clusterUpdateSucceededCond, updateRun.Generation) { + if condition.IsConditionStatusFalse(clusterUpdateSucceededCond, updateRun.GetGeneration()) { // The cluster is marked as failed to update. failedErr := fmt.Errorf("the cluster `%s` in the stage %s has failed", clusterStatus.ClusterName, updatingStageStatus.StageName) - klog.ErrorS(failedErr, "The cluster has failed to be updated", "clusterStagedUpdateRun", updateRunRef) + klog.ErrorS(failedErr, "The cluster has failed to be updated", "updateRun", updateRunRef) return 0, fmt.Errorf("%w: %s", errStagedUpdatedAborted, failedErr.Error()) } - if condition.IsConditionStatusTrue(clusterUpdateSucceededCond, updateRun.Generation) { + if condition.IsConditionStatusTrue(clusterUpdateSucceededCond, updateRun.GetGeneration()) { // The cluster has been updated successfully. finishedClusterCount++ continue } // The cluster is either updating or not started yet. binding := toBeUpdatedBindingsMap[clusterStatus.ClusterName] - if !condition.IsConditionStatusTrue(clusterStartedCond, updateRun.Generation) { + if !condition.IsConditionStatusTrue(clusterStartedCond, updateRun.GetGeneration()) { // The cluster has not started updating yet. if !isBindingSyncedWithClusterStatus(resourceSnapshotName, updateRun, binding, clusterStatus) { - klog.V(2).InfoS("Found the first cluster that needs to be updated", "cluster", clusterStatus.ClusterName, "stage", updatingStageStatus.StageName, "clusterStagedUpdateRun", updateRunRef) + klog.V(2).InfoS("Found the first cluster that needs to be updated", "cluster", clusterStatus.ClusterName, "stage", updatingStageStatus.StageName, "updateRun", updateRunRef) // The binding is not up-to-date with the cluster status. - binding.Spec.State = placementv1beta1.BindingStateBound - binding.Spec.ResourceSnapshotName = resourceSnapshotName - binding.Spec.ResourceOverrideSnapshots = clusterStatus.ResourceOverrideSnapshots - binding.Spec.ClusterResourceOverrideSnapshots = clusterStatus.ClusterResourceOverrideSnapshots - binding.Spec.ApplyStrategy = updateRun.Status.ApplyStrategy + bindingSpec := binding.GetBindingSpec() + bindingSpec.State = placementv1beta1.BindingStateBound + bindingSpec.ResourceSnapshotName = resourceSnapshotName + bindingSpec.ResourceOverrideSnapshots = clusterStatus.ResourceOverrideSnapshots + bindingSpec.ClusterResourceOverrideSnapshots = clusterStatus.ClusterResourceOverrideSnapshots + bindingSpec.ApplyStrategy = updateRunStatus.ApplyStrategy if err := r.Client.Update(ctx, binding); err != nil { - klog.ErrorS(err, "Failed to update binding to be bound with the matching spec of the updateRun", "binding", klog.KObj(binding), "clusterStagedUpdateRun", updateRunRef) + klog.ErrorS(err, "Failed to update binding to be bound with the matching spec of the updateRun", "binding", klog.KObj(binding), "updateRun", updateRunRef) return 0, controller.NewUpdateIgnoreConflictError(err) } - klog.V(2).InfoS("Updated the status of a binding to bound", "binding", klog.KObj(binding), "cluster", clusterStatus.ClusterName, "stage", updatingStageStatus.StageName, "clusterStagedUpdateRun", updateRunRef) + klog.V(2).InfoS("Updated the status of a binding to bound", "binding", klog.KObj(binding), "cluster", clusterStatus.ClusterName, "stage", updatingStageStatus.StageName, "updateRun", updateRunRef) if err := r.updateBindingRolloutStarted(ctx, binding, updateRun); err != nil { return 0, err } } else { - klog.V(2).InfoS("Found the first binding that is updating but the cluster status has not been updated", "cluster", clusterStatus.ClusterName, "stage", updatingStageStatus.StageName, "clusterStagedUpdateRun", updateRunRef) - if binding.Spec.State != placementv1beta1.BindingStateBound { - binding.Spec.State = placementv1beta1.BindingStateBound + klog.V(2).InfoS("Found the first binding that is updating but the cluster status has not been updated", "cluster", clusterStatus.ClusterName, "stage", updatingStageStatus.StageName, "updateRun", updateRunRef) + bindingSpec := binding.GetBindingSpec() + if bindingSpec.State != placementv1beta1.BindingStateBound { + bindingSpec.State = placementv1beta1.BindingStateBound if err := r.Client.Update(ctx, binding); err != nil { - klog.ErrorS(err, "Failed to update a binding to be bound", "binding", klog.KObj(binding), "cluster", clusterStatus.ClusterName, "stage", updatingStageStatus.StageName, "clusterStagedUpdateRun", updateRunRef) + klog.ErrorS(err, "Failed to update a binding to be bound", "binding", klog.KObj(binding), "cluster", clusterStatus.ClusterName, "stage", updatingStageStatus.StageName, "updateRun", updateRunRef) return 0, controller.NewUpdateIgnoreConflictError(err) } - klog.V(2).InfoS("Updated the status of a binding to bound", "binding", klog.KObj(binding), "cluster", clusterStatus.ClusterName, "stage", updatingStageStatus.StageName, "clusterStagedUpdateRun", updateRunRef) + klog.V(2).InfoS("Updated the status of a binding to bound", "binding", klog.KObj(binding), "cluster", clusterStatus.ClusterName, "stage", updatingStageStatus.StageName, "updateRun", updateRunRef) if err := r.updateBindingRolloutStarted(ctx, binding, updateRun); err != nil { return 0, err } - } else if !condition.IsConditionStatusTrue(meta.FindStatusCondition(binding.Status.Conditions, string(placementv1beta1.ResourceBindingRolloutStarted)), binding.Generation) { - klog.V(2).InfoS("The binding is bound and up-to-date but the generation is updated by the scheduler, update rolloutStarted status again", "binding", klog.KObj(binding), "cluster", clusterStatus.ClusterName, "stage", updatingStageStatus.StageName, "clusterStagedUpdateRun", updateRunRef) + } else if !condition.IsConditionStatusTrue(meta.FindStatusCondition(binding.GetBindingStatus().Conditions, string(placementv1beta1.ResourceBindingRolloutStarted)), binding.GetGeneration()) { + klog.V(2).InfoS("The binding is bound and up-to-date but the generation is updated by the scheduler, update rolloutStarted status again", "binding", klog.KObj(binding), "cluster", clusterStatus.ClusterName, "stage", updatingStageStatus.StageName, "updateRun", updateRunRef) if err := r.updateBindingRolloutStarted(ctx, binding, updateRun); err != nil { return 0, err } @@ -161,9 +168,9 @@ func (r *Reconciler) executeUpdatingStage( } } } - markClusterUpdatingStarted(clusterStatus, updateRun.Generation) + markClusterUpdatingStarted(clusterStatus, updateRun.GetGeneration()) if finishedClusterCount == 0 { - markStageUpdatingStarted(updatingStageStatus, updateRun.Generation) + markStageUpdatingStarted(updatingStageStatus, updateRun.GetGeneration()) } // No need to continue as we only support one cluster updating at a time for now. return clusterUpdatingWaitTime, nil @@ -171,17 +178,18 @@ func (r *Reconciler) executeUpdatingStage( // Now the cluster has to be updating, the binding should point to the right resource snapshot and the binding should be bound. inSync := isBindingSyncedWithClusterStatus(resourceSnapshotName, updateRun, binding, clusterStatus) - rolloutStarted := condition.IsConditionStatusTrue(meta.FindStatusCondition(binding.Status.Conditions, string(placementv1beta1.ResourceBindingRolloutStarted)), binding.Generation) - if !inSync || !rolloutStarted || binding.Spec.State != placementv1beta1.BindingStateBound { + rolloutStarted := condition.IsConditionStatusTrue(meta.FindStatusCondition(binding.GetBindingStatus().Conditions, string(placementv1beta1.ResourceBindingRolloutStarted)), binding.GetGeneration()) + bindingSpec := binding.GetBindingSpec() + if !inSync || !rolloutStarted || bindingSpec.State != placementv1beta1.BindingStateBound { // This issue mostly happens when there are concurrent updateRuns referencing the same clusterResourcePlacement but releasing different versions. // After the 1st updateRun updates the binding, and before the controller re-checks the binding status, the 2nd updateRun updates the same binding, and thus the 1st updateRun is preempted and observes the binding not matching the desired state. - preemptedErr := controller.NewUserError(fmt.Errorf("the clusterResourceBinding of the updating cluster `%s` in the stage `%s` is not up-to-date with the desired status, "+ + preemptedErr := controller.NewUserError(fmt.Errorf("the binding of the updating cluster `%s` in the stage `%s` is not up-to-date with the desired status, "+ "please check the status of binding `%s` and see if there is a concurrent updateRun referencing the same clusterResourcePlacement and updating the same cluster", clusterStatus.ClusterName, updatingStageStatus.StageName, klog.KObj(binding))) klog.ErrorS(preemptedErr, "The binding has been changed during updating", - "bindingSpecInSync", inSync, "bindingState", binding.Spec.State, - "bindingRolloutStarted", rolloutStarted, "binding", klog.KObj(binding), "clusterStagedUpdateRun", updateRunRef) - markClusterUpdatingFailed(clusterStatus, updateRun.Generation, preemptedErr.Error()) + "bindingSpecInSync", inSync, "bindingState", bindingSpec.State, + "bindingRolloutStarted", rolloutStarted, "binding", klog.KObj(binding), "updateRun", updateRunRef) + markClusterUpdatingFailed(clusterStatus, updateRun.GetGeneration(), preemptedErr.Error()) return 0, fmt.Errorf("%w: %s", errStagedUpdatedAborted, preemptedErr.Error()) } @@ -194,7 +202,7 @@ func (r *Reconciler) executeUpdatingStage( // If cluster update has been running for more than "updateRunStuckThreshold", mark the update run as stuck. timeElapsed := time.Since(clusterStartedCond.LastTransitionTime.Time) if timeElapsed > updateRunStuckThreshold { - klog.V(2).InfoS("Time waiting for cluster update to finish passes threshold, mark the update run as stuck", "time elapsed", timeElapsed, "threshold", updateRunStuckThreshold, "cluster", clusterStatus.ClusterName, "stage", updatingStageStatus.StageName, "clusterStagedUpdateRun", updateRunRef) + klog.V(2).InfoS("Time waiting for cluster update to finish passes threshold, mark the update run as stuck", "time elapsed", timeElapsed, "threshold", updateRunStuckThreshold, "cluster", clusterStatus.ClusterName, "stage", updatingStageStatus.StageName, "updateRun", updateRunRef) markUpdateRunStuck(updateRun, updatingStageStatus.StageName, clusterStatus.ClusterName) } } @@ -205,8 +213,8 @@ func (r *Reconciler) executeUpdatingStage( if finishedClusterCount == len(updatingStageStatus.Clusters) { // All the clusters in the stage have been updated. markUpdateRunWaiting(updateRun, updatingStageStatus.StageName) - markStageUpdatingWaiting(updatingStageStatus, updateRun.Generation) - klog.V(2).InfoS("The stage has finished all cluster updating", "stage", updatingStageStatus.StageName, "clusterStagedUpdateRun", updateRunRef) + markStageUpdatingWaiting(updatingStageStatus, updateRun.GetGeneration()) + klog.V(2).InfoS("The stage has finished all cluster updating", "stage", updatingStageStatus.StageName, "updateRun", updateRunRef) // Check if the after stage tasks are ready. approved, waitTime, err := r.checkAfterStageTasksStatus(ctx, updatingStageIndex, updateRun) if err != nil { @@ -214,7 +222,7 @@ func (r *Reconciler) executeUpdatingStage( } if approved { markUpdateRunProgressing(updateRun) - markStageUpdatingSucceeded(updatingStageStatus, updateRun.Generation) + markStageUpdatingSucceeded(updatingStageStatus, updateRun.GetGeneration()) // No need to wait to get to the next stage. return 0, nil } @@ -227,65 +235,67 @@ func (r *Reconciler) executeUpdatingStage( return clusterUpdatingWaitTime, nil } -// executeDeleteStage executes the delete stage by deleting the clusterResourceBindings. +// executeDeleteStage executes the delete stage by deleting the bindings. func (r *Reconciler) executeDeleteStage( ctx context.Context, - updateRun *placementv1beta1.ClusterStagedUpdateRun, - toBeDeletedBindings []*placementv1beta1.ClusterResourceBinding, + updateRun placementv1beta1.UpdateRunObj, + toBeDeletedBindings []placementv1beta1.BindingObj, ) (bool, error) { updateRunRef := klog.KObj(updateRun) - existingDeleteStageStatus := updateRun.Status.DeletionStageStatus + updateRunStatus := updateRun.GetUpdateRunStatus() + existingDeleteStageStatus := updateRunStatus.DeletionStageStatus existingDeleteStageClusterMap := make(map[string]*placementv1beta1.ClusterUpdatingStatus, len(existingDeleteStageStatus.Clusters)) for i := range existingDeleteStageStatus.Clusters { existingDeleteStageClusterMap[existingDeleteStageStatus.Clusters[i].ClusterName] = &existingDeleteStageStatus.Clusters[i] } // Mark the delete stage as started in case it's not. - markStageUpdatingStarted(updateRun.Status.DeletionStageStatus, updateRun.Generation) + markStageUpdatingStarted(updateRunStatus.DeletionStageStatus, updateRun.GetGeneration()) for _, binding := range toBeDeletedBindings { - curCluster, exist := existingDeleteStageClusterMap[binding.Spec.TargetCluster] + bindingSpec := binding.GetBindingSpec() + curCluster, exist := existingDeleteStageClusterMap[bindingSpec.TargetCluster] if !exist { // This is unexpected because we already checked in validation. - missingErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the to be deleted cluster `%s` is not in the deleting stage during execution", binding.Spec.TargetCluster)) - klog.ErrorS(missingErr, "The cluster in the deleting stage does not include all the to be deleted binding", "clusterStagedUpdateRun", updateRunRef) + missingErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the to be deleted cluster `%s` is not in the deleting stage during execution", bindingSpec.TargetCluster)) + klog.ErrorS(missingErr, "The cluster in the deleting stage does not include all the to be deleted binding", "updateRun", updateRunRef) return false, fmt.Errorf("%w: %s", errStagedUpdatedAborted, missingErr.Error()) } // In validation, we already check the binding must exist in the status. - delete(existingDeleteStageClusterMap, binding.Spec.TargetCluster) + delete(existingDeleteStageClusterMap, bindingSpec.TargetCluster) // Make sure the cluster is not marked as deleted as the binding is still there. - if condition.IsConditionStatusTrue(meta.FindStatusCondition(curCluster.Conditions, string(placementv1beta1.ClusterUpdatingConditionSucceeded)), updateRun.Generation) { - unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the deleted cluster `%s` in the deleting stage still has a clusterResourceBinding", binding.Spec.TargetCluster)) - klog.ErrorS(unexpectedErr, "The cluster in the deleting stage is not removed yet but marked as deleted", "cluster", curCluster.ClusterName, "clusterStagedUpdateRun", updateRunRef) + if condition.IsConditionStatusTrue(meta.FindStatusCondition(curCluster.Conditions, string(placementv1beta1.ClusterUpdatingConditionSucceeded)), updateRun.GetGeneration()) { + unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the deleted cluster `%s` in the deleting stage still has a binding", bindingSpec.TargetCluster)) + klog.ErrorS(unexpectedErr, "The cluster in the deleting stage is not removed yet but marked as deleted", "cluster", curCluster.ClusterName, "updateRun", updateRunRef) return false, fmt.Errorf("%w: %s", errStagedUpdatedAborted, unexpectedErr.Error()) } - if condition.IsConditionStatusTrue(meta.FindStatusCondition(curCluster.Conditions, string(placementv1beta1.ClusterUpdatingConditionStarted)), updateRun.Generation) { + if condition.IsConditionStatusTrue(meta.FindStatusCondition(curCluster.Conditions, string(placementv1beta1.ClusterUpdatingConditionStarted)), updateRun.GetGeneration()) { // The cluster status is marked as being deleted. - if binding.DeletionTimestamp.IsZero() { + if binding.GetDeletionTimestamp().IsZero() { // The cluster is marked as deleting but the binding is not deleting. unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the cluster `%s` in the deleting stage is marked as deleting but its corresponding binding is not deleting", curCluster.ClusterName)) - klog.ErrorS(unexpectedErr, "The binding should be deleting before we mark a cluster deleting", "clusterStatus", curCluster, "clusterStagedUpdateRun", updateRunRef) + klog.ErrorS(unexpectedErr, "The binding should be deleting before we mark a cluster deleting", "clusterStatus", curCluster, "updateRun", updateRunRef) return false, fmt.Errorf("%w: %s", errStagedUpdatedAborted, unexpectedErr.Error()) } continue } // The cluster status is not deleting yet if err := r.Client.Delete(ctx, binding); err != nil { - klog.ErrorS(err, "Failed to delete a binding in the update run", "binding", klog.KObj(binding), "cluster", curCluster.ClusterName, "clusterStagedUpdateRun", updateRunRef) + klog.ErrorS(err, "Failed to delete a binding in the update run", "binding", klog.KObj(binding), "cluster", curCluster.ClusterName, "updateRun", updateRunRef) return false, controller.NewAPIServerError(false, err) } - klog.V(2).InfoS("Deleted a binding pointing to a to be deleted cluster", "binding", klog.KObj(binding), "cluster", curCluster.ClusterName, "clusterStagedUpdateRun", updateRunRef) - markClusterUpdatingStarted(curCluster, updateRun.Generation) + klog.V(2).InfoS("Deleted a binding pointing to a to be deleted cluster", "binding", klog.KObj(binding), "cluster", curCluster.ClusterName, "updateRun", updateRunRef) + markClusterUpdatingStarted(curCluster, updateRun.GetGeneration()) } // The rest of the clusters in the stage are not in the toBeDeletedBindings so it should be marked as delete succeeded. for _, clusterStatus := range existingDeleteStageClusterMap { // Make sure the cluster is marked as deleted. - if !condition.IsConditionStatusTrue(meta.FindStatusCondition(clusterStatus.Conditions, string(placementv1beta1.ClusterUpdatingConditionStarted)), updateRun.Generation) { - markClusterUpdatingStarted(clusterStatus, updateRun.Generation) + if !condition.IsConditionStatusTrue(meta.FindStatusCondition(clusterStatus.Conditions, string(placementv1beta1.ClusterUpdatingConditionStarted)), updateRun.GetGeneration()) { + markClusterUpdatingStarted(clusterStatus, updateRun.GetGeneration()) } - markClusterUpdatingSucceeded(clusterStatus, updateRun.Generation) + markClusterUpdatingSucceeded(clusterStatus, updateRun.GetGeneration()) } - klog.InfoS("The delete stage is progressing", "numberOfDeletingClusters", len(toBeDeletedBindings), "clusterStagedUpdateRun", updateRunRef) + klog.InfoS("The delete stage is progressing", "numberOfDeletingClusters", len(toBeDeletedBindings), "updateRun", updateRunRef) if len(toBeDeletedBindings) == 0 { - markStageUpdatingSucceeded(updateRun.Status.DeletionStageStatus, updateRun.Generation) + markStageUpdatingSucceeded(updateRunStatus.DeletionStageStatus, updateRun.GetGeneration()) } return len(toBeDeletedBindings) == 0, nil } @@ -293,12 +303,13 @@ func (r *Reconciler) executeDeleteStage( // checkAfterStageTasksStatus checks if the after stage tasks have finished. // It returns if the after stage tasks have finished or error if the after stage tasks failed. // It also returns the time to wait before rechecking the wait type of task. It turns -1 if the task is not a wait type. -func (r *Reconciler) checkAfterStageTasksStatus(ctx context.Context, updatingStageIndex int, updateRun *placementv1beta1.ClusterStagedUpdateRun) (bool, time.Duration, error) { +func (r *Reconciler) checkAfterStageTasksStatus(ctx context.Context, updatingStageIndex int, updateRun placementv1beta1.UpdateRunObj) (bool, time.Duration, error) { updateRunRef := klog.KObj(updateRun) - updatingStageStatus := &updateRun.Status.StagesStatus[updatingStageIndex] - updatingStage := &updateRun.Status.StagedUpdateStrategySnapshot.Stages[updatingStageIndex] + updateRunStatus := updateRun.GetUpdateRunStatus() + updatingStageStatus := &updateRunStatus.StagesStatus[updatingStageIndex] + updatingStage := &updateRunStatus.UpdateStrategySnapshot.Stages[updatingStageIndex] if updatingStage.AfterStageTasks == nil { - klog.V(2).InfoS("There is no after stage task for this stage", "stage", updatingStage.Name, "clusterStagedUpdateRun", updateRunRef) + klog.V(2).InfoS("There is no after stage task for this stage", "stage", updatingStage.Name, "updateRun", updateRunRef) return true, 0, nil } passed := true @@ -310,78 +321,67 @@ func (r *Reconciler) checkAfterStageTasksStatus(ctx context.Context, updatingSta // Check if the wait time has passed. waitTime := time.Until(waitStartTime.Add(task.WaitTime.Duration)) if waitTime > 0 { - klog.V(2).InfoS("The after stage task still need to wait", "waitStartTime", waitStartTime, "waitTime", task.WaitTime, "stage", updatingStage.Name, "clusterStagedUpdateRun", updateRunRef) + klog.V(2).InfoS("The after stage task still need to wait", "waitStartTime", waitStartTime, "waitTime", task.WaitTime, "stage", updatingStage.Name, "updateRun", updateRunRef) passed = false afterStageWaitTime = waitTime } else { - markAfterStageWaitTimeElapsed(&updatingStageStatus.AfterStageTaskStatus[i], updateRun.Generation) - klog.V(2).InfoS("The after stage wait task has completed", "stage", updatingStage.Name, "clusterStagedUpdateRun", updateRunRef) + markAfterStageWaitTimeElapsed(&updatingStageStatus.AfterStageTaskStatus[i], updateRun.GetGeneration()) + klog.V(2).InfoS("The after stage wait task has completed", "stage", updatingStage.Name, "updateRun", updateRunRef) } case placementv1beta1.AfterStageTaskTypeApproval: - afterStageTaskApproved := condition.IsConditionStatusTrue(meta.FindStatusCondition(updatingStageStatus.AfterStageTaskStatus[i].Conditions, string(placementv1beta1.AfterStageTaskConditionApprovalRequestApproved)), updateRun.Generation) + afterStageTaskApproved := condition.IsConditionStatusTrue(meta.FindStatusCondition(updatingStageStatus.AfterStageTaskStatus[i].Conditions, string(placementv1beta1.AfterStageTaskConditionApprovalRequestApproved)), updateRun.GetGeneration()) if afterStageTaskApproved { // The afterStageTask has been approved. continue } // Check if the approval request has been created. - approvalRequest := placementv1beta1.ClusterApprovalRequest{ - ObjectMeta: metav1.ObjectMeta{ - Name: updatingStageStatus.AfterStageTaskStatus[i].ApprovalRequestName, - Labels: map[string]string{ - placementv1beta1.TargetUpdatingStageNameLabel: updatingStage.Name, - placementv1beta1.TargetUpdateRunLabel: updateRun.Name, - placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", - }, - }, - Spec: placementv1beta1.ApprovalRequestSpec{ - TargetUpdateRun: updateRun.Name, - TargetStage: updatingStage.Name, - }, - } - requestRef := klog.KObj(&approvalRequest) - if err := r.Client.Create(ctx, &approvalRequest); err != nil { + approvalRequest := buildApprovalRequestObject(types.NamespacedName{Name: updatingStageStatus.AfterStageTaskStatus[i].ApprovalRequestName, Namespace: updateRun.GetNamespace()}, updatingStage.Name, updateRun.GetName()) + requestRef := klog.KObj(approvalRequest) + if err := r.Client.Create(ctx, approvalRequest); err != nil { if apierrors.IsAlreadyExists(err) { // The approval task already exists. - markAfterStageRequestCreated(&updatingStageStatus.AfterStageTaskStatus[i], updateRun.Generation) - if err = r.Client.Get(ctx, client.ObjectKeyFromObject(&approvalRequest), &approvalRequest); err != nil { - klog.ErrorS(err, "Failed to get the already existing approval request", "approvalRequest", requestRef, "stage", updatingStage.Name, "clusterStagedUpdateRun", updateRunRef) + markAfterStageRequestCreated(&updatingStageStatus.AfterStageTaskStatus[i], updateRun.GetGeneration()) + if err = r.Client.Get(ctx, client.ObjectKeyFromObject(approvalRequest), approvalRequest); err != nil { + klog.ErrorS(err, "Failed to get the already existing approval request", "approvalRequest", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) return false, -1, controller.NewAPIServerError(true, err) } - if approvalRequest.Spec.TargetStage != updatingStage.Name || approvalRequest.Spec.TargetUpdateRun != updateRun.Name { - unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the approval request task `%s` is targeting update run `%s` and stage `%s` ", approvalRequest.Name, approvalRequest.Spec.TargetStage, approvalRequest.Spec.TargetUpdateRun)) - klog.ErrorS(unexpectedErr, "Found an approval request targeting wrong stage", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "clusterStagedUpdateRun", updateRunRef) + approvalRequestSpec := approvalRequest.GetApprovalRequestSpec() + if approvalRequestSpec.TargetStage != updatingStage.Name || approvalRequestSpec.TargetUpdateRun != updateRun.GetName() { + unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the approval request task `%s/%s` is targeting update run `%s/%s` and stage `%s`", approvalRequest.GetNamespace(), approvalRequest.GetName(), approvalRequest.GetNamespace(), approvalRequestSpec.TargetUpdateRun, approvalRequestSpec.TargetStage)) + klog.ErrorS(unexpectedErr, "Found an approval request targeting wrong stage", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) return false, -1, fmt.Errorf("%w: %s", errStagedUpdatedAborted, unexpectedErr.Error()) } - approvalAccepted := condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequest.Status.Conditions, string(placementv1beta1.ApprovalRequestConditionApprovalAccepted)), approvalRequest.Generation) - approved := condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequest.Status.Conditions, string(placementv1beta1.ApprovalRequestConditionApproved)), approvalRequest.Generation) + approvalRequestStatus := approvalRequest.GetApprovalRequestStatus() + approvalAccepted := condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequestStatus.Conditions, string(placementv1beta1.ApprovalRequestConditionApprovalAccepted)), approvalRequest.GetGeneration()) + approved := condition.IsConditionStatusTrue(meta.FindStatusCondition(approvalRequestStatus.Conditions, string(placementv1beta1.ApprovalRequestConditionApproved)), approvalRequest.GetGeneration()) if !approvalAccepted && !approved { - klog.V(2).InfoS("The approval request has not been approved yet", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "clusterStagedUpdateRun", updateRunRef) + klog.V(2).InfoS("The approval request has not been approved yet", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) passed = false continue } if approved { - klog.V(2).InfoS("The approval request has been approved", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "clusterStagedUpdateRun", updateRunRef) + klog.V(2).InfoS("The approval request has been approved", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) if !approvalAccepted { - if err = r.updateApprovalRequestAccepted(ctx, &approvalRequest); err != nil { - klog.ErrorS(err, "Failed to accept the approved approval request", "approvalRequest", requestRef, "stage", updatingStage.Name, "clusterStagedUpdateRun", updateRunRef) + if err = r.updateApprovalRequestAccepted(ctx, approvalRequest); err != nil { + klog.ErrorS(err, "Failed to accept the approved approval request", "approvalRequest", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) // retriable err return false, -1, err } } } else { // Approved state should not change once the approval is accepted. - klog.V(2).InfoS("The approval request has been approval-accepted, ignoring changing back to unapproved", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "clusterStagedUpdateRun", updateRunRef) + klog.V(2).InfoS("The approval request has been approval-accepted, ignoring changing back to unapproved", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) } - markAfterStageRequestApproved(&updatingStageStatus.AfterStageTaskStatus[i], updateRun.Generation) + markAfterStageRequestApproved(&updatingStageStatus.AfterStageTaskStatus[i], updateRun.GetGeneration()) } else { // retriable error - klog.ErrorS(err, "Failed to create the approval request", "approvalRequest", requestRef, "stage", updatingStage.Name, "clusterStagedUpdateRun", updateRunRef) + klog.ErrorS(err, "Failed to create the approval request", "approvalRequest", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) return false, -1, controller.NewAPIServerError(false, err) } } else { // The approval request has been created for the first time. - klog.V(2).InfoS("The approval request has been created", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "clusterStagedUpdateRun", updateRunRef) - markAfterStageRequestCreated(&updatingStageStatus.AfterStageTaskStatus[i], updateRun.Generation) + klog.V(2).InfoS("The approval request has been created", "approvalRequestTask", requestRef, "stage", updatingStage.Name, "updateRun", updateRunRef) + markAfterStageRequestCreated(&updatingStageStatus.AfterStageTaskStatus[i], updateRun.GetGeneration()) passed = false } } @@ -393,59 +393,61 @@ func (r *Reconciler) checkAfterStageTasksStatus(ctx context.Context, updatingSta } // updateBindingRolloutStarted updates the binding status to indicate the rollout has started. -func (r *Reconciler) updateBindingRolloutStarted(ctx context.Context, binding *placementv1beta1.ClusterResourceBinding, updateRun *placementv1beta1.ClusterStagedUpdateRun) error { +func (r *Reconciler) updateBindingRolloutStarted(ctx context.Context, binding placementv1beta1.BindingObj, updateRun placementv1beta1.UpdateRunObj) error { // first reset the condition to reflect the latest lastTransitionTime binding.RemoveCondition(string(placementv1beta1.ResourceBindingRolloutStarted)) cond := metav1.Condition{ Type: string(placementv1beta1.ResourceBindingRolloutStarted), Status: metav1.ConditionTrue, - ObservedGeneration: binding.Generation, + ObservedGeneration: binding.GetGeneration(), Reason: condition.RolloutStartedReason, - Message: fmt.Sprintf("Detected the new changes on the resources and started the rollout process, resourceSnapshotIndex: %s, clusterStagedUpdateRun: %s", updateRun.Spec.ResourceSnapshotIndex, updateRun.Name), + Message: fmt.Sprintf("Detected the new changes on the resources and started the rollout process, resourceSnapshotIndex: %s, updateRun: %s", updateRun.GetUpdateRunSpec().ResourceSnapshotIndex, updateRun.GetName()), } binding.SetConditions(cond) if err := r.Client.Status().Update(ctx, binding); err != nil { - klog.ErrorS(err, "Failed to update binding status", "clusterResourceBinding", klog.KObj(binding), "condition", cond) + klog.ErrorS(err, "Failed to update binding status", "binding", klog.KObj(binding), "condition", cond) return controller.NewUpdateIgnoreConflictError(err) } - klog.V(2).InfoS("Updated binding as rolloutStarted", "clusterResourceBinding", klog.KObj(binding), "condition", cond) + klog.V(2).InfoS("Updated binding as rolloutStarted", "binding", klog.KObj(binding), "condition", cond) return nil } -// updateApprovalRequestAccepted updates the *approved* clusterApprovalRequest status to indicate the approval accepted. -func (r *Reconciler) updateApprovalRequestAccepted(ctx context.Context, appReq *placementv1beta1.ClusterApprovalRequest) error { +// updateApprovalRequestAccepted updates the *approved* approval request status to indicate the approval accepted. +func (r *Reconciler) updateApprovalRequestAccepted(ctx context.Context, appReq placementv1beta1.ApprovalRequestObj) error { cond := metav1.Condition{ Type: string(placementv1beta1.ApprovalRequestConditionApprovalAccepted), Status: metav1.ConditionTrue, - ObservedGeneration: appReq.Generation, + ObservedGeneration: appReq.GetGeneration(), Reason: condition.ApprovalRequestApprovalAcceptedReason, Message: "The approval request has been approved and cannot be reverted", } - meta.SetStatusCondition(&appReq.Status.Conditions, cond) + approvalRequestStatus := appReq.GetApprovalRequestStatus() + meta.SetStatusCondition(&approvalRequestStatus.Conditions, cond) if err := r.Client.Status().Update(ctx, appReq); err != nil { - klog.ErrorS(err, "Failed to update approval request status", "clusterApprovalRequest", klog.KObj(appReq), "condition", cond) + klog.ErrorS(err, "Failed to update approval request status", "approvalRequest", klog.KObj(appReq), "condition", cond) return controller.NewUpdateIgnoreConflictError(err) } - klog.V(2).InfoS("Updated approval request as approval accepted", "clusterApprovalRequest", klog.KObj(appReq), "condition", cond) + klog.V(2).InfoS("Updated approval request as approval accepted", "approvalRequest", klog.KObj(appReq), "condition", cond) return nil } // isBindingSyncedWithClusterStatus checks if the binding is up-to-date with the cluster status. -func isBindingSyncedWithClusterStatus(resourceSnapshotName string, updateRun *placementv1beta1.ClusterStagedUpdateRun, binding *placementv1beta1.ClusterResourceBinding, cluster *placementv1beta1.ClusterUpdatingStatus) bool { - if binding.Spec.ResourceSnapshotName != resourceSnapshotName { - klog.ErrorS(fmt.Errorf("binding has different resourceSnapshotName, want: %s, got: %s", resourceSnapshotName, binding.Spec.ResourceSnapshotName), "ClusterResourceBinding is not up-to-date", "clusterResourceBinding", klog.KObj(binding), "clusterStagedUpdateRun", klog.KObj(updateRun)) +func isBindingSyncedWithClusterStatus(resourceSnapshotName string, updateRun placementv1beta1.UpdateRunObj, binding placementv1beta1.BindingObj, cluster *placementv1beta1.ClusterUpdatingStatus) bool { + bindingSpec := binding.GetBindingSpec() + if bindingSpec.ResourceSnapshotName != resourceSnapshotName { + klog.ErrorS(fmt.Errorf("binding has different resourceSnapshotName, want: %s, got: %s", resourceSnapshotName, bindingSpec.ResourceSnapshotName), "binding is not up-to-date", "binding", klog.KObj(binding), "updateRun", klog.KObj(updateRun)) return false } - if !reflect.DeepEqual(cluster.ResourceOverrideSnapshots, binding.Spec.ResourceOverrideSnapshots) { - klog.ErrorS(fmt.Errorf("binding has different resourceOverrideSnapshots, want: %v, got: %v", cluster.ResourceOverrideSnapshots, binding.Spec.ResourceOverrideSnapshots), "ClusterResourceBinding is not up-to-date", "clusterResourceBinding", klog.KObj(binding), "clusterStagedUpdateRun", klog.KObj(updateRun)) + if !reflect.DeepEqual(cluster.ResourceOverrideSnapshots, bindingSpec.ResourceOverrideSnapshots) { + klog.ErrorS(fmt.Errorf("binding has different resourceOverrideSnapshots, want: %v, got: %v", cluster.ResourceOverrideSnapshots, bindingSpec.ResourceOverrideSnapshots), "binding is not up-to-date", "binding", klog.KObj(binding), "updateRun", klog.KObj(updateRun)) return false } - if !reflect.DeepEqual(cluster.ClusterResourceOverrideSnapshots, binding.Spec.ClusterResourceOverrideSnapshots) { - klog.ErrorS(fmt.Errorf("binding has different clusterResourceOverrideSnapshots, want: %v, got: %v", cluster.ClusterResourceOverrideSnapshots, binding.Spec.ClusterResourceOverrideSnapshots), "ClusterResourceBinding is not up-to-date", "clusterResourceBinding", klog.KObj(binding), "clusterStagedUpdateRun", klog.KObj(updateRun)) + if !reflect.DeepEqual(cluster.ClusterResourceOverrideSnapshots, bindingSpec.ClusterResourceOverrideSnapshots) { + klog.ErrorS(fmt.Errorf("binding has different clusterResourceOverrideSnapshots, want: %v, got: %v", cluster.ClusterResourceOverrideSnapshots, bindingSpec.ClusterResourceOverrideSnapshots), "binding is not up-to-date", "binding", klog.KObj(binding), "updateRun", klog.KObj(updateRun)) return false } - if !reflect.DeepEqual(binding.Spec.ApplyStrategy, updateRun.Status.ApplyStrategy) { - klog.ErrorS(fmt.Errorf("binding has different applyStrategy, want: %v, got: %v", updateRun.Status.ApplyStrategy, binding.Spec.ApplyStrategy), "ClusterResourceBinding is not up-to-date", "clusterResourceBinding", klog.KObj(binding), "clusterStagedUpdateRun", klog.KObj(updateRun)) + if !reflect.DeepEqual(bindingSpec.ApplyStrategy, updateRun.GetUpdateRunStatus().ApplyStrategy) { + klog.ErrorS(fmt.Errorf("binding has different applyStrategy, want: %v, got: %v", updateRun.GetUpdateRunStatus().ApplyStrategy, bindingSpec.ApplyStrategy), "binding is not up-to-date", "binding", klog.KObj(binding), "updateRun", klog.KObj(updateRun)) return false } return true @@ -454,45 +456,86 @@ func isBindingSyncedWithClusterStatus(resourceSnapshotName string, updateRun *pl // checkClusterUpdateResult checks if the resources have been updated successfully on a given cluster. // It returns true if the resources have been updated successfully or any error if the update failed. func checkClusterUpdateResult( - binding *placementv1beta1.ClusterResourceBinding, + binding placementv1beta1.BindingObj, clusterStatus *placementv1beta1.ClusterUpdatingStatus, updatingStage *placementv1beta1.StageUpdatingStatus, - updateRun *placementv1beta1.ClusterStagedUpdateRun, + updateRun placementv1beta1.UpdateRunObj, ) (bool, error) { availCond := binding.GetCondition(string(placementv1beta1.ResourceBindingAvailable)) diffReportCondition := binding.GetCondition(string(placementv1beta1.ResourceBindingDiffReported)) - if condition.IsConditionStatusTrue(availCond, binding.Generation) || - condition.IsConditionStatusTrue(diffReportCondition, binding.Generation) { + if condition.IsConditionStatusTrue(availCond, binding.GetGeneration()) || + condition.IsConditionStatusTrue(diffReportCondition, binding.GetGeneration()) { // The resource updated on the cluster is available or diff is successfully reported. - klog.InfoS("The cluster has been updated", "cluster", clusterStatus.ClusterName, "stage", updatingStage.StageName, "clusterStagedUpdateRun", klog.KObj(updateRun)) - markClusterUpdatingSucceeded(clusterStatus, updateRun.Generation) + klog.InfoS("The cluster has been updated", "cluster", clusterStatus.ClusterName, "stage", updatingStage.StageName, "updateRun", klog.KObj(updateRun)) + markClusterUpdatingSucceeded(clusterStatus, updateRun.GetGeneration()) return true, nil } - if bindingutils.HasBindingFailed(binding) || condition.IsConditionStatusFalse(diffReportCondition, binding.Generation) { + if bindingutils.HasBindingFailed(binding) || condition.IsConditionStatusFalse(diffReportCondition, binding.GetGeneration()) { // We have no way to know if the failed condition is recoverable or not so we just let it run - klog.InfoS("The cluster updating encountered an error", "cluster", clusterStatus.ClusterName, "stage", updatingStage.StageName, "clusterStagedUpdateRun", klog.KObj(updateRun)) + klog.InfoS("The cluster updating encountered an error", "cluster", clusterStatus.ClusterName, "stage", updatingStage.StageName, "updateRun", klog.KObj(updateRun)) // TODO(wantjian): identify some non-recoverable error and mark the cluster updating as failed - return false, fmt.Errorf("the cluster updating encountered an error at stage `%s`, updateRun := `%s`", updatingStage.StageName, updateRun.Name) + return false, fmt.Errorf("the cluster updating encountered an error at stage `%s`, updateRun := `%s`", updatingStage.StageName, updateRun.GetName()) } - klog.InfoS("The application on the cluster is in the mid of being updated", "cluster", clusterStatus.ClusterName, "stage", updatingStage.StageName, "clusterStagedUpdateRun", klog.KObj(updateRun)) + klog.InfoS("The application on the cluster is in the mid of being updated", "cluster", clusterStatus.ClusterName, "stage", updatingStage.StageName, "updateRun", klog.KObj(updateRun)) return false, nil } +// buildApprovalRequestObject creates an approval request object for after-stage tasks. +// It returns a ClusterApprovalRequest if namespace is empty, otherwise returns an ApprovalRequest. +func buildApprovalRequestObject(namespacedName types.NamespacedName, stageName, updateRunName string) placementv1beta1.ApprovalRequestObj { + var approvalRequest placementv1beta1.ApprovalRequestObj + if namespacedName.Namespace == "" { + approvalRequest = &placementv1beta1.ClusterApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespacedName.Name, + Labels: map[string]string{ + placementv1beta1.TargetUpdatingStageNameLabel: stageName, + placementv1beta1.TargetUpdateRunLabel: updateRunName, + placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", + }, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: updateRunName, + TargetStage: stageName, + }, + } + } else { + approvalRequest = &placementv1beta1.ApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespacedName.Name, + Namespace: namespacedName.Namespace, + Labels: map[string]string{ + placementv1beta1.TargetUpdatingStageNameLabel: stageName, + placementv1beta1.TargetUpdateRunLabel: updateRunName, + placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", + }, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: updateRunName, + TargetStage: stageName, + }, + } + } + return approvalRequest +} + // markUpdateRunProgressing marks the update run as progressing in memory. -func markUpdateRunProgressing(updateRun *placementv1beta1.ClusterStagedUpdateRun) { - meta.SetStatusCondition(&updateRun.Status.Conditions, metav1.Condition{ +func markUpdateRunProgressing(updateRun placementv1beta1.UpdateRunObj) { + updateRunStatus := updateRun.GetUpdateRunStatus() + meta.SetStatusCondition(&updateRunStatus.Conditions, metav1.Condition{ Type: string(placementv1beta1.StagedUpdateRunConditionProgressing), Status: metav1.ConditionTrue, - ObservedGeneration: updateRun.Generation, + ObservedGeneration: updateRun.GetGeneration(), Reason: condition.UpdateRunProgressingReason, Message: "The update run is making progress", }) } // markUpdateRunProgressingIfNotWaitingOrStuck marks the update run as proegressing in memory if it's not marked as waiting or stuck already. -func markUpdateRunProgressingIfNotWaitingOrStuck(updateRun *placementv1beta1.ClusterStagedUpdateRun) { - progressingCond := meta.FindStatusCondition(updateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionProgressing)) - if condition.IsConditionStatusFalse(progressingCond, updateRun.Generation) && +func markUpdateRunProgressingIfNotWaitingOrStuck(updateRun placementv1beta1.UpdateRunObj) { + updateRunStatus := updateRun.GetUpdateRunStatus() + progressingCond := meta.FindStatusCondition(updateRunStatus.Conditions, string(placementv1beta1.StagedUpdateRunConditionProgressing)) + if condition.IsConditionStatusFalse(progressingCond, updateRun.GetGeneration()) && (progressingCond.Reason == condition.UpdateRunWaitingReason || progressingCond.Reason == condition.UpdateRunStuckReason) { // The updateRun is waiting or stuck, no need to mark it as started. return @@ -501,22 +544,24 @@ func markUpdateRunProgressingIfNotWaitingOrStuck(updateRun *placementv1beta1.Clu } // markUpdateRunStuck marks the updateRun as stuck in memory. -func markUpdateRunStuck(updateRun *placementv1beta1.ClusterStagedUpdateRun, stageName, clusterName string) { - meta.SetStatusCondition(&updateRun.Status.Conditions, metav1.Condition{ +func markUpdateRunStuck(updateRun placementv1beta1.UpdateRunObj, stageName, clusterName string) { + updateRunStatus := updateRun.GetUpdateRunStatus() + meta.SetStatusCondition(&updateRunStatus.Conditions, metav1.Condition{ Type: string(placementv1beta1.StagedUpdateRunConditionProgressing), Status: metav1.ConditionFalse, - ObservedGeneration: updateRun.Generation, + ObservedGeneration: updateRun.GetGeneration(), Reason: condition.UpdateRunStuckReason, - Message: fmt.Sprintf("The updateRun is stuck waiting for cluster %s in stage %s to finish updating, please check crp status for potential errors", clusterName, stageName), + Message: fmt.Sprintf("The updateRun is stuck waiting for cluster %s in stage %s to finish updating, please check placement status for potential errors", clusterName, stageName), }) } // markUpdateRunWaiting marks the updateRun as waiting in memory. -func markUpdateRunWaiting(updateRun *placementv1beta1.ClusterStagedUpdateRun, stageName string) { - meta.SetStatusCondition(&updateRun.Status.Conditions, metav1.Condition{ +func markUpdateRunWaiting(updateRun placementv1beta1.UpdateRunObj, stageName string) { + updateRunStatus := updateRun.GetUpdateRunStatus() + meta.SetStatusCondition(&updateRunStatus.Conditions, metav1.Condition{ Type: string(placementv1beta1.StagedUpdateRunConditionProgressing), Status: metav1.ConditionFalse, - ObservedGeneration: updateRun.Generation, + ObservedGeneration: updateRun.GetGeneration(), Reason: condition.UpdateRunWaitingReason, Message: fmt.Sprintf("The updateRun is waiting for after-stage tasks in stage %s to complete", stageName), }) @@ -629,7 +674,7 @@ func markAfterStageRequestCreated(afterStageTaskStatus *placementv1beta1.AfterSt Status: metav1.ConditionTrue, ObservedGeneration: generation, Reason: condition.AfterStageTaskApprovalRequestCreatedReason, - Message: "ClusterApprovalRequest is created", + Message: "ApprovalRequest object is created", }) } @@ -640,7 +685,7 @@ func markAfterStageRequestApproved(afterStageTaskStatus *placementv1beta1.AfterS Status: metav1.ConditionTrue, ObservedGeneration: generation, Reason: condition.AfterStageTaskApprovalRequestApprovedReason, - Message: "ClusterApprovalRequest is approved", + Message: "ApprovalRequest object is approved", }) } diff --git a/pkg/controllers/updaterun/execution_integration_test.go b/pkg/controllers/updaterun/execution_integration_test.go index 6b1dd5d7b..322bc00a1 100644 --- a/pkg/controllers/updaterun/execution_integration_test.go +++ b/pkg/controllers/updaterun/execution_integration_test.go @@ -45,7 +45,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { var unscheduledClusters []*clusterv1beta1.MemberCluster var resourceSnapshot *placementv1beta1.ClusterResourceSnapshot var clusterResourceOverride *placementv1beta1.ClusterResourceOverrideSnapshot - var wantStatus *placementv1beta1.StagedUpdateRunStatus + var wantStatus *placementv1beta1.UpdateRunStatus var numTargetClusters int var numUnscheduledClusters int @@ -586,7 +586,7 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { var resourceBindings []*placementv1beta1.ClusterResourceBinding var targetClusters []*clusterv1beta1.MemberCluster var resourceSnapshot *placementv1beta1.ClusterResourceSnapshot - var wantStatus *placementv1beta1.StagedUpdateRunStatus + var wantStatus *placementv1beta1.UpdateRunStatus BeforeEach(OncePerOrdered, func() { testUpdateRunName = "updaterun-" + utils.RandStr() diff --git a/pkg/controllers/updaterun/execution_test.go b/pkg/controllers/updaterun/execution_test.go index 563a7c948..22b2bd928 100644 --- a/pkg/controllers/updaterun/execution_test.go +++ b/pkg/controllers/updaterun/execution_test.go @@ -19,8 +19,10 @@ package updaterun import ( "testing" + "github.com/google/go-cmp/cmp" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" "go.goms.io/fleet/pkg/utils/condition" @@ -103,7 +105,7 @@ func TestIsBindingSyncedWithClusterStatus(t *testing.T) { name: "isBindingSyncedWithClusterStatus should return false if binding and updateRun have different applyStrategy", resourceSnapshotName: "test-1-snapshot", updateRun: &placementv1beta1.ClusterStagedUpdateRun{ - Status: placementv1beta1.StagedUpdateRunStatus{ + Status: placementv1beta1.UpdateRunStatus{ ApplyStrategy: &placementv1beta1.ApplyStrategy{ Type: placementv1beta1.ApplyStrategyTypeClientSideApply, }, @@ -135,7 +137,7 @@ func TestIsBindingSyncedWithClusterStatus(t *testing.T) { name: "isBindingSyncedWithClusterStatus should return true if resourceSnapshot, applyStrategy, and override lists are all deep equal", resourceSnapshotName: "test-1-snapshot", updateRun: &placementv1beta1.ClusterStagedUpdateRun{ - Status: placementv1beta1.StagedUpdateRunStatus{ + Status: placementv1beta1.UpdateRunStatus{ ApplyStrategy: &placementv1beta1.ApplyStrategy{ Type: placementv1beta1.ApplyStrategyTypeReportDiff, }, @@ -327,3 +329,72 @@ func TestCheckClusterUpdateResult(t *testing.T) { }) } } + +func TestBuildApprovalRequestObject(t *testing.T) { + tests := []struct { + name string + namespacedName types.NamespacedName + stageName string + updateRunName string + want placementv1beta1.ApprovalRequestObj + }{ + { + name: "should create ClusterApprovalRequest when namespace is empty", + namespacedName: types.NamespacedName{ + Name: "test-approval-request", + Namespace: "", + }, + stageName: "test-stage", + updateRunName: "test-update-run", + want: &placementv1beta1.ClusterApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-approval-request", + Labels: map[string]string{ + placementv1beta1.TargetUpdatingStageNameLabel: "test-stage", + placementv1beta1.TargetUpdateRunLabel: "test-update-run", + placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", + }, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: "test-update-run", + TargetStage: "test-stage", + }, + }, + }, + { + name: "should create namespaced ApprovalRequest when namespace is provided", + namespacedName: types.NamespacedName{ + Name: "test-approval-request", + Namespace: "test-namespace", + }, + stageName: "test-stage", + updateRunName: "test-update-run", + want: &placementv1beta1.ApprovalRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-approval-request", + Namespace: "test-namespace", + Labels: map[string]string{ + placementv1beta1.TargetUpdatingStageNameLabel: "test-stage", + placementv1beta1.TargetUpdateRunLabel: "test-update-run", + placementv1beta1.IsLatestUpdateRunApprovalLabel: "true", + }, + }, + Spec: placementv1beta1.ApprovalRequestSpec{ + TargetUpdateRun: "test-update-run", + TargetStage: "test-stage", + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := buildApprovalRequestObject(test.namespacedName, test.stageName, test.updateRunName) + + // Compare the whole objects using cmp.Diff with ignore options + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("buildApprovalRequestObject() mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/pkg/controllers/updaterun/initialization.go b/pkg/controllers/updaterun/initialization.go index cc7ebdd63..b3428dd19 100644 --- a/pkg/controllers/updaterun/initialization.go +++ b/pkg/controllers/updaterun/initialization.go @@ -27,6 +27,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" @@ -35,227 +36,262 @@ import ( "go.goms.io/fleet/pkg/utils/annotations" "go.goms.io/fleet/pkg/utils/condition" "go.goms.io/fleet/pkg/utils/controller" + "go.goms.io/fleet/pkg/utils/defaulter" "go.goms.io/fleet/pkg/utils/overrider" ) -// initialize initializes the ClusterStagedUpdateRun object with all the stages computed during the initialization. -// This function is called only once during the initialization of the ClusterStagedUpdateRun. +// initialize initializes the UpdateRun object with all the stages computed during the initialization. +// This function is called only once during the initialization of the UpdateRun. func (r *Reconciler) initialize( ctx context.Context, - updateRun *placementv1beta1.ClusterStagedUpdateRun, -) ([]*placementv1beta1.ClusterResourceBinding, []*placementv1beta1.ClusterResourceBinding, error) { - // Validate the ClusterResourcePlace object referenced by the ClusterStagedUpdateRun. - placementName, err := r.validateCRP(ctx, updateRun) + updateRun placementv1beta1.UpdateRunObj, +) ([]placementv1beta1.BindingObj, []placementv1beta1.BindingObj, error) { + // Validate the Placement object referenced by the UpdateRun. + placementNamespacedName, err := r.validatePlacement(ctx, updateRun) if err != nil { return nil, nil, err } - // Record the latest policy snapshot associated with the ClusterStagedUpdateRun. - latestPolicySnapshot, _, err := r.determinePolicySnapshot(ctx, placementName, updateRun) + // Record the latest policy snapshot associated with the placement. + latestPolicySnapshot, _, err := r.determinePolicySnapshot(ctx, placementNamespacedName, updateRun) if err != nil { return nil, nil, err } - // Collect the scheduled clusters by the corresponding ClusterResourcePlacement with the latest policy snapshot. - scheduledBindings, toBeDeletedBindings, err := r.collectScheduledClusters(ctx, placementName, latestPolicySnapshot, updateRun) + // Collect the scheduled clusters by the corresponding placement with the latest policy snapshot. + scheduledBindings, toBeDeletedBindings, err := r.collectScheduledClusters(ctx, placementNamespacedName, latestPolicySnapshot, updateRun) if err != nil { return nil, nil, err } - // Compute the stages based on the StagedUpdateStrategy. + + // Compute the stages based on the UpdateStrategy. if err := r.generateStagesByStrategy(ctx, scheduledBindings, toBeDeletedBindings, updateRun); err != nil { return nil, nil, err } // Record the override snapshots associated with each cluster. - if err := r.recordOverrideSnapshots(ctx, placementName, updateRun); err != nil { + if err := r.recordOverrideSnapshots(ctx, placementNamespacedName, updateRun); err != nil { return nil, nil, err } return scheduledBindings, toBeDeletedBindings, r.recordInitializationSucceeded(ctx, updateRun) } -// validateCRP validates the ClusterResourcePlacement object referenced by the ClusterStagedUpdateRun. -func (r *Reconciler) validateCRP(ctx context.Context, updateRun *placementv1beta1.ClusterStagedUpdateRun) (string, error) { +// validatePlacement validates the Placement object referenced by the UpdateRun. +func (r *Reconciler) validatePlacement(ctx context.Context, updateRun placementv1beta1.UpdateRunObj) (types.NamespacedName, error) { updateRunRef := klog.KObj(updateRun) - // Fetch the ClusterResourcePlacement object. - placementName := updateRun.Spec.PlacementName - var crp placementv1beta1.ClusterResourcePlacement - if err := r.Client.Get(ctx, client.ObjectKey{Name: placementName}, &crp); err != nil { - klog.ErrorS(err, "Failed to get ClusterResourcePlacement", "clusterResourcePlacement", placementName, "clusterStagedUpdateRun", updateRunRef) + placementName := updateRun.GetUpdateRunSpec().PlacementName + + // Create NamespacedName for the placement. + placementKey := types.NamespacedName{ + Name: placementName, + Namespace: updateRun.GetNamespace(), + } + + // Fetch the placement object. + placement, err := controller.FetchPlacementFromNamespacedName(ctx, r.Client, placementKey) + if err != nil { if apierrors.IsNotFound(err) { - // we won't continue or retry the initialization if the ClusterResourcePlacement is not found. - crpNotFoundErr := controller.NewUserError(fmt.Errorf("parent clusterResourcePlacement not found")) - return "", fmt.Errorf("%w: %s", errInitializedFailed, crpNotFoundErr.Error()) + placementNotFoundErr := controller.NewUserError(fmt.Errorf("parent placement not found")) + klog.ErrorS(err, "Failed to get placement", "placement", placementKey, "updateRun", updateRunRef) + return types.NamespacedName{}, fmt.Errorf("%w: %s", errInitializedFailed, placementNotFoundErr.Error()) } - return "", controller.NewAPIServerError(true, err) + klog.ErrorS(err, "Failed to get placement", "placement", placementKey, "updateRun", updateRunRef) + return types.NamespacedName{}, controller.NewAPIServerError(true, err) } - // Check if the ClusterResourcePlacement has an external rollout strategy. - if crp.Spec.Strategy.Type != placementv1beta1.ExternalRolloutStrategyType { - klog.V(2).InfoS("The clusterResourcePlacement does not have an external rollout strategy", "clusterResourcePlacement", placementName, "clusterStagedUpdateRun", updateRunRef) - wrongRolloutTypeErr := controller.NewUserError(errors.New("parent clusterResourcePlacement does not have an external rollout strategy, current strategy: " + string(crp.Spec.Strategy.Type))) - return "", fmt.Errorf("%w: %s", errInitializedFailed, wrongRolloutTypeErr.Error()) + + // fill out all the default values for placement, mutation webhook is not setup for resource placement. + // TODO: setup mutation webhook for resource placement. + defaulter.SetPlacementDefaults(placement) + + // Check if the Placement has an external rollout strategy. + placementSpec := placement.GetPlacementSpec() + if placementSpec.Strategy.Type != placementv1beta1.ExternalRolloutStrategyType { + klog.V(2).InfoS("The placement does not have an external rollout strategy", "placement", placementKey, "updateRun", updateRunRef) + wrongRolloutTypeErr := controller.NewUserError(errors.New("parent placement does not have an external rollout strategy, current strategy: " + string(placementSpec.Strategy.Type))) + return types.NamespacedName{}, fmt.Errorf("%w: %s", errInitializedFailed, wrongRolloutTypeErr.Error()) } - updateRun.Status.ApplyStrategy = crp.Spec.Strategy.ApplyStrategy - return crp.Name, nil + + updateRunStatus := updateRun.GetUpdateRunStatus() + updateRunStatus.ApplyStrategy = placementSpec.Strategy.ApplyStrategy + + return placementKey, nil } -// determinePolicySnapshot retrieves the latest policy snapshot associated with the ClusterResourcePlacement, -// and validates it and records it in the ClusterStagedUpdateRun status. +// determinePolicySnapshot retrieves the latest policy snapshot associated with the Placement, +// and validates it and records it in the UpdateRun status. func (r *Reconciler) determinePolicySnapshot( ctx context.Context, - placementName string, - updateRun *placementv1beta1.ClusterStagedUpdateRun, -) (*placementv1beta1.ClusterSchedulingPolicySnapshot, int, error) { + placementKey types.NamespacedName, + updateRun placementv1beta1.UpdateRunObj, +) (placementv1beta1.PolicySnapshotObj, int, error) { updateRunRef := klog.KObj(updateRun) + // Get the latest policy snapshot. - var policySnapshotList placementv1beta1.ClusterSchedulingPolicySnapshotList - latestPolicyMatcher := client.MatchingLabels{ - placementv1beta1.PlacementTrackingLabel: placementName, - placementv1beta1.IsLatestSnapshotLabel: "true", - } - if err := r.Client.List(ctx, &policySnapshotList, latestPolicyMatcher); err != nil { - klog.ErrorS(err, "Failed to list the latest policy snapshots", "clusterResourcePlacement", placementName, "clusterStagedUpdateRun", updateRunRef) + policySnapshotList, err := controller.FetchLatestPolicySnapshot(ctx, r.Client, placementKey) + if err != nil { + klog.ErrorS(err, "Failed to list the latest policy snapshots", "placement", placementKey, "updateRun", updateRunRef) // err can be retried. return nil, -1, controller.NewAPIServerError(true, err) } - if len(policySnapshotList.Items) != 1 { - if len(policySnapshotList.Items) > 1 { - err := controller.NewUnexpectedBehaviorError(fmt.Errorf("more than one (%d in actual) latest policy snapshots associated with the clusterResourcePlacement: %s", len(policySnapshotList.Items), placementName)) - klog.ErrorS(err, "Failed to find the latest policy snapshot", "clusterResourcePlacement", placementName, "clusterStagedUpdateRun", updateRunRef) + + policySnapshotObjs := policySnapshotList.GetPolicySnapshotObjs() + if len(policySnapshotObjs) != 1 { + if len(policySnapshotObjs) > 1 { + err := controller.NewUnexpectedBehaviorError(fmt.Errorf("more than one (%d in actual) latest policy snapshots associated with the placement: `%s`", len(policySnapshotObjs), placementKey)) + klog.ErrorS(err, "Failed to find the latest policy snapshot", "placement", placementKey, "updateRun", updateRunRef) // no more retries for this error. return nil, -1, fmt.Errorf("%w: %s", errInitializedFailed, err.Error()) } // no latest policy snapshot found. - err := fmt.Errorf("no latest policy snapshot associated with the clusterResourcePlacement: %s", placementName) - klog.ErrorS(err, "Failed to find the latest policy snapshot", "clusterResourcePlacement", placementName, "clusterStagedUpdateRun", updateRunRef) + err := fmt.Errorf("no latest policy snapshot associated with the placement: `%s`", placementKey) + klog.ErrorS(err, "Failed to find the latest policy snapshot", "placement", placementKey, "updateRun", updateRunRef) // again, no more retries here. return nil, -1, fmt.Errorf("%w: %s", errInitializedFailed, err.Error()) } - latestPolicySnapshot := policySnapshotList.Items[0] - policyIndex, foundIndex := latestPolicySnapshot.Labels[placementv1beta1.PolicyIndexLabel] + latestPolicySnapshot := policySnapshotObjs[0] + policySnapshotRef := klog.KObj(latestPolicySnapshot) + policyIndex, foundIndex := latestPolicySnapshot.GetLabels()[placementv1beta1.PolicyIndexLabel] if !foundIndex || len(policyIndex) == 0 { - noIndexErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("policy snapshot `%s` does not have a policy index label", latestPolicySnapshot.Name)) - klog.ErrorS(noIndexErr, "Failed to get the policy index from the latestPolicySnapshot", "clusterResourcePlacement", placementName, "latestPolicySnapshot", latestPolicySnapshot.Name, "clusterStagedUpdateRun", updateRunRef) + noIndexErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("policy snapshot `%s/%s` does not have a policy index label", latestPolicySnapshot.GetNamespace(), latestPolicySnapshot.GetName())) + klog.ErrorS(noIndexErr, "Failed to get the policy index from the latestPolicySnapshot", "placement", placementKey, "policySnapshot", policySnapshotRef, "updateRun", updateRunRef) // no more retries here. return nil, -1, fmt.Errorf("%w: %s", errInitializedFailed, noIndexErr.Error()) } - updateRun.Status.PolicySnapshotIndexUsed = policyIndex + + updateRunStatus := updateRun.GetUpdateRunStatus() + updateRunStatus.PolicySnapshotIndexUsed = policyIndex // For pickAll policy, the observed cluster count is not included in the policy snapshot. // We set it to -1. It will be validated in the binding stages. // If policy is nil, it's default to pickAll. clusterCount := -1 - if latestPolicySnapshot.Spec.Policy != nil { - if latestPolicySnapshot.Spec.Policy.PlacementType == placementv1beta1.PickNPlacementType { - count, err := annotations.ExtractNumOfClustersFromPolicySnapshot(&latestPolicySnapshot) + policySnapshotSpec := latestPolicySnapshot.GetPolicySnapshotSpec() + if policySnapshotSpec.Policy != nil { + if policySnapshotSpec.Policy.PlacementType == placementv1beta1.PickNPlacementType { + count, err := annotations.ExtractNumOfClustersFromPolicySnapshot(latestPolicySnapshot) if err != nil { - annErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("%w: the policy snapshot `%s` doesn't have valid cluster count annotation", err, latestPolicySnapshot.Name)) - klog.ErrorS(annErr, "Failed to get the cluster count from the latestPolicySnapshot", "clusterResourcePlacement", placementName, "latestPolicySnapshot", latestPolicySnapshot.Name, "clusterStagedUpdateRun", updateRunRef) + annErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("%w: the policy snapshot `%s/%s` doesn't have valid cluster count annotation", err, latestPolicySnapshot.GetNamespace(), latestPolicySnapshot.GetName())) + klog.ErrorS(annErr, "Failed to get the cluster count from the latestPolicySnapshot", "placement", placementKey, "policySnapshot", policySnapshotRef, "updateRun", updateRunRef) // no more retries here. return nil, -1, fmt.Errorf("%w, %s", errInitializedFailed, annErr.Error()) } clusterCount = count - } else if latestPolicySnapshot.Spec.Policy.PlacementType == placementv1beta1.PickFixedPlacementType { - clusterCount = len(latestPolicySnapshot.Spec.Policy.ClusterNames) + } else if policySnapshotSpec.Policy.PlacementType == placementv1beta1.PickFixedPlacementType { + clusterCount = len(policySnapshotSpec.Policy.ClusterNames) } } - updateRun.Status.PolicyObservedClusterCount = clusterCount - klog.V(2).InfoS("Found the latest policy snapshot", "latestPolicySnapshot", latestPolicySnapshot.Name, "observedClusterCount", updateRun.Status.PolicyObservedClusterCount, "clusterStagedUpdateRun", updateRunRef) + updateRunStatus.PolicyObservedClusterCount = clusterCount + klog.V(2).InfoS("Found the latest policy snapshot", "policySnapshot", policySnapshotRef, "observedClusterCount", updateRunStatus.PolicyObservedClusterCount, "updateRun", updateRunRef) - if !condition.IsConditionStatusTrue(latestPolicySnapshot.GetCondition(string(placementv1beta1.PolicySnapshotScheduled)), latestPolicySnapshot.Generation) { - scheduleErr := fmt.Errorf("policy snapshot `%s` not fully scheduled yet", latestPolicySnapshot.Name) - klog.ErrorS(scheduleErr, "The policy snapshot is not scheduled successfully", "clusterResourcePlacement", placementName, "latestPolicySnapshot", latestPolicySnapshot.Name, "clusterStagedUpdateRun", updateRunRef) + if !condition.IsConditionStatusTrue(latestPolicySnapshot.GetCondition(string(placementv1beta1.PolicySnapshotScheduled)), latestPolicySnapshot.GetGeneration()) { + scheduleErr := fmt.Errorf("policy snapshot `%s` not fully scheduled yet", latestPolicySnapshot.GetName()) + klog.ErrorS(scheduleErr, "The policy snapshot is not scheduled successfully", "placement", placementKey, "policySnapshot", policySnapshotRef, "updateRun", updateRunRef) return nil, -1, fmt.Errorf("%w: %s", errInitializedFailed, scheduleErr.Error()) } - return &latestPolicySnapshot, clusterCount, nil + return latestPolicySnapshot, clusterCount, nil } // collectScheduledClusters retrieves the schedules clusters from the latest policy snapshot // and lists all the bindings according to its SchedulePolicyTrackingLabel. func (r *Reconciler) collectScheduledClusters( ctx context.Context, - placementName string, - latestPolicySnapshot *placementv1beta1.ClusterSchedulingPolicySnapshot, - updateRun *placementv1beta1.ClusterStagedUpdateRun, -) ([]*placementv1beta1.ClusterResourceBinding, []*placementv1beta1.ClusterResourceBinding, error) { + placementKey types.NamespacedName, + latestPolicySnapshot placementv1beta1.PolicySnapshotObj, + updateRun placementv1beta1.UpdateRunObj, +) ([]placementv1beta1.BindingObj, []placementv1beta1.BindingObj, error) { updateRunRef := klog.KObj(updateRun) - // List all the bindings according to the ClusterResourcePlacement. - var bindingList placementv1beta1.ClusterResourceBindingList - resourceBindingMatcher := client.MatchingLabels{ - placementv1beta1.PlacementTrackingLabel: placementName, - } - if err := r.Client.List(ctx, &bindingList, resourceBindingMatcher); err != nil { - klog.ErrorS(err, "Failed to list clusterResourceBindings", "clusterResourcePlacement", placementName, "latestPolicySnapshot", latestPolicySnapshot.Name, "clusterStagedUpdateRun", updateRunRef) + policySnapshotRef := klog.KObj(latestPolicySnapshot) + + bindingObjs, err := controller.ListBindingsFromKey(ctx, r.Client, placementKey) + if err != nil { + klog.ErrorS(err, "Failed to list bindings", "placement", placementKey, "policySnapshot", policySnapshotRef, "updateRun", updateRunRef) // list err can be retried. return nil, nil, controller.NewAPIServerError(true, err) } - var toBeDeletedBindings, selectedBindings []*placementv1beta1.ClusterResourceBinding - for i, binding := range bindingList.Items { - if binding.Spec.SchedulingPolicySnapshotName == latestPolicySnapshot.Name { - if binding.Spec.State == placementv1beta1.BindingStateUnscheduled { - klog.V(2).InfoS("Found an unscheduled binding with the latest policy snapshot, delete it", "binding", binding.Name, "clusterResourcePlacement", placementName, - "latestPolicySnapshot", latestPolicySnapshot.Name, "clusterStagedUpdateRun", updateRunRef) - toBeDeletedBindings = append(toBeDeletedBindings, &bindingList.Items[i]) + var toBeDeletedBindings, selectedBindings []placementv1beta1.BindingObj + for i, binding := range bindingObjs { + bindingRef := klog.KObj(bindingObjs[i]) + bindingSpec := binding.GetBindingSpec() + if bindingSpec.SchedulingPolicySnapshotName == latestPolicySnapshot.GetName() { + if bindingSpec.State == placementv1beta1.BindingStateUnscheduled { + klog.V(2).InfoS("Found an unscheduled binding with the latest policy snapshot, delete it", "binding", bindingRef, "placement", placementKey, + "policySnapshot", policySnapshotRef, "updateRun", updateRunRef) + toBeDeletedBindings = append(toBeDeletedBindings, bindingObjs[i]) } else { - klog.V(2).InfoS("Found a scheduled binding", "binding", binding.Name, "clusterResourcePlacement", placementName, - "latestPolicySnapshot", latestPolicySnapshot.Name, "clusterStagedUpdateRun", updateRunRef) - selectedBindings = append(selectedBindings, &bindingList.Items[i]) + klog.V(2).InfoS("Found a scheduled binding", "binding", bindingRef, "placement", placementKey, + "policySnapshot", policySnapshotRef, "updateRun", updateRunRef) + selectedBindings = append(selectedBindings, bindingObjs[i]) } } else { - if binding.Spec.State != placementv1beta1.BindingStateUnscheduled { - stateErr := fmt.Errorf("binding `%s` with old policy snapshot %s has state %s, we might observe a transient state, need retry", binding.Name, binding.Spec.SchedulingPolicySnapshotName, binding.Spec.State) - klog.V(2).InfoS("Found a not-unscheduled binding with old policy snapshot, retrying", "binding", binding.Name, "clusterResourcePlacement", placementName, - "latestPolicySnapshot", latestPolicySnapshot.Name, "clusterStagedUpdateRun", updateRunRef) + if bindingSpec.State != placementv1beta1.BindingStateUnscheduled { + stateErr := fmt.Errorf("binding `%s/%s` with old policy snapshot `%s/%s` has state %s, we might observe a transient state, need retry", + binding.GetNamespace(), binding.GetName(), binding.GetNamespace(), bindingSpec.SchedulingPolicySnapshotName, bindingSpec.State) + + klog.V(2).InfoS("Found a not-unscheduled binding with old policy snapshot, retrying", "binding", bindingRef, "placement", placementKey, + "policySnapshot", policySnapshotRef, "updateRun", updateRunRef) // Transient state can be retried. return nil, nil, stateErr } - klog.V(2).InfoS("Found an unscheduled binding with old policy snapshot", "binding", binding.Name, "cluster", binding.Spec.TargetCluster, - "clusterResourcePlacement", placementName, "latestPolicySnapshot", latestPolicySnapshot.Name, "clusterStagedUpdateRun", updateRunRef) - toBeDeletedBindings = append(toBeDeletedBindings, &bindingList.Items[i]) + klog.V(2).InfoS("Found an unscheduled binding with old policy snapshot", "binding", bindingRef, "cluster", bindingSpec.TargetCluster, + "placement", placementKey, "policySnapshot", policySnapshotRef, "updateRun", updateRunRef) + toBeDeletedBindings = append(toBeDeletedBindings, bindingObjs[i]) } } if len(selectedBindings) == 0 && len(toBeDeletedBindings) == 0 { - nobindingErr := fmt.Errorf("no scheduled or to-be-deleted clusterResourceBindings found for the latest policy snapshot %s", latestPolicySnapshot.Name) - klog.ErrorS(nobindingErr, "Failed to collect clusterResourceBindings", "clusterResourcePlacement", placementName, "latestPolicySnapshot", latestPolicySnapshot.Name, "clusterStagedUpdateRun", updateRunRef) + nobindingErr := fmt.Errorf("no scheduled or to-be-deleted bindings found for the latest policy snapshot %s/%s", latestPolicySnapshot.GetNamespace(), latestPolicySnapshot.GetName()) + klog.ErrorS(nobindingErr, "Failed to collect bindings", "placement", placementKey, "policySnapshot", policySnapshotRef, "updateRun", updateRunRef) // no more retries here. return nil, nil, fmt.Errorf("%w: %s", errInitializedFailed, nobindingErr.Error()) } - if updateRun.Status.PolicyObservedClusterCount == -1 { + updateRunStatus := updateRun.GetUpdateRunStatus() + if updateRunStatus.PolicyObservedClusterCount == -1 { // For pickAll policy, the observed cluster count is not included in the policy snapshot. We set it to the number of selected bindings. // TODO (wantjian): refactor this part to update PolicyObservedClusterCount in one place. - updateRun.Status.PolicyObservedClusterCount = len(selectedBindings) - } else if updateRun.Status.PolicyObservedClusterCount != len(selectedBindings) { - countErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the number of selected bindings %d is not equal to the observed cluster count %d", len(selectedBindings), updateRun.Status.PolicyObservedClusterCount)) - klog.ErrorS(countErr, "Failed to collect clusterResourceBindings", "clusterResourcePlacement", placementName, "latestPolicySnapshot", latestPolicySnapshot.Name, "clusterStagedUpdateRun", updateRunRef) + updateRunStatus.PolicyObservedClusterCount = len(selectedBindings) + } else if updateRunStatus.PolicyObservedClusterCount != len(selectedBindings) { + countErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the number of selected bindings %d is not equal to the observed cluster count %d", len(selectedBindings), updateRunStatus.PolicyObservedClusterCount)) + klog.ErrorS(countErr, "Failed to collect bindings", "placement", placementKey, "policySnapshot", policySnapshotRef, "updateRun", updateRunRef) // no more retries here. return nil, nil, fmt.Errorf("%w: %s", errInitializedFailed, countErr.Error()) } return selectedBindings, toBeDeletedBindings, nil } -// generateStagesByStrategy computes the stages based on the ClusterStagedUpdateStrategy referenced by the ClusterStagedUpdateRun. +// generateStagesByStrategy computes the stages based on the UpdateStrategy referenced by the UpdateRun. func (r *Reconciler) generateStagesByStrategy( ctx context.Context, - scheduledBindings []*placementv1beta1.ClusterResourceBinding, - toBeDeletedBindings []*placementv1beta1.ClusterResourceBinding, - updateRun *placementv1beta1.ClusterStagedUpdateRun, + scheduledBindings []placementv1beta1.BindingObj, + toBeDeletedBindings []placementv1beta1.BindingObj, + updateRun placementv1beta1.UpdateRunObj, ) error { updateRunRef := klog.KObj(updateRun) - // Fetch the StagedUpdateStrategy referenced by StagedUpdateStrategyName. - var updateStrategy placementv1beta1.ClusterStagedUpdateStrategy - if err := r.Client.Get(ctx, client.ObjectKey{Name: updateRun.Spec.StagedUpdateStrategyName}, &updateStrategy); err != nil { - klog.ErrorS(err, "Failed to get StagedUpdateStrategy", "stagedUpdateStrategy", updateRun.Spec.StagedUpdateStrategyName, "clusterStagedUpdateRun", updateRunRef) + updateRunSpec := updateRun.GetUpdateRunSpec() + + // Create NamespacedName for the UpdateStrategy. + strategyKey := types.NamespacedName{ + Name: updateRunSpec.StagedUpdateStrategyName, + Namespace: updateRun.GetNamespace(), + } + + // Fetch the UpdateStrategy. + updateStrategy, err := controller.FetchUpdateStrategyFromNamespacedName(ctx, r.Client, strategyKey) + if err != nil { + klog.ErrorS(err, "Failed to get UpdateStrategy", "updateStrategy", strategyKey, "updateRun", updateRunRef) if apierrors.IsNotFound(err) { - // we won't continue or retry the initialization if the StagedUpdateStrategy is not found. - strategyNotFoundErr := controller.NewUserError(errors.New("referenced clusterStagedUpdateStrategy not found: " + updateRun.Spec.StagedUpdateStrategyName)) + // we won't continue or retry the initialization if the UpdateStrategy is not found. + strategyNotFoundErr := controller.NewUserError(fmt.Errorf("referenced updateStrategy not found: `%s`", strategyKey)) return fmt.Errorf("%w: %s", errInitializedFailed, strategyNotFoundErr.Error()) } // other err can be retried. return controller.NewAPIServerError(true, err) } - // This won't change even if the stagedUpdateStrategy changes or is deleted after the updateRun is initialized. - updateRun.Status.StagedUpdateStrategySnapshot = &updateStrategy.Spec + // This won't change even if the updateStrategy changes or is deleted after the updateRun is initialized. + updateRunStatus := updateRun.GetUpdateRunStatus() + updateStrategySpec := updateStrategy.GetUpdateStrategySpec() + updateRunStatus.UpdateStrategySnapshot = updateStrategySpec + // Remove waitTime from the updateRun status for AfterStageTask for type Approval. removeWaitTimeFromUpdateRunStatus(updateRun) @@ -265,43 +301,53 @@ func (r *Reconciler) generateStagesByStrategy( } toBeDeletedClusters := make([]placementv1beta1.ClusterUpdatingStatus, len(toBeDeletedBindings)) for i, binding := range toBeDeletedBindings { - klog.V(2).InfoS("Adding a cluster to the delete stage", "cluster", binding.Spec.TargetCluster, "clusterStagedUpdateStrategy", updateStrategy.Name, "clusterStagedUpdateRun", updateRunRef) - toBeDeletedClusters[i].ClusterName = binding.Spec.TargetCluster + bindingSpec := binding.GetBindingSpec() + klog.V(2).InfoS("Adding a cluster to the delete stage", "cluster", bindingSpec.TargetCluster, "updateStrategy", strategyKey, "updateRun", updateRunRef) + toBeDeletedClusters[i].ClusterName = bindingSpec.TargetCluster } // Sort the clusters in the deletion stage by their names. sort.Slice(toBeDeletedClusters, func(i, j int) bool { return toBeDeletedClusters[i].ClusterName < toBeDeletedClusters[j].ClusterName }) - updateRun.Status.DeletionStageStatus = &placementv1beta1.StageUpdatingStatus{ + + // Update deletion stage status using interface methods. + updateRunStatus.DeletionStageStatus = &placementv1beta1.StageUpdatingStatus{ StageName: placementv1beta1.UpdateRunDeleteStageName, Clusters: toBeDeletedClusters, } return nil } -// computeRunStageStatus computes the stages based on the ClusterStagedUpdateStrategy and records them in the ClusterStagedUpdateRun status. +// computeRunStageStatus computes the stages based on the UpdateStrategy and records them in the UpdateRun status. func (r *Reconciler) computeRunStageStatus( ctx context.Context, - scheduledBindings []*placementv1beta1.ClusterResourceBinding, - updateRun *placementv1beta1.ClusterStagedUpdateRun, + scheduledBindings []placementv1beta1.BindingObj, + updateRun placementv1beta1.UpdateRunObj, ) error { updateRunRef := klog.KObj(updateRun) - updateStrategyName := updateRun.Spec.StagedUpdateStrategyName + updateRunSpec := updateRun.GetUpdateRunSpec() + updateRunStatus := updateRun.GetUpdateRunStatus() + + // Create NamespacedName for the UpdateStrategy. + strategyKey := types.NamespacedName{ + Name: updateRunSpec.StagedUpdateStrategyName, + Namespace: updateRun.GetNamespace(), + } // Map to track clusters and ensure they appear in one and only one stage. allSelectedClusters := make(map[string]struct{}, len(scheduledBindings)) allPlacedClusters := make(map[string]struct{}) for _, binding := range scheduledBindings { - allSelectedClusters[binding.Spec.TargetCluster] = struct{}{} + allSelectedClusters[binding.GetBindingSpec().TargetCluster] = struct{}{} } - stagesStatus := make([]placementv1beta1.StageUpdatingStatus, 0, len(updateRun.Status.StagedUpdateStrategySnapshot.Stages)) + stagesStatus := make([]placementv1beta1.StageUpdatingStatus, 0, len(updateRunStatus.UpdateStrategySnapshot.Stages)) - // Apply the label selectors from the ClusterStagedUpdateStrategy to filter the clusters. - for _, stage := range updateRun.Status.StagedUpdateStrategySnapshot.Stages { + // Apply the label selectors from the UpdateStrategy to filter the clusters. + for _, stage := range updateRunStatus.UpdateStrategySnapshot.Stages { if err := validateAfterStageTask(stage.AfterStageTasks); err != nil { - klog.ErrorS(err, "Failed to validate the after stage tasks", "clusterStagedUpdateStrategy", updateStrategyName, "stage name", stage.Name, "clusterStagedUpdateRun", updateRunRef) + klog.ErrorS(err, "Failed to validate the after stage tasks", "updateStrategy", strategyKey, "stageName", stage.Name, "updateRun", updateRunRef) // no more retries here. - invalidAfterStageErr := controller.NewUserError(fmt.Errorf("the after stage tasks are invalid, clusterStagedUpdateStrategy: %s, stage: %s, err: %s", updateStrategyName, stage.Name, err.Error())) + invalidAfterStageErr := controller.NewUserError(fmt.Errorf("the after stage tasks are invalid, updateStrategy: `%s`, stage: %s, err: %s", strategyKey, stage.Name, err.Error())) return fmt.Errorf("%w: %s", errInitializedFailed, invalidAfterStageErr.Error()) } @@ -309,15 +355,15 @@ func (r *Reconciler) computeRunStageStatus( var curStageClusters []clusterv1beta1.MemberCluster labelSelector, err := metav1.LabelSelectorAsSelector(stage.LabelSelector) if err != nil { - klog.ErrorS(err, "Failed to convert label selector", "clusterStagedUpdateStrategy", updateStrategyName, "stage name", stage.Name, "labelSelector", stage.LabelSelector, "clusterStagedUpdateRun", updateRunRef) + klog.ErrorS(err, "Failed to convert label selector", "updateStrategy", strategyKey, "stageName", stage.Name, "labelSelector", stage.LabelSelector, "updateRun", updateRunRef) // no more retries here. - invalidLabelErr := controller.NewUserError(fmt.Errorf("the stage label selector is invalid, clusterStagedUpdateStrategy: %s, stage: %s, err: %s", updateStrategyName, stage.Name, err.Error())) + invalidLabelErr := controller.NewUserError(fmt.Errorf("the stage label selector is invalid, updateStrategy: `%s`, stage: %s, err: %s", strategyKey, stage.Name, err.Error())) return fmt.Errorf("%w: %s", errInitializedFailed, invalidLabelErr.Error()) } // List all the clusters that match the label selector. var clusterList clusterv1beta1.MemberClusterList if err := r.Client.List(ctx, &clusterList, &client.ListOptions{LabelSelector: labelSelector}); err != nil { - klog.ErrorS(err, "Failed to list clusters for the stage", "clusterStagedUpdateStrategy", updateStrategyName, "stage name", stage.Name, "labelSelector", stage.LabelSelector, "clusterStagedUpdateRun", updateRunRef) + klog.ErrorS(err, "Failed to list clusters for the stage", "updateStrategy", strategyKey, "stageName", stage.Name, "labelSelector", stage.LabelSelector, "updateRun", updateRunRef) // list err can be retried. return controller.NewAPIServerError(true, err) } @@ -328,7 +374,7 @@ func (r *Reconciler) computeRunStageStatus( if _, ok := allPlacedClusters[cluster.Name]; ok { // a cluster can only appear in one stage. dupErr := controller.NewUserError(fmt.Errorf("cluster `%s` appears in more than one stages", cluster.Name)) - klog.ErrorS(dupErr, "Failed to compute the stage", "clusterStagedUpdateStrategy", updateStrategyName, "stage name", stage.Name, "clusterStagedUpdateRun", updateRunRef) + klog.ErrorS(dupErr, "Failed to compute the stage", "updateStrategy", strategyKey, "stageName", stage.Name, "updateRun", updateRunRef) // no more retries here. return fmt.Errorf("%w: %s", errInitializedFailed, dupErr.Error()) } @@ -336,7 +382,7 @@ func (r *Reconciler) computeRunStageStatus( // interpret the label values as integers. if _, err := strconv.Atoi(cluster.Labels[*stage.SortingLabelKey]); err != nil { keyErr := controller.NewUserError(fmt.Errorf("the sorting label `%s:%s` on cluster `%s` is not valid: %s", *stage.SortingLabelKey, cluster.Labels[*stage.SortingLabelKey], cluster.Name, err.Error())) - klog.ErrorS(keyErr, "Failed to sort clusters in the stage", "clusterStagedUpdateStrategy", updateStrategyName, "stage name", stage.Name, "clusterStagedUpdateRun", updateRunRef) + klog.ErrorS(keyErr, "Failed to sort clusters in the stage", "updateStrategy", strategyKey, "stageName", stage.Name, "updateRun", updateRunRef) // no more retries here. return fmt.Errorf("%w: %s", errInitializedFailed, keyErr.Error()) } @@ -349,7 +395,7 @@ func (r *Reconciler) computeRunStageStatus( // Check if the stage is empty. if len(curStageClusters) == 0 { // since we allow no selected bindings, a stage can be empty. - klog.InfoS("No cluster is selected for the stage", "clusterStagedUpdateStrategy", updateStrategyName, "stage name", stage.Name, "clusterStagedUpdateRun", updateRunRef) + klog.InfoS("No cluster is selected for the stage", "updateStrategy", strategyKey, "stageName", stage.Name, "updateRun", updateRunRef) } else { // Sort the clusters in the stage based on the SortingLabelKey and cluster name. sort.Slice(curStageClusters, func(i, j int) bool { @@ -368,7 +414,7 @@ func (r *Reconciler) computeRunStageStatus( // Record the clusters in the stage. curStageUpdatingStatus.Clusters = make([]placementv1beta1.ClusterUpdatingStatus, len(curStageClusters)) for i, cluster := range curStageClusters { - klog.V(2).InfoS("Adding a cluster to the stage", "cluster", cluster.Name, "clusterStagedUpdateStrategy", updateStrategyName, "stage name", stage.Name, "clusterStagedUpdateRun", updateRunRef) + klog.V(2).InfoS("Adding a cluster to the stage", "cluster", cluster.Name, "updateStrategy", strategyKey, "stageName", stage.Name, "updateRun", updateRunRef) curStageUpdatingStatus.Clusters[i].ClusterName = cluster.Name } @@ -377,12 +423,12 @@ func (r *Reconciler) computeRunStageStatus( for i, task := range stage.AfterStageTasks { curStageUpdatingStatus.AfterStageTaskStatus[i].Type = task.Type if task.Type == placementv1beta1.AfterStageTaskTypeApproval { - curStageUpdatingStatus.AfterStageTaskStatus[i].ApprovalRequestName = fmt.Sprintf(placementv1beta1.ApprovalTaskNameFmt, updateRun.Name, stage.Name) + curStageUpdatingStatus.AfterStageTaskStatus[i].ApprovalRequestName = fmt.Sprintf(placementv1beta1.ApprovalTaskNameFmt, updateRun.GetName(), stage.Name) } } stagesStatus = append(stagesStatus, curStageUpdatingStatus) } - updateRun.Status.StagesStatus = stagesStatus + updateRunStatus.StagesStatus = stagesStatus // Check if the clusters are all placed. if len(allPlacedClusters) != len(allSelectedClusters) { @@ -395,14 +441,14 @@ func (r *Reconciler) computeRunStageStatus( } // Sort the missing clusters by their names to generate a stable error message. sort.Strings(missingClusters) - klog.ErrorS(missingErr, "Clusters are missing in any stage", "clusters", strings.Join(missingClusters, ", "), "clusterStagedUpdateStrategy", updateStrategyName, "clusterStagedUpdateRun", updateRunRef) - // no more retries here, only show the first 10 missing clusters in the CRP status. + klog.ErrorS(missingErr, "Clusters are missing in any stage", "clusters", strings.Join(missingClusters, ", "), "updateStrategy", strategyKey, "updateRun", updateRunRef) + // no more retries here, only show the first 10 missing clusters in the placement status. return fmt.Errorf("%w: %s, total %d, showing up to 10: %s", errInitializedFailed, missingErr.Error(), len(missingClusters), strings.Join(missingClusters[:min(10, len(missingClusters))], ", ")) } return nil } -// validateAfterStageTask valides the afterStageTasks in the stage defined in the clusterStagedUpdateStrategy. +// validateAfterStageTask valides the afterStageTasks in the stage defined in the UpdateStrategy. // The error returned from this function is not retryable. func validateAfterStageTask(tasks []placementv1beta1.AfterStageTask) error { if len(tasks) == 2 && tasks[0].Type == tasks[1].Type { @@ -421,107 +467,112 @@ func validateAfterStageTask(tasks []placementv1beta1.AfterStageTask) error { return nil } -// recordOverrideSnapshots finds all the override snapshots that are associated with each cluster and record them in the ClusterStagedUpdateRun status. -func (r *Reconciler) recordOverrideSnapshots(ctx context.Context, placementName string, updateRun *placementv1beta1.ClusterStagedUpdateRun) error { +// recordOverrideSnapshots finds all the override snapshots that are associated with each cluster and record them in the UpdateRun status. +func (r *Reconciler) recordOverrideSnapshots(ctx context.Context, placementKey types.NamespacedName, updateRun placementv1beta1.UpdateRunObj) error { updateRunRef := klog.KObj(updateRun) + updateRunSpec := updateRun.GetUpdateRunSpec() + placementName := placementKey.Name - snapshotIndex, err := strconv.Atoi(updateRun.Spec.ResourceSnapshotIndex) + snapshotIndex, err := strconv.Atoi(updateRunSpec.ResourceSnapshotIndex) if err != nil || snapshotIndex < 0 { - err := controller.NewUserError(fmt.Errorf("invalid resource snapshot index `%s` provided, expected an integer >= 0", updateRun.Spec.ResourceSnapshotIndex)) - klog.ErrorS(err, "Failed to parse the resource snapshot index", "clusterStagedUpdateRun", updateRunRef) + err := controller.NewUserError(fmt.Errorf("invalid resource snapshot index `%s` provided, expected an integer >= 0", updateRunSpec.ResourceSnapshotIndex)) + klog.ErrorS(err, "Failed to parse the resource snapshot index", "updateRun", updateRunRef) // no more retries here. return fmt.Errorf("%w: %s", errInitializedFailed, err.Error()) } - // TODO: use the lib to fetch the master resource snapshot using interface instead of concrete type - var masterResourceSnapshot *placementv1beta1.ClusterResourceSnapshot - labelMatcher := client.MatchingLabels{ - placementv1beta1.PlacementTrackingLabel: placementName, - placementv1beta1.ResourceIndexLabel: updateRun.Spec.ResourceSnapshotIndex, - } - resourceSnapshotList := &placementv1beta1.ClusterResourceSnapshotList{} - if err := r.Client.List(ctx, resourceSnapshotList, labelMatcher); err != nil { - klog.ErrorS(err, "Failed to list the clusterResourceSnapshots associated with the clusterResourcePlacement", - "clusterResourcePlacement", placementName, "resourceSnapshotIndex", snapshotIndex, "clusterStagedUpdateRun", updateRunRef) + + resourceSnapshotList, err := controller.ListAllResourceSnapshotWithAnIndex(ctx, r.Client, updateRunSpec.ResourceSnapshotIndex, placementName, placementKey.Namespace) + if err != nil { + klog.ErrorS(err, "Failed to list the resourceSnapshots associated with the placement", + "placement", placementKey, "resourceSnapshotIndex", snapshotIndex, "updateRun", updateRunRef) // err can be retried. return controller.NewAPIServerError(true, err) } - if len(resourceSnapshotList.Items) == 0 { - err := controller.NewUserError(fmt.Errorf("no clusterResourceSnapshots with index `%d` found for clusterResourcePlacement `%s`", snapshotIndex, placementName)) - klog.ErrorS(err, "No specified clusterResourceSnapshots found", "clusterStagedUpdateRun", updateRunRef) + resourceSnapshotObjs := resourceSnapshotList.GetResourceSnapshotObjs() + if len(resourceSnapshotObjs) == 0 { + err := controller.NewUserError(fmt.Errorf("no resourceSnapshots with index `%d` found for placement `%s`", snapshotIndex, placementKey)) + klog.ErrorS(err, "No specified resourceSnapshots found", "updateRun", updateRunRef) // no more retries here. return fmt.Errorf("%w: %s", errInitializedFailed, err.Error()) } - // Look for the master clusterResourceSnapshot. - for i, resourceSnapshot := range resourceSnapshotList.Items { - // only master has this annotation - if len(resourceSnapshot.Annotations[placementv1beta1.ResourceGroupHashAnnotation]) != 0 { - masterResourceSnapshot = &resourceSnapshotList.Items[i] + // Look for the master resourceSnapshot. + var masterResourceSnapshot placementv1beta1.ResourceSnapshotObj + for _, resourceSnapshot := range resourceSnapshotObjs { + // only master has this annotation. + if len(resourceSnapshot.GetAnnotations()[placementv1beta1.ResourceGroupHashAnnotation]) != 0 { + masterResourceSnapshot = resourceSnapshot break } } - // No clusterResourceSnapshot found + // No masterResourceSnapshot found. if masterResourceSnapshot == nil { - err := controller.NewUnexpectedBehaviorError(fmt.Errorf("no master clusterResourceSnapshot found for clusterResourcePlacement `%s` with index `%d`", placementName, snapshotIndex)) - klog.ErrorS(err, "Failed to find master clusterResourceSnapshot", "clusterStagedUpdateRun", updateRunRef) + err := controller.NewUnexpectedBehaviorError(fmt.Errorf("no master resourceSnapshot found for placement `%s` with index `%d`", placementKey, snapshotIndex)) + klog.ErrorS(err, "Failed to find master resourceSnapshot", "updateRun", updateRunRef) // no more retries here. return fmt.Errorf("%w: %s", errInitializedFailed, err.Error()) } - klog.V(2).InfoS("Found master clusterResourceSnapshot", "clusterResourcePlacement", placementName, "index", snapshotIndex, "clusterStagedUpdateRun", updateRunRef) + klog.V(2).InfoS("Found master resourceSnapshot", "placement", placementKey, "index", snapshotIndex, "updateRun", updateRunRef) + resourceSnapshotRef := klog.KObj(masterResourceSnapshot) // Fetch all the matching overrides. - matchedCRO, matchedRO, err := overrider.FetchAllMatchingOverridesForResourceSnapshot(ctx, r.Client, r.InformerManager, updateRun.Spec.PlacementName, masterResourceSnapshot) + matchedCRO, matchedRO, err := overrider.FetchAllMatchingOverridesForResourceSnapshot(ctx, r.Client, r.InformerManager, updateRunSpec.PlacementName, masterResourceSnapshot) if err != nil { - klog.ErrorS(err, "Failed to find all matching overrides for the clusterStagedUpdateRun", "masterResourceSnapshot", klog.KObj(masterResourceSnapshot), "clusterStagedUpdateRun", updateRunRef) + klog.ErrorS(err, "Failed to find all matching overrides for the stagedUpdateRun", "resourceSnapshot", resourceSnapshotRef, "updateRun", updateRunRef) // no more retries here. return fmt.Errorf("%w: %s", errInitializedFailed, err.Error()) } + // Pick the overrides associated with each target cluster. - for _, stageStatus := range updateRun.Status.StagesStatus { + updateRunStatus := updateRun.GetUpdateRunStatus() + for _, stageStatus := range updateRunStatus.StagesStatus { for i := range stageStatus.Clusters { clusterStatus := &stageStatus.Clusters[i] clusterStatus.ClusterResourceOverrideSnapshots, clusterStatus.ResourceOverrideSnapshots, err = overrider.PickFromResourceMatchedOverridesForTargetCluster(ctx, r.Client, clusterStatus.ClusterName, matchedCRO, matchedRO) if err != nil { - klog.ErrorS(err, "Failed to pick the override snapshots for cluster", "cluster", clusterStatus.ClusterName, "masterResourceSnapshot", klog.KObj(masterResourceSnapshot), "clusterStagedUpdateRun", updateRunRef) + klog.ErrorS(err, "Failed to pick the override snapshots for cluster", "cluster", clusterStatus.ClusterName, "resourceSnapshot", resourceSnapshotRef, "updateRun", updateRunRef) // no more retries here. return fmt.Errorf("%w: %s", errInitializedFailed, err.Error()) } } } + return nil } -// recordInitializationSucceeded records the successful initialization condition in the ClusterStagedUpdateRun status. -func (r *Reconciler) recordInitializationSucceeded(ctx context.Context, updateRun *placementv1beta1.ClusterStagedUpdateRun) error { - meta.SetStatusCondition(&updateRun.Status.Conditions, metav1.Condition{ +// recordInitializationSucceeded records the successful initialization condition in the UpdateRun status. +func (r *Reconciler) recordInitializationSucceeded(ctx context.Context, updateRun placementv1beta1.UpdateRunObj) error { + updateRunStatus := updateRun.GetUpdateRunStatus() + meta.SetStatusCondition(&updateRunStatus.Conditions, metav1.Condition{ Type: string(placementv1beta1.StagedUpdateRunConditionInitialized), Status: metav1.ConditionTrue, - ObservedGeneration: updateRun.Generation, + ObservedGeneration: updateRun.GetGeneration(), Reason: condition.UpdateRunInitializeSucceededReason, Message: "ClusterStagedUpdateRun initialized successfully", }) if updateErr := r.Client.Status().Update(ctx, updateRun); updateErr != nil { - klog.ErrorS(updateErr, "Failed to update the ClusterStagedUpdateRun status as initialized", "clusterStagedUpdateRun", klog.KObj(updateRun)) + klog.ErrorS(updateErr, "Failed to update the UpdateRun status as initialized", "updateRun", klog.KObj(updateRun)) // updateErr can be retried. return controller.NewUpdateIgnoreConflictError(updateErr) } return nil } -// recordInitializationFailed records the failed initialization condition in the ClusterStagedUpdateRun status. -func (r *Reconciler) recordInitializationFailed(ctx context.Context, updateRun *placementv1beta1.ClusterStagedUpdateRun, message string) error { - meta.SetStatusCondition(&updateRun.Status.Conditions, metav1.Condition{ +// recordInitializationFailed records the failed initialization condition in the updateRun status. +func (r *Reconciler) recordInitializationFailed(ctx context.Context, updateRun placementv1beta1.UpdateRunObj, message string) error { + updateRunStatus := updateRun.GetUpdateRunStatus() + meta.SetStatusCondition(&updateRunStatus.Conditions, metav1.Condition{ Type: string(placementv1beta1.StagedUpdateRunConditionInitialized), Status: metav1.ConditionFalse, - ObservedGeneration: updateRun.Generation, + ObservedGeneration: updateRun.GetGeneration(), Reason: condition.UpdateRunInitializeFailedReason, Message: message, }) if updateErr := r.Client.Status().Update(ctx, updateRun); updateErr != nil { - klog.ErrorS(updateErr, "Failed to update the ClusterStagedUpdateRun status as failed to initialize", "clusterStagedUpdateRun", klog.KObj(updateRun)) + klog.ErrorS(updateErr, "Failed to update the updateRun status as failed to initialize", "updateRun", klog.KObj(updateRun)) // updateErr can be retried. return controller.NewUpdateIgnoreConflictError(updateErr) } diff --git a/pkg/controllers/updaterun/initialization_integration_test.go b/pkg/controllers/updaterun/initialization_integration_test.go index c1d65dcac..a7d3d3648 100644 --- a/pkg/controllers/updaterun/initialization_integration_test.go +++ b/pkg/controllers/updaterun/initialization_integration_test.go @@ -138,7 +138,7 @@ var _ = Describe("Updaterun initialization tests", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization failed") - validateFailedInitCondition(ctx, updateRun, "parent clusterResourcePlacement not found") + validateFailedInitCondition(ctx, updateRun, "parent placement not found") }) It("Should fail to initialize if CRP does not have external rollout strategy type", func() { @@ -151,7 +151,7 @@ var _ = Describe("Updaterun initialization tests", func() { By("Validating the initialization failed") validateFailedInitCondition(ctx, updateRun, - "parent clusterResourcePlacement does not have an external rollout strategy") + "parent placement does not have an external rollout strategy") }) It("Should copy the ApplyStrategy in the CRP to the UpdateRun", func() { @@ -405,7 +405,7 @@ var _ = Describe("Updaterun initialization tests", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization failed") - validateFailedInitCondition(ctx, updateRun, "no scheduled or to-be-deleted clusterResourceBindings found") + validateFailedInitCondition(ctx, updateRun, "no scheduled or to-be-deleted bindings found") }) It("Should fail to initialize if the number of selected bindings does not match the observed cluster count", func() { @@ -505,7 +505,7 @@ var _ = Describe("Updaterun initialization tests", func() { By("Validating the initialization not failed due to no selected cluster") // it should fail due to strategy not found - validateFailedInitCondition(ctx, updateRun, "referenced clusterStagedUpdateStrategy not found") + validateFailedInitCondition(ctx, updateRun, "referenced updateStrategy not found") }) It("Should update the ObservedClusterCount to the number of scheduled bindings if it's pickAll policy", func() { @@ -517,7 +517,7 @@ var _ = Describe("Updaterun initialization tests", func() { By("Validating the initialization not failed due to no selected cluster") // it should fail due to strategy not found - validateFailedInitCondition(ctx, updateRun, "referenced clusterStagedUpdateStrategy not found") + validateFailedInitCondition(ctx, updateRun, "referenced updateStrategy not found") By("Validating the ObservedClusterCount is updated") Expect(updateRun.Status.PolicyObservedClusterCount).To(Equal(1), "failed to update the updateRun PolicyObservedClusterCount status") @@ -565,7 +565,7 @@ var _ = Describe("Updaterun initialization tests", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization failed") - validateFailedInitCondition(ctx, updateRun, "referenced clusterStagedUpdateStrategy not found") + validateFailedInitCondition(ctx, updateRun, "referenced updateStrategy not found") }) Context("Test computeRunStageStatus", func() { @@ -777,7 +777,7 @@ var _ = Describe("Updaterun initialization tests", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization failed") - validateFailedInitCondition(ctx, updateRun, "no clusterResourceSnapshots with index `0` found") + validateFailedInitCondition(ctx, updateRun, "no resourceSnapshots with index `0` found") By("Checking update run status metrics are emitted") validateUpdateRunMetricsEmitted(generateInitializationFailedMetric(updateRun)) @@ -792,7 +792,7 @@ var _ = Describe("Updaterun initialization tests", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization failed") - validateFailedInitCondition(ctx, updateRun, "no clusterResourceSnapshots with index `0` found") + validateFailedInitCondition(ctx, updateRun, "no resourceSnapshots with index `0` found") By("Checking update run status metrics are emitted") validateUpdateRunMetricsEmitted(generateInitializationFailedMetric(updateRun)) @@ -807,7 +807,7 @@ var _ = Describe("Updaterun initialization tests", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization failed") - validateFailedInitCondition(ctx, updateRun, "no clusterResourceSnapshots with index `0` found") + validateFailedInitCondition(ctx, updateRun, "no resourceSnapshots with index `0` found") By("Checking update run status metrics are emitted") validateUpdateRunMetricsEmitted(generateInitializationFailedMetric(updateRun)) @@ -822,7 +822,7 @@ var _ = Describe("Updaterun initialization tests", func() { Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) By("Validating the initialization failed") - validateFailedInitCondition(ctx, updateRun, "no master clusterResourceSnapshot found for clusterResourcePlacement") + validateFailedInitCondition(ctx, updateRun, "no master resourceSnapshot found for placement") By("Checking update run status metrics are emitted") validateUpdateRunMetricsEmitted(generateInitializationFailedMetric(updateRun)) @@ -875,12 +875,12 @@ func generateSucceededInitializationStatus( policySnapshot *placementv1beta1.ClusterSchedulingPolicySnapshot, updateStrategy *placementv1beta1.ClusterStagedUpdateStrategy, clusterResourceOverride *placementv1beta1.ClusterResourceOverrideSnapshot, -) *placementv1beta1.StagedUpdateRunStatus { - status := &placementv1beta1.StagedUpdateRunStatus{ - PolicySnapshotIndexUsed: policySnapshot.Labels[placementv1beta1.PolicyIndexLabel], - PolicyObservedClusterCount: 10, - ApplyStrategy: crp.Spec.Strategy.ApplyStrategy.DeepCopy(), - StagedUpdateStrategySnapshot: &updateStrategy.Spec, +) *placementv1beta1.UpdateRunStatus { + status := &placementv1beta1.UpdateRunStatus{ + PolicySnapshotIndexUsed: policySnapshot.Labels[placementv1beta1.PolicyIndexLabel], + PolicyObservedClusterCount: 10, + ApplyStrategy: crp.Spec.Strategy.ApplyStrategy.DeepCopy(), + UpdateStrategySnapshot: &updateStrategy.Spec, StagesStatus: []placementv1beta1.StageUpdatingStatus{ { StageName: "stage1", @@ -935,12 +935,12 @@ func generateSucceededInitializationStatusForSmallClusters( updateRun *placementv1beta1.ClusterStagedUpdateRun, policySnapshot *placementv1beta1.ClusterSchedulingPolicySnapshot, updateStrategy *placementv1beta1.ClusterStagedUpdateStrategy, -) *placementv1beta1.StagedUpdateRunStatus { - status := &placementv1beta1.StagedUpdateRunStatus{ - PolicySnapshotIndexUsed: policySnapshot.Labels[placementv1beta1.PolicyIndexLabel], - PolicyObservedClusterCount: 3, - ApplyStrategy: crp.Spec.Strategy.ApplyStrategy.DeepCopy(), - StagedUpdateStrategySnapshot: &updateStrategy.Spec, +) *placementv1beta1.UpdateRunStatus { + status := &placementv1beta1.UpdateRunStatus{ + PolicySnapshotIndexUsed: policySnapshot.Labels[placementv1beta1.PolicyIndexLabel], + PolicyObservedClusterCount: 3, + ApplyStrategy: crp.Spec.Strategy.ApplyStrategy.DeepCopy(), + UpdateStrategySnapshot: &updateStrategy.Spec, StagesStatus: []placementv1beta1.StageUpdatingStatus{ { StageName: "stage1", @@ -976,8 +976,8 @@ func generateSucceededInitializationStatusForSmallClusters( func generateExecutionStartedStatus( updateRun *placementv1beta1.ClusterStagedUpdateRun, - initialized *placementv1beta1.StagedUpdateRunStatus, -) *placementv1beta1.StagedUpdateRunStatus { + initialized *placementv1beta1.UpdateRunStatus, +) *placementv1beta1.UpdateRunStatus { // Mark updateRun execution has started. initialized.Conditions = append(initialized.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) // Mark updateRun 1st stage has started. diff --git a/pkg/controllers/updaterun/suite_test.go b/pkg/controllers/updaterun/suite_test.go index ab4e712bf..9f1e00fa5 100644 --- a/pkg/controllers/updaterun/suite_test.go +++ b/pkg/controllers/updaterun/suite_test.go @@ -35,13 +35,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" - ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" clusterv1beta1 "go.goms.io/fleet/apis/cluster/v1beta1" placementv1alpha1 "go.goms.io/fleet/apis/placement/v1alpha1" placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" - "go.goms.io/fleet/pkg/metrics" "go.goms.io/fleet/pkg/utils" "go.goms.io/fleet/pkg/utils/informer" ) @@ -116,12 +114,9 @@ var _ = BeforeSuite(func() { err = (&Reconciler{ Client: k8sClient, InformerManager: dynamicInformerManager, - }).SetupWithManager(mgr) + }).SetupWithManagerForClusterStagedUpdateRun(mgr) Expect(err).Should(Succeed()) - // Register metrics. - ctrlmetrics.Registry.MustRegister(metrics.FleetUpdateRunStatusLastTimestampSeconds) - go func() { defer GinkgoRecover() err = mgr.Start(ctx) diff --git a/pkg/controllers/updaterun/validation.go b/pkg/controllers/updaterun/validation.go index 81cf53093..d517290d6 100644 --- a/pkg/controllers/updaterun/validation.go +++ b/pkg/controllers/updaterun/validation.go @@ -29,53 +29,55 @@ import ( "go.goms.io/fleet/pkg/utils/controller" ) -// validate validates the clusterStagedUpdateRun status and ensures the update can be continued. +// validate validates the updateRun status and ensures the update can be continued. // The function returns the index of the stage that is updating, and the list of clusters that are scheduled to be deleted. -// If the updating stage index is -1, it means all stages are finished, and the clusterStageUpdateRun should be marked as finished. +// If the updating stage index is -1, it means all stages are finished, and the updateRun should be marked as finished. // If the updating stage index is 0, the next stage to be updated is the first stage. -// If the updating stage index is len(updateRun.Status.StagesStatus), the next stage to be updated will be the delete stage. +// If the updating stage index is len(updateRun.GetUpdateRunStatus().StagesStatus), the next stage to be updated will be the delete stage. func (r *Reconciler) validate( ctx context.Context, - updateRun *placementv1beta1.ClusterStagedUpdateRun, -) (int, []*placementv1beta1.ClusterResourceBinding, []*placementv1beta1.ClusterResourceBinding, error) { + updateRun placementv1beta1.UpdateRunObj, +) (int, []placementv1beta1.BindingObj, []placementv1beta1.BindingObj, error) { // Some of the validating function changes the object, so we need to make a copy of the object. updateRunRef := klog.KObj(updateRun) - updateRunCopy := updateRun.DeepCopy() - klog.V(2).InfoS("Start to validate the clusterStagedUpdateRun", "clusterStagedUpdateRun", updateRunRef) + updateRunStatus := updateRun.GetUpdateRunStatus() + updateRunCopy := updateRun.DeepCopyObject().(placementv1beta1.UpdateRunObj) + updateRunCopyStatus := updateRunCopy.GetUpdateRunStatus() + klog.V(2).InfoS("Start to validate the updateRun", "updateRun", updateRunRef) - // Validate the ClusterResourcePlacement object referenced by the ClusterStagedUpdateRun. - placementName, err := r.validateCRP(ctx, updateRunCopy) + // Validate the Placement object referenced by the UpdateRun. + placementNamespacedName, err := r.validatePlacement(ctx, updateRunCopy) if err != nil { return -1, nil, nil, err } // Validate the applyStrategy. - if !reflect.DeepEqual(updateRun.Status.ApplyStrategy, updateRunCopy.Status.ApplyStrategy) { - mismatchErr := controller.NewUserError(fmt.Errorf("the applyStrategy in the clusterStagedUpdateRun is outdated, latest: %v, recorded: %v", updateRunCopy.Status.ApplyStrategy, updateRun.Status.ApplyStrategy)) - klog.ErrorS(mismatchErr, "the applyStrategy in the clusterResourcePlacement has changed", "clusterResourcePlacement", placementName, "clusterStagedUpdateRun", updateRunRef) + if !reflect.DeepEqual(updateRunStatus.ApplyStrategy, updateRunCopyStatus.ApplyStrategy) { + mismatchErr := controller.NewUserError(fmt.Errorf("the applyStrategy in the updateRun is outdated, latest: %v, recorded: %v", updateRunCopyStatus.ApplyStrategy, updateRunStatus.ApplyStrategy)) + klog.ErrorS(mismatchErr, "the applyStrategy in the placement has changed", "placement", placementNamespacedName, "updateRun", updateRunRef) return -1, nil, nil, fmt.Errorf("%w: %s", errStagedUpdatedAborted, mismatchErr.Error()) } // Retrieve the latest policy snapshot. - latestPolicySnapshot, clusterCount, err := r.determinePolicySnapshot(ctx, placementName, updateRunCopy) + latestPolicySnapshot, clusterCount, err := r.determinePolicySnapshot(ctx, placementNamespacedName, updateRunCopy) if err != nil { return -1, nil, nil, err } // Make sure the latestPolicySnapshot has not changed. - if updateRun.Status.PolicySnapshotIndexUsed != updateRunCopy.Status.PolicySnapshotIndexUsed { - mismatchErr := fmt.Errorf("the policy snapshot index used in the clusterStagedUpdateRun is outdated, latest: %s, recorded: %s", updateRunCopy.Status.PolicySnapshotIndexUsed, updateRun.Status.PolicySnapshotIndexUsed) - klog.ErrorS(mismatchErr, "there's a new latest policy snapshot", "clusterResourcePlacement", placementName, "clusterStagedUpdateRun", updateRunRef) + if updateRunStatus.PolicySnapshotIndexUsed != updateRunCopyStatus.PolicySnapshotIndexUsed { + mismatchErr := fmt.Errorf("the policy snapshot index used in the updateRun is outdated, latest: %s, recorded: %s", updateRunCopyStatus.PolicySnapshotIndexUsed, updateRunStatus.PolicySnapshotIndexUsed) + klog.ErrorS(mismatchErr, "there's a new latest policy snapshot", "placement", placementNamespacedName, "updateRun", updateRunRef) return -1, nil, nil, fmt.Errorf("%w: %s", errStagedUpdatedAborted, mismatchErr.Error()) } // Make sure the cluster count in the policy snapshot has not changed. // PickAll policy case will be verified in validateStagesStatus. - if clusterCount != -1 && updateRun.Status.PolicyObservedClusterCount != clusterCount { - mismatchErr := fmt.Errorf("the cluster count initialized in the clusterStagedUpdateRun is outdated, latest: %d, recorded: %d", clusterCount, updateRun.Status.PolicyObservedClusterCount) - klog.ErrorS(mismatchErr, "the cluster count in the policy snapshot has changed", "clusterResourcePlacement", placementName, "clusterStagedUpdateRun", updateRunRef) + if clusterCount != -1 && updateRunStatus.PolicyObservedClusterCount != clusterCount { + mismatchErr := fmt.Errorf("the cluster count initialized in the updateRun is outdated, latest: %d, recorded: %d", clusterCount, updateRunStatus.PolicyObservedClusterCount) + klog.ErrorS(mismatchErr, "the cluster count in the policy snapshot has changed", "placement", placementNamespacedName, "updateRun", updateRunRef) return -1, nil, nil, fmt.Errorf("%w: %s", errStagedUpdatedAborted, mismatchErr.Error()) } - // Collect the clusters by the corresponding ClusterResourcePlacement with the latest policy snapshot. - scheduledBindings, toBeDeletedBindings, err := r.collectScheduledClusters(ctx, placementName, latestPolicySnapshot, updateRunCopy) + // Collect the clusters by the corresponding placement with the latest policy snapshot. + scheduledBindings, toBeDeletedBindings, err := r.collectScheduledClusters(ctx, placementNamespacedName, latestPolicySnapshot, updateRunCopy) if err != nil { return -1, nil, nil, err } @@ -85,27 +87,28 @@ func (r *Reconciler) validate( if err != nil { return -1, nil, nil, err } + return updatingStageIndex, scheduledBindings, toBeDeletedBindings, nil } -// validateStagesStatus validates both the update and delete stages of the ClusterStagedUpdateRun. +// validateStagesStatus validates both the update and delete stages of the UpdateRun. // The function returns the stage index that is updating, or any error encountered. -// If the updating stage index is -1, it means all stages are finished, and the clusterStageUpdateRun should be marked as finished. +// If the updating stage index is -1, it means all stages are finished, and the updateRun should be marked as finished. // If the updating stage index is 0, the next stage to be updated will be the first stage. -// If the updating stage index is len(updateRun.Status.StagesStatus), the next stage to be updated will be the delete stage. +// If the updating stage index is len(updateRun.GetUpdateRunStatus().StagesStatus), the next stage to be updated will be the delete stage. func (r *Reconciler) validateStagesStatus( ctx context.Context, - scheduledBindings, toBeDeletedBindings []*placementv1beta1.ClusterResourceBinding, - updateRun, updateRunCopy *placementv1beta1.ClusterStagedUpdateRun, + scheduledBindings, toBeDeletedBindings []placementv1beta1.BindingObj, + updateRun, updateRunCopy placementv1beta1.UpdateRunObj, ) (int, error) { updateRunRef := klog.KObj(updateRun) // Recompute the stage status which does not include the delete stage. - // Note that the compute process uses the StagedUpdateStrategySnapshot in status, + // Note that the compute process uses the UpdateStrategySnapshot in status, // so it won't affect anything if the actual updateStrategy has changed. - if updateRun.Status.StagedUpdateStrategySnapshot == nil { - unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the clusterStagedUpdateRun has nil stagedUpdateStrategySnapshot")) - klog.ErrorS(unexpectedErr, "Failed to find the stagedUpdateStrategySnapshot in the clusterStagedUpdateRun", "clusterStagedUpdateRun", updateRunRef) + if updateRun.GetUpdateRunStatus().UpdateStrategySnapshot == nil { + unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the updateRun has nil updateStrategySnapshot")) + klog.ErrorS(unexpectedErr, "Failed to find the updateStrategySnapshot in the updateRun", "updateRun", updateRunRef) return -1, fmt.Errorf("%w: %s", errStagedUpdatedAborted, unexpectedErr.Error()) } if err := r.computeRunStageStatus(ctx, scheduledBindings, updateRunCopy); err != nil { @@ -113,10 +116,10 @@ func (r *Reconciler) validateStagesStatus( } // Validate the stages in the updateRun and return the updating stage index. - existingStageStatus := updateRun.Status.StagesStatus + existingStageStatus := updateRun.GetUpdateRunStatus().StagesStatus if existingStageStatus == nil { - unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the clusterStagedUpdateRun has nil stagesStatus")) - klog.ErrorS(unexpectedErr, "Failed to find the stagesStatus in the clusterStagedUpdateRun", "clusterStagedUpdateRun", updateRunRef) + unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the updateRun has nil stagesStatus")) + klog.ErrorS(unexpectedErr, "Failed to find the stagesStatus in the updateRun", "updateRun", updateRunRef) return -1, fmt.Errorf("%w: %s", errStagedUpdatedAborted, unexpectedErr.Error()) } updatingStageIndex, lastFinishedStageIndex, validateErr := validateUpdateStagesStatus(existingStageStatus, updateRunCopy) @@ -127,37 +130,37 @@ func (r *Reconciler) validateStagesStatus( return validateDeleteStageStatus(updatingStageIndex, lastFinishedStageIndex, len(existingStageStatus), toBeDeletedBindings, updateRunCopy) } -// validateUpdateStagesStatus is a helper function to validate the updating stages in the clusterStagedUpdateRun. +// validateUpdateStagesStatus is a helper function to validate the updating stages in the updateRun. // It compares the existing stage status with the latest list of clusters to be updated. // It returns the index of the updating stage, the index of the last finished stage and any error encountered. -func validateUpdateStagesStatus(existingStageStatus []placementv1beta1.StageUpdatingStatus, updateRun *placementv1beta1.ClusterStagedUpdateRun) (int, int, error) { +func validateUpdateStagesStatus(existingStageStatus []placementv1beta1.StageUpdatingStatus, updateRun placementv1beta1.UpdateRunObj) (int, int, error) { updatingStageIndex := -1 lastFinishedStageIndex := -1 // Remember the newly computed stage status. - newStageStatus := updateRun.Status.StagesStatus - // Make sure the number of stages in the clusterStagedUpdateRun are still the same. + newStageStatus := updateRun.GetUpdateRunStatus().StagesStatus + // Make sure the number of stages in the updateRun are still the same. if len(existingStageStatus) != len(newStageStatus) { - mismatchErr := fmt.Errorf("the number of stages in the clusterStagedUpdateRun has changed, new: %d, existing: %d", len(newStageStatus), len(existingStageStatus)) - klog.ErrorS(mismatchErr, "The number of stages in the clusterStagedUpdateRun has changed", "clusterStagedUpdateRun", klog.KObj(updateRun)) + mismatchErr := fmt.Errorf("the number of stages in the updateRun has changed, new: %d, existing: %d", len(newStageStatus), len(existingStageStatus)) + klog.ErrorS(mismatchErr, "The number of stages in the updateRun has changed", "updateRun", klog.KObj(updateRun)) return -1, -1, fmt.Errorf("%w: %s", errStagedUpdatedAborted, mismatchErr.Error()) } // Make sure the stages in the updateRun are still the same. for curStage := range existingStageStatus { if existingStageStatus[curStage].StageName != newStageStatus[curStage].StageName { - mismatchErr := fmt.Errorf("index `%d` stage name in the clusterStagedUpdateRun has changed, new: %s, existing: %s", curStage, newStageStatus[curStage].StageName, existingStageStatus[curStage].StageName) - klog.ErrorS(mismatchErr, "The stage name in the clusterStagedUpdateRun has changed", "clusterStagedUpdateRun", klog.KObj(updateRun)) + mismatchErr := fmt.Errorf("index `%d` stage name in the updateRun has changed, new: %s, existing: %s", curStage, newStageStatus[curStage].StageName, existingStageStatus[curStage].StageName) + klog.ErrorS(mismatchErr, "The stage name in the updateRun has changed", "updateRun", klog.KObj(updateRun)) return -1, -1, fmt.Errorf("%w: %s", errStagedUpdatedAborted, mismatchErr.Error()) } if len(existingStageStatus[curStage].Clusters) != len(newStageStatus[curStage].Clusters) { mismatchErr := fmt.Errorf("the number of clusters in index `%d` stage has changed, new: %d, existing: %d", curStage, len(newStageStatus[curStage].Clusters), len(existingStageStatus[curStage].Clusters)) - klog.ErrorS(mismatchErr, "The number of clusters in the stage has changed", "clusterStagedUpdateRun", klog.KObj(updateRun)) + klog.ErrorS(mismatchErr, "The number of clusters in the stage has changed", "updateRun", klog.KObj(updateRun)) return -1, -1, fmt.Errorf("%w: %s", errStagedUpdatedAborted, mismatchErr.Error()) } // Check that the clusters in the stage are still the same. for j := range existingStageStatus[curStage].Clusters { if existingStageStatus[curStage].Clusters[j].ClusterName != newStageStatus[curStage].Clusters[j].ClusterName { mismatchErr := fmt.Errorf("the `%d` cluster in the `%d` stage has changed, new: %s, existing: %s", j, curStage, newStageStatus[curStage].Clusters[j].ClusterName, existingStageStatus[curStage].Clusters[j].ClusterName) - klog.ErrorS(mismatchErr, "The cluster in the stage has changed", "clusterStagedUpdateRun", klog.KObj(updateRun)) + klog.ErrorS(mismatchErr, "The cluster in the stage has changed", "updateRun", klog.KObj(updateRun)) return -1, -1, fmt.Errorf("%w: %s", errStagedUpdatedAborted, mismatchErr.Error()) } } @@ -178,16 +181,16 @@ func validateUpdateStagesStatus(existingStageStatus []placementv1beta1.StageUpda func validateClusterUpdatingStatus( curStage, updatingStageIndex, lastFinishedStageIndex int, stageStatus *placementv1beta1.StageUpdatingStatus, - updateRun *placementv1beta1.ClusterStagedUpdateRun, + updateRun placementv1beta1.UpdateRunObj, ) (int, int, error) { stageSucceedCond := meta.FindStatusCondition(stageStatus.Conditions, string(placementv1beta1.StageUpdatingConditionSucceeded)) stageStartedCond := meta.FindStatusCondition(stageStatus.Conditions, string(placementv1beta1.StageUpdatingConditionProgressing)) - if condition.IsConditionStatusTrue(stageSucceedCond, updateRun.Generation) { + if condition.IsConditionStatusTrue(stageSucceedCond, updateRun.GetGeneration()) { // The stage has finished. if updatingStageIndex != -1 && curStage > updatingStageIndex { // The finished stage is after the updating stage. unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the finished stage `%d` is after the updating stage `%d`", curStage, updatingStageIndex)) - klog.ErrorS(unexpectedErr, "The finished stage is after the updating stage", "clusterStagedUpdateRun", klog.KObj(updateRun)) + klog.ErrorS(unexpectedErr, "The finished stage is after the updating stage", "updateRun", klog.KObj(updateRun)) return -1, -1, fmt.Errorf("%w: %s", errStagedUpdatedAborted, unexpectedErr.Error()) } // Make sure that all the clusters are updated. @@ -196,38 +199,38 @@ func validateClusterUpdatingStatus( if !condition.IsConditionStatusTrue(meta.FindStatusCondition( stageStatus.Clusters[curCluster].Conditions, string(placementv1beta1.ClusterUpdatingConditionSucceeded)), - updateRun.Generation) { + updateRun.GetGeneration()) { // The clusters in the finished stage should all have finished too. unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("cluster `%s` in the finished stage `%s` has not succeeded", stageStatus.Clusters[curCluster].ClusterName, stageStatus.StageName)) - klog.ErrorS(unexpectedErr, "The cluster in a finished stage is still updating", "clusterStagedUpdateRun", klog.KObj(updateRun)) + klog.ErrorS(unexpectedErr, "The cluster in a finished stage is still updating", "updateRun", klog.KObj(updateRun)) return -1, -1, fmt.Errorf("%w: %s", errStagedUpdatedAborted, unexpectedErr.Error()) } } if curStage != lastFinishedStageIndex+1 { // The current finished stage is not right after the last finished stage. unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the finished stage `%s` is not right after the last finished stage with index `%d`", stageStatus.StageName, lastFinishedStageIndex)) - klog.ErrorS(unexpectedErr, "There's not yet started stage before the finished stage", "clusterStagedUpdateRun", klog.KObj(updateRun)) + klog.ErrorS(unexpectedErr, "There's not yet started stage before the finished stage", "updateRun", klog.KObj(updateRun)) return -1, -1, fmt.Errorf("%w: %s", errStagedUpdatedAborted, unexpectedErr.Error()) } // Record the last finished stage so we can continue from the next stage if no stage is updating. lastFinishedStageIndex = curStage - } else if condition.IsConditionStatusFalse(stageSucceedCond, updateRun.Generation) { + } else if condition.IsConditionStatusFalse(stageSucceedCond, updateRun.GetGeneration()) { // The stage has failed. failedErr := fmt.Errorf("the stage `%s` has failed, err: %s", stageStatus.StageName, stageSucceedCond.Message) - klog.ErrorS(failedErr, "The stage has failed", "stageCond", stageSucceedCond, "clusterStagedUpdateRun", klog.KObj(updateRun)) + klog.ErrorS(failedErr, "The stage has failed", "stageCond", stageSucceedCond, "updateRun", klog.KObj(updateRun)) return -1, -1, fmt.Errorf("%w: %s", errStagedUpdatedAborted, failedErr.Error()) } else if stageStartedCond != nil { // The stage is still updating. if updatingStageIndex != -1 { // There should be only one stage updating at a time. unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the stage `%s` is updating, but there is already a stage with index `%d` updating", stageStatus.StageName, updatingStageIndex)) - klog.ErrorS(unexpectedErr, "Detected more than one updating stages", "clusterStagedUpdateRun", klog.KObj(updateRun)) + klog.ErrorS(unexpectedErr, "Detected more than one updating stages", "updateRun", klog.KObj(updateRun)) return -1, -1, fmt.Errorf("%w: %s", errStagedUpdatedAborted, unexpectedErr.Error()) } if curStage != lastFinishedStageIndex+1 { // The current updating stage is not right after the last finished stage. unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the updating stage `%s` is not right after the last finished stage with index `%d`", stageStatus.StageName, lastFinishedStageIndex)) - klog.ErrorS(unexpectedErr, "There's not yet started stage before the updating stage", "clusterStagedUpdateRun", klog.KObj(updateRun)) + klog.ErrorS(unexpectedErr, "There's not yet started stage before the updating stage", "updateRun", klog.KObj(updateRun)) return -1, -1, fmt.Errorf("%w: %s", errStagedUpdatedAborted, unexpectedErr.Error()) } updatingStageIndex = curStage @@ -236,8 +239,8 @@ func validateClusterUpdatingStatus( for j := range stageStatus.Clusters { clusterStartedCond := meta.FindStatusCondition(stageStatus.Clusters[j].Conditions, string(placementv1beta1.ClusterUpdatingConditionStarted)) clusterFinishedCond := meta.FindStatusCondition(stageStatus.Clusters[j].Conditions, string(placementv1beta1.ClusterUpdatingConditionSucceeded)) - if condition.IsConditionStatusTrue(clusterStartedCond, updateRun.Generation) && - !(condition.IsConditionStatusTrue(clusterFinishedCond, updateRun.Generation) || condition.IsConditionStatusFalse(clusterFinishedCond, updateRun.Generation)) { + if condition.IsConditionStatusTrue(clusterStartedCond, updateRun.GetGeneration()) && + !(condition.IsConditionStatusTrue(clusterFinishedCond, updateRun.GetGeneration()) || condition.IsConditionStatusFalse(clusterFinishedCond, updateRun.GetGeneration())) { updatingClusters = append(updatingClusters, stageStatus.Clusters[j].ClusterName) } } @@ -245,39 +248,40 @@ func validateClusterUpdatingStatus( // TODO(wantjian): support multiple clusters updating at the same time. if len(updatingClusters) > 1 { unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("more than one cluster is updating in the stage `%s`, clusters: %v", stageStatus.StageName, updatingClusters)) - klog.ErrorS(unexpectedErr, "Detected more than one updating clusters in the stage", "clusterStagedUpdateRun", klog.KObj(updateRun)) + klog.ErrorS(unexpectedErr, "Detected more than one updating clusters in the stage", "updateRun", klog.KObj(updateRun)) return -1, -1, fmt.Errorf("%w: %s", errStagedUpdatedAborted, unexpectedErr.Error()) } } return updatingStageIndex, lastFinishedStageIndex, nil } -// validateDeleteStageStatus validates the delete stage in the clusterStagedUpdateRun. +// validateDeleteStageStatus validates the delete stage in the updateRun. // It returns the updating stage index, or any error encountered. func validateDeleteStageStatus( updatingStageIndex, lastFinishedStageIndex, totalStages int, - toBeDeletedBindings []*placementv1beta1.ClusterResourceBinding, - updateRun *placementv1beta1.ClusterStagedUpdateRun, + toBeDeletedBindings []placementv1beta1.BindingObj, + updateRun placementv1beta1.UpdateRunObj, ) (int, error) { updateRunRef := klog.KObj(updateRun) - existingDeleteStageStatus := updateRun.Status.DeletionStageStatus + existingDeleteStageStatus := updateRun.GetUpdateRunStatus().DeletionStageStatus if existingDeleteStageStatus == nil { - unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the clusterStagedUpdateRun has nil deletionStageStatus")) - klog.ErrorS(unexpectedErr, "Failed to find the deletionStageStatus in the clusterStagedUpdateRun", "clusterStagedUpdateRun", updateRunRef) + unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the updateRun has nil deletionStageStatus")) + klog.ErrorS(unexpectedErr, "Failed to find the deletionStageStatus in the updateRun", "updateRun", updateRunRef) return -1, fmt.Errorf("%w: %s", errStagedUpdatedAborted, unexpectedErr.Error()) } - // Validate whether toBeDeletedBindings are a subnet of the clusters in the delete stage status. - // We only validate if it's a subnet because we will delete the bindings during the deleteStage execution so they can disappear. + // Validate whether toBeDeletedBindings are a subset of the clusters in the delete stage status. + // We only validate if it's a subset because we will delete the bindings during the deleteStage execution so they can disappear. // We only need to check the existence, not the order, because clusters are always sorted by name in the delete stage. deletingClusterMap := make(map[string]struct{}, len(existingDeleteStageStatus.Clusters)) for _, cluster := range existingDeleteStageStatus.Clusters { deletingClusterMap[cluster.ClusterName] = struct{}{} } for _, binding := range toBeDeletedBindings { - if _, ok := deletingClusterMap[binding.Spec.TargetCluster]; !ok { - unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the cluster `%s` to be deleted is not in the delete stage", binding.Spec.TargetCluster)) - klog.ErrorS(unexpectedErr, "Detect new cluster to be unscheduled", "clusterResourceBinding", klog.KObj(binding), "clusterStagedUpdateRun", updateRunRef) + bindingSpec := binding.GetBindingSpec() + if _, ok := deletingClusterMap[bindingSpec.TargetCluster]; !ok { + unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the cluster `%s` to be deleted is not in the delete stage", bindingSpec.TargetCluster)) + klog.ErrorS(unexpectedErr, "Detect new cluster to be unscheduled", "binding", klog.KObj(binding), "updateRun", updateRunRef) return -1, fmt.Errorf("%w: %s", errStagedUpdatedAborted, unexpectedErr.Error()) } } @@ -287,11 +291,11 @@ func validateDeleteStageStatus( // Check if there is any active updating stage if updatingStageIndex != -1 || lastFinishedStageIndex < totalStages-1 { // There are still stages updating before the delete stage, make sure the delete stage is not active/finished. - if condition.IsConditionStatusTrue(deleteStageFinishedCond, updateRun.Generation) || - condition.IsConditionStatusFalse(deleteStageFinishedCond, updateRun.Generation) || - condition.IsConditionStatusTrue(deleteStageProgressingCond, updateRun.Generation) { + if condition.IsConditionStatusTrue(deleteStageFinishedCond, updateRun.GetGeneration()) || + condition.IsConditionStatusFalse(deleteStageFinishedCond, updateRun.GetGeneration()) || + condition.IsConditionStatusTrue(deleteStageProgressingCond, updateRun.GetGeneration()) { unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the delete stage is active, but there are still stages updating, updatingStageIndex: %d, lastFinishedStageIndex: %d", updatingStageIndex, lastFinishedStageIndex)) - klog.ErrorS(unexpectedErr, "the delete stage is active, but there are still stages updating", "clusterStagedUpdateRun", updateRunRef) + klog.ErrorS(unexpectedErr, "the delete stage is active, but there are still stages updating", "updateRun", updateRunRef) return -1, fmt.Errorf("%w: %s", errStagedUpdatedAborted, unexpectedErr.Error()) } @@ -303,16 +307,16 @@ func validateDeleteStageStatus( return updatingStageIndex, nil } - klog.InfoS("All stages are finished, continue from the delete stage", "clusterStagedUpdateRun", updateRunRef) + klog.InfoS("All stages are finished, continue from the delete stage", "updateRun", updateRunRef) // Check if the delete stage has finished successfully. - if condition.IsConditionStatusTrue(deleteStageFinishedCond, updateRun.Generation) { - klog.InfoS("The delete stage has finished successfully, no more stages to update", "clusterStagedUpdateRun", updateRunRef) + if condition.IsConditionStatusTrue(deleteStageFinishedCond, updateRun.GetGeneration()) { + klog.InfoS("The delete stage has finished successfully, no more stages to update", "updateRun", updateRunRef) return -1, nil } // Check if the delete stage has failed. - if condition.IsConditionStatusFalse(deleteStageFinishedCond, updateRun.Generation) { + if condition.IsConditionStatusFalse(deleteStageFinishedCond, updateRun.GetGeneration()) { failedErr := fmt.Errorf("the delete stage has failed, err: %s", deleteStageFinishedCond.Message) - klog.ErrorS(failedErr, "The delete stage has failed", "stageCond", deleteStageFinishedCond, "clusterStagedUpdateRun", updateRunRef) + klog.ErrorS(failedErr, "The delete stage has failed", "stageCond", deleteStageFinishedCond, "updateRun", updateRunRef) return -1, fmt.Errorf("%w: %s", errStagedUpdatedAborted, failedErr.Error()) } // The delete stage is still updating or just to start. diff --git a/pkg/controllers/updaterun/validation_integration_test.go b/pkg/controllers/updaterun/validation_integration_test.go index 0d40aa038..a9fb61de2 100644 --- a/pkg/controllers/updaterun/validation_integration_test.go +++ b/pkg/controllers/updaterun/validation_integration_test.go @@ -46,7 +46,7 @@ var _ = Describe("UpdateRun validation tests", func() { var unscheduledClusters []*clusterv1beta1.MemberCluster var resourceSnapshot *placementv1beta1.ClusterResourceSnapshot var clusterResourceOverride *placementv1beta1.ClusterResourceOverrideSnapshot - var wantStatus *placementv1beta1.StagedUpdateRunStatus + var wantStatus *placementv1beta1.UpdateRunStatus BeforeEach(func() { testUpdateRunName = "updaterun-" + utils.RandStr() @@ -172,7 +172,7 @@ var _ = Describe("UpdateRun validation tests", func() { By("Validating the validation failed") wantStatus = generateFailedValidationStatus(updateRun, wantStatus) - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "parent clusterResourcePlacement not found") + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "parent placement not found") }) It("Should fail to validate if CRP does not have external rollout strategy type", func() { @@ -183,7 +183,7 @@ var _ = Describe("UpdateRun validation tests", func() { By("Validating the validation failed") wantStatus = generateFailedValidationStatus(updateRun, wantStatus) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, - "parent clusterResourcePlacement does not have an external rollout strategy") + "parent placement does not have an external rollout strategy") }) It("Should fail to valdiate if the ApplyStrategy in the CRP has changed", func() { @@ -193,7 +193,7 @@ var _ = Describe("UpdateRun validation tests", func() { By("Validating the validation failed") wantStatus = generateFailedValidationStatus(updateRun, wantStatus) - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "the applyStrategy in the clusterStagedUpdateRun is outdated") + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "the applyStrategy in the updateRun is outdated") }) }) @@ -230,7 +230,7 @@ var _ = Describe("UpdateRun validation tests", func() { By("Validating the validation failed") wantStatus = generateFailedValidationStatus(updateRun, wantStatus) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, - "the policy snapshot index used in the clusterStagedUpdateRun is outdated") + "the policy snapshot index used in the updateRun is outdated") By("Deleting the new policySnapshot") Expect(k8sClient.Delete(ctx, newPolicySnapshot)).Should(Succeed()) @@ -247,7 +247,7 @@ var _ = Describe("UpdateRun validation tests", func() { By("Validating the validation failed") wantStatus = generateFailedValidationStatus(updateRun, wantStatus) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, - "the cluster count initialized in the clusterStagedUpdateRun is outdated") + "the cluster count initialized in the updateRun is outdated") By("Checking update run status metrics are emitted") validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun), generateFailedMetric(updateRun)) @@ -260,15 +260,15 @@ var _ = Describe("UpdateRun validation tests", func() { validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun), generateFailedMetric(updateRun)) }) - It("Should fail to validate if the StagedUpdateStrategySnapshot is nil", func() { - By("Updating the status.StagedUpdateStrategySnapshot to nil") - updateRun.Status.StagedUpdateStrategySnapshot = nil + It("Should fail to validate if the UpdateStrategySnapshot is nil", func() { + By("Updating the status.UpdateStrategySnapshot to nil") + updateRun.Status.UpdateStrategySnapshot = nil Expect(k8sClient.Status().Update(ctx, updateRun)).Should(Succeed()) By("Validating the validation failed") wantStatus = generateFailedValidationStatus(updateRun, wantStatus) - wantStatus.StagedUpdateStrategySnapshot = nil - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "the clusterStagedUpdateRun has nil stagedUpdateStrategySnapshot") + wantStatus.UpdateStrategySnapshot = nil + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "the updateRun has nil updateStrategySnapshot") }) It("Should fail to validate if the StagesStatus is nil", func() { @@ -279,7 +279,7 @@ var _ = Describe("UpdateRun validation tests", func() { By("Validating the validation failed") wantStatus = generateFailedValidationStatus(updateRun, wantStatus) wantStatus.StagesStatus = nil - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "the clusterStagedUpdateRun has nil stagesStatus") + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "the updateRun has nil stagesStatus") }) It("Should fail to validate if the DeletionStageStatus is nil", func() { @@ -290,12 +290,12 @@ var _ = Describe("UpdateRun validation tests", func() { By("Validating the validation failed") wantStatus = generateFailedValidationStatus(updateRun, wantStatus) wantStatus.DeletionStageStatus = nil - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "the clusterStagedUpdateRun has nil deletionStageStatus") + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "the updateRun has nil deletionStageStatus") }) It("Should fail to validate if the number of stages has changed", func() { By("Adding a stage to the updateRun status") - updateRun.Status.StagedUpdateStrategySnapshot.Stages = append(updateRun.Status.StagedUpdateStrategySnapshot.Stages, placementv1beta1.StageConfig{ + updateRun.Status.UpdateStrategySnapshot.Stages = append(updateRun.Status.UpdateStrategySnapshot.Stages, placementv1beta1.StageConfig{ Name: "stage3", LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -308,7 +308,7 @@ var _ = Describe("UpdateRun validation tests", func() { By("Validating the validation failed") wantStatus = generateFailedValidationStatus(updateRun, wantStatus) - wantStatus.StagedUpdateStrategySnapshot.Stages = append(wantStatus.StagedUpdateStrategySnapshot.Stages, placementv1beta1.StageConfig{ + wantStatus.UpdateStrategySnapshot.Stages = append(wantStatus.UpdateStrategySnapshot.Stages, placementv1beta1.StageConfig{ Name: "stage3", LabelSelector: &metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -317,18 +317,18 @@ var _ = Describe("UpdateRun validation tests", func() { }, }, }) - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "the number of stages in the clusterStagedUpdateRun has changed") + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "the number of stages in the updateRun has changed") }) It("Should fail to validate if stage name has changed", func() { By("Changing the name of a stage") - updateRun.Status.StagedUpdateStrategySnapshot.Stages[0].Name = "stage3" + updateRun.Status.UpdateStrategySnapshot.Stages[0].Name = "stage3" Expect(k8sClient.Status().Update(ctx, updateRun)).Should(Succeed()) By("Validating the validation failed") wantStatus = generateFailedValidationStatus(updateRun, wantStatus) - wantStatus.StagedUpdateStrategySnapshot.Stages[0].Name = "stage3" - validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "index `0` stage name in the clusterStagedUpdateRun has changed") + wantStatus.UpdateStrategySnapshot.Stages[0].Name = "stage3" + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "index `0` stage name in the updateRun has changed") }) It("Should fail to validate if the number of clusters has changed in a stage", func() { @@ -365,7 +365,7 @@ var _ = Describe("UpdateRun validation tests", func() { var unscheduledClusters []*clusterv1beta1.MemberCluster var resourceSnapshot *placementv1beta1.ClusterResourceSnapshot var clusterResourceOverride *placementv1beta1.ClusterResourceOverrideSnapshot - var wantStatus *placementv1beta1.StagedUpdateRunStatus + var wantStatus *placementv1beta1.UpdateRunStatus BeforeEach(func() { testUpdateRunName = "updaterun-" + utils.RandStr() testCRPName = "crp-" + utils.RandStr() @@ -506,7 +506,7 @@ var _ = Describe("UpdateRun validation tests", func() { func validateClusterStagedUpdateRunStatus( ctx context.Context, updateRun *placementv1beta1.ClusterStagedUpdateRun, - want *placementv1beta1.StagedUpdateRunStatus, + want *placementv1beta1.UpdateRunStatus, message string, ) { Eventually(func() error { @@ -530,7 +530,7 @@ func validateClusterStagedUpdateRunStatus( func validateClusterStagedUpdateRunStatusConsistently( ctx context.Context, updateRun *placementv1beta1.ClusterStagedUpdateRun, - want *placementv1beta1.StagedUpdateRunStatus, + want *placementv1beta1.UpdateRunStatus, message string, ) { Consistently(func() error { @@ -553,8 +553,8 @@ func validateClusterStagedUpdateRunStatusConsistently( func generateFailedValidationStatus( updateRun *placementv1beta1.ClusterStagedUpdateRun, - started *placementv1beta1.StagedUpdateRunStatus, -) *placementv1beta1.StagedUpdateRunStatus { + started *placementv1beta1.UpdateRunStatus, +) *placementv1beta1.UpdateRunStatus { started.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, false) started.Conditions = append(started.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) return started diff --git a/pkg/controllers/updaterun/validation_test.go b/pkg/controllers/updaterun/validation_test.go index b75ea2297..a2d6e1315 100644 --- a/pkg/controllers/updaterun/validation_test.go +++ b/pkg/controllers/updaterun/validation_test.go @@ -244,7 +244,7 @@ func TestValidateDeleteStageStatus(t *testing.T) { name string updatingStageIndex int lastFinishedStageIndex int - toBeDeletedBindings []*placementv1beta1.ClusterResourceBinding + toBeDeletedBindings []placementv1beta1.BindingObj deleteStageStatus *placementv1beta1.StageUpdatingStatus wantErr error wantUpdatingStageIndex int @@ -252,17 +252,17 @@ func TestValidateDeleteStageStatus(t *testing.T) { { name: "validateDeleteStageStatus should return error if delete stage status is nil", deleteStageStatus: nil, - wantErr: wrapErr(true, fmt.Errorf("the clusterStagedUpdateRun has nil deletionStageStatus")), + wantErr: wrapErr(true, fmt.Errorf("the updateRun has nil deletionStageStatus")), wantUpdatingStageIndex: -1, }, { name: "validateDeleteStageStatus should return error if there's new to-be-deleted bindings", - toBeDeletedBindings: []*placementv1beta1.ClusterResourceBinding{ - { + toBeDeletedBindings: []placementv1beta1.BindingObj{ + &placementv1beta1.ClusterResourceBinding{ ObjectMeta: metav1.ObjectMeta{Name: "binding-1"}, Spec: placementv1beta1.ResourceBindingSpec{TargetCluster: "cluster-1"}, }, - { + &placementv1beta1.ClusterResourceBinding{ ObjectMeta: metav1.ObjectMeta{Name: "binding-2"}, Spec: placementv1beta1.ResourceBindingSpec{TargetCluster: "cluster-2"}, }, @@ -280,8 +280,8 @@ func TestValidateDeleteStageStatus(t *testing.T) { name: "validateDeleteStageStatus should not return error if there's fewer to-be-deleted bindings", updatingStageIndex: -1, lastFinishedStageIndex: -1, - toBeDeletedBindings: []*placementv1beta1.ClusterResourceBinding{ - { + toBeDeletedBindings: []placementv1beta1.BindingObj{ + &placementv1beta1.ClusterResourceBinding{ ObjectMeta: metav1.ObjectMeta{Name: "binding-1"}, Spec: placementv1beta1.ResourceBindingSpec{TargetCluster: "cluster-1"}, }, @@ -300,8 +300,8 @@ func TestValidateDeleteStageStatus(t *testing.T) { name: "validateDeleteStageStatus should not return error if there are equal to-be-deleted bindings", updatingStageIndex: -1, lastFinishedStageIndex: -1, - toBeDeletedBindings: []*placementv1beta1.ClusterResourceBinding{ - { + toBeDeletedBindings: []placementv1beta1.BindingObj{ + &placementv1beta1.ClusterResourceBinding{ ObjectMeta: metav1.ObjectMeta{Name: "binding-1"}, Spec: placementv1beta1.ResourceBindingSpec{TargetCluster: "cluster-1"}, }, diff --git a/pkg/controllers/workapplier/controller_test.go b/pkg/controllers/workapplier/controller_test.go index 27b0b32b4..2dadd9722 100644 --- a/pkg/controllers/workapplier/controller_test.go +++ b/pkg/controllers/workapplier/controller_test.go @@ -34,10 +34,8 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes/scheme" "k8s.io/utils/ptr" - ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" - "go.goms.io/fleet/pkg/metrics" ) const ( @@ -261,12 +259,6 @@ func TestMain(m *testing.M) { // Initialize the variables. initializeVariables() - // Register the metrics. - ctrlmetrics.Registry.MustRegister( - metrics.FleetWorkProcessingRequestsTotal, - metrics.FleetManifestProcessingRequestsTotal, - ) - os.Exit(m.Run()) } diff --git a/pkg/controllers/workapplier/metrics.go b/pkg/controllers/workapplier/metrics.go index f3126e4ca..c046d799a 100644 --- a/pkg/controllers/workapplier/metrics.go +++ b/pkg/controllers/workapplier/metrics.go @@ -24,7 +24,7 @@ import ( "k8s.io/klog/v2" fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" - "go.goms.io/fleet/pkg/metrics" + membermetrics "go.goms.io/fleet/pkg/metrics/member" "go.goms.io/fleet/pkg/utils/controller" ) @@ -123,7 +123,7 @@ func trackWorkAndManifestProcessingRequestMetrics(work *fleetv1beta1.Work) { return } - metrics.FleetWorkProcessingRequestsTotal.WithLabelValues( + membermetrics.FleetWorkProcessingRequestsTotal.WithLabelValues( workApplyStatus, workAvailabilityStatus, workDiffReportedStatus, @@ -200,7 +200,7 @@ func trackWorkAndManifestProcessingRequestMetrics(work *fleetv1beta1.Work) { manifestDiffDetectionStatus = manifestDriftOrDiffDetectionStatusFound } - metrics.FleetManifestProcessingRequestsTotal.WithLabelValues( + membermetrics.FleetManifestProcessingRequestsTotal.WithLabelValues( manifestApplyStatus, manifestAvailabilityStatus, manifestDiffReportedStatus, diff --git a/pkg/controllers/workapplier/metrics_test.go b/pkg/controllers/workapplier/metrics_test.go index 4ffcd14c4..f2d6eb7ab 100644 --- a/pkg/controllers/workapplier/metrics_test.go +++ b/pkg/controllers/workapplier/metrics_test.go @@ -24,7 +24,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" - "go.goms.io/fleet/pkg/metrics" + membermetrics "go.goms.io/fleet/pkg/metrics/member" "go.goms.io/fleet/pkg/utils/condition" ) @@ -522,23 +522,23 @@ func TestTrackWorkAndManifestProcessingRequestMetrics(t *testing.T) { trackWorkAndManifestProcessingRequestMetrics(tc.work) // Collect the metrics. - if c := testutil.CollectAndCount(metrics.FleetWorkProcessingRequestsTotal); c != tc.wantWorkMetricCount { + if c := testutil.CollectAndCount(membermetrics.FleetWorkProcessingRequestsTotal); c != tc.wantWorkMetricCount { t.Fatalf("unexpected work metric count: got %d, want %d", c, tc.wantWorkMetricCount) } if err := testutil.CollectAndCompare( - metrics.FleetWorkProcessingRequestsTotal, + membermetrics.FleetWorkProcessingRequestsTotal, strings.NewReader(workMetricMetadata+tc.wantWorkCounter), ); err != nil { t.Fatalf("unexpected work counter value:\n%v", err) } - if c := testutil.CollectAndCount(metrics.FleetManifestProcessingRequestsTotal); c != tc.wantManifestMetricCount { + if c := testutil.CollectAndCount(membermetrics.FleetManifestProcessingRequestsTotal); c != tc.wantManifestMetricCount { t.Fatalf("unexpected manifest metric count: got %d, want %d", c, tc.wantManifestMetricCount) } if err := testutil.CollectAndCompare( - metrics.FleetManifestProcessingRequestsTotal, + membermetrics.FleetManifestProcessingRequestsTotal, strings.NewReader(manifestMetricMetadata+tc.wantManifestCounter), ); err != nil { t.Fatalf("unexpected manifest counter value:\n%v", err) diff --git a/pkg/controllers/workapplier/process.go b/pkg/controllers/workapplier/process.go index daaaedb2e..9c437e6c0 100644 --- a/pkg/controllers/workapplier/process.go +++ b/pkg/controllers/workapplier/process.go @@ -229,7 +229,7 @@ func (r *Reconciler) takeOverInMemberClusterObjectIfApplicable( ) (shouldSkipProcessing bool) { if !shouldInitiateTakeOverAttempt(bundle.inMemberClusterObj, work.Spec.ApplyStrategy, expectedAppliedWorkOwnerRef) { // Takeover is not necessary; proceed with the processing. - klog.V(2).InfoS("Takeover is not needed; skip the step") + klog.V(2).InfoS("Takeover is not needed; skip the step", "work", klog.KObj(work), "GVR", *bundle.gvr, "manifestObj", bundle.workResourceIdentifierStr) return false } diff --git a/pkg/controllers/workv1alpha1/apply_controller.go b/pkg/controllers/workv1alpha1/apply_controller.go index e909dfac1..e73a51ddf 100644 --- a/pkg/controllers/workv1alpha1/apply_controller.go +++ b/pkg/controllers/workv1alpha1/apply_controller.go @@ -60,7 +60,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" workv1alpha1 "sigs.k8s.io/work-api/pkg/apis/v1alpha1" - "go.goms.io/fleet/pkg/metrics" + membermetrics "go.goms.io/fleet/pkg/metrics/member" "go.goms.io/fleet/pkg/utils" ) @@ -171,7 +171,7 @@ func (r *ApplyWorkReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( klog.ErrorS(parseErr, "failed to parse the last work update time", "work", logObjRef) } else { latency := time.Since(workUpdateTime) - metrics.WorkApplyTime.WithLabelValues(work.GetName()).Observe(latency.Seconds()) + membermetrics.WorkApplyTime.WithLabelValues(work.GetName()).Observe(latency.Seconds()) klog.V(2).InfoS("work is applied", "work", work.GetName(), "latency", latency.Milliseconds()) } } else { diff --git a/pkg/metrics/hub/metrics.go b/pkg/metrics/hub/metrics.go new file mode 100644 index 000000000..c61f85ff0 --- /dev/null +++ b/pkg/metrics/hub/metrics.go @@ -0,0 +1,92 @@ +/* +Copyright 2025 The KubeFleet Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package hub contains metrics exclusively emitted by the hub-agent. +package hub + +import ( + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +var ( + // These 2 metrics are used in v1alpha1 controller, should be soon deprecated. + PlacementApplyFailedCount = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "placement_apply_failed_counter", + Help: "Number of failed to apply cluster resource placement", + }, []string{"name"}) + PlacementApplySucceedCount = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "placement_apply_succeed_counter", + Help: "Number of successfully applied cluster resource placement", + }, []string{"name"}) + + // FleetPlacementStatusLastTimeStampSeconds is a prometheus metric which keeps track of the last placement status. + FleetPlacementStatusLastTimeStampSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "fleet_workload_placement_status_last_timestamp_seconds", + Help: "Last update timestamp of placement status in seconds", + }, []string{"namespace", "name", "generation", "conditionType", "status", "reason"}) + + // FleetEvictionStatus is prometheus metrics which holds the + // status of eviction completion. + FleetEvictionStatus = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "fleet_workload_eviction_complete", + Help: "Last update timestamp of eviction complete status in seconds", + }, []string{"name", "isCompleted", "isValid"}) + + // FleetUpdateRunStatusLastTimestampSeconds is a prometheus metric which holds the + // last update timestamp of update run status in seconds. + FleetUpdateRunStatusLastTimestampSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "fleet_workload_update_run_status_last_timestamp_seconds", + Help: "Last update timestamp of update run status in seconds", + }, []string{"namespace", "name", "generation", "condition", "status", "reason"}) +) + +// The scheduler related metrics. +var ( + // SchedulingCycleDurationMilliseconds is a Fleet scheduler metric that tracks how long it + // takes to complete a scheduling loop run. + SchedulingCycleDurationMilliseconds = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "scheduling_cycle_duration_milliseconds", + Help: "The duration of a scheduling cycle run in milliseconds", + Buckets: []float64{ + 10, 50, 100, 500, 1000, 5000, 10000, 50000, + }, + }, + []string{ + "is_failed", + "needs_requeue", + }, + ) + + // SchedulerActiveWorkers is a prometheus metric which holds the number of active scheduler loop. + SchedulerActiveWorkers = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "scheduling_active_workers", + Help: "Number of currently running scheduling loop", + }, []string{}) +) + +func init() { + metrics.Registry.MustRegister( + PlacementApplyFailedCount, + PlacementApplySucceedCount, + FleetPlacementStatusLastTimeStampSeconds, + FleetEvictionStatus, + FleetUpdateRunStatusLastTimestampSeconds, + SchedulingCycleDurationMilliseconds, + SchedulerActiveWorkers, + ) +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/member/metrics.go similarity index 55% rename from pkg/metrics/metrics.go rename to pkg/metrics/member/metrics.go index 8e4756781..41b34dde3 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/member/metrics.go @@ -14,97 +14,22 @@ See the License for the specific language governing permissions and limitations under the License. */ -package metrics +// Package member contains metrics exclusively emitted by the member-agent. +package member import ( "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" ) var ( - JoinResultMetrics = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "join_result_counter", - Help: "Number of successful Join operations", - }, []string{"result"}) - LeaveResultMetrics = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "leave_result_counter", - Help: "Number of successful Leave operations", - }, []string{"result"}) + // This metric is used in v1alpha1 controller, should be soon deprecated. WorkApplyTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{ Name: "work_apply_time_seconds", Help: "Length of time between when a work resource is created/updated to when it is applied on the member cluster", Buckets: []float64{0.01, 0.025, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.4, 0.5, 0.7, 0.9, 1.0, 1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, 7, 9, 10, 15, 20, 30, 60, 120}, }, []string{"name"}) - PlacementApplyFailedCount = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "placement_apply_failed_counter", - Help: "Number of failed to apply cluster resource placement", - }, []string{"name"}) - PlacementApplySucceedCount = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "placement_apply_succeed_counter", - Help: "Number of successfully applied cluster resource placement", - }, []string{"name"}) - - // FleetPlacementStatusLastTimeStampSeconds is a prometheus metric which keeps track of the last placement status. - FleetPlacementStatusLastTimeStampSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "fleet_workload_placement_status_last_timestamp_seconds", - Help: "Last update timestamp of placement status in seconds", - }, []string{"namespace", "name", "generation", "conditionType", "status", "reason"}) - - // FleetEvictionStatus is prometheus metrics which holds the - // status of eviction completion. - FleetEvictionStatus = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "fleet_workload_eviction_complete", - Help: "Last update timestamp of eviction complete status in seconds", - }, []string{"name", "isCompleted", "isValid"}) - - // FleetUpdateRunStatusLastTimestampSeconds is a prometheus metric which holds the - // last update timestamp of update run status in seconds. - FleetUpdateRunStatusLastTimestampSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "fleet_workload_update_run_status_last_timestamp_seconds", - Help: "Last update timestamp of update run status in seconds", - }, []string{"name", "generation", "condition", "status", "reason"}) -) - -var ( - ReportJoinResultMetric = func() { - JoinResultMetrics.With(prometheus.Labels{ - // Per team agreement, the failure result won't be reported from the agents as k8s controller would retry - // failed reconciliations. - "result": "success", - }).Inc() - } - ReportLeaveResultMetric = func() { - LeaveResultMetrics.With(prometheus.Labels{ - // Per team agreement, the failure result won't be reported from the agents as k8s controller would retry - // failed reconciliations. - "result": "success", - }).Inc() - } -) - -// The scheduler related metrics. -var ( - // SchedulingCycleDurationMilliseconds is a Fleet scheduler metric that tracks how long it - // takes to complete a scheduling loop run. - SchedulingCycleDurationMilliseconds = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "scheduling_cycle_duration_milliseconds", - Help: "The duration of a scheduling cycle run in milliseconds", - Buckets: []float64{ - 10, 50, 100, 500, 1000, 5000, 10000, 50000, - }, - }, - []string{ - "is_failed", - "needs_requeue", - }, - ) - - // SchedulerActiveWorkers is a prometheus metric which holds the number of active scheduler loop. - SchedulerActiveWorkers = prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: "scheduling_active_workers", - Help: "Number of currently running scheduling loop", - }, []string{}) ) // The work applier related metrics. @@ -154,3 +79,11 @@ var ( Help: "Total number of processing requests of manifest objects, including retries and periodic checks", }, []string{"apply_status", "availability_status", "diff_reporting_status", "drift_detection_status", "diff_detection_status"}) ) + +func init() { + metrics.Registry.MustRegister( + WorkApplyTime, + FleetWorkProcessingRequestsTotal, + FleetManifestProcessingRequestsTotal, + ) +} diff --git a/pkg/metrics/shared/metrics.go b/pkg/metrics/shared/metrics.go new file mode 100644 index 000000000..fdcd5a1f2 --- /dev/null +++ b/pkg/metrics/shared/metrics.go @@ -0,0 +1,58 @@ +/* +Copyright 2025 The KubeFleet Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package shared contains metrics emitted by both the hub-agent and member-agent. +package shared + +import ( + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +var ( + JoinResultMetrics = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "join_result_counter", + Help: "Number of successful Join operations", + }, []string{"result"}) + LeaveResultMetrics = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "leave_result_counter", + Help: "Number of successful Leave operations", + }, []string{"result"}) +) + +var ( + ReportJoinResultMetric = func() { + JoinResultMetrics.With(prometheus.Labels{ + // Per team agreement, the failure result won't be reported from the agents as k8s controller would retry + // failed reconciliations. + "result": "success", + }).Inc() + } + ReportLeaveResultMetric = func() { + LeaveResultMetrics.With(prometheus.Labels{ + // Per team agreement, the failure result won't be reported from the agents as k8s controller would retry + // failed reconciliations. + "result": "success", + }).Inc() + } +) + +func init() { + metrics.Registry.MustRegister( + JoinResultMetrics, + LeaveResultMetrics, + ) +} diff --git a/pkg/propertyprovider/azure/provider.go b/pkg/propertyprovider/azure/provider.go index d13f289e6..196556351 100644 --- a/pkg/propertyprovider/azure/provider.go +++ b/pkg/propertyprovider/azure/provider.go @@ -53,6 +53,8 @@ const ( // a Kubernetes cluster. PerGBMemoryCostProperty = "kubernetes.azure.com/per-gb-memory-cost" + NodeCountPerSKUPropertyTmpl = "kubernetes.azure.com/vm-size/%s/count" + CostPrecisionTemplate = "%.3f" ) @@ -284,13 +286,10 @@ func (p *PropertyProvider) Collect(ctx context.Context) propertyprovider.Propert conds := make([]metav1.Condition, 0, 1) // Collect the non-resource properties. - - // Collect the node count property. properties := make(map[clusterv1beta1.PropertyName]clusterv1beta1.PropertyValue) - properties[propertyprovider.NodeCountProperty] = clusterv1beta1.PropertyValue{ - Value: fmt.Sprintf("%d", p.nodeTracker.NodeCount()), - ObservationTime: metav1.Now(), - } + + // Collect node-count related properties. + p.collectNodeCountRelatedProperties(ctx, properties) // Collect the cost properties (if enabled). if p.isCostCollectionEnabled { @@ -318,6 +317,27 @@ func (p *PropertyProvider) Collect(ctx context.Context) propertyprovider.Propert } } +// collectNodeCountRelatedProperties collects the node-count related properties. +func (p *PropertyProvider) collectNodeCountRelatedProperties(_ context.Context, properties map[clusterv1beta1.PropertyName]clusterv1beta1.PropertyValue) { + now := metav1.Now() + + // Collect the total node count as a property. + properties[propertyprovider.NodeCountProperty] = clusterv1beta1.PropertyValue{ + Value: fmt.Sprintf("%d", p.nodeTracker.NodeCount()), + ObservationTime: now, + } + + // Collect the per-SKU node counts as properties. + nodeCountPerSKU := p.nodeTracker.NodeCountPerSKU() + for sku, count := range nodeCountPerSKU { + pName := fmt.Sprintf(NodeCountPerSKUPropertyTmpl, sku) + properties[clusterv1beta1.PropertyName(pName)] = clusterv1beta1.PropertyValue{ + Value: fmt.Sprintf("%d", count), + ObservationTime: now, + } + } +} + // collectCosts collects the cost information. func (p *PropertyProvider) collectCosts(_ context.Context, properties map[clusterv1beta1.PropertyName]clusterv1beta1.PropertyValue) []metav1.Condition { conds := make([]metav1.Condition, 0, 1) diff --git a/pkg/propertyprovider/azure/provider_integration_test.go b/pkg/propertyprovider/azure/provider_integration_test.go index 7b168df4d..e7cd687dd 100644 --- a/pkg/propertyprovider/azure/provider_integration_test.go +++ b/pkg/propertyprovider/azure/provider_integration_test.go @@ -187,6 +187,16 @@ var ( isPricingDataStale = true } + nodeCountPerSKU := map[string]int{} + for idx := range nodes { + node := nodes[idx] + sku := node.Labels[trackers.AKSClusterNodeSKULabelName] + if sku == "" { + sku = trackers.ReservedNameForUndefinedSKU + } + nodeCountPerSKU[sku]++ + } + for idx := range nodes { node := nodes[idx] sku := node.Labels[trackers.AKSClusterNodeSKULabelName] @@ -230,6 +240,11 @@ var ( Value: fmt.Sprintf("%d", len(nodes)), }, } + for sku, count := range nodeCountPerSKU { + wantProperties[clusterv1beta1.PropertyName(fmt.Sprintf(NodeCountPerSKUPropertyTmpl, sku))] = clusterv1beta1.PropertyValue{ + Value: fmt.Sprintf("%d", count), + } + } if costCollectionErr == nil && isCostsEnabled { wantProperties[PerCPUCoreCostProperty] = clusterv1beta1.PropertyValue{ @@ -305,11 +320,8 @@ var ( ) var ( - // The nodes below use actual capacities of their respective AKS SKUs; for more information, - // see: - // https://learn.microsoft.com/en-us/azure/virtual-machines/av2-series (for A-series nodes), - // https://learn.microsoft.com/en-us/azure/virtual-machines/sizes-b-series-burstable (for B-series nodes), and - // https://learn.microsoft.com/en-us/azure/virtual-machines/dv2-dsv2-series (for D/DS v2 series nodes). + // The nodes below use actual capacities of their respective AKS SKUs; We need to change them periodically as + // they will be deprecated over time. nodes = []corev1.Node{ { ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/propertyprovider/azure/provider_test.go b/pkg/propertyprovider/azure/provider_test.go index f5618842e..ce8df9295 100644 --- a/pkg/propertyprovider/azure/provider_test.go +++ b/pkg/propertyprovider/azure/provider_test.go @@ -220,6 +220,12 @@ func TestCollect(t *testing.T) { propertyprovider.NodeCountProperty: { Value: "2", }, + clusterv1beta1.PropertyName(fmt.Sprintf(NodeCountPerSKUPropertyTmpl, nodeSKU1)): { + Value: "1", + }, + clusterv1beta1.PropertyName(fmt.Sprintf(NodeCountPerSKUPropertyTmpl, nodeSKU2)): { + Value: "1", + }, PerCPUCoreCostProperty: { Value: "0.167", }, @@ -311,6 +317,12 @@ func TestCollect(t *testing.T) { propertyprovider.NodeCountProperty: { Value: "2", }, + clusterv1beta1.PropertyName(fmt.Sprintf(NodeCountPerSKUPropertyTmpl, nodeSKU1)): { + Value: "1", + }, + clusterv1beta1.PropertyName(fmt.Sprintf(NodeCountPerSKUPropertyTmpl, nodeSKU2)): { + Value: "1", + }, PerCPUCoreCostProperty: { Value: "0.167", }, @@ -391,6 +403,9 @@ func TestCollect(t *testing.T) { propertyprovider.NodeCountProperty: { Value: "2", }, + clusterv1beta1.PropertyName(fmt.Sprintf(NodeCountPerSKUPropertyTmpl, nodeSKU3)): { + Value: "2", + }, }, Resources: clusterv1beta1.ResourceUsage{ Capacity: corev1.ResourceList{ @@ -467,6 +482,12 @@ func TestCollect(t *testing.T) { propertyprovider.NodeCountProperty: { Value: "2", }, + clusterv1beta1.PropertyName(fmt.Sprintf(NodeCountPerSKUPropertyTmpl, nodeSKU1)): { + Value: "1", + }, + clusterv1beta1.PropertyName(fmt.Sprintf(NodeCountPerSKUPropertyTmpl, nodeSKU3)): { + Value: "1", + }, }, Resources: clusterv1beta1.ResourceUsage{ Capacity: corev1.ResourceList{ @@ -540,6 +561,12 @@ func TestCollect(t *testing.T) { propertyprovider.NodeCountProperty: { Value: "2", }, + clusterv1beta1.PropertyName(fmt.Sprintf(NodeCountPerSKUPropertyTmpl, nodeSKU1)): { + Value: "1", + }, + clusterv1beta1.PropertyName(fmt.Sprintf(NodeCountPerSKUPropertyTmpl, trackers.ReservedNameForUndefinedSKU)): { + Value: "1", + }, }, Resources: clusterv1beta1.ResourceUsage{ Capacity: corev1.ResourceList{ @@ -616,6 +643,12 @@ func TestCollect(t *testing.T) { propertyprovider.NodeCountProperty: { Value: "2", }, + clusterv1beta1.PropertyName(fmt.Sprintf(NodeCountPerSKUPropertyTmpl, nodeSKU1)): { + Value: "1", + }, + clusterv1beta1.PropertyName(fmt.Sprintf(NodeCountPerSKUPropertyTmpl, nodeKnownMissingSKU)): { + Value: "1", + }, }, Resources: clusterv1beta1.ResourceUsage{ Capacity: corev1.ResourceList{ @@ -653,6 +686,12 @@ func TestCollect(t *testing.T) { propertyprovider.NodeCountProperty: { Value: "2", }, + clusterv1beta1.PropertyName(fmt.Sprintf(NodeCountPerSKUPropertyTmpl, nodeSKU1)): { + Value: "1", + }, + clusterv1beta1.PropertyName(fmt.Sprintf(NodeCountPerSKUPropertyTmpl, nodeSKU2)): { + Value: "1", + }, PerCPUCoreCostProperty: { Value: "0.167", }, @@ -791,6 +830,9 @@ func TestCollectWithDisabledFeatures(t *testing.T) { propertyprovider.NodeCountProperty: { Value: "1", }, + clusterv1beta1.PropertyName(fmt.Sprintf(NodeCountPerSKUPropertyTmpl, nodeSKU1)): { + Value: "1", + }, }, Resources: clusterv1beta1.ResourceUsage{ Capacity: corev1.ResourceList{ @@ -819,6 +861,9 @@ func TestCollectWithDisabledFeatures(t *testing.T) { propertyprovider.NodeCountProperty: { Value: "1", }, + clusterv1beta1.PropertyName(fmt.Sprintf(NodeCountPerSKUPropertyTmpl, nodeSKU1)): { + Value: "1", + }, PerCPUCoreCostProperty: { Value: "0.250", }, @@ -856,6 +901,9 @@ func TestCollectWithDisabledFeatures(t *testing.T) { propertyprovider.NodeCountProperty: { Value: "1", }, + clusterv1beta1.PropertyName(fmt.Sprintf(NodeCountPerSKUPropertyTmpl, nodeSKU1)): { + Value: "1", + }, }, Resources: clusterv1beta1.ResourceUsage{ Capacity: corev1.ResourceList{ diff --git a/pkg/propertyprovider/azure/suite_test.go b/pkg/propertyprovider/azure/suite_test.go index 84c674730..871234d61 100644 --- a/pkg/propertyprovider/azure/suite_test.go +++ b/pkg/propertyprovider/azure/suite_test.go @@ -43,10 +43,10 @@ const ( const ( region = "eastus" - - aksNodeSKU1 = "Standard_B4ms" - aksNodeSKU2 = "Standard_A4_v2" - aksNodeSKU3 = "Standard_F4s" + // we need to regularly update the SKUs used in tests as some of them get deprecated over time. + aksNodeSKU1 = "Standard_D8s_v5" + aksNodeSKU2 = "Standard_E16_v5" + aksNodeSKU3 = "Standard_M16ms" // Note (chenyu1): cross-reference between the Azure VM SKU list and the Azure Retail Prices API // for a list of currently known SKUs to be missing from the Azure Retail Prices API. aksNodeKnownMissingSKU1 = "Standard_DS2_v2" diff --git a/pkg/propertyprovider/azure/trackers/nodes.go b/pkg/propertyprovider/azure/trackers/nodes.go index 2fa01d1cf..7060db591 100644 --- a/pkg/propertyprovider/azure/trackers/nodes.go +++ b/pkg/propertyprovider/azure/trackers/nodes.go @@ -37,6 +37,8 @@ const ( // AKSClusterNodeSKULabelName is the node label added by AKS, which indicated the SKU // of the node. AKSClusterNodeSKULabelName = "beta.kubernetes.io/instance-type" + + ReservedNameForUndefinedSKU = "undefined" ) const ( @@ -327,7 +329,7 @@ func (nt *NodeTracker) trackSKU(node *corev1.Node) bool { return true default: // No further action is needed if the node's SKU remains the same. - klog.V(2).InfoS("The node's SKU has not changed", "sku", sku, "node", klog.KObj(node)) + klog.V(3).InfoS("The node's SKU has not changed", "sku", sku, "node", klog.KObj(node)) return false } } @@ -583,3 +585,20 @@ func (nt *NodeTracker) Costs() (perCPUCoreCost, perGBMemoryCost float64, warning } return nt.costs.perCPUCoreHourlyCost, nt.costs.perGBMemoryHourlyCost, nt.costs.warnings, nt.costs.err } + +// NodeCountPerSKU returns a counter that tracks the number of nodes per SKU in the cluster. +func (nt *NodeTracker) NodeCountPerSKU() map[string]int { + nt.mu.RLock() + defer nt.mu.RUnlock() + + // Return a copy to avoid leaks/unexpected edits. + res := make(map[string]int, len(nt.nodeSetBySKU)) + for sku, ns := range nt.nodeSetBySKU { + // For those nodes without a SKU, use `undefined` as the SKU name. + if len(sku) == 0 { + sku = ReservedNameForUndefinedSKU + } + res[sku] = len(ns) + } + return res +} diff --git a/pkg/propertyprovider/azure/trackers/trackers_test.go b/pkg/propertyprovider/azure/trackers/trackers_test.go index 942781a41..d1ef7bb91 100644 --- a/pkg/propertyprovider/azure/trackers/trackers_test.go +++ b/pkg/propertyprovider/azure/trackers/trackers_test.go @@ -152,6 +152,20 @@ var ( corev1.ResourceMemory: resource.MustParse("1.8Gi"), }, }, + nodeSetBySKU: map[string]NodeSet{ + nodeSKU1: { + nodeName1: true, + nodeName3: true, + }, + nodeSKU2: { + nodeName2: true, + }, + }, + skuByNode: map[string]string{ + nodeName1: nodeSKU1, + nodeName2: nodeSKU2, + nodeName3: nodeSKU1, + }, pricingProvider: &dummyPricingProvider{}, } @@ -2213,6 +2227,48 @@ func TestNodeTrackerCosts(t *testing.T) { } } +func TestNodeTrackerNodeCountPerSKU(t *testing.T) { + testCases := []struct { + name string + nt *NodeTracker + wantCounter map[string]int + }{ + { + name: "can return the counter (all nodes have SKUs assigned)", + nt: nodeTrackerWith3Nodes, + wantCounter: map[string]int{ + nodeSKU1: 2, + nodeSKU2: 1, + }, + }, + { + name: "can return the counter (with undefined SKUs)", + nt: &NodeTracker{ + nodeSetBySKU: map[string]NodeSet{ + nodeSKU1: {nodeName1: true}, + "": { + nodeName2: true, + nodeName3: true, + }, + }, + }, + wantCounter: map[string]int{ + nodeSKU1: 1, + ReservedNameForUndefinedSKU: 2, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + counter := tc.nt.NodeCountPerSKU() + if diff := cmp.Diff(counter, tc.wantCounter, cmpopts.EquateEmpty()); diff != "" { + t.Fatalf("NodeCountPerSKU() diff (-got, +want):\n%s", diff) + } + }) + } +} + // TestNodeTrackerAddOrUpdate tests the AddOrUpdate method of the PodTracker. func TestPodTrackerAddOrUpdate(t *testing.T) { testCases := []struct { diff --git a/pkg/scheduler/framework/framework.go b/pkg/scheduler/framework/framework.go index 10bb5a601..025970fa6 100644 --- a/pkg/scheduler/framework/framework.go +++ b/pkg/scheduler/framework/framework.go @@ -22,6 +22,7 @@ import ( "context" "fmt" "sort" + "strings" "sync/atomic" "time" @@ -71,6 +72,9 @@ const ( // The array length limit of the cluster decision array in the scheduling policy snapshot // status API. clustersDecisionArrayLengthLimitInAPI = 1000 + + // maxClusterInfoForDebugging controls the maximum number of cluster information entries to include in the scheduler debugging output + maxClusterInfoForDebugging = 32 ) // Handle is an interface which allows plugins to access some shared structs (e.g., client, manager) @@ -461,7 +465,7 @@ func (f *framework) runSchedulingCycleForPickAllPlacementType( // longer picked in the current run. // // Fields in the returned bindings are fulfilled and/or refreshed as applicable. - klog.V(2).InfoS("Cross-referencing bindings with picked clusters", "policySnapshot", policyRef) + klog.V(2).InfoS("Cross-referencing bindings with picked clusters", "policySnapshot", policyRef, "scoredClusters", scored, "filteredClusters", filtered) toCreate, toDelete, toPatch, err := crossReferencePickedClustersAndDeDupBindings(placementKey, policy, scored, unscheduled, obsolete) if err != nil { klog.ErrorS(err, "Failed to cross-reference bindings with picked clusters", "policySnapshot", policyRef) @@ -603,8 +607,22 @@ type filteredClusterWithStatus struct { status *Status } +// helper type to pretty print a list of filteredClusterWithStatus +type filteredClusterWithStatusList []*filteredClusterWithStatus + +func (cs filteredClusterWithStatusList) String() string { + if len(cs) > maxClusterInfoForDebugging { + cs = cs[:maxClusterInfoForDebugging] // limit the number of entries to print + } + filteredClusters := make([]string, 0, len(cs)) + for _, fc := range cs { + filteredClusters = append(filteredClusters, fmt.Sprintf("{cluster: %s, status: %s}", fc.cluster.Name, fc.status)) + } + return fmt.Sprintf("filteredClusters[%s]", strings.Join(filteredClusters, ", ")) +} + // runFilterPlugins runs filter plugins on clusters in parallel. -func (f *framework) runFilterPlugins(ctx context.Context, state *CycleState, policy placementv1beta1.PolicySnapshotObj, clusters []clusterv1beta1.MemberCluster) (passed []*clusterv1beta1.MemberCluster, filtered []*filteredClusterWithStatus, err error) { +func (f *framework) runFilterPlugins(ctx context.Context, state *CycleState, policy placementv1beta1.PolicySnapshotObj, clusters []clusterv1beta1.MemberCluster) (passed []*clusterv1beta1.MemberCluster, filtered filteredClusterWithStatusList, err error) { // Create a child context. childCtx, cancel := context.WithCancel(ctx) @@ -906,7 +924,7 @@ func (f *framework) runSchedulingCycleForPickNPlacementType( } // Pick the top scored clusters. - klog.V(2).InfoS("Picking clusters", "policySnapshot", policyRef) + klog.V(2).InfoS("Picking clusters", "policySnapshot", policyRef, "filtered", filtered, "scored", scored) // Calculate the number of clusters to pick. numOfClustersToPick := calcNumOfClustersToSelect(state.desiredBatchSize, state.batchSizeLimit, len(scored)) @@ -936,7 +954,7 @@ func (f *framework) runSchedulingCycleForPickNPlacementType( // longer picked in the current run. // // Fields in the returned bindings are fulfilled and/or refreshed as applicable. - klog.V(2).InfoS("Cross-referencing bindings with picked clusters", "policySnapshot", policyRef, "numOfClustersToPick", numOfClustersToPick) + klog.V(2).InfoS("Cross-referencing bindings with picked clusters", "policySnapshot", policyRef, "numOfClustersToPick", numOfClustersToPick, "picked", picked, "notPicked", notPicked) toCreate, toDelete, toPatch, err := crossReferencePickedClustersAndDeDupBindings(placementKey, policy, picked, unscheduled, obsolete) if err != nil { klog.ErrorS(err, "Failed to cross-reference bindings with picked clusters", "policySnapshot", policyRef) diff --git a/pkg/scheduler/framework/framework_test.go b/pkg/scheduler/framework/framework_test.go index bab8a0574..83973e854 100644 --- a/pkg/scheduler/framework/framework_test.go +++ b/pkg/scheduler/framework/framework_test.go @@ -871,7 +871,7 @@ func TestRunFilterPlugins(t *testing.T) { name string filterPlugins []FilterPlugin wantClusters []*clusterv1beta1.MemberCluster - wantFiltered []*filteredClusterWithStatus + wantFiltered filteredClusterWithStatusList expectedToFail bool }{ { @@ -907,7 +907,7 @@ func TestRunFilterPlugins(t *testing.T) { }, }, }, - wantFiltered: []*filteredClusterWithStatus{}, + wantFiltered: filteredClusterWithStatusList{}, }, { name: "three clusters, two filter plugins, two filtered", @@ -938,7 +938,7 @@ func TestRunFilterPlugins(t *testing.T) { }, }, }, - wantFiltered: []*filteredClusterWithStatus{ + wantFiltered: filteredClusterWithStatusList{ { cluster: &clusterv1beta1.MemberCluster{ ObjectMeta: metav1.ObjectMeta{ @@ -986,7 +986,7 @@ func TestRunFilterPlugins(t *testing.T) { }, }, }, - wantFiltered: []*filteredClusterWithStatus{}, + wantFiltered: filteredClusterWithStatusList{}, }, { name: "three clusters, two filter plugins, one success, one internal error on specific cluster", diff --git a/pkg/scheduler/framework/score.go b/pkg/scheduler/framework/score.go index 1f776359e..6745bd3c3 100644 --- a/pkg/scheduler/framework/score.go +++ b/pkg/scheduler/framework/score.go @@ -17,6 +17,9 @@ limitations under the License. package framework import ( + "fmt" + "strings" + clusterv1beta1 "go.goms.io/fleet/apis/cluster/v1beta1" ) @@ -111,3 +114,15 @@ func (sc ScoredClusters) Less(i, j int) bool { func (sc ScoredClusters) Swap(i, j int) { sc[i], sc[j] = sc[j], sc[i] } + +func (sc ScoredClusters) String() string { + if len(sc) > maxClusterInfoForDebugging { + sc = sc[:maxClusterInfoForDebugging] // limit the number of entries to print + } + + names := make([]string, 0, len(sc)) + for _, s := range sc { + names = append(names, fmt.Sprintf("Cluster{Name: %s, Score: %v}", s.Cluster.Name, s.Score)) + } + return fmt.Sprintf("ScoredClusters{%s}", strings.Join(names, ", ")) +} diff --git a/pkg/scheduler/framework/score_test.go b/pkg/scheduler/framework/score_test.go index c0dfbd4fa..90511059f 100644 --- a/pkg/scheduler/framework/score_test.go +++ b/pkg/scheduler/framework/score_test.go @@ -17,7 +17,9 @@ limitations under the License. package framework import ( + "fmt" "sort" + "strings" "testing" "github.com/google/go-cmp/cmp" @@ -538,3 +540,117 @@ func TestScoredClustersSort(t *testing.T) { }) } } + +func TestScoredClustersString(t *testing.T) { + clusterA := &clusterv1beta1.MemberCluster{ObjectMeta: metav1.ObjectMeta{Name: "cluster-a"}} + clusterB := &clusterv1beta1.MemberCluster{ObjectMeta: metav1.ObjectMeta{Name: "cluster-b"}} + clusterC := &clusterv1beta1.MemberCluster{ObjectMeta: metav1.ObjectMeta{Name: "cluster-c"}} + + testCases := []struct { + name string + scs ScoredClusters + expected string + }{ + { + name: "empty slice", + scs: ScoredClusters{}, + expected: "ScoredClusters{}", + }, + { + name: "single cluster", + scs: ScoredClusters{ + { + Cluster: clusterA, + Score: &ClusterScore{ + TopologySpreadScore: 1, + AffinityScore: 2, + ObsoletePlacementAffinityScore: 0, + }, + }, + }, + expected: "ScoredClusters{Cluster{Name: cluster-a, Score: &{1 2 0}}}", + }, + { + name: "multiple clusters", + scs: ScoredClusters{ + { + Cluster: clusterA, + Score: &ClusterScore{ + TopologySpreadScore: 100, + AffinityScore: 50, + ObsoletePlacementAffinityScore: 1, + }, + }, + { + Cluster: clusterB, + Score: &ClusterScore{ + TopologySpreadScore: 0, + AffinityScore: 0, + ObsoletePlacementAffinityScore: 0, + }, + }, + { + Cluster: clusterC, + Score: &ClusterScore{ + TopologySpreadScore: -10, + AffinityScore: -5, + ObsoletePlacementAffinityScore: 0, + }, + }, + }, + expected: "ScoredClusters{Cluster{Name: cluster-a, Score: &{100 50 1}}, Cluster{Name: cluster-b, Score: &{0 0 0}}, Cluster{Name: cluster-c, Score: &{-10 -5 0}}}", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got := tc.scs.String() + if got != tc.expected { + t.Fatalf("String() = %q, want %q", got, tc.expected) + } + }) + } +} + +func TestScoredClustersStringTruncation(t *testing.T) { + var scs ScoredClusters + var i int32 + for i = 0; i < maxClusterInfoForDebugging+5; i++ { + cluster := &clusterv1beta1.MemberCluster{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("cluster-%02d", i)}} + scs = append(scs, &ScoredCluster{ + Cluster: cluster, + Score: &ClusterScore{ + TopologySpreadScore: i, + AffinityScore: i * 2, + ObsoletePlacementAffinityScore: int(i) % 2, + }, + }) + } + + output := scs.String() + + if got, want := strings.Count(output, "Cluster{Name:"), maxClusterInfoForDebugging; got != want { + t.Fatalf("String() truncated cluster count = %d, want %d", got, want) + } + + if !strings.HasPrefix(output, "ScoredClusters{") { + t.Fatalf("String() prefix mismatch: %q", output) + } + + if !strings.HasSuffix(output, "}") { + t.Fatalf("String() suffix mismatch: %q", output) + } + + if !strings.Contains(output, "cluster-00") { + t.Fatalf("String() missing first cluster: %q", output) + } + + lastIncluded := fmt.Sprintf("cluster-%02d", maxClusterInfoForDebugging-1) + if !strings.Contains(output, lastIncluded) { + t.Fatalf("String() missing last retained cluster %q: %q", lastIncluded, output) + } + + if strings.Contains(output, fmt.Sprintf("cluster-%02d", maxClusterInfoForDebugging)) { + t.Fatalf("String() should not contain overflow cluster: %q", output) + } +} diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 3ad1d685d..edf0423a9 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -36,7 +36,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" - "go.goms.io/fleet/pkg/metrics" + hubmetrics "go.goms.io/fleet/pkg/metrics/hub" "go.goms.io/fleet/pkg/scheduler/framework" "go.goms.io/fleet/pkg/scheduler/queue" "go.goms.io/fleet/pkg/utils/controller" @@ -120,8 +120,8 @@ func (s *Scheduler) scheduleOnce(ctx context.Context, worker int) { }() // keep track of the number of active scheduling loop - metrics.SchedulerActiveWorkers.WithLabelValues().Add(1) - defer metrics.SchedulerActiveWorkers.WithLabelValues().Add(-1) + hubmetrics.SchedulerActiveWorkers.WithLabelValues().Add(1) + defer hubmetrics.SchedulerActiveWorkers.WithLabelValues().Add(-1) startTime := time.Now() klog.V(2).InfoS("Schedule once started", "placement", placementKey, "worker", worker) @@ -415,7 +415,7 @@ func (s *Scheduler) addSchedulerCleanUpFinalizer(ctx context.Context, placement // observeSchedulingCycleMetrics adds a data point to the scheduling cycle duration metric. func observeSchedulingCycleMetrics(startTime time.Time, isFailed, needsRequeue bool) { - metrics.SchedulingCycleDurationMilliseconds. + hubmetrics.SchedulingCycleDurationMilliseconds. WithLabelValues(fmt.Sprintf("%t", isFailed), fmt.Sprintf("%t", needsRequeue)). Observe(float64(time.Since(startTime).Milliseconds())) } diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go index 9af4ebdaa..acbf8a482 100644 --- a/pkg/scheduler/scheduler_test.go +++ b/pkg/scheduler/scheduler_test.go @@ -33,7 +33,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" fleetv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" - "go.goms.io/fleet/pkg/metrics" + hubmetrics "go.goms.io/fleet/pkg/metrics/hub" ) const ( @@ -627,11 +627,11 @@ func TestObserveSchedulingCycleMetrics(t *testing.T) { t.Run(tc.name, func(t *testing.T) { observeSchedulingCycleMetrics(tc.cycleStartTime, false, false) - if c := testutil.CollectAndCount(metrics.SchedulingCycleDurationMilliseconds); c != tc.wantMetricCount { + if c := testutil.CollectAndCount(hubmetrics.SchedulingCycleDurationMilliseconds); c != tc.wantMetricCount { t.Fatalf("metric counts, got %d, want %d", c, tc.wantMetricCount) } - if err := testutil.CollectAndCompare(metrics.SchedulingCycleDurationMilliseconds, strings.NewReader(metricMetadata+tc.wantHistogram)); err != nil { + if err := testutil.CollectAndCompare(hubmetrics.SchedulingCycleDurationMilliseconds, strings.NewReader(metricMetadata+tc.wantHistogram)); err != nil { t.Errorf("%s", err) } }) diff --git a/pkg/utils/controller/placement_resolver.go b/pkg/utils/controller/placement_resolver.go index 1c6fb60c7..992ed114a 100644 --- a/pkg/utils/controller/placement_resolver.go +++ b/pkg/utils/controller/placement_resolver.go @@ -42,20 +42,22 @@ func FetchPlacementFromKey(ctx context.Context, c client.Reader, placementKey qu if err != nil { return nil, err } + return FetchPlacementFromNamespacedName(ctx, c, types.NamespacedName{Namespace: namespace, Name: name}) +} + +// FetchPlacementFromNamespacedName resolves a NamespacedName to a concrete placement object that implements PlacementObj. +func FetchPlacementFromNamespacedName(ctx context.Context, c client.Reader, nn types.NamespacedName) (fleetv1beta1.PlacementObj, error) { // Check if the key contains a namespace separator var placement fleetv1beta1.PlacementObj - if namespace != "" { + if nn.Namespace != "" { // This is a namespaced ResourcePlacement placement = &fleetv1beta1.ResourcePlacement{} } else { // This is a cluster-scoped ClusterResourcePlacement placement = &fleetv1beta1.ClusterResourcePlacement{} } - key := types.NamespacedName{ - Namespace: namespace, - Name: name, - } - if err := c.Get(ctx, key, placement); err != nil { + + if err := c.Get(ctx, nn, placement); err != nil { return nil, err } return placement, nil @@ -94,6 +96,14 @@ func GetObjectKeyFromNamespaceName(namespace, name string) string { } } +// GetNamespacedNameFromObject generates a NamespacedName from a meta object. +func GetNamespacedNameFromObject(obj metav1.Object) types.NamespacedName { + return types.NamespacedName{ + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + } +} + // ExtractNamespaceNameFromKey resolves a PlacementKey to a (namespace, name) tuple of the placement object. func ExtractNamespaceNameFromKey(key queue.PlacementKey) (string, string, error) { return ExtractNamespaceNameFromKeyStr(string(key)) diff --git a/pkg/utils/controller/strategy_resolver.go b/pkg/utils/controller/strategy_resolver.go new file mode 100644 index 000000000..a55a12080 --- /dev/null +++ b/pkg/utils/controller/strategy_resolver.go @@ -0,0 +1,45 @@ +/* +Copyright 2025 The KubeFleet Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" +) + +// FetchUpdateStrategyFromNamespacedName resolves a NamespacedName to a concrete staged update strategy object that implements UpdateStrategyObj. +// If Namespace is empty, it fetches a ClusterStagedUpdateStrategy (cluster-scoped). +// If Namespace is not empty, it fetches a StagedUpdateStrategy (namespace-scoped). +func FetchUpdateStrategyFromNamespacedName(ctx context.Context, c client.Reader, strategyKey types.NamespacedName) (placementv1beta1.UpdateStrategyObj, error) { + var strategy placementv1beta1.UpdateStrategyObj + // If Namespace is empty, it's a cluster-scoped strategy (ClusterStagedUpdateStrategy) + if strategyKey.Namespace == "" { + strategy = &placementv1beta1.ClusterStagedUpdateStrategy{} + } else { + // Otherwise, it's a namespaced strategy (StagedUpdateStrategy) + strategy = &placementv1beta1.StagedUpdateStrategy{} + } + err := c.Get(ctx, strategyKey, strategy) + if err != nil { + return nil, err + } + return strategy, nil +} diff --git a/pkg/utils/controller/strategy_resolver_test.go b/pkg/utils/controller/strategy_resolver_test.go new file mode 100644 index 000000000..8720ae4fd --- /dev/null +++ b/pkg/utils/controller/strategy_resolver_test.go @@ -0,0 +1,357 @@ +/* +Copyright 2025 The KubeFleet Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "errors" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" +) + +func TestFetchUpdateStrategyFromNamespacedName(t *testing.T) { + ctx := context.Background() + + tests := []struct { + name string + strategyKey types.NamespacedName + objects []client.Object + wantErr bool + wantStrategy placementv1beta1.UpdateStrategyObj + }{ + { + name: "cluster-scoped strategy key - ClusterStagedUpdateStrategy found", + strategyKey: types.NamespacedName{Name: "test-strategy"}, + objects: []client.Object{ + &placementv1beta1.ClusterStagedUpdateStrategy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-strategy", + }, + Spec: placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: "stage-1", + }, + }, + }, + }, + }, + wantErr: false, + wantStrategy: &placementv1beta1.ClusterStagedUpdateStrategy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-strategy", + }, + Spec: placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: "stage-1", + }, + }, + }, + }, + }, + { + name: "cluster-scoped strategy key - ClusterStagedUpdateStrategy not found", + strategyKey: types.NamespacedName{Name: "test-strategy"}, + objects: []client.Object{ + &placementv1beta1.ClusterStagedUpdateStrategy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "other-strategy", + }, + Spec: placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: "stage-1", + }, + }, + }, + }, + &placementv1beta1.StagedUpdateStrategy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-strategy", + Namespace: "test-namespace", + }, + Spec: placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: "stage-1", + }, + }, + }, + }, + }, + wantErr: true, + wantStrategy: nil, + }, + { + name: "namespaced strategy key - StagedUpdateStrategy found", + strategyKey: types.NamespacedName{Namespace: "test-namespace", Name: "test-strategy"}, + objects: []client.Object{ + &placementv1beta1.StagedUpdateStrategy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-strategy", + Namespace: "test-namespace", + }, + Spec: placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: "stage-1", + }, + }, + }, + }, + &placementv1beta1.StagedUpdateStrategy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-strategy", + Namespace: "other-namespace", + }, + Spec: placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: "stage-2", + }, + }, + }, + }, + &placementv1beta1.StagedUpdateStrategy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "other-strategy", + Namespace: "test-namespace", + }, + Spec: placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: "stage-3", + }, + }, + }, + }, + }, + wantErr: false, + wantStrategy: &placementv1beta1.StagedUpdateStrategy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-strategy", + Namespace: "test-namespace", + }, + Spec: placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: "stage-1", + }, + }, + }, + }, + }, + { + name: "namespaced strategy key - StagedUpdateStrategy not found", + strategyKey: types.NamespacedName{Namespace: "test-namespace", Name: "test-strategy"}, + objects: []client.Object{ + &placementv1beta1.ClusterStagedUpdateStrategy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-strategy", + }, + Spec: placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: "stage-1", + }, + }, + }, + }, + &placementv1beta1.StagedUpdateStrategy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-strategy", + Namespace: "other-namespace", + }, + Spec: placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: "stage-1", + }, + }, + }, + }, + &placementv1beta1.StagedUpdateStrategy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "other-strategy", + Namespace: "test-namespace", + }, + Spec: placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: "stage-2", + }, + }, + }, + }, + }, + wantErr: true, + wantStrategy: nil, + }, + { + name: "mixed setup - fetch cluster-scoped strategy", + strategyKey: types.NamespacedName{Name: "test-strategy"}, + objects: []client.Object{ + &placementv1beta1.ClusterStagedUpdateStrategy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-strategy", + }, + Spec: placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: "cluster-stage-1", + }, + }, + }, + }, + &placementv1beta1.StagedUpdateStrategy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-strategy", + Namespace: "test-namespace", + }, + Spec: placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: "namespaced-stage-1", + }, + }, + }, + }, + }, + wantErr: false, + wantStrategy: &placementv1beta1.ClusterStagedUpdateStrategy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-strategy", + }, + Spec: placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: "cluster-stage-1", + }, + }, + }, + }, + }, + { + name: "mixed setup - fetch namespaced strategy", + strategyKey: types.NamespacedName{Namespace: "test-namespace", Name: "test-strategy"}, + objects: []client.Object{ + &placementv1beta1.ClusterStagedUpdateStrategy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-strategy", + }, + Spec: placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: "cluster-stage-1", + }, + }, + }, + }, + &placementv1beta1.StagedUpdateStrategy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-strategy", + Namespace: "test-namespace", + }, + Spec: placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: "namespaced-stage-1", + }, + }, + }, + }, + }, + wantErr: false, + wantStrategy: &placementv1beta1.StagedUpdateStrategy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-strategy", + Namespace: "test-namespace", + }, + Spec: placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: "namespaced-stage-1", + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := runtime.NewScheme() + _ = placementv1beta1.AddToScheme(scheme) + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(tt.objects...). + Build() + + got, err := FetchUpdateStrategyFromNamespacedName(ctx, fakeClient, tt.strategyKey) + + if (err != nil) != tt.wantErr { + t.Fatalf("FetchUpdateStrategyFromNamespacedName() error = %v, wantErr %v", err, tt.wantErr) + } + + if diff := cmp.Diff(tt.wantStrategy, got, cmpopts.IgnoreFields(metav1.ObjectMeta{}, "ResourceVersion")); diff != "" { + t.Errorf("FetchUpdateStrategyFromNamespacedName() mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestFetchUpdateStrategyFromNamespacedName_ClientError(t *testing.T) { + ctx := context.Background() + + // Create a client that will return an error. + scheme := runtime.NewScheme() + _ = placementv1beta1.AddToScheme(scheme) + + testError := errors.New("test client error") + + // Use interceptor to make Get calls fail. + fakeClient := interceptor.NewClient( + fake.NewClientBuilder().WithScheme(scheme).Build(), + interceptor.Funcs{ + Get: func(ctx context.Context, client client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + return testError + }, + }, + ) + + _, err := FetchUpdateStrategyFromNamespacedName(ctx, fakeClient, types.NamespacedName{Name: "test-strategy"}) + + if err == nil { + t.Fatalf("FetchUpdateStrategyFromNamespacedName() expected error but got nil") + } + + if !errors.Is(err, testError) { + t.Fatalf("FetchUpdateStrategyFromNamespacedName() expected testError but got %v", err) + } +} diff --git a/pkg/utils/controller/updaterun_resolver.go b/pkg/utils/controller/updaterun_resolver.go new file mode 100644 index 000000000..174523821 --- /dev/null +++ b/pkg/utils/controller/updaterun_resolver.go @@ -0,0 +1,44 @@ +/* +Copyright 2025 The KubeFleet Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" +) + +// FetchUpdateRunFromRequest resolves a controller runtime request to a concrete update run object that implements UpdateRunObj. +func FetchUpdateRunFromRequest(ctx context.Context, c client.Reader, req ctrl.Request) (placementv1beta1.UpdateRunObj, error) { + var updateRun placementv1beta1.UpdateRunObj + if req.NamespacedName.Namespace != "" { + // This is a namespaced StagedUpdateRun + updateRun = &placementv1beta1.StagedUpdateRun{} + } else { + // This is a cluster-scoped ClusterStagedUpdateRun + updateRun = &placementv1beta1.ClusterStagedUpdateRun{} + } + + if err := c.Get(ctx, types.NamespacedName{Namespace: req.NamespacedName.Namespace, Name: req.NamespacedName.Name}, updateRun); err != nil { + return nil, err + } + return updateRun, nil +} diff --git a/pkg/utils/controller/updaterun_resolver_test.go b/pkg/utils/controller/updaterun_resolver_test.go new file mode 100644 index 000000000..c663e78af --- /dev/null +++ b/pkg/utils/controller/updaterun_resolver_test.go @@ -0,0 +1,358 @@ +/* +Copyright 2025 The KubeFleet Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "errors" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" +) + +func TestFetchUpdateRunFromRequest(t *testing.T) { + ctx := context.Background() + + tests := []struct { + name string + request ctrl.Request + objects []client.Object + wantErr bool + wantRun placementv1beta1.UpdateRunObj + }{ + { + name: "cluster-scoped request - ClusterStagedUpdateRun found", + request: ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "test-run"}, + }, + objects: []client.Object{ + &placementv1beta1.ClusterStagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-run", + }, + Spec: placementv1beta1.UpdateRunSpec{ + PlacementName: "test-placement", + ResourceSnapshotIndex: "snapshot-1", + StagedUpdateStrategyName: "test-strategy", + }, + }, + }, + wantErr: false, + wantRun: &placementv1beta1.ClusterStagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-run", + }, + Spec: placementv1beta1.UpdateRunSpec{ + PlacementName: "test-placement", + ResourceSnapshotIndex: "snapshot-1", + StagedUpdateStrategyName: "test-strategy", + }, + }, + }, + { + name: "cluster-scoped request - ClusterStagedUpdateRun not found", + request: ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "test-run"}, + }, + objects: []client.Object{ + &placementv1beta1.ClusterStagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "other-run", + }, + Spec: placementv1beta1.UpdateRunSpec{ + PlacementName: "test-placement", + ResourceSnapshotIndex: "snapshot-1", + StagedUpdateStrategyName: "test-strategy", + }, + }, + &placementv1beta1.StagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-run", + Namespace: "test-namespace", + }, + Spec: placementv1beta1.UpdateRunSpec{ + PlacementName: "test-placement", + ResourceSnapshotIndex: "snapshot-1", + StagedUpdateStrategyName: "test-strategy", + }, + }, + }, + wantErr: true, + wantRun: nil, + }, + { + name: "namespaced request - StagedUpdateRun found", + request: ctrl.Request{ + NamespacedName: types.NamespacedName{Namespace: "test-namespace", Name: "test-run"}, + }, + objects: []client.Object{ + &placementv1beta1.StagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-run", + Namespace: "test-namespace", + }, + Spec: placementv1beta1.UpdateRunSpec{ + PlacementName: "test-placement", + ResourceSnapshotIndex: "snapshot-1", + StagedUpdateStrategyName: "test-strategy", + }, + }, + &placementv1beta1.StagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-run", + Namespace: "other-namespace", + }, + Spec: placementv1beta1.UpdateRunSpec{ + PlacementName: "other-placement", + ResourceSnapshotIndex: "snapshot-2", + StagedUpdateStrategyName: "other-strategy", + }, + }, + &placementv1beta1.StagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "other-run", + Namespace: "test-namespace", + }, + Spec: placementv1beta1.UpdateRunSpec{ + PlacementName: "test-placement", + ResourceSnapshotIndex: "snapshot-1", + StagedUpdateStrategyName: "test-strategy", + }, + }, + }, + wantErr: false, + wantRun: &placementv1beta1.StagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-run", + Namespace: "test-namespace", + }, + Spec: placementv1beta1.UpdateRunSpec{ + PlacementName: "test-placement", + ResourceSnapshotIndex: "snapshot-1", + StagedUpdateStrategyName: "test-strategy", + }, + }, + }, + { + name: "namespaced request - StagedUpdateRun not found", + request: ctrl.Request{ + NamespacedName: types.NamespacedName{Namespace: "test-namespace", Name: "test-run"}, + }, + objects: []client.Object{ + &placementv1beta1.ClusterStagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-run", + }, + Spec: placementv1beta1.UpdateRunSpec{ + PlacementName: "test-placement", + ResourceSnapshotIndex: "snapshot-1", + StagedUpdateStrategyName: "test-strategy", + }, + }, + &placementv1beta1.StagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-run", + Namespace: "other-namespace", + }, + Spec: placementv1beta1.UpdateRunSpec{ + PlacementName: "test-placement", + ResourceSnapshotIndex: "snapshot-1", + StagedUpdateStrategyName: "test-strategy", + }, + }, + &placementv1beta1.StagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "other-run", + Namespace: "test-namespace", + }, + Spec: placementv1beta1.UpdateRunSpec{ + PlacementName: "test-placement", + ResourceSnapshotIndex: "snapshot-1", + StagedUpdateStrategyName: "test-strategy", + }, + }, + }, + wantErr: true, + wantRun: nil, + }, + { + name: "mixed setup - fetch cluster-scoped run", + request: ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "test-run"}, + }, + objects: []client.Object{ + &placementv1beta1.ClusterStagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-run", + }, + Spec: placementv1beta1.UpdateRunSpec{ + PlacementName: "cluster-placement", + ResourceSnapshotIndex: "cluster-snapshot-1", + StagedUpdateStrategyName: "cluster-strategy", + }, + }, + &placementv1beta1.StagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-run", + Namespace: "test-namespace", + }, + Spec: placementv1beta1.UpdateRunSpec{ + PlacementName: "namespaced-placement", + ResourceSnapshotIndex: "namespaced-snapshot-1", + StagedUpdateStrategyName: "namespaced-strategy", + }, + }, + }, + wantErr: false, + wantRun: &placementv1beta1.ClusterStagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-run", + }, + Spec: placementv1beta1.UpdateRunSpec{ + PlacementName: "cluster-placement", + ResourceSnapshotIndex: "cluster-snapshot-1", + StagedUpdateStrategyName: "cluster-strategy", + }, + }, + }, + { + name: "mixed setup - fetch namespaced run", + request: ctrl.Request{ + NamespacedName: types.NamespacedName{Namespace: "test-namespace", Name: "test-run"}, + }, + objects: []client.Object{ + &placementv1beta1.ClusterStagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-run", + }, + Spec: placementv1beta1.UpdateRunSpec{ + PlacementName: "cluster-placement", + ResourceSnapshotIndex: "cluster-snapshot-1", + StagedUpdateStrategyName: "cluster-strategy", + }, + }, + &placementv1beta1.StagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-run", + Namespace: "test-namespace", + }, + Spec: placementv1beta1.UpdateRunSpec{ + PlacementName: "namespaced-placement", + ResourceSnapshotIndex: "namespaced-snapshot-1", + StagedUpdateStrategyName: "namespaced-strategy", + }, + }, + }, + wantErr: false, + wantRun: &placementv1beta1.StagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-run", + Namespace: "test-namespace", + }, + Spec: placementv1beta1.UpdateRunSpec{ + PlacementName: "namespaced-placement", + ResourceSnapshotIndex: "namespaced-snapshot-1", + StagedUpdateStrategyName: "namespaced-strategy", + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := runtime.NewScheme() + _ = placementv1beta1.AddToScheme(scheme) + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(tt.objects...). + Build() + + got, err := FetchUpdateRunFromRequest(ctx, fakeClient, tt.request) + + if (err != nil) != tt.wantErr { + t.Fatalf("FetchUpdateRunFromRequest() error = %v, wantErr %v", err, tt.wantErr) + } + + if diff := cmp.Diff(tt.wantRun, got, cmpopts.IgnoreFields(metav1.ObjectMeta{}, "ResourceVersion")); diff != "" { + t.Errorf("FetchUpdateRunFromRequest() mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestFetchUpdateRunFromRequest_ClientError(t *testing.T) { + ctx := context.Background() + + // Create a client that will return an error. + scheme := runtime.NewScheme() + _ = placementv1beta1.AddToScheme(scheme) + + testError := errors.New("test client error") + + // Use interceptor to make Get calls fail. + fakeClient := interceptor.NewClient( + fake.NewClientBuilder().WithScheme(scheme).Build(), + interceptor.Funcs{ + Get: func(ctx context.Context, client client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + return testError + }, + }, + ) + + tests := []struct { + name string + request ctrl.Request + }{ + { + name: "cluster-scoped request with client error", + request: ctrl.Request{ + NamespacedName: types.NamespacedName{Name: "test-run"}, + }, + }, + { + name: "namespaced request with client error", + request: ctrl.Request{ + NamespacedName: types.NamespacedName{Namespace: "test-namespace", Name: "test-run"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := FetchUpdateRunFromRequest(ctx, fakeClient, tt.request) + + if err == nil { + t.Fatalf("FetchUpdateRunFromRequest() want error but got nil") + } + + if !errors.Is(err, testError) { + t.Fatalf("FetchUpdateRunFromRequest() want testError but got %v", err) + } + }) + } +} diff --git a/test/apis/placement/v1beta1/api_validation_integration_test.go b/test/apis/placement/v1beta1/api_validation_integration_test.go index f6ca7a495..dbc30847d 100644 --- a/test/apis/placement/v1beta1/api_validation_integration_test.go +++ b/test/apis/placement/v1beta1/api_validation_integration_test.go @@ -1097,7 +1097,7 @@ var _ = Describe("Test placement v1beta1 API validation", func() { ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf(validupdateRunNameTemplate, GinkgoParallelProcess()), }, - Spec: placementv1beta1.StagedUpdateRunSpec{ + Spec: placementv1beta1.UpdateRunSpec{ PlacementName: "test-placement", }, } @@ -1118,7 +1118,7 @@ var _ = Describe("Test placement v1beta1 API validation", func() { ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf(updateRunStrategyNameTemplate, GinkgoParallelProcess()), }, - Spec: placementv1beta1.StagedUpdateStrategySpec{ + Spec: placementv1beta1.UpdateStrategySpec{ Stages: []placementv1beta1.StageConfig{ { Name: fmt.Sprintf(updateRunStageNameTemplate, GinkgoParallelProcess(), 1), @@ -1165,7 +1165,7 @@ var _ = Describe("Test placement v1beta1 API validation", func() { ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf(updateRunStrategyNameTemplate, GinkgoParallelProcess()), }, - Spec: placementv1beta1.StagedUpdateStrategySpec{ + Spec: placementv1beta1.UpdateStrategySpec{ Stages: []placementv1beta1.StageConfig{ { Name: fmt.Sprintf(invalidupdateRunStageNameTemplate, GinkgoParallelProcess(), 1), @@ -1184,7 +1184,7 @@ var _ = Describe("Test placement v1beta1 API validation", func() { ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf(updateRunStrategyNameTemplate, GinkgoParallelProcess()), }, - Spec: placementv1beta1.StagedUpdateStrategySpec{ + Spec: placementv1beta1.UpdateStrategySpec{ Stages: []placementv1beta1.StageConfig{ { Name: fmt.Sprintf(updateRunStageNameTemplate, GinkgoParallelProcess(), 1) + "-A", @@ -1203,7 +1203,7 @@ var _ = Describe("Test placement v1beta1 API validation", func() { ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf(updateRunStrategyNameTemplate, GinkgoParallelProcess()), }, - Spec: placementv1beta1.StagedUpdateStrategySpec{ + Spec: placementv1beta1.UpdateStrategySpec{ Stages: []placementv1beta1.StageConfig{ { Name: fmt.Sprintf(updateRunStageNameTemplate, GinkgoParallelProcess(), 1), @@ -1234,7 +1234,7 @@ var _ = Describe("Test placement v1beta1 API validation", func() { ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf(updateRunStrategyNameTemplate, GinkgoParallelProcess()), }, - Spec: placementv1beta1.StagedUpdateStrategySpec{ + Spec: placementv1beta1.UpdateStrategySpec{ Stages: []placementv1beta1.StageConfig{ { Name: fmt.Sprintf(updateRunStageNameTemplate, GinkgoParallelProcess(), 1), @@ -1263,7 +1263,7 @@ var _ = Describe("Test placement v1beta1 API validation", func() { ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf(updateRunStrategyNameTemplate, GinkgoParallelProcess()), }, - Spec: placementv1beta1.StagedUpdateStrategySpec{ + Spec: placementv1beta1.UpdateStrategySpec{ Stages: []placementv1beta1.StageConfig{ { Name: fmt.Sprintf(updateRunStageNameTemplate, GinkgoParallelProcess(), 1), @@ -1292,7 +1292,7 @@ var _ = Describe("Test placement v1beta1 API validation", func() { ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf(updateRunStrategyNameTemplate, GinkgoParallelProcess()), }, - Spec: placementv1beta1.StagedUpdateStrategySpec{ + Spec: placementv1beta1.UpdateStrategySpec{ Stages: []placementv1beta1.StageConfig{ { Name: fmt.Sprintf(updateRunStageNameTemplate, GinkgoParallelProcess(), 1), @@ -1316,7 +1316,7 @@ var _ = Describe("Test placement v1beta1 API validation", func() { ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf(updateRunStrategyNameTemplate, GinkgoParallelProcess()), }, - Spec: placementv1beta1.StagedUpdateStrategySpec{ + Spec: placementv1beta1.UpdateStrategySpec{ Stages: []placementv1beta1.StageConfig{ { Name: fmt.Sprintf(updateRunStageNameTemplate, GinkgoParallelProcess(), 1), diff --git a/test/e2e/actuals_test.go b/test/e2e/actuals_test.go index 90db251b5..db6796ddb 100644 --- a/test/e2e/actuals_test.go +++ b/test/e2e/actuals_test.go @@ -519,6 +519,23 @@ func crpRolloutPendingDueToExternalStrategyConditions(generation int64) []metav1 } } +func rpRolloutPendingDueToExternalStrategyConditions(generation int64) []metav1.Condition { + return []metav1.Condition{ + { + Type: string(placementv1beta1.ResourcePlacementScheduledConditionType), + Status: metav1.ConditionTrue, + Reason: scheduler.FullyScheduledReason, + ObservedGeneration: generation, + }, + { + Type: string(placementv1beta1.ResourcePlacementRolloutStartedConditionType), + Status: metav1.ConditionUnknown, + Reason: condition.RolloutControlledByExternalControllerReason, + ObservedGeneration: generation, + }, + } +} + func rpAppliedFailedConditions(generation int64) []metav1.Condition { return []metav1.Condition{ { @@ -1413,9 +1430,6 @@ func crpStatusWithExternalStrategyActual( wantResourceOverrides map[string][]placementv1beta1.NamespacedName, ) func() error { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) - nsName := fmt.Sprintf(workNamespaceNameTemplate, GinkgoParallelProcess()) - cmName := fmt.Sprintf(appConfigMapNameTemplate, GinkgoParallelProcess()) - return func() error { crp := &placementv1beta1.ClusterResourcePlacement{} if err := hubClient.Get(ctx, types.NamespacedName{Name: crpName}, crp); err != nil { @@ -1424,71 +1438,8 @@ func crpStatusWithExternalStrategyActual( reportDiff := crp.Spec.Strategy.ApplyStrategy != nil && crp.Spec.Strategy.ApplyStrategy.Type == placementv1beta1.ApplyStrategyTypeReportDiff - var wantPlacementStatus []placementv1beta1.PerClusterPlacementStatus - crpHasOverrides := false - for i, name := range wantSelectedClusters { - if !wantRolloutCompletedPerCluster[i] { - // No observed resource index for this cluster, assume rollout is still pending. - wantPlacementStatus = append(wantPlacementStatus, placementv1beta1.PerClusterPlacementStatus{ - ClusterName: name, - Conditions: perClusterRolloutUnknownConditions(crp.Generation), - ObservedResourceIndex: wantObservedResourceIndexPerCluster[i], - }) - } else { - wantResourceOverrides, hasRO := wantResourceOverrides[name] - wantClusterResourceOverrides, hasCRO := wantClusterResourceOverrides[name] - hasOverrides := (hasRO && len(wantResourceOverrides) > 0) || (hasCRO && len(wantClusterResourceOverrides) > 0) - if hasOverrides { - crpHasOverrides = true - } - if reportDiff { - wantPlacementStatus = append(wantPlacementStatus, placementv1beta1.PerClusterPlacementStatus{ - ClusterName: name, - Conditions: perClusterDiffReportedConditions(crp.Generation), - ApplicableResourceOverrides: wantResourceOverrides, - ApplicableClusterResourceOverrides: wantClusterResourceOverrides, - ObservedResourceIndex: wantObservedResourceIndexPerCluster[i], - DiffedPlacements: []placementv1beta1.DiffedResourcePlacement{ - { - ResourceIdentifier: placementv1beta1.ResourceIdentifier{ - Version: "v1", - Kind: "Namespace", - Name: nsName, - }, - ObservedDiffs: []placementv1beta1.PatchDetail{ - { - Path: "/", - ValueInHub: "(the whole object)", - }, - }, - }, - { - ResourceIdentifier: placementv1beta1.ResourceIdentifier{ - Version: "v1", - Kind: "ConfigMap", - Name: cmName, - Namespace: nsName, - }, - ObservedDiffs: []placementv1beta1.PatchDetail{ - { - Path: "/", - ValueInHub: "(the whole object)", - }, - }, - }, - }, - }) - } else { - wantPlacementStatus = append(wantPlacementStatus, placementv1beta1.PerClusterPlacementStatus{ - ClusterName: name, - Conditions: perClusterRolloutCompletedConditions(crp.Generation, true, hasOverrides), - ApplicableResourceOverrides: wantResourceOverrides, - ApplicableClusterResourceOverrides: wantClusterResourceOverrides, - ObservedResourceIndex: wantObservedResourceIndexPerCluster[i], - }) - } - } - } + wantPlacementStatus, crpHasOverrides := buildPerClusterPlacementStatusesAndHasOverrides( + crp.Generation, reportDiff, wantSelectedClusters, wantObservedResourceIndexPerCluster, wantRolloutCompletedPerCluster, wantClusterResourceOverrides, wantResourceOverrides) wantStatus := placementv1beta1.PlacementStatus{ PerClusterPlacementStatuses: wantPlacementStatus, @@ -1506,12 +1457,137 @@ func crpStatusWithExternalStrategyActual( } if diff := cmp.Diff(crp.Status, wantStatus, placementStatusCmpOptions...); diff != "" { - return fmt.Errorf("CRP status diff (-got, +want): %s", diff) + return fmt.Errorf("CRP status diff (-got, +want): %s for CRP %s", diff, crpName) } return nil } } +func rpStatusWithExternalStrategyActual( + wantSelectedResourceIdentifiers []placementv1beta1.ResourceIdentifier, + wantObservedResourceIndex string, + wantRPRolloutCompleted bool, + wantSelectedClusters []string, + wantObservedResourceIndexPerCluster []string, + wantRolloutCompletedPerCluster []bool, + wantClusterResourceOverrides map[string][]string, + wantResourceOverrides map[string][]placementv1beta1.NamespacedName, +) func() error { + rpName := fmt.Sprintf(rpNameTemplate, GinkgoParallelProcess()) + rpNamespace := fmt.Sprintf(workNamespaceNameTemplate, GinkgoParallelProcess()) + return func() error { + rp := &placementv1beta1.ResourcePlacement{} + if err := hubClient.Get(ctx, client.ObjectKey{Name: rpName, Namespace: rpNamespace}, rp); err != nil { + return err + } + + reportDiff := rp.Spec.Strategy.ApplyStrategy != nil && rp.Spec.Strategy.ApplyStrategy.Type == placementv1beta1.ApplyStrategyTypeReportDiff + + wantPlacementStatus, rpHasOverrides := buildPerClusterPlacementStatusesAndHasOverrides( + rp.Generation, reportDiff, wantSelectedClusters, wantObservedResourceIndexPerCluster, wantRolloutCompletedPerCluster, wantClusterResourceOverrides, wantResourceOverrides) + + wantStatus := placementv1beta1.PlacementStatus{ + PerClusterPlacementStatuses: wantPlacementStatus, + SelectedResources: wantSelectedResourceIdentifiers, + ObservedResourceIndex: wantObservedResourceIndex, + } + if wantRPRolloutCompleted { + if reportDiff { + wantStatus.Conditions = rpDiffReportedConditions(rp.Generation, rpHasOverrides) + } else { + wantStatus.Conditions = rpRolloutCompletedConditions(rp.Generation, rpHasOverrides) + } + } else { + wantStatus.Conditions = rpRolloutPendingDueToExternalStrategyConditions(rp.Generation) + } + + if diff := cmp.Diff(rp.Status, wantStatus, placementStatusCmpOptions...); diff != "" { + return fmt.Errorf("RP status diff (-got, +want): %s", diff) + } + return nil + } +} + +func buildPerClusterPlacementStatusesAndHasOverrides( + generation int64, + reportDiff bool, + wantSelectedClusters []string, + wantObservedResourceIndexPerCluster []string, + wantRolloutCompletedPerCluster []bool, + wantClusterResourceOverrides map[string][]string, + wantResourceOverrides map[string][]placementv1beta1.NamespacedName, +) ([]placementv1beta1.PerClusterPlacementStatus, bool) { + nsName := fmt.Sprintf(workNamespaceNameTemplate, GinkgoParallelProcess()) + cmName := fmt.Sprintf(appConfigMapNameTemplate, GinkgoParallelProcess()) + var wantPlacementStatuses []placementv1beta1.PerClusterPlacementStatus + placementHasOverrides := false + for i, name := range wantSelectedClusters { + if !wantRolloutCompletedPerCluster[i] { + // No observed resource index for this cluster, assume rollout is still pending. + wantPlacementStatuses = append(wantPlacementStatuses, placementv1beta1.PerClusterPlacementStatus{ + ClusterName: name, + Conditions: perClusterRolloutUnknownConditions(generation), + ObservedResourceIndex: wantObservedResourceIndexPerCluster[i], + }) + } else { + wantResourceOverrides, hasRO := wantResourceOverrides[name] + wantClusterResourceOverrides, hasCRO := wantClusterResourceOverrides[name] + hasOverrides := (hasRO && len(wantResourceOverrides) > 0) || (hasCRO && len(wantClusterResourceOverrides) > 0) + if hasOverrides { + placementHasOverrides = true + } + if reportDiff { + wantPlacementStatuses = append(wantPlacementStatuses, placementv1beta1.PerClusterPlacementStatus{ + ClusterName: name, + Conditions: perClusterDiffReportedConditions(generation), + ApplicableResourceOverrides: wantResourceOverrides, + ApplicableClusterResourceOverrides: wantClusterResourceOverrides, + ObservedResourceIndex: wantObservedResourceIndexPerCluster[i], + // TODO(arvindth): Need to update this when we have more test cases for staged UpdateRun. + DiffedPlacements: []placementv1beta1.DiffedResourcePlacement{ + { + ResourceIdentifier: placementv1beta1.ResourceIdentifier{ + Version: "v1", + Kind: "Namespace", + Name: nsName, + }, + ObservedDiffs: []placementv1beta1.PatchDetail{ + { + Path: "/", + ValueInHub: "(the whole object)", + }, + }, + }, + { + ResourceIdentifier: placementv1beta1.ResourceIdentifier{ + Version: "v1", + Kind: "ConfigMap", + Name: cmName, + Namespace: nsName, + }, + ObservedDiffs: []placementv1beta1.PatchDetail{ + { + Path: "/", + ValueInHub: "(the whole object)", + }, + }, + }, + }, + }) + } else { + wantPlacementStatuses = append(wantPlacementStatuses, placementv1beta1.PerClusterPlacementStatus{ + ClusterName: name, + Conditions: perClusterRolloutCompletedConditions(generation, true, hasOverrides), + ApplicableResourceOverrides: wantResourceOverrides, + ApplicableClusterResourceOverrides: wantClusterResourceOverrides, + ObservedResourceIndex: wantObservedResourceIndexPerCluster[i], + }) + } + } + } + return wantPlacementStatuses, placementHasOverrides +} + func customizedPlacementStatusUpdatedActual( placementKey types.NamespacedName, wantSelectedResourceIdentifiers []placementv1beta1.ResourceIdentifier, @@ -1526,8 +1602,15 @@ func customizedPlacementStatusUpdatedActual( } wantStatus := buildWantPlacementStatus(placementKey, placement.GetGeneration(), wantSelectedResourceIdentifiers, wantSelectedClusters, wantUnselectedClusters, wantObservedResourceIndex, resourceIsTrackable) - if diff := cmp.Diff(placement.GetPlacementStatus(), wantStatus, placementStatusCmpOptions...); diff != "" { - return fmt.Errorf("Placement status diff (-got, +want): %s", diff) + if wantObservedResourceIndex == "0" { + // When the observedResourceIndex is "0", it means the placement is just created and the placement controller + if diff := cmp.Diff(placement.GetPlacementStatus(), wantStatus, placementStatusCmpOptionsOnCreate...); diff != "" { + return fmt.Errorf("Placement status diff (-got, +want): %s for placement %v", diff, placementKey) + } + } else { + if diff := cmp.Diff(placement.GetPlacementStatus(), wantStatus, placementStatusCmpOptions...); diff != "" { + return fmt.Errorf("Placement status diff (-got, +want): %s for placement %v", diff, placementKey) + } } return nil } @@ -2004,12 +2087,12 @@ func updateRunSucceedConditions(generation int64) []metav1.Condition { } } -func updateRunStatusSucceededActual( +func clusterStagedUpdateRunStatusSucceededActual( updateRunName string, wantPolicyIndex string, wantClusterCount int, wantApplyStrategy *placementv1beta1.ApplyStrategy, - wantStrategySpec *placementv1beta1.StagedUpdateStrategySpec, + wantStrategySpec *placementv1beta1.UpdateStrategySpec, wantSelectedClusters [][]string, wantUnscheduledClusters []string, wantCROs map[string][]string, @@ -2021,45 +2104,49 @@ func updateRunStatusSucceededActual( return err } - wantStatus := placementv1beta1.StagedUpdateRunStatus{ - PolicySnapshotIndexUsed: wantPolicyIndex, - PolicyObservedClusterCount: wantClusterCount, - ApplyStrategy: wantApplyStrategy.DeepCopy(), - StagedUpdateStrategySnapshot: wantStrategySpec, - } - stagesStatus := make([]placementv1beta1.StageUpdatingStatus, len(wantStrategySpec.Stages)) - for i, stage := range wantStrategySpec.Stages { - stagesStatus[i].StageName = stage.Name - stagesStatus[i].Clusters = make([]placementv1beta1.ClusterUpdatingStatus, len(wantSelectedClusters[i])) - for j := range stagesStatus[i].Clusters { - stagesStatus[i].Clusters[j].ClusterName = wantSelectedClusters[i][j] - stagesStatus[i].Clusters[j].ClusterResourceOverrideSnapshots = wantCROs[wantSelectedClusters[i][j]] - stagesStatus[i].Clusters[j].ResourceOverrideSnapshots = wantROs[wantSelectedClusters[i][j]] - stagesStatus[i].Clusters[j].Conditions = updateRunClusterRolloutSucceedConditions(updateRun.Generation) - } - stagesStatus[i].AfterStageTaskStatus = make([]placementv1beta1.AfterStageTaskStatus, len(stage.AfterStageTasks)) - for j, task := range stage.AfterStageTasks { - stagesStatus[i].AfterStageTaskStatus[j].Type = task.Type - if task.Type == placementv1beta1.AfterStageTaskTypeApproval { - stagesStatus[i].AfterStageTaskStatus[j].ApprovalRequestName = fmt.Sprintf(placementv1beta1.ApprovalTaskNameFmt, updateRun.Name, stage.Name) - } - stagesStatus[i].AfterStageTaskStatus[j].Conditions = updateRunAfterStageTaskSucceedConditions(updateRun.Generation, task.Type) - } - stagesStatus[i].Conditions = updateRunStageRolloutSucceedConditions(updateRun.Generation) + wantStatus := placementv1beta1.UpdateRunStatus{ + PolicySnapshotIndexUsed: wantPolicyIndex, + PolicyObservedClusterCount: wantClusterCount, + ApplyStrategy: wantApplyStrategy.DeepCopy(), + UpdateStrategySnapshot: wantStrategySpec, + } + + wantStatus.StagesStatus = buildStageUpdatingStatuses(wantStrategySpec, wantSelectedClusters, wantCROs, wantROs, updateRun) + wantStatus.DeletionStageStatus = buildDeletionStageStatus(wantUnscheduledClusters, updateRun) + wantStatus.Conditions = updateRunSucceedConditions(updateRun.Generation) + if diff := cmp.Diff(updateRun.Status, wantStatus, updateRunStatusCmpOption...); diff != "" { + return fmt.Errorf("UpdateRun status diff (-got, +want): %s", diff) } + return nil + } +} - deleteStageStatus := &placementv1beta1.StageUpdatingStatus{ - StageName: "kubernetes-fleet.io/deleteStage", +func stagedUpdateRunStatusSucceededActual( + updateRunName, namespace string, + wantPolicyIndex string, + wantClusterCount int, + wantApplyStrategy *placementv1beta1.ApplyStrategy, + wantStrategySpec *placementv1beta1.UpdateStrategySpec, + wantSelectedClusters [][]string, + wantUnscheduledClusters []string, + wantCROs map[string][]string, + wantROs map[string][]placementv1beta1.NamespacedName, +) func() error { + return func() error { + updateRun := &placementv1beta1.StagedUpdateRun{} + if err := hubClient.Get(ctx, client.ObjectKey{Name: updateRunName, Namespace: namespace}, updateRun); err != nil { + return err } - deleteStageStatus.Clusters = make([]placementv1beta1.ClusterUpdatingStatus, len(wantUnscheduledClusters)) - for i := range deleteStageStatus.Clusters { - deleteStageStatus.Clusters[i].ClusterName = wantUnscheduledClusters[i] - deleteStageStatus.Clusters[i].Conditions = updateRunClusterRolloutSucceedConditions(updateRun.Generation) + + wantStatus := placementv1beta1.UpdateRunStatus{ + PolicySnapshotIndexUsed: wantPolicyIndex, + PolicyObservedClusterCount: wantClusterCount, + ApplyStrategy: wantApplyStrategy.DeepCopy(), + UpdateStrategySnapshot: wantStrategySpec, } - deleteStageStatus.Conditions = updateRunStageRolloutSucceedConditions(updateRun.Generation) - wantStatus.StagesStatus = stagesStatus - wantStatus.DeletionStageStatus = deleteStageStatus + wantStatus.StagesStatus = buildStageUpdatingStatuses(wantStrategySpec, wantSelectedClusters, wantCROs, wantROs, updateRun) + wantStatus.DeletionStageStatus = buildDeletionStageStatus(wantUnscheduledClusters, updateRun) wantStatus.Conditions = updateRunSucceedConditions(updateRun.Generation) if diff := cmp.Diff(updateRun.Status, wantStatus, updateRunStatusCmpOption...); diff != "" { return fmt.Errorf("UpdateRun status diff (-got, +want): %s", diff) @@ -2068,7 +2155,53 @@ func updateRunStatusSucceededActual( } } -func updateRunAndApprovalRequestsRemovedActual(updateRunName string) func() error { +func buildStageUpdatingStatuses( + wantStrategySpec *placementv1beta1.UpdateStrategySpec, + wantSelectedClusters [][]string, + wantCROs map[string][]string, + wantROs map[string][]placementv1beta1.NamespacedName, + updateRun placementv1beta1.UpdateRunObj, +) []placementv1beta1.StageUpdatingStatus { + stagesStatus := make([]placementv1beta1.StageUpdatingStatus, len(wantStrategySpec.Stages)) + for i, stage := range wantStrategySpec.Stages { + stagesStatus[i].StageName = stage.Name + stagesStatus[i].Clusters = make([]placementv1beta1.ClusterUpdatingStatus, len(wantSelectedClusters[i])) + for j := range stagesStatus[i].Clusters { + stagesStatus[i].Clusters[j].ClusterName = wantSelectedClusters[i][j] + stagesStatus[i].Clusters[j].ClusterResourceOverrideSnapshots = wantCROs[wantSelectedClusters[i][j]] + stagesStatus[i].Clusters[j].ResourceOverrideSnapshots = wantROs[wantSelectedClusters[i][j]] + stagesStatus[i].Clusters[j].Conditions = updateRunClusterRolloutSucceedConditions(updateRun.GetGeneration()) + } + stagesStatus[i].AfterStageTaskStatus = make([]placementv1beta1.AfterStageTaskStatus, len(stage.AfterStageTasks)) + for j, task := range stage.AfterStageTasks { + stagesStatus[i].AfterStageTaskStatus[j].Type = task.Type + if task.Type == placementv1beta1.AfterStageTaskTypeApproval { + stagesStatus[i].AfterStageTaskStatus[j].ApprovalRequestName = fmt.Sprintf(placementv1beta1.ApprovalTaskNameFmt, updateRun.GetName(), stage.Name) + } + stagesStatus[i].AfterStageTaskStatus[j].Conditions = updateRunAfterStageTaskSucceedConditions(updateRun.GetGeneration(), task.Type) + } + stagesStatus[i].Conditions = updateRunStageRolloutSucceedConditions(updateRun.GetGeneration()) + } + return stagesStatus +} + +func buildDeletionStageStatus( + wantUnscheduledClusters []string, + updateRun placementv1beta1.UpdateRunObj, +) *placementv1beta1.StageUpdatingStatus { + deleteStageStatus := &placementv1beta1.StageUpdatingStatus{ + StageName: "kubernetes-fleet.io/deleteStage", + } + deleteStageStatus.Clusters = make([]placementv1beta1.ClusterUpdatingStatus, len(wantUnscheduledClusters)) + for i := range deleteStageStatus.Clusters { + deleteStageStatus.Clusters[i].ClusterName = wantUnscheduledClusters[i] + deleteStageStatus.Clusters[i].Conditions = updateRunClusterRolloutSucceedConditions(updateRun.GetGeneration()) + } + deleteStageStatus.Conditions = updateRunStageRolloutSucceedConditions(updateRun.GetGeneration()) + return deleteStageStatus +} + +func clusterStagedUpdateRunAndClusterApprovalRequestsRemovedActual(updateRunName string) func() error { return func() error { if err := hubClient.Get(ctx, types.NamespacedName{Name: updateRunName}, &placementv1beta1.ClusterStagedUpdateRun{}); !errors.IsNotFound(err) { return fmt.Errorf("UpdateRun still exists or an unexpected error occurred: %w", err) @@ -2087,7 +2220,26 @@ func updateRunAndApprovalRequestsRemovedActual(updateRunName string) func() erro } } -func updateRunStrategyRemovedActual(strategyName string) func() error { +func stagedUpdateRunAndApprovalRequestsRemovedActual(updateRunName, namespace string) func() error { + return func() error { + if err := hubClient.Get(ctx, client.ObjectKey{Name: updateRunName, Namespace: namespace}, &placementv1beta1.StagedUpdateRun{}); !errors.IsNotFound(err) { + return fmt.Errorf("UpdateRun still exists or an unexpected error occurred: %w", err) + } + + appReqList := &placementv1beta1.ApprovalRequestList{} + if err := hubClient.List(ctx, appReqList, client.InNamespace(namespace), client.MatchingLabels{ + placementv1beta1.TargetUpdateRunLabel: updateRunName, + }); err != nil { + return fmt.Errorf("failed to list ApprovalRequests: %w", err) + } + if len(appReqList.Items) > 0 { + return fmt.Errorf("ApprovalRequests still exist: %v", appReqList.Items) + } + return nil + } +} + +func clusterUpdateRunStrategyRemovedActual(strategyName string) func() error { return func() error { if err := hubClient.Get(ctx, types.NamespacedName{Name: strategyName}, &placementv1beta1.ClusterStagedUpdateStrategy{}); !errors.IsNotFound(err) { return fmt.Errorf("ClusterStagedUpdateStrategy still exists or an unexpected error occurred: %w", err) diff --git a/test/e2e/updaterun_test.go b/test/e2e/cluster_staged_updaterun_test.go similarity index 81% rename from test/e2e/updaterun_test.go rename to test/e2e/cluster_staged_updaterun_test.go index f13137ef7..13d28a5ab 100644 --- a/test/e2e/updaterun_test.go +++ b/test/e2e/cluster_staged_updaterun_test.go @@ -49,7 +49,7 @@ const ( ) var ( - // defaultApplyStrategy is the default apply strategy that CRP mutation webhook will use + // defaultApplyStrategy is the default apply strategy that CRP mutation webhook will use. defaultApplyStrategy = &placementv1beta1.ApplyStrategy{ ComparisonOption: "PartialComparison", WhenToApply: "Always", @@ -61,7 +61,7 @@ var ( // Note that this container will run in parallel with other containers. var _ = Describe("test CRP rollout with staged update run", func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) - strategyName := fmt.Sprintf(updateRunStrategyNameTemplate, GinkgoParallelProcess()) + strategyName := fmt.Sprintf(clusterStagedUpdateRunStrategyNameTemplate, GinkgoParallelProcess()) Context("Test resource rollout and rollback with staged update run", Ordered, func() { updateRunNames := []string{} @@ -90,10 +90,10 @@ var _ = Describe("test CRP rollout with staged update run", func() { Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP") // Create the clusterStagedUpdateStrategy. - strategy = createStagedUpdateStrategySucceed(strategyName) + strategy = createClusterStagedUpdateStrategySucceed(strategyName) for i := 0; i < 3; i++ { - updateRunNames = append(updateRunNames, fmt.Sprintf(updateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), i)) + updateRunNames = append(updateRunNames, fmt.Sprintf(clusterStagedUpdateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), i)) } oldConfigMap = appConfigMap() @@ -107,21 +107,21 @@ var _ = Describe("test CRP rollout with staged update run", func() { // Remove all the clusterStagedUpdateRuns. for _, name := range updateRunNames { - ensureUpdateRunDeletion(name) + ensureClusterStagedUpdateRunDeletion(name) } // Delete the clusterStagedUpdateStrategy. - ensureUpdateRunStrategyDeletion(strategyName) + ensureClusterUpdateRunStrategyDeletion(strategyName) }) It("Should not rollout any resources to member clusters as there's no update run yet", checkIfRemovedWorkResourcesFromAllMemberClustersConsistently) It("Should have the latest resource snapshot", func() { - validateLatestResourceSnapshot(crpName, resourceSnapshotIndex1st) + validateLatestClusterResourceSnapshot(crpName, resourceSnapshotIndex1st) }) It("Should successfully schedule the crp", func() { - validateLatestPolicySnapshot(crpName, policySnapshotIndex1st, 3) + validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) }) It("Should update crp status as pending rollout", func() { @@ -129,8 +129,8 @@ var _ = Describe("test CRP rollout with staged update run", func() { Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) }) - It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName) + It("Should create a cluster staged update run successfully", func() { + createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -148,9 +148,9 @@ var _ = Describe("test CRP rollout with staged update run", func() { checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[0]}) }) - It("Should rollout resources to all the members and complete the staged update run successfully", func() { - updateRunSucceededActual := updateRunStatusSucceededActual(updateRunNames[0], policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) - Eventually(updateRunSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) + It("Should rollout resources to all the members and complete the cluster staged update run successfully", func() { + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -191,8 +191,8 @@ var _ = Describe("test CRP rollout with staged update run", func() { }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed get the new latest resourcensnapshot") }) - It("Should create a new staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex2nd, strategyName) + It("Should create a new cluster staged update run successfully", func() { + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex2nd, strategyName) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -213,9 +213,9 @@ var _ = Describe("test CRP rollout with staged update run", func() { validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary) }) - It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { - updateRunSucceededActual := updateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) - Eventually(updateRunSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) + It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the cluster staged update run successfully", func() { + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) By("Verify that new the configmap is updated on all member clusters") for idx := range allMemberClusters { configMapActual := configMapPlacedOnClusterActual(allMemberClusters[idx], &newConfigMap) @@ -230,7 +230,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a new staged update run with old resourceSnapshotIndex successfully to rollback", func() { - createStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName) }) It("Should rollback resources to member-cluster-2 only and completes stage canary", func() { @@ -251,9 +251,9 @@ var _ = Describe("test CRP rollout with staged update run", func() { validateAndApproveClusterApprovalRequests(updateRunNames[2], envCanary) }) - It("Should rollback resources to member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { - updateRunSucceededActual := updateRunStatusSucceededActual(updateRunNames[2], policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) - Eventually(updateRunSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) + It("Should rollback resources to member-cluster-1 and member-cluster-3 too and complete the cluster staged update run successfully", func() { + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) for idx := range allMemberClusters { configMapActual := configMapPlacedOnClusterActual(allMemberClusters[idx], &oldConfigMap) Eventually(configMapActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to rollback the configmap %s data on cluster %s as expected", oldConfigMap.Name, allMemberClusterNames[idx]) @@ -297,10 +297,10 @@ var _ = Describe("test CRP rollout with staged update run", func() { Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP") // Create the clusterStagedUpdateStrategy. - strategy = createStagedUpdateStrategySucceed(strategyName) + strategy = createClusterStagedUpdateStrategySucceed(strategyName) for i := 0; i < 3; i++ { - updateRunNames = append(updateRunNames, fmt.Sprintf(updateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), i)) + updateRunNames = append(updateRunNames, fmt.Sprintf(clusterStagedUpdateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), i)) } }) @@ -310,21 +310,21 @@ var _ = Describe("test CRP rollout with staged update run", func() { // Remove all the clusterStagedUpdateRuns. for _, name := range updateRunNames { - ensureUpdateRunDeletion(name) + ensureClusterStagedUpdateRunDeletion(name) } // Delete the clusterStagedUpdateStrategy. - ensureUpdateRunStrategyDeletion(strategyName) + ensureClusterUpdateRunStrategyDeletion(strategyName) }) It("Should not rollout any resources to member clusters as there's no update run yet", checkIfRemovedWorkResourcesFromAllMemberClustersConsistently) It("Should have the latest resource snapshot", func() { - validateLatestResourceSnapshot(crpName, resourceSnapshotIndex1st) + validateLatestClusterResourceSnapshot(crpName, resourceSnapshotIndex1st) }) It("Should successfully schedule the crp", func() { - validateLatestPolicySnapshot(crpName, policySnapshotIndex1st, 2) + validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 2) }) It("Should update crp status as pending rollout", func() { @@ -332,8 +332,8 @@ var _ = Describe("test CRP rollout with staged update run", func() { Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) }) - It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName) + It("Should create a cluster staged update run successfully", func() { + createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -347,9 +347,9 @@ var _ = Describe("test CRP rollout with staged update run", func() { validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary) }) - It("Should rollout resources to member-cluster-1 too but not member-cluster-3 and complete the staged update run successfully", func() { - updateRunSucceededActual := updateRunStatusSucceededActual(updateRunNames[0], policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0]}}, nil, nil, nil) - Eventually(updateRunSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) + It("Should rollout resources to member-cluster-1 too but not member-cluster-3 and complete the cluster staged update run successfully", func() { + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0]}}, nil, nil, nil) + Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[2]}) }) @@ -372,7 +372,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should successfully schedule the crp", func() { - validateLatestPolicySnapshot(crpName, policySnapshotIndex2nd, 3) + validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex2nd, 3) }) It("Should update crp status as rollout pending", func() { @@ -380,8 +380,8 @@ var _ = Describe("test CRP rollout with staged update run", func() { Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) }) - It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName) + It("Should create a cluster staged update run successfully", func() { + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName) }) It("Should still have resources on member-cluster-1 and member-cluster-2 only and completes stage canary", func() { @@ -397,9 +397,9 @@ var _ = Describe("test CRP rollout with staged update run", func() { validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary) }) - It("Should rollout resources to member-cluster-3 too and complete the staged update run successfully", func() { - updateRunSucceededActual := updateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex2nd, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) - Eventually(updateRunSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) + It("Should rollout resources to member-cluster-3 too and complete the cluster staged update run successfully", func() { + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex2nd, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -421,7 +421,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should successfully schedule the crp", func() { - validateLatestPolicySnapshot(crpName, policySnapshotIndex3rd, 1) + validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex3rd, 1) }) It("Should update crp status as rollout pending with member-cluster-3 only", func() { @@ -429,8 +429,8 @@ var _ = Describe("test CRP rollout with staged update run", func() { Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) }) - It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName) + It("Should create a cluster staged update run successfully", func() { + createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName) }) It("Should still have resources on all member clusters and complete stage canary", func() { @@ -443,10 +443,10 @@ var _ = Describe("test CRP rollout with staged update run", func() { validateAndApproveClusterApprovalRequests(updateRunNames[2], envCanary) }) - It("Should remove resources on member-cluster-1 and member-cluster-2 and complete the staged update run successfully", func() { + It("Should remove resources on member-cluster-1 and member-cluster-2 and complete the cluster staged update run successfully", func() { // need to go through two stages - updateRunSucceededActual := updateRunStatusSucceededActual(updateRunNames[2], policySnapshotIndex3rd, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0], allMemberClusterNames[1]}, nil, nil) - Eventually(updateRunSucceededActual, 2*updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[2]) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], policySnapshotIndex3rd, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0], allMemberClusterNames[1]}, nil, nil) + Eventually(csurSucceededActual, 2*updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[2]) checkIfRemovedWorkResourcesFromMemberClusters([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) checkIfPlacedWorkResourcesOnMemberClustersConsistently([]*framework.Cluster{allMemberClusters[2]}) }) @@ -487,10 +487,10 @@ var _ = Describe("test CRP rollout with staged update run", func() { Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP") // Create the clusterStagedUpdateStrategy. - strategy = createStagedUpdateStrategySucceed(strategyName) + strategy = createClusterStagedUpdateStrategySucceed(strategyName) for i := 0; i < 3; i++ { - updateRunNames = append(updateRunNames, fmt.Sprintf(updateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), i)) + updateRunNames = append(updateRunNames, fmt.Sprintf(clusterStagedUpdateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), i)) } }) @@ -500,21 +500,21 @@ var _ = Describe("test CRP rollout with staged update run", func() { // Remove all the clusterStagedUpdateRuns. for _, name := range updateRunNames { - ensureUpdateRunDeletion(name) + ensureClusterStagedUpdateRunDeletion(name) } // Delete the clusterStagedUpdateStrategy. - ensureUpdateRunStrategyDeletion(strategyName) + ensureClusterUpdateRunStrategyDeletion(strategyName) }) It("Should not rollout any resources to member clusters as there's no update run yet", checkIfRemovedWorkResourcesFromAllMemberClustersConsistently) It("Should have the latest resource snapshot", func() { - validateLatestResourceSnapshot(crpName, resourceSnapshotIndex1st) + validateLatestClusterResourceSnapshot(crpName, resourceSnapshotIndex1st) }) It("Should successfully schedule the crp", func() { - validateLatestPolicySnapshot(crpName, policySnapshotIndex1st, 1) + validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 1) }) It("Should update crp status as pending rollout", func() { @@ -522,8 +522,8 @@ var _ = Describe("test CRP rollout with staged update run", func() { Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) }) - It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName) + It("Should create a cluster staged update run successfully", func() { + createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName) }) It("Should not rollout any resources to member clusters and complete stage canary", func() { @@ -536,9 +536,9 @@ var _ = Describe("test CRP rollout with staged update run", func() { validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary) }) - It("Should rollout resources to member-cluster-3 and complete the staged update run successfully", func() { - updateRunSucceededActual := updateRunStatusSucceededActual(updateRunNames[0], policySnapshotIndex1st, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, nil, nil, nil) - Eventually(updateRunSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) + It("Should rollout resources to member-cluster-3 and complete the cluster staged update run successfully", func() { + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], policySnapshotIndex1st, 1, defaultApplyStrategy, &strategy.Spec, [][]string{{}, {allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[2]}) checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[1]}) }) @@ -561,7 +561,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should successfully schedule the crp without creating a new policy snapshot", func() { - validateLatestPolicySnapshot(crpName, policySnapshotIndex1st, 3) + validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) }) It("Should update crp status as rollout pending", func() { @@ -569,8 +569,8 @@ var _ = Describe("test CRP rollout with staged update run", func() { Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) }) - It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName) + It("Should create a cluster staged update run successfully", func() { + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName) }) It("Should still have resources on member-cluster-2 and member-cluster-3 only and completes stage canary", func() { @@ -585,9 +585,9 @@ var _ = Describe("test CRP rollout with staged update run", func() { validateAndApproveClusterApprovalRequests(updateRunNames[1], envCanary) }) - It("Should rollout resources to member-cluster-1 too and complete the staged update run successfully", func() { - updateRunSucceededActual := updateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) - Eventually(updateRunSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) + It("Should rollout resources to member-cluster-1 too and complete the cluster staged update run successfully", func() { + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -609,7 +609,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should successfully schedule the crp without creating a new policy snapshot", func() { - validateLatestPolicySnapshot(crpName, policySnapshotIndex1st, 2) + validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 2) }) It("Should update crp status as rollout completed with member-cluster-2 and member-cluster-3", func() { @@ -617,8 +617,8 @@ var _ = Describe("test CRP rollout with staged update run", func() { Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) }) - It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName) + It("Should create a cluster staged update run successfully", func() { + createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName) }) It("Should still have resources on all member clusters and complete stage canary", func() { @@ -631,9 +631,9 @@ var _ = Describe("test CRP rollout with staged update run", func() { validateAndApproveClusterApprovalRequests(updateRunNames[2], envCanary) }) - It("Should remove resources on member-cluster-1 and complete the staged update run successfully", func() { - updateRunSucceededActual := updateRunStatusSucceededActual(updateRunNames[2], policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil) - Eventually(updateRunSucceededActual, 2*updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[2]) + It("Should remove resources on member-cluster-1 and complete the cluster staged update run successfully", func() { + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[2], policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil) + Eventually(csurSucceededActual, 2*updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[2]) checkIfRemovedWorkResourcesFromMemberClusters([]*framework.Cluster{allMemberClusters[0]}) checkIfPlacedWorkResourcesOnMemberClustersConsistently([]*framework.Cluster{allMemberClusters[1], allMemberClusters[2]}) }) @@ -646,7 +646,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { Context("Test staged update run with overrides", Ordered, func() { var strategy *placementv1beta1.ClusterStagedUpdateStrategy - updateRunName := fmt.Sprintf(updateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), 0) + updateRunName := fmt.Sprintf(clusterStagedUpdateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), 0) croName := fmt.Sprintf(croNameTemplate, GinkgoParallelProcess()) roName := fmt.Sprintf(roNameTemplate, GinkgoParallelProcess()) roNamespace := fmt.Sprintf(workNamespaceNameTemplate, GinkgoParallelProcess()) @@ -752,7 +752,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP") // Create the clusterStagedUpdateStrategy. - strategy = createStagedUpdateStrategySucceed(strategyName) + strategy = createClusterStagedUpdateStrategySucceed(strategyName) }) AfterAll(func() { @@ -760,10 +760,10 @@ var _ = Describe("test CRP rollout with staged update run", func() { ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) // Delete the clusterStagedUpdateRun. - ensureUpdateRunDeletion(updateRunName) + ensureClusterStagedUpdateRunDeletion(updateRunName) // Delete the clusterStagedUpdateStrategy. - ensureUpdateRunStrategyDeletion(strategyName) + ensureClusterUpdateRunStrategyDeletion(strategyName) // Delete the clusterResourceOverride. cleanupClusterResourceOverride(croName) @@ -775,7 +775,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { It("Should not rollout any resources to member clusters as there's no update run yet", checkIfRemovedWorkResourcesFromAllMemberClustersConsistently) It("Should have the latest resource snapshot", func() { - validateLatestResourceSnapshot(crpName, resourceSnapshotIndex1st) + validateLatestClusterResourceSnapshot(crpName, resourceSnapshotIndex1st) }) It("Should update crp status as pending rollout", func() { @@ -784,11 +784,11 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should successfully schedule the crp", func() { - validateLatestPolicySnapshot(crpName, policySnapshotIndex1st, 3) + validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) }) - It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) + It("Should create a cluster staged update run successfully", func() { + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -803,9 +803,9 @@ var _ = Describe("test CRP rollout with staged update run", func() { validateAndApproveClusterApprovalRequests(updateRunName, envCanary) }) - It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { - updateRunSucceededActual := updateRunStatusSucceededActual(updateRunName, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, wantCROs, wantROs) - Eventually(updateRunSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) + It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the cluster staged update run successfully", func() { + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, wantCROs, wantROs) + Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -827,7 +827,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { Context("Test staged update run with reportDiff mode", Ordered, func() { var strategy *placementv1beta1.ClusterStagedUpdateStrategy var applyStrategy *placementv1beta1.ApplyStrategy - updateRunName := fmt.Sprintf(updateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), 0) + updateRunName := fmt.Sprintf(clusterStagedUpdateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), 0) BeforeAll(func() { // Create a test namespace and a configMap inside it on the hub cluster. @@ -861,7 +861,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP") // Create the clusterStagedUpdateStrategy. - strategy = createStagedUpdateStrategySucceed(strategyName) + strategy = createClusterStagedUpdateStrategySucceed(strategyName) }) AfterAll(func() { @@ -869,16 +869,16 @@ var _ = Describe("test CRP rollout with staged update run", func() { ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) // Delete the clusterStagedUpdateRun. - ensureUpdateRunDeletion(updateRunName) + ensureClusterStagedUpdateRunDeletion(updateRunName) // Delete the clusterStagedUpdateStrategy. - ensureUpdateRunStrategyDeletion(strategyName) + ensureClusterUpdateRunStrategyDeletion(strategyName) }) It("Should not rollout any resources to member clusters as there's no update run yet", checkIfRemovedWorkResourcesFromAllMemberClustersConsistently) It("Should have the latest resource snapshot", func() { - validateLatestResourceSnapshot(crpName, resourceSnapshotIndex1st) + validateLatestClusterResourceSnapshot(crpName, resourceSnapshotIndex1st) }) It("Should update crp status as pending rollout", func() { @@ -887,11 +887,11 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should successfully schedule the crp", func() { - validateLatestPolicySnapshot(crpName, policySnapshotIndex1st, 3) + validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) }) - It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) + It("Should create a cluster staged update run successfully", func() { + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) }) It("Should report diff for member-cluster-2 only and completes stage canary", func() { @@ -903,9 +903,9 @@ var _ = Describe("test CRP rollout with staged update run", func() { validateAndApproveClusterApprovalRequests(updateRunName, envCanary) }) - It("Should report diff for member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { - updateRunSucceededActual := updateRunStatusSucceededActual(updateRunName, policySnapshotIndex1st, len(allMemberClusters), applyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) - Eventually(updateRunSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) + It("Should report diff for member-cluster-1 and member-cluster-3 too and complete the cluster staged update run successfully", func() { + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, policySnapshotIndex1st, len(allMemberClusters), applyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) }) It("Should update crp status as diff reported", func() { @@ -919,7 +919,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { Context("Test CRP rollout strategy transition from rollingUpdate to external", Ordered, func() { var strategy *placementv1beta1.ClusterStagedUpdateStrategy - updateRunName := fmt.Sprintf(updateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), 0) + updateRunName := fmt.Sprintf(clusterStagedUpdateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), 0) var oldConfigMap, newConfigMap corev1.ConfigMap BeforeAll(func() { @@ -944,7 +944,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP") // Create the clusterStagedUpdateStrategy for later use. - strategy = createStagedUpdateStrategySucceed(strategyName) + strategy = createClusterStagedUpdateStrategySucceed(strategyName) oldConfigMap = appConfigMap() newConfigMap = appConfigMap() @@ -956,10 +956,10 @@ var _ = Describe("test CRP rollout with staged update run", func() { ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) // Delete the clusterStagedUpdateRun. - ensureUpdateRunDeletion(updateRunName) + ensureClusterStagedUpdateRunDeletion(updateRunName) // Delete the clusterStagedUpdateStrategy. - ensureUpdateRunStrategyDeletion(strategyName) + ensureClusterUpdateRunStrategyDeletion(strategyName) }) It("Should rollout resources to all member clusters with rollingUpdate strategy", func() { @@ -998,7 +998,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should have the new resource snapshot but CRP status should remain completed with old snapshot", func() { - validateLatestResourceSnapshot(crpName, resourceSnapshotIndex2nd) + validateLatestClusterResourceSnapshot(crpName, resourceSnapshotIndex2nd) // CRP status should still show completed with old snapshot crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames, @@ -1007,7 +1007,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Create a staged update run with new resourceSnapshotIndex and verify rollout happens", func() { - createStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex2nd, strategyName) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex2nd, strategyName) // Verify rollout to canary cluster first By("Verify that the new configmap is updated on member-cluster-2 during canary stage") @@ -1017,8 +1017,8 @@ var _ = Describe("test CRP rollout with staged update run", func() { validateAndApproveClusterApprovalRequests(updateRunName, envCanary) // Verify complete rollout - updateRunSucceededActual := updateRunStatusSucceededActual(updateRunName, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) - Eventually(updateRunSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) // Verify new configmap is on all member clusters for _, cluster := range allMemberClusters { @@ -1036,7 +1036,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { Context("Test kubectl-fleet approve plugin with cluster approval requests", Ordered, func() { var strategy *placementv1beta1.ClusterStagedUpdateStrategy - updateRunName := fmt.Sprintf(updateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), 0) + updateRunName := fmt.Sprintf(clusterStagedUpdateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), 0) BeforeAll(func() { // Create a test namespace and a configMap inside it on the hub cluster. @@ -1060,7 +1060,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP") // Create the clusterStagedUpdateStrategy. - strategy = createStagedUpdateStrategySucceed(strategyName) + strategy = createClusterStagedUpdateStrategySucceed(strategyName) }) AfterAll(func() { @@ -1068,16 +1068,16 @@ var _ = Describe("test CRP rollout with staged update run", func() { ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) // Delete the clusterStagedUpdateRun. - ensureUpdateRunDeletion(updateRunName) + ensureClusterStagedUpdateRunDeletion(updateRunName) // Delete the clusterStagedUpdateStrategy. - ensureUpdateRunStrategyDeletion(strategyName) + ensureClusterUpdateRunStrategyDeletion(strategyName) }) It("Should create a staged update run and verify cluster approval request is created", func() { - validateLatestResourceSnapshot(crpName, resourceSnapshotIndex1st) - validateLatestPolicySnapshot(crpName, policySnapshotIndex1st, 3) - createStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) + validateLatestClusterResourceSnapshot(crpName, resourceSnapshotIndex1st) + validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) // Verify that cluster approval request is created for canary stage. Eventually(func() error { @@ -1143,8 +1143,8 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should complete the staged update run after approval", func() { - updateRunSucceededActual := updateRunStatusSucceededActual(updateRunName, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) - Eventually(updateRunSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -1157,7 +1157,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { Context("Test CRP rollout strategy transition from external to rollingUpdate", Ordered, func() { var strategy *placementv1beta1.ClusterStagedUpdateStrategy - updateRunName := fmt.Sprintf(updateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), 0) + updateRunName := fmt.Sprintf(clusterStagedUpdateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), 0) var oldConfigMap, newConfigMap corev1.ConfigMap BeforeAll(func() { @@ -1182,7 +1182,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP") // Create the clusterStagedUpdateStrategy. - strategy = createStagedUpdateStrategySucceed(strategyName) + strategy = createClusterStagedUpdateStrategySucceed(strategyName) oldConfigMap = appConfigMap() newConfigMap = appConfigMap() @@ -1194,16 +1194,16 @@ var _ = Describe("test CRP rollout with staged update run", func() { ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) // Delete the clusterStagedUpdateRun. - ensureUpdateRunDeletion(updateRunName) + ensureClusterStagedUpdateRunDeletion(updateRunName) // Delete the clusterStagedUpdateStrategy. - ensureUpdateRunStrategyDeletion(strategyName) + ensureClusterUpdateRunStrategyDeletion(strategyName) }) It("Should not rollout any resources to member clusters with external strategy", checkIfRemovedWorkResourcesFromAllMemberClustersConsistently) It("Should have the latest resource snapshot", func() { - validateLatestResourceSnapshot(crpName, resourceSnapshotIndex1st) + validateLatestClusterResourceSnapshot(crpName, resourceSnapshotIndex1st) }) It("Should update crp status as pending rollout", func() { @@ -1212,12 +1212,12 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Create updateRun and verify resources are rolled out", func() { - createStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) validateAndApproveClusterApprovalRequests(updateRunName, envCanary) - updateRunSucceededActual := updateRunStatusSucceededActual(updateRunName, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) - Eventually(updateRunSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunName, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunName) checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun(allMemberClusters) }) @@ -1239,7 +1239,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should have new resource snapshot but CRP status should remain completed with old snapshot", func() { - validateLatestResourceSnapshot(crpName, resourceSnapshotIndex2nd) + validateLatestClusterResourceSnapshot(crpName, resourceSnapshotIndex2nd) // CRP status should still show completed with old snapshot crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames, @@ -1277,7 +1277,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { // Note that this container cannot run in parallel with other containers. var _ = Describe("Test member cluster join and leave flow with updateRun", Label("joinleave"), Serial, func() { crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) - strategyName := fmt.Sprintf(updateRunStrategyNameTemplate, GinkgoParallelProcess()) + strategyName := fmt.Sprintf(clusterStagedUpdateRunStrategyNameTemplate, GinkgoParallelProcess()) var strategy *placementv1beta1.ClusterStagedUpdateStrategy updateRunNames := []string{} @@ -1316,7 +1316,7 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label ObjectMeta: metav1.ObjectMeta{ Name: strategyName, }, - Spec: placementv1beta1.StagedUpdateStrategySpec{ + Spec: placementv1beta1.UpdateStrategySpec{ Stages: []placementv1beta1.StageConfig{ { Name: "all", @@ -1329,21 +1329,21 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label Expect(hubClient.Create(ctx, strategy)).To(Succeed(), "Failed to create ClusterStagedUpdateStrategy") for i := 0; i < 2; i++ { - updateRunNames = append(updateRunNames, fmt.Sprintf(updateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), i)) + updateRunNames = append(updateRunNames, fmt.Sprintf(clusterStagedUpdateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), i)) } checkIfRemovedWorkResourcesFromAllMemberClustersConsistently() By("Validating created resource snapshot and policy snapshot") - validateLatestResourceSnapshot(crpName, resourceSnapshotIndex1st) - validateLatestPolicySnapshot(crpName, policySnapshotIndex1st, 3) + validateLatestClusterResourceSnapshot(crpName, resourceSnapshotIndex1st) + validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) By("Creating the first staged update run") - createStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName) By("Validating staged update run has succeeded") - updateRunSucceededActual := updateRunStatusSucceededActual(updateRunNames[0], policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) - Eventually(updateRunSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) By("Validating CRP status as completed") crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames, @@ -1374,12 +1374,12 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label // Remove all the clusterStagedUpdateRuns. By("Deleting updateRuns") for _, name := range updateRunNames { - ensureUpdateRunDeletion(name) + ensureClusterStagedUpdateRunDeletion(name) } // Delete the clusterStagedUpdateStrategy. By("Deleting clusterStagedUpdateStrategy") - ensureUpdateRunStrategyDeletion(strategyName) + ensureClusterUpdateRunStrategyDeletion(strategyName) }) Context("UpdateRun should delete the binding of a left cluster but resources are kept", Label("joinleave"), Ordered, Serial, func() { @@ -1389,13 +1389,13 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label }) It("Should create another staged update run for the same CRP", func() { - validateLatestPolicySnapshot(crpName, policySnapshotIndex1st, 2) - createStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName) + validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 2) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName) }) It("Should complete the second staged update run and complete the CRP", func() { - updateRunSucceededActual := updateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1], allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil) - Eventually(updateRunSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex1st, 2, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1], allMemberClusterNames[2]}}, []string{allMemberClusterNames[0]}, nil, nil) + Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames[1:], []string{resourceSnapshotIndex1st, resourceSnapshotIndex1st}, []bool{true, true}, nil, nil) @@ -1436,14 +1436,14 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label checkIfMemberClusterHasJoined(allMemberClusters[0]) }) - It("Should reschedule to member cluster 1 and create a new staged update run successfully", func() { - validateLatestPolicySnapshot(crpName, policySnapshotIndex1st, 3) - createStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName) + It("Should reschedule to member cluster 1 and create a new cluster staged update run successfully", func() { + validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName) }) It("Should complete the staged update run, complete CRP, and rollout resources to all member clusters", func() { - updateRunSucceededActual := updateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) - Eventually(updateRunSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames, []string{resourceSnapshotIndex1st, resourceSnapshotIndex1st, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) @@ -1476,17 +1476,17 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label }) It("Should have the latest resource snapshot with updated resources", func() { - validateLatestResourceSnapshot(crpName, resourceSnapshotIndex2nd) + validateLatestClusterResourceSnapshot(crpName, resourceSnapshotIndex2nd) }) - It("Should reschedule to member cluster 1 and create a new staged update run successfully", func() { - validateLatestPolicySnapshot(crpName, policySnapshotIndex1st, 3) - createStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex2nd, strategyName) + It("Should reschedule to member cluster 1 and create a new cluster staged update run successfully", func() { + validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex2nd, strategyName) }) It("Should complete the staged update run, complete CRP, and rollout updated resources to all member clusters", func() { - updateRunSucceededActual := updateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) - Eventually(updateRunSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex2nd, true, allMemberClusterNames, []string{resourceSnapshotIndex2nd, resourceSnapshotIndex2nd, resourceSnapshotIndex2nd}, []bool{true, true, true}, nil, nil) @@ -1518,14 +1518,14 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label checkIfMemberClusterHasJoined(allMemberClusters[0]) }) - It("Should reschedule to member cluster 1 and create a new staged update run successfully", func() { - validateLatestPolicySnapshot(crpName, policySnapshotIndex1st, 3) - createStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName) + It("Should reschedule to member cluster 1 and create a new cluster staged update run successfully", func() { + validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName) }) It("Should complete the staged update run, complete CRP, and re-place resources to all member clusters", func() { - updateRunSucceededActual := updateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) - Eventually(updateRunSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) + csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[1], policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(workResourceIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames, []string{resourceSnapshotIndex1st, resourceSnapshotIndex1st, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) @@ -1574,12 +1574,12 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label }) }) -func createStagedUpdateStrategySucceed(strategyName string) *placementv1beta1.ClusterStagedUpdateStrategy { +func createClusterStagedUpdateStrategySucceed(strategyName string) *placementv1beta1.ClusterStagedUpdateStrategy { strategy := &placementv1beta1.ClusterStagedUpdateStrategy{ ObjectMeta: metav1.ObjectMeta{ Name: strategyName, }, - Spec: placementv1beta1.StagedUpdateStrategySpec{ + Spec: placementv1beta1.UpdateStrategySpec{ Stages: []placementv1beta1.StageConfig{ { Name: envCanary, @@ -1615,7 +1615,7 @@ func createStagedUpdateStrategySucceed(strategyName string) *placementv1beta1.Cl return strategy } -func validateLatestPolicySnapshot(crpName, wantPolicySnapshotIndex string, wantSelectedClusterCount int) { +func validateLatestClusterSchedulingPolicySnapshot(crpName, wantPolicySnapshotIndex string, wantSelectedClusterCount int) { Eventually(func() (string, error) { var policySnapshotList placementv1beta1.ClusterSchedulingPolicySnapshotList if err := hubClient.List(ctx, &policySnapshotList, client.MatchingLabels{ @@ -1645,7 +1645,7 @@ func validateLatestPolicySnapshot(crpName, wantPolicySnapshotIndex string, wantS }, eventuallyDuration, eventuallyInterval).Should(Equal(wantPolicySnapshotIndex), "Policy snapshot index does not match") } -func validateLatestResourceSnapshot(crpName, wantResourceSnapshotIndex string) { +func validateLatestClusterResourceSnapshot(crpName, wantResourceSnapshotIndex string) { Eventually(func() (string, error) { crsList := &placementv1beta1.ClusterResourceSnapshotList{} if err := hubClient.List(ctx, crsList, client.MatchingLabels{ @@ -1661,12 +1661,12 @@ func validateLatestResourceSnapshot(crpName, wantResourceSnapshotIndex string) { }, eventuallyDuration, eventuallyInterval).Should(Equal(wantResourceSnapshotIndex), "Resource snapshot index does not match") } -func createStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex, strategyName string) { +func createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex, strategyName string) { updateRun := &placementv1beta1.ClusterStagedUpdateRun{ ObjectMeta: metav1.ObjectMeta{ Name: updateRunName, }, - Spec: placementv1beta1.StagedUpdateRunSpec{ + Spec: placementv1beta1.UpdateRunSpec{ PlacementName: crpName, ResourceSnapshotIndex: resourceSnapshotIndex, StagedUpdateStrategyName: strategyName, diff --git a/test/e2e/mixed_placement_test.go b/test/e2e/mixed_placement_test.go index 050f29925..f82aed417 100644 --- a/test/e2e/mixed_placement_test.go +++ b/test/e2e/mixed_placement_test.go @@ -277,15 +277,12 @@ var _ = Describe("mixed ClusterResourcePlacement and ResourcePlacement negative Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s status as expected", rpName) By("creating a CRP that uses PickAll strategy") + // Note that this CRP will take over namespaces. createCRP(crpName) }) AfterAll(func() { ensureRPAndRelatedResourcesDeleted(types.NamespacedName{Name: rpName, Namespace: workNamespaceName}, allMemberClusters) - By("CRP should take over the work resources") - // ConfigMap on the hub was deleted in the previous step. - crpStatusUpdatedActual := crpStatusUpdatedActual(workNamespaceIdentifiers(), allMemberClusterNames, nil, "1") - Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected") ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) @@ -345,5 +342,34 @@ var _ = Describe("mixed ClusterResourcePlacement and ResourcePlacement negative } Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s status as expected", rpName) }) + + // The CRP must give up on the configMap first, otherwise it might attempt re-create the configMap + // during the cleanup process. + It("can update the CRP to give up on the configMap", func() { + Eventually(func() error { + crp := &placementv1beta1.ClusterResourcePlacement{} + if err := hubClient.Get(ctx, types.NamespacedName{Name: crpName}, crp); err != nil { + return fmt.Errorf("failed to get the CRP %s: %w", crpName, err) + } + + crp.Spec.ResourceSelectors = namespaceOnlySelector() + if err := hubClient.Update(ctx, crp); err != nil { + return fmt.Errorf("failed to update the CRP %s: %w", crpName, err) + } + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update the CRP %s to give up on the configMap", crpName) + }) + + It("should update CRP status as expected", func() { + nsOnlyResIds := []placementv1beta1.ResourceIdentifier{ + { + Kind: "Namespace", + Name: workNamespaceName, + Version: "v1", + }, + } + crpStatusUpdatedActual := crpStatusUpdatedActual(nsOnlyResIds, allMemberClusterNames, nil, "1") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected") + }) }) }) diff --git a/test/e2e/placement_apply_strategy_test.go b/test/e2e/placement_apply_strategy_test.go index 3f0b24d44..a4c07dac8 100644 --- a/test/e2e/placement_apply_strategy_test.go +++ b/test/e2e/placement_apply_strategy_test.go @@ -25,6 +25,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" @@ -601,9 +602,30 @@ var _ = Describe("validating CRP when resources exists", Ordered, func() { }) AfterAll(func() { - ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + // Must delete the conflicting CRP first, otherwise it might re-create the resources when we check + // if the original CRP has been fully deleted. + // + // And here the test suite will not use the shared common deletion logic as it will attempt to verify + // resources that are not managed by the conflicting CRP. + Eventually(func() error { + conflictedCRP := &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: conflictedCRPName, + }, + } + + if err := hubClient.Delete(ctx, conflictedCRP); err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete CRP %s: %w", conflictedCRPName, err) + } - ensureCRPAndRelatedResourcesDeleted(conflictedCRPName, allMemberClusters) + // Wait until the CRP is fully deleted. + if err := hubClient.Get(ctx, types.NamespacedName{Name: conflictedCRPName}, conflictedCRP); !errors.IsNotFound(err) { + return fmt.Errorf("CRP %s is still present or an unexpected error has occurred: %w", conflictedCRPName, err) + } + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to delete CRP %s", conflictedCRPName) + + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) }) }) diff --git a/test/e2e/placement_drift_diff_test.go b/test/e2e/placement_drift_diff_test.go index ea2f3e360..eb6c85daa 100644 --- a/test/e2e/placement_drift_diff_test.go +++ b/test/e2e/placement_drift_diff_test.go @@ -504,10 +504,14 @@ var _ = Describe("take over existing resources", func() { }) AfterAll(func() { + // The CRP must be deleted first, otherwise the pre-existing namespace might get re-created. + // + // Also, do not attempt to verify deletion on the first member cluster, as a pre-existing namespace + // lives there. + ensureCRPAndRelatedResourcesDeleted(crpName, []*framework.Cluster{memberCluster2EastCanary, memberCluster3WestProd}) + // The pre-existing namespace has not been taken over and must be deleted manually. cleanWorkResourcesOnCluster(memberCluster1EastProd) - - ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) }) }) diff --git a/test/e2e/placement_eviction_test.go b/test/e2e/placement_eviction_test.go index d4742d7b4..ab0af6bc7 100644 --- a/test/e2e/placement_eviction_test.go +++ b/test/e2e/placement_eviction_test.go @@ -18,6 +18,7 @@ package e2e import ( "fmt" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -152,13 +153,25 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding, no taint s Eventually(crpEvictionStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement eviction status as expected") }) - // specifying a longer timeout, the namespace is being evicted while a new namespace is propagated with the same name leading to a failed takeover. + // The two test specs below use a longer timeout; due to the async nature of our controllers, the controllers + // might not act fast enough even if the eviction object has reported a successful eviction (the eviction controller + // considers an eviction to be successful as soon as the target binding has been marked for deletion; it will + // not wait for its final removal), which might lead to a number of inconsistencies. + // + // TO-DO: longer timeouts are not an ultimate solution; identify better signals that can prove that the CRP + // status has been refreshed (LTT?). It("should ensure evicted cluster is picked again by scheduler & update cluster resource placement status as expected", func() { crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") - Eventually(crpStatusUpdatedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected") + Eventually(crpStatusUpdatedActual, workloadEventuallyDuration, time.Second*5).Should(Succeed(), "Failed to update cluster resource placement status as expected") }) - It("should still place resources on the all available member clusters", checkIfPlacedWorkResourcesOnAllMemberClusters) + It("should still place resources on the all available member clusters", func() { + for idx := range allMemberClusters { + memberCluster := allMemberClusters[idx] + workResourcesPlacedActual := workNamespaceAndConfigMapPlacedOnClusterActual(memberCluster) + Eventually(workResourcesPlacedActual, workloadEventuallyDuration, time.Second*5).Should(Succeed(), "Failed to place work resources on member cluster %s", memberCluster.ClusterName) + } + }) }) var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickAll CRP, PDB with MinAvailable specified as Integer to protect resources on all clusters, eviction denied", Ordered, func() { diff --git a/test/e2e/placement_pickall_test.go b/test/e2e/placement_pickall_test.go index fffc403d7..9d60fa804 100644 --- a/test/e2e/placement_pickall_test.go +++ b/test/e2e/placement_pickall_test.go @@ -960,11 +960,16 @@ var _ = Describe("placing resources using a CRP of PickAll placement type", func }, PropertySelector: &placementv1beta1.PropertySelector{ MatchExpressions: []placementv1beta1.PropertySelectorRequirement{ + // Important: the test environment has the Azure VM SKUs set up manually, + // but the CPU/memory total capacities will always use the actual values, + // which are the number of CPU cores/the amount of RAM installed on the + // local machine (or remote VM). As a result, there are chances where + // the per CPU core/per GB memory costs will become 0. { Name: azure.PerGBMemoryCostProperty, - Operator: placementv1beta1.PropertySelectorEqualTo, + Operator: placementv1beta1.PropertySelectorGreaterThanOrEqualTo, Values: []string{ - "0", + "9999", }, }, }, diff --git a/test/e2e/placement_pickfixed_test.go b/test/e2e/placement_pickfixed_test.go index 496680708..a6f8acb97 100644 --- a/test/e2e/placement_pickfixed_test.go +++ b/test/e2e/placement_pickfixed_test.go @@ -240,7 +240,7 @@ var _ = Describe("placing resources using a CRP of PickFixed placement type", fu Eventually(resourcePlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place resources on specified clusters") }) - It("should add finalizer to work resources on the specified clusters", func() { + It("can add finalizer to work resources on the specified clusters", func() { Eventually(func() error { if err := memberCluster1EastProd.KubeClient.Get(ctx, types.NamespacedName{Namespace: workNamespaceName, Name: appConfigMapName}, ¤tConfigMap); err != nil { return err @@ -270,22 +270,52 @@ var _ = Describe("placing resources using a CRP of PickFixed placement type", fu }) It("should have a deletion timestamp on work objects", func() { - work := &placementv1beta1.Work{} - Expect(hubClient.Get(ctx, types.NamespacedName{Namespace: fmt.Sprintf("fleet-member-%s", memberCluster1EastProdName), Name: fmt.Sprintf("%s-work", crpName)}, work)).Should(Succeed(), "Failed to get work") - Expect(work.DeletionTimestamp).ShouldNot(BeNil(), "Work should have a deletion timestamp") + // Use an Eventually block as the Fleet controllers might not be fast enough. + // + // Note that the CRP controller will ignore deleted bindings when reporting status. + Eventually(func() error { + work := &placementv1beta1.Work{} + reservedMemberNSName := fmt.Sprintf("fleet-member-%s", memberCluster1EastProdName) + workName := fmt.Sprintf("%s-work", crpName) + if err := hubClient.Get(ctx, types.NamespacedName{Namespace: reservedMemberNSName, Name: workName}, work); err != nil { + return fmt.Errorf("failed to get work: %w", err) + } + + if work.DeletionTimestamp == nil { + return fmt.Errorf("work %s in namespace %s has not yet been marked for deletion", workName, reservedMemberNSName) + } + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to verify that work has been marked for deletion") - appliedWork := &placementv1beta1.AppliedWork{} - Expect(memberCluster1EastProd.KubeClient.Get(ctx, types.NamespacedName{Name: fmt.Sprintf("%s-work", crpName)}, appliedWork)).Should(Succeed(), "Failed to get appliedwork") - Expect(appliedWork.DeletionTimestamp).ShouldNot(BeNil(), "AppliedWork should have a deletion timestamp") + Eventually(func() error { + appliedWork := &placementv1beta1.AppliedWork{} + appliedWorkName := fmt.Sprintf("%s-work", crpName) + if err := memberCluster1EastProd.KubeClient.Get(ctx, types.NamespacedName{Name: appliedWorkName}, appliedWork); err != nil { + return fmt.Errorf("failed to get appliedwork: %w", err) + } + + if appliedWork.DeletionTimestamp == nil { + return fmt.Errorf("appliedwork %s has not yet been marked for deletion", appliedWorkName) + } + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to verify that appliedwork has been marked for deletion") }) It("configmap should still exists on previously specified cluster and be in deleting state", func() { - configMap := &corev1.ConfigMap{} - Expect(memberCluster1EastProd.KubeClient.Get(ctx, types.NamespacedName{Namespace: workNamespaceName, Name: appConfigMapName}, configMap)).Should(Succeed(), "Failed to get configmap") - Expect(configMap.DeletionTimestamp).ShouldNot(BeNil(), "ConfigMap should have a deletion timestamp") + // Use an Eventually block as the Fleet agent might not be fast enough. + Eventually(func() error { + configMap := &corev1.ConfigMap{} + if err := memberCluster1EastProd.KubeClient.Get(ctx, types.NamespacedName{Namespace: workNamespaceName, Name: appConfigMapName}, configMap); err != nil { + return err + } + if configMap.DeletionTimestamp == nil { + return fmt.Errorf("configMap %s has not yet been marked for deletion", appConfigMapName) + } + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to verify that configMap has been marked for deletion") }) - It("should remove finalizer from work resources on the specified clusters", func() { + It("can remove finalizer from work resources on the specified clusters", func() { configMap := &corev1.ConfigMap{} Expect(memberCluster1EastProd.KubeClient.Get(ctx, types.NamespacedName{Namespace: workNamespaceName, Name: appConfigMapName}, configMap)).Should(Succeed(), "Failed to get configmap") controllerutil.RemoveFinalizer(configMap, "example.com/finalizer") diff --git a/test/e2e/placement_ro_test.go b/test/e2e/placement_ro_test.go index ef03cff3e..92352c0c5 100644 --- a/test/e2e/placement_ro_test.go +++ b/test/e2e/placement_ro_test.go @@ -26,6 +26,8 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" scheduler "go.goms.io/fleet/pkg/scheduler/framework" @@ -581,8 +583,45 @@ var _ = Context("creating resourceOverride and resource becomes invalid after ov BeforeAll(func() { By("creating work resources") createWorkResources() + // Create the CRP. - createCRP(crpName) + crp := &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + // Add a custom finalizer; this would allow us to better observe + // the behavior of the controllers. + Finalizers: []string{customDeletionBlockerFinalizer}, + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: workResourceSelector(), + Strategy: placementv1beta1.RolloutStrategy{ + Type: placementv1beta1.RollingUpdateRolloutStrategyType, + RollingUpdate: &placementv1beta1.RollingUpdateConfig{ + UnavailablePeriodSeconds: ptr.To(2), + MaxUnavailable: ptr.To(intstr.FromString("100%")), + }, + }, + }, + } + Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP %s", crpName) + }) + + AfterAll(func() { + By(fmt.Sprintf("deleting placement %s and related resources", crpName)) + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + + By(fmt.Sprintf("deleting resourceOverride %s", roName)) + cleanupResourceOverride(roName, roNamespace) + }) + + // Verify the status before creating the overrides, so that we can be certain about the resource index + // to check for when the override is actually being picked up by Fleet agents. + It("should update CRP status as expected", func() { + crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) + }) + + It("can create a override that breaks the resource", func() { // Create the ro. ro := &placementv1beta1.ResourceOverride{ ObjectMeta: metav1.ObjectMeta{ @@ -616,14 +655,6 @@ var _ = Context("creating resourceOverride and resource becomes invalid after ov Expect(hubClient.Create(ctx, ro)).To(Succeed(), "Failed to create resourceOverride %s", roName) }) - AfterAll(func() { - By(fmt.Sprintf("deleting placement %s and related resources", crpName)) - ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) - - By(fmt.Sprintf("deleting resourceOverride %s", roName)) - cleanupResourceOverride(roName, roNamespace) - }) - It("should update CRP status as expected", func() { wantRONames := []placementv1beta1.NamespacedName{ {Namespace: roNamespace, Name: fmt.Sprintf(placementv1beta1.OverrideSnapshotNameFmt, roName, 0)}, @@ -632,8 +663,13 @@ var _ = Context("creating resourceOverride and resource becomes invalid after ov Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) }) - // This check will ignore the annotation of resources. - It("should not place the selected resources on member clusters", checkIfRemovedWorkResourcesFromAllMemberClusters) + // For simplicity reasons, this test spec will only check if the annotation hasn't been added. + It("should not place the selected resources on member clusters", func() { + for idx := range allMemberClusters { + memberCluster := allMemberClusters[idx] + Eventually(validateConfigMapNoAnnotationKeyOnCluster(memberCluster, roTestAnnotationKey)).Should(Succeed(), "Failed to find the annotation of config map on %s", memberCluster.ClusterName) + } + }) }) var _ = Context("creating resourceOverride with a templated rules with cluster name to override configMap", Ordered, func() { diff --git a/test/e2e/resource_placement_apply_strategy_test.go b/test/e2e/resource_placement_apply_strategy_test.go index 528606335..8ce402920 100644 --- a/test/e2e/resource_placement_apply_strategy_test.go +++ b/test/e2e/resource_placement_apply_strategy_test.go @@ -25,7 +25,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" @@ -575,8 +575,31 @@ var _ = Describe("validating resource placement using different apply strategies }) AfterAll(func() { + // Must delete the conflicting CRP first, otherwise it might re-create the resources when we check + // if the original CRP has been fully deleted. + // + // And here the test suite will not use the shared common deletion logic as it will attempt to verify + // resources that are not managed by the conflicting CRP. + Eventually(func() error { + conflictedRP := &placementv1beta1.ResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: conflictedRPName, + Namespace: workNamespaceName, + }, + } + + if err := hubClient.Delete(ctx, conflictedRP); err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete RP %s: %w", conflictedRPName, err) + } + + // Wait until the RP is fully deleted. + if err := hubClient.Get(ctx, types.NamespacedName{Name: conflictedRPName, Namespace: workNamespaceName}, conflictedRP); !errors.IsNotFound(err) { + return fmt.Errorf("RP %s is still present or an unexpected error has occurred: %w", conflictedRPName, err) + } + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to delete RP %s", conflictedRPName) + ensureRPAndRelatedResourcesDeleted(types.NamespacedName{Name: rpName, Namespace: workNamespaceName}, allMemberClusters) - ensureRPAndRelatedResourcesDeleted(types.NamespacedName{Name: conflictedRPName, Namespace: workNamespaceName}, allMemberClusters) }) }) }) @@ -1412,7 +1435,7 @@ func createAnotherConfigMap(name types.NamespacedName) { func cleanupAnotherConfigMap(name types.NamespacedName) { cm := &corev1.ConfigMap{} err := hubClient.Get(ctx, name, cm) - if err != nil && apierrors.IsNotFound(err) { + if err != nil && errors.IsNotFound(err) { return } Expect(err).To(Succeed(), "Failed to get config map %s", name) @@ -1423,7 +1446,7 @@ func cleanupAnotherConfigMap(name types.NamespacedName) { cm := &corev1.ConfigMap{} err := hubClient.Get(ctx, name, cm) if err != nil { - if apierrors.IsNotFound(err) { + if errors.IsNotFound(err) { return nil } return err diff --git a/test/e2e/resource_placement_drift_diff_test.go b/test/e2e/resource_placement_drift_diff_test.go index d9b547671..ec2131786 100644 --- a/test/e2e/resource_placement_drift_diff_test.go +++ b/test/e2e/resource_placement_drift_diff_test.go @@ -69,6 +69,7 @@ var _ = Describe("take over existing resources using RP", Label("resourceplaceme Expect(memberCluster1EastProdClient.Create(ctx, &cm2)).To(Succeed()) By("Create the CRP with Namespace-only selector") + // Note that this CRP will take over pre-existing namespaces on member clusters. crpName = fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) crp := &placementv1beta1.ClusterResourcePlacement{ ObjectMeta: metav1.ObjectMeta{ @@ -98,13 +99,7 @@ var _ = Describe("take over existing resources using RP", Label("resourceplaceme }) AfterEach(OncePerOrdered, func() { - // The pre-existing resource that have not been taken over and must be deleted manually. - cleanupAnotherConfigMapOnMemberCluster(types.NamespacedName{Name: cm2Name, Namespace: nsName}, memberCluster1EastProd) - cleanWorkResourcesOnCluster(memberCluster1EastProd) - - // Clean up second configMap on the hub cluster. - cleanupAnotherConfigMap(types.NamespacedName{Name: cm2Name, Namespace: nsName}) - + // No need to clean up pre-existing resources as the namespace has been taken over. ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) @@ -618,13 +613,10 @@ var _ = Describe("detect drifts on placed resources using RP", Ordered, Label("r }) AfterEach(OncePerOrdered, func() { - // The pre-existing resource that have not been taken over and must be deleted manually. - cleanupAnotherConfigMapOnMemberCluster(types.NamespacedName{Name: cm2Name, Namespace: nsName}, memberCluster1EastProd) - cleanWorkResourcesOnCluster(memberCluster1EastProd) - // Clean up second configMap on the hub cluster. cleanupAnotherConfigMap(types.NamespacedName{Name: cm2Name, Namespace: nsName}) + // The CRP owns the namespace; do not verify if the namespace has been deleted until the CRP is gone. ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) @@ -1201,7 +1193,9 @@ var _ = Describe("report diff mode using RP", Label("resourceplacement"), func() } Expect(memberCluster1EastProdClient.Create(ctx, &cm2)).To(Succeed()) - // Create the CRP with Namespace-only selector + // Create the CRP with Namespace-only selector. + // + // Note that this CRP will take over existing namespaces. crp := &placementv1beta1.ClusterResourcePlacement{ ObjectMeta: metav1.ObjectMeta{ Name: crpName, @@ -1577,11 +1571,14 @@ var _ = Describe("report diff mode using RP", Label("resourceplacement"), func() }) AfterAll(func() { - // The pre-existing namespace resources that have not been taken over and must be deleted manually. - cleanWorkResourcesOnCluster(memberCluster1EastProd) - cleanupAnotherConfigMapOnMemberCluster(types.NamespacedName{Name: cm2Name, Namespace: nsName}, memberCluster1EastProd) + // Clean up the created resources. - ensureRPAndRelatedResourcesDeleted(types.NamespacedName{Name: rpName, Namespace: nsName}, allMemberClusters) + // For the RP-related resources, ignore the first member cluster as it has pre-existing resources. + ensureRPAndRelatedResourcesDeleted(types.NamespacedName{Name: rpName, Namespace: nsName}, []*framework.Cluster{ + memberCluster2EastCanary, + memberCluster3WestProd, + }) + // Note that the pre-existing namespace (not the configMaps) has been taken over by the CRP. ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) }) }) diff --git a/test/e2e/resource_placement_pickall_test.go b/test/e2e/resource_placement_pickall_test.go index 7b9c49c35..0925eb545 100644 --- a/test/e2e/resource_placement_pickall_test.go +++ b/test/e2e/resource_placement_pickall_test.go @@ -811,11 +811,16 @@ var _ = Describe("placing namespaced scoped resources using a RP with PickAll po }, PropertySelector: &placementv1beta1.PropertySelector{ MatchExpressions: []placementv1beta1.PropertySelectorRequirement{ + // Important: the test environment has the Azure VM SKUs set up manually, + // but the CPU/memory total capacities will always use the actual values, + // which are the number of CPU cores/the amount of RAM installed on the + // local machine (or remote VM). As a result, there are chances where + // the per CPU core/per GB memory costs will become 0. { Name: azure.PerGBMemoryCostProperty, - Operator: placementv1beta1.PropertySelectorEqualTo, + Operator: placementv1beta1.PropertySelectorGreaterThanOrEqualTo, Values: []string{ - "0", + "9999", }, }, }, diff --git a/test/e2e/resource_placement_pickfixed_test.go b/test/e2e/resource_placement_pickfixed_test.go index e1442d766..87c168a7c 100644 --- a/test/e2e/resource_placement_pickfixed_test.go +++ b/test/e2e/resource_placement_pickfixed_test.go @@ -248,7 +248,7 @@ var _ = Describe("placing namespaced scoped resources using an RP with PickFixed Eventually(resourcePlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place resources on specified clusters") }) - It("should add finalizer to work resources on the specified clusters", func() { + It("can add finalizer to work resources on the specified clusters", func() { Eventually(func() error { if err := memberCluster1EastProd.KubeClient.Get(ctx, types.NamespacedName{Namespace: workNamespaceName, Name: appConfigMapName}, ¤tConfigMap); err != nil { return err @@ -278,19 +278,39 @@ var _ = Describe("placing namespaced scoped resources using an RP with PickFixed }) It("should have a deletion timestamp on work objects", func() { - work := &placementv1beta1.Work{} - workName := fmt.Sprintf("%s.%s-work", rpKey.Namespace, rpName) - Expect(hubClient.Get(ctx, types.NamespacedName{Namespace: fmt.Sprintf("fleet-member-%s", memberCluster1EastProdName), Name: workName}, work)).Should(Succeed(), "Failed to get work") - Expect(work.DeletionTimestamp).ShouldNot(BeNil(), "Work should have a deletion timestamp") + // Use an Eventually block as the Fleet controllers might not be fast enough. + // + // Note that the CRP controller will ignore deleted bindings when reporting status. + Eventually(func() error { + work := &placementv1beta1.Work{} + reservedMemberNSName := fmt.Sprintf("fleet-member-%s", memberCluster1EastProdName) + workName := fmt.Sprintf("%s.%s-work", rpKey.Namespace, rpName) + if err := hubClient.Get(ctx, types.NamespacedName{Namespace: reservedMemberNSName, Name: workName}, work); err != nil { + return fmt.Errorf("failed to get work: %w", err) + } + + if work.DeletionTimestamp == nil { + return fmt.Errorf("work %s in namespace %s has not yet been marked for deletion", workName, reservedMemberNSName) + } + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to verify that work has been marked for deletion") }) It("configmap should still exists on previously specified cluster and be in deleting state", func() { - configMap := &corev1.ConfigMap{} - Expect(memberCluster1EastProd.KubeClient.Get(ctx, types.NamespacedName{Namespace: workNamespaceName, Name: appConfigMapName}, configMap)).Should(Succeed(), "Failed to get configmap") - Expect(configMap.DeletionTimestamp).ShouldNot(BeNil(), "ConfigMap should have a deletion timestamp") + // Use an Eventually block as the Fleet agent might not be fast enough. + Eventually(func() error { + configMap := &corev1.ConfigMap{} + if err := memberCluster1EastProd.KubeClient.Get(ctx, types.NamespacedName{Namespace: workNamespaceName, Name: appConfigMapName}, configMap); err != nil { + return err + } + if configMap.DeletionTimestamp == nil { + return fmt.Errorf("configMap %s has not yet been marked for deletion", appConfigMapName) + } + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to verify that configMap has been marked for deletion") }) - It("should remove finalizer from work resources on the specified clusters", func() { + It("can remove finalizer from work resources on the specified clusters", func() { configMap := &corev1.ConfigMap{} Expect(memberCluster1EastProd.KubeClient.Get(ctx, types.NamespacedName{Namespace: workNamespaceName, Name: appConfigMapName}, configMap)).Should(Succeed(), "Failed to get configmap") controllerutil.RemoveFinalizer(configMap, "example.com/finalizer") diff --git a/test/e2e/resource_placement_ro_test.go b/test/e2e/resource_placement_ro_test.go index dcbdb92bd..c71ebde12 100644 --- a/test/e2e/resource_placement_ro_test.go +++ b/test/e2e/resource_placement_ro_test.go @@ -26,6 +26,8 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" scheduler "go.goms.io/fleet/pkg/scheduler/framework" @@ -489,8 +491,43 @@ var _ = Describe("placing namespaced scoped resources using a RP with ResourceOv createConfigMap() // Create the RP. - createRP(workNamespace, rpName) + rp := &placementv1beta1.ResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: rpName, + Namespace: fmt.Sprintf(workNamespaceNameTemplate, GinkgoParallelProcess()), + // Add a custom finalizer; this would allow us to better observe + // the behavior of the controllers. + Finalizers: []string{customDeletionBlockerFinalizer}, + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: configMapSelector(), + Strategy: placementv1beta1.RolloutStrategy{ + Type: placementv1beta1.RollingUpdateRolloutStrategyType, + RollingUpdate: &placementv1beta1.RollingUpdateConfig{ + UnavailablePeriodSeconds: ptr.To(2), + MaxUnavailable: ptr.To(intstr.FromString("100%")), + }, + }, + }, + } + Expect(hubClient.Create(ctx, rp)).To(Succeed(), "Failed to create resource placement %s/%s", workNamespace, rpName) + }) + + AfterAll(func() { + By(fmt.Sprintf("deleting resource placement %s/%s and related resources", workNamespace, rpName)) + ensureRPAndRelatedResourcesDeleted(types.NamespacedName{Name: rpName, Namespace: workNamespace}, allMemberClusters) + + By(fmt.Sprintf("deleting resourceOverride %s", roName)) + cleanupResourceOverride(roName, workNamespace) + }) + // Verify the status before creating the overrides for consistency reasons. + It("should update the RP status as expected", func() { + rpStatusUpdatedActual := rpStatusUpdatedActual(appConfigMapIdentifiers(), allMemberClusterNames, nil, "0") + Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s status as expected", rpName) + }) + + It("can create the resource override that breaks the resource", func() { // Create the ro. ro := &placementv1beta1.ResourceOverride{ ObjectMeta: metav1.ObjectMeta{ @@ -525,14 +562,6 @@ var _ = Describe("placing namespaced scoped resources using a RP with ResourceOv Expect(hubClient.Create(ctx, ro)).To(Succeed(), "Failed to create resourceOverride %s", roName) }) - AfterAll(func() { - By(fmt.Sprintf("deleting resource placement %s/%s and related resources", workNamespace, rpName)) - ensureRPAndRelatedResourcesDeleted(types.NamespacedName{Name: rpName, Namespace: workNamespace}, allMemberClusters) - - By(fmt.Sprintf("deleting resourceOverride %s", roName)) - cleanupResourceOverride(roName, workNamespace) - }) - It("should update RP status as expected", func() { wantRONames := []placementv1beta1.NamespacedName{ {Namespace: workNamespace, Name: fmt.Sprintf(placementv1beta1.OverrideSnapshotNameFmt, roName, 0)}, @@ -541,8 +570,13 @@ var _ = Describe("placing namespaced scoped resources using a RP with ResourceOv Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s status as expected", rpName) }) - // This check will ignore the annotation of resources. - It("should not place the selected resources on member clusters", checkIfRemovedConfigMapFromAllMemberClusters) + // For simplicity reasons, this test spec will only check if the annotation hasn't been added. + It("should not place the selected resources on member clusters", func() { + for idx := range allMemberClusters { + memberCluster := allMemberClusters[idx] + Eventually(validateConfigMapNoAnnotationKeyOnCluster(memberCluster, roTestAnnotationKey)).Should(Succeed(), "Failed to find the annotation of config map on %s", memberCluster.ClusterName) + } + }) }) Context("creating resourceOverride with templated rules with cluster name to override configMap for ResourcePlacement", Ordered, func() { diff --git a/test/e2e/resources/statefulset-with-volume.yaml b/test/e2e/resources/statefulset-with-volume.yaml index 36d7d8321..cb7f5dd69 100644 --- a/test/e2e/resources/statefulset-with-volume.yaml +++ b/test/e2e/resources/statefulset-with-volume.yaml @@ -1,35 +1,28 @@ apiVersion: apps/v1 kind: StatefulSet metadata: - name: web + name: test-ss-with-volume spec: selector: matchLabels: - app: nginx # has to match .spec.template.metadata.labels - serviceName: "nginx" + app: test-ss-with-volume # has to match .spec.template.metadata.labels + serviceName: "test-ss-with-volume-svc" replicas: 3 # by default is 1 - minReadySeconds: 10 # by default is 0 template: metadata: labels: - app: nginx # has to match .spec.selector.matchLabels + app: test-ss-with-volume # has to match .spec.selector.matchLabels spec: terminationGracePeriodSeconds: 10 containers: - - name: nginx - image: registry.k8s.io/nginx-slim:0.24 - ports: - - containerPort: 80 - name: web - volumeMounts: - - name: www - mountPath: /usr/share/nginx/html + - name: pause + image: k8s.gcr.io/pause:3.8 volumeClaimTemplates: - - metadata: - name: www - spec: - accessModes: [ "ReadWriteOnce" ] - storageClassName: "my-storage-class" - resources: - requests: - storage: 1Gi \ No newline at end of file + - metadata: + name: test-ss-with-volume-pvc + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: "non-existent-sc" + resources: + requests: + storage: 1Gi \ No newline at end of file diff --git a/test/e2e/resources/test-daemonset.yaml b/test/e2e/resources/test-daemonset.yaml index e901b8964..e546f7407 100644 --- a/test/e2e/resources/test-daemonset.yaml +++ b/test/e2e/resources/test-daemonset.yaml @@ -1,34 +1,27 @@ apiVersion: apps/v1 kind: DaemonSet metadata: - name: fluentd-elasticsearch + name: test-ds namespace: test-ns labels: - k8s-app: fluentd-logging + k8s-app: test-ds spec: selector: matchLabels: - name: fluentd-elasticsearch + name: test-ds template: metadata: labels: - name: fluentd-elasticsearch + name: test-ds spec: containers: - - name: fluentd-elasticsearch - image: quay.io/fluentd_elasticsearch/fluentd:v2.5.2 + - name: pause + image: k8s.gcr.io/pause:3.8 resources: limits: - cpu: 250m - memory: 200Mi - requests: cpu: 100m memory: 200Mi - volumeMounts: - - name: varlog - mountPath: /var/log + requests: + cpu: 1m + memory: 1Mi terminationGracePeriodSeconds: 30 - volumes: - - name: varlog - hostPath: - path: /var/logask \ No newline at end of file diff --git a/test/e2e/resources/test-deployment.yaml b/test/e2e/resources/test-deployment.yaml index 5cb9bec0a..f675d1d4f 100644 --- a/test/e2e/resources/test-deployment.yaml +++ b/test/e2e/resources/test-deployment.yaml @@ -1,41 +1,25 @@ -# Copyright 2021 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - apiVersion: apps/v1 kind: Deployment metadata: - name: test-nginx + name: test-deploy namespace: default spec: selector: matchLabels: - app: test-nginx + app: test-deploy replicas: 2 template: metadata: labels: - app: test-nginx + app: test-deploy spec: containers: - - name: nginx - image: nginx:1.14.2 + - name: pause + image: k8s.gcr.io/pause:3.8 resources: requests: - cpu: 50m - memory: 200Mi + cpu: 1m + memory: 2Mi limits: - cpu: 250m - memory: 400Mi - ports: - - containerPort: 80 + cpu: 100m + memory: 200Mi diff --git a/test/e2e/resources/test-job.yaml b/test/e2e/resources/test-job.yaml index 0e5c219d1..24d9f0116 100644 --- a/test/e2e/resources/test-job.yaml +++ b/test/e2e/resources/test-job.yaml @@ -1,13 +1,13 @@ apiVersion: batch/v1 kind: Job metadata: - name: pi + name: sleep spec: template: spec: containers: - - name: pi - image: perl:5.34.0 - command: ["perl", "-Mbignum=bpi", "-wle", "print bpi(2000)"] + - name: sleep + image: alpine:3 + command: ["sh", "-c", "sleep 3"] restartPolicy: Never backoffLimit: 4 \ No newline at end of file diff --git a/test/e2e/resources/test-service.yaml b/test/e2e/resources/test-service.yaml index abed7143a..02e10c81d 100644 --- a/test/e2e/resources/test-service.yaml +++ b/test/e2e/resources/test-service.yaml @@ -1,17 +1,3 @@ -# Copyright 2021 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - apiVersion: v1 kind: Service metadata: diff --git a/test/e2e/resources/test-statefulset.yaml b/test/e2e/resources/test-statefulset.yaml index a3e9b624f..eb1e66b12 100644 --- a/test/e2e/resources/test-statefulset.yaml +++ b/test/e2e/resources/test-statefulset.yaml @@ -1,31 +1,29 @@ apiVersion: apps/v1 kind: StatefulSet metadata: - name: test-statefulset + name: test-ss namespace: test-ns labels: test-key: test-value spec: selector: matchLabels: - app: custom # has to match .spec.template.metadata.labels - serviceName: "test-service" + app: test-ss # has to match .spec.template.metadata.labels + serviceName: "test-ss-svc" replicas: 3 # by default is 1 template: metadata: labels: - app: custom # has to match .spec.selector.matchLabels + app: test-ss # has to match .spec.selector.matchLabels spec: containers: - - name: nginx - image: nginx + - name: pause + image: k8s.gcr.io/pause:3.8 resources: requests: - cpu: 50m - memory: 200Mi + cpu: 1m + memory: 2Mi limits: - cpu: 250m - memory: 400Mi - ports: - - containerPort: 80 - protocol: TCP + cpu: 100m + memory: 200Mi + diff --git a/test/e2e/resources_test.go b/test/e2e/resources_test.go index 895077f6d..ed3a7bb2f 100644 --- a/test/e2e/resources_test.go +++ b/test/e2e/resources_test.go @@ -35,22 +35,24 @@ import ( ) const ( - workNamespaceNameTemplate = "application-%d" - appConfigMapNameTemplate = "app-config-%d" - appDeploymentNameTemplate = "app-deploy-%d" - appSecretNameTemplate = "app-secret-%d" // #nosec G101 - crpNameTemplate = "crp-%d" - rpNameTemplate = "rp-%d" - crpNameWithSubIndexTemplate = "crp-%d-%d" - croNameTemplate = "cro-%d" - roNameTemplate = "ro-%d" - mcNameTemplate = "mc-%d" - internalServiceExportNameTemplate = "ise-%d" - internalServiceImportNameTemplate = "isi-%d" - endpointSliceExportNameTemplate = "ep-%d" - crpEvictionNameTemplate = "crpe-%d" - updateRunStrategyNameTemplate = "curs-%d" - updateRunNameWithSubIndexTemplate = "cur-%d-%d" + workNamespaceNameTemplate = "application-%d" + appConfigMapNameTemplate = "app-config-%d" + appDeploymentNameTemplate = "app-deploy-%d" + appSecretNameTemplate = "app-secret-%d" // #nosec G101 + crpNameTemplate = "crp-%d" + rpNameTemplate = "rp-%d" + crpNameWithSubIndexTemplate = "crp-%d-%d" + croNameTemplate = "cro-%d" + roNameTemplate = "ro-%d" + mcNameTemplate = "mc-%d" + internalServiceExportNameTemplate = "ise-%d" + internalServiceImportNameTemplate = "isi-%d" + endpointSliceExportNameTemplate = "ep-%d" + crpEvictionNameTemplate = "crpe-%d" + clusterStagedUpdateRunStrategyNameTemplate = "curs-%d" + clusterStagedUpdateRunNameWithSubIndexTemplate = "cur-%d-%d" + stagedUpdateRunStrategyNameTemplate = "sus-%d" + stagedUpdateRunNameWithSubIndexTemplate = "sur-%d-%d" customDeletionBlockerFinalizer = "kubernetes-fleet.io/custom-deletion-blocker-finalizer" workNamespaceLabelName = "process" @@ -172,20 +174,20 @@ func appDeployment() appsv1.Deployment { Replicas: ptr.To(int32(1)), Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - "app": "nginx", + "app": "pause", }, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - "app": "nginx", + "app": "pause", }, }, Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: "nginx", - Image: "nginx", + Name: "pause", + Image: "k8s.gcr.io/pause:3.8", }, }, TerminationGracePeriodSeconds: ptr.To(int64(60)), diff --git a/test/e2e/rollout_test.go b/test/e2e/rollout_test.go index fee968dd7..6c0157271 100644 --- a/test/e2e/rollout_test.go +++ b/test/e2e/rollout_test.go @@ -612,18 +612,22 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() { memberCluster := allMemberClusters[idx].ClusterName namespaceName := fmt.Sprintf(utils.NamespaceNameFormat, memberCluster) workName := fmt.Sprintf(placementv1beta1.FirstWorkNameFmt, crpName) - work := placementv1beta1.Work{} - Expect(hubClient.Get(ctx, types.NamespacedName{Name: workName, Namespace: namespaceName}, &work)).Should(Succeed(), "Failed to get the work") - if work.Status.ManifestConditions != nil { - work.Status.ManifestConditions = nil - } else { - meta.SetStatusCondition(&work.Status.Conditions, metav1.Condition{ - Type: placementv1beta1.WorkConditionTypeAvailable, - Status: metav1.ConditionFalse, - Reason: "WorkNotAvailable", - }) - } - Expect(hubClient.Status().Update(ctx, &work)).Should(Succeed(), "Failed to update the work") + Eventually(func() error { + work := placementv1beta1.Work{} + if err := hubClient.Get(ctx, types.NamespacedName{Name: workName, Namespace: namespaceName}, &work); err != nil { + return err + } + if work.Status.ManifestConditions != nil { + work.Status.ManifestConditions = nil + } else { + meta.SetStatusCondition(&work.Status.Conditions, metav1.Condition{ + Type: placementv1beta1.WorkConditionTypeAvailable, + Status: metav1.ConditionFalse, + Reason: "WorkNotAvailable", + }) + } + return hubClient.Status().Update(ctx, &work) + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update the work") } }) diff --git a/test/e2e/setup.sh b/test/e2e/setup.sh index 97f1d7775..b58487cb1 100755 --- a/test/e2e/setup.sh +++ b/test/e2e/setup.sh @@ -6,7 +6,7 @@ set -o pipefail # Before updating the default kind image to use, verify that the version is supported # by the current kind release. -KIND_IMAGE="${KIND_IMAGE:-kindest/node:v1.31.0}" +KIND_IMAGE="${KIND_IMAGE:-kindest/node:v1.33.4}" KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config}" MEMBER_CLUSTER_COUNT=$1 @@ -46,7 +46,7 @@ AKS_NODE_REGIONS=("westus" "northeurope" "eastasia") # # Note that this is for information only; kind nodes always use the same fixed setup # (total/allocatable capacity = host capacity). -AKS_NODE_SKUS=("Standard_A4_v2" "Standard_B4ms" "Standard_D8s_v5" "Standard_E16_v5" "Standard_M16ms") +AKS_NODE_SKUS=("Standard_B2ats_v2" "Standard_B2ts_v2" "Standard_D8s_v5" "Standard_E16_v5" "Standard_M16ms") AKS_SKU_COUNT=${#AKS_NODE_SKUS[@]} # The number of clusters that has pre-defined configuration for testing purposes. RESERVED_CLUSTER_COUNT=${MEMBER_CLUSTER_COUNT} @@ -134,7 +134,8 @@ helm install hub-agent ../../charts/hub-agent/ \ --set webhookClientConnectionType=service \ --set forceDeleteWaitTime="1m0s" \ --set clusterUnhealthyThreshold="3m0s" \ - --set logFileMaxSize=1000000 \ + --set logFileMaxSize=100000 \ + --set MaxConcurrentClusterPlacement=200 \ --set resourceSnapshotCreationMinimumInterval=$RESOURCE_SNAPSHOT_CREATION_MINIMUM_INTERVAL \ --set resourceChangesCollectionDuration=$RESOURCE_CHANGES_COLLECTION_DURATION diff --git a/test/e2e/setup_test.go b/test/e2e/setup_test.go index 05bffea25..d30a7a8d8 100644 --- a/test/e2e/setup_test.go +++ b/test/e2e/setup_test.go @@ -218,9 +218,11 @@ var ( // disappear from the status of the MemberCluster object. c.Type == string(clusterv1beta1.ConditionTypeClusterPropertyProviderStarted) }) - ignoreTimeTypeFields = cmpopts.IgnoreTypes(time.Time{}, metav1.Time{}) - ignorePlacementStatusDriftedPlacementsTimestampFields = cmpopts.IgnoreFields(placementv1beta1.DriftedResourcePlacement{}, "ObservationTime", "FirstDriftedObservedTime") - ignorePlacementStatusDiffedPlacementsTimestampFields = cmpopts.IgnoreFields(placementv1beta1.DiffedResourcePlacement{}, "ObservationTime", "FirstDiffedObservedTime") + ignoreTimeTypeFields = cmpopts.IgnoreTypes(time.Time{}, metav1.Time{}) + ignorePlacementStatusDriftedPlacementsTimestampFields = cmpopts.IgnoreFields(placementv1beta1.DriftedResourcePlacement{}, "ObservationTime", "FirstDriftedObservedTime") + ignorePlacementStatusDiffedPlacementsTimestampFields = cmpopts.IgnoreFields(placementv1beta1.DiffedResourcePlacement{}, "ObservationTime", "FirstDiffedObservedTime") + ignorePerClusterPlacementStatusObservedResourceIndexField = cmpopts.IgnoreFields(placementv1beta1.PerClusterPlacementStatus{}, "ObservedResourceIndex") + ignorePlacementStatusObservedResourceIndexField = cmpopts.IgnoreFields(placementv1beta1.PlacementStatus{}, "ObservedResourceIndex") placementStatusCmpOptions = cmp.Options{ cmpopts.SortSlices(lessFuncCondition), @@ -235,6 +237,21 @@ var ( cmpopts.EquateEmpty(), } + placementStatusCmpOptionsOnCreate = cmp.Options{ + cmpopts.SortSlices(lessFuncCondition), + cmpopts.SortSlices(lessFuncPlacementStatus), + cmpopts.SortSlices(utils.LessFuncResourceIdentifier), + cmpopts.SortSlices(utils.LessFuncFailedResourcePlacements), + cmpopts.SortSlices(utils.LessFuncDiffedResourcePlacements), + cmpopts.SortSlices(utils.LessFuncDriftedResourcePlacements), + utils.IgnoreConditionLTTAndMessageFields, + ignorePlacementStatusDriftedPlacementsTimestampFields, + ignorePlacementStatusDiffedPlacementsTimestampFields, + ignorePlacementStatusObservedResourceIndexField, + ignorePerClusterPlacementStatusObservedResourceIndexField, + cmpopts.EquateEmpty(), + } + // We don't sort ResourcePlacementStatus by their name since we don't know which cluster will become unavailable first, // prompting the rollout to be blocked for remaining clusters. safeRolloutPlacementStatusCmpOptions = cmp.Options{ diff --git a/test/e2e/staged_updaterun_test.go b/test/e2e/staged_updaterun_test.go new file mode 100644 index 000000000..a86f08e09 --- /dev/null +++ b/test/e2e/staged_updaterun_test.go @@ -0,0 +1,400 @@ +/* +Copyright 2025 The KubeFleet Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1" + "go.goms.io/fleet/pkg/utils/condition" + "go.goms.io/fleet/test/e2e/framework" +) + +// Note that this container will run in parallel with other containers. +var _ = Describe("test RP rollout with staged update run", func() { + crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess()) + rpName := fmt.Sprintf(rpNameTemplate, GinkgoParallelProcess()) + strategyName := fmt.Sprintf(stagedUpdateRunStrategyNameTemplate, GinkgoParallelProcess()) + testNamespace := fmt.Sprintf(workNamespaceNameTemplate, GinkgoParallelProcess()) + + Context("Test resource rollout and rollback with staged update run", Ordered, func() { + updateRunNames := []string{} + var strategy *placementv1beta1.StagedUpdateStrategy + var oldConfigMap, newConfigMap corev1.ConfigMap + + BeforeAll(func() { + // Create a test namespace and a configMap inside it on the hub cluster. + createWorkResources() + + // Create the CRP with Namespace-only selector. + createNamespaceOnlyCRP(crpName) + + By("should update CRP status as expected") + crpStatusUpdatedActual := crpStatusUpdatedActual(workNamespaceIdentifiers(), allMemberClusterNames, nil, "0") + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected") + + // Create the RP with external rollout strategy. + rp := &placementv1beta1.ResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: rpName, + Namespace: testNamespace, + // Add a custom finalizer; this would allow us to better observe + // the behavior of the controllers. + Finalizers: []string{customDeletionBlockerFinalizer}, + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: configMapSelector(), + Strategy: placementv1beta1.RolloutStrategy{ + Type: placementv1beta1.ExternalRolloutStrategyType, + }, + }, + } + Expect(hubClient.Create(ctx, rp)).To(Succeed(), "Failed to create RP") + + // Create the stagedUpdateStrategy. + strategy = createStagedUpdateStrategySucceed(strategyName, testNamespace) + + for i := 0; i < 3; i++ { + updateRunNames = append(updateRunNames, fmt.Sprintf(stagedUpdateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), i)) + } + + oldConfigMap = appConfigMap() + newConfigMap = appConfigMap() + newConfigMap.Data["data"] = testConfigMapDataValue + }) + + AfterAll(func() { + // Remove the custom deletion blocker finalizer from the RP. + ensureRPAndRelatedResourcesDeleted(types.NamespacedName{Name: rpName, Namespace: testNamespace}, allMemberClusters) + + // Remove all the stagedUpdateRuns. + for _, name := range updateRunNames { + ensureStagedUpdateRunDeletion(name, testNamespace) + } + + // Delete the stagedUpdateStrategy. + ensureStagedUpdateRunStrategyDeletion(strategyName, testNamespace) + // Delete the namespace only CRP. + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + }) + + It("Should not rollout any resources to member clusters as there's no update run yet", checkIfRemovedConfigMapFromAllMemberClusters) + + It("Should have the latest resource snapshot", func() { + validateLatestResourceSnapshot(rpName, testNamespace, resourceSnapshotIndex1st) + }) + + It("Should successfully schedule the rp", func() { + validateLatestSchedulingPolicySnapshot(rpName, testNamespace, policySnapshotIndex1st, 3) + }) + + It("Should update rp status as pending rollout", func() { + rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", "", ""}, []bool{false, false, false}, nil, nil) + Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s status as expected", rpName) + }) + + It("Should create a staged update run successfully", func() { + createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + }) + + It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { + checkIfPlacedConfigMapOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[1]}) + checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) + + By("Validating rp status as member-cluster-2 updated") + rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) + Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s status as expected", rpName) + + validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary) + }) + + It("Should rollout resources to member-cluster-1 first because of its name", func() { + checkIfPlacedConfigMapOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[0]}) + }) + + It("Should rollout resources to all the members and complete the staged update run successfully", func() { + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) + checkIfPlacedConfigMapOnMemberClustersInUpdateRun(allMemberClusters) + }) + + It("Should update rp status as completed", func() { + rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(appConfigMapIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames, + []string{resourceSnapshotIndex1st, resourceSnapshotIndex1st, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) + Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s status as expected", rpName) + }) + + It("Should update the configmap successfully on hub but not change member clusters", func() { + updateConfigMapSucceed(&newConfigMap) + + for _, cluster := range allMemberClusters { + configMapActual := configMapPlacedOnClusterActual(cluster, &oldConfigMap) + Consistently(configMapActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to keep configmap %s data as expected", oldConfigMap.Name) + } + }) + + It("Should not update rp status, should still be completed", func() { + rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(appConfigMapIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames, + []string{resourceSnapshotIndex1st, resourceSnapshotIndex1st, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) + Consistently(rpStatusUpdatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to keep RP %s status as expected", rpName) + }) + + It("Should create a new latest resource snapshot", func() { + rsList := &placementv1beta1.ResourceSnapshotList{} + Eventually(func() error { + if err := hubClient.List(ctx, rsList, client.InNamespace(testNamespace), client.MatchingLabels{placementv1beta1.PlacementTrackingLabel: rpName, placementv1beta1.IsLatestSnapshotLabel: "true"}); err != nil { + return fmt.Errorf("failed to list the resourcesnapshot: %w", err) + } + if len(rsList.Items) != 1 { + return fmt.Errorf("got %d latest resourcesnapshots, want 1", len(rsList.Items)) + } + if rsList.Items[0].Labels[placementv1beta1.ResourceIndexLabel] != resourceSnapshotIndex2nd { + return fmt.Errorf("got resource snapshot index %s, want %s", rsList.Items[0].Labels[placementv1beta1.ResourceIndexLabel], resourceSnapshotIndex2nd) + } + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed get the new latest resourcensnapshot") + }) + + It("Should create a new staged update run successfully", func() { + createStagedUpdateRunSucceed(updateRunNames[1], testNamespace, rpName, resourceSnapshotIndex2nd, strategyName) + }) + + It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { + By("Verify that the new configmap is updated on member-cluster-2") + configMapActual := configMapPlacedOnClusterActual(allMemberClusters[1], &newConfigMap) + Eventually(configMapActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update to the new configmap %s on cluster %s", newConfigMap.Name, allMemberClusterNames[1]) + By("Verify that the configmap is not updated on member-cluster-1 and member-cluster-3") + for _, cluster := range []*framework.Cluster{allMemberClusters[0], allMemberClusters[2]} { + configMapActual := configMapPlacedOnClusterActual(cluster, &oldConfigMap) + Consistently(configMapActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to keep configmap %s data as expected", newConfigMap.Name) + } + + By("Validating rp status as member-cluster-2 updated") + rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, + []string{resourceSnapshotIndex1st, resourceSnapshotIndex2nd, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) + Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s status as expected", rpName) + + validateAndApproveNamespacedApprovalRequests(updateRunNames[1], testNamespace, envCanary) + }) + + It("Should rollout resources to member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[1], testNamespace, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) + By("Verify that new the configmap is updated on all member clusters") + for idx := range allMemberClusters { + configMapActual := configMapPlacedOnClusterActual(allMemberClusters[idx], &newConfigMap) + Eventually(configMapActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update to the new configmap %s on cluster %s as expected", newConfigMap.Name, allMemberClusterNames[idx]) + } + }) + + It("Should update rp status as completed", func() { + rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(appConfigMapIdentifiers(), resourceSnapshotIndex2nd, true, allMemberClusterNames, + []string{resourceSnapshotIndex2nd, resourceSnapshotIndex2nd, resourceSnapshotIndex2nd}, []bool{true, true, true}, nil, nil) + Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s status as expected", rpName) + }) + + It("Should create a new staged update run with old resourceSnapshotIndex successfully to rollback", func() { + createStagedUpdateRunSucceed(updateRunNames[2], testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + }) + + It("Should rollback resources to member-cluster-2 only and completes stage canary", func() { + By("Verify that the configmap is rolled back on member-cluster-2") + configMapActual := configMapPlacedOnClusterActual(allMemberClusters[1], &oldConfigMap) + Eventually(configMapActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to rollback the configmap change on cluster %s", allMemberClusterNames[1]) + By("Verify that the configmap is not rolled back on member-cluster-1 and member-cluster-3") + for _, cluster := range []*framework.Cluster{allMemberClusters[0], allMemberClusters[2]} { + configMapActual := configMapPlacedOnClusterActual(cluster, &newConfigMap) + Consistently(configMapActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to keep configmap %s data as expected", newConfigMap.Name) + } + + By("Validating rp status as member-cluster-2 updated") + rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, + []string{resourceSnapshotIndex2nd, resourceSnapshotIndex1st, resourceSnapshotIndex2nd}, []bool{true, true, true}, nil, nil) + Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s status as expected", rpName) + + validateAndApproveNamespacedApprovalRequests(updateRunNames[2], testNamespace, envCanary) + }) + + It("Should rollback resources to member-cluster-1 and member-cluster-3 too and complete the staged update run successfully", func() { + surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[2], testNamespace, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[1]) + for idx := range allMemberClusters { + configMapActual := configMapPlacedOnClusterActual(allMemberClusters[idx], &oldConfigMap) + Eventually(configMapActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to rollback the configmap %s data on cluster %s as expected", oldConfigMap.Name, allMemberClusterNames[idx]) + } + }) + + It("Should update rp status as completed", func() { + rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(appConfigMapIdentifiers(), resourceSnapshotIndex1st, true, allMemberClusterNames, + []string{resourceSnapshotIndex1st, resourceSnapshotIndex1st, resourceSnapshotIndex1st}, []bool{true, true, true}, nil, nil) + Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s status as expected", rpName) + }) + }) +}) + +func createStagedUpdateStrategySucceed(strategyName, namespace string) *placementv1beta1.StagedUpdateStrategy { + strategy := &placementv1beta1.StagedUpdateStrategy{ + ObjectMeta: metav1.ObjectMeta{ + Name: strategyName, + Namespace: namespace, + }, + Spec: placementv1beta1.UpdateStrategySpec{ + Stages: []placementv1beta1.StageConfig{ + { + Name: envCanary, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + envLabelName: envCanary, // member-cluster-2 + }, + }, + AfterStageTasks: []placementv1beta1.AfterStageTask{ + { + Type: placementv1beta1.AfterStageTaskTypeApproval, + }, + { + Type: placementv1beta1.AfterStageTaskTypeTimedWait, + WaitTime: &metav1.Duration{ + Duration: time.Second * 5, + }, + }, + }, + }, + { + Name: envProd, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + envLabelName: envProd, // member-cluster-1 and member-cluster-3 + }, + }, + }, + }, + }, + } + Expect(hubClient.Create(ctx, strategy)).To(Succeed(), "Failed to create StagedUpdateStrategy") + return strategy +} + +func validateLatestSchedulingPolicySnapshot(rpName, namespace, wantPolicySnapshotIndex string, wantSelectedClusterCount int) { + Eventually(func() (string, error) { + var policySnapshotList placementv1beta1.SchedulingPolicySnapshotList + if err := hubClient.List(ctx, &policySnapshotList, client.InNamespace(namespace), client.MatchingLabels{ + placementv1beta1.PlacementTrackingLabel: rpName, + placementv1beta1.IsLatestSnapshotLabel: "true", + }); err != nil { + return "", fmt.Errorf("failed to list the latest scheduling policy snapshot: %w", err) + } + if len(policySnapshotList.Items) != 1 { + return "", fmt.Errorf("failed to find the latest scheduling policy snapshot") + } + latestPolicySnapshot := policySnapshotList.Items[0] + if !condition.IsConditionStatusTrue(latestPolicySnapshot.GetCondition(string(placementv1beta1.PolicySnapshotScheduled)), latestPolicySnapshot.Generation) { + return "", fmt.Errorf("the latest scheduling policy snapshot is not scheduled yet") + } + + selectedClusterCount := 0 + for _, decision := range latestPolicySnapshot.Status.ClusterDecisions { + if decision.Selected { + selectedClusterCount++ + } + } + if selectedClusterCount != wantSelectedClusterCount { + return "", fmt.Errorf("want %d selected clusters, got %d", wantSelectedClusterCount, selectedClusterCount) + } + return latestPolicySnapshot.Labels[placementv1beta1.PolicyIndexLabel], nil + }, eventuallyDuration, eventuallyInterval).Should(Equal(wantPolicySnapshotIndex), "Policy snapshot index does not match") +} + +func validateLatestResourceSnapshot(rpName, namespace, wantResourceSnapshotIndex string) { + Eventually(func() (string, error) { + rsList := &placementv1beta1.ResourceSnapshotList{} + if err := hubClient.List(ctx, rsList, client.InNamespace(namespace), client.MatchingLabels{ + placementv1beta1.PlacementTrackingLabel: rpName, + placementv1beta1.IsLatestSnapshotLabel: "true", + }); err != nil { + return "", fmt.Errorf("failed to list the latestresourcesnapshot: %w", err) + } + if len(rsList.Items) != 1 { + return "", fmt.Errorf("got %d resourcesnapshots, want 1", len(rsList.Items)) + } + return rsList.Items[0].Labels[placementv1beta1.ResourceIndexLabel], nil + }, eventuallyDuration, eventuallyInterval).Should(Equal(wantResourceSnapshotIndex), "Resource snapshot index does not match") +} + +func createStagedUpdateRunSucceed(updateRunName, namespace, rpName, resourceSnapshotIndex, strategyName string) { + updateRun := &placementv1beta1.StagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: updateRunName, + Namespace: namespace, + }, + Spec: placementv1beta1.UpdateRunSpec{ + PlacementName: rpName, + ResourceSnapshotIndex: resourceSnapshotIndex, + StagedUpdateStrategyName: strategyName, + }, + } + Expect(hubClient.Create(ctx, updateRun)).To(Succeed(), "Failed to create StagedUpdateRun %s", updateRunName) +} + +func validateAndApproveNamespacedApprovalRequests(updateRunName, namespace, stageName string) { + Eventually(func() error { + appReqList := &placementv1beta1.ApprovalRequestList{} + if err := hubClient.List(ctx, appReqList, client.InNamespace(namespace), client.MatchingLabels{ + placementv1beta1.TargetUpdatingStageNameLabel: stageName, + placementv1beta1.TargetUpdateRunLabel: updateRunName, + }); err != nil { + return fmt.Errorf("failed to list approval requests: %w", err) + } + + if len(appReqList.Items) != 1 { + return fmt.Errorf("got %d approval requests, want 1", len(appReqList.Items)) + } + appReq := &appReqList.Items[0] + meta.SetStatusCondition(&appReq.Status.Conditions, metav1.Condition{ + Status: metav1.ConditionTrue, + Type: string(placementv1beta1.ApprovalRequestConditionApproved), + ObservedGeneration: appReq.GetGeneration(), + Reason: "lgtm", + }) + return hubClient.Status().Update(ctx, appReq) + }, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to get or approve approval request") +} + +func checkIfPlacedConfigMapOnMemberClustersInUpdateRun(clusters []*framework.Cluster) { + for idx := range clusters { + memberCluster := clusters[idx] + configMap := appConfigMap() + configMapPlacedActual := configMapPlacedOnClusterActual(memberCluster, &configMap) + Eventually(configMapPlacedActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place config map on member cluster %s", memberCluster.ClusterName) + } +} + +func checkIfRemovedConfigMapFromMemberClustersConsistently(clusters []*framework.Cluster) { + for idx := range clusters { + memberCluster := clusters[idx] + configMapRemovedActual := namespacedResourcesRemovedFromClusterActual(memberCluster) + Consistently(configMapRemovedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to remove config map from member cluster %s consistently", memberCluster.ClusterName) + } +} diff --git a/test/e2e/utils_test.go b/test/e2e/utils_test.go index e28baa500..2e7dfd00b 100644 --- a/test/e2e/utils_test.go +++ b/test/e2e/utils_test.go @@ -20,6 +20,8 @@ import ( "encoding/json" "errors" "fmt" + "math" + "strconv" "strings" "time" @@ -305,13 +307,50 @@ func checkIfAzurePropertyProviderIsWorking() { // the diff output (if any) to omit certain fields. // Diff the non-resource properties. + + // Cost properties are checked separately to account for approximation margins. + ignoreCostProperties := cmpopts.IgnoreMapEntries(func(k clusterv1beta1.PropertyName, v clusterv1beta1.PropertyValue) bool { + return k == azure.PerCPUCoreCostProperty || k == azure.PerGBMemoryCostProperty + }) if diff := cmp.Diff( mcObj.Status.Properties, wantStatus.Properties, ignoreTimeTypeFields, + ignoreCostProperties, ); diff != "" { return fmt.Errorf("member cluster status properties diff (-got, +want):\n%s", diff) } + // Check the cost properties separately. + // + // The test suite consider cost outputs with a margin of no more than 0.002 to be acceptable. + perCPUCoreCostProperty, found := mcObj.Status.Properties[azure.PerCPUCoreCostProperty] + wantPerCPUCoreCostProperty, wantFound := wantStatus.Properties[azure.PerCPUCoreCostProperty] + if found != wantFound { + return fmt.Errorf("member cluster per CPU core cost property diff: found=%v, wantFound=%v", found, wantFound) + } + perCPUCoreCost, err := strconv.ParseFloat(perCPUCoreCostProperty.Value, 64) + wantPerCPUCoreCost, wantErr := strconv.ParseFloat(wantPerCPUCoreCostProperty.Value, 64) + if err != nil || wantErr != nil { + return fmt.Errorf("failed to parse per CPU core cost property: val=%s, err=%w, wantVal=%s, wantErr=%w", perCPUCoreCostProperty.Value, err, wantPerCPUCoreCostProperty.Value, wantErr) + } + if diff := math.Abs(perCPUCoreCost - wantPerCPUCoreCost); diff > 0.002 { + return fmt.Errorf("member cluster per CPU core cost property diff: got=%f, want=%f, diff=%f", perCPUCoreCost, wantPerCPUCoreCost, diff) + } + + perGBMemoryCostProperty, found := mcObj.Status.Properties[azure.PerGBMemoryCostProperty] + wantPerGBMemoryCostProperty, wantFound := wantStatus.Properties[azure.PerGBMemoryCostProperty] + if found != wantFound { + return fmt.Errorf("member cluster per GB memory cost property diff: found=%v, wantFound=%v", found, wantFound) + } + perGBMemoryCost, err := strconv.ParseFloat(perGBMemoryCostProperty.Value, 64) + wantPerGBMemoryCost, wantErr := strconv.ParseFloat(wantPerGBMemoryCostProperty.Value, 64) + if err != nil || wantErr != nil { + return fmt.Errorf("failed to parse per GB memory cost property: val=%s, err=%w, wantVal=%s, wantErr=%w", perGBMemoryCostProperty.Value, err, wantPerGBMemoryCostProperty.Value, wantErr) + } + if diff := math.Abs(perGBMemoryCost - wantPerGBMemoryCost); diff > 0.002 { + return fmt.Errorf("member cluster per GB memory cost property diff: got=%f, want=%f, diff=%f", perGBMemoryCost, wantPerGBMemoryCost, diff) + } + // Diff the resource usage. if diff := cmp.Diff( mcObj.Status.ResourceUsage, wantStatus.ResourceUsage, @@ -330,7 +369,7 @@ func checkIfAzurePropertyProviderIsWorking() { return fmt.Errorf("member cluster status conditions diff (-got, +want):\n%s", diff) } return nil - }, longEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to confirm that Azure property provider is up and running") + }, longEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to confirm that Azure property provider is up and running on cluster", memberCluster.ClusterName) } } @@ -351,6 +390,17 @@ func summarizeAKSClusterProperties(memberCluster *framework.Cluster, mcObj *clus return nil, fmt.Errorf("no nodes are found") } + nodeCountBySKU := map[string]int{} + for idx := range nodeList.Items { + node := nodeList.Items[idx] + + nodeSKU := node.Labels[trackers.AKSClusterNodeSKULabelName] + if len(nodeSKU) == 0 { + nodeSKU = trackers.ReservedNameForUndefinedSKU + } + nodeCountBySKU[nodeSKU]++ + } + totalCPUCapacity := resource.Quantity{} totalMemoryCapacity := resource.Quantity{} allocatableCPUCapacity := resource.Quantity{} @@ -370,6 +420,9 @@ func summarizeAKSClusterProperties(memberCluster *framework.Cluster, mcObj *clus totalHourlyRate += hourlyRate } } + if totalHourlyRate <= 0.002 { + return nil, fmt.Errorf("total hourly rate is zero or too small; there might be unrecognized SKUs or incorrect pricing data") + } cpuCores := totalCPUCapacity.AsApproximateFloat64() if cpuCores < 0.001 { @@ -405,18 +458,25 @@ func summarizeAKSClusterProperties(memberCluster *framework.Cluster, mcObj *clus availableMemoryCapacity.Sub(requestedMemoryCapacity) } - status := clusterv1beta1.MemberClusterStatus{ - Properties: map[clusterv1beta1.PropertyName]clusterv1beta1.PropertyValue{ - propertyprovider.NodeCountProperty: { - Value: fmt.Sprintf("%d", nodeCount), - }, - azure.PerCPUCoreCostProperty: { - Value: fmt.Sprintf(azure.CostPrecisionTemplate, perCPUCoreCost), - }, - azure.PerGBMemoryCostProperty: { - Value: fmt.Sprintf(azure.CostPrecisionTemplate, perGBMemoryCost), - }, + properties := map[clusterv1beta1.PropertyName]clusterv1beta1.PropertyValue{ + propertyprovider.NodeCountProperty: { + Value: fmt.Sprintf("%d", nodeCount), + }, + azure.PerCPUCoreCostProperty: { + Value: fmt.Sprintf(azure.CostPrecisionTemplate, perCPUCoreCost), + }, + azure.PerGBMemoryCostProperty: { + Value: fmt.Sprintf(azure.CostPrecisionTemplate, perGBMemoryCost), }, + } + for sku, count := range nodeCountBySKU { + pName := clusterv1beta1.PropertyName(fmt.Sprintf(azure.NodeCountPerSKUPropertyTmpl, sku)) + properties[pName] = clusterv1beta1.PropertyValue{ + Value: fmt.Sprintf("%d", count), + } + } + status := clusterv1beta1.MemberClusterStatus{ + Properties: properties, ResourceUsage: clusterv1beta1.ResourceUsage{ Capacity: corev1.ResourceList{ corev1.ResourceCPU: totalCPUCapacity, @@ -652,7 +712,11 @@ func ensureMemberClusterAndRelatedResourcesDeletion(memberClusterName string) { Eventually(func() error { ns := corev1.Namespace{} if err := hubClient.Get(ctx, types.NamespacedName{Name: reservedNSName}, &ns); !k8serrors.IsNotFound(err) { - return fmt.Errorf("namespace still exists or an unexpected error occurred: %w", err) + if err == nil { + return fmt.Errorf("work namespace %s still exists on cluster %s: deletion timestamp: %v, current timestamp: %v", + ns.Name, memberClusterName, ns.GetDeletionTimestamp(), time.Now()) + } + return fmt.Errorf("an unexpected error occurred: %w", err) } return nil }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove reserved namespace") @@ -717,8 +781,19 @@ func createWorkResources() { } func createNamespace() { - ns := appNamespace() - Expect(hubClient.Create(ctx, &ns)).To(Succeed(), "Failed to create namespace %s", ns.Name) + var ns corev1.Namespace + Eventually(func() error { + ns = appNamespace() + err := hubClient.Create(ctx, &ns) + if k8serrors.IsAlreadyExists(err) { + err = hubClient.Get(ctx, types.NamespacedName{Name: ns.Name}, &ns) + if err != nil { + return fmt.Errorf("failed to get the namespace %s, err is %+w", ns.Name, err) + } + return fmt.Errorf("namespace %s already exists, delete time is %v", ns.Name, ns.GetDeletionTimestamp()) + } + return err + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to create namespace %s", ns.Name) } func createConfigMap() { @@ -736,7 +811,7 @@ func cleanWorkResourcesOnCluster(cluster *framework.Cluster) { Expect(client.IgnoreNotFound(cluster.KubeClient.Delete(ctx, &ns))).To(Succeed(), "Failed to delete namespace %s", ns.Name) workResourcesRemovedActual := workNamespaceRemovedFromClusterActual(cluster) - Eventually(workResourcesRemovedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove work resources from %s cluster", cluster.ClusterName) + Eventually(workResourcesRemovedActual, workloadEventuallyDuration, time.Second*5).Should(Succeed(), "Failed to remove work resources from %s cluster", cluster.ClusterName) } // cleanupConfigMap deletes the ConfigMap created by createWorkResources and waits until the resource is not found. @@ -752,29 +827,6 @@ func cleanupConfigMapOnCluster(cluster *framework.Cluster) { Eventually(configMapRemovedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove config map from %s cluster", cluster.ClusterName) } -func cleanupAnotherConfigMapOnMemberCluster(name types.NamespacedName, cluster *framework.Cluster) { - cm := &corev1.ConfigMap{} - err := cluster.KubeClient.Get(ctx, name, cm) - if err != nil && k8serrors.IsNotFound(err) { - return - } - Expect(err).To(Succeed(), "Failed to get config map %s", name) - - Expect(cluster.KubeClient.Delete(ctx, cm)).To(Succeed(), "Failed to delete config map %s", name) - - Eventually(func() error { - cm := &corev1.ConfigMap{} - err := cluster.KubeClient.Get(ctx, name, cm) - if err != nil { - if k8serrors.IsNotFound(err) { - return nil - } - return err - } - return fmt.Errorf("config map %s still exists", name) - }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to wait for config map %s to be deleted", name) -} - // setMemberClusterToLeave sets a specific member cluster to leave the fleet. func setMemberClusterToLeave(memberCluster *framework.Cluster) { mcObj := &clusterv1beta1.MemberCluster{ @@ -974,7 +1026,7 @@ func checkNamespaceExistsWithOwnerRefOnMemberCluster(nsName, crpName string) { } func checkConfigMapExistsWithOwnerRefOnMemberCluster(namespace, cmName, rpName string) { - Consistently(func() error { + cmHasNoWorkOwnerRefActual := func() error { cm := &corev1.ConfigMap{} if err := allMemberClusters[0].KubeClient.Get(ctx, types.NamespacedName{Name: cmName, Namespace: namespace}, cm); err != nil { return fmt.Errorf("failed to get configmap %s/%s: %w", namespace, cmName, err) @@ -992,7 +1044,11 @@ func checkConfigMapExistsWithOwnerRefOnMemberCluster(namespace, cmName, rpName s } } return nil - }, consistentlyDuration, consistentlyInterval).Should(Succeed(), "ConfigMap which is not owned by the RP should not be deleted") + } + + // Must use Eventually checks first, as Fleet agents might not act fast enough in the test environment. + Eventually(cmHasNoWorkOwnerRefActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "ConfigMap still has AppliedWork owner reference") + Consistently(cmHasNoWorkOwnerRefActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "ConfigMap which is not owned by the RP should not be deleted") } // cleanupPlacement deletes the placement and waits until the resources are not found. @@ -1169,7 +1225,7 @@ func ensureCRPAndRelatedResourcesDeleted(crpName string, memberClusters []*frame memberCluster := memberClusters[idx] workResourcesRemovedActual := workNamespaceRemovedFromClusterActual(memberCluster) - Eventually(workResourcesRemovedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove work resources from member cluster %s", memberCluster.ClusterName) + Eventually(workResourcesRemovedActual, workloadEventuallyDuration, time.Second*5).Should(Succeed(), "Failed to remove work resources from member cluster %s", memberCluster.ClusterName) } // Verify that related finalizers have been removed from the CRP. @@ -1618,8 +1674,8 @@ func createNamespaceOnlyCRP(crpName string) { createCRPWithApplyStrategy(crpName, nil, namespaceOnlySelector()) } -// ensureUpdateRunDeletion deletes the update run with the given name and checks all related approval requests are also deleted. -func ensureUpdateRunDeletion(updateRunName string) { +// ensureClusterStagedUpdateRunDeletion deletes the cluster staged update run with the given name and checks all related cluster approval requests are also deleted. +func ensureClusterStagedUpdateRunDeletion(updateRunName string) { updateRun := &placementv1beta1.ClusterStagedUpdateRun{ ObjectMeta: metav1.ObjectMeta{ Name: updateRunName, @@ -1627,22 +1683,52 @@ func ensureUpdateRunDeletion(updateRunName string) { } Expect(client.IgnoreNotFound(hubClient.Delete(ctx, updateRun))).Should(Succeed(), "Failed to delete ClusterStagedUpdateRun %s", updateRunName) - removedActual := updateRunAndApprovalRequestsRemovedActual(updateRunName) + removedActual := clusterStagedUpdateRunAndClusterApprovalRequestsRemovedActual(updateRunName) Eventually(removedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "ClusterStagedUpdateRun or ClusterApprovalRequests still exists") } -// ensureUpdateRunStrategyDeletion deletes the update run strategy with the given name. -func ensureUpdateRunStrategyDeletion(strategyName string) { +// ensureStagedUpdateRunDeletion deletes the staged update run with the given name and checks all related approval requests are also deleted. +func ensureStagedUpdateRunDeletion(updateRunName, namespace string) { + updateRun := &placementv1beta1.StagedUpdateRun{ + ObjectMeta: metav1.ObjectMeta{ + Name: updateRunName, + Namespace: namespace, + }, + } + Expect(client.IgnoreNotFound(hubClient.Delete(ctx, updateRun))).Should(Succeed(), "Failed to delete StagedUpdateRun %s", updateRunName) + + removedActual := stagedUpdateRunAndApprovalRequestsRemovedActual(updateRunName, namespace) + Eventually(removedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "StagedUpdateRun or ApprovalRequests still exists") +} + +// ensureClusterUpdateRunStrategyDeletion deletes the cluster update run strategy with the given name. +func ensureClusterUpdateRunStrategyDeletion(strategyName string) { strategy := &placementv1beta1.ClusterStagedUpdateStrategy{ ObjectMeta: metav1.ObjectMeta{ Name: strategyName, }, } Expect(client.IgnoreNotFound(hubClient.Delete(ctx, strategy))).Should(Succeed(), "Failed to delete ClusterStagedUpdateStrategy %s", strategyName) - removedActual := updateRunStrategyRemovedActual(strategyName) + removedActual := clusterUpdateRunStrategyRemovedActual(strategyName) Eventually(removedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "ClusterStagedUpdateStrategy still exists") } +func ensureStagedUpdateRunStrategyDeletion(strategyName, namespace string) { + Eventually(func() error { + strategy := &placementv1beta1.StagedUpdateStrategy{} + if err := hubClient.Get(ctx, client.ObjectKey{Name: strategyName, Namespace: namespace}, strategy); err != nil { + return client.IgnoreNotFound(err) + } + return hubClient.Delete(ctx, strategy) + }, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to delete StagedUpdateStrategy %s", strategyName) + + // Wait for the staged update strategy to be deleted. + Eventually(func() bool { + strategy := &placementv1beta1.StagedUpdateStrategy{} + return hubClient.Get(ctx, client.ObjectKey{Name: strategyName, Namespace: namespace}, strategy) != nil + }, eventuallyDuration, eventuallyInterval).Should(BeTrue(), "Failed to delete StagedUpdateStrategy %s", strategyName) +} + // ensureRPAndRelatedResourcesDeleted deletes rp and verifies resources in the specified namespace placed by the rp are removed from the cluster. // It checks if the placed configMap is removed by default, as this is tested in most of the test cases. // For tests with additional resources placed, e.g. deployments, daemonSets, add those to placedResources. @@ -1661,7 +1747,7 @@ func ensureRPAndRelatedResourcesDeleted(rpKey types.NamespacedName, memberCluste memberCluster := memberClusters[idx] workResourcesRemovedActual := namespacedResourcesRemovedFromClusterActual(memberCluster, placedResources...) - Eventually(workResourcesRemovedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove work resources from member cluster %s", memberCluster.ClusterName) + Eventually(workResourcesRemovedActual, workloadEventuallyDuration, time.Second*5).Should(Succeed(), "Failed to remove work resources from member cluster %s", memberCluster.ClusterName) } // Verify that related finalizers have been removed from the ResourcePlacement. diff --git a/test/e2e/v1alpha1/webhook_test.go b/test/e2e/v1alpha1/webhook_test.go index 01ae7289c..5e123bcf2 100644 --- a/test/e2e/v1alpha1/webhook_test.go +++ b/test/e2e/v1alpha1/webhook_test.go @@ -113,15 +113,8 @@ var _ = Describe("Fleet's Hub cluster webhook tests", func() { Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: "nginx", - Image: "nginx:1.14.2", - Ports: []corev1.ContainerPort{ - { - Name: "http", - Protocol: corev1.ProtocolTCP, - ContainerPort: 80, - }, - }, + Name: "pause", + Image: "k8s.gcr.io/pause:3.8", }, }, }, @@ -168,15 +161,8 @@ var _ = Describe("Fleet's Hub cluster webhook tests", func() { Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: "nginx", - Image: "nginx:1.14.2", - Ports: []corev1.ContainerPort{ - { - Name: "http", - Protocol: corev1.ProtocolTCP, - ContainerPort: 80, - }, - }, + Name: "pause", + Image: "k8s.gcr.io/pause:3.8", }, }, }, @@ -456,15 +442,8 @@ var _ = Describe("Fleet's Hub cluster webhook tests", func() { Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: "nginx", - Image: "nginx", - Ports: []corev1.ContainerPort{ - { - Name: "http", - Protocol: corev1.ProtocolTCP, - ContainerPort: 80, - }, - }, + Name: "pause", + Image: "k8s.gcr.io/pause:3.8", }, }, }, @@ -513,15 +492,8 @@ var _ = Describe("Fleet's Hub cluster webhook tests", func() { Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: "nginx", - Image: "nginx", - Ports: []corev1.ContainerPort{ - { - Name: "http", - Protocol: corev1.ProtocolTCP, - ContainerPort: 80, - }, - }, + Name: "pause", + Image: "k8s.gcr.io/pause:3.8", }, }, }, diff --git a/test/upgrade/setup.sh b/test/upgrade/setup.sh index 1aa568aef..e09c8fab9 100755 --- a/test/upgrade/setup.sh +++ b/test/upgrade/setup.sh @@ -81,7 +81,7 @@ helm install hub-agent charts/hub-agent/ \ --set webhookClientConnectionType=service \ --set forceDeleteWaitTime="1m0s" \ --set clusterUnhealthyThreshold="3m0s" \ - --set logFileMaxSize=1000000 \ + --set logFileMaxSize=100000 \ --set resourceSnapshotCreationMinimumInterval=0m \ --set resourceChangesCollectionDuration=0m diff --git a/test/upgrade/upgrade.sh b/test/upgrade/upgrade.sh index b84e0b060..4b7000cdf 100755 --- a/test/upgrade/upgrade.sh +++ b/test/upgrade/upgrade.sh @@ -70,7 +70,7 @@ if [ -n "$UPGRADE_HUB_SIDE" ]; then --set webhookClientConnectionType=service \ --set forceDeleteWaitTime="1m0s" \ --set clusterUnhealthyThreshold="3m0s" \ - --set logFileMaxSize=1000000 \ + --set logFileMaxSize=100000 \ --set resourceSnapshotCreationMinimumInterval=0m \ --set resourceChangesCollectionDuration=0m fi