chore: Backport v20260428

.github/workflows/codespell.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@6c3c2f2c1c457b00c10c4848d6f5491db3b629df # v2.18.0
+        uses: step-security/harden-runner@8d3c67de8e2fe68ef647c8db1e6a09f647780f40 # v2.19.0
         with:
           egress-policy: audit
 

.github/workflows/trivy.yml

@@ -64,7 +64,7 @@ jobs:
           TAG: ${{ env.IMAGE_VERSION }}
 
       - name: Scan ${{ env.REGISTRY }}/${{ env.HUB_AGENT_IMAGE_NAME }}:${{ env.IMAGE_VERSION }}
-        uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
+        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
         with:
           image-ref: ${{ env.REGISTRY }}/${{ env.HUB_AGENT_IMAGE_NAME }}:${{ env.IMAGE_VERSION }}
           format: 'table'
@@ -80,7 +80,7 @@ jobs:
 
 
       - name: Scan ${{ env.REGISTRY }}/${{ env.MEMBER_AGENT_IMAGE_NAME }}:${{ env.IMAGE_VERSION }}
-        uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
+        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
         with:
           image-ref: ${{ env.REGISTRY }}/${{ env.MEMBER_AGENT_IMAGE_NAME }}:${{ env.IMAGE_VERSION }}
           format: 'table'
@@ -95,7 +95,7 @@ jobs:
           TRIVY_DB_REPOSITORY: mcr.microsoft.com/mirror/ghcr/aquasecurity/trivy-db 
 
       - name: Scan ${{ env.REGISTRY }}/${{ env.REFRESH_TOKEN_IMAGE_NAME }}:${{ env.IMAGE_VERSION }}
-        uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0
+        uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
         with:
           image-ref: ${{ env.REGISTRY }}/${{ env.REFRESH_TOKEN_IMAGE_NAME }}:${{ env.IMAGE_VERSION }}
           format: 'table'

pkg/controllers/updaterun/controller.go

@@ -109,8 +109,12 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim
 		return runtime.Result{}, err
 	}
 
+	// Track errors for metrics emission. The error is used to determine the failure type
+	// (user_error vs internal_error) in the emitted metrics.
+	var reconcileErr error
 	// Emit the update run status metric based on status conditions in the updateRun.
-	defer emitUpdateRunStatusMetric(updateRun)
+	// Use a closure to capture reconcileErr by reference, so it reflects any updates made during reconciliation.
+	defer func() { emitUpdateRunStatusMetric(updateRun, reconcileErr) }()
 
 	state := updateRun.GetUpdateRunSpec().State
 
@@ -126,14 +130,13 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim
 		}
 
 		// Initialize the updateRun.
-		var initErr error
-		if toBeUpdatedBindings, toBeDeletedBindings, initErr = r.initialize(ctx, updateRun); initErr != nil {
-			klog.ErrorS(initErr, "Failed to initialize the updateRun", "updateRun", runObjRef)
+		if toBeUpdatedBindings, toBeDeletedBindings, reconcileErr = r.initialize(ctx, updateRun); reconcileErr != nil {
+			klog.ErrorS(reconcileErr, "Failed to initialize the updateRun", "updateRun", runObjRef)
 			// errStagedUpdatedAborted cannot be retried.
-			if errors.Is(initErr, errStagedUpdatedAborted) {
-				return runtime.Result{}, r.recordInitializationFailed(ctx, updateRun, initErr.Error())
+			if errors.Is(reconcileErr, errStagedUpdatedAborted) {
+				return runtime.Result{}, r.recordInitializationFailed(ctx, updateRun, reconcileErr.Error())
 			}
-			return runtime.Result{}, initErr
+			return runtime.Result{}, reconcileErr
 		}
 		updatingStageIndex = 0 // start from the first stage (typically for Initialize or Run states).
 		klog.V(2).InfoS("Initialized the updateRun", "state", state, "updateRun", runObjRef)
@@ -145,14 +148,14 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim
 			klog.V(2).InfoS("The updateRun is finished", "finishedSuccessfully", finishedCond.Status, "updateRun", runObjRef)
 			return runtime.Result{}, nil
 		}
-		var validateErr error
 		// Validate the updateRun status to ensure the update can be continued and get the updating stage index and cluster indices.
-		if updatingStageIndex, toBeUpdatedBindings, toBeDeletedBindings, validateErr = r.validate(ctx, updateRun); validateErr != nil {
+		if updatingStageIndex, toBeUpdatedBindings, toBeDeletedBindings, reconcileErr = r.validate(ctx, updateRun); reconcileErr != nil {
+			klog.ErrorS(reconcileErr, "Failed to validate the updateRun", "updateRun", runObjRef)
 			// errStagedUpdatedAborted cannot be retried.
-			if errors.Is(validateErr, errStagedUpdatedAborted) {
-				return runtime.Result{}, r.recordUpdateRunFailed(ctx, updateRun, validateErr.Error())
+			if errors.Is(reconcileErr, errStagedUpdatedAborted) {
+				return runtime.Result{}, r.recordUpdateRunFailed(ctx, updateRun, reconcileErr.Error())
 			}
-			return runtime.Result{}, validateErr
+			return runtime.Result{}, reconcileErr
 		}
 		klog.V(2).InfoS("The updateRun is validated", "updateRun", runObjRef)
 	}
@@ -163,45 +166,48 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim
 		return runtime.Result{}, r.recordUpdateRunSucceeded(ctx, updateRun)
 	}
 
+	var finished bool
+	var waitTime time.Duration
 	switch state {
 	case placementv1beta1.StateInitialize:
 		klog.V(2).InfoS("The updateRun is initialized but not executed, waiting to execute", "state", state, "updateRun", runObjRef)
 	case placementv1beta1.StateRun:
 		// Execute the updateRun.
 		klog.V(2).InfoS("Continue to execute the updateRun", "updatingStageIndex", updatingStageIndex, "updateRun", runObjRef)
-		finished, waitTime, execErr := r.execute(ctx, updateRun, updatingStageIndex, toBeUpdatedBindings, toBeDeletedBindings)
-		if errors.Is(execErr, errStagedUpdatedAborted) {
+		finished, waitTime, reconcileErr = r.execute(ctx, updateRun, updatingStageIndex, toBeUpdatedBindings, toBeDeletedBindings)
+		if errors.Is(reconcileErr, errStagedUpdatedAborted) {
 			// errStagedUpdatedAborted cannot be retried.
-			return runtime.Result{}, r.recordUpdateRunFailed(ctx, updateRun, execErr.Error())
+			return runtime.Result{}, r.recordUpdateRunFailed(ctx, updateRun, reconcileErr.Error())
 		}
 
 		if finished {
 			klog.V(2).InfoS("The updateRun is completed", "updateRun", runObjRef)
 			return runtime.Result{}, r.recordUpdateRunSucceeded(ctx, updateRun)
 		}
 
-		return r.handleIncompleteUpdateRun(ctx, updateRun, waitTime, execErr, state, runObjRef)
+		return r.handleIncompleteUpdateRun(ctx, updateRun, waitTime, reconcileErr, state, runObjRef)
 	case placementv1beta1.StateStop:
 		// Stop the updateRun.
 		klog.V(2).InfoS("Stopping the updateRun", "state", state, "updatingStageIndex", updatingStageIndex, "updateRun", runObjRef)
-		finished, waitTime, stopErr := r.stop(updateRun, updatingStageIndex, toBeUpdatedBindings, toBeDeletedBindings)
-		if errors.Is(stopErr, errStagedUpdatedAborted) {
+		finished, waitTime, reconcileErr = r.stop(updateRun, updatingStageIndex, toBeUpdatedBindings, toBeDeletedBindings)
+		if errors.Is(reconcileErr, errStagedUpdatedAborted) {
 			// errStagedUpdatedAborted cannot be retried.
-			return runtime.Result{}, r.recordUpdateRunFailed(ctx, updateRun, stopErr.Error())
+			return runtime.Result{}, r.recordUpdateRunFailed(ctx, updateRun, reconcileErr.Error())
 		}
 
 		if finished {
 			klog.V(2).InfoS("The updateRun is stopped", "updateRun", runObjRef)
 			return runtime.Result{}, r.recordUpdateRunStopped(ctx, updateRun)
 		}
 
-		return r.handleIncompleteUpdateRun(ctx, updateRun, waitTime, stopErr, state, runObjRef)
+		return r.handleIncompleteUpdateRun(ctx, updateRun, waitTime, reconcileErr, state, runObjRef)
 
 	default:
 		// Initialize, Run, or Stop are the only supported states.
-		unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("found unsupported updateRun state: %s", state))
-		klog.ErrorS(unexpectedErr, "Invalid updateRun state", "state", state, "updateRun", runObjRef)
-		return runtime.Result{}, r.recordUpdateRunFailed(ctx, updateRun, unexpectedErr.Error())
+		reconcileErr = controller.NewUnexpectedBehaviorError(fmt.Errorf("found unsupported updateRun state: %s", state))
+		klog.ErrorS(reconcileErr, "Invalid updateRun state", "state", state, "updateRun", runObjRef)
+		// This is an internal error - unsupported state should not happen
+		return runtime.Result{}, r.recordUpdateRunFailed(ctx, updateRun, reconcileErr.Error())
 	}
 	return runtime.Result{}, nil
 }

pkg/controllers/updaterun/controller_integration_test.go

@@ -377,7 +377,7 @@ func generateApprovalStageTaskMetric(
 // the current updateRun state if the updateRun has transitioned since then.
 func generateMetricsLabels(
 	updateRun *placementv1beta1.ClusterStagedUpdateRun,
-	state, condition, status, reason string,
+	state, condition, status, reason, failureType string,
 ) []*prometheusclientmodel.LabelPair {
 	return []*prometheusclientmodel.LabelPair{
 		{Name: ptr.To("namespace"), Value: &updateRun.Namespace},
@@ -386,23 +386,24 @@ func generateMetricsLabels(
 		{Name: ptr.To("condition"), Value: ptr.To(condition)},
 		{Name: ptr.To("status"), Value: ptr.To(status)},
 		{Name: ptr.To("reason"), Value: ptr.To(reason)},
+		{Name: ptr.To("failureType"), Value: ptr.To(failureType)},
 	}
 }
 
 func generateInitializationSucceededMetric(state placementv1beta1.State, updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric {
 	return &prometheusclientmodel.Metric{
 		Label: generateMetricsLabels(updateRun, string(state), string(placementv1beta1.StagedUpdateRunConditionInitialized),
-			string(metav1.ConditionTrue), condition.UpdateRunInitializeSucceededReason),
+			string(metav1.ConditionTrue), condition.UpdateRunInitializeSucceededReason, string(hubmetrics.UpdateRunFailureTypeNone)),
 		Gauge: &prometheusclientmodel.Gauge{
 			Value: ptr.To(float64(time.Now().UnixNano()) / 1e9),
 		},
 	}
 }
 
-func generateInitializationFailedMetric(state placementv1beta1.State, updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric {
+func generateInitializationFailedMetric(state placementv1beta1.State, updateRun *placementv1beta1.ClusterStagedUpdateRun, failureType string) *prometheusclientmodel.Metric {
 	return &prometheusclientmodel.Metric{
 		Label: generateMetricsLabels(updateRun, string(state), string(placementv1beta1.StagedUpdateRunConditionInitialized),
-			string(metav1.ConditionFalse), condition.UpdateRunInitializeFailedReason),
+			string(metav1.ConditionFalse), condition.UpdateRunInitializeFailedReason, failureType),
 		Gauge: &prometheusclientmodel.Gauge{
 			Value: ptr.To(float64(time.Now().UnixNano()) / 1e9),
 		},
@@ -412,7 +413,7 @@ func generateInitializationFailedMetric(state placementv1beta1.State, updateRun
 func generateProgressingMetric(state placementv1beta1.State, updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric {
 	return &prometheusclientmodel.Metric{
 		Label: generateMetricsLabels(updateRun, string(state), string(placementv1beta1.StagedUpdateRunConditionProgressing),
-			string(metav1.ConditionTrue), condition.UpdateRunProgressingReason),
+			string(metav1.ConditionTrue), condition.UpdateRunProgressingReason, string(hubmetrics.UpdateRunFailureTypeNone)),
 		Gauge: &prometheusclientmodel.Gauge{
 			Value: ptr.To(float64(time.Now().UnixNano()) / 1e9),
 		},
@@ -422,7 +423,7 @@ func generateProgressingMetric(state placementv1beta1.State, updateRun *placemen
 func generateWaitingMetric(state placementv1beta1.State, updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric {
 	return &prometheusclientmodel.Metric{
 		Label: generateMetricsLabels(updateRun, string(state), string(placementv1beta1.StagedUpdateRunConditionProgressing),
-			string(metav1.ConditionFalse), condition.UpdateRunWaitingReason),
+			string(metav1.ConditionFalse), condition.UpdateRunWaitingReason, string(hubmetrics.UpdateRunFailureTypeNone)),
 		Gauge: &prometheusclientmodel.Gauge{
 			Value: ptr.To(float64(time.Now().UnixNano()) / 1e9),
 		},
@@ -432,17 +433,17 @@ func generateWaitingMetric(state placementv1beta1.State, updateRun *placementv1b
 func generateStuckMetric(state placementv1beta1.State, updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric {
 	return &prometheusclientmodel.Metric{
 		Label: generateMetricsLabels(updateRun, string(state), string(placementv1beta1.StagedUpdateRunConditionProgressing),
-			string(metav1.ConditionFalse), condition.UpdateRunStuckReason),
+			string(metav1.ConditionFalse), condition.UpdateRunStuckReason, string(hubmetrics.UpdateRunFailureTypeInternalError)),
 		Gauge: &prometheusclientmodel.Gauge{
 			Value: ptr.To(float64(time.Now().UnixNano()) / 1e9),
 		},
 	}
 }
 
-func generateFailedMetric(state placementv1beta1.State, updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric {
+func generateFailedMetric(state placementv1beta1.State, updateRun *placementv1beta1.ClusterStagedUpdateRun, failureType string) *prometheusclientmodel.Metric {
 	return &prometheusclientmodel.Metric{
 		Label: generateMetricsLabels(updateRun, string(state), string(placementv1beta1.StagedUpdateRunConditionSucceeded),
-			string(metav1.ConditionFalse), condition.UpdateRunFailedReason),
+			string(metav1.ConditionFalse), condition.UpdateRunFailedReason, failureType),
 		Gauge: &prometheusclientmodel.Gauge{
 			Value: ptr.To(float64(time.Now().UnixNano()) / 1e9),
 		},
@@ -452,7 +453,7 @@ func generateFailedMetric(state placementv1beta1.State, updateRun *placementv1be
 func generateStoppingMetric(state placementv1beta1.State, updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric {
 	return &prometheusclientmodel.Metric{
 		Label: generateMetricsLabels(updateRun, string(state), string(placementv1beta1.StagedUpdateRunConditionProgressing),
-			string(metav1.ConditionUnknown), condition.UpdateRunStoppingReason),
+			string(metav1.ConditionUnknown), condition.UpdateRunStoppingReason, string(hubmetrics.UpdateRunFailureTypeNone)),
 		Gauge: &prometheusclientmodel.Gauge{
 			Value: ptr.To(float64(time.Now().UnixNano()) / 1e9),
 		},
@@ -462,7 +463,7 @@ func generateStoppingMetric(state placementv1beta1.State, updateRun *placementv1
 func generateStoppedMetric(state placementv1beta1.State, updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric {
 	return &prometheusclientmodel.Metric{
 		Label: generateMetricsLabels(updateRun, string(state), string(placementv1beta1.StagedUpdateRunConditionProgressing),
-			string(metav1.ConditionFalse), condition.UpdateRunStoppedReason),
+			string(metav1.ConditionFalse), condition.UpdateRunStoppedReason, string(hubmetrics.UpdateRunFailureTypeNone)),
 		Gauge: &prometheusclientmodel.Gauge{
 			Value: ptr.To(float64(time.Now().UnixNano()) / 1e9),
 		},
@@ -472,7 +473,7 @@ func generateStoppedMetric(state placementv1beta1.State, updateRun *placementv1b
 func generateSucceededMetric(state placementv1beta1.State, updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric {
 	return &prometheusclientmodel.Metric{
 		Label: generateMetricsLabels(updateRun, string(state), string(placementv1beta1.StagedUpdateRunConditionSucceeded),
-			string(metav1.ConditionTrue), condition.UpdateRunSucceededReason),
+			string(metav1.ConditionTrue), condition.UpdateRunSucceededReason, string(hubmetrics.UpdateRunFailureTypeNone)),
 		Gauge: &prometheusclientmodel.Gauge{
 			Value: ptr.To(float64(time.Now().UnixNano()) / 1e9),
 		},
@@ -963,16 +964,16 @@ func generateFalseCondition(obj client.Object, condType any) metav1.Condition {
 	}
 }
 
-func generateFalseProgressingCondition(obj client.Object, condType any, reason string) metav1.Condition {
+func generateFalseConditionWithReason(obj client.Object, condType any, reason string) metav1.Condition {
 	falseCond := generateFalseCondition(obj, condType)
 	falseCond.Reason = reason
 	return falseCond
 }
 
-func generateFalseConditionWithReason(obj client.Object, condType any, reason string) metav1.Condition {
-	falseCond := generateFalseCondition(obj, condType)
-	falseCond.Reason = reason
-	return falseCond
+func generateTrueConditionWithReason(obj client.Object, condType any, reason string) metav1.Condition {
+	trueCond := generateTrueCondition(obj, condType)
+	trueCond.Reason = reason
+	return trueCond
 }
 
 func generateProgressingUnknownConditionWithReason(obj client.Object, reason string) metav1.Condition {

pkg/controllers/updaterun/execution.go

@@ -76,6 +76,13 @@ func (r *Reconciler) execute(
 	markUpdateRunProgressingIfNotWaitingOrStuck(updateRun)
 	if updatingStageIndex < len(updateRunStatus.StagesStatus) {
 		updatingStageStatus = &updateRunStatus.StagesStatus[updatingStageIndex]
+		// Skip the entire stage when there are 0 clusters.
+		if len(updatingStageStatus.Clusters) == 0 {
+			klog.V(2).InfoS("The stage has 0 clusters, skipping the entire stage", "stage", updatingStageStatus.StageName, "updateRun", klog.KObj(updateRun))
+			markStageUpdatingSkippedNoClusters(updatingStageStatus, updateRun.GetGeneration(), "Stage skipped because it has no clusters")
+			// No need to wait to get to the next stage.
+			return false, 0, nil
+		}
 		approved, err := r.checkBeforeStageTasksStatus(ctx, updatingStageIndex, updateRun)
 		if err != nil {
 			return false, 0, err
@@ -249,7 +256,7 @@ func (r *Reconciler) executeUpdatingStage(
 				"bindingSpecInSync", inSync, "bindingState", bindingSpec.State,
 				"bindingRolloutStarted", rolloutStarted, "binding", klog.KObj(binding), "updateRun", updateRunRef)
 			markClusterUpdatingFailed(clusterStatus, updateRun.GetGeneration(), preemptedErr.Error())
-			clusterUpdateErrors = append(clusterUpdateErrors, fmt.Errorf("%w: %s", errStagedUpdatedAborted, preemptedErr.Error()))
+			clusterUpdateErrors = append(clusterUpdateErrors, fmt.Errorf("%w: %w", errStagedUpdatedAborted, preemptedErr))
 			continue
 		}
 
@@ -773,6 +780,30 @@ func markStageUpdatingSucceeded(stageUpdatingStatus *placementv1beta1.StageUpdat
 	})
 }
 
+// markStageUpdatingSkippedNoClusters marks the stage updating status as skipped due to no clusters in memory.
+func markStageUpdatingSkippedNoClusters(stageUpdatingStatus *placementv1beta1.StageUpdatingStatus, generation int64, message string) {
+	if stageUpdatingStatus.StartTime == nil {
+		stageUpdatingStatus.StartTime = &metav1.Time{Time: time.Now()}
+	}
+	if stageUpdatingStatus.EndTime == nil {
+		stageUpdatingStatus.EndTime = &metav1.Time{Time: time.Now()}
+	}
+	meta.SetStatusCondition(&stageUpdatingStatus.Conditions, metav1.Condition{
+		Type:               string(placementv1beta1.StageUpdatingConditionProgressing),
+		Status:             metav1.ConditionFalse,
+		ObservedGeneration: generation,
+		Reason:             condition.StageUpdatingSkippedNoClustersReason,
+		Message:            message,
+	})
+	meta.SetStatusCondition(&stageUpdatingStatus.Conditions, metav1.Condition{
+		Type:               string(placementv1beta1.StageUpdatingConditionSucceeded),
+		Status:             metav1.ConditionTrue,
+		ObservedGeneration: generation,
+		Reason:             condition.StageUpdatingSkippedNoClustersReason,
+		Message:            message,
+	})
+}
+
 // markStageUpdatingFailed marks the stage updating status as failed in memory.
 func markStageUpdatingFailed(stageUpdatingStatus *placementv1beta1.StageUpdatingStatus, generation int64, message string) {
 	if stageUpdatingStatus.EndTime == nil {