OCPBUGS-98220: fix(cpo,hcco): prevent premature KAS convergence during KMS key rotation [release-4.22]

control-plane-operator/controllers/hostedcontrolplane/v2/kas/deployment.go

@@ -15,6 +15,7 @@ import (
 	"github.com/openshift/hypershift/support/config"
 	component "github.com/openshift/hypershift/support/controlplane-component"
 	"github.com/openshift/hypershift/support/proxy"
+	"github.com/openshift/hypershift/support/secretencryption"
 	"github.com/openshift/hypershift/support/util"
 
 	configv1 "github.com/openshift/api/config/v1"
@@ -123,14 +124,20 @@ func adaptDeployment(cpContext component.WorkloadContext, deployment *appsv1.Dep
 
 	if secretEncryption := hcp.Spec.SecretEncryption; secretEncryption != nil {
 		applyGenericSecretEncryptionConfig(&deployment.Spec.Template.Spec)
+		encConfigSecret := manifests.KASSecretEncryptionConfigFile(hcp.Namespace)
+		currentConfig, configBytes, err := readCurrentEncryptionConfig(cpContext, encConfigSecret)
+		if err != nil {
+			return fmt.Errorf("failed to read current encryption config: %w", err)
+		}
+		// Record a dedicated encryption-config hash on the pod template. The CPO v2
+		// framework already computes a composite config-hash from all mounted secrets,
+		// but that hash mixes multiple secrets together. This separate annotation lets
+		// the HCCO verify that KAS has rolled out with a specific encryption config.
+		secretencryption.SetEncryptionConfigHashAnnotation(&deployment.Spec.Template, configBytes)
 		switch secretEncryption.Type {
 		case hyperv1.KMS:
-			encConfigSecret := manifests.KASSecretEncryptionConfigFile(hcp.Namespace)
-			currentConfig, err := readCurrentEncryptionConfig(cpContext, encConfigSecret)
-			if err != nil {
-				return fmt.Errorf("failed to read current encryption config: %w", err)
-			}
-			kasReady, err := isKASConverged(cpContext)
+			configHash := secretencryption.EncryptionConfigHash(configBytes)
+			kasReady, err := isKASConverged(cpContext, configHash)
 			if err != nil {
 				return fmt.Errorf("failed to check KAS convergence: %w", err)
 			}

control-plane-operator/controllers/hostedcontrolplane/v2/kas/secretencryption.go

@@ -7,7 +7,6 @@ import (
 	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests"
 	component "github.com/openshift/hypershift/support/controlplane-component"
 	"github.com/openshift/hypershift/support/secretencryption"
-	"github.com/openshift/hypershift/support/util"
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -35,13 +34,14 @@ func adaptSecretEncryptionConfig(cpContext component.WorkloadContext, secret *co
 	encStatus := &cpContext.HCP.Status.SecretEncryption
 
 	// Read the live encryption config from the cluster to derive the two-stage rollout state.
-	currentConfig, err := readCurrentEncryptionConfig(cpContext, secret)
+	currentConfig, configBytes, err := readCurrentEncryptionConfig(cpContext, secret)
 	if err != nil {
 		return fmt.Errorf("failed to read current encryption config: %w", err)
 	}
 
 	// Check KAS convergence — needed to decide whether to promote the target key.
-	kasConverged, err := isKASConverged(cpContext)
+	configHash := secretencryption.EncryptionConfigHash(configBytes)
+	kasConverged, err := isKASConverged(cpContext, configHash)
 	if err != nil {
 		return fmt.Errorf("failed to check KAS convergence: %w", err)
 	}
@@ -67,9 +67,10 @@ func adaptSecretEncryptionConfig(cpContext component.WorkloadContext, secret *co
 	return nil
 }
 
-// isKASConverged checks if the KAS Deployment has fully rolled out.
+// isKASConverged checks if the KAS Deployment has fully rolled out with the
+// encryption config snapshot identified by expectedConfigHash.
 // Returns false (not error) if the deployment doesn't exist yet.
-func isKASConverged(cpContext component.WorkloadContext) (bool, error) {
+func isKASConverged(cpContext component.WorkloadContext, expectedConfigHash string) (bool, error) {
 	kasDeployment := &appsv1.Deployment{}
 	kasRef := manifests.KASDeployment(cpContext.HCP.Namespace)
 	if err := cpContext.Client.Get(cpContext, client.ObjectKeyFromObject(kasRef), kasDeployment); err != nil {
@@ -78,27 +79,31 @@ func isKASConverged(cpContext component.WorkloadContext) (bool, error) {
 		}
 		return false, fmt.Errorf("failed to get KAS deployment: %w", err)
 	}
-	return util.IsDeploymentReady(cpContext, kasDeployment), nil
+	return secretencryption.KASDeploymentConvergedWithEncryptionConfig(cpContext, cpContext.Client, kasDeployment, expectedConfigHash), nil
 }
 
 // readCurrentEncryptionConfig reads the live encryption config secret from the
 // cluster and parses its EncryptionConfiguration. Returns nil (not an error)
 // if the secret does not exist yet.
-func readCurrentEncryptionConfig(cpContext component.WorkloadContext, templateSecret *corev1.Secret) (*apiserverv1.EncryptionConfiguration, error) {
+func readCurrentEncryptionConfig(cpContext component.WorkloadContext, templateSecret *corev1.Secret) (*apiserverv1.EncryptionConfiguration, []byte, error) {
 	existingSecret := &corev1.Secret{}
 	if err := cpContext.Client.Get(cpContext, client.ObjectKeyFromObject(templateSecret), existingSecret); err != nil {
 		if apierrors.IsNotFound(err) {
-			return nil, nil
+			return nil, nil, nil
 		}
-		return nil, err
+		return nil, nil, err
 	}
 
 	configBytes := existingSecret.Data[secretEncryptionConfigurationKey]
 	if len(configBytes) == 0 {
-		return nil, nil
+		return nil, nil, nil
 	}
 
-	return secretencryption.DecodeEncryptionConfiguration(configBytes)
+	currentConfig, err := secretencryption.DecodeEncryptionConfiguration(configBytes)
+	if err != nil {
+		return nil, configBytes, err
+	}
+	return currentConfig, configBytes, nil
 }
 
 // deriveAESCBCEncryptionConfig determines the AESCBC write and read keys using

control-plane-operator/hostedclusterconfigoperator/controllers/reencryption/reencryption.go

@@ -213,12 +213,12 @@ func (r *Reconciler) handleInProgressRotation(ctx context.Context, log logr.Logg
 	}
 
 	// Derive the current phase from observable state.
-	role, err := r.deriveTargetKeyRole(ctx, hcp)
+	role, configHash, err := r.deriveTargetKeyRole(ctx, hcp)
 	if err != nil {
 		return reconcile.Result{}, fmt.Errorf("failed to derive target key role from EncryptionConfiguration: %w", err)
 	}
 
-	converged, err := r.isKASConverged(ctx, log, hcp)
+	converged, err := r.isKASConverged(ctx, log, hcp, configHash)
 	if err != nil {
 		return reconcile.Result{}, fmt.Errorf("failed to check KAS convergence: %w", err)
 	}
@@ -289,35 +289,39 @@ func (r *Reconciler) handleInProgressRotation(ctx context.Context, log logr.Logg
 
 // deriveTargetKeyRole reads the kas-secret-encryption-config Secret, parses the
 // EncryptionConfiguration, and determines where the target key appears.
-func (r *Reconciler) deriveTargetKeyRole(ctx context.Context, hcp *hyperv1.HostedControlPlane) (secretencryption.TargetKeyRole, error) {
+// It also returns a hash of the config bytes it read so callers can verify that
+// KAS has rolled out with that exact config snapshot.
+func (r *Reconciler) deriveTargetKeyRole(ctx context.Context, hcp *hyperv1.HostedControlPlane) (secretencryption.TargetKeyRole, string, error) {
 	secret := &corev1.Secret{}
 	secretKey := crclient.ObjectKey{
 		Namespace: hcp.Namespace,
 		Name:      manifests.KASSecretEncryptionConfigFile("").Name,
 	}
 	if err := r.cpClient.Get(ctx, secretKey, secret); err != nil {
 		if apierrors.IsNotFound(err) {
-			return secretencryption.TargetKeyAbsent, nil
+			return secretencryption.TargetKeyAbsent, "", nil
 		}
-		return secretencryption.TargetKeyAbsent, fmt.Errorf("failed to get encryption config secret: %w", err)
+		return secretencryption.TargetKeyAbsent, "", fmt.Errorf("failed to get encryption config secret: %w", err)
 	}
 
 	configBytes := secret.Data[secretencryption.EncryptionConfigurationKey]
 	if len(configBytes) == 0 {
-		return secretencryption.TargetKeyAbsent, nil
+		return secretencryption.TargetKeyAbsent, "", nil
 	}
 
+	configHash := secretencryption.EncryptionConfigHash(configBytes)
+
 	currentConfig, err := secretencryption.DecodeEncryptionConfiguration(configBytes)
 	if err != nil {
-		return secretencryption.TargetKeyAbsent, err
+		return secretencryption.TargetKeyAbsent, configHash, err
 	}
 
 	targetName, err := r.computeTargetKeyProviderName(ctx, hcp)
 	if err != nil {
-		return secretencryption.TargetKeyAbsent, fmt.Errorf("failed to compute target key provider name: %w", err)
+		return secretencryption.TargetKeyAbsent, configHash, fmt.Errorf("failed to compute target key provider name: %w", err)
 	}
 
-	return secretencryption.FindKeyRole(currentConfig, targetName, hcp.Spec.SecretEncryption.Type), nil
+	return secretencryption.FindKeyRole(currentConfig, targetName, hcp.Spec.SecretEncryption.Type), configHash, nil
 }
 
 // computeTargetKeyProviderName computes the provider name that the CPO would
@@ -464,8 +468,9 @@ func (r *Reconciler) completeRotation(log logr.Logger, hcp *hyperv1.HostedContro
 	return reconcile.Result{}, nil
 }
 
-// isKASConverged checks if the KAS Deployment has fully rolled out.
-func (r *Reconciler) isKASConverged(ctx context.Context, log logr.Logger, hcp *hyperv1.HostedControlPlane) (bool, error) {
+// isKASConverged checks if the KAS Deployment has fully rolled out with the
+// encryption config snapshot identified by expectedConfigHash.
+func (r *Reconciler) isKASConverged(ctx context.Context, log logr.Logger, hcp *hyperv1.HostedControlPlane, expectedConfigHash string) (bool, error) {
 	deployment := &appsv1.Deployment{}
 	kasRef := manifests.KASDeployment(hcp.Namespace)
 	if err := r.cpClient.Get(ctx, crclient.ObjectKeyFromObject(kasRef), deployment); err != nil {
@@ -475,11 +480,13 @@ func (r *Reconciler) isKASConverged(ctx context.Context, log logr.Logger, hcp *h
 		return false, fmt.Errorf("failed to get KAS deployment: %w", err)
 	}
 
-	ready := util.IsDeploymentReady(ctx, deployment)
-	if !ready {
-		log.V(2).Info("KAS not converged")
+	converged := secretencryption.KASDeploymentConvergedWithEncryptionConfig(ctx, r.cpClient, deployment, expectedConfigHash)
+	if !converged {
+		log.V(2).Info("KAS not converged with encryption config",
+			"expectedConfigHash", expectedConfigHash,
+			"deploymentConfigHash", deployment.Spec.Template.Annotations[secretencryption.EncryptionConfigHashAnnotation])
 	}
-	return ready, nil
+	return converged, nil
 }
 
 // keyStatusFromSpec computes a SecretEncryptionKeyStatus from the HCP spec.