From 623b0189c936eb75331145cb15e3e64b1cd8555b Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 17 Oct 2023 10:47:15 +0530 Subject: [PATCH 1/4] Handle errors while scaling kubernetes cluster --- .../cluster/actionworkers/KubernetesClusterScaleWorker.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index e3708673f9d0..ab11d366a356 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -32,7 +32,6 @@ import org.apache.log4j.Level; import com.cloud.dc.DataCenter; -import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.ManagementServerException; import com.cloud.exception.NetworkRuleConflictException; @@ -299,8 +298,8 @@ private void scaleKubernetesClusterOffering() throws CloudRuntimeException { boolean result = false; try { result = userVmManager.upgradeVirtualMachine(userVM.getId(), serviceOffering.getId(), new HashMap()); - } catch (ResourceUnavailableException | ManagementServerException | ConcurrentOperationException | VirtualMachineMigrationException e) { - logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster : %s failed, unable to scale cluster VM : %s", kubernetesCluster.getName(), userVM.getDisplayName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); + } catch (CloudRuntimeException | ResourceUnavailableException | ManagementServerException | VirtualMachineMigrationException e) { + logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster : %s failed, unable to scale cluster VM : %s due to %s", kubernetesCluster.getName(), userVM.getDisplayName(), e.getMessage()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); } if (!result) { logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster : %s failed, unable to scale cluster VM : %s", kubernetesCluster.getName(), userVM.getDisplayName()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); From 03d1295e8ac07d40e6907fd97a9f18feeca130b4 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 7 Dec 2023 13:02:54 +0530 Subject: [PATCH 2/4] Fixed state issues and also applying new compute offering to all nodes --- .../KubernetesClusterResourceModifierActionWorker.java | 9 ++++++--- .../actionworkers/KubernetesClusterScaleWorker.java | 6 +++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 726aadff4d06..4b6972e4c592 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -768,11 +768,11 @@ protected String getKubernetesClusterNodeNamePrefix() { } protected KubernetesClusterVO updateKubernetesClusterEntry(final Long cores, final Long memory, - final Long size, final Long serviceOfferingId, final Boolean autoscaleEnabled, final Long minSize, final Long maxSize) { + final Long size, final Long serviceOfferingId, final Boolean autoscaleEnabled, final Long minSize, final Long maxSize, KubernetesCluster.State state) { return Transaction.execute(new TransactionCallback() { @Override public KubernetesClusterVO doInTransaction(TransactionStatus status) { - KubernetesClusterVO updatedCluster = kubernetesClusterDao.createForUpdate(kubernetesCluster.getId()); + KubernetesClusterVO updatedCluster = kubernetesClusterDao.findById(kubernetesCluster.getId()); if (cores != null) { updatedCluster.setCores(cores); } @@ -788,6 +788,9 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { if (autoscaleEnabled != null) { updatedCluster.setAutoscalingEnabled(autoscaleEnabled.booleanValue()); } + if (state != null) { + updatedCluster.setState(state); + } updatedCluster.setMinSize(minSize); updatedCluster.setMaxSize(maxSize); return kubernetesClusterDao.persist(updatedCluster); @@ -796,7 +799,7 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { } private KubernetesClusterVO updateKubernetesClusterEntry(final Boolean autoscaleEnabled, final Long minSize, final Long maxSize) throws CloudRuntimeException { - KubernetesClusterVO kubernetesClusterVO = updateKubernetesClusterEntry(null, null, null, null, autoscaleEnabled, minSize, maxSize); + KubernetesClusterVO kubernetesClusterVO = updateKubernetesClusterEntry(null, null, null, null, autoscaleEnabled, minSize, maxSize, null); if (kubernetesClusterVO == null) { logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster %s failed, unable to update Kubernetes cluster", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index ab11d366a356..2667dffdcd41 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -182,7 +182,7 @@ private KubernetesClusterVO updateKubernetesClusterEntry(final Long newSize, fin final long cores = serviceOffering.getCpu() * size; final long memory = serviceOffering.getRamSize() * size; KubernetesClusterVO kubernetesClusterVO = updateKubernetesClusterEntry(cores, memory, newSize, serviceOfferingId, - kubernetesCluster.getAutoscalingEnabled(), kubernetesCluster.getMinSize(), kubernetesCluster.getMaxSize()); + kubernetesCluster.getAutoscalingEnabled(), kubernetesCluster.getMinSize(), kubernetesCluster.getMaxSize(), kubernetesCluster.getState()); if (kubernetesClusterVO == null) { logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster %s failed, unable to update Kubernetes cluster", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); @@ -298,7 +298,7 @@ private void scaleKubernetesClusterOffering() throws CloudRuntimeException { boolean result = false; try { result = userVmManager.upgradeVirtualMachine(userVM.getId(), serviceOffering.getId(), new HashMap()); - } catch (CloudRuntimeException | ResourceUnavailableException | ManagementServerException | VirtualMachineMigrationException e) { + } catch (RuntimeException | ResourceUnavailableException | ManagementServerException | VirtualMachineMigrationException e) { logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster : %s failed, unable to scale cluster VM : %s due to %s", kubernetesCluster.getName(), userVM.getDisplayName(), e.getMessage()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed, e); } if (!result) { @@ -411,7 +411,7 @@ private void scaleKubernetesClusterSize() throws CloudRuntimeException { } else { // upscale, same node count handled above scaleUpKubernetesClusterSize(newVmRequiredCount); } - kubernetesCluster = updateKubernetesClusterEntry(clusterSize, null); + kubernetesCluster = updateKubernetesClusterEntry(clusterSize, serviceOffering); } private boolean isAutoscalingChanged() { From 6a419bca92bcc26bb5b844e0ce7ddd530b57f51a Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 7 Dec 2023 13:32:27 +0530 Subject: [PATCH 3/4] Format code --- .../KubernetesClusterResourceModifierActionWorker.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 4b6972e4c592..710359b195db 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -767,8 +767,8 @@ protected String getKubernetesClusterNodeNamePrefix() { return prefix; } - protected KubernetesClusterVO updateKubernetesClusterEntry(final Long cores, final Long memory, - final Long size, final Long serviceOfferingId, final Boolean autoscaleEnabled, final Long minSize, final Long maxSize, KubernetesCluster.State state) { + protected KubernetesClusterVO updateKubernetesClusterEntry(final Long cores, final Long memory, final Long size, + final Long serviceOfferingId, final Boolean autoscaleEnabled, final Long minSize, final Long maxSize, KubernetesCluster.State state) { return Transaction.execute(new TransactionCallback() { @Override public KubernetesClusterVO doInTransaction(TransactionStatus status) { From ccf9d1ea0976afdb00e7f0436c53f9bc20ce5321 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Fri, 8 Dec 2023 15:24:50 +0530 Subject: [PATCH 4/4] revert manual state changes --- .../KubernetesClusterResourceModifierActionWorker.java | 7 ++----- .../actionworkers/KubernetesClusterScaleWorker.java | 4 ++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 710359b195db..0ae22bf8c8d5 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -768,7 +768,7 @@ protected String getKubernetesClusterNodeNamePrefix() { } protected KubernetesClusterVO updateKubernetesClusterEntry(final Long cores, final Long memory, final Long size, - final Long serviceOfferingId, final Boolean autoscaleEnabled, final Long minSize, final Long maxSize, KubernetesCluster.State state) { + final Long serviceOfferingId, final Boolean autoscaleEnabled, final Long minSize, final Long maxSize) { return Transaction.execute(new TransactionCallback() { @Override public KubernetesClusterVO doInTransaction(TransactionStatus status) { @@ -788,9 +788,6 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { if (autoscaleEnabled != null) { updatedCluster.setAutoscalingEnabled(autoscaleEnabled.booleanValue()); } - if (state != null) { - updatedCluster.setState(state); - } updatedCluster.setMinSize(minSize); updatedCluster.setMaxSize(maxSize); return kubernetesClusterDao.persist(updatedCluster); @@ -799,7 +796,7 @@ public KubernetesClusterVO doInTransaction(TransactionStatus status) { } private KubernetesClusterVO updateKubernetesClusterEntry(final Boolean autoscaleEnabled, final Long minSize, final Long maxSize) throws CloudRuntimeException { - KubernetesClusterVO kubernetesClusterVO = updateKubernetesClusterEntry(null, null, null, null, autoscaleEnabled, minSize, maxSize, null); + KubernetesClusterVO kubernetesClusterVO = updateKubernetesClusterEntry(null, null, null, null, autoscaleEnabled, minSize, maxSize); if (kubernetesClusterVO == null) { logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster %s failed, unable to update Kubernetes cluster", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 2667dffdcd41..df94642a8810 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -182,7 +182,7 @@ private KubernetesClusterVO updateKubernetesClusterEntry(final Long newSize, fin final long cores = serviceOffering.getCpu() * size; final long memory = serviceOffering.getRamSize() * size; KubernetesClusterVO kubernetesClusterVO = updateKubernetesClusterEntry(cores, memory, newSize, serviceOfferingId, - kubernetesCluster.getAutoscalingEnabled(), kubernetesCluster.getMinSize(), kubernetesCluster.getMaxSize(), kubernetesCluster.getState()); + kubernetesCluster.getAutoscalingEnabled(), kubernetesCluster.getMinSize(), kubernetesCluster.getMaxSize()); if (kubernetesClusterVO == null) { logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster %s failed, unable to update Kubernetes cluster", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); @@ -411,7 +411,7 @@ private void scaleKubernetesClusterSize() throws CloudRuntimeException { } else { // upscale, same node count handled above scaleUpKubernetesClusterSize(newVmRequiredCount); } - kubernetesCluster = updateKubernetesClusterEntry(clusterSize, serviceOffering); + kubernetesCluster = updateKubernetesClusterEntry(clusterSize, null); } private boolean isAutoscalingChanged() {