From 010a187246f78e77e1d6bb8da05644b25154f3d4 Mon Sep 17 00:00:00 2001 From: miyadav Date: Mon, 3 Nov 2025 11:12:30 +0000 Subject: [PATCH 1/3] Add e2e test for OCPBUGS-63519 --- pkg/framework/framework.go | 10 ++ pkg/operators/cluster-machine-approver.go | 123 ++++++++++++++++++++++ 2 files changed, 133 insertions(+) diff --git a/pkg/framework/framework.go b/pkg/framework/framework.go index f39865b2d..800c79b23 100644 --- a/pkg/framework/framework.go +++ b/pkg/framework/framework.go @@ -296,6 +296,16 @@ func SkipIfNotTechPreviewNoUpgrade(oc *gatherer.CLI, cl runtimeclient.Client) { } } +// SkipIfNotCustomNoUpgrade skip test if a cluster is not a CustomNoUpgrade cluster. +func SkipIfNotCustomNoUpgrade(oc *gatherer.CLI, cl runtimeclient.Client) { + featureSet, err := oc.WithoutNamespace().Run("get").Args("featuregate", "cluster", "-o=jsonpath={.spec.featureSet}").Output() + Expect(err).NotTo(HaveOccurred(), "Failed to get featureSet") + + if featureSet != string(configv1.CustomNoUpgrade) { + Skip("FeatureSet is not CustomNoUpgrade, skip it!") + } +} + // GetCredentialsFromCluster get credentials from cluster. func GetCredentialsFromCluster(oc *gatherer.CLI) ([]byte, []byte, string) { awscreds, err := oc.WithoutNamespace().Run("get").Args("secret/aws-creds", "-n", "kube-system", "-o", "json").Output() diff --git a/pkg/operators/cluster-machine-approver.go b/pkg/operators/cluster-machine-approver.go index e2ccddcfa..35a1b7531 100644 --- a/pkg/operators/cluster-machine-approver.go +++ b/pkg/operators/cluster-machine-approver.go @@ -1,8 +1,13 @@ package operators import ( + "strconv" + "strings" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + discoveryv1 "k8s.io/api/discovery/v1" + runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/openshift/cluster-api-actuator-pkg/pkg/framework" ) @@ -11,6 +16,12 @@ const ( cmaDeployment = "machine-approver" cmaClusterOperator = "machine-approver" cmaNamespace = "openshift-cluster-machine-approver" + cmaDeploymentcapi = "machine-approver-capi" + cmaServicecapi = "machine-approver-capi" + cmaServicePortcapi = 9194 + cmaService = "machine-approver" + cmaServicePort = 9192 + cmaMetricsPort = 9193 ) var _ = Describe("Cluster Machine Approver deployment", framework.LabelMachineApprover, framework.LabelLEVEL0, func() { @@ -36,3 +47,115 @@ var _ = Describe("Cluster Machine Approver Cluster Operator Status", framework.L "Failed to wait for cluster-machine-approver Cluster Operator to be available") }) }) + +var _ = Describe("Cluster Machine Approver CAPI Integration", framework.LabelMachineApprover, framework.LabelLEVEL0, func() { + It("cluster-machine-approver must have endpoint slices for open ports the operator uses", func() { + ctx := framework.GetContext() + client, err := framework.LoadClient() + Expect(err).NotTo(HaveOccurred(), "Failed to load client") + + // Pre-check: Skip if cluster doesn't have TechPreviewNoUpgrade or CustomNoUpgrade featuregate enabled + oc, err := framework.NewCLI() + Expect(err).NotTo(HaveOccurred(), "Failed to create CLI") + + featureSet, err := oc.WithoutNamespace().Run("get").Args("featuregate", "cluster", "-o=jsonpath={.spec.featureSet}").Output() + Expect(err).NotTo(HaveOccurred(), "Failed to get featureSet") + + if featureSet != "TechPreviewNoUpgrade" && featureSet != "CustomNoUpgrade" { + Skip("Cluster does not have TechPreviewNoUpgrade or CustomNoUpgrade featuregate enabled. Not a valid cluster for this test.") + } + + // Test Case 1: Verify machine-approver-capi deployment has port 9193 configured with internal IP 127.0.0.1 + deployment, err := framework.GetDeployment(ctx, client, cmaDeploymentcapi, cmaNamespace) + Expect(err).NotTo(HaveOccurred(), "Failed to get machine-approver-capi deployment") + + // Check containers for the port configuration + foundInternalPort := false + foundUpstreamArg := false + for _, container := range deployment.Spec.Template.Spec.Containers { + // Check for upstream argument with 127.0.0.1:9193 + for _, arg := range container.Args { + if strings.Contains(arg, "--upstream=") && strings.Contains(arg, "127.0.0.1:"+strconv.Itoa(cmaMetricsPort)) { + foundUpstreamArg = true + } + } + + // Check for environment variable with port 9193 + for _, env := range container.Env { + if env.Value == "9193" || strings.Contains(env.Value, "9193") { + foundInternalPort = true + } + } + } + + Expect(foundUpstreamArg && foundInternalPort).To(BeTrue(), + "Deployment machine-approver-capi should have port 9193 configured with 127.0.0.1 (internal)") + + // Test Case 2: Verify services exist at ports 9192 and 9194 + // Check machine-approver service (port 9192) + maSvc, err := framework.GetService(ctx, client, cmaService, cmaNamespace) + Expect(err).NotTo(HaveOccurred(), "Failed to get machine-approver service") + + foundMaPort := false + for _, port := range maSvc.Spec.Ports { + if port.Port == cmaServicePort { + foundMaPort = true + break + } + } + Expect(foundMaPort).To(BeTrue(), + "Service machine-approver should have port 9192") + + // Check machine-approver-capi service (port 9194) + maCapiSvc, err := framework.GetService(ctx, client, cmaServicecapi, cmaNamespace) + Expect(err).NotTo(HaveOccurred(), "Failed to get machine-approver-capi service") + + foundCapiPort := false + for _, port := range maCapiSvc.Spec.Ports { + if port.Port == cmaServicePortcapi { + foundCapiPort = true + break + } + } + Expect(foundCapiPort).To(BeTrue(), + "Service machine-approver-capi should have port 9194") + + // Test Case 3: Verify endpoint slices exist for above ports + endpointSlices := &discoveryv1.EndpointSliceList{} + err = client.List(ctx, endpointSlices, runtimeclient.InNamespace(cmaNamespace)) + Expect(err).NotTo(HaveOccurred(), "Failed to list endpoint slices") + + // Find endpoint slices for machine-approver (port 9192) + foundMaEndpoint := false + foundCapiEndpoint := false + + for _, slice := range endpointSlices.Items { + serviceName := slice.Labels["kubernetes.io/service-name"] + + // Check for machine-approver endpoint slice with port 9192 + if serviceName == cmaService { + for _, port := range slice.Ports { + if port.Port != nil && *port.Port == cmaServicePort { + foundMaEndpoint = true + break + } + } + } + + // Check for machine-approver-capi endpoint slice with port 9194 + if serviceName == cmaServicecapi { + for _, port := range slice.Ports { + if port.Port != nil && *port.Port == cmaServicePortcapi { + foundCapiEndpoint = true + break + } + } + } + } + + Expect(foundMaEndpoint).To(BeTrue(), + "EndpointSlice for machine-approver service (port 9192) should exist") + Expect(foundCapiEndpoint).To(BeTrue(), + "EndpointSlice for machine-approver-capi service (port 9194) should exist") + }) +}) From bdc907940a49a68cd87d3518e6e264a7794e3a63 Mon Sep 17 00:00:00 2001 From: miyadav Date: Fri, 7 Nov 2025 10:59:01 +0000 Subject: [PATCH 2/3] moved client load check in beforeeach for all tests --- pkg/operators/cluster-machine-approver.go | 97 ++++++++++++----------- 1 file changed, 52 insertions(+), 45 deletions(-) diff --git a/pkg/operators/cluster-machine-approver.go b/pkg/operators/cluster-machine-approver.go index 35a1b7531..83765d7db 100644 --- a/pkg/operators/cluster-machine-approver.go +++ b/pkg/operators/cluster-machine-approver.go @@ -1,11 +1,13 @@ package operators import ( + "context" "strconv" "strings" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + configv1 "github.com/openshift/api/config/v1" discoveryv1 "k8s.io/api/discovery/v1" runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -24,12 +26,20 @@ const ( cmaMetricsPort = 9193 ) +var ( + ctx context.Context + client runtimeclient.Client +) + +var _ = BeforeEach(func() { + var err error + ctx = framework.GetContext() + client, err = framework.LoadClient() + Expect(err).NotTo(HaveOccurred(), "Failed to load client") +}) + var _ = Describe("Cluster Machine Approver deployment", framework.LabelMachineApprover, framework.LabelLEVEL0, func() { It("should be available", func() { - ctx := framework.GetContext() - - client, err := framework.LoadClient() - Expect(err).NotTo(HaveOccurred(), "Failed to load client") Expect(framework.IsDeploymentAvailable(ctx, client, cmaDeployment, cmaNamespace)).To(BeTrue(), "Failed to wait for cluster-machine-approver Deployment to become available") @@ -38,22 +48,13 @@ var _ = Describe("Cluster Machine Approver deployment", framework.LabelMachineAp var _ = Describe("Cluster Machine Approver Cluster Operator Status", framework.LabelMachineApprover, framework.LabelLEVEL0, func() { It("should be available", func() { - client, err := framework.LoadClient() - Expect(err).NotTo(HaveOccurred(), "Failed to load client") - - ctx := framework.GetContext() - Expect(framework.WaitForStatusAvailableShort(ctx, client, cmaClusterOperator)).To(BeTrue(), "Failed to wait for cluster-machine-approver Cluster Operator to be available") }) }) -var _ = Describe("Cluster Machine Approver CAPI Integration", framework.LabelMachineApprover, framework.LabelLEVEL0, func() { - It("cluster-machine-approver must have endpoint slices for open ports the operator uses", func() { - ctx := framework.GetContext() - client, err := framework.LoadClient() - Expect(err).NotTo(HaveOccurred(), "Failed to load client") - +var _ = Describe("Cluster Machine Approver CAPI Integration", framework.LabelMachineApprover, func() { + BeforeEach(func() { // Pre-check: Skip if cluster doesn't have TechPreviewNoUpgrade or CustomNoUpgrade featuregate enabled oc, err := framework.NewCLI() Expect(err).NotTo(HaveOccurred(), "Failed to create CLI") @@ -61,9 +62,13 @@ var _ = Describe("Cluster Machine Approver CAPI Integration", framework.LabelMac featureSet, err := oc.WithoutNamespace().Run("get").Args("featuregate", "cluster", "-o=jsonpath={.spec.featureSet}").Output() Expect(err).NotTo(HaveOccurred(), "Failed to get featureSet") - if featureSet != "TechPreviewNoUpgrade" && featureSet != "CustomNoUpgrade" { + // Check against the same constants used by SkipIfNotTechPreviewNoUpgrade and SkipIfNotCustomNoUpgrade + if featureSet != string(configv1.TechPreviewNoUpgrade) && featureSet != string(configv1.CustomNoUpgrade) { Skip("Cluster does not have TechPreviewNoUpgrade or CustomNoUpgrade featuregate enabled. Not a valid cluster for this test.") } + }) + + It("cluster-machine-approver must have endpoint slices for open ports the operator uses", func() { // Test Case 1: Verify machine-approver-capi deployment has port 9193 configured with internal IP 127.0.0.1 deployment, err := framework.GetDeployment(ctx, client, cmaDeploymentcapi, cmaNamespace) @@ -121,41 +126,43 @@ var _ = Describe("Cluster Machine Approver CAPI Integration", framework.LabelMac "Service machine-approver-capi should have port 9194") // Test Case 3: Verify endpoint slices exist for above ports - endpointSlices := &discoveryv1.EndpointSliceList{} - err = client.List(ctx, endpointSlices, runtimeclient.InNamespace(cmaNamespace)) - Expect(err).NotTo(HaveOccurred(), "Failed to list endpoint slices") - - // Find endpoint slices for machine-approver (port 9192) - foundMaEndpoint := false - foundCapiEndpoint := false - - for _, slice := range endpointSlices.Items { - serviceName := slice.Labels["kubernetes.io/service-name"] - - // Check for machine-approver endpoint slice with port 9192 - if serviceName == cmaService { - for _, port := range slice.Ports { - if port.Port != nil && *port.Port == cmaServicePort { - foundMaEndpoint = true - break + Eventually(func() bool { + endpointSlices := &discoveryv1.EndpointSliceList{} + err := client.List(ctx, endpointSlices, runtimeclient.InNamespace(cmaNamespace)) + if err != nil { + return false + } + + // Find endpoint slices for machine-approver (port 9192) + foundMaEndpoint := false + foundCapiEndpoint := false + + for _, slice := range endpointSlices.Items { + serviceName := slice.Labels["kubernetes.io/service-name"] + + // Check for machine-approver endpoint slice with port 9192 + if serviceName == cmaService { + for _, port := range slice.Ports { + if port.Port != nil && *port.Port == cmaServicePort { + foundMaEndpoint = true + break + } } } - } - // Check for machine-approver-capi endpoint slice with port 9194 - if serviceName == cmaServicecapi { - for _, port := range slice.Ports { - if port.Port != nil && *port.Port == cmaServicePortcapi { - foundCapiEndpoint = true - break + // Check for machine-approver-capi endpoint slice with port 9194 + if serviceName == cmaServicecapi { + for _, port := range slice.Ports { + if port.Port != nil && *port.Port == cmaServicePortcapi { + foundCapiEndpoint = true + break + } } } } - } - Expect(foundMaEndpoint).To(BeTrue(), - "EndpointSlice for machine-approver service (port 9192) should exist") - Expect(foundCapiEndpoint).To(BeTrue(), - "EndpointSlice for machine-approver-capi service (port 9194) should exist") + return foundMaEndpoint && foundCapiEndpoint + }, framework.WaitShort, framework.RetryMedium).Should(BeTrue(), + "EndpointSlices for machine-approver (port 9192) and machine-approver-capi (port 9194) should exist") }) }) From a0fc164aa4bb5713bf2a15589b7fa6e0f901e1d5 Mon Sep 17 00:00:00 2001 From: miyadav Date: Thu, 13 Nov 2025 07:41:11 +0000 Subject: [PATCH 3/3] added skipfunc --- pkg/framework/framework.go | 10 ++++++++++ pkg/operators/cluster-machine-approver.go | 9 +-------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/pkg/framework/framework.go b/pkg/framework/framework.go index 800c79b23..9d64bd7f2 100644 --- a/pkg/framework/framework.go +++ b/pkg/framework/framework.go @@ -306,6 +306,16 @@ func SkipIfNotCustomNoUpgrade(oc *gatherer.CLI, cl runtimeclient.Client) { } } +// SkipIfNotTechPreviewNoUpgradeOrCustomNoUpgrade skip test if a cluster does not have TechPreviewNoUpgrade or CustomNoUpgrade featuregate enabled. +func SkipIfNotTechPreviewNoUpgradeOrCustomNoUpgrade(oc *gatherer.CLI, cl runtimeclient.Client) { + featureSet, err := oc.WithoutNamespace().Run("get").Args("featuregate", "cluster", "-o=jsonpath={.spec.featureSet}").Output() + Expect(err).NotTo(HaveOccurred(), "Failed to get featureSet") + + if featureSet != string(configv1.TechPreviewNoUpgrade) && featureSet != string(configv1.CustomNoUpgrade) { + Skip("Cluster does not have TechPreviewNoUpgrade or CustomNoUpgrade featuregate enabled. Not a valid cluster for this test.") + } +} + // GetCredentialsFromCluster get credentials from cluster. func GetCredentialsFromCluster(oc *gatherer.CLI) ([]byte, []byte, string) { awscreds, err := oc.WithoutNamespace().Run("get").Args("secret/aws-creds", "-n", "kube-system", "-o", "json").Output() diff --git a/pkg/operators/cluster-machine-approver.go b/pkg/operators/cluster-machine-approver.go index 83765d7db..dd401fbe7 100644 --- a/pkg/operators/cluster-machine-approver.go +++ b/pkg/operators/cluster-machine-approver.go @@ -7,7 +7,6 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - configv1 "github.com/openshift/api/config/v1" discoveryv1 "k8s.io/api/discovery/v1" runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -59,13 +58,7 @@ var _ = Describe("Cluster Machine Approver CAPI Integration", framework.LabelMac oc, err := framework.NewCLI() Expect(err).NotTo(HaveOccurred(), "Failed to create CLI") - featureSet, err := oc.WithoutNamespace().Run("get").Args("featuregate", "cluster", "-o=jsonpath={.spec.featureSet}").Output() - Expect(err).NotTo(HaveOccurred(), "Failed to get featureSet") - - // Check against the same constants used by SkipIfNotTechPreviewNoUpgrade and SkipIfNotCustomNoUpgrade - if featureSet != string(configv1.TechPreviewNoUpgrade) && featureSet != string(configv1.CustomNoUpgrade) { - Skip("Cluster does not have TechPreviewNoUpgrade or CustomNoUpgrade featuregate enabled. Not a valid cluster for this test.") - } + framework.SkipIfNotTechPreviewNoUpgradeOrCustomNoUpgrade(oc, client) }) It("cluster-machine-approver must have endpoint slices for open ports the operator uses", func() {