From 12384240b0c4b1b849873e0490496174e5ba1dca Mon Sep 17 00:00:00 2001 From: Yuedong Wu Date: Fri, 10 Apr 2026 08:21:31 +0800 Subject: [PATCH 1/2] Refactor proxy helpers to be generic and support pre-configured proxy clusters Extract MAPI-specific proxy env var verification into reusable helpers that work for any deployment. Add 'IsClusterProxyEnabled' (inspired by o/origin) to detect clusters with pre-existing proxy config, allowing proxy tests to skip MITM proxy deployment on proxy-enabled CI jobs and avoid cleanup conflict failure. --- pkg/framework/proxies.go | 132 +++++++++++++++----------- pkg/operators/machine-api-operator.go | 39 ++++++-- 2 files changed, 103 insertions(+), 68 deletions(-) diff --git a/pkg/framework/proxies.go b/pkg/framework/proxies.go index 29d4bd253..7378a21c0 100644 --- a/pkg/framework/proxies.go +++ b/pkg/framework/proxies.go @@ -45,16 +45,14 @@ import ( ) const ( - mapiControllersDeploymentName = "machine-api-controllers" - machineControllerContainerName string = "machine-controller" - proxyNamespace = MachineAPINamespace - proxyName = "mitm-proxy" - mitmSignerName = "mitm-signer" - mitmBootstrapName = "mitm-bootstrap" - mitmCustomPKIName = "mitm-custom-pki" - mitmCustomPKINamespace = "openshift-config" - mitmDaemonsetName = proxyName - mitmServiceName = proxyName + proxyNamespace = MachineAPINamespace + proxyName = "mitm-proxy" + mitmSignerName = "mitm-signer" + mitmBootstrapName = "mitm-bootstrap" + mitmCustomPKIName = "mitm-custom-pki" + mitmCustomPKINamespace = "openshift-config" + mitmDaemonsetName = proxyName + mitmServiceName = proxyName ) const proxySetup = ` @@ -65,6 +63,16 @@ tar xvf mitmproxy-5.3.0-linux.tar.gz HOME=/.mitmproxy ./mitmdump --ssl-insecure ` +// IsClusterProxyEnabled returns true if the cluster has a global proxy enabled. +func IsClusterProxyEnabled(ctx context.Context, c client.Client) (bool, error) { + proxy := &configv1.Proxy{} + if err := c.Get(ctx, client.ObjectKey{Name: "cluster"}, proxy); err != nil { + return false, fmt.Errorf("failed to get cluster proxy: %w", err) + } + + return len(proxy.Status.HTTPProxy) > 0 || len(proxy.Status.HTTPSProxy) > 0, nil +} + // DeployProxy deploys a MITM Proxy to the cluster. func DeployProxy(c client.Client, gomegaArgs ...interface{}) { ctx := context.Background() @@ -142,42 +150,11 @@ func ConfigureClusterWideProxy(c client.Client, gomegaArgs ...interface{}) { Name: mitmCustomPKIName, } }), gomegaArgs...).Should(Succeed(), "cluster wide proxy set be able to be updated") - - deploy := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: mapiControllersDeploymentName, - Namespace: proxyNamespace, - }, - } - - By("Waiting for machine-api-controller deployment to reflect configured cluster-wide proxy") - - Eventually(kom.Object(deploy), time.Minute*5).Should( - HaveField("Spec.Template.Spec.Containers", ContainElement(SatisfyAll( - HaveField("Name", Equal(machineControllerContainerName)), - HaveField("Env", SatisfyAll( - ContainElement(SatisfyAll( - HaveField("Name", "NO_PROXY"), - HaveField("Value", Not(BeEmpty())), - )), - ContainElement(SatisfyAll( - HaveField("Name", "HTTPS_PROXY"), - HaveField("Value", Not(BeEmpty())), - )), - ContainElement(SatisfyAll( - HaveField("Name", "HTTP_PROXY"), - HaveField("Value", Not(BeEmpty())), - )), - )), - ))), - "Cluster-wide proxy environment variables were not set.", - ) } // UnconfigureClusterWideProxy configures the Cluster-Wide Proxy to stop using the MITM Proxy. func UnconfigureClusterWideProxy(c client.Client, gomegaArgs ...interface{}) { ctx := context.Background() - kom := komega.New(c) proxy := &configv1.Proxy{} Eventually(c.Get(ctx, client.ObjectKey{Name: "cluster"}, proxy)).Should(Succeed(), "timed out getting Proxy named 'cluster.'") @@ -188,31 +165,70 @@ func UnconfigureClusterWideProxy(c client.Client, gomegaArgs ...interface{}) { {"op": "remove", "path": "/spec/noProxy"}, {"op": "remove", "path": "/spec/trustedCA"} ]`)))).Should(Succeed(), "timed out patching Proxy Spec.") +} + +// WaitForAllContainersProxyEnvVars waits for all containers in the given Deployment +// to have HTTP_PROXY, HTTPS_PROXY, and NO_PROXY environment variables set. +func WaitForAllContainersProxyEnvVars(c client.Client, namespace, deploymentName string) { + kom := komega.New(c) deploy := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: mapiControllersDeploymentName, - Namespace: proxyNamespace, + Name: deploymentName, + Namespace: namespace, }, } - By("Waiting for machine-api-controller deployment to reflect unconfigured cluster-wide proxy") + proxyEnvMatcher := SatisfyAll( + ContainElement(SatisfyAll( + HaveField("Name", "NO_PROXY"), + HaveField("Value", Not(BeEmpty())), + )), + ContainElement(SatisfyAll( + HaveField("Name", "HTTPS_PROXY"), + HaveField("Value", Not(BeEmpty())), + )), + ContainElement(SatisfyAll( + HaveField("Name", "HTTP_PROXY"), + HaveField("Value", Not(BeEmpty())), + )), + ) + + By(fmt.Sprintf("Waiting for all containers in %s/%s to reflect configured cluster-wide proxy", namespace, deploymentName)) + + Eventually(kom.Object(deploy), time.Minute*5).Should( + HaveField("Spec.Template.Spec.Containers", Not(ContainElement( + HaveField("Env", Not(proxyEnvMatcher)), + ))), + fmt.Sprintf("Cluster-wide proxy environment variables were not set on all containers of %s/%s.", namespace, deploymentName), + ) +} + +// WaitForAllContainersNoProxyEnvVars waits for all containers in the given Deployment +// to have HTTP_PROXY, HTTPS_PROXY, and NO_PROXY environment variables removed. +func WaitForAllContainersNoProxyEnvVars(c client.Client, namespace, deploymentName string) { + kom := komega.New(c) + + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: deploymentName, + Namespace: namespace, + }, + } + + noProxyEnvMatcher := SatisfyAll( + Not(ContainElement(HaveField("Name", "NO_PROXY"))), + Not(ContainElement(HaveField("Name", "HTTPS_PROXY"))), + Not(ContainElement(HaveField("Name", "HTTP_PROXY"))), + ) + + By(fmt.Sprintf("Waiting for all containers in %s/%s to reflect unconfigured cluster-wide proxy", namespace, deploymentName)) + Eventually(kom.Object(deploy), time.Minute*5).Should( - HaveField("Spec.Template.Spec.Containers", ContainElement(SatisfyAll( - HaveField("Name", Equal(machineControllerContainerName)), - HaveField("Env", SatisfyAll( - Not(ContainElement(SatisfyAll( - HaveField("Name", "NO_PROXY"), - ))), - Not(ContainElement(SatisfyAll( - HaveField("Name", "HTTPS_PROXY"), - ))), - Not(ContainElement(SatisfyAll( - HaveField("Name", "HTTP_PROXY"), - ))), - )), + HaveField("Spec.Template.Spec.Containers", Not(ContainElement( + HaveField("Env", Not(noProxyEnvMatcher)), ))), - "Cluster-wide proxy environmenet variables were still set.", + fmt.Sprintf("Cluster-wide proxy environment variables were still set on some containers of %s/%s.", namespace, deploymentName), ) } diff --git a/pkg/operators/machine-api-operator.go b/pkg/operators/machine-api-operator.go index 09b9fa944..f294a3fec 100644 --- a/pkg/operators/machine-api-operator.go +++ b/pkg/operators/machine-api-operator.go @@ -272,9 +272,10 @@ var _ = Describe( Serial, func() { var ( - gatherer *gatherer.StateGatherer - client runtimeclient.Client - ctx context.Context + gatherer *gatherer.StateGatherer + client runtimeclient.Client + ctx context.Context + isProxyJob bool ) BeforeEach(func() { @@ -288,11 +289,21 @@ var _ = Describe( gatherer, err = framework.NewGatherer() Expect(err).ToNot(HaveOccurred(), "Failed to load gatherer") - By("deploying an HTTP proxy") - framework.DeployProxy(client) + isProxyJob, err = framework.IsClusterProxyEnabled(ctx, client) + Expect(err).ToNot(HaveOccurred(), "Failed to check cluster proxy configuration") - By("configuring cluster-wide proxy") - framework.ConfigureClusterWideProxy(client) + if isProxyJob { + By("cluster-wide proxy is already configured on proxy cluster, skipping MITM proxy deployment") + } else { + By("deploying an HTTP proxy") + framework.DeployProxy(client) + + By("configuring cluster-wide proxy") + framework.ConfigureClusterWideProxy(client) + } + + By("waiting for machine-api-controllers to reflect proxy configuration") + framework.WaitForAllContainersProxyEnvVars(client, framework.MachineAPINamespace, maoManagedDeployment) }) // Machines required for test: 1 @@ -312,14 +323,22 @@ var _ = Describe( }) AfterEach(func() { - By("unconfiguring cluster-wide proxy") - framework.UnconfigureClusterWideProxy(client) - specReport := CurrentSpecReport() if specReport.Failed() { Expect(gatherer.WithSpecReport(specReport).GatherAll()).To(Succeed(), "Failed to GatherAll") } + if isProxyJob { + By("cluster-wide proxy was pre-existing, skipping MITM proxy cleanup") + return + } + + By("unconfiguring cluster-wide proxy") + framework.UnconfigureClusterWideProxy(client) + + By("waiting for machine-api-controllers to reflect proxy removal") + framework.WaitForAllContainersNoProxyEnvVars(client, framework.MachineAPINamespace, maoManagedDeployment) + By("waiting for MAO, KAPI and KCM cluster operators to become available") client, err := framework.LoadClient() From af21b1de5f634949a2b5c3332ceed92a35407a00 Mon Sep 17 00:00:00 2001 From: Yuedong Wu Date: Fri, 10 Apr 2026 08:22:04 +0800 Subject: [PATCH 2/2] Add E2E test for CAPI cluster-wide proxy support Add test verifying that when a cluster-wide proxy is configured, all Cluster API provider deployments have HTTP_PROXY, HTTPS_PROXY, and NO_PROXY environment variables injected into their containers. --- pkg/operators/cluster-capi-operator.go | 133 +++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 pkg/operators/cluster-capi-operator.go diff --git a/pkg/operators/cluster-capi-operator.go b/pkg/operators/cluster-capi-operator.go new file mode 100644 index 000000000..279d373c6 --- /dev/null +++ b/pkg/operators/cluster-capi-operator.go @@ -0,0 +1,133 @@ +package operators + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + configv1 "github.com/openshift/api/config/v1" + appsv1 "k8s.io/api/apps/v1" + runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/openshift/cluster-api-actuator-pkg/pkg/framework" + "github.com/openshift/cluster-api-actuator-pkg/pkg/framework/gatherer" +) + +var _ = Describe( + "[sig-cluster-lifecycle] Cluster API When cluster-wide proxy is configured, Cluster API provider deployments should", + framework.LabelDisruptive, framework.LabelConnectedOnly, framework.LabelPeriodic, framework.LabelCAPI, + Serial, + func() { + var ( + gatherer *gatherer.StateGatherer + client runtimeclient.Client + ctx context.Context + isProxyJob bool + ) + + BeforeEach(func() { + var err error + + client, err = framework.LoadClient() + Expect(err).NotTo(HaveOccurred(), "Failed to load client") + + ctx = framework.GetContext() + + gatherer, err = framework.NewGatherer() + Expect(err).ToNot(HaveOccurred(), "Failed to load gatherer") + + isProxyJob, err = framework.IsClusterProxyEnabled(ctx, client) + Expect(err).ToNot(HaveOccurred(), "Failed to check cluster proxy configuration") + + if isProxyJob { + By("cluster-wide proxy is already configured on proxy cluster, skipping MITM proxy deployment") + } else { + By("deploying an HTTP proxy") + framework.DeployProxy(client) + + By("configuring cluster-wide proxy") + framework.ConfigureClusterWideProxy(client) + } + }) + + It("have proxy environment variables injected", func() { + By("waiting for Cluster API provider deployments to reflect proxy configuration") + + deployments := &appsv1.DeploymentList{} + Eventually(client.List(ctx, deployments, + runtimeclient.InNamespace(framework.ClusterAPINamespace), + runtimeclient.HasLabels{"cluster.x-k8s.io/provider"}, + )).Should(Succeed(), "timed out listing Cluster API provider Deployments.") + Expect(deployments.Items).NotTo(BeEmpty(), "no Cluster API provider Deployments found") + + for _, deploy := range deployments.Items { + By(fmt.Sprintf("verifying proxy env vars on Deployment %s", deploy.Name)) + framework.WaitForAllContainersProxyEnvVars(client, framework.ClusterAPINamespace, deploy.Name) + } + }) + + AfterEach(func() { + specReport := CurrentSpecReport() + if specReport.Failed() { + Expect(gatherer.WithSpecReport(specReport).GatherAll()).To(Succeed(), "Failed to GatherAll") + } + + if isProxyJob { + By("cluster-wide proxy was pre-existing, skipping MITM proxy cleanup") + return + } + + By("unconfiguring cluster-wide proxy") + framework.UnconfigureClusterWideProxy(client) + + var err error + + client, err = framework.LoadClient() + Expect(err).NotTo(HaveOccurred(), "Failed to refresh client after proxy teardown") + + By("verifying proxy env vars are removed from Cluster API provider Deployments") + + deployments := &appsv1.DeploymentList{} + Eventually(client.List(ctx, deployments, + runtimeclient.InNamespace(framework.ClusterAPINamespace), + runtimeclient.HasLabels{"cluster.x-k8s.io/provider"}, + )).Should(Succeed(), "timed out listing Cluster API provider Deployments.") + Expect(deployments.Items).NotTo(BeEmpty(), "no Cluster API provider Deployments found") + + for _, deploy := range deployments.Items { + By(fmt.Sprintf("verifying proxy env vars removed from Deployment %s", deploy.Name)) + framework.WaitForAllContainersNoProxyEnvVars(client, framework.ClusterAPINamespace, deploy.Name) + } + + By("waiting for KAPI cluster operator to become available") + Expect(framework.WaitForStatusAvailableOverLong(ctx, client, "kube-apiserver")).To(BeTrue(), + "Failed to wait for kube-apiserver Cluster Operator to become available") + + By("waiting for KCM cluster operator to become available") + Expect(framework.WaitForStatusAvailableOverLong(ctx, client, "kube-controller-manager")).To(BeTrue(), + "Failed to wait for kube-controller-manager Cluster Operator to become available") + + By("waiting for cluster-api cluster operator to become available") + Expect(framework.WaitForStatusAvailableMedium(ctx, client, "cluster-api")).To(BeTrue(), + "Failed to wait for cluster-api Cluster Operator to become available") + + By("waiting for all nodes to become ready") + Expect(framework.WaitUntilAllNodesAreReady(ctx, client)).To(Succeed(), + "Failed to wait for all nodes to become ready") + + By("waiting for all cluster operators to become available") + + coList := &configv1.ClusterOperatorList{} + Eventually(client.List(ctx, coList)).Should(Succeed(), "failed to list ClusterOperators.") + + for _, co := range coList.Items { + Expect(framework.WaitForStatusAvailableOverLong(ctx, client, co.Name)).To(BeTrue(), + "Failed to wait for %s Cluster Operator to become available", co.Name) + } + + By("removing the mitm-proxy") + framework.DeleteProxy(client) + }) + })