OCPCLOUD-3466: Add E2E test for CAPI controllers cluster-wide proxy support #473
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,133 @@ | ||
| package operators | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
|
|
||
| . "github.com/onsi/ginkgo/v2" | ||
| . "github.com/onsi/gomega" | ||
|
|
||
| configv1 "github.com/openshift/api/config/v1" | ||
| appsv1 "k8s.io/api/apps/v1" | ||
| runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" | ||
|
|
||
| "github.com/openshift/cluster-api-actuator-pkg/pkg/framework" | ||
| "github.com/openshift/cluster-api-actuator-pkg/pkg/framework/gatherer" | ||
| ) | ||
|
|
||
| var _ = Describe( | ||
| "[sig-cluster-lifecycle] Cluster API When cluster-wide proxy is configured, Cluster API provider deployments should", | ||
| framework.LabelDisruptive, framework.LabelConnectedOnly, framework.LabelPeriodic, framework.LabelCAPI, | ||
| Serial, | ||
| func() { | ||
| var ( | ||
| gatherer *gatherer.StateGatherer | ||
| client runtimeclient.Client | ||
| ctx context.Context | ||
| isProxyJob bool | ||
| ) | ||
|
|
||
| BeforeEach(func() { | ||
| var err error | ||
|
|
||
| client, err = framework.LoadClient() | ||
| Expect(err).NotTo(HaveOccurred(), "Failed to load client") | ||
|
Contributor
There was a problem hiding this comment. Skip if the CAPIMachineManagement (or whatever) feature gate is not set?
Author
There was a problem hiding this comment. Assuming the goal here is to skip test execution on non-applicable clusters (CAPI is not enabled): I checked how existing tests handle similar situations. They simply call I can add the same here. As for a more granular check (by feature set name, like |
||
|
|
||
| ctx = framework.GetContext() | ||
|
|
||
| gatherer, err = framework.NewGatherer() | ||
| Expect(err).ToNot(HaveOccurred(), "Failed to load gatherer") | ||
|
|
||
| isProxyJob, err = framework.IsClusterProxyEnabled(ctx, client) | ||
| Expect(err).ToNot(HaveOccurred(), "Failed to check cluster proxy configuration") | ||
|
|
||
| if isProxyJob { | ||
| By("cluster-wide proxy is already configured on proxy cluster, skipping MITM proxy deployment") | ||
| } else { | ||
| By("deploying an HTTP proxy") | ||
| framework.DeployProxy(client) | ||
|
|
||
| By("configuring cluster-wide proxy") | ||
| framework.ConfigureClusterWideProxy(client) | ||
| } | ||
| }) | ||
|
|
||
| It("have proxy environment variables injected", func() { | ||
| By("waiting for Cluster API provider deployments to reflect proxy configuration") | ||
|
|
||
| deployments := &appsv1.DeploymentList{} | ||
| Eventually(client.List(ctx, deployments, | ||
| runtimeclient.InNamespace(framework.ClusterAPINamespace), | ||
| runtimeclient.HasLabels{"cluster.x-k8s.io/provider"}, | ||
|
Contributor
There was a problem hiding this comment. I don't think we should continue to rely on these labels. Maybe the installer revision label? That would unambiguously identify any CAPI operand.
Author
There was a problem hiding this comment. Just to confirm my understanding: are you suggesting using newly added |
||
| )).Should(Succeed(), "timed out listing Cluster API provider Deployments.") | ||
| Expect(deployments.Items).NotTo(BeEmpty(), "no Cluster API provider Deployments found") | ||
|
Contributor
There was a problem hiding this comment. For extra robustness I'd add a wait, probably in BeforeEach, that generation == observedRevisionGeneration and desiredRevision == currentRevision on the ClusterAPI object. |
||
|
|
||
| for _, deploy := range deployments.Items { | ||
| By(fmt.Sprintf("verifying proxy env vars on Deployment %s", deploy.Name)) | ||
| framework.WaitForAllContainersProxyEnvVars(client, framework.ClusterAPINamespace, deploy.Name) | ||
| } | ||
| }) | ||
|
|
||
| AfterEach(func() { | ||
| specReport := CurrentSpecReport() | ||
| if specReport.Failed() { | ||
| Expect(gatherer.WithSpecReport(specReport).GatherAll()).To(Succeed(), "Failed to GatherAll") | ||
| } | ||
|
|
||
| if isProxyJob { | ||
| By("cluster-wide proxy was pre-existing, skipping MITM proxy cleanup") | ||
| return | ||
| } | ||
|
|
||
| By("unconfiguring cluster-wide proxy") | ||
| framework.UnconfigureClusterWideProxy(client) | ||
|
|
||
| var err error | ||
|
|
||
| client, err = framework.LoadClient() | ||
| Expect(err).NotTo(HaveOccurred(), "Failed to refresh client after proxy teardown") | ||
|
|
||
| By("verifying proxy env vars are removed from Cluster API provider Deployments") | ||
|
|
||
| deployments := &appsv1.DeploymentList{} | ||
| Eventually(client.List(ctx, deployments, | ||
| runtimeclient.InNamespace(framework.ClusterAPINamespace), | ||
| runtimeclient.HasLabels{"cluster.x-k8s.io/provider"}, | ||
| )).Should(Succeed(), "timed out listing Cluster API provider Deployments.") | ||
| Expect(deployments.Items).NotTo(BeEmpty(), "no Cluster API provider Deployments found") | ||
|
|
||
| for _, deploy := range deployments.Items { | ||
| By(fmt.Sprintf("verifying proxy env vars removed from Deployment %s", deploy.Name)) | ||
| framework.WaitForAllContainersNoProxyEnvVars(client, framework.ClusterAPINamespace, deploy.Name) | ||
| } | ||
|
|
||
| By("waiting for KAPI cluster operator to become available") | ||
| Expect(framework.WaitForStatusAvailableOverLong(ctx, client, "kube-apiserver")).To(BeTrue(), | ||
| "Failed to wait for kube-apiserver Cluster Operator to become available") | ||
|
|
||
| By("waiting for KCM cluster operator to become available") | ||
| Expect(framework.WaitForStatusAvailableOverLong(ctx, client, "kube-controller-manager")).To(BeTrue(), | ||
| "Failed to wait for kube-controller-manager Cluster Operator to become available") | ||
|
|
||
| By("waiting for cluster-api cluster operator to become available") | ||
| Expect(framework.WaitForStatusAvailableMedium(ctx, client, "cluster-api")).To(BeTrue(), | ||
| "Failed to wait for cluster-api Cluster Operator to become available") | ||
|
|
||
| By("waiting for all nodes to become ready") | ||
| Expect(framework.WaitUntilAllNodesAreReady(ctx, client)).To(Succeed(), | ||
| "Failed to wait for all nodes to become ready") | ||
|
|
||
| By("waiting for all cluster operators to become available") | ||
|
|
||
| coList := &configv1.ClusterOperatorList{} | ||
| Eventually(client.List(ctx, coList)).Should(Succeed(), "failed to list ClusterOperators.") | ||
|
|
||
| for _, co := range coList.Items { | ||
| Expect(framework.WaitForStatusAvailableOverLong(ctx, client, co.Name)).To(BeTrue(), | ||
| "Failed to wait for %s Cluster Operator to become available", co.Name) | ||
| } | ||
|
|
||
| By("removing the mitm-proxy") | ||
| framework.DeleteProxy(client) | ||
| }) | ||
| }) | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I don't know how to review this label, but I suspect it's important. @damdo ?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I referenced MAPI's:
cluster-api-actuator-pkg/pkg/operators/machine-api-operator.go
Lines 270 to 271 in 7528e67
Custom labels being defined: https://github.com/openshift/cluster-api-actuator-pkg/blob/master/pkg/framework/ginkgo-labels.go