From 67af14fc8f7d89137810d69798e92aa0d998f877 Mon Sep 17 00:00:00 2001 From: "David L. Chandler" Date: Thu, 30 Jul 2026 00:18:24 -0400 Subject: [PATCH 1/9] flake: validation and client TLS e2e Signed-off-by: David L. Chandler --- ...x-flaky-validation-and-client-tls-e2e.yaml | 10 +++++++ .../e2e/features/client_tls/suite.go | 28 ++++++++++++++++--- .../e2e/tests/validation_strict_tests.go | 18 ++++++++++-- 3 files changed, 49 insertions(+), 7 deletions(-) create mode 100644 changelog/v1.22.0-beta11/fix-flaky-validation-and-client-tls-e2e.yaml diff --git a/changelog/v1.22.0-beta11/fix-flaky-validation-and-client-tls-e2e.yaml b/changelog/v1.22.0-beta11/fix-flaky-validation-and-client-tls-e2e.yaml new file mode 100644 index 00000000000..8936e00f4cf --- /dev/null +++ b/changelog/v1.22.0-beta11/fix-flaky-validation-and-client-tls-e2e.yaml @@ -0,0 +1,10 @@ +changelog: + - type: NON_USER_FACING + resolvesIssue: false + description: >- + Fix two flaky kube e2e tests. Run the ValidationSplitWebhook suite last, since it + cannot undo its changes to the ValidatingWebhookConfiguration (a helm hook resource + that `helm rollback` does not restore) and its matchConditions would otherwise + disable secret deletion validation for later suites. Retry the client_tls + VirtualService applies, which can be transiently rejected with a domain conflict + against a VirtualService that a prior test already deleted. diff --git a/test/kubernetes/e2e/features/client_tls/suite.go b/test/kubernetes/e2e/features/client_tls/suite.go index df1b96ab481..27faa3f6ea9 100644 --- a/test/kubernetes/e2e/features/client_tls/suite.go +++ b/test/kubernetes/e2e/features/client_tls/suite.go @@ -2,6 +2,7 @@ package client_tls import ( "context" + "time" "github.com/onsi/gomega" "github.com/stretchr/testify/suite" @@ -75,8 +76,7 @@ func (s *clientTlsTestingSuite) TestRouteSecureRequestToUpstream() { err := s.testInstallation.Actions.Kubectl().Apply(s.ctx, NginxUpstreamsYaml) s.NoError(err, "can apply upstream manifest file") - err = s.testInstallation.Actions.Kubectl().Apply(s.ctx, VSTargetingUpstreamYaml, "-n", ns) - s.NoError(err, "can apply vs targeting upstream manifest file") + s.eventuallyApply(VSTargetingUpstreamYaml, "can apply vs targeting upstream manifest file", "-n", ns) s.assertEventualResponseForPath("nginx", expectedHealthyResponse) @@ -106,8 +106,7 @@ func (s *clientTlsTestingSuite) TestRouteSecureRequestToAnnotatedService() { err := s.testInstallation.Actions.Kubectl().Apply(s.ctx, NginxAnnotatedServicesYaml) s.NoError(err, "can apply services manifest file") - err = s.testInstallation.Actions.Kubectl().Apply(s.ctx, VSTargetingKubeYaml, "-n", ns) - s.NoError(err, "can apply vs targeting services manifest file") + s.eventuallyApply(VSTargetingKubeYaml, "can apply vs targeting services manifest file", "-n", ns) s.assertEventualResponseForPath("nginx", expectedHealthyResponse) @@ -188,6 +187,27 @@ func (s *clientTlsTestingSuite) TestOneWayTlsDoesNotRequestClientCertificate() { }).WithContext(s.ctx).Should(gomega.Succeed()) } +// eventuallyApply applies a manifest, retrying for as long as the Edge validating admission +// webhook rejects it. +// +// TestRouteSecureRequestToUpstream and TestRouteSecureRequestToAnnotatedService each create a +// VirtualService that claims the nginx.example.com domain, and each deletes its VirtualService +// during cleanup. Deleting a VirtualService removes it from the validator's in-memory snapshot, +// but a snapshot that was already in flight when the delete happened can transiently re-add it +// (see Sync in projects/gateway/pkg/validation/validator.go, which replaces latestSnapshot +// wholesale). Until that settles, applying the other VirtualService is rejected with a domain +// conflict against a VirtualService that no longer exists in the cluster. +func (s *clientTlsTestingSuite) eventuallyApply(manifest []byte, description string, extraArgs ...string) { + s.testInstallation.AssertionsT(s.T()).Gomega.Eventually(func(g gomega.Gomega) { + err := s.testInstallation.Actions.Kubectl().Apply(s.ctx, manifest, extraArgs...) + g.Expect(err).NotTo(gomega.HaveOccurred(), description) + }). + WithContext(s.ctx). + WithTimeout(time.Minute). + WithPolling(time.Second). + Should(gomega.Succeed(), description) +} + func (s *clientTlsTestingSuite) assertEventualResponseForPath(path string, matcher *matchers.HttpResponse) { s.testInstallation.AssertionsT(s.T()).AssertEventualCurlResponse( s.ctx, diff --git a/test/kubernetes/e2e/tests/validation_strict_tests.go b/test/kubernetes/e2e/tests/validation_strict_tests.go index 63bcb005674..242f07c6df8 100644 --- a/test/kubernetes/e2e/tests/validation_strict_tests.go +++ b/test/kubernetes/e2e/tests/validation_strict_tests.go @@ -9,8 +9,17 @@ import ( // ValidationStrictSuiteRunnerAll is used to run all the validation tests, including ones that depend on the helm chart/values/helpers // This is the function that should be used to run the validation tests in this repo +// +// These suites are ordered so that "ValidationSplitWebhook" always runs last. That suite mutates +// the ValidatingWebhookConfiguration (failurePolicy, matchConditions) via `helm upgrade` and then +// reverts it with `helm rollback`. However, the ValidatingWebhookConfiguration is a helm hook +// resource ("helm.sh/hook": pre-install, pre-upgrade), so it is not part of the release manifest +// and `helm rollback` does not restore it. The values it installs - notably the +// `kubeCoreMatchConditions` that skip validation of secrets - therefore outlive the suite and +// silently disable validation for any suite that runs after it. func ValidationStrictSuiteRunnerAll() e2e.SuiteRunner { - validationSuiteRunner := ValidationStrictSuiteRunner() + validationSuiteRunner := e2e.NewSuiteRunner(true) + registerValidationStrictSuites(validationSuiteRunner) validationSuiteRunner.Register("ValidationSplitWebhook", split_webhook.NewTestingSuite) return validationSuiteRunner @@ -23,9 +32,12 @@ func ValidationStrictSuiteRunnerAll() e2e.SuiteRunner { // If more tests are added that depend on the helm chart/values/helpers, the above issue should be resolved instead of using this approach func ValidationStrictSuiteRunner() e2e.SuiteRunner { validationSuiteRunner := e2e.NewSuiteRunner(false) + registerValidationStrictSuites(validationSuiteRunner) + return validationSuiteRunner +} + +func registerValidationStrictSuites(validationSuiteRunner e2e.SuiteRunner) { validationSuiteRunner.Register("ValidationStrictWarnings", validation_strict_warnings.NewTestingSuite) validationSuiteRunner.Register("ValidationRejectInvalid", validation_reject_invalid.NewTestingSuite) - - return validationSuiteRunner } From 160958c9fbc73ad49bcd32dd892b94e75474cad2 Mon Sep 17 00:00:00 2001 From: changelog-bot Date: Fri, 31 Jul 2026 18:02:45 +0000 Subject: [PATCH 2/9] Adding changelog file to new location --- .../fix-flaky-validation-and-client-tls-e2e.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 changelog/v1.22.0-beta12/fix-flaky-validation-and-client-tls-e2e.yaml diff --git a/changelog/v1.22.0-beta12/fix-flaky-validation-and-client-tls-e2e.yaml b/changelog/v1.22.0-beta12/fix-flaky-validation-and-client-tls-e2e.yaml new file mode 100644 index 00000000000..8936e00f4cf --- /dev/null +++ b/changelog/v1.22.0-beta12/fix-flaky-validation-and-client-tls-e2e.yaml @@ -0,0 +1,10 @@ +changelog: + - type: NON_USER_FACING + resolvesIssue: false + description: >- + Fix two flaky kube e2e tests. Run the ValidationSplitWebhook suite last, since it + cannot undo its changes to the ValidatingWebhookConfiguration (a helm hook resource + that `helm rollback` does not restore) and its matchConditions would otherwise + disable secret deletion validation for later suites. Retry the client_tls + VirtualService applies, which can be transiently rejected with a domain conflict + against a VirtualService that a prior test already deleted. From d4be5dc57869973f7afeeb099d51b340684cd449 Mon Sep 17 00:00:00 2001 From: changelog-bot Date: Fri, 31 Jul 2026 18:02:45 +0000 Subject: [PATCH 3/9] Deleting changelog file from old location --- .../fix-flaky-validation-and-client-tls-e2e.yaml | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 changelog/v1.22.0-beta11/fix-flaky-validation-and-client-tls-e2e.yaml diff --git a/changelog/v1.22.0-beta11/fix-flaky-validation-and-client-tls-e2e.yaml b/changelog/v1.22.0-beta11/fix-flaky-validation-and-client-tls-e2e.yaml deleted file mode 100644 index 8936e00f4cf..00000000000 --- a/changelog/v1.22.0-beta11/fix-flaky-validation-and-client-tls-e2e.yaml +++ /dev/null @@ -1,10 +0,0 @@ -changelog: - - type: NON_USER_FACING - resolvesIssue: false - description: >- - Fix two flaky kube e2e tests. Run the ValidationSplitWebhook suite last, since it - cannot undo its changes to the ValidatingWebhookConfiguration (a helm hook resource - that `helm rollback` does not restore) and its matchConditions would otherwise - disable secret deletion validation for later suites. Retry the client_tls - VirtualService applies, which can be transiently rejected with a domain conflict - against a VirtualService that a prior test already deleted. From f56f145458e72f26032e035c66f9800f8bc98992 Mon Sep 17 00:00:00 2001 From: changelog-bot Date: Sun, 2 Aug 2026 23:11:40 +0000 Subject: [PATCH 4/9] Adding changelog file to new location --- .../fix-flaky-validation-and-client-tls-e2e.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 changelog/v1.22.0-beta13/fix-flaky-validation-and-client-tls-e2e.yaml diff --git a/changelog/v1.22.0-beta13/fix-flaky-validation-and-client-tls-e2e.yaml b/changelog/v1.22.0-beta13/fix-flaky-validation-and-client-tls-e2e.yaml new file mode 100644 index 00000000000..8936e00f4cf --- /dev/null +++ b/changelog/v1.22.0-beta13/fix-flaky-validation-and-client-tls-e2e.yaml @@ -0,0 +1,10 @@ +changelog: + - type: NON_USER_FACING + resolvesIssue: false + description: >- + Fix two flaky kube e2e tests. Run the ValidationSplitWebhook suite last, since it + cannot undo its changes to the ValidatingWebhookConfiguration (a helm hook resource + that `helm rollback` does not restore) and its matchConditions would otherwise + disable secret deletion validation for later suites. Retry the client_tls + VirtualService applies, which can be transiently rejected with a domain conflict + against a VirtualService that a prior test already deleted. From 0938a4348bef8d47cb820fe83765aacdca1e1ba9 Mon Sep 17 00:00:00 2001 From: changelog-bot Date: Sun, 2 Aug 2026 23:11:40 +0000 Subject: [PATCH 5/9] Deleting changelog file from old location --- .../fix-flaky-validation-and-client-tls-e2e.yaml | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 changelog/v1.22.0-beta12/fix-flaky-validation-and-client-tls-e2e.yaml diff --git a/changelog/v1.22.0-beta12/fix-flaky-validation-and-client-tls-e2e.yaml b/changelog/v1.22.0-beta12/fix-flaky-validation-and-client-tls-e2e.yaml deleted file mode 100644 index 8936e00f4cf..00000000000 --- a/changelog/v1.22.0-beta12/fix-flaky-validation-and-client-tls-e2e.yaml +++ /dev/null @@ -1,10 +0,0 @@ -changelog: - - type: NON_USER_FACING - resolvesIssue: false - description: >- - Fix two flaky kube e2e tests. Run the ValidationSplitWebhook suite last, since it - cannot undo its changes to the ValidatingWebhookConfiguration (a helm hook resource - that `helm rollback` does not restore) and its matchConditions would otherwise - disable secret deletion validation for later suites. Retry the client_tls - VirtualService applies, which can be transiently rejected with a domain conflict - against a VirtualService that a prior test already deleted. From db891db99b993b44d4263d5c9f005d2b7e71e0c1 Mon Sep 17 00:00:00 2001 From: changelog-bot Date: Tue, 4 Aug 2026 01:00:16 +0000 Subject: [PATCH 6/9] Adding changelog file to new location --- .../fix-flaky-validation-and-client-tls-e2e.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 changelog/v1.22.1/fix-flaky-validation-and-client-tls-e2e.yaml diff --git a/changelog/v1.22.1/fix-flaky-validation-and-client-tls-e2e.yaml b/changelog/v1.22.1/fix-flaky-validation-and-client-tls-e2e.yaml new file mode 100644 index 00000000000..8936e00f4cf --- /dev/null +++ b/changelog/v1.22.1/fix-flaky-validation-and-client-tls-e2e.yaml @@ -0,0 +1,10 @@ +changelog: + - type: NON_USER_FACING + resolvesIssue: false + description: >- + Fix two flaky kube e2e tests. Run the ValidationSplitWebhook suite last, since it + cannot undo its changes to the ValidatingWebhookConfiguration (a helm hook resource + that `helm rollback` does not restore) and its matchConditions would otherwise + disable secret deletion validation for later suites. Retry the client_tls + VirtualService applies, which can be transiently rejected with a domain conflict + against a VirtualService that a prior test already deleted. From df8fef68eb048bedd2094ee24337e3504e8a1295 Mon Sep 17 00:00:00 2001 From: changelog-bot Date: Tue, 4 Aug 2026 01:00:17 +0000 Subject: [PATCH 7/9] Deleting changelog file from old location --- .../fix-flaky-validation-and-client-tls-e2e.yaml | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 changelog/v1.22.0-beta13/fix-flaky-validation-and-client-tls-e2e.yaml diff --git a/changelog/v1.22.0-beta13/fix-flaky-validation-and-client-tls-e2e.yaml b/changelog/v1.22.0-beta13/fix-flaky-validation-and-client-tls-e2e.yaml deleted file mode 100644 index 8936e00f4cf..00000000000 --- a/changelog/v1.22.0-beta13/fix-flaky-validation-and-client-tls-e2e.yaml +++ /dev/null @@ -1,10 +0,0 @@ -changelog: - - type: NON_USER_FACING - resolvesIssue: false - description: >- - Fix two flaky kube e2e tests. Run the ValidationSplitWebhook suite last, since it - cannot undo its changes to the ValidatingWebhookConfiguration (a helm hook resource - that `helm rollback` does not restore) and its matchConditions would otherwise - disable secret deletion validation for later suites. Retry the client_tls - VirtualService applies, which can be transiently rejected with a domain conflict - against a VirtualService that a prior test already deleted. From 79c3d3df8843a57a96133fbae1addc5b0495939e Mon Sep 17 00:00:00 2001 From: changelog-bot Date: Tue, 4 Aug 2026 22:04:08 +0000 Subject: [PATCH 8/9] Adding changelog file to new location --- .../fix-flaky-validation-and-client-tls-e2e.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 changelog/v1.22.2/fix-flaky-validation-and-client-tls-e2e.yaml diff --git a/changelog/v1.22.2/fix-flaky-validation-and-client-tls-e2e.yaml b/changelog/v1.22.2/fix-flaky-validation-and-client-tls-e2e.yaml new file mode 100644 index 00000000000..8936e00f4cf --- /dev/null +++ b/changelog/v1.22.2/fix-flaky-validation-and-client-tls-e2e.yaml @@ -0,0 +1,10 @@ +changelog: + - type: NON_USER_FACING + resolvesIssue: false + description: >- + Fix two flaky kube e2e tests. Run the ValidationSplitWebhook suite last, since it + cannot undo its changes to the ValidatingWebhookConfiguration (a helm hook resource + that `helm rollback` does not restore) and its matchConditions would otherwise + disable secret deletion validation for later suites. Retry the client_tls + VirtualService applies, which can be transiently rejected with a domain conflict + against a VirtualService that a prior test already deleted. From 1576bb6669044d243386f9825fd8daee9144e92d Mon Sep 17 00:00:00 2001 From: changelog-bot Date: Tue, 4 Aug 2026 22:04:09 +0000 Subject: [PATCH 9/9] Deleting changelog file from old location --- .../fix-flaky-validation-and-client-tls-e2e.yaml | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 changelog/v1.22.1/fix-flaky-validation-and-client-tls-e2e.yaml diff --git a/changelog/v1.22.1/fix-flaky-validation-and-client-tls-e2e.yaml b/changelog/v1.22.1/fix-flaky-validation-and-client-tls-e2e.yaml deleted file mode 100644 index 8936e00f4cf..00000000000 --- a/changelog/v1.22.1/fix-flaky-validation-and-client-tls-e2e.yaml +++ /dev/null @@ -1,10 +0,0 @@ -changelog: - - type: NON_USER_FACING - resolvesIssue: false - description: >- - Fix two flaky kube e2e tests. Run the ValidationSplitWebhook suite last, since it - cannot undo its changes to the ValidatingWebhookConfiguration (a helm hook resource - that `helm rollback` does not restore) and its matchConditions would otherwise - disable secret deletion validation for later suites. Retry the client_tls - VirtualService applies, which can be transiently rejected with a domain conflict - against a VirtualService that a prior test already deleted.