diff --git a/Makefile b/Makefile index c0f29671..3f56edc0 100644 --- a/Makefile +++ b/Makefile @@ -212,8 +212,8 @@ test-unit: vet ## Run unit tests. # E2E_TIMEOUT is the timeout for e2e tests. E2E_TIMEOUT ?= 1h -# E2E_GINKGO_LABEL_FILTER is ginkgo label query for selecting tests. See -# https://onsi.github.io/ginkgo/#spec-labels. Default runs Platform:AWS and Platform:Generic tests; excludes Feature:Proxy, Feature:Upgrade, and Provider:Bitwarden. +# E2E_GINKGO_LABEL_FILTER is ginkgo label query for selecting tests. See https://onsi.github.io/ginkgo/#spec-labels. +# Default runs Platform:AWS and Platform:Generic tests; excludes Feature:Proxy, Feature:Upgrade, and Provider:Bitwarden. E2E_GINKGO_LABEL_FILTER ?= Platform: isSubsetOf {AWS,Generic} && !(Feature: containsAny {Proxy, Upgrade}) && !(Provider: containsAny Bitwarden) .PHONY: test-e2e test-e2e: ## Run e2e tests against a cluster. diff --git a/test/e2e/README.md b/test/e2e/README.md index 0524987f..2f6b498f 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -19,7 +19,7 @@ make test-e2e E2E_GINKGO_LABEL_FILTER="" | Key | Values | Meaning | |-----|--------|---------| | `Platform` | `AWS`, `GCP`, `Generic` | Cluster or portability requirement | -| `Provider` | `AWS`, `Bitwarden` | External secret backend integration | +| `Provider` | `AWS`, `Bitwarden`, `Vault` | External secret backend integration | | `API` | `Bitwarden` | Direct HTTP tests against bitwarden-sdk-server | | `Feature` | see below | Optional capability or functional area | @@ -34,7 +34,8 @@ make test-e2e E2E_GINKGO_LABEL_FILTER="" | `CustomLabels` | Custom and managed label lifecycle | | `NetworkPolicy` | Static and custom network policy naming | | `Proxy` | Proxy egress network policy (requires cluster-wide OpenShift proxy) | -| `TrustedCABundle` | trustedCABundle ConfigMap mounting and validation | +| `TrustedCABundle` | trustedCABundle ConfigMap mounting/validation and Vault TLS failure→recovery | +| `ExternalSecretsTemplating` | ExternalSecret template merge (Kubernetes + Vault → dockerconfigjson) | | `Upgrade` | Post-upgrade migration checks (temporary) | ## Default filter @@ -51,10 +52,6 @@ This runs portable tests plus AWS provider tests, and **API:Bitwarden** health/a ### Secrets you provision -```bash -hack/e2e-setup-secrets.sh setup -``` - | Secret | Namespace | Keys | Required when filter includes | |--------|-----------|------|-------------------------------| | `aws-creds` | `kube-system` | `aws_access_key_id`, `aws_secret_access_key` | `Provider:AWS` (`Platform:AWS` or `Platform:GCP`) | @@ -73,6 +70,7 @@ The bitwarden-sdk-server plugin uses **`bitwarden-tls-certs`** (TLS materials fo | Requirement | Required when filter includes | |-------------|-------------------------------| | Cluster-wide OpenShift proxy (`proxy.config.openshift.io/cluster`) | `Feature:Proxy` | +| OpenShift with `redhat-operators` catalog (`openshift-marketplace`) | Main e2e Describe (`e2e_test.go`) — root `BeforeAll` installs Red Hat cert-manager Operator via OLM if not already present | If a prerequisite is missing, the affected spec **fails** with a message pointing here — it does not skip. @@ -86,9 +84,11 @@ If a prerequisite is missing, the affected spec **fails** with a message pointin | `Platform:GCP && Provider:AWS` | GCP cluster using AWS Secrets Manager | | `Provider:AWS` | Any AWS Secrets Manager integration (`Platform:AWS` or `Platform:GCP`) | | `Provider:Bitwarden` | Bitwarden provider sync and API Secrets API | +| `Provider:Vault` | Vault HTTPS + trustedCABundle failure/recovery (uses suite-installed Red Hat cert-manager Operator) | | `API:Bitwarden` | bitwarden-sdk-server HTTP API (deploys plugin + `bitwarden-tls-certs` automatically) | | `API:Bitwarden \|\| Provider:Bitwarden` | All Bitwarden HTTP and provider tests (requires `bitwarden-creds` for Secrets API / provider sync) | -| `Feature:TrustedCABundle` | Trusted CA bundle suite | +| `Feature:TrustedCABundle` | Trusted CA bundle suite (mount/validation + Vault TLS path when combined with `Provider:Vault`) | +| `Feature:ExternalSecretsTemplating` | ExternalSecret templating merge (Kubernetes + Vault dockerconfig) | | `Feature:Proxy` | Proxy egress network policy | | `Feature:Upgrade` | Post-upgrade network policy migration check | | `Feature:NetworkPolicy` | Static and custom network policy naming | @@ -110,6 +110,9 @@ make test-e2e E2E_GINKGO_LABEL_FILTER="Feature:NetworkPolicy || Feature:Proxy" # AWS integration only (any platform label that uses AWS SM) make test-e2e E2E_GINKGO_LABEL_FILTER="Provider:AWS" + +# Vault TLS + trustedCABundle (uses cert-manager from root BeforeAll) +make test-e2e E2E_GINKGO_LABEL_FILTER="Provider:Vault" ``` ## Specs by label @@ -147,10 +150,29 @@ File: `trusted_ca_bundle_test.go` | Feature | Describe | |---------|----------| -| `TrustedCABundle` | Trusted CA Bundle | +| `TrustedCABundle` | Trusted CA Bundle (mount / `SSL_CERT_DIR` / Degraded / watch-label restore) | The **Custom Network Policy Naming** spec adds a dummy egress port to `ExternalSecretsConfig` (if not already present — entries cannot be removed due to CEL immutability), verifies the operator creates `eso-user-e2e-test-custom-np` in the operand namespace (`external-secrets`), and leaves the CR entry in place. +### `Provider:Vault` + `Feature:TrustedCABundle` + +File: `e2e_test.go` — **Vault Secret Manager** + +- **BeforeAll:** cert-manager CA (`isCA`) + server Certificate for Vault HTTPS; Vault deploy/init +- Uses Red Hat cert-manager Operator installed by the root `BeforeAll` of the main e2e Describe (`ensureCertManagerOperatorReady`) +- Asserts SecretStore `Ready=False` / `InvalidProviderConfig` when `trustedCABundle` points at a valid but non-matching CA (`vault-e2e-sample-ca`), then Ready after switching to the Vault CA ConfigMap (`vault-server-ca` from the isCA Certificate) +- After trustedCABundle switch: SecretStore Ready → PushSecret → ExternalSecret → verify synced Secret +- Excluded from the default label filter (longer Vault HTTPS setup) +- Does not uninstall cert-manager on teardown (shared for other suites) + +### `Provider:Vault` + `Feature:ExternalSecretsTemplating` + +File: `e2e_test.go` — **Vault Secret Manager** (same Context) + +- Creates a namespaced Kubernetes SecretStore (`kubernetes-backend`) with dedicated `eso-secret-reader` SA/Role/RoleBinding (no existing e2e Kubernetes-provider fixture) +- Pushes a dockerconfigjson Secret to Vault, reads another from Kubernetes, merges both via ExternalSecret `target.template` into `merged-registry-pull-secret` +- Asserts the target Secret is `kubernetes.io/dockerconfigjson` and contains both registry auth hosts + ### `Provider:Bitwarden` File: `bitwarden_es_test.go` — **Bitwarden Provider** diff --git a/test/e2e/cert_manager_helpers_test.go b/test/e2e/cert_manager_helpers_test.go new file mode 100644 index 00000000..ffb01871 --- /dev/null +++ b/test/e2e/cert_manager_helpers_test.go @@ -0,0 +1,184 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2026. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + + . "github.com/onsi/ginkgo/v2" + + "github.com/openshift/external-secrets-operator/test/utils" +) + +const ( + certManagerOperatorNamespace = "cert-manager-operator" + certManagerOperandNamespace = "cert-manager" + certManagerOperatorManifest = "testdata/cert-manager/operator.yaml" + + certManagerOperatorPodPrefix = "cert-manager-operator-controller-manager-" + certManagerCAInjectorPodPrefix = "cert-manager-cainjector-" + certManagerWebhookPodPrefix = "cert-manager-webhook-" + + certManagerInstallTimeout = 10 * time.Minute +) + +var certificateGVR = schema.GroupVersionResource{ + Group: "cert-manager.io", + Version: "v1", + Resource: "certificates", +} + +// ensureCertManagerOperatorReady installs the Red Hat cert-manager Operator via OLM if needed, +// then waits until the operator, operand pods, and Certificate API are ready. +// Idempotent: skips install when the Certificate API is already usable and operands are Ready. +func ensureCertManagerOperatorReady(ctx context.Context, clientset *kubernetes.Clientset, dynamicClient dynamic.Interface) error { + By("Ensuring Red Hat cert-manager Operator is ready") + + if err := certManagerCertificateCRDInstalled(ctx, dynamicClient); err == nil { + if err := waitForCertManagerOperandPods(ctx, clientset, 2*time.Minute); err == nil { + By("cert-manager Certificate API and operand pods already ready") + return nil + } + } + + By("Applying cert-manager Operator OLM manifests (Namespace, OperatorGroup, Subscription)") + if err := utils.ApplyManifestFromReader(ctx, dynamicClient, testassets.ReadFile, certManagerOperatorManifest); err != nil { + return fmt.Errorf("apply cert-manager operator manifests: %w", err) + } + + By(fmt.Sprintf("Waiting for cert-manager operator pod in namespace %s", certManagerOperatorNamespace)) + if err := waitForReadyPodsByNamePrefixes(ctx, clientset, certManagerOperatorNamespace, []string{certManagerOperatorPodPrefix}, certManagerInstallTimeout); err != nil { + return fmt.Errorf("wait for cert-manager operator pod: %w", err) + } + + By(fmt.Sprintf("Waiting for cert-manager operand pods in namespace %s", certManagerOperandNamespace)) + if err := waitForCertManagerOperandPods(ctx, clientset, certManagerInstallTimeout); err != nil { + return fmt.Errorf("wait for cert-manager operand pods: %w", err) + } + + By("Waiting for cert-manager Certificate API") + if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { + if err := certManagerCertificateCRDInstalled(ctx, dynamicClient); err != nil { + return false, nil + } + return true, nil + }); err != nil { + return fmt.Errorf("cert-manager Certificate API not available after install: %w", err) + } + + By("Red Hat cert-manager Operator is ready") + return nil +} + +// certManagerCertificateCRDInstalled returns nil when the cert-manager Certificate API is available. +func certManagerCertificateCRDInstalled(ctx context.Context, dynamicClient dynamic.Interface) error { + _, err := dynamicClient.Resource(certificateGVR).Namespace("default").List(ctx, metav1.ListOptions{Limit: 1}) + if err != nil { + return fmt.Errorf("cert-manager Certificate API unavailable: %w", err) + } + return nil +} + +// waitForCertManagerOperandPods waits until controller, cainjector, and webhook pods are Ready. +func waitForCertManagerOperandPods(ctx context.Context, clientset kubernetes.Interface, timeout time.Duration) error { + return wait.PollUntilContextTimeout(ctx, 5*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + pods, err := clientset.CoreV1().Pods(certManagerOperandNamespace).List(ctx, metav1.ListOptions{}) + if err != nil { + if strings.Contains(err.Error(), "not found") { + return false, nil + } + return false, err + } + + var hasController, hasCAInjector, hasWebhook bool + for i := range pods.Items { + pod := &pods.Items[i] + if pod.Status.Phase != corev1.PodRunning || !isPodReadyForCertManager(pod) { + continue + } + name := pod.Name + switch { + case strings.HasPrefix(name, certManagerCAInjectorPodPrefix): + hasCAInjector = true + case strings.HasPrefix(name, certManagerWebhookPodPrefix): + hasWebhook = true + case strings.HasPrefix(name, "cert-manager-") && + !strings.HasPrefix(name, certManagerCAInjectorPodPrefix) && + !strings.HasPrefix(name, certManagerWebhookPodPrefix) && + !strings.Contains(name, "startupapicheck"): + // Main controller Deployment pods: cert-manager-- + hasController = true + } + } + return hasController && hasCAInjector && hasWebhook, nil + }) +} + +func waitForReadyPodsByNamePrefixes(ctx context.Context, clientset kubernetes.Interface, namespace string, prefixes []string, timeout time.Duration) error { + return wait.PollUntilContextTimeout(ctx, 5*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + pods, err := clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return false, err + } + matched := make(map[string]bool, len(prefixes)) + for _, prefix := range prefixes { + matched[prefix] = false + } + for i := range pods.Items { + pod := &pods.Items[i] + if pod.Status.Phase != corev1.PodRunning || !isPodReadyForCertManager(pod) { + continue + } + for _, prefix := range prefixes { + if strings.HasPrefix(pod.Name, prefix) { + matched[prefix] = true + } + } + } + for _, ok := range matched { + if !ok { + return false, nil + } + } + return true, nil + }) +} + +func isPodReadyForCertManager(pod *corev1.Pod) bool { + ready := map[string]bool{ + "Ready": false, + "ContainersReady": false, + } + for _, cond := range pod.Status.Conditions { + if _, ok := ready[string(cond.Type)]; ok && cond.Status == corev1.ConditionTrue { + ready[string(cond.Type)] = true + } + } + return ready["Ready"] && ready["ContainersReady"] +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 6d432d34..25729549 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -23,6 +23,7 @@ import ( "context" "embed" "encoding/base64" + "encoding/json" "fmt" "maps" "net/url" @@ -79,11 +80,20 @@ const ( testNamespacePrefix = "external-secrets-e2e-test-" vaultNamespace = "vault-test" vaultManifestFile = "testdata/vault/vault.yaml" + vaultIssuerFile = "testdata/vault/issuer.yaml" + vaultCACertificateFile = "testdata/vault/ca_certificate.yaml" + vaultCAIssuerFile = "testdata/vault/ca_issuer.yaml" + vaultCertificateFile = "testdata/vault/certificate.yaml" vaultServiceName = "vault" - vaultAddr = "http://vault.vault-test.svc.cluster.local:8200" - targetSecretName = "k8s-secret-to-create" //must match with external_secret.yaml target.name - vaultEgressNetworkPolicyName = "allow-vault-egress" // logical name stored in ExternalSecretsConfig spec + vaultAddr = "https://vault.vault-test.svc.cluster.local:8200" + vaultTLSSecretName = "vault-server-tls" + vaultCASecretName = "vault-ca" + vaultCAConfigMapName = "vault-server-ca" + vaultSampleCAConfigMapName = "vault-e2e-sample-ca" + targetSecretName = "k8s-secret-to-create" //must match with external_secret.yaml target.name + vaultEgressNetworkPolicyName = "allow-vault-egress" // logical name stored in ExternalSecretsConfig spec vaultEgressNetworkPolicyK8sName = "eso-user-allow-vault-egress" // actual Kubernetes object name (operator prepends "eso-user-") + invalidProviderConfigReason = "InvalidProviderConfig" ) const ( @@ -151,6 +161,10 @@ var _ = Describe("External Secrets Operator End-to-End test scenarios", Ordered, operatorPodPrefix, })).To(Succeed()) + By("Ensuring Red Hat cert-manager Operator is installed and ready") + Expect(ensureCertManagerOperatorReady(ctx, clientset, dynamicClient)).To(Succeed(), + "Red Hat cert-manager Operator is required for e2e (OLM install from redhat-operators); see test/e2e/README.md") + By("Ensuring ExternalSecretsConfig cluster CR exists and is Ready") Expect(ensureExternalSecretsConfigReady(ctx)).To(Succeed(), "ExternalSecretsConfig should have Ready=True and Degraded=False conditions") @@ -1871,11 +1885,12 @@ var _ = Describe("External Secrets Operator End-to-End test scenarios", Ordered, }) - Context("Vault Secret Manager", Label("Platform:Generic", "Provider:Vault", "Skipped:Disconnected"), func() { + Context("Vault Secret Manager", Ordered, Label("Platform:Generic", "Provider:Vault", "Skipped:Disconnected"), func() { const ( - vaultSecretName = "foo" - vaultSecretKey = "my-value" - vaultSecretValue = "bar" + vaultSecretValue = "bar" + vaultPushSourceSecretFile = "testdata/vault/push_source_secret.yaml" + vaultPushSecretFile = "testdata/vault/push_secret.yaml" + vaultPushSecretResourceName = "vault-push-secret" ) var ( @@ -1887,6 +1902,33 @@ var _ = Describe("External Secrets Operator End-to-End test scenarios", Ordered, // Re-use the suite rest.Config so that kubeconfig trust settings are preserved. config = cfg + By("Ensuring Vault namespace exists") + Expect(ensureVaultNamespace(ctx, clientset)).To(Succeed()) + + By("Creating cert-manager self-signed Issuer") + vaultReplacements := map[string]string{ + "{{VAULT_NAMESPACE}}": vaultNamespace, + "{{VAULT_ADDR}}": vaultAddr, + } + loader.CreateFromFileWithReplacements(testassets.ReadFile, vaultIssuerFile, "", vaultReplacements) + + By("Creating Vault CA Certificate (isCA=true)") + loader.CreateFromFileWithReplacements(testassets.ReadFile, vaultCACertificateFile, "", vaultReplacements) + + By("Waiting for Vault CA secret from cert-manager") + Expect(waitForVaultCASecret(ctx, clientset)).To(Succeed(), + "vault CA secret %s/%s was not issued by cert-manager", vaultNamespace, vaultCASecretName) + + By("Creating cert-manager CA Issuer for Vault server TLS") + loader.CreateFromFileWithReplacements(testassets.ReadFile, vaultCAIssuerFile, "", vaultReplacements) + + By("Creating Vault server Certificate signed by the CA Issuer") + loader.CreateFromFileWithReplacements(testassets.ReadFile, vaultCertificateFile, "", vaultReplacements) + + By("Waiting for Vault TLS secret from cert-manager") + Expect(waitForVaultTLSSecret(ctx, clientset)).To(Succeed(), + "vault TLS secret %s/%s was not issued by cert-manager", vaultNamespace, vaultTLSSecretName) + By("Deploying Vault") Expect(applyVault(ctx, dynamicClient, clientset)).To(Succeed(), "applyVault: failed to apply vault manifest to namespace %s", vaultNamespace) @@ -1907,21 +1949,23 @@ var _ = Describe("External Secrets Operator End-to-End test scenarios", Ordered, By("Creating vault-token Secret") Expect(createVaultTokenSecret(ctx, clientset, rootToken)).To(Succeed(), "createVaultTokenSecret: failed to create vault-token Secret in namespace %s", vaultNamespace) - - By("Create test secret in vault") - Expect(createVaultTestSecret( - ctx, - clientset, - config, - rootToken, - vaultSecretName, - vaultSecretKey, - vaultSecretValue, - )).To(Succeed(), - "createVaultTestSecret: failed to write secret %q (key %q) to vault", vaultSecretName, vaultSecretKey) }) AfterAll(func() { + // Cluster-scoped / operand-namespace leftovers that namespace deletion cannot cover. + By("Clearing trustedCABundle from ExternalSecretsConfig") + clearTrustedCABundle(ctx) + + By("Deleting Vault CA ConfigMaps from operand namespace") + _ = clientset.CoreV1().ConfigMaps(operandNamespace).Delete(ctx, vaultCAConfigMapName, metav1.DeleteOptions{}) + _ = clientset.CoreV1().ConfigMaps(operandNamespace).Delete(ctx, vaultSampleCAConfigMapName, metav1.DeleteOptions{}) + + // Vault egress NetworkPolicy entry on ExternalSecretsConfig is CEL-immutable once added; + // leave it (same pattern as other custom NP e2e). The K8s NetworkPolicy in the operand + // namespace is owned by the operator from that ESC entry. + + // Namespaced Vault fixture (Issuer, Certificate, Vault, SecretStores, Push/ExternalSecrets, + // templating RBAC/Secrets, synced Secrets) is removed with the namespace. By("Cleaning up Vault namespace") safeDelete(ctx, "delete", @@ -1931,17 +1975,26 @@ var _ = Describe("External Secrets Operator End-to-End test scenarios", Ordered, ) }) - It("should create secret mentioned in ExternalSecret using the referenced SecretStore", func() { + It("should fail SecretStore with a non-matching trustedCABundle then succeed after switching to the Vault CA, PushSecret, and ExternalSecret", Label("Feature:TrustedCABundle"), func() { var ( - // test bindata for Vault externalsecretsConfigFile = "testdata/vault/externalsecretsconfig.yaml" vaultSecretStoreFile = "testdata/vault/secret_store.yaml" vaultExternalSecretFile = "testdata/vault/external_secret.yaml" secretStoreResourceName = "vault-backend" externalSecretResourceName = "vault-example" - targetSecretKey = "password" //must match with external_secret.yaml data.secretKey + targetSecretKey = "password" // must match external_secret.yaml data.secretKey ) + secretStoreReplacements := map[string]string{ + "{{VAULT_NAMESPACE}}": vaultNamespace, + "{{VAULT_ADDR}}": vaultAddr, + } + secretStoreGVR := schema.GroupVersionResource{ + Group: externalSecretsGroupName, + Version: v1APIVersion, + Resource: secretStoresKind, + } + By("Ensuring ExternalSecretsConfig has Vault egress network policy") updated, err := ensureVaultEgressOnExternalSecretsConfig(ctx, runtimeClient, externalsecretsConfigFile) Expect(err).NotTo(HaveOccurred()) @@ -1956,35 +2009,69 @@ var _ = Describe("External Secrets Operator End-to-End test scenarios", Ordered, }, 30*time.Second, 2*time.Second).Should(Succeed(), "NetworkPolicy %s should be created in namespace %s", vaultEgressNetworkPolicyK8sName, operandNamespace) } - By("Creating SecretStore") - // Create template replacements map for SecretStore - secretStoreReplacements := map[string]string{ - "{{VAULT_NAMESPACE}}": vaultNamespace, - "{{VAULT_ADDR}}": vaultAddr, - } + By("Creating Vault CA ConfigMap and a sample CA ConfigMap in the operand namespace") + Expect(createVaultCAConfigMap(ctx, clientset)).To(Succeed()) + Expect(createSampleCAConfigMap(ctx, clientset)).To(Succeed()) + + By("Creating SecretStore without caBundle/caProvider") loader.CreateFromFileWithReplacements( testassets.ReadFile, vaultSecretStoreFile, "", secretStoreReplacements, ) + defer loader.DeleteFromFileWithReplacements(testassets.ReadFile, vaultSecretStoreFile, "", secretStoreReplacements) + + By("Pointing trustedCABundle at the sample CA ConfigMap") + setTrustedCABundle(ctx, vaultSampleCAConfigMapName, externalsecrets.UserCABundleKeyPath) + Expect(utils.WaitForExternalSecretsConfigReady(ctx, dynamicClient, common.ExternalSecretsConfigObjectName, 2*time.Minute)).To(Succeed()) + + By("Waiting for SecretStore Ready=False with reason InvalidProviderConfig (sample CA does not trust Vault)") + Expect(utils.WaitForESOResourceCondition(ctx, dynamicClient, + secretStoreGVR, + vaultNamespace, secretStoreResourceName, + "Ready", "False", invalidProviderConfigReason, 2*time.Minute, + )).To(Succeed()) + + By("Switching trustedCABundle to the Vault CA ConfigMap") + setTrustedCABundle(ctx, vaultCAConfigMapName, externalsecrets.UserCABundleKeyPath) + + By("Waiting for ExternalSecretsConfig to become Ready after trustedCABundle switch") + Expect(utils.WaitForExternalSecretsConfigReady(ctx, dynamicClient, common.ExternalSecretsConfigObjectName, 2*time.Minute)).To(Succeed()) + + By("Waiting for SecretStore to become Ready after switching to the Vault CA") + Expect(utils.WaitForESOResourceReady(ctx, dynamicClient, + secretStoreGVR, + vaultNamespace, secretStoreResourceName, 2*time.Minute, + )).To(Succeed()) + + By("Creating source Secret for PushSecret") + pushReplacements := map[string]string{ + "{{PUSH_SECRET_VALUE}}": vaultSecretValue, + } + loader.CreateFromFileWithReplacements(testassets.ReadFile, vaultPushSourceSecretFile, "", pushReplacements) + defer loader.DeleteFromFileWithReplacements(testassets.ReadFile, vaultPushSourceSecretFile, "", pushReplacements) + + By("Creating PushSecret") + loader.CreateFromFile(testassets.ReadFile, vaultPushSecretFile, "") + defer loader.DeleteFromFile(testassets.ReadFile, vaultPushSecretFile, "") - By("Waiting for SecretStore to become Ready") + By("Waiting for PushSecret to become Ready") Expect(utils.WaitForESOResourceReady(ctx, dynamicClient, schema.GroupVersionResource{ Group: externalSecretsGroupName, - Version: v1APIVersion, - Resource: secretStoresKind, + Version: v1alpha1APIVersion, + Resource: PushSecretsKind, }, - vaultNamespace, secretStoreResourceName, time.Minute, + vaultNamespace, vaultPushSecretResourceName, 2*time.Minute, )).To(Succeed()) By("Creating ExternalSecret") - loader.CreateFromFile( - testassets.ReadFile, - vaultExternalSecretFile, - "", - ) + loader.CreateFromFile(testassets.ReadFile, vaultExternalSecretFile, "") + defer loader.DeleteFromFile(testassets.ReadFile, vaultExternalSecretFile, "") + defer func() { + _ = clientset.CoreV1().Secrets(vaultNamespace).Delete(ctx, targetSecretName, metav1.DeleteOptions{}) + }() By("Waiting for ExternalSecret to become Ready") Expect(utils.WaitForESOResourceReady(ctx, dynamicClient, @@ -1993,7 +2080,7 @@ var _ = Describe("External Secrets Operator End-to-End test scenarios", Ordered, Version: v1APIVersion, Resource: externalSecretsKind, }, - vaultNamespace, externalSecretResourceName, time.Minute, + vaultNamespace, externalSecretResourceName, 2*time.Minute, )).To(Succeed()) By("Verifying the generated Kubernetes Secret contains expected value") @@ -2010,7 +2097,147 @@ var _ = Describe("External Secrets Operator End-to-End test scenarios", Ordered, g.Expect(string(value)).To(Equal(vaultSecretValue), "Secret %q key %q does not match expected value", targetSecretName, targetSecretKey) + }, time.Minute, 5*time.Second).Should(Succeed()) + }) + + It("should merge Kubernetes and Vault dockerconfig secrets via ExternalSecret templating", Label("Feature:ExternalSecretsTemplating"), func() { + const ( + externalsecretsConfigFile = "testdata/vault/externalsecretsconfig.yaml" + vaultSecretStoreFile = "testdata/vault/secret_store.yaml" + templatingK8sBackendFile = "testdata/vault/templating_k8s_backend.yaml" + templatingSourceSecretsFile = "testdata/vault/templating_source_secrets.yaml" + templatingPushSecretFile = "testdata/vault/templating_push_secret.yaml" + templatingExternalSecretFile = "testdata/vault/templating_external_secret.yaml" + vaultSecretStoreResourceName = "vault-backend" + k8sSecretStoreResourceName = "kubernetes-backend" + pushSecretResourceName = "push-registry-auth-to-vault" + externalSecretResourceName = "merge-registry-auths" + mergedSecretName = "merged-registry-pull-secret" + baselineRegistryHost = "cluster.registry.example.com" + vaultRegistryHost = "private.registry.example.com" + baselineSecretName = "registry-auth-baseline" + vaultSourceSecretName = "registry-auth-vault-source" + secretReaderName = "eso-secret-reader" + ) + + vaultReplacements := map[string]string{ + "{{VAULT_NAMESPACE}}": vaultNamespace, + "{{VAULT_ADDR}}": vaultAddr, + } + + By("Ensuring ExternalSecretsConfig has Vault egress network policy") + updated, err := ensureVaultEgressOnExternalSecretsConfig(ctx, runtimeClient, externalsecretsConfigFile) + Expect(err).NotTo(HaveOccurred()) + if updated { + By("Waiting for ExternalSecretsConfig to reconcile with Vault egress policy") + Expect(utils.WaitForExternalSecretsConfigReady(ctx, dynamicClient, "cluster", 2*time.Minute)).To(Succeed()) + + By("Waiting for Vault egress NetworkPolicy to be created") + Eventually(func() error { + _, err := clientset.NetworkingV1().NetworkPolicies(operandNamespace).Get(ctx, vaultEgressNetworkPolicyK8sName, metav1.GetOptions{}) + return err + }, 30*time.Second, 2*time.Second).Should(Succeed(), "NetworkPolicy %s should be created in namespace %s", vaultEgressNetworkPolicyK8sName, operandNamespace) + } + + By("Ensuring Vault CA ConfigMap and trustedCABundle are configured") + Expect(createVaultCAConfigMap(ctx, clientset)).To(Succeed()) + setTrustedCABundle(ctx, vaultCAConfigMapName, externalsecrets.UserCABundleKeyPath) + Expect(utils.WaitForExternalSecretsConfigReady(ctx, dynamicClient, common.ExternalSecretsConfigObjectName, 2*time.Minute)).To(Succeed()) + + By("Creating Vault SecretStore") + Expect(utils.ApplyManifestFromReaderWithReplacements(ctx, dynamicClient, testassets.ReadFile, vaultSecretStoreFile, vaultReplacements)).To(Succeed()) + defer loader.DeleteFromFileWithReplacements(testassets.ReadFile, vaultSecretStoreFile, "", vaultReplacements) + + By("Waiting for Vault SecretStore to become Ready") + Expect(utils.WaitForESOResourceReady(ctx, dynamicClient, + schema.GroupVersionResource{ + Group: externalSecretsGroupName, + Version: v1APIVersion, + Resource: secretStoresKind, + }, + vaultNamespace, vaultSecretStoreResourceName, 2*time.Minute, + )).To(Succeed()) + + By("Creating Kubernetes SecretStore reader RBAC and SecretStore") + Expect(utils.ApplyManifestFromReaderWithReplacements(ctx, dynamicClient, testassets.ReadFile, templatingK8sBackendFile, vaultReplacements)).To(Succeed()) + defer func() { + _ = dynamicClient.Resource(schema.GroupVersionResource{ + Group: externalSecretsGroupName, Version: v1APIVersion, Resource: secretStoresKind, + }).Namespace(vaultNamespace).Delete(ctx, k8sSecretStoreResourceName, metav1.DeleteOptions{}) + _ = clientset.RbacV1().RoleBindings(vaultNamespace).Delete(ctx, secretReaderName, metav1.DeleteOptions{}) + _ = clientset.RbacV1().Roles(vaultNamespace).Delete(ctx, secretReaderName, metav1.DeleteOptions{}) + _ = clientset.CoreV1().ServiceAccounts(vaultNamespace).Delete(ctx, secretReaderName, metav1.DeleteOptions{}) + }() + + By("Waiting for Kubernetes SecretStore to become Ready") + Expect(utils.WaitForESOResourceReady(ctx, dynamicClient, + schema.GroupVersionResource{ + Group: externalSecretsGroupName, + Version: v1APIVersion, + Resource: secretStoresKind, + }, + vaultNamespace, k8sSecretStoreResourceName, 2*time.Minute, + )).To(Succeed()) + + By("Creating baseline (Kubernetes) and vault-source dockerconfig Secrets") + Expect(utils.ApplyManifestFromReaderWithReplacements(ctx, dynamicClient, testassets.ReadFile, templatingSourceSecretsFile, vaultReplacements)).To(Succeed()) + defer func() { + _ = clientset.CoreV1().Secrets(vaultNamespace).Delete(ctx, baselineSecretName, metav1.DeleteOptions{}) + _ = clientset.CoreV1().Secrets(vaultNamespace).Delete(ctx, vaultSourceSecretName, metav1.DeleteOptions{}) + }() + + By("Pushing vault-source dockerconfig to Vault") + Expect(utils.ApplyManifestFromReaderWithReplacements(ctx, dynamicClient, testassets.ReadFile, templatingPushSecretFile, vaultReplacements)).To(Succeed()) + defer func() { + _ = dynamicClient.Resource(schema.GroupVersionResource{ + Group: externalSecretsGroupName, Version: v1alpha1APIVersion, Resource: PushSecretsKind, + }).Namespace(vaultNamespace).Delete(ctx, pushSecretResourceName, metav1.DeleteOptions{}) + }() + + By("Waiting for PushSecret to become Ready") + Expect(utils.WaitForESOResourceReady(ctx, dynamicClient, + schema.GroupVersionResource{ + Group: externalSecretsGroupName, + Version: v1alpha1APIVersion, + Resource: PushSecretsKind, + }, + vaultNamespace, pushSecretResourceName, 2*time.Minute, + )).To(Succeed()) + By("Creating ExternalSecret that templates a merge of Kubernetes + Vault dockerconfigs") + Expect(utils.ApplyManifestFromReaderWithReplacements(ctx, dynamicClient, testassets.ReadFile, templatingExternalSecretFile, vaultReplacements)).To(Succeed()) + defer func() { + _ = dynamicClient.Resource(schema.GroupVersionResource{ + Group: externalSecretsGroupName, Version: v1APIVersion, Resource: externalSecretsKind, + }).Namespace(vaultNamespace).Delete(ctx, externalSecretResourceName, metav1.DeleteOptions{}) + _ = clientset.CoreV1().Secrets(vaultNamespace).Delete(ctx, mergedSecretName, metav1.DeleteOptions{}) + }() + + By("Waiting for ExternalSecret to become Ready") + Expect(utils.WaitForESOResourceReady(ctx, dynamicClient, + schema.GroupVersionResource{ + Group: externalSecretsGroupName, + Version: v1APIVersion, + Resource: externalSecretsKind, + }, + vaultNamespace, externalSecretResourceName, 2*time.Minute, + )).To(Succeed()) + + By("Verifying merged dockerconfigjson contains both registry auths") + Eventually(func(g Gomega) { + secret, err := clientset.CoreV1().Secrets(vaultNamespace).Get(ctx, mergedSecretName, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(secret.Type).To(Equal(corev1.SecretTypeDockerConfigJson)) + + raw, ok := secret.Data[corev1.DockerConfigJsonKey] + g.Expect(ok).To(BeTrue(), "merged secret missing %s", corev1.DockerConfigJsonKey) + + var parsed struct { + Auths map[string]json.RawMessage `json:"auths"` + } + g.Expect(json.Unmarshal(raw, &parsed)).To(Succeed()) + g.Expect(parsed.Auths).To(HaveKey(baselineRegistryHost)) + g.Expect(parsed.Auths).To(HaveKey(vaultRegistryHost)) }, time.Minute, 5*time.Second).Should(Succeed()) }) }) @@ -2064,48 +2291,13 @@ func expectedProxyPorts(httpsProxy, httpProxy string) []int32 { return ports } -// Apply vault manifest using dynamic client with architecture-specific image substitution +// Apply vault manifest using dynamic client with architecture-specific image substitution. +// Caller must ensure the vault TLS secret already exists (cert-manager Certificate Ready). func applyVault(ctx context.Context, dynamicClient *dynamic.DynamicClient, clientset *kubernetes.Clientset) error { By(fmt.Sprintf("Applying vault manifest from: %s", vaultManifestFile)) - // Ensure vault namespace exists and is not terminating - By(fmt.Sprintf("Ensuring namespace %s exists", vaultNamespace)) - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: vaultNamespace, - }, - } - - // Try to get the namespace - existingNs, err := clientset.CoreV1().Namespaces().Get(ctx, vaultNamespace, metav1.GetOptions{}) - if err != nil { - if !k8serrors.IsNotFound(err) { - return fmt.Errorf("failed to check namespace: %w", err) - } - // Namespace doesn't exist, create it - _, err = clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) - if err != nil { - return fmt.Errorf("failed to create namespace: %w", err) - } - By(fmt.Sprintf("Created namespace %s", vaultNamespace)) - } else if existingNs.Status.Phase == corev1.NamespaceTerminating { - // Namespace is terminating — wait for full deletion then recreate. - By(fmt.Sprintf("Namespace %s is terminating, waiting for deletion (up to 2 minutes)", vaultNamespace)) - Eventually(func() bool { - _, err := clientset.CoreV1().Namespaces().Get(ctx, vaultNamespace, metav1.GetOptions{}) - return k8serrors.IsNotFound(err) - }).WithTimeout(2*time.Minute).WithPolling(2*time.Second).Should(BeTrue(), - "namespace %s was not fully deleted within 2 minutes", vaultNamespace) - By(fmt.Sprintf("Namespace %s has been deleted", vaultNamespace)) - - // Retry creating the namespace until the API server cache is fully cleared. - Eventually(func() error { - _, createErr := clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) - return createErr - }).WithTimeout(15*time.Second).WithPolling(1*time.Second).Should(Succeed()) - By(fmt.Sprintf("Recreated namespace %s after termination", vaultNamespace)) - } else { - By(fmt.Sprintf("Namespace %s already exists", vaultNamespace)) + if err := ensureVaultNamespace(ctx, clientset); err != nil { + return err } // Get node information for debugging @@ -2241,11 +2433,12 @@ func setupVault(ctx context.Context, client *kubernetes.Clientset, config *rest. By(fmt.Sprintf("Initializing Vault, pod=%s", podName)) - // Step 1: Initialize Vault + // Step 1: Initialize Vault (skip TLS verify for in-pod self-signed cert) stdout, stderr, err := utils.ExecCommandInPod(ctx, client, config, utils.PodExecOptions{ Namespace: vaultNamespace, PodName: podName, - Command: []string{"vault", "operator", "init", "-key-shares=1", "-key-threshold=1"}, + Command: []string{"sh", "-c", + "VAULT_SKIP_VERIFY=true vault operator init -key-shares=1 -key-threshold=1"}, }) if err != nil { @@ -2277,7 +2470,8 @@ func setupVault(ctx context.Context, client *kubernetes.Clientset, config *rest. stdout, stderr, err = utils.ExecCommandInPod(ctx, client, config, utils.PodExecOptions{ Namespace: vaultNamespace, PodName: podName, - Command: []string{"vault", "operator", "unseal", unsealKey}, + Command: []string{"sh", "-c", + fmt.Sprintf("VAULT_SKIP_VERIFY=true vault operator unseal '%s'", escapeShellString(unsealKey))}, }) if err != nil { @@ -2325,8 +2519,8 @@ func enableKVEngine(ctx context.Context, client *kubernetes.Clientset, config *r // Only suppress the error when the secret/ mount is already enabled (exit code 2 from vault). // Any other failure (permissions, CLI error) is propagated by checking vault secrets list. command := fmt.Sprintf( - "VAULT_TOKEN='%s' vault secrets enable -path=secret kv-v2 || "+ - "VAULT_TOKEN='%s' vault secrets list | grep -q '^secret/'", + "VAULT_SKIP_VERIFY=true VAULT_TOKEN='%s' vault secrets enable -path=secret kv-v2 || "+ + "VAULT_SKIP_VERIFY=true VAULT_TOKEN='%s' vault secrets list | grep -q '^secret/'", escapeShellString(token), escapeShellString(token), ) @@ -2346,37 +2540,6 @@ func enableKVEngine(ctx context.Context, client *kubernetes.Clientset, config *r return err } -// Create a vault test secret using client-go -func createVaultTestSecret(ctx context.Context, client *kubernetes.Clientset, config *rest.Config, token string, secretname string, key, value string) error { - podName, err := getVaultPodName(ctx, client) - if err != nil { - return err - } - - // Use shell command with VAULT_TOKEN environment variable for security. - // secretname, key and value are escaped so shell metacharacters cannot alter the command. - command := fmt.Sprintf("VAULT_TOKEN='%s' vault kv put secret/%s '%s'='%s'", - escapeShellString(token), - escapeShellString(secretname), - escapeShellString(key), - escapeShellString(value), - ) - - _, _, err = utils.ExecCommandInPod(ctx, client, config, utils.PodExecOptions{ - Namespace: vaultNamespace, - PodName: podName, - Command: []string{"sh", "-c", command}, - }) - - if err != nil { - return fmt.Errorf("failed to create vault secret '%s': %w", secretname, err) - } - - By(fmt.Sprintf("Vault secret '%s' created successfully", secretname)) - - return err -} - func createVaultTokenSecret(ctx context.Context, client *kubernetes.Clientset, token string) error { secretsClient := client.CoreV1().Secrets(vaultNamespace) existing, err := secretsClient.Get(ctx, "vault-token", metav1.GetOptions{}) @@ -2516,6 +2679,156 @@ func loadExternalSecretsConfigFromFile(assetFunc func(string) ([]byte, error), f return esc, nil } +// ensureVaultNamespace creates vault-test if missing, or waits/recreates if terminating. +func ensureVaultNamespace(ctx context.Context, clientset *kubernetes.Clientset) error { + By(fmt.Sprintf("Ensuring namespace %s exists", vaultNamespace)) + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: vaultNamespace, + }, + } + + existingNs, err := clientset.CoreV1().Namespaces().Get(ctx, vaultNamespace, metav1.GetOptions{}) + if err != nil { + if !k8serrors.IsNotFound(err) { + return fmt.Errorf("failed to check namespace: %w", err) + } + _, err = clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create namespace: %w", err) + } + By(fmt.Sprintf("Created namespace %s", vaultNamespace)) + return nil + } + + if existingNs.Status.Phase == corev1.NamespaceTerminating { + By(fmt.Sprintf("Namespace %s is terminating, waiting for deletion (up to 2 minutes)", vaultNamespace)) + Eventually(func() bool { + _, err := clientset.CoreV1().Namespaces().Get(ctx, vaultNamespace, metav1.GetOptions{}) + return k8serrors.IsNotFound(err) + }).WithTimeout(2 * time.Minute).WithPolling(2 * time.Second).Should(BeTrue(), + "namespace %s was not fully deleted within 2 minutes", vaultNamespace) + + Eventually(func() error { + _, createErr := clientset.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + return createErr + }).WithTimeout(15 * time.Second).WithPolling(1 * time.Second).Should(Succeed()) + By(fmt.Sprintf("Recreated namespace %s after termination", vaultNamespace)) + return nil + } + + By(fmt.Sprintf("Namespace %s already exists", vaultNamespace)) + return nil +} + +// waitForVaultCASecret waits until cert-manager has issued vault-ca with a CA certificate. +func waitForVaultCASecret(ctx context.Context, clientset *kubernetes.Clientset) error { + return wait.PollUntilContextTimeout(ctx, 5*time.Second, 3*time.Minute, true, func(ctx context.Context) (bool, error) { + secret, err := clientset.CoreV1().Secrets(vaultNamespace).Get(ctx, vaultCASecretName, metav1.GetOptions{}) + if err != nil { + if k8serrors.IsNotFound(err) { + return false, nil + } + return false, err + } + // For isCA Certificates, tls.crt is the CA certificate used by trustedCABundle. + if len(secret.Data["tls.crt"]) == 0 || len(secret.Data["tls.key"]) == 0 { + return false, nil + } + return true, nil + }) +} + +// waitForVaultTLSSecret waits until cert-manager has issued vault-server-tls with cert, key, and CA. +func waitForVaultTLSSecret(ctx context.Context, clientset *kubernetes.Clientset) error { + return wait.PollUntilContextTimeout(ctx, 5*time.Second, 3*time.Minute, true, func(ctx context.Context) (bool, error) { + secret, err := clientset.CoreV1().Secrets(vaultNamespace).Get(ctx, vaultTLSSecretName, metav1.GetOptions{}) + if err != nil { + if k8serrors.IsNotFound(err) { + return false, nil + } + return false, err + } + if len(secret.Data["tls.crt"]) == 0 || len(secret.Data["tls.key"]) == 0 || len(secret.Data["ca.crt"]) == 0 { + return false, nil + } + return true, nil + }) +} + +// createVaultCAConfigMap copies the Vault issuing CA into the operand namespace for trustedCABundle. +// Prefer vault-ca tls.crt (isCA Certificate) so the bundle passes operator CA validation. +func createVaultCAConfigMap(ctx context.Context, clientset *kubernetes.Clientset) error { + caSecret, err := clientset.CoreV1().Secrets(vaultNamespace).Get(ctx, vaultCASecretName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("get vault CA secret: %w", err) + } + caPEM := caSecret.Data["tls.crt"] + if len(caPEM) == 0 { + return fmt.Errorf("vault CA secret %s/%s missing tls.crt", vaultNamespace, vaultCASecretName) + } + + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: vaultCAConfigMapName, + Namespace: operandNamespace, + }, + Data: map[string]string{ + externalsecrets.UserCABundleKeyPath: string(caPEM), + }, + } + existing, err := clientset.CoreV1().ConfigMaps(operandNamespace).Get(ctx, vaultCAConfigMapName, metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { + _, err = clientset.CoreV1().ConfigMaps(operandNamespace).Create(ctx, cm, metav1.CreateOptions{}) + return err + } + if err != nil { + return err + } + existing.Data = cm.Data + _, err = clientset.CoreV1().ConfigMaps(operandNamespace).Update(ctx, existing, metav1.UpdateOptions{}) + return err +} + +// createSampleCAConfigMap creates a valid CA ConfigMap that does not issue Vault's TLS cert. +// Used to assert trustedCABundle switches: sample CA keeps SecretStore InvalidProviderConfig. +func createSampleCAConfigMap(ctx context.Context, clientset *kubernetes.Clientset) error { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: vaultSampleCAConfigMapName, + Namespace: operandNamespace, + }, + Data: map[string]string{ + externalsecrets.UserCABundleKeyPath: testCACertPEM(), + }, + } + existing, err := clientset.CoreV1().ConfigMaps(operandNamespace).Get(ctx, vaultSampleCAConfigMapName, metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { + _, err = clientset.CoreV1().ConfigMaps(operandNamespace).Create(ctx, cm, metav1.CreateOptions{}) + return err + } + if err != nil { + return err + } + existing.Data = cm.Data + _, err = clientset.CoreV1().ConfigMaps(operandNamespace).Update(ctx, existing, metav1.UpdateOptions{}) + return err +} + +// clearTrustedCABundle clears ExternalSecretsConfig.spec.controllerConfig.trustedCABundle. +func clearTrustedCABundle(ctx context.Context) { + GinkgoHelper() + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + esc := &operatorv1alpha1.ExternalSecretsConfig{} + if err := suiteRuntimeClient.Get(ctx, client.ObjectKey{Name: common.ExternalSecretsConfigObjectName}, esc); err != nil { + return err + } + esc.Spec.ControllerConfig.TrustedCABundle = nil + return suiteRuntimeClient.Update(ctx, esc) + }) + Expect(err).NotTo(HaveOccurred(), "should clear trustedCABundle from ExternalSecretsConfig") +} + // ensureVaultEgressOnExternalSecretsConfig ensures the cluster ExternalSecretsConfig has the Vault egress // network policies. If policies are missing or different, they are updated. Returns true if an update was made. func ensureVaultEgressOnExternalSecretsConfig(ctx context.Context, c client.Client, vaultConfigFile string) (bool, error) { diff --git a/test/e2e/testdata/cert-manager/operator.yaml b/test/e2e/testdata/cert-manager/operator.yaml new file mode 100644 index 00000000..1175bdc1 --- /dev/null +++ b/test/e2e/testdata/cert-manager/operator.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: cert-manager-operator +--- +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: openshift-cert-manager-operator + namespace: cert-manager-operator +spec: {} +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: openshift-cert-manager-operator + namespace: cert-manager-operator +spec: + channel: stable-v1 + name: openshift-cert-manager-operator + source: redhat-operators + sourceNamespace: openshift-marketplace + installPlanApproval: Automatic diff --git a/test/e2e/testdata/vault/ca_certificate.yaml b/test/e2e/testdata/vault/ca_certificate.yaml new file mode 100644 index 00000000..57f430fc --- /dev/null +++ b/test/e2e/testdata/vault/ca_certificate.yaml @@ -0,0 +1,21 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: vault-ca + namespace: {{VAULT_NAMESPACE}} +spec: + isCA: true + commonName: vault-e2e-ca + secretName: vault-ca + duration: 2160h + renewBefore: 360h + privateKey: + algorithm: RSA + size: 2048 + usages: + - cert sign + - crl sign + issuerRef: + name: vault-selfsigned + kind: Issuer + group: cert-manager.io diff --git a/test/e2e/testdata/vault/ca_issuer.yaml b/test/e2e/testdata/vault/ca_issuer.yaml new file mode 100644 index 00000000..eeb3eead --- /dev/null +++ b/test/e2e/testdata/vault/ca_issuer.yaml @@ -0,0 +1,8 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: vault-ca-issuer + namespace: {{VAULT_NAMESPACE}} +spec: + ca: + secretName: vault-ca diff --git a/test/e2e/testdata/vault/certificate.yaml b/test/e2e/testdata/vault/certificate.yaml new file mode 100644 index 00000000..749c7c87 --- /dev/null +++ b/test/e2e/testdata/vault/certificate.yaml @@ -0,0 +1,26 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: vault-server-tls + namespace: {{VAULT_NAMESPACE}} +spec: + secretName: vault-server-tls + duration: 2160h + renewBefore: 360h + commonName: vault.{{VAULT_NAMESPACE}}.svc + dnsNames: + - vault + - vault.{{VAULT_NAMESPACE}} + - vault.{{VAULT_NAMESPACE}}.svc + - vault.{{VAULT_NAMESPACE}}.svc.cluster.local + issuerRef: + name: vault-ca-issuer + kind: Issuer + group: cert-manager.io + privateKey: + algorithm: RSA + size: 2048 + usages: + - server auth + - digital signature + - key encipherment diff --git a/test/e2e/testdata/vault/external_secret.yaml b/test/e2e/testdata/vault/external_secret.yaml index 76942a49..b8c3920f 100644 --- a/test/e2e/testdata/vault/external_secret.yaml +++ b/test/e2e/testdata/vault/external_secret.yaml @@ -13,5 +13,5 @@ spec: data: - secretKey: password remoteRef: - key: foo - property: my-value \ No newline at end of file + key: e2e/foo + property: my-value diff --git a/test/e2e/testdata/vault/issuer.yaml b/test/e2e/testdata/vault/issuer.yaml new file mode 100644 index 00000000..0aff5a29 --- /dev/null +++ b/test/e2e/testdata/vault/issuer.yaml @@ -0,0 +1,7 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: vault-selfsigned + namespace: {{VAULT_NAMESPACE}} +spec: + selfSigned: {} diff --git a/test/e2e/testdata/vault/push_secret.yaml b/test/e2e/testdata/vault/push_secret.yaml new file mode 100644 index 00000000..8bcfcaf6 --- /dev/null +++ b/test/e2e/testdata/vault/push_secret.yaml @@ -0,0 +1,22 @@ +apiVersion: external-secrets.io/v1alpha1 +kind: PushSecret +metadata: + name: vault-push-secret + namespace: vault-test + labels: + app.kubernetes.io/name: vault-push-secret + app.kubernetes.io/managed-by: external-secrets-operator-e2e +spec: + refreshInterval: 15s + secretStoreRefs: + - name: vault-backend + kind: SecretStore + selector: + secret: + name: vault-push-source + data: + - match: + secretKey: my-value + remoteRef: + remoteKey: e2e/foo + property: my-value diff --git a/test/e2e/testdata/vault/push_source_secret.yaml b/test/e2e/testdata/vault/push_source_secret.yaml new file mode 100644 index 00000000..1b7ff873 --- /dev/null +++ b/test/e2e/testdata/vault/push_source_secret.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: vault-push-source + namespace: vault-test + labels: + app.kubernetes.io/managed-by: external-secrets-operator-e2e +type: Opaque +stringData: + my-value: "{{PUSH_SECRET_VALUE}}" diff --git a/test/e2e/testdata/vault/templating_external_secret.yaml b/test/e2e/testdata/vault/templating_external_secret.yaml new file mode 100644 index 00000000..9578a719 --- /dev/null +++ b/test/e2e/testdata/vault/templating_external_secret.yaml @@ -0,0 +1,37 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: merge-registry-auths + namespace: {{VAULT_NAMESPACE}} + labels: + app.kubernetes.io/managed-by: external-secrets-operator-e2e +spec: + refreshInterval: 15s + secretStoreRef: + name: vault-backend + kind: SecretStore + target: + name: merged-registry-pull-secret + creationPolicy: Owner + template: + engineVersion: v2 + type: kubernetes.io/dockerconfigjson + data: + .dockerconfigjson: | + {{- $base := .baseline | fromJson -}} + {{- $extra := .vault | fromJson -}} + {{- $merged := merge $base $extra -}} + {{- $merged | toJson -}} + data: + - secretKey: vault + remoteRef: + key: registry-auth/extra + property: dockerconfigjson + - secretKey: baseline + remoteRef: + key: registry-auth-baseline + property: .dockerconfigjson + sourceRef: + storeRef: + name: kubernetes-backend + kind: SecretStore diff --git a/test/e2e/testdata/vault/templating_k8s_backend.yaml b/test/e2e/testdata/vault/templating_k8s_backend.yaml new file mode 100644 index 00000000..69ee3c7c --- /dev/null +++ b/test/e2e/testdata/vault/templating_k8s_backend.yaml @@ -0,0 +1,59 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: eso-secret-reader + namespace: {{VAULT_NAMESPACE}} + labels: + app.kubernetes.io/managed-by: external-secrets-operator-e2e +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: eso-secret-reader + namespace: {{VAULT_NAMESPACE}} + labels: + app.kubernetes.io/managed-by: external-secrets-operator-e2e +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["authorization.k8s.io"] + resources: ["selfsubjectrulesreviews"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: eso-secret-reader + namespace: {{VAULT_NAMESPACE}} + labels: + app.kubernetes.io/managed-by: external-secrets-operator-e2e +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: eso-secret-reader +subjects: + - kind: ServiceAccount + name: eso-secret-reader + namespace: {{VAULT_NAMESPACE}} +--- +apiVersion: external-secrets.io/v1 +kind: SecretStore +metadata: + name: kubernetes-backend + namespace: {{VAULT_NAMESPACE}} + labels: + app.kubernetes.io/managed-by: external-secrets-operator-e2e +spec: + provider: + kubernetes: + remoteNamespace: {{VAULT_NAMESPACE}} + server: + url: "https://kubernetes.default.svc" + caProvider: + type: ConfigMap + name: kube-root-ca.crt + key: ca.crt + auth: + serviceAccount: + name: eso-secret-reader diff --git a/test/e2e/testdata/vault/templating_push_secret.yaml b/test/e2e/testdata/vault/templating_push_secret.yaml new file mode 100644 index 00000000..f9291ffe --- /dev/null +++ b/test/e2e/testdata/vault/templating_push_secret.yaml @@ -0,0 +1,23 @@ +apiVersion: external-secrets.io/v1alpha1 +kind: PushSecret +metadata: + name: push-registry-auth-to-vault + namespace: {{VAULT_NAMESPACE}} + labels: + app.kubernetes.io/managed-by: external-secrets-operator-e2e +spec: + refreshInterval: 15s + updatePolicy: Replace + deletionPolicy: None + secretStoreRefs: + - name: vault-backend + kind: SecretStore + selector: + secret: + name: registry-auth-vault-source + data: + - match: + secretKey: .dockerconfigjson + remoteRef: + remoteKey: registry-auth/extra + property: dockerconfigjson diff --git a/test/e2e/testdata/vault/templating_source_secrets.yaml b/test/e2e/testdata/vault/templating_source_secrets.yaml new file mode 100644 index 00000000..0f3c4ea7 --- /dev/null +++ b/test/e2e/testdata/vault/templating_source_secrets.yaml @@ -0,0 +1,39 @@ +apiVersion: v1 +kind: Secret +metadata: + name: registry-auth-baseline + namespace: {{VAULT_NAMESPACE}} + labels: + app.kubernetes.io/managed-by: external-secrets-operator-e2e +type: kubernetes.io/dockerconfigjson +stringData: + .dockerconfigjson: | + { + "auths": { + "cluster.registry.example.com": { + "username": "cluster-user", + "password": "cluster-password", + "auth": "Y2x1c3Rlci11c2VyOmNsdXN0ZXItcGFzc3dvcmQ=" + } + } + } +--- +apiVersion: v1 +kind: Secret +metadata: + name: registry-auth-vault-source + namespace: {{VAULT_NAMESPACE}} + labels: + app.kubernetes.io/managed-by: external-secrets-operator-e2e +type: kubernetes.io/dockerconfigjson +stringData: + .dockerconfigjson: | + { + "auths": { + "private.registry.example.com": { + "username": "extra-user", + "password": "extra-password", + "auth": "ZXh0cmEtdXNlcjpleHRyYS1wYXNzd29yZA==" + } + } + } diff --git a/test/e2e/testdata/vault/vault.yaml b/test/e2e/testdata/vault/vault.yaml index 2b7da658..427c93b8 100644 --- a/test/e2e/testdata/vault/vault.yaml +++ b/test/e2e/testdata/vault/vault.yaml @@ -22,8 +22,9 @@ data: ui = true listener "tcp" { - address = "0.0.0.0:8200" - tls_disable = 1 + address = "0.0.0.0:8200" + tls_cert_file = "/vault/tls/tls.crt" + tls_key_file = "/vault/tls/tls.key" } storage "file" { @@ -47,7 +48,7 @@ spec: metadata: labels: app: vault - annotations: + annotations: openshift.io/scc: restricted-v2 spec: serviceAccountName: vault @@ -75,7 +76,9 @@ spec: - name: VAULT_API_ADDR value: {{VAULT_ADDR}} - name: VAULT_ADDR - value: http://127.0.0.1:8200 + value: https://127.0.0.1:8200 + - name: VAULT_SKIP_VERIFY + value: "true" ports: - containerPort: 8200 securityContext: @@ -94,18 +97,19 @@ spec: - name: config mountPath: /vault/config - name: data - mountPath: /vault/data + mountPath: /vault/data + - name: tls + mountPath: /vault/tls + readOnly: true startupProbe: - httpGet: - path: /v1/sys/health?standbyok=true&sealedcode=204&uninitcode=204 + tcpSocket: port: 8200 initialDelaySeconds: 5 periodSeconds: 5 timeoutSeconds: 5 failureThreshold: 24 readinessProbe: - httpGet: - path: /v1/sys/health?standbyok=true&sealedcode=204&uninitcode=204 + tcpSocket: port: 8200 initialDelaySeconds: 0 periodSeconds: 5 @@ -117,6 +121,9 @@ spec: name: vault-config - name: data emptyDir: {} + - name: tls + secret: + secretName: vault-server-tls --- apiVersion: v1 @@ -128,6 +135,6 @@ spec: selector: app: vault ports: - - name: http + - name: https port: 8200 targetPort: 8200 diff --git a/test/e2e/trusted_ca_bundle_test.go b/test/e2e/trusted_ca_bundle_test.go index cc5817c6..50bdb643 100644 --- a/test/e2e/trusted_ca_bundle_test.go +++ b/test/e2e/trusted_ca_bundle_test.go @@ -222,6 +222,45 @@ var _ = Describe("Trusted CA Bundle", Ordered, Label("Platform:Generic", "Featur }, time.Minute, 5*time.Second).Should(Succeed(), "core controller should have user CA bundle removed after clearing") }) + It("should restore the watch label on the trustedCABundle ConfigMap after external removal", func() { + By("Creating a ConfigMap and configuring trustedCABundle") + createTestCAConfigMap(ctx, trustedCABundleTestCMName, externalsecrets.UserCABundleKeyPath, testCACertPEM(), nil) + setTrustedCABundle(ctx, trustedCABundleTestCMName, externalsecrets.UserCABundleKeyPath) + + By("Waiting for ExternalSecretsConfig to be Ready") + Expect(utils.WaitForExternalSecretsConfigReady(ctx, suiteDynamicClient, common.ExternalSecretsConfigObjectName, 2*time.Minute)).To(Succeed()) + + By("Verifying the operator applied the watch label on the referenced ConfigMap") + Eventually(func(g Gomega) { + cm, err := suiteClientset.CoreV1().ConfigMaps(operandNamespace).Get(ctx, trustedCABundleTestCMName, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(cm.Labels).To(HaveKeyWithValue(externalsecrets.WatchedResourceLabelKey, externalsecrets.WatchedResourceLabelValue)) + }, time.Minute, 5*time.Second).Should(Succeed()) + + By("Removing the watch label from the trustedCABundle ConfigMap") + Expect(retry.RetryOnConflict(retry.DefaultRetry, func() error { + cm, err := suiteClientset.CoreV1().ConfigMaps(operandNamespace).Get(ctx, trustedCABundleTestCMName, metav1.GetOptions{}) + if err != nil { + return err + } + if cm.Labels == nil { + return nil + } + delete(cm.Labels, externalsecrets.WatchedResourceLabelKey) + _, err = suiteClientset.CoreV1().ConfigMaps(operandNamespace).Update(ctx, cm, metav1.UpdateOptions{}) + return err + })).To(Succeed(), "should remove the watch label") + + By("Waiting for the operator to restore the watch label") + Eventually(func(g Gomega) { + cm, err := suiteClientset.CoreV1().ConfigMaps(operandNamespace).Get(ctx, trustedCABundleTestCMName, metav1.GetOptions{}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(cm.Labels).To(HaveKeyWithValue(externalsecrets.WatchedResourceLabelKey, externalsecrets.WatchedResourceLabelValue), + "operator should restore %s=%s on trustedCABundle ConfigMap %s", + externalsecrets.WatchedResourceLabelKey, externalsecrets.WatchedResourceLabelValue, trustedCABundleTestCMName) + }, 2*time.Minute, 5*time.Second).Should(Succeed()) + }) + It("should set ExternalSecretsConfig to Degraded when ConfigMap does not exist", func() { By("Setting trustedCABundle pointing to a non-existent ConfigMap") setTrustedCABundle(ctx, "does-not-exist-ca-bundle", externalsecrets.UserCABundleKeyPath) diff --git a/test/utils/conditions.go b/test/utils/conditions.go index 3d8732cd..766a87d5 100644 --- a/test/utils/conditions.go +++ b/test/utils/conditions.go @@ -113,7 +113,23 @@ func WaitForESOResourceReady( namespace, name string, timeout time.Duration, ) error { - return wait.PollUntilContextTimeout(ctx, 5*time.Second, timeout, true, func(ctx context.Context) (bool, error) { + return WaitForESOResourceCondition(ctx, client, gvr, namespace, name, "Ready", "True", "", timeout) +} + +// WaitForESOResourceCondition polls until the named condition on an ESO custom resource +// (SecretStore, PushSecret, ExternalSecret, etc.) has the expected status. When reason is +// non-empty, the condition's reason must match exactly. Message matching is intentionally +// omitted so callers can assert status/reason without scraping logs or message text. +func WaitForESOResourceCondition( + ctx context.Context, + client dynamic.Interface, + gvr schema.GroupVersionResource, + namespace, name, condType, condStatus, reason string, + timeout time.Duration, +) error { + var lastCondition map[string]interface{} + + err := wait.PollUntilContextTimeout(ctx, 5*time.Second, timeout, true, func(ctx context.Context) (bool, error) { u, err := client.Resource(gvr).Namespace(namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil { return false, nil // retry @@ -129,20 +145,42 @@ func WaitForESOResourceReady( if !ok { continue } - t := cond["type"] - s := cond["status"] - msg := cond["message"] - - if t == "Ready" { - if s == "True" { - return true, nil - } else { - fmt.Printf("resource %s/%s not ready: %v\n", namespace, name, msg) + if cond["type"] != condType { + continue + } + lastCondition = cond + if cond["status"] != condStatus { + fmt.Printf("resource %s/%s condition %s status=%v reason=%v message=%v\n", + namespace, name, condType, cond["status"], cond["reason"], cond["message"]) + return false, nil + } + if reason != "" { + gotReason, _ := cond["reason"].(string) + if gotReason != reason { + fmt.Printf("resource %s/%s condition %s status=%v reason=%v (want %s) message=%v\n", + namespace, name, condType, cond["status"], cond["reason"], reason, cond["message"]) + return false, nil } } + return true, nil } return false, nil }) + + if err != nil && wait.Interrupted(err) { + got := "not set" + if lastCondition != nil { + got = fmt.Sprintf("status=%v reason=%v message=%v", + lastCondition["status"], lastCondition["reason"], lastCondition["message"]) + } + want := fmt.Sprintf("status=%s", condStatus) + if reason != "" { + want = fmt.Sprintf("%s reason=%s", want, reason) + } + return fmt.Errorf("timeout waiting for %s/%s condition %s (%s): last observed %s", + namespace, name, condType, want, got) + } + return err } // WaitForExternalSecretsConfigReady waits for the ExternalSecretsConfig CR to have both Ready and Degraded diff --git a/test/utils/dynamic_resources.go b/test/utils/dynamic_resources.go index 11222438..bc7a2d5d 100644 --- a/test/utils/dynamic_resources.go +++ b/test/utils/dynamic_resources.go @@ -59,13 +59,19 @@ func NewDynamicResourceLoader(context context.Context, t *testing.T) DynamicReso } func (d DynamicResourceLoader) DeleteFromFile(assetFunc func(name string) ([]byte, error), filename string, overrideNamespace string) { + d.DeleteFromFileWithReplacements(assetFunc, filename, overrideNamespace, nil) +} + +// DeleteFromFileWithReplacements deletes a resource from a file after applying template replacements +// (same placeholders as CreateFromFileWithReplacements). +func (d DynamicResourceLoader) DeleteFromFileWithReplacements(assetFunc func(name string) ([]byte, error), filename string, overrideNamespace string, replacements map[string]string) { d.t.Logf("Deleting resource %v\n", filename) deleteFunc := func(t *testing.T, unstructured *unstructured.Unstructured, dynamicResourceInterface dynamic.ResourceInterface) { err := dynamicResourceInterface.Delete(d.context, unstructured.GetName(), metav1.DeleteOptions{}) d.noErrorSkipNotExisting(err) } - d.do(deleteFunc, assetFunc, filename, overrideNamespace) + d.doWithReplacements(deleteFunc, assetFunc, filename, overrideNamespace, replacements) d.t.Logf("Resource %v deleted\n", filename) }