feat: dns utils

e2e/main_test.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"testing"
 
+	"github.com/openmcp-project/openmcp-operator/api/clusters/v1alpha1"
 	"github.com/openmcp-project/openmcp-testing/pkg/platformservices"
 	"github.com/openmcp-project/openmcp-testing/pkg/providers"
 	"github.com/openmcp-project/openmcp-testing/pkg/setup"
@@ -26,6 +27,13 @@ func TestMain(m *testing.M) {
 			Image:        "ghcr.io/openmcp-project/images/openmcp-operator:v1.3.0",
 			Environment:  "debug",
 			PlatformName: "platform",
+			ExtraClusterPurposeMapping: []providers.ClusterPurposeMapping{
+				{
+					Purpose: "dns",
+					Profile: "kind",
+					Tenancy: v1alpha1.TENANCY_SHARED,
+				},
+			},
 		},
 		ClusterProviders: []providers.ClusterProviderSetup{
 			{

e2e/serviceprovider_test.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/openmcp-project/openmcp-testing/pkg/conditions"
+	"github.com/openmcp-project/openmcp-testing/pkg/setup/dns"
 	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/e2e-framework/klient/wait"
 	"sigs.k8s.io/e2e-framework/pkg/envconf"
@@ -18,6 +19,7 @@ import (
 
 func TestServiceProvider(t *testing.T) {
 	basicProviderTest := features.New("provider test").
+		Setup(dns.CreateExternalService()).
 		Setup(providers.CreateMCP("test-mcp", wait.WithTimeout(2*time.Minute))).
 		Setup(providers.ImportServiceProviderAPIs("serviceproviderobjects", wait.WithTimeout(time.Minute))).
 		Setup(providers.ImportDomainAPIs("test-mcp", "domainobjects", wait.WithTimeout(time.Minute))).

pkg/clusterutils/apiserver/README.md

@@ -0,0 +1,5 @@
+# API Server Updater
+
+Package `clusterutils/apiserver` provides functionality for updating the static pod manifest of the Kubernetes API Server of [kind](https://kind.sigs.k8s.io/) control planes.
+
+See package [setup/dns](../../setup/dns/) and the [service-provider-template](https://github.com/openmcp-project/service-provider-template) for example usages to test service provider webhooks in [OpenControlPlane](https://open-control-plane.io/).

pkg/clusterutils/apiserver/updater.go

@@ -0,0 +1,213 @@
+package apiserver
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
+	kindcluster "sigs.k8s.io/kind/pkg/cluster"
+	"sigs.k8s.io/yaml"
+)
+
+// Updater is a helper to adjust the static pod manifest of the kube-apiserver in a kind control plane container.
+type Updater struct {
+	// The CLI to interact with the kind container. Defaults to docker, can be replaced with docker compatible CLIs like podman.
+	dockerCLI string
+	// The kind control plane to update. Defaults to the onboarcing cluster container.
+	kindContainer string
+	// The path to the api-server manifest in the kind container.
+	apiServerManifestPath string
+	// The timeout for the API server restart.
+	timeout time.Duration
+}
+
+type Option func(*Updater)
+
+// NewUpdater returns a new Updater. The onboarding cluster container is the default cluster target.
+func NewUpdater(opts ...Option) (*Updater, error) {
+	updater := &Updater{
+		dockerCLI:             "docker",
+		apiServerManifestPath: "/etc/kubernetes/manifests/kube-apiserver.yaml",
+		timeout:               time.Minute * 3,
+	}
+	for _, o := range opts {
+		o(updater)
+	}
+	if updater.kindContainer == "" {
+		onboardingClusterContainer, err := onboardingClusterContainer()
+		if err != nil {
+			return nil, err
+		}
+		updater.kindContainer = onboardingClusterContainer
+	}
+	return updater, nil
+}
+
+func WithAPIServerManifestPath(path string) Option {
+	return func(c *Updater) {
+		c.apiServerManifestPath = path
+	}
+}
+
+func WithDockerCLI(cli string) Option {
+	return func(c *Updater) {
+		c.dockerCLI = cli
+	}
+}
+
+func WithKindContainer(name string) Option {
+	return func(c *Updater) {
+		c.kindContainer = name
+	}
+}
+
+func WithTimeout(timeout time.Duration) Option {
+	return func(c *Updater) {
+		c.timeout = timeout
+	}
+}
+
+// AddHostAlias adds the given hostname -> ip mapping as host alias to the kube-apiserver (static pod) manifest
+// inside a kind container and waits for the kubelet to restart the API server.
+func (u *Updater) AddHostAlias(hostname, ip string) error {
+	klog.Infof("add host %s with ip %s to /etc/hosts of the (%s) kube-apiserver", hostname, ip, u.kindContainer)
+	pod, err := u.getStaticPod()
+	if err != nil {
+		return err
+	}
+	pod.Spec.HostAliases = append(pod.Spec.HostAliases, corev1.HostAlias{
+		IP: ip,
+		Hostnames: []string{
+			hostname,
+		},
+	})
+	if err := u.writeToContainerFS(pod); err != nil {
+		return err
+	}
+	if err := u.waitForRestart(); err != nil {
+		return err
+	}
+	return nil
+}
+
+// AddNameserver adds the nameserver ip to the DNS config of the kube-apiserver (static pod) manifest
+// inside a kind container and waits for the kubelet to restart the API server.
+func (u *Updater) AddNameserver(ip string) error {
+	klog.Infof("add nameserver with ip %s (coredns) to dns config of (%s) kube-apiserver", ip, u.kindContainer)
+	pod, err := u.getStaticPod()
+	if err != nil {
+		return err
+	}
+	pod.Spec.DNSPolicy = corev1.DNSNone
+	pod.Spec.DNSConfig = &corev1.PodDNSConfig{
+		Nameservers: []string{
+			ip,
+		},
+	}
+	if err := u.writeToContainerFS(pod); err != nil {
+		return err
+	}
+	if err := u.waitForRestart(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (u *Updater) writeToContainerFS(pod *corev1.Pod) error {
+	tmpFile, err := os.CreateTemp("", "kube-apiserver.yaml")
+	if err != nil {
+		return fmt.Errorf("failed to create temp file: %w", err)
+	}
+	defer func() {
+		tmpFile.Close()
+		os.Remove(tmpFile.Name())
+	}()
+	data, err := yaml.Marshal(pod)
+	if err != nil {
+		return fmt.Errorf("failed to marshal pod to yaml: %w", err)
+	}
+	if _, err := tmpFile.Write(data); err != nil {
+		return fmt.Errorf("failed to write temp file: %w", err)
+	}
+	var stderr bytes.Buffer
+	cmd := exec.Command(u.dockerCLI, "cp", tmpFile.Name(), u.kindContainer+":"+u.apiServerManifestPath)
+	cmd.Stderr = &stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("failed to copy updated manifest to %s: %w: %s", u.kindContainer, err, stderr.String())
+	}
+	return nil
+}
+
+// retrieve the kube-apiserver manifest from the kind container filesystem.
+func (u *Updater) getStaticPod() (*corev1.Pod, error) {
+	var stdout, stderr bytes.Buffer
+	cmd := exec.Command(u.dockerCLI, "exec", u.kindContainer, "cat", u.apiServerManifestPath)
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	if err := cmd.Run(); err != nil {
+		return nil, fmt.Errorf("failed to read %s from %s: %w: %s", u.apiServerManifestPath, u.kindContainer, err, stderr.String())
+	}
+	podManifest := stdout.String()
+	pod := &corev1.Pod{}
+	if err := yaml.Unmarshal([]byte(podManifest), pod); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal pod manifest: %w", err)
+	}
+	return pod, nil
+}
+
+// waitForRestart polls the kube-apiserver /livez endpoint inside the kind container.
+// It waits for the API server to go down and come back healthy.
+func (u *Updater) waitForRestart() error {
+	timeout := time.Now().Add(u.timeout)
+	klog.Infof("wait for (%s) kube-apiserver restart...", u.kindContainer)
+	// wait for the server to become unavailable
+	for time.Now().Before(timeout) {
+		if !u.apiServerAvailable() {
+			klog.Infof("(%s) kube-apiserver unavailable", u.kindContainer)
+			break
+		}
+		klog.Infof("wait for (%s) kube-apiserver to become unavailable...", u.kindContainer)
+		time.Sleep(2 * time.Second)
+	}
+	if !time.Now().Before(timeout) {
+		return fmt.Errorf("kube-apiserver in %s did not go down within %s", u.kindContainer, u.timeout)
+	}
+	// wait for the server to become healthy again
+	for time.Now().Before(timeout) {
+		if u.apiServerAvailable() {
+			klog.Infof("(%s) kube-apiserver available", u.kindContainer)
+			return nil
+		}
+		klog.Infof("wait for (%s) kube-apiserver to become available...", u.kindContainer)
+		time.Sleep(2 * time.Second)
+	}
+	return fmt.Errorf("kube-apiserver in %s did not become healthy within %s", u.kindContainer, u.timeout)
+}
+
+func (u *Updater) apiServerAvailable() bool {
+	return exec.Command(u.dockerCLI, "exec", u.kindContainer, "curl", "--silent", "--fail", "--insecure", "https://localhost:6443/livez").Run() == nil
+}
+
+func onboardingClusterContainer() (string, error) {
+	kind := kindcluster.NewProvider()
+	clusters, err := kind.List()
+	if err != nil {
+		return "", err
+	}
+	for _, clusterName := range clusters {
+		if strings.HasPrefix(clusterName, "onboarding") {
+			nodes, err := kind.ListNodes(clusterName)
+			if err != nil {
+				return "", fmt.Errorf("failed to retrieve onboarding cluster nodes: %w", err)
+			}
+			return nodes[0].String(), nil
+		}
+	}
+	return "", errors.New("onboarding cluster not found")
+}

pkg/setup/bootstrap.go

@@ -9,7 +9,7 @@ import (
 
 	clustersv1alpha1 "github.com/openmcp-project/openmcp-operator/api/clusters/v1alpha1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	apimachinerytypes "k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/e2e-framework/klient/wait"
 	"sigs.k8s.io/e2e-framework/klient/wait/conditions"
@@ -104,10 +104,26 @@ func (s *OpenMCPSetup) cleanup(tmpFiles ...string) types.EnvFunc {
 				klog.Errorf("delete platform service failed: %v", err)
 			}
 		}
-		if err := providers.DeleteCluster(ctx, c, apimachinerytypes.NamespacedName{Namespace: s.Namespace, Name: "onboarding"},
-			s.WaitOpts...); err != nil {
-			klog.Errorf("delete cluster failed: %v", err)
+		// delete clusters
+		clusterRequests := &unstructured.UnstructuredList{}
+		clusterRequests.SetGroupVersionKind(schema.GroupVersionKind{
+			Group:   "clusters.openmcp.cloud",
+			Version: "v1alpha1",
+			Kind:    "clusterrequest",
+		})
+		if err := c.Client().Resources().List(ctx, clusterRequests); err != nil {
+			klog.Errorf("failed to retrieve cluster requests: %v", err)
 		}
+		for _, clusterRequest := range clusterRequests.Items {
+			if err := resources.DeleteObject(ctx, c, &clusterRequest, s.WaitOpts...); err != nil {
+				klog.Errorf("failed to delete cluster request: %v", err)
+			}
+			if err := wait.For(conditions.New(c.Client().Resources()).
+				ResourceDeleted(&clusterRequest), s.WaitOpts...); err != nil {
+				klog.Errorf("failed to delete cluster request: %v", err)
+			}
+		}
+		// delete cluster providers
 		for _, cp := range s.ClusterProviders {
 			if err := providers.DeleteClusterProvider(ctx, c, cp.Name, cp.WaitOpts...); err != nil {
 				klog.Errorf("delete cluster provider failed: %v", err)

pkg/setup/dns/README.md

@@ -0,0 +1,9 @@
+# DNS
+
+Package `setup/dns` provides functionality for testing webhooks in [OpenControlPlane](https://open-control-plane.io/) with [platform-service-gateway](https://github.com/openmcp-project/platform-service-gateway) and [platform-service-dns](https://github.com/openmcp-project/platform-service-dns).
+
+`dns.CreateExternalService()` creates a dedicated DNS service to test scenarios where dynamic service discovery from multiple sources like `Service`, `TLSRoute`, `HTTPRoute` is required.
+
+The configuration is based on the [CoreDNS with etcd](https://kubernetes-sigs.github.io/external-dns/latest/docs/tutorials/coredns-etcd/#overview) description of [external-dns](https://github.com/kubernetes-sigs/external-dns/).
+
+See [service_test.go](../../../e2e/serviceprovider_test.go) for an example of how to integrate it in a service provider test.

pkg/setup/dns/cluster.go

@@ -0,0 +1,90 @@
+package dns
+
+import (
+	"context"
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/e2e-framework/klient/wait"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+
+	"github.com/openmcp-project/openmcp-testing/pkg/providers"
+	"github.com/openmcp-project/openmcp-testing/pkg/resources"
+
+	openmcpconditions "github.com/openmcp-project/openmcp-testing/pkg/conditions"
+)
+
+const clusterRequestTemplate = `
+apiVersion: clusters.openmcp.cloud/v1alpha1
+kind: ClusterRequest
+metadata:
+  name: {{.Name}}
+  namespace: {{.Namespace}}
+spec:
+  purpose: {{.Purpose}}
+`
+
+const accessRequestTemplate = `
+apiVersion: clusters.openmcp.cloud/v1alpha1
+kind: AccessRequest
+metadata:
+  name: {{.Name}}
+  namespace: {{.Namespace}}
+spec:
+  requestRef:
+    name: {{.RequestName}}
+    namespace: {{.Namespace}}
+  token:
+    roleRefs:
+    - kind: ClusterRole
+      name: cluster-admin
+`
+
+type clusterRequest struct {
+	Name      string
+	Namespace string
+	Purpose   string
+}
+
+type accessRequest struct {
+	Name        string
+	Namespace   string
+	RequestName string
+}
+
+func createCluster(ctx context.Context, config *envconf.Config, cr clusterRequest) error {
+	klog.Info("create dns cluster")
+	crObj, err := resources.CreateObjectFromTemplate(ctx, config, clusterRequestTemplate, cr)
+	if err != nil {
+		return fmt.Errorf("failed to create dns cluster request: %w", err)
+	}
+	if err := wait.For(openmcpconditions.Status(crObj, config, "phase", "Granted")); err != nil {
+		return fmt.Errorf("dns cluster request failed to get ready: %w", err)
+	}
+	if err := providers.ClustersReady(ctx, config); err != nil {
+		return fmt.Errorf("dns cluster failed to get ready: %w", err)
+	}
+	ar := accessRequest{
+		Name:        cr.Name,
+		Namespace:   cr.Namespace,
+		RequestName: cr.Name,
+	}
+	arObj, err := resources.CreateObjectFromTemplate(ctx, config, accessRequestTemplate, ar)
+	if err != nil {
+		return fmt.Errorf("failed to request dns cluster access: %w", err)
+	}
+	if err := wait.For(openmcpconditions.Status(arObj, config, "phase", "Granted")); err != nil {
+		return fmt.Errorf("dns cluster access not granted: %w", err)
+	}
+	if err := wait.For(func(ctx context.Context) (bool, error) {
+		if err := config.Client().Resources().Get(ctx, ar.Name, ar.Namespace, arObj); err != nil {
+			return false, err
+		}
+		_, found, err := unstructured.NestedFieldNoCopy(arObj.Object, "status", "secretRef")
+		return found, err
+	}); err != nil {
+		return fmt.Errorf("failed to retrieve kubeconfig to access dns cluster")
+	}
+	return nil
+}