From cd19328125165006f5a21cb120890a95918d4553 Mon Sep 17 00:00:00 2001 From: dsanatar Date: Wed, 8 Jul 2026 11:05:06 -0400 Subject: [PATCH 1/2] add new overlay csi func test involves edits to sync and test.sh scripts to deploy the overlay configuration when env var is set Signed-off-by: dsanatar --- cluster-sync/sync.sh | 7 + .../hostpathprovisioner_nfs_overlay_cr.yaml | 20 ++ .../tests/storageclass_wffc_nfs_overlay.yaml | 9 + hack/test.sh | 6 + tests/prometheus_test.go | 8 +- tests/storage_pool_test.go | 217 ++++++++++++++++++ 6 files changed, 264 insertions(+), 3 deletions(-) create mode 100644 deploy/tests/hostpathprovisioner_nfs_overlay_cr.yaml create mode 100644 deploy/tests/storageclass_wffc_nfs_overlay.yaml create mode 100644 tests/storage_pool_test.go diff --git a/cluster-sync/sync.sh b/cluster-sync/sync.sh index f36516c4f..ee39bce09 100755 --- a/cluster-sync/sync.sh +++ b/cluster-sync/sync.sh @@ -89,11 +89,18 @@ _kubectl get pods -n hostpath-provisioner _kubectl patch deployment hostpath-provisioner-operator -n hostpath-provisioner --patch-file cluster-sync/patch.yaml _kubectl rollout status -n hostpath-provisioner deployment/hostpath-provisioner-operator --timeout=120s + +# allow for the webhook server to be ready +sleep 5 + HPP_CR_PATH="${OPERATOR_URL}/hostpathprovisioner_legacy_cr.yaml" HPP_CSI_SC="${OPERATOR_URL}/storageclass-wffc-legacy-csi.yaml" if [ "${KUBEVIRT_STORAGE}" == "rook-ceph-default" ] && [ "${HPP_CR_TYPE}" == "storagepool-pvc-template" ]; then HPP_CR_PATH="deploy/tests/hostpathprovisioner_ceph_pvc_pool_cr.yaml" HPP_CSI_SC="deploy/tests/storageclass_wffc_ceph_pool.yaml" +elif [ "${HPP_CR_TYPE}" == "overlay-csi" ]; then + HPP_CR_PATH="deploy/tests/hostpathprovisioner_nfs_overlay_cr.yaml" + HPP_CSI_SC="deploy/tests/storageclass_wffc_nfs_overlay.yaml" fi _kubectl apply -f $HPP_CR_PATH _kubectl apply -f ${OPERATOR_URL}/storageclass-wffc-legacy.yaml diff --git a/deploy/tests/hostpathprovisioner_nfs_overlay_cr.yaml b/deploy/tests/hostpathprovisioner_nfs_overlay_cr.yaml new file mode 100644 index 000000000..a1277bb24 --- /dev/null +++ b/deploy/tests/hostpathprovisioner_nfs_overlay_cr.yaml @@ -0,0 +1,20 @@ +apiVersion: hostpathprovisioner.kubevirt.io/v1beta1 +kind: HostPathProvisioner +metadata: + name: hostpath-provisioner +spec: + imagePullPolicy: Always + storagePools: + - name: "test-pool" + path: "/nfs-vol" + overlayClassName: "hpp-overlay" + pvcTemplate: + accessModes: + - ReadWriteMany + storageClassName: "nfs-csi" + resources: + requests: + storage: 10Gi + workload: + nodeSelector: + kubernetes.io/os: linux diff --git a/deploy/tests/storageclass_wffc_nfs_overlay.yaml b/deploy/tests/storageclass_wffc_nfs_overlay.yaml new file mode 100644 index 000000000..9f7180b0f --- /dev/null +++ b/deploy/tests/storageclass_wffc_nfs_overlay.yaml @@ -0,0 +1,9 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: hostpath-csi +provisioner: kubevirt.io.hostpath-provisioner +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer +parameters: + storagePool: test-pool diff --git a/hack/test.sh b/hack/test.sh index 7b4756ef2..91043ffba 100755 --- a/hack/test.sh +++ b/hack/test.sh @@ -17,6 +17,12 @@ set -e readonly ARTIFACTS_PATH="${ARTIFACTS}" export KUBEVIRT_NUM_NODES=2 export KUBEVIRT_DEPLOY_PROMETHEUS=true + +if [ "${HPP_CR_TYPE}" == "overlay-csi" ]; then + export KUBEVIRT_DEPLOY_NFS_CSI=true + export KUBEVIRT_NFS_DIR=/var/lib/containers/nfs-data +fi + make cluster-down make cluster-up diff --git a/tests/prometheus_test.go b/tests/prometheus_test.go index ac268af2a..08b50ca56 100644 --- a/tests/prometheus_test.go +++ b/tests/prometheus_test.go @@ -88,6 +88,9 @@ type promMetric struct { func TestPrometheusMetrics(t *testing.T) { k8sClient, _, token := prometheusTestSetup(t) + hppCrType := os.Getenv("HPP_CR_TYPE") + isNFSOverlay := hppCrType == "overlay-csi" + // Wait for Prometheus to scrape metrics at least once waitForPrometheusMetrics(token) @@ -100,10 +103,9 @@ func TestPrometheusMetrics(t *testing.T) { t.Run("HPP pool sharing path with OS", func(t *testing.T) { promRulePoolShared := "0" backingStorage := os.Getenv("KUBEVIRT_STORAGE") - hppCrType := os.Getenv("HPP_CR_TYPE") // Our only CI setup that avoids sharing path with OS // is a backing rook-ceph-block PVC of the HPP storage pool - shared := backingStorage != "rook-ceph-default" || hppCrType != "storagepool-pvc-template" + shared := backingStorage != "rook-ceph-default" || (hppCrType != "storagepool-pvc-template" && !isNFSOverlay) if shared { promRulePoolShared = "1" } @@ -160,7 +162,7 @@ func TestPrometheusAlerts(t *testing.T) { t.Run("HPPSharingPoolPathWithOS", func(t *testing.T) { backingStorage := os.Getenv("KUBEVIRT_STORAGE") hppCrType := os.Getenv("HPP_CR_TYPE") - if backingStorage == "rook-ceph-default" && hppCrType == "storagepool-pvc-template" { + if (backingStorage == "rook-ceph-default" && hppCrType == "storagepool-pvc-template") || hppCrType == "overlay-csi" { t.Skip("HPP pool is not shared with OS in this CI config") } diff --git a/tests/storage_pool_test.go b/tests/storage_pool_test.go new file mode 100644 index 000000000..23178d9e5 --- /dev/null +++ b/tests/storage_pool_test.go @@ -0,0 +1,217 @@ +/* +Copyright 2024 The hostpath provisioner Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package tests + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" + hostpathprovisionerv1 "kubevirt.io/hostpath-provisioner-operator/pkg/apis/hostpathprovisioner/v1beta1" +) + +const ( + hppNamespace = "hostpath-provisioner" + hppCRName = "hostpath-provisioner" + poolMountName = "pool-volume" +) + +// sharedPoolPVCName returns the pool PVC name for a shared (RWX) storage pool. +func sharedPoolPVCName(poolName string) string { + return fmt.Sprintf("hpp-pool-%s-shared", poolName) +} + +// findNFSOverlayPool returns the first storage pool backed by an NFS PVC with an +// overlay class configured, or nil if none exists. +func findNFSOverlayPool(k8sClient *kubernetes.Clientset) *hostpathprovisionerv1.StoragePool { + hppClient, err := getHPPClient() + Expect(err).ToNot(HaveOccurred()) + cr, err := hppClient.HostpathprovisionerV1beta1().HostPathProvisioners().Get(context.TODO(), "hostpath-provisioner", metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + + for i, pool := range cr.Spec.StoragePools { + if pool.PVCTemplate == nil || pool.OverlayClassName == "" || pool.PVCTemplate.StorageClassName == nil { + continue + } + sc, err := k8sClient.StorageV1().StorageClasses().Get(context.TODO(), *pool.PVCTemplate.StorageClassName, metav1.GetOptions{}) + if err != nil { + continue + } + if strings.Contains(sc.Provisioner, "nfs") { + return &cr.Spec.StoragePools[i] + } + } + return nil +} + +// TestNFSStoragePoolMounterRecovery verifies that the hpp-pool mounter pod recovers correctly +// when multiple pods are mounting the same NFS-backed pool PVC simultaneously. +// This is a regression test for https://github.com/kubevirt/hostpath-provisioner-operator/issues/721. +func TestNFSStoragePoolMounterRecovery(t *testing.T) { + RegisterTestingT(t) + + k8sClient, err := getKubeClient() + Expect(err).ToNot(HaveOccurred()) + + pool := findNFSOverlayPool(k8sClient) + if pool == nil { + t.Skip("No NFS-backed overlay storage pool found — skipping") + } + t.Logf("Found NFS overlay pool: %s", pool.Name) + + pvcName := sharedPoolPVCName(pool.Name) + poolPVC, err := k8sClient.CoreV1().PersistentVolumeClaims(hppNamespace).Get(context.TODO(), pvcName, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + t.Logf("Pool PVC: %s", poolPVC.Name) + + // Find a node running an hpp-pool pod for this pool. + poolPods, err := k8sClient.CoreV1().Pods(hppNamespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: fmt.Sprintf("k8s-app=hostpath-provisioner,kubevirt.io.hostpath-provisioner/storagePool=%s-hpp", pool.Name), + }) + Expect(err).ToNot(HaveOccurred()) + if len(poolPods.Items) == 0 { + t.Skip("No hpp-pool pods found for pool — skipping") + } + targetNode := poolPods.Items[0].Spec.NodeName + t.Logf("Targeting node: %s", targetNode) + + // Create a second pod that directly mounts the pool PVC on the same node. + // This simulates the scenario from issue #721 where multiple pod mounts of + // the same NFS source cause the mounter to crash on restart. + consumerPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "pool-consumer-test-", + Namespace: hppNamespace, + }, + Spec: corev1.PodSpec{ + NodeName: targetNode, + RestartPolicy: corev1.RestartPolicyNever, + Containers: []corev1.Container{ + { + Name: "consumer", + Image: "quay.io/kubevirt/cdi-importer:latest-amd64", + Command: []string{"/bin/sh", "-c", "sleep infinity"}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: poolMountName, + MountPath: "/mnt/pool", + }, + }, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + RunAsNonRoot: ptr.To(true), + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: poolMountName, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: poolPVC.Name, + }, + }, + }, + }, + }, + } + consumerPod, err = k8sClient.CoreV1().Pods(hppNamespace).Create(context.TODO(), consumerPod, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + defer func() { + t.Logf("Cleaning up consumer pod: %s", consumerPod.Name) + _ = k8sClient.CoreV1().Pods(hppNamespace).Delete(context.TODO(), consumerPod.Name, metav1.DeleteOptions{}) + }() + + t.Logf("Waiting for consumer pod %s to be Running", consumerPod.Name) + Eventually(func() corev1.PodPhase { + consumerPod, err = k8sClient.CoreV1().Pods(hppNamespace).Get(context.TODO(), consumerPod.Name, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + return consumerPod.Status.Phase + }, 60*time.Second, 2*time.Second).Should(Equal(corev1.PodRunning)) + + // Delete the hpp-pool mounter pod on the target node to force a restart. + // With multiple pod mounts on the same NFS source, the mounter must correctly + // identify the bind source and not crash. + originalPodName := poolPods.Items[0].Name + t.Logf("Deleting hpp-pool pod %s to trigger restart", originalPodName) + err = k8sClient.CoreV1().Pods(hppNamespace).Delete(context.TODO(), originalPodName, metav1.DeleteOptions{}) + Expect(err).ToNot(HaveOccurred()) + + // Verify the new mounter pod comes up Running and does not enter CrashLoopBackOff. + t.Logf("Waiting for hpp-pool pod on %s to recover", targetNode) + Eventually(func() bool { + pods, err := k8sClient.CoreV1().Pods(hppNamespace).List(context.TODO(), metav1.ListOptions{ + LabelSelector: fmt.Sprintf("k8s-app=hostpath-provisioner,kubevirt.io.hostpath-provisioner/storagePool=%s-hpp", pool.Name), + FieldSelector: fmt.Sprintf("spec.nodeName=%s", targetNode), + }) + if err != nil { + return false + } + for _, pod := range pods.Items { + if pod.Name == originalPodName { + continue // still the old pod being terminated + } + if pod.Status.Phase == corev1.PodRunning { + t.Logf("hpp-pool pod %s is Running", pod.Name) + return true + } + // Fail fast if it enters CrashLoopBackOff + for _, cs := range pod.Status.ContainerStatuses { + if cs.State.Waiting != nil && cs.State.Waiting.Reason == "CrashLoopBackOff" { + t.Errorf("hpp-pool mounter entered CrashLoopBackOff: %s", cs.State.Waiting.Message) + return false + } + } + } + return false + }, 120*time.Second, 2*time.Second).Should(BeTrue(), "hpp-pool mounter should recover to Running with multiple NFS pod mounts present") + + // Verify the overlay storage class can still provision PVCs after mounter recovery. + // This confirms the bind mount at the pool path was correctly re-established. + t.Logf("Verifying PVC provisioning from overlay class %s after recovery", pool.OverlayClassName) + overlayPVC := createPVCDef(hppNamespace, pool.OverlayClassName, map[string]string{}) + overlayPVC, err = k8sClient.CoreV1().PersistentVolumeClaims(hppNamespace).Create(context.TODO(), overlayPVC, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + defer func() { + t.Logf("Cleaning up overlay PVC: %s", overlayPVC.Name) + _ = k8sClient.CoreV1().PersistentVolumeClaims(hppNamespace).Delete(context.TODO(), overlayPVC.Name, metav1.DeleteOptions{}) + }() + + overlayPod := createPodUsingPVCWithCommand(hppNamespace, "overlay-verify-pod", overlayPVC, "sleep 1", map[string]string{}) + overlayPod, err = k8sClient.CoreV1().Pods(hppNamespace).Create(context.TODO(), overlayPod, metav1.CreateOptions{}) + Expect(err).ToNot(HaveOccurred()) + defer func() { + t.Logf("Cleaning up overlay pod: %s", overlayPod.Name) + _ = k8sClient.CoreV1().Pods(hppNamespace).Delete(context.TODO(), overlayPod.Name, metav1.DeleteOptions{}) + }() + + Eventually(func() bool { + overlayPod, err = k8sClient.CoreV1().Pods(hppNamespace).Get(context.TODO(), overlayPod.Name, metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + return overlayPod.Status.Phase == corev1.PodRunning || overlayPod.Status.Phase == corev1.PodSucceeded + }, 90*time.Second, 2*time.Second).Should(BeTrue(), "pod using overlay PVC should run successfully after mounter recovery") + t.Logf("Overlay PVC provisioning and pod scheduling verified successfully") +} From 5e003150adbf7ac9cbbbb57cc61e0b117ade79e1 Mon Sep 17 00:00:00 2001 From: dsanatar Date: Thu, 9 Jul 2026 09:37:50 -0400 Subject: [PATCH 2/2] extend test to verify bind mount on host Signed-off-by: dsanatar --- tests/storage_pool_test.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/storage_pool_test.go b/tests/storage_pool_test.go index 23178d9e5..069186da3 100644 --- a/tests/storage_pool_test.go +++ b/tests/storage_pool_test.go @@ -33,7 +33,6 @@ import ( const ( hppNamespace = "hostpath-provisioner" - hppCRName = "hostpath-provisioner" poolMountName = "pool-volume" ) @@ -65,6 +64,15 @@ func findNFSOverlayPool(k8sClient *kubernetes.Clientset) *hostpathprovisionerv1. return nil } +// exec into the mounter pod to retrive total block size of a mount pount +func getMountSize(mounterPod, mountPath string) string { + blockSize, err := RunKubeCtlCommand("exec", "-n", hppNamespace, mounterPod, "-c", "mounter", "--", + "/bin/sh", "-c", fmt.Sprintf("df %s | sed 1d | awk '{print $2}'", mountPath)) + Expect(err).ToNot(HaveOccurred()) + Expect(blockSize).ToNot(BeEmpty()) + return blockSize +} + // TestNFSStoragePoolMounterRecovery verifies that the hpp-pool mounter pod recovers correctly // when multiple pods are mounting the same NFS-backed pool PVC simultaneously. // This is a regression test for https://github.com/kubevirt/hostpath-provisioner-operator/issues/721. @@ -162,6 +170,7 @@ func TestNFSStoragePoolMounterRecovery(t *testing.T) { // Verify the new mounter pod comes up Running and does not enter CrashLoopBackOff. t.Logf("Waiting for hpp-pool pod on %s to recover", targetNode) + newPoolPod := "" Eventually(func() bool { pods, err := k8sClient.CoreV1().Pods(hppNamespace).List(context.TODO(), metav1.ListOptions{ LabelSelector: fmt.Sprintf("k8s-app=hostpath-provisioner,kubevirt.io.hostpath-provisioner/storagePool=%s-hpp", pool.Name), @@ -176,6 +185,7 @@ func TestNFSStoragePoolMounterRecovery(t *testing.T) { } if pod.Status.Phase == corev1.PodRunning { t.Logf("hpp-pool pod %s is Running", pod.Name) + newPoolPod = pod.Name return true } // Fail fast if it enters CrashLoopBackOff @@ -189,6 +199,12 @@ func TestNFSStoragePoolMounterRecovery(t *testing.T) { return false }, 120*time.Second, 2*time.Second).Should(BeTrue(), "hpp-pool mounter should recover to Running with multiple NFS pod mounts present") + // Check that the directory bind mounted on the host is the actual nfs share by comparing the total block sizes + t.Logf("Verifying bind mount size matches the NFS share") + hostMountSize := getMountSize(newPoolPod, fmt.Sprintf("/host/%s/csi", pool.Path)) + sourceMountSize := getMountSize(newPoolPod, "/source") + Expect(hostMountSize).To(Equal(sourceMountSize)) + // Verify the overlay storage class can still provision PVCs after mounter recovery. // This confirms the bind mount at the pool path was correctly re-established. t.Logf("Verifying PVC provisioning from overlay class %s after recovery", pool.OverlayClassName)