From d00c07fe46c34d1f8a086a8afcd0ae4748766b89 Mon Sep 17 00:00:00 2001 From: Simon Croome Date: Wed, 13 Jan 2021 13:20:01 +0000 Subject: [PATCH 1/5] Remove support for k8s 1.14 and earlier and NFSv1 --- deploy/storageos-operators.configmap.yaml | 22 +- internal/pkg/image/image.go | 52 +-- internal/pkg/image/image_test.go | 17 +- .../storageos/v1/storageoscluster_types.go | 103 +---- pkg/controller/add_nfsserver.go | 10 - .../nfsserver/nfsserver_controller.go | 278 ------------- .../nfsserver/nfsserver_controller_test.go | 171 -------- .../storageoscluster_controller.go | 14 +- pkg/nfs/configmap.go | 198 ---------- pkg/nfs/configmap_test.go | 279 ------------- pkg/nfs/delete.go | 42 -- pkg/nfs/delete_test.go | 147 ------- pkg/nfs/deploy.go | 171 -------- pkg/nfs/deploy_test.go | 98 ----- pkg/nfs/deployment.go | 41 -- pkg/nfs/pvc.go | 22 -- pkg/nfs/service.go | 68 ---- pkg/nfs/statefulset.go | 128 ------ pkg/nfs/status.go | 105 ----- pkg/storageos/configmap.go | 10 +- pkg/storageos/configmap_test.go | 28 +- pkg/storageos/csi_helper.go | 64 +-- pkg/storageos/delete.go | 8 - pkg/storageos/deploy.go | 90 ++--- pkg/storageos/deploy_test.go | 369 ++++++++---------- pkg/storageos/podspec.go | 255 ++++++------ pkg/storageos/rbac.go | 30 -- pkg/storageos/storageclass.go | 62 +-- test/e2e/clusterCSINodeV2_test.go | 18 +- 29 files changed, 383 insertions(+), 2517 deletions(-) delete mode 100644 pkg/controller/add_nfsserver.go delete mode 100644 pkg/controller/nfsserver/nfsserver_controller.go delete mode 100644 pkg/controller/nfsserver/nfsserver_controller_test.go delete mode 100644 pkg/nfs/configmap.go delete mode 100644 pkg/nfs/configmap_test.go delete mode 100644 pkg/nfs/delete.go delete mode 100644 pkg/nfs/delete_test.go delete mode 100644 pkg/nfs/deploy.go delete mode 100644 pkg/nfs/deploy_test.go delete mode 100644 pkg/nfs/deployment.go delete mode 100644 pkg/nfs/pvc.go delete mode 100644 pkg/nfs/service.go delete mode 100644 pkg/nfs/statefulset.go delete mode 100644 pkg/nfs/status.go diff --git a/deploy/storageos-operators.configmap.yaml b/deploy/storageos-operators.configmap.yaml index afcc72d3c..50b465352 100644 --- a/deploy/storageos-operators.configmap.yaml +++ b/deploy/storageos-operators.configmap.yaml @@ -1283,27 +1283,15 @@ data: value: "" - name: RELATED_IMAGE_STORAGEOS_INIT value: "" - - name: RELATED_IMAGE_CSIV1_CLUSTER_DRIVER_REGISTRAR + - name: RELATED_IMAGE_CSI_NODE_DRIVER_REGISTRAR value: "" - - name: RELATED_IMAGE_CSIV1_NODE_DRIVER_REGISTRAR + - name: RELATED_IMAGE_CSI_EXTERNAL_PROVISIONER value: "" - - name: RELATED_IMAGE_CSIV1_EXTERNAL_PROVISIONER + - name: RELATED_IMAGE_CSI_EXTERNAL_ATTACHER value: "" - - name: RELATED_IMAGE_CSIV1_EXTERNAL_ATTACHER + - name: RELATED_IMAGE_CSI_EXTERNAL_RESIZER value: "" - - name: RELATED_IMAGE_CSIV1_EXTERNAL_ATTACHER_V2 - value: "" - - name: RELATED_IMAGE_CSIV1_EXTERNAL_RESIZER - value: "" - - name: RELATED_IMAGE_CSIV1_LIVENESS_PROBE - value: "" - - name: RELATED_IMAGE_CSIV0_DRIVER_REGISTRAR - value: "" - - name: RELATED_IMAGE_CSIV0_EXTERNAL_PROVISIONER - value: "" - - name: RELATED_IMAGE_CSIV0_EXTERNAL_ATTACHER - value: "" - - name: RELATED_IMAGE_NFS + - name: RELATED_IMAGE_CSI_LIVENESS_PROBE value: "" - name: RELATED_IMAGE_KUBE_SCHEDULER value: "" diff --git a/internal/pkg/image/image.go b/internal/pkg/image/image.go index aca35d35b..9822a261f 100644 --- a/internal/pkg/image/image.go +++ b/internal/pkg/image/image.go @@ -4,47 +4,29 @@ import "os" // Default image constant variables. const ( - DefaultNodeContainerImage = "storageos/node:v2.3.1" - DefaultInitContainerImage = "storageos/init:v2.1.0" - CSIv1ClusterDriverRegistrarContainerImage = "quay.io/k8scsi/csi-cluster-driver-registrar:v1.0.1" - CSIv1NodeDriverRegistrarContainerImage = "quay.io/k8scsi/csi-node-driver-registrar:v1.2.0" - CSIv1ExternalProvisionerContainerImageV1 = "storageos/csi-provisioner:v1.4.0" - CSIv1ExternalProvisionerContainerImageV2 = "storageos/csi-provisioner:v1.6.0-patched" - CSIv1ExternalAttacherContainerImage = "quay.io/k8scsi/csi-attacher:v1.2.1" - CSIv1ExternalAttacherv2ContainerImage = "quay.io/k8scsi/csi-attacher:v2.2.0" - CSIv1ExternalResizerContainerImage = "quay.io/k8scsi/csi-resizer:v0.5.0" - CSIv1LivenessProbeContainerImage = "quay.io/k8scsi/livenessprobe:v1.1.0" - CSIv0DriverRegistrarContainerImage = "quay.io/k8scsi/driver-registrar:v0.4.2" - CSIv0ExternalProvisionerContainerImage = "storageos/csi-provisioner:v0.4.3" - CSIv0ExternalAttacherContainerImage = "quay.io/k8scsi/csi-attacher:v0.4.2" - DefaultNFSContainerImage = "storageos/nfs:1.0.0" - DefaultAPIManagerImage = "storageos/api-manager:v1.0.0" - - DefaultHyperkubeContainerRegistry = "gcr.io/google_containers/hyperkube" + DefaultNodeContainerImage = "storageos/node:v2.3.1" + DefaultInitContainerImage = "storageos/init:v2.1.0" + CSINodeDriverRegistrarContainerImage = "quay.io/k8scsi/csi-node-driver-registrar:v1.2.0" + CSIExternalProvisionerContainerImage = "storageos/csi-provisioner:v1.6.0-patched" + CSIExternalAttacherContainerImage = "quay.io/k8scsi/csi-attacher:v2.2.0" + CSIExternalResizerContainerImage = "quay.io/k8scsi/csi-resizer:v0.5.0" + CSILivenessProbeContainerImage = "quay.io/k8scsi/livenessprobe:v1.1.0" + DefaultAPIManagerImage = "storageos/api-manager:v1.0.0" DefaultKubeSchedulerContainerRegistry = "k8s.gcr.io/kube-scheduler" ) // Environment variables for setting default images. const ( - StorageOSNodeImageEnvVar = "RELATED_IMAGE_STORAGEOS_NODE" - StorageOSInitImageEnvVar = "RELATED_IMAGE_STORAGEOS_INIT" - - CSIv1ClusterDriverRegistrarImageEnvVar = "RELATED_IMAGE_CSIV1_CLUSTER_DRIVER_REGISTRAR" - CSIv1NodeDriverRegistrarImageEnvVar = "RELATED_IMAGE_CSIV1_NODE_DRIVER_REGISTRAR" - CSIv1ExternalProvisionerImageEnvVar = "RELATED_IMAGE_CSIV1_EXTERNAL_PROVISIONER" - CSIv1ExternalAttacherImageEnvVar = "RELATED_IMAGE_CSIV1_EXTERNAL_ATTACHER" - CSIv1ExternalAttacherv2ImageEnvVar = "RELATED_IMAGE_CSIV1_EXTERNAL_ATTACHER_V2" - CSIv1ExternalResizerContainerImageEnvVar = "RELATED_IMAGE_CSIV1_EXTERNAL_RESIZER" - CSIv1LivenessProbeImageEnvVar = "RELATED_IMAGE_CSIV1_LIVENESS_PROBE" - - CSIv0DriverRegistrarImageEnvVar = "RELATED_IMAGE_CSIV0_DRIVER_REGISTRAR" - CSIv0ExternalProvisionerImageEnvVar = "RELATED_IMAGE_CSIV0_EXTERNAL_PROVISIONER" - CSIv0ExternalAttacherImageEnvVar = "RELATED_IMAGE_CSIV0_EXTERNAL_ATTACHER" - - NFSImageEnvVar = "RELATED_IMAGE_NFS" - KubeSchedulerImageEnvVar = "RELATED_IMAGE_KUBE_SCHEDULER" - APIManagerEnvVar = "RELATED_IMAGE_API_MANAGER" + StorageOSNodeImageEnvVar = "RELATED_IMAGE_STORAGEOS_NODE" + StorageOSInitImageEnvVar = "RELATED_IMAGE_STORAGEOS_INIT" + CSINodeDriverRegistrarImageEnvVar = "RELATED_IMAGE_CSI_NODE_DRIVER_REGISTRAR" + CSIExternalProvisionerImageEnvVar = "RELATED_IMAGE_CSI_EXTERNAL_PROVISIONER" + CSIExternalAttacherImageEnvVar = "RELATED_IMAGE_CSI_EXTERNAL_ATTACHER" + CSIExternalResizerContainerImageEnvVar = "RELATED_IMAGE_CSI_EXTERNAL_RESIZER" + CSILivenessProbeImageEnvVar = "RELATED_IMAGE_CSI_LIVENESS_PROBE" + KubeSchedulerImageEnvVar = "RELATED_IMAGE_KUBE_SCHEDULER" + APIManagerEnvVar = "RELATED_IMAGE_API_MANAGER" ) // GetDefaultImage checks the environment variable for an image. If not found, diff --git a/internal/pkg/image/image_test.go b/internal/pkg/image/image_test.go index ca93040b7..06404b7f4 100644 --- a/internal/pkg/image/image_test.go +++ b/internal/pkg/image/image_test.go @@ -6,7 +6,6 @@ import ( ) func TestGetDefaultImage(t *testing.T) { - fakeNFSDefaultImage := "nfs/foo:1" fakeStorageOSNodeImage := "stos/foo:1" testcases := []struct { @@ -18,15 +17,12 @@ func TestGetDefaultImage(t *testing.T) { { name: "images from env vars", envVars: map[string]string{ - NFSImageEnvVar: fakeNFSDefaultImage, StorageOSNodeImageEnvVar: fakeStorageOSNodeImage, }, defaultImages: map[string]string{ - NFSImageEnvVar: DefaultNFSContainerImage, StorageOSNodeImageEnvVar: DefaultNodeContainerImage, }, wantImages: map[string]string{ - NFSImageEnvVar: fakeNFSDefaultImage, StorageOSNodeImageEnvVar: fakeStorageOSNodeImage, }, }, @@ -34,29 +30,24 @@ func TestGetDefaultImage(t *testing.T) { name: "images not in env var", envVars: map[string]string{}, defaultImages: map[string]string{ - NFSImageEnvVar: DefaultNFSContainerImage, StorageOSNodeImageEnvVar: DefaultNodeContainerImage, }, wantImages: map[string]string{ - NFSImageEnvVar: DefaultNFSContainerImage, StorageOSNodeImageEnvVar: DefaultNodeContainerImage, }, }, { name: "some images in env var and some defaults", envVars: map[string]string{ - NFSImageEnvVar: fakeNFSDefaultImage, StorageOSNodeImageEnvVar: fakeStorageOSNodeImage, }, defaultImages: map[string]string{ - NFSImageEnvVar: DefaultNFSContainerImage, - StorageOSNodeImageEnvVar: DefaultNodeContainerImage, - CSIv1LivenessProbeImageEnvVar: CSIv1LivenessProbeContainerImage, + StorageOSNodeImageEnvVar: DefaultNodeContainerImage, + CSILivenessProbeImageEnvVar: CSILivenessProbeContainerImage, }, wantImages: map[string]string{ - NFSImageEnvVar: fakeNFSDefaultImage, - StorageOSNodeImageEnvVar: fakeStorageOSNodeImage, - CSIv1LivenessProbeImageEnvVar: CSIv1LivenessProbeContainerImage, + StorageOSNodeImageEnvVar: fakeStorageOSNodeImage, + CSILivenessProbeImageEnvVar: CSILivenessProbeContainerImage, }, }, } diff --git a/pkg/apis/storageos/v1/storageoscluster_types.go b/pkg/apis/storageos/v1/storageoscluster_types.go index bc8e155a2..1da5d8aec 100644 --- a/pkg/apis/storageos/v1/storageoscluster_types.go +++ b/pkg/apis/storageos/v1/storageoscluster_types.go @@ -49,7 +49,6 @@ const ( DefaultIngressHostname = "storageos.local" DefaultPluginRegistrationPath = "/var/lib/kubelet/plugins_registry" - OldPluginRegistrationPath = "/var/lib/kubelet/plugins" DefaultCSIEndpoint = "/storageos/csi.sock" DefaultCSIRegistrarSocketDir = "/var/lib/kubelet/device-plugins/" @@ -292,50 +291,28 @@ func (s StorageOSClusterSpec) GetInitContainerImage() string { } // GetCSINodeDriverRegistrarImage returns CSI node driver registrar container image. -func (s StorageOSClusterSpec) GetCSINodeDriverRegistrarImage(csiv1 bool) string { +func (s StorageOSClusterSpec) GetCSINodeDriverRegistrarImage() string { if s.Images.CSINodeDriverRegistrarContainer != "" { return s.Images.CSINodeDriverRegistrarContainer } - if csiv1 { - return image.GetDefaultImage(image.CSIv1NodeDriverRegistrarImageEnvVar, image.CSIv1NodeDriverRegistrarContainerImage) - } - return image.GetDefaultImage(image.CSIv0DriverRegistrarImageEnvVar, image.CSIv0DriverRegistrarContainerImage) -} - -// GetCSIClusterDriverRegistrarImage returns CSI cluster driver registrar -// container image. -func (s StorageOSClusterSpec) GetCSIClusterDriverRegistrarImage() string { - if s.Images.CSIClusterDriverRegistrarContainer != "" { - return s.Images.CSIClusterDriverRegistrarContainer - } - return image.GetDefaultImage(image.CSIv1ClusterDriverRegistrarImageEnvVar, image.CSIv1ClusterDriverRegistrarContainerImage) + return image.GetDefaultImage(image.CSINodeDriverRegistrarImageEnvVar, image.CSINodeDriverRegistrarContainerImage) } // GetCSIExternalProvisionerImage returns CSI external provisioner container image. -func (s StorageOSClusterSpec) GetCSIExternalProvisionerImage(csiv1 bool) string { +func (s StorageOSClusterSpec) GetCSIExternalProvisionerImage() string { if s.Images.CSIExternalProvisionerContainer != "" { return s.Images.CSIExternalProvisionerContainer } - if csiv1 { - return image.GetDefaultImage(image.CSIv1ExternalProvisionerImageEnvVar, image.CSIv1ExternalProvisionerContainerImageV2) - } - return image.GetDefaultImage(image.CSIv0ExternalProvisionerImageEnvVar, image.CSIv0ExternalProvisionerContainerImage) + return image.GetDefaultImage(image.CSIExternalProvisionerImageEnvVar, image.CSIExternalProvisionerContainerImage) } // GetCSIExternalAttacherImage returns CSI external attacher container image. -// CSI v0, CSI v1 on k8s 1.13 and CSI v1 on k8s 1.14+ require different versions -// of external attacher. -func (s StorageOSClusterSpec) GetCSIExternalAttacherImage(csiv1 bool, attacherv2Supported bool) string { +// Require different versions of external attacher. +func (s StorageOSClusterSpec) GetCSIExternalAttacherImage() string { if s.Images.CSIExternalAttacherContainer != "" { return s.Images.CSIExternalAttacherContainer } - if csiv1 { - if attacherv2Supported { - return image.GetDefaultImage(image.CSIv1ExternalAttacherv2ImageEnvVar, image.CSIv1ExternalAttacherv2ContainerImage) - } - return image.GetDefaultImage(image.CSIv1ExternalAttacherImageEnvVar, image.CSIv1ExternalAttacherContainerImage) - } - return image.GetDefaultImage(image.CSIv0ExternalAttacherImageEnvVar, image.CSIv0ExternalAttacherContainerImage) + return image.GetDefaultImage(image.CSIExternalAttacherImageEnvVar, image.CSIExternalAttacherContainerImage) } // GetCSIExternalResizerImage returns CSI external resizer container image. @@ -343,7 +320,7 @@ func (s StorageOSClusterSpec) GetCSIExternalResizerImage() string { if s.Images.CSIExternalResizerContainer != "" { return s.Images.CSIExternalResizerContainer } - return image.GetDefaultImage(image.CSIv1ExternalResizerContainerImageEnvVar, image.CSIv1ExternalResizerContainerImage) + return image.GetDefaultImage(image.CSIExternalResizerContainerImageEnvVar, image.CSIExternalResizerContainerImage) } // GetCSILivenessProbeImage returns CSI liveness probe container image. @@ -351,21 +328,7 @@ func (s StorageOSClusterSpec) GetCSILivenessProbeImage() string { if s.Images.CSILivenessProbeContainer != "" { return s.Images.CSILivenessProbeContainer } - return image.GetDefaultImage(image.CSIv1LivenessProbeImageEnvVar, image.CSIv1LivenessProbeContainerImage) -} - -// GetHyperkubeImage returns hyperkube container image for a given k8s version. -// If an image is set explicitly in the cluster configuration, that image is -// returned. -func (s StorageOSClusterSpec) GetHyperkubeImage(k8sVersion string) string { - if s.Images.HyperkubeContainer != "" { - return s.Images.HyperkubeContainer - } - - // NOTE: Hyperkube is not being used anywhere for now. Hyperkube image is - // not available to be set via environment variable. - // Add version prefix "v" in the tag. - return fmt.Sprintf("%s:v%s", image.DefaultHyperkubeContainerRegistry, k8sVersion) + return image.GetDefaultImage(image.CSILivenessProbeImageEnvVar, image.CSILivenessProbeContainerImage) } // GetKubeSchedulerImage returns kube-scheduler container image for a given k8s @@ -387,15 +350,6 @@ func (s StorageOSClusterSpec) GetKubeSchedulerImage(k8sVersion string) string { return image.GetDefaultImage(image.KubeSchedulerImageEnvVar, fallbackImage) } -// GetNFSServerImage returns NFS server container image used as the default -// image in the cluster. -func (s StorageOSClusterSpec) GetNFSServerImage() string { - if s.Images.NFSContainer != "" { - return s.Images.NFSContainer - } - return image.GetDefaultImage(image.NFSImageEnvVar, image.DefaultNFSContainerImage) -} - // GetAPIManagerImage returns the API Manager container image used as the // default image in the cluster. func (s StorageOSClusterSpec) GetAPIManagerImage() string { @@ -446,14 +400,11 @@ func (s StorageOSClusterSpec) GetIngressHostname() string { } // GetCSIEndpoint returns the CSI unix socket endpoint path. -func (s StorageOSClusterSpec) GetCSIEndpoint(csiv1 bool) string { +func (s StorageOSClusterSpec) GetCSIEndpoint() string { if s.CSI.Endpoint != "" { return s.CSI.Endpoint } - if csiv1 { - return getDefaultCSIEndpoint(DefaultPluginRegistrationPath) - } - return getDefaultCSIEndpoint(OldPluginRegistrationPath) + return getDefaultCSIEndpoint(DefaultPluginRegistrationPath) } // GetCSIRegistrarSocketDir returns the CSI registrar socket dir. @@ -473,14 +424,11 @@ func (s StorageOSClusterSpec) GetCSIKubeletDir() string { } // GetCSIPluginDir returns the CSI plugin dir. -func (s StorageOSClusterSpec) GetCSIPluginDir(csiv1 bool) string { +func (s StorageOSClusterSpec) GetCSIPluginDir() string { if s.CSI.PluginDir != "" { return s.CSI.PluginDir } - if csiv1 { - return getDefaultCSIPluginDir(DefaultPluginRegistrationPath) - } - return getDefaultCSIPluginDir(OldPluginRegistrationPath) + return getDefaultCSIPluginDir(DefaultPluginRegistrationPath) } // GetCSIDeviceDir returns the CSI device dir. @@ -492,26 +440,19 @@ func (s StorageOSClusterSpec) GetCSIDeviceDir() string { } // GetCSIRegistrationDir returns the CSI registration dir. -func (s StorageOSClusterSpec) GetCSIRegistrationDir(csiv1 bool) string { +func (s StorageOSClusterSpec) GetCSIRegistrationDir() string { if s.CSI.RegistrationDir != "" { return s.CSI.RegistrationDir } - if csiv1 { - return DefaultCSIRegistrationDir - } - // CSI Registration Dir and Plugin Registration Path are the same. - return OldPluginRegistrationPath + return DefaultCSIRegistrationDir } // GetCSIKubeletRegistrationPath returns the CSI Kubelet Registration Path. -func (s StorageOSClusterSpec) GetCSIKubeletRegistrationPath(csiv1 bool) string { +func (s StorageOSClusterSpec) GetCSIKubeletRegistrationPath() string { if s.CSI.KubeletRegistrationPath != "" { return s.CSI.KubeletRegistrationPath } - if csiv1 { - return getDefaultCSIKubeletRegistrationPath(DefaultPluginRegistrationPath) - } - return getDefaultCSIKubeletRegistrationPath(OldPluginRegistrationPath) + return getDefaultCSIKubeletRegistrationPath(DefaultPluginRegistrationPath) } // GetCSIDriverRegistrationMode returns the CSI Driver Registration Mode. @@ -531,14 +472,8 @@ func (s StorageOSClusterSpec) GetCSIDriverRequiresAttachment() string { } // GetCSIVersion returns the CSI Driver version. -func (s StorageOSClusterSpec) GetCSIVersion(csiv1 bool) string { - if s.CSI.Version != "" { - return s.CSI.Version - } - if csiv1 { - return "v1" - } - return "v0" +func (s StorageOSClusterSpec) GetCSIVersion() string { + return "v1" } // GetCSIDeploymentStrategy returns the CSI helper deployment strategy value. diff --git a/pkg/controller/add_nfsserver.go b/pkg/controller/add_nfsserver.go deleted file mode 100644 index cae80a3be..000000000 --- a/pkg/controller/add_nfsserver.go +++ /dev/null @@ -1,10 +0,0 @@ -package controller - -import ( - "github.com/storageos/cluster-operator/pkg/controller/nfsserver" -) - -func init() { - // AddToManagerFuncs is a list of functions to create controllers and add them to a manager. - AddToManagerFuncs = append(AddToManagerFuncs, nfsserver.Add) -} diff --git a/pkg/controller/nfsserver/nfsserver_controller.go b/pkg/controller/nfsserver/nfsserver_controller.go deleted file mode 100644 index 16b95164c..000000000 --- a/pkg/controller/nfsserver/nfsserver_controller.go +++ /dev/null @@ -1,278 +0,0 @@ -package nfsserver - -import ( - "context" - goerrors "errors" - "strings" - "time" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/record" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/handler" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - - "github.com/storageos/cluster-operator/internal/pkg/storageoscluster" - storageosv1 "github.com/storageos/cluster-operator/pkg/apis/storageos/v1" - stosClientset "github.com/storageos/cluster-operator/pkg/client/clientset/versioned" - "github.com/storageos/cluster-operator/pkg/nfs" - "github.com/storageos/cluster-operator/pkg/util/k8s" -) - -// ErrNoCluster is the error when there's no associated running StorageOS -// cluster found for NFS server. -var ErrNoCluster = goerrors.New("no storageos cluster found") - -var log = logf.Log.WithName("controller_nfsserver") - -const ( - finalizer = "finalizer.nfsserver.storageos.com" - appComponent = "nfs-server" - - reconcilePeriodSeconds = 15 -) - -// Add creates a new NFSServer Controller and adds it to the Manager. The Manager will set fields on the Controller -// and Start it when the Manager is Started. -func Add(mgr manager.Manager) error { - return add(mgr, newReconciler(mgr)) -} - -// newReconciler returns a new reconcile.Reconciler -func newReconciler(mgr manager.Manager) reconcile.Reconciler { - storageos := stosClientset.NewForConfigOrDie(mgr.GetConfig()) - return &ReconcileNFSServer{ - client: mgr.GetClient(), - kConfig: mgr.GetConfig(), - scheme: mgr.GetScheme(), - recorder: mgr.GetEventRecorderFor("storageos-nfsserver"), - stosClientset: storageos, - } -} - -// add adds a new Controller to mgr with r as the reconcile.Reconciler -func add(mgr manager.Manager, r reconcile.Reconciler) error { - // Create a new controller - c, err := controller.New("nfsserver-controller", mgr, controller.Options{Reconciler: r}) - if err != nil { - return err - } - - // Watch for changes to primary resource NFSServer. - err = c.Watch(&source.Kind{Type: &storageosv1.NFSServer{}}, &handler.EnqueueRequestForObject{}) - if err != nil { - return err - } - - // Watch for changes to secondary resource StatefulSet and requeue the owner - // NFSServer. - err = c.Watch(&source.Kind{Type: &appsv1.StatefulSet{}}, &handler.EnqueueRequestForOwner{ - IsController: true, - OwnerType: &storageosv1.NFSServer{}, - }) - if err != nil { - return err - } - - // Watch for changes to secondary resource Service and requeue the owner - // NFSServer. - // - // This is used to update the NFSServer Status with the connection endpoint - // once it comes online. - err = c.Watch(&source.Kind{Type: &corev1.Service{}}, &handler.EnqueueRequestForOwner{ - IsController: true, - OwnerType: &storageosv1.NFSServer{}, - }) - if err != nil { - return err - } - - return nil -} - -// blank assignment to verify that ReconcileNFSServer implements reconcile.Reconciler -var _ reconcile.Reconciler = &ReconcileNFSServer{} - -// ReconcileNFSServer reconciles a NFSServer object -type ReconcileNFSServer struct { - // This client, initialized using mgr.Client() above, is a split client - // that reads objects from the cache and writes to the apiserver - client client.Client - stosClientset stosClientset.Interface - scheme *runtime.Scheme - recorder record.EventRecorder - // k8s rest config is needed for creating a k8s discovery client, used by - // the osdk's metrics helpers to create Prometheus ServiceMonitor for NFS - // Server. - kConfig *rest.Config -} - -// Reconcile reads that state of the cluster for a NFSServer object and makes changes based on the state read -// and what is in the NFSServer.Spec -// Note: -// The Controller will requeue the Request to be processed again if the returned error is non-nil or -// Result.Requeue is true, otherwise upon completion it will remove the work from the queue. -func (r *ReconcileNFSServer) Reconcile(request reconcile.Request) (reconcile.Result, error) { - reqLogger := log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name) - // reqLogger.Info("Reconciling NFSServer") - - reconcilePeriod := reconcilePeriodSeconds * time.Second - reconcileResult := reconcile.Result{RequeueAfter: reconcilePeriod} - - // Fetch the NFSServer instance - instance := &storageosv1.NFSServer{} - err := r.client.Get(context.TODO(), request.NamespacedName, instance) - if err != nil { - if errors.IsNotFound(err) { - // Request object not found, could have been deleted after reconcile request. - // Owned objects are automatically garbage collected. For additional cleanup logic use finalizers. - // Return and don't requeue - return reconcile.Result{}, nil - } - // Error reading the object - requeue the request. - return reconcileResult, err - } - - if err := r.reconcile(instance); err != nil { - reqLogger.Info("Reconcile failed", "error", err) - return reconcileResult, nil - } - - return reconcileResult, nil -} - -func (r *ReconcileNFSServer) reconcile(instance *storageosv1.NFSServer) error { - // Add our finalizer immediately so we can cleanup a partial deployment. If - // this is not set, the CR can simply be deleted. - if len(instance.GetFinalizers()) == 0 { - // Add our finalizer so that we control deletion. - if err := r.addFinalizer(instance); err != nil { - return err - } - - // Return here, as the update to add the finalizer will trigger another - // reconcile. - return nil - } - - // Get a StorageOS cluster to associate the NFS server with. - stosCluster, err := storageoscluster.GetCurrentStorageOSCluster(r.client) - if err != nil { - return err - } - - // Update NFS spec with values inferred from the StorageOS cluster. - updated, err := r.updateSpec(instance, stosCluster) - if err != nil { - return err - } - - // Return here if the CR has been updated as the current instance is - // outdated. - if updated { - return nil - } - - // Prepare for NFS deployment. - - // Labels to be applied on all the k8s resources that are created for NFS - // server. Inherit the labels from the CR. - labels := instance.Labels - if labels == nil { - labels = map[string]string{} - } - // Add default labels. - // TODO: This is legacy label. Remove this with care. Ensure it's not used - // by any label selectors. - labels["app"] = "storageos" - - // Set the app component. - labels[k8s.AppComponent] = appComponent - - // Add default resource app labels. - labels = k8s.AddDefaultAppLabels(stosCluster.Name, labels) - - d := nfs.NewDeployment(r.client, r.kConfig, stosCluster, instance, labels, r.recorder, r.scheme) - - // If the CR has not been marked for deletion, ensure it is deployed. - if instance.GetDeletionTimestamp() == nil { - if err := d.Deploy(); err != nil { - // Ignore "Operation cannot be fulfilled" error. It happens when the - // actual state of object is different from what is known to the operator. - // Operator would resync and retry the failed operation on its own. - if !strings.HasPrefix(err.Error(), "Operation cannot be fulfilled") { - r.recorder.Event(instance, corev1.EventTypeWarning, "FailedCreation", err.Error()) - } - return err - } - } else { - // Delete the deployment once the finalizers are set on the cluster - // resource. - r.recorder.Event(instance, corev1.EventTypeNormal, "Terminating", "Deleting the NFS server.") - - if err := d.Delete(); err != nil { - return err - } - - // Reset finalizers and let k8s delete the object. - // When finalizers are set on an object, metadata.deletionTimestamp is - // also set. deletionTimestamp helps the garbage collector identify - // when to delete an object. k8s deletes the object only once the - // list of finalizers is empty. - instance.SetFinalizers([]string{}) - return r.client.Update(context.Background(), instance) - } - - return nil -} - -func (r *ReconcileNFSServer) addFinalizer(instance *storageosv1.NFSServer) error { - instance.SetFinalizers(append(instance.GetFinalizers(), finalizer)) - - // Update CR - err := r.client.Update(context.TODO(), instance) - if err != nil { - return err - } - return nil -} - -// updateSpec takes a NFSServer CR and a StorageOSCluster CR and updates -// NFSServer if needed. It returns true if there was an update. This result can -// be used to decide if the caller should continue with reconcile or return from -// reconcile due to an outdated CR instance. -func (r *ReconcileNFSServer) updateSpec(instance *storageosv1.NFSServer, cluster *storageosv1.StorageOSCluster) (bool, error) { - needUpdate := false - - // Check if any CR property needs to be updated. - - sc := instance.Spec.GetStorageClassName(cluster.Spec.GetStorageClassName()) - if instance.Spec.StorageClassName != sc { - instance.Spec.StorageClassName = sc - needUpdate = true - } - - image := instance.Spec.GetContainerImage(cluster.Spec.GetNFSServerImage()) - if instance.Spec.NFSContainer != image { - instance.Spec.NFSContainer = image - needUpdate = true - } - - if needUpdate { - // Update CR. - err := r.client.Update(context.TODO(), instance) - if err != nil { - return false, err - } - return true, nil - } - return false, nil -} diff --git a/pkg/controller/nfsserver/nfsserver_controller_test.go b/pkg/controller/nfsserver/nfsserver_controller_test.go deleted file mode 100644 index 650678f0d..000000000 --- a/pkg/controller/nfsserver/nfsserver_controller_test.go +++ /dev/null @@ -1,171 +0,0 @@ -package nfsserver - -import ( - "context" - "reflect" - "testing" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - - "github.com/storageos/cluster-operator/internal/pkg/image" - storageosapis "github.com/storageos/cluster-operator/pkg/apis" - storageosv1 "github.com/storageos/cluster-operator/pkg/apis/storageos/v1" -) - -//nolint // This function is shown as unused by the linter. -// getTestCluster returns a StorageOSCluster object with the given properties. -func getTestCluster( - name string, namespace string, - spec storageosv1.StorageOSClusterSpec, - status storageosv1.StorageOSClusterStatus) *storageosv1.StorageOSCluster { - return &storageosv1.StorageOSCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: spec, - Status: status, - } -} - -//nolint // This function is shown as unused by the linter. -// getTestNFSServer returns a NFSServer object with the given properties. -func getTestNFSServer( - name string, namespace string, - spec storageosv1.NFSServerSpec, - status storageosv1.NFSServerStatus) *storageosv1.NFSServer { - return &storageosv1.NFSServer{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: spec, - Status: status, - } -} - -func TestUpdateSpec(t *testing.T) { - // This test used to work with the controller-runtime fake client. The fake - // client has been deprecated and this test fails due to unexpected issues. - // NFS Controller is no longer used in StorageOS v2. This test will be - // removed with the NFS controller. - t.Skip("skipping... fails with the controller-runtime fake client") - - emptyClusterSpec := storageosv1.StorageOSClusterSpec{} - emptyClusterStatus := storageosv1.StorageOSClusterStatus{} - emptyNFSSpec := storageosv1.NFSServerSpec{} - emptyNFSStatus := storageosv1.NFSServerStatus{} - - testcases := []struct { - name string - cluster *storageosv1.StorageOSCluster - nfsServer *storageosv1.NFSServer - wantNFSServer *storageosv1.NFSServer - wantUpdate bool - wantErr error - }{ - { - name: "inherit attributes from cluster", - cluster: getTestCluster("cluster1", "default", emptyClusterSpec, emptyClusterStatus), - nfsServer: getTestNFSServer("nfs1", "default", emptyNFSSpec, emptyNFSStatus), - wantNFSServer: getTestNFSServer("nfs1", "default", - storageosv1.NFSServerSpec{ - StorageClassName: "fast", - NFSContainer: image.DefaultNFSContainerImage, - }, - emptyNFSStatus, - ), - wantUpdate: true, - }, - { - // Check if the overridden cluster level defaults are inherited to - // the NFS Server. - name: "update the default properties in cluster", - cluster: getTestCluster( - "cluster1", "default", - storageosv1.StorageOSClusterSpec{ - StorageClassName: "testsc", - Images: storageosv1.ContainerImages{ - NFSContainer: "test-image", - }, - }, emptyClusterStatus), - nfsServer: getTestNFSServer("nfs1", "default", emptyNFSSpec, emptyNFSStatus), - wantNFSServer: getTestNFSServer("nfs1", "default", storageosv1.NFSServerSpec{ - StorageClassName: "testsc", - NFSContainer: "test-image", - }, emptyNFSStatus), - wantUpdate: true, - }, - { - // Check that there's no update when the NFS Server CR is already - // up-to-date. - name: "no new attributes to update", - cluster: getTestCluster("cluster1", "default", emptyClusterSpec, emptyClusterStatus), - nfsServer: getTestNFSServer( - "nfs1", "default", - storageosv1.NFSServerSpec{ - StorageClassName: "fast", - NFSContainer: image.DefaultNFSContainerImage, - }, emptyNFSStatus), - wantUpdate: false, - }, - { - // When the attributes are defined in NFS Server CR, no CR update - // should happen. - name: "override default attributes", - cluster: getTestCluster("cluster1", "default", emptyClusterSpec, emptyClusterStatus), - nfsServer: getTestNFSServer("nfs1", "default", storageosv1.NFSServerSpec{ - StorageClassName: "testsc", - NFSContainer: "test-image", - }, emptyNFSStatus), - wantUpdate: false, - }, - } - - for _, tc := range testcases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - // Create a new scheme and add StorageOS APIs to it. Pass this to the - // k8s client so that it can create StorageOS resources. - testScheme := runtime.NewScheme() - if err := storageosapis.AddToScheme(testScheme); err != nil { - t.Fatal(err) - } - - client := fake.NewFakeClientWithScheme(testScheme, tc.cluster, tc.nfsServer) - - reconciler := ReconcileNFSServer{ - client: client, - } - - // Update NFSServer instance with the StorageOS Cluster and check the - // results. - result, err := reconciler.updateSpec(tc.nfsServer, tc.cluster) - if err != nil { - t.Fatalf("error while updating spec: %v", err) - } - - if result != tc.wantUpdate { - t.Errorf("unexpected update spec result:\n\t(WNT) %t\n\t(GOT) %t", tc.wantUpdate, result) - } - - // If there was an update, get the NFS Server and check if it's as - // expected. - if tc.wantUpdate { - namespacedNameNFS := types.NamespacedName{Name: tc.nfsServer.Name, Namespace: tc.nfsServer.Namespace} - nfsServer := &storageosv1.NFSServer{} - - if err := client.Get(context.TODO(), namespacedNameNFS, nfsServer); err != nil { - t.Fatalf("failed to get NFS Server: %v", err) - } - - if !reflect.DeepEqual(nfsServer, tc.wantNFSServer) { - t.Errorf("unexpected NFS Server:\n\t(WNT) %v\n\t(GOT) %v", tc.wantNFSServer, nfsServer) - } - } - }) - } -} diff --git a/pkg/controller/storageoscluster/storageoscluster_controller.go b/pkg/controller/storageoscluster/storageoscluster_controller.go index 20cf7b682..72cf772cf 100644 --- a/pkg/controller/storageoscluster/storageoscluster_controller.go +++ b/pkg/controller/storageoscluster/storageoscluster_controller.go @@ -332,16 +332,10 @@ func (r *ReconcileStorageOSCluster) updateSpec(m *storageosv1.StorageOSCluster) // CSI related string properties. These must be set always because CSI is // the only supported deployment. - properties[&m.Spec.Images.CSINodeDriverRegistrarContainer] = m.Spec.GetCSINodeDriverRegistrarImage(storageos.CSIV1Supported(r.k8sVersion)) - - if storageos.CSIV1Supported(r.k8sVersion) { - properties[&m.Spec.Images.CSIClusterDriverRegistrarContainer] = m.Spec.GetCSIClusterDriverRegistrarImage() - properties[&m.Spec.Images.CSILivenessProbeContainer] = m.Spec.GetCSILivenessProbeImage() - } - - properties[&m.Spec.Images.CSIExternalProvisionerContainer] = m.Spec.GetCSIExternalProvisionerImage(storageos.CSIV1Supported(r.k8sVersion)) - - properties[&m.Spec.Images.CSIExternalAttacherContainer] = m.Spec.GetCSIExternalAttacherImage(storageos.CSIV1Supported(r.k8sVersion), storageos.CSIExternalAttacherV2Supported(r.k8sVersion)) + properties[&m.Spec.Images.CSINodeDriverRegistrarContainer] = m.Spec.GetCSINodeDriverRegistrarImage() + properties[&m.Spec.Images.CSILivenessProbeContainer] = m.Spec.GetCSILivenessProbeImage() + properties[&m.Spec.Images.CSIExternalProvisionerContainer] = m.Spec.GetCSIExternalProvisionerImage() + properties[&m.Spec.Images.CSIExternalAttacherContainer] = m.Spec.GetCSIExternalAttacherImage() // Add external resizer image if storageos v2 and supported k8s // version. diff --git a/pkg/nfs/configmap.go b/pkg/nfs/configmap.go deleted file mode 100644 index 908a3e23c..000000000 --- a/pkg/nfs/configmap.go +++ /dev/null @@ -1,198 +0,0 @@ -package nfs - -import ( - "bytes" - "fmt" - "strings" - "text/template" - - storageosv1 "github.com/storageos/cluster-operator/pkg/apis/storageos/v1" - corev1 "k8s.io/api/core/v1" -) - -// NFS server configuration constants. -const ( - DefaultExportName = "NFSExport" - DefaultExportPVCReadOnly = false - DefaultAccessType = "readwrite" - DefaultSquash = "none" - DefaultLogLevel = "DEBUG" - DefaultGraceless = true - DefaultFsidDevice = true -) - -func createConfig(instance *storageosv1.NFSServer) (string, error) { - // id needs to be unique for each export on the server node. - id := 57 - - var exportCfg string - - // Get export spec and obtain export config. - exportSpec := getExportSpec(instance) - export, err := exportConfig(id, exportSpec.PersistentVolumeClaim.ClaimName, exportSpec.Server.AccessMode, exportSpec.Server.Squash) - if err != nil { - return "", err - } - exportCfg = export - - globalCfg, err := globalConfig(DefaultGraceless, DefaultFsidDevice) - if err != nil { - return "", err - } - - logCfg, err := logConfig(DefaultLogLevel) - if err != nil { - return "", err - } - - return fmt.Sprintf("%s\n%s\n%s", globalCfg, logCfg, exportCfg), nil -} - -// getExportSpec returns a NFS ExportSpec based on the NFSServer instance. -func getExportSpec(instance *storageosv1.NFSServer) storageosv1.ExportSpec { - // Default export spec using the instance name as the PVC name. - // ExportSpec.Name is not used anywhere, use a default name. - exportSpec := storageosv1.ExportSpec{ - Name: DefaultExportName, - PersistentVolumeClaim: corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: instance.Name, - ReadOnly: DefaultExportPVCReadOnly, - }, - Server: storageosv1.ServerSpec{ - AccessMode: DefaultAccessType, - Squash: DefaultSquash, - }, - } - - // If PVC is specified in the spec, use the specified PVC. - if instance.Spec.PersistentVolumeClaim.ClaimName != "" { - exportSpec.PersistentVolumeClaim = instance.Spec.PersistentVolumeClaim - } - - // If Export is specified in the spec, use the specified export. - if instance.Spec.Export.Name != "" { - exportSpec = instance.Spec.Export - } - - return exportSpec -} - -// nfsExportConfig is the NFS server export configuration. -type nfsExportConfig struct { - ID int - Name string - AccessType string - Squash string -} - -func exportConfig(id int, ref string, access string, squash string) (string, error) { - exportConfigTemplate := ` -EXPORT { - Export_Id = {{.ID}}; - Path = /export/{{.Name}}; - Pseudo = /{{.Name}}; - Protocols = 4; - Transports = TCP; - Sectype = sys; - Access_Type = {{.AccessType}}; - Squash = {{.Squash}}; - FSAL { - Name = VFS; - } -}` - exportConfigData := nfsExportConfig{ - ID: id, - Name: ref, - AccessType: getAccessMode(access), - Squash: getSquash(squash), - } - return renderConfig("exportConfig", exportConfigTemplate, exportConfigData) -} - -type nfsGlobalConfig struct { - Graceless bool - FSIDDevice bool -} - -func globalConfig(graceless, fsidDevice bool) (string, error) { - globalConfigTemplate := ` -NFSv4 { - Graceless = {{.Graceless}}; -} -NFS_Core_Param { - fsid_device = {{.FSIDDevice}}; -}` - globalConfigData := nfsGlobalConfig{ - Graceless: graceless, - FSIDDevice: fsidDevice, - } - return renderConfig("globalConfig", globalConfigTemplate, globalConfigData) -} - -type nfsLogConfig struct { - LogLevel string -} - -// TODO, use default "EVENT" level. -func logConfig(logLevel string) (string, error) { - logConfigTemplate := ` -LOG { - default_log_level = {{.LogLevel}}; - Components { - ALL = {{.LogLevel}}; - } -}` - logConfigData := nfsLogConfig{ - LogLevel: logLevel, - } - return renderConfig("logConfig", logConfigTemplate, logConfigData) -} - -// renderConfig takes template name, template of a configuration and config data -// and returns a rendered configuration. -func renderConfig(templateName, configTemplate string, config interface{}) (string, error) { - var configuration bytes.Buffer - tmpl, err := template.New(templateName).Parse(configTemplate) - if err != nil { - return "", err - } - - if err := tmpl.Execute(&configuration, config); err != nil { - return "", err - } - - return configuration.String(), nil -} - -// getAccessMode converts the access mode in NFSServer config to nfs-ganesha -// access modes. -func getAccessMode(mode string) string { - switch strings.ToLower(mode) { - case "none": - return "None" - case "readonly": - return "RO" - default: - return "RW" - } -} - -func getSquash(squash string) string { - if squash != "" { - return strings.ToLower(squash) - } - return "none" -} - -func (d *Deployment) createNFSConfigMap() error { - nfsConfig, err := createConfig(d.nfsServer) - if err != nil { - return err - } - - data := map[string]string{ - d.nfsServer.Name: nfsConfig, - } - - return d.k8sResourceManager.ConfigMap(d.nfsServer.Name, d.nfsServer.Namespace, nil, data).Create() -} diff --git a/pkg/nfs/configmap_test.go b/pkg/nfs/configmap_test.go deleted file mode 100644 index 3f2044ee9..000000000 --- a/pkg/nfs/configmap_test.go +++ /dev/null @@ -1,279 +0,0 @@ -package nfs - -import ( - "reflect" - "strings" - "testing" - - storageosv1 "github.com/storageos/cluster-operator/pkg/apis/storageos/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -type testConfig struct { - Foo string - Bar string -} - -func TestRenderConfig(t *testing.T) { - someTemplate := ` -foo = {{.Foo}} -bar = {{.Bar}} -` - wantRender := ` -foo = foo1 -bar = bar1 -` - - someData := testConfig{ - Foo: "foo1", - Bar: "bar1", - } - - render, err := renderConfig("somefoo", someTemplate, someData) - if err != nil { - t.Error("failed to render config", err) - } - - if render != wantRender { - t.Errorf("unexpected template renders:\n\t(WNT) %v\n\t(GOT) %v", wantRender, render) - } -} - -func TestCreateConfig(t *testing.T) { - testcases := []struct { - name string - nfsServerSpec storageosv1.NFSServerSpec - wantConfig string - wantErr bool - }{ - { - name: "default nfs server spec", - nfsServerSpec: storageosv1.NFSServerSpec{}, - wantConfig: ` -NFSv4 { - Graceless = true; -} -NFS_Core_Param { - fsid_device = true; -} - -LOG { - default_log_level = DEBUG; - Components { - ALL = DEBUG; - } -} - -EXPORT { - Export_Id = 57; - Path = /export/test-nfs; - Pseudo = /test-nfs; - Protocols = 4; - Transports = TCP; - Sectype = sys; - Access_Type = RW; - Squash = none; - FSAL { - Name = VFS; - } -}`, - }, - { - name: "nfs server spec with default export server spec", - nfsServerSpec: storageosv1.NFSServerSpec{ - Export: storageosv1.ExportSpec{ - Name: "export1", - Server: storageosv1.ServerSpec{}, - PersistentVolumeClaim: corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: "test-claim", - ReadOnly: false, - }, - }, - }, - wantConfig: ` -NFSv4 { - Graceless = true; -} -NFS_Core_Param { - fsid_device = true; -} - -LOG { - default_log_level = DEBUG; - Components { - ALL = DEBUG; - } -} - -EXPORT { - Export_Id = 57; - Path = /export/test-claim; - Pseudo = /test-claim; - Protocols = 4; - Transports = TCP; - Sectype = sys; - Access_Type = RW; - Squash = none; - FSAL { - Name = VFS; - } -}`, - }, - - { - name: "nfs server spec with custom export server spec", - nfsServerSpec: storageosv1.NFSServerSpec{ - Export: storageosv1.ExportSpec{ - Name: "export1", - Server: storageosv1.ServerSpec{ - AccessMode: "readonly", - Squash: "test-squash", - }, - PersistentVolumeClaim: corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: "test-claim", - ReadOnly: false, - }, - }, - }, - wantConfig: ` -NFSv4 { - Graceless = true; -} -NFS_Core_Param { - fsid_device = true; -} - -LOG { - default_log_level = DEBUG; - Components { - ALL = DEBUG; - } -} - -EXPORT { - Export_Id = 57; - Path = /export/test-claim; - Pseudo = /test-claim; - Protocols = 4; - Transports = TCP; - Sectype = sys; - Access_Type = RO; - Squash = test-squash; - FSAL { - Name = VFS; - } -}`, - }, - } - - for _, tc := range testcases { - nfsServer := &storageosv1.NFSServer{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-nfs", - Namespace: "default", - }, - Spec: tc.nfsServerSpec, - } - - gotConfig, err := createConfig(nfsServer) - if err != nil { - t.Fatal("failed to create config", err) - } - - if strings.TrimSpace(tc.wantConfig) != strings.TrimSpace(gotConfig) { - t.Errorf("unexpected nfs config:\n\t(WNT) %s\n\t(GOT) %s", tc.wantConfig, gotConfig) - } - } -} - -func TestGetExportSpec(t *testing.T) { - defaultExportSpecServer := storageosv1.ServerSpec{ - AccessMode: DefaultAccessType, - Squash: DefaultSquash, - } - - nfsServerName := "testNFSServer" - - testcases := []struct { - name string - nfsServerSpec storageosv1.NFSServerSpec - wantExportSpec storageosv1.ExportSpec - }{ - { - name: "Default export spec", - nfsServerSpec: storageosv1.NFSServerSpec{}, - wantExportSpec: storageosv1.ExportSpec{ - Name: DefaultExportName, - Server: defaultExportSpecServer, - PersistentVolumeClaim: corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: nfsServerName, - ReadOnly: DefaultExportPVCReadOnly, - }, - }, - }, - { - name: "External PVC", - nfsServerSpec: storageosv1.NFSServerSpec{ - PersistentVolumeClaim: corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: "testPVC", - ReadOnly: true, - }, - }, - wantExportSpec: storageosv1.ExportSpec{ - Name: DefaultExportName, - Server: defaultExportSpecServer, - PersistentVolumeClaim: corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: "testPVC", - ReadOnly: true, - }, - }, - }, - { - name: "Export spec specified", - nfsServerSpec: storageosv1.NFSServerSpec{ - Export: storageosv1.ExportSpec{ - Name: "test-export", - Server: storageosv1.ServerSpec{ - AccessMode: "fooaccess", - Squash: "foosquash", - }, - PersistentVolumeClaim: corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: "test-export-pvc", - ReadOnly: true, - }, - }, - }, - wantExportSpec: storageosv1.ExportSpec{ - Name: "test-export", - Server: storageosv1.ServerSpec{ - AccessMode: "fooaccess", - Squash: "foosquash", - }, - PersistentVolumeClaim: corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: "test-export-pvc", - ReadOnly: true, - }, - }, - }, - } - - for _, tc := range testcases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - nfsServer := &storageosv1.NFSServer{ - ObjectMeta: metav1.ObjectMeta{ - Name: nfsServerName, - Namespace: "default", - }, - Spec: tc.nfsServerSpec, - } - - gotExport := getExportSpec(nfsServer) - - if !reflect.DeepEqual(gotExport, tc.wantExportSpec) { - t.Errorf("unexpected export spec:\n\t(WNT) %v\n\t(GOT) %v", tc.wantExportSpec, gotExport) - } - }) - } -} diff --git a/pkg/nfs/delete.go b/pkg/nfs/delete.go deleted file mode 100644 index 72e074a7f..000000000 --- a/pkg/nfs/delete.go +++ /dev/null @@ -1,42 +0,0 @@ -package nfs - -// Delete deletes all the storageos resources. -// This explicit delete is implemented instead of depending on the garbage -// collector because sometimes the garbage collector deletes the resources -// with owner reference as a CRD without the parent being deleted. This happens -// especially when a cluster reboots. Althrough the operator re-creates the -// resources, we want to avoid this behavior by implementing an explcit delete. -func (d *Deployment) Delete() error { - if err := d.k8sResourceManager.StatefulSet(d.nfsServer.Name, d.nfsServer.Namespace, nil, nil).Delete(); err != nil { - return err - } - if err := d.k8sResourceManager.ConfigMap(d.nfsServer.Name, d.nfsServer.Namespace, nil, nil).Delete(); err != nil { - return err - } - if err := d.k8sResourceManager.Service(d.nfsServer.Name, d.nfsServer.Namespace, nil, nil, nil).Delete(); err != nil { - return err - } - if err := d.k8sResourceManager.Service(d.getMetricsServiceName(), d.nfsServer.Namespace, nil, nil, nil).Delete(); err != nil { - return err - } - if err := d.k8sResourceManager.ClusterRoleBinding(d.getClusterRoleBindingName(), nil, nil, nil).Delete(); err != nil { - return err - } - if err := d.k8sResourceManager.ServiceAccount(d.getServiceAccountName(), d.nfsServer.Namespace, nil).Delete(); err != nil { - return err - } - - // Delete PVC if it was not specified and dynamically created for NFS - // Server. - // NOTE: Reclaim policy is not respected here because NFS Server need not - // have its own reclaim policy options. The StorageClass reclaim policy - // must be used to set volume reclaim policy. NFS Server spec reclaim policy - // will be removed in StorageOS cluster-operator v2 APIs. - if d.nfsServer.Spec.PersistentVolumeClaim.ClaimName == "" { - if err := d.k8sResourceManager.PersistentVolumeClaim(d.nfsServer.Name, d.nfsServer.Namespace, nil, nil).Delete(); err != nil { - return err - } - } - - return nil -} diff --git a/pkg/nfs/delete_test.go b/pkg/nfs/delete_test.go deleted file mode 100644 index ffeda5a1e..000000000 --- a/pkg/nfs/delete_test.go +++ /dev/null @@ -1,147 +0,0 @@ -package nfs - -import ( - "context" - "testing" - - storageosv1 "github.com/storageos/cluster-operator/pkg/apis/storageos/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -func TestDelete(t *testing.T) { - // Existing PVC that can be used with a NFS Server. - existingPVC := &corev1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: "existing-pvc", - Namespace: "default", - }, - Spec: corev1.PersistentVolumeClaimSpec{}, - } - - testcases := []struct { - name string - nfsServerSpec storageosv1.NFSServerSpec - wantDefaultPVCDeleted bool - }{ - { - name: "delete dynamically created volume", - nfsServerSpec: storageosv1.NFSServerSpec{ - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - wantDefaultPVCDeleted: true, - }, - { - // NFS Server spec reclaim policy must not be respected. - name: "volume reclaim policy - retain", - nfsServerSpec: storageosv1.NFSServerSpec{ - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimDelete, - }, - wantDefaultPVCDeleted: true, - }, - { - // Existing volume must not be deleted. - name: "specify existing volume claim", - nfsServerSpec: storageosv1.NFSServerSpec{ - PersistentVolumeClaim: corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: existingPVC.Name, - }, - }, - }, - } - - for _, tc := range testcases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - client := fake.NewFakeClient(existingPVC) - kConfig := &rest.Config{} - - // NFSServer config. - nfsServer := &storageosv1.NFSServer{ - ObjectMeta: metav1.ObjectMeta{ - Name: "some-nfs-server", - Namespace: "default", - }, - Spec: tc.nfsServerSpec, - } - - // StorageOS Cluster config. - stosCluster := &storageosv1.StorageOSCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "some-stos-cluster", - Namespace: "default", - }, - Spec: storageosv1.StorageOSClusterSpec{}, - } - - // NFSServer deployment. - deployment := NewDeployment(client, kConfig, stosCluster, nfsServer, nil, nil, nil) - - // Deploy NFS Server. - if err := deployment.Deploy(); err != nil { - t.Errorf("unexpected error while deploying: %v", err) - } - - // Default dynamically created PVC for NFS Server. - createdPVC := &corev1.PersistentVolumeClaim{} - pvcNSName := types.NamespacedName{ - Name: nfsServer.Name, - Namespace: nfsServer.Namespace, - } - - // If PVC volume source is not specified, check if the default PVC - // is created. - if nfsServer.Spec.PersistentVolumeClaim.ClaimName == "" { - if err := client.Get(context.Background(), pvcNSName, createdPVC); err != nil { - t.Fatalf("failed to get the created PVC: %v", err) - } - } - - // Delete NFS Server. - if err := deployment.Delete(); err != nil { - t.Errorf("unexpected error: %v", err) - } - - // If PVC Volume Source was not provided, check their existence. - if nfsServer.Spec.PersistentVolumeClaim.ClaimName == "" { - // Get the default PVC. - err := client.Get(context.Background(), pvcNSName, createdPVC) - - // If the PVC is expected to be deleted, the error must be NOT - // FOUND. - if tc.wantDefaultPVCDeleted { - if !errors.IsNotFound(err) { - t.Errorf("expected the PVC to be deleted") - } - } else { - if err != nil { - t.Errorf("expected the default volume to exist") - } - } - } else { - // Ensure that the existing provided PVC has not been deleted. - existingPVCNSName := types.NamespacedName{ - Name: existingPVC.Name, - Namespace: existingPVC.Namespace, - } - if err := client.Get(context.Background(), existingPVCNSName, existingPVC); err != nil { - t.Error("expected existing PVC to not be deleted") - } - } - }) - } -} diff --git a/pkg/nfs/deploy.go b/pkg/nfs/deploy.go deleted file mode 100644 index 25fa33a55..000000000 --- a/pkg/nfs/deploy.go +++ /dev/null @@ -1,171 +0,0 @@ -package nfs - -import ( - "fmt" - "strings" - - "github.com/operator-framework/operator-sdk/pkg/metrics" - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - "k8s.io/apimachinery/pkg/api/errors" - logf "sigs.k8s.io/controller-runtime/pkg/log" - - "github.com/storageos/cluster-operator/pkg/storageos" -) - -const ( - appName = "storageos" - - serviceAccountPrefix = "storageos-nfs" - - // DefaultNFSPort is the default port for NFS server. - DefaultNFSPort = 2049 - // DefaultHTTPPort is the default port for NFS server health and metrics. - DefaultHTTPPort = 80 - // NFSPortName is the name of the port that exposes the NFS server. - NFSPortName = "nfs" - // MetricsPortName is the name of the port that exposes the NFS metrics. - MetricsPortName = "metrics" - - // HealthEndpointPath is the path to query on the HTTP Port for health. - // This is hardcoded in the NFS container and not settable by the user. - HealthEndpointPath = "/healthz" -) - -var log = logf.Log.WithName("storageos.nfsserver") - -// Deploy deploys a NFS server. -func (d *Deployment) Deploy() error { - err := d.ensureService(DefaultNFSPort) - if err != nil { - return err - } - - // Create metrics service. - // Since we use ServiceMonitor, a separate service dedicated to metrics - // ports helps avoid Prometheus targets endpoints that don't serve metrics. - if err := d.createMetricsService(DefaultHTTPPort); err != nil { - return err - } - - if err := d.createNFSConfigMap(); err != nil { - return err - } - - if err := d.createServiceAccountForNFSServer(); err != nil { - return err - } - - // Grant OpenShift SCC permission for StatefulSet using the ClusterRole - // created for the StorageOSCluster. - if strings.Contains(d.cluster.Spec.K8sDistro, storageos.K8SDistroOpenShift) { - if err := d.createClusterRoleBindingForSCC(); err != nil { - return err - } - } - - // Get the NFS capacity. - requestedCapacity := d.nfsServer.Spec.GetRequestedCapacity() - size := &requestedCapacity - - pvcVS := d.nfsServer.Spec.PersistentVolumeClaim - - // If no existing PVC Volume Source is specified in the spec, create a new - // PVC with NFS Server name. - if pvcVS.ClaimName == "" { - // Create a PVC with the same name as the NFS Server. - if err := d.createPVC(size); err != nil { - return err - } - pvcVS = corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: d.nfsServer.Name, - } - } - - // Create a StatefulSet NFS Server with PVC Volume Source. - if err := d.createStatefulSet(&pvcVS, DefaultNFSPort, DefaultHTTPPort); err != nil { - return err - } - - status, err := d.getStatus() - if err != nil { - return err - } - - if err := d.updateStatus(status); err != nil { - log.Info("Failed to update status", "error", err) - } - - if err := d.createServiceMonitor(); err != nil { - // Ignore if the ServiceMonitor already exists. - if !errors.IsAlreadyExists(err) { - log.Info("Failed to create service monitor for metrics", "error", err) - } - } - - return nil -} - -// Due to https://github.com/kubernetes/kubernetes/issues/74916 fixed in -// 1.15, labels intended for the PVC must be set on the Pod template. -// In 1.15 and later we can just set the "app" and "nfsserver" labels here. For -// now, pass all labels rather than check k8s versions. The only downside is -// that the nfs pod gets storageos.com labels that don't do anything directly. -func (d *Deployment) labelsForStatefulSet() map[string]string { - // Get labels from the NFS k8s resource manager and add NFS Server specific - // labels. - ssLabels := d.k8sResourceManager.GetLabels() - // TODO: This is legacy label. Remove this with care. Ensure it's not used - // by any label selectors. - ssLabels["app"] = appName - ssLabels["nfsserver"] = d.nfsServer.Name - - if !d.cluster.Spec.DisableFencing { - ssLabels["storageos.com/fenced"] = "true" - } - - return ssLabels -} - -func (d *Deployment) createClusterRoleBindingForSCC() error { - subjects := []rbacv1.Subject{ - { - Kind: rbacv1.ServiceAccountKind, - Name: d.getServiceAccountName(), - Namespace: d.nfsServer.Namespace, - }, - } - roleRef := &rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: storageos.OpenShiftSCCClusterRoleName, - APIGroup: "rbac.authorization.k8s.io", - } - return d.k8sResourceManager.ClusterRoleBinding(d.getClusterRoleBindingName(), nil, subjects, roleRef).Create() -} - -func (d *Deployment) getClusterRoleBindingName() string { - return fmt.Sprintf("storageos:openshift-scc-nfs-%s", d.nfsServer.Name) -} - -func (d *Deployment) getServiceAccountName() string { - return fmt.Sprintf("%s-%s", serviceAccountPrefix, d.nfsServer.Name) -} - -func (d *Deployment) createServiceAccountForNFSServer() error { - return d.k8sResourceManager.ServiceAccount(d.getServiceAccountName(), d.nfsServer.Namespace, nil).Create() -} - -func (d *Deployment) createServiceMonitor() error { - metricsService, err := d.getMetricsService() - if err != nil { - return err - } - - // Create the ServiceMonitor resource for the metrics service. - _, err = metrics.CreateServiceMonitors(d.kConfig, d.nfsServer.Namespace, []*corev1.Service{metricsService}) - if err != nil { - return err - } - - return nil -} diff --git a/pkg/nfs/deploy_test.go b/pkg/nfs/deploy_test.go deleted file mode 100644 index 69726e14d..000000000 --- a/pkg/nfs/deploy_test.go +++ /dev/null @@ -1,98 +0,0 @@ -package nfs - -import ( - "context" - "testing" - - storageosv1 "github.com/storageos/cluster-operator/pkg/apis/storageos/v1" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -func TestDeploy(t *testing.T) { - testcases := []struct { - name string - nfsServerSpec storageosv1.NFSServerSpec - }{ - { - name: "default nfs server spec", - nfsServerSpec: storageosv1.NFSServerSpec{ - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse("1Gi"), - }, - }, - }, - }, - { - name: "specify existing volume claim", - nfsServerSpec: storageosv1.NFSServerSpec{ - PersistentVolumeClaim: corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: "some-pvc", - }, - }, - }, - } - - for _, tc := range testcases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - client := fake.NewFakeClient() - kConfig := &rest.Config{} - - // NFSServer config. - nfsServer := &storageosv1.NFSServer{ - ObjectMeta: metav1.ObjectMeta{ - Name: "some-nfs-server", - Namespace: "default", - }, - Spec: tc.nfsServerSpec, - } - - // StorageOS Cluster config. - stosCluster := &storageosv1.StorageOSCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "some-stos-cluster", - Namespace: "default", - }, - Spec: storageosv1.StorageOSClusterSpec{}, - } - - // NFSServer deployment. - deployment := NewDeployment(client, kConfig, stosCluster, nfsServer, nil, nil, nil) - - // Deploy NFS Server. - if err := deployment.Deploy(); err != nil { - t.Errorf("unexpected error while deploying: %v", err) - } - - // If PVC volume source is not specified, check if the default PVC - // is created. - if nfsServer.Spec.PersistentVolumeClaim.ClaimName == "" { - createdPVC := &corev1.PersistentVolumeClaim{} - pvcNSName := types.NamespacedName{ - Name: nfsServer.Name, - Namespace: nfsServer.Namespace, - } - if err := client.Get(context.Background(), pvcNSName, createdPVC); err != nil { - t.Fatalf("failed to get the created PVC: %v", err) - } - } - - // Check if the StatefulSet was created. - createdStatefulSet := &appsv1.StatefulSet{} - ssNSName := types.NamespacedName{ - Name: nfsServer.Name, - Namespace: nfsServer.Namespace, - } - if err := client.Get(context.Background(), ssNSName, createdStatefulSet); err != nil { - t.Fatalf("failed to get the created statefulset: %v", err) - } - }) - } -} diff --git a/pkg/nfs/deployment.go b/pkg/nfs/deployment.go deleted file mode 100644 index 2e2f078b7..000000000 --- a/pkg/nfs/deployment.go +++ /dev/null @@ -1,41 +0,0 @@ -package nfs - -import ( - storageosv1 "github.com/storageos/cluster-operator/pkg/apis/storageos/v1" - "github.com/storageos/cluster-operator/pkg/util/k8s" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/rest" - "k8s.io/client-go/tools/record" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// Deployment manages the NFS server deployment. -type Deployment struct { - client client.Client - kConfig *rest.Config - nfsServer *storageosv1.NFSServer - recorder record.EventRecorder - scheme *runtime.Scheme - cluster *storageosv1.StorageOSCluster - k8sResourceManager *k8s.ResourceManager -} - -// NewDeployment returns an initialized Deployment. -func NewDeployment( - client client.Client, - kConfig *rest.Config, - stosCluster *storageosv1.StorageOSCluster, - nfsServer *storageosv1.NFSServer, - labels map[string]string, - recorder record.EventRecorder, - scheme *runtime.Scheme) *Deployment { - return &Deployment{ - client: client, - kConfig: kConfig, - nfsServer: nfsServer, - recorder: recorder, - scheme: scheme, - cluster: stosCluster, - k8sResourceManager: k8s.NewResourceManager(client).SetLabels(labels), - } -} diff --git a/pkg/nfs/pvc.go b/pkg/nfs/pvc.go deleted file mode 100644 index fbf732e97..000000000 --- a/pkg/nfs/pvc.go +++ /dev/null @@ -1,22 +0,0 @@ -package nfs - -import ( - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" -) - -func (d *Deployment) createPVC(size *resource.Quantity) error { - scName := d.nfsServer.Spec.GetStorageClassName(d.cluster.Spec.GetStorageClassName()) - - spec := &corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - StorageClassName: &scName, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: *size, - }, - }, - } - - return d.k8sResourceManager.PersistentVolumeClaim(d.nfsServer.Name, d.nfsServer.Namespace, nil, spec).Create() -} diff --git a/pkg/nfs/service.go b/pkg/nfs/service.go deleted file mode 100644 index 5d62df25b..000000000 --- a/pkg/nfs/service.go +++ /dev/null @@ -1,68 +0,0 @@ -package nfs - -import ( - "fmt" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/intstr" - - "github.com/storageos/cluster-operator/pkg/util/k8s" -) - -func (d *Deployment) ensureService(nfsPort int) error { - // If no error in getting the service, service already exists, do nothing. - if _, err := d.getServerService(); err == nil { - return nil - } - - labels := map[string]string{ - k8s.ServiceFor: "nfs-server", - } - - // Couldn't get any existing service. Create a new service. - if err := d.createService(d.nfsServer.Name, NFSPortName, nfsPort, labels); err != nil { - return err - } - return nil -} - -// createMetricsService creates a Service for metrics at the given port number. -func (d *Deployment) createMetricsService(metricsPort int) error { - labels := map[string]string{ - k8s.ServiceFor: "nfs-metrics", - } - if err := d.createService(d.getMetricsServiceName(), MetricsPortName, metricsPort, labels); err != nil { - return err - } - return nil -} - -func (d *Deployment) createService(name string, portName string, port int, labels map[string]string) error { - spec := &corev1.ServiceSpec{ - Selector: d.labelsForStatefulSet(), - Type: corev1.ServiceTypeClusterIP, - Ports: []corev1.ServicePort{ - { - Name: portName, - Port: int32(port), - TargetPort: intstr.FromInt(port), - }, - }, - } - return d.k8sResourceManager.Service(name, d.nfsServer.Namespace, labels, nil, spec).Create() -} - -// getServerService returns the NFS Server endpoint service. -func (d *Deployment) getServerService() (*corev1.Service, error) { - return d.k8sResourceManager.Service(d.nfsServer.Name, d.nfsServer.Namespace, nil, nil, nil).Get() -} - -// getMetricsServiceName returns the name of the metrics service. -func (d *Deployment) getMetricsServiceName() string { - return fmt.Sprintf("%s-%s", d.nfsServer.Name, MetricsPortName) -} - -// getMetricsService returns the NFS Server metrics endpoint service. -func (d *Deployment) getMetricsService() (*corev1.Service, error) { - return d.k8sResourceManager.Service(d.getMetricsServiceName(), d.nfsServer.Namespace, nil, nil, nil).Get() -} diff --git a/pkg/nfs/statefulset.go b/pkg/nfs/statefulset.go deleted file mode 100644 index 67d477d0d..000000000 --- a/pkg/nfs/statefulset.go +++ /dev/null @@ -1,128 +0,0 @@ -package nfs - -import ( - "github.com/storageos/cluster-operator/pkg/util" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" -) - -const ( - // DataVolName is the NFS data volume name. - DataVolName = "nfs-data" -) - -func (d *Deployment) createStatefulSet(pvcVS *corev1.PersistentVolumeClaimVolumeSource, nfsPort int, httpPort int) error { - replicas := int32(1) - - spec := &appsv1.StatefulSetSpec{ - ServiceName: d.nfsServer.Name, - Replicas: &replicas, - Selector: &metav1.LabelSelector{ - MatchLabels: d.labelsForStatefulSet(), - }, - Template: d.createPodTemplateSpec(nfsPort, httpPort), - } - - // Add the block volume in the pod spec volumes. - vol := corev1.Volume{ - Name: DataVolName, - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: pvcVS, - }, - } - spec.Template.Spec.Volumes = append(spec.Template.Spec.Volumes, vol) - - if err := util.AddTolerations(&spec.Template.Spec, d.nfsServer.Spec.Tolerations); err != nil { - return err - } - - // If the cluster was configured with node selectors to only run on certain - // nodes, use the same selectors to selct the nodes that the NFS pods can - // run on. NFSServer does not currently allow setting node selectors or - // affinity directly. - util.AddRequiredNodeAffinity(&spec.Template.Spec, d.cluster.Spec.NodeSelectorTerms) - - return d.k8sResourceManager.StatefulSet(d.nfsServer.Name, d.nfsServer.Namespace, nil, spec).Create() -} - -func (d *Deployment) createPodTemplateSpec(nfsPort int, httpPort int) corev1.PodTemplateSpec { - return corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: d.labelsForStatefulSet(), - }, - Spec: corev1.PodSpec{ - ServiceAccountName: d.getServiceAccountName(), - Containers: []corev1.Container{ - { - ImagePullPolicy: "IfNotPresent", - Name: "nfsd", - Image: d.nfsServer.Spec.GetContainerImage(d.cluster.Spec.GetNFSServerImage()), - Env: []corev1.EnvVar{ - { - Name: "GANESHA_CONFIGFILE", - Value: "/config/" + d.nfsServer.Name, - }, - { - Name: "NAME", - Value: d.nfsServer.Name, - }, - { - Name: "NAMESPACE", - Value: d.nfsServer.Namespace, - }, - }, - Ports: []corev1.ContainerPort{ - { - Name: "nfs-port", - ContainerPort: int32(nfsPort), - }, - { - Name: "http-port", - ContainerPort: int32(httpPort), - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "nfs-config", - MountPath: "/config", - }, - { - Name: DataVolName, - MountPath: "/export", - }, - }, - SecurityContext: &corev1.SecurityContext{ - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{ - "SYS_ADMIN", - "DAC_READ_SEARCH", - }, - }, - }, - ReadinessProbe: &corev1.Probe{ - Handler: corev1.Handler{ - HTTPGet: &corev1.HTTPGetAction{ - Port: intstr.FromInt(httpPort), - Path: HealthEndpointPath, - }, - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "nfs-config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: d.nfsServer.Name, - }, - }, - }, - }, - }, - }, - } -} diff --git a/pkg/nfs/status.go b/pkg/nfs/status.go deleted file mode 100644 index 1c02e6a08..000000000 --- a/pkg/nfs/status.go +++ /dev/null @@ -1,105 +0,0 @@ -package nfs - -import ( - "context" - "fmt" - "reflect" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - "sigs.k8s.io/controller-runtime/pkg/client" - - storageosv1 "github.com/storageos/cluster-operator/pkg/apis/storageos/v1" -) - -const ( - // nfsServerPodLabelSelector is the label that can be used to select all the - // pods of NFS Server. - nfsServerPodLabelSelector = "nfsserver" -) - -func (s *Deployment) updateStatus(status *storageosv1.NFSServerStatus) error { - if reflect.DeepEqual(s.nfsServer.Status, *status) { - return nil - } - - // When there's a difference in remote target, broadcast the status change - // event. - if s.nfsServer.Status.RemoteTarget != status.RemoteTarget { - if status.RemoteTarget != "" { - if s.recorder != nil { - s.recorder.Event(s.nfsServer, corev1.EventTypeNormal, "ChangedStatus", fmt.Sprintf("NFS server is now functional: %s", status.RemoteTarget)) - } - } - } - - // Update subresource status. - s.nfsServer.Status = *status - return s.client.Status().Update(context.Background(), s.nfsServer) -} - -// getStatus determines the status of the NFS Server deployment. -func (s *Deployment) getStatus() (*storageosv1.NFSServerStatus, error) { - status := &storageosv1.NFSServerStatus{ - Phase: storageosv1.PhaseUnknown, - RemoteTarget: "", - AccessModes: "", - } - - // Check if the StatefulSet exists. - _, err := s.k8sResourceManager.StatefulSet(s.nfsServer.Name, s.nfsServer.Namespace, nil, nil).Get() - if err != nil { - if errors.IsNotFound(err) { - // Return empty status without any error. Resources haven't been - // created yet. - return status, nil - } - return status, err - } - - // Check if the Service exists. - svc, err := s.k8sResourceManager.Service(s.nfsServer.Name, s.nfsServer.Namespace, nil, nil, nil).Get() - if err != nil { - if errors.IsNotFound(err) { - // Return empty status without any error. Resources haven't been - // created yet. - return status, nil - } - return status, err - } - - // We got both StatefulSet and Service without error, so upgrade to Pending. - status.Phase = storageosv1.PhasePending - - // Set access mode. - if s.nfsServer.Spec.Export.Name == "" { - status.AccessModes = getAccessMode(DefaultAccessType) - } else { - status.AccessModes = getAccessMode(s.nfsServer.Spec.Export.Server.AccessMode) - } - - // If the service is created, set the cluster address as the endpoint. - if svc.Spec.ClusterIP != "" { - status.RemoteTarget = svc.Spec.ClusterIP - - // Get the NFS Server pods and check their status. - listOpts := []client.ListOption{ - client.MatchingLabels{nfsServerPodLabelSelector: s.nfsServer.Name}, - } - podList := &corev1.PodList{} - if err := s.client.List(context.Background(), podList, listOpts...); err != nil { - return status, err - } - - // If any of the NFS pods are ready, then we can mark the NFS Server as - // online. - for _, pod := range podList.Items { - if pod.Status.Phase == corev1.PodRunning { - status.Phase = storageosv1.PhaseRunning - break - } - } - } - - return status, nil -} diff --git a/pkg/storageos/configmap.go b/pkg/storageos/configmap.go index 5d2880f43..a1fbb364c 100644 --- a/pkg/storageos/configmap.go +++ b/pkg/storageos/configmap.go @@ -123,7 +123,7 @@ const ( // createService creates a ConfigMap to store the node container configuration. func (s *Deployment) createConfigMap() error { - config := configFromSpec(s.stos.Spec, CSIV1Supported(s.k8sVersion)) + config := configFromSpec(s.stos.Spec) labels := make(map[string]string) @@ -140,7 +140,7 @@ func (s *Deployment) createConfigMap() error { // - HOSTNAME (reads from spec.nodeName) // - ADVERTISE_IP (reads from status.podIP) // - BOOTSTRAP_USERNAME, BOOTSTRAP_PASSWORD (reads from secret) -func configFromSpec(spec storageosv1.StorageOSClusterSpec, csiv1 bool) map[string]string { +func configFromSpec(spec storageosv1.StorageOSClusterSpec) map[string]string { config := make(map[string]string) // ETCD_ENDPOINTS must be set to a comma separated list of endpoints. @@ -176,9 +176,9 @@ func configFromSpec(spec storageosv1.StorageOSClusterSpec, csiv1 bool) map[strin config[k8sDistroEnvVar] = spec.K8sDistro } - // CSI is always enabled. - config[csiEndpointEnvVar] = spec.GetCSIEndpoint(true) - config[csiVersionEnvVar] = spec.GetCSIVersion(true) + // CSI v1 is always enabled. + config[csiEndpointEnvVar] = spec.GetCSIEndpoint() + config[csiVersionEnvVar] = spec.GetCSIVersion() // Since we're running in k8s, always listen on the the scheduler extender // api endpoints. The feature can be disabled with the operator. This diff --git a/pkg/storageos/configmap_test.go b/pkg/storageos/configmap_test.go index 8450cbc1b..c7b46f4e6 100644 --- a/pkg/storageos/configmap_test.go +++ b/pkg/storageos/configmap_test.go @@ -29,14 +29,12 @@ func Test_configFromSpec(t *testing.T) { name string spec storageosv1.StorageOSClusterSpec env map[string]string - csiv1 bool wantbase map[string]string wantcustom map[string]string }{ { name: "defaults", spec: defaultSpec, - csiv1: true, wantbase: defaultConfig, }, { @@ -46,29 +44,13 @@ func Test_configFromSpec(t *testing.T) { Enable: true, }, }, - csiv1: true, wantbase: defaultConfig, }, - { - name: "csi v0 - override to csi v1", - spec: storageosv1.StorageOSClusterSpec{ - CSI: storageosv1.StorageOSClusterCSI{ - Enable: true, - }, - }, - csiv1: false, - wantbase: defaultConfig, - wantcustom: map[string]string{ - csiEndpointEnvVar: "unix:///var/lib/kubelet/plugins_registry/storageos/csi.sock", - csiVersionEnvVar: "v1", - }, - }, { name: "shared-dir", spec: storageosv1.StorageOSClusterSpec{ SharedDir: "some-dir-path", }, - csiv1: true, wantbase: defaultConfig, wantcustom: map[string]string{ deviceDirEnvVar: "some-dir-path/devices", @@ -79,7 +61,6 @@ func Test_configFromSpec(t *testing.T) { spec: storageosv1.StorageOSClusterSpec{ DisableTelemetry: true, }, - csiv1: true, wantbase: defaultConfig, wantcustom: map[string]string{ disableTelemetryEnvVar: "true", @@ -93,7 +74,6 @@ func Test_configFromSpec(t *testing.T) { // spec: storageosv1.StorageOSClusterSpec{ // DisableFencing: true, // }, - // csiv1: true, // wantbase: v2DefaultConfig, // wantcustom: map[string]string{ // disableFencingEnvVar: "true", @@ -104,7 +84,6 @@ func Test_configFromSpec(t *testing.T) { spec: storageosv1.StorageOSClusterSpec{ DisableTCMU: true, }, - csiv1: true, wantbase: defaultConfig, wantcustom: map[string]string{ disableTCMUEnvVar: "true", @@ -115,7 +94,6 @@ func Test_configFromSpec(t *testing.T) { spec: storageosv1.StorageOSClusterSpec{ ForceTCMU: true, }, - csiv1: true, wantbase: defaultConfig, wantcustom: map[string]string{ forceTCMUEnvVar: "true", @@ -127,7 +105,6 @@ func Test_configFromSpec(t *testing.T) { TLSEtcdSecretRefName: "etcd-certs", TLSEtcdSecretRefNamespace: "default", }, - csiv1: true, wantbase: defaultConfig, wantcustom: map[string]string{ etcdTLSClientCAEnvVar: "/run/storageos/pki/etcd-client-ca.crt", @@ -140,7 +117,6 @@ func Test_configFromSpec(t *testing.T) { spec: storageosv1.StorageOSClusterSpec{ K8sDistro: "some-distro-name", }, - csiv1: true, wantbase: defaultConfig, wantcustom: map[string]string{ k8sDistroEnvVar: "some-distro-name", @@ -152,7 +128,6 @@ func Test_configFromSpec(t *testing.T) { env: map[string]string{ jaegerEndpointEnvVar: "http:/1.2.3.4:1234", }, - csiv1: true, wantbase: defaultConfig, wantcustom: map[string]string{ jaegerEndpointEnvVar: "http:/1.2.3.4:1234", @@ -164,7 +139,6 @@ func Test_configFromSpec(t *testing.T) { env: map[string]string{ jaegerServiceNameEnvVar: "test-1234", }, - csiv1: true, wantbase: defaultConfig, wantcustom: map[string]string{ jaegerServiceNameEnvVar: "test-1234", @@ -193,7 +167,7 @@ func Test_configFromSpec(t *testing.T) { want[k] = v } - if got := configFromSpec(tt.spec, tt.csiv1); !reflect.DeepEqual(got, want) { + if got := configFromSpec(tt.spec); !reflect.DeepEqual(got, want) { t.Errorf("configFromSpec() got:\n%v\n want:\n%v\n", got, want) } }) diff --git a/pkg/storageos/csi_helper.go b/pkg/storageos/csi_helper.go index 2a548c17b..7860e7d6d 100644 --- a/pkg/storageos/csi_helper.go +++ b/pkg/storageos/csi_helper.go @@ -110,7 +110,7 @@ func (s Deployment) csiHelperContainers() ([]corev1.Container, error) { privileged := true containers := []corev1.Container{ { - Image: s.stos.Spec.GetCSIExternalProvisionerImage(CSIV1Supported(s.k8sVersion)), + Image: s.stos.Spec.GetCSIExternalProvisionerImage(), Name: "csi-external-provisioner", ImagePullPolicy: corev1.PullIfNotPresent, Args: []string{ @@ -136,7 +136,7 @@ func (s Deployment) csiHelperContainers() ([]corev1.Container, error) { }, }, { - Image: s.stos.Spec.GetCSIExternalAttacherImage(CSIV1Supported(s.k8sVersion), CSIExternalAttacherV2Supported(s.k8sVersion)), + Image: s.stos.Spec.GetCSIExternalAttacherImage(), Name: "csi-external-attacher", ImagePullPolicy: corev1.PullIfNotPresent, Args: []string{ @@ -191,64 +191,6 @@ func (s Deployment) csiHelperContainers() ([]corev1.Container, error) { containers = append(containers, resizer) } - // CSI v1 requires running CSI driver registrar to register the driver along - // with the other CSI helpers. - // CSI v0 requires the driver registrar to be run with the driver instances - // only. - // In k8s 1.13, csi-cluster-driver-registrar was required to be run along - // with the CSI helpers. This was responsible for the creation of CSIDriver - // resource belonging to the CRD csidrivers.csi.storage.k8s.io. In k8s - // 1.14+ this was replaced by a CSIDriver built-in resource belonging to - // API group csidrivers.storage.k8s.io. This is no longer automatically - // created. The deployment tools should create this resource. - // - // Add csi-cluster-driver-registrar if the built-in csidrivers resource is - // not supported by the k8s api server. - supportsCSIDriver, err := HasCSIDriverKind(s.discoveryClient) - if err != nil { - return containers, err - } - - // If CSIDriver is not supported but CSI v1 is supported, run - // cluster-driver-registrar. - if !supportsCSIDriver && CSIV1Supported(s.k8sVersion) { - driverReg := corev1.Container{ - Image: s.stos.Spec.GetCSIClusterDriverRegistrarImage(), - Name: "csi-driver-k8s-registrar", - ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{ - "--v=5", - "--csi-address=$(ADDRESS)", - "--pod-info-mount-version=v1", - }, - Env: []corev1.EnvVar{ - { - Name: addressEnvVar, - Value: "/csi/csi.sock", - }, - { - Name: kubeNodeNameEnvVar, - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - APIVersion: "v1", - FieldPath: "spec.nodeName", - }, - }, - }, - }, - SecurityContext: &corev1.SecurityContext{ - Privileged: &privileged, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "plugin-dir", - MountPath: "/csi", - }, - }, - } - containers = append(containers, driverReg) - } - return containers, nil } @@ -261,7 +203,7 @@ func (s Deployment) csiHelperVolumes() []corev1.Volume { Name: "plugin-dir", VolumeSource: corev1.VolumeSource{ HostPath: &corev1.HostPathVolumeSource{ - Path: s.stos.Spec.GetCSIPluginDir(CSIV1Supported(s.k8sVersion)), + Path: s.stos.Spec.GetCSIPluginDir(), Type: &hostpathDirOrCreate, }, }, diff --git a/pkg/storageos/delete.go b/pkg/storageos/delete.go index 95daf93a4..6c4d8c6c2 100644 --- a/pkg/storageos/delete.go +++ b/pkg/storageos/delete.go @@ -51,14 +51,6 @@ func (s *Deployment) Delete() error { return err } - if err := s.k8sResourceManager.ClusterRoleBinding(NFSClusterBindingName, nil, nil, nil).Delete(); err != nil { - return err - } - - if err := s.k8sResourceManager.ClusterRole(NFSClusterRoleName, nil, nil).Delete(); err != nil { - return err - } - if err := s.k8sResourceManager.ServiceAccount(DaemonsetSA, namespace, nil).Delete(); err != nil { return err } diff --git a/pkg/storageos/deploy.go b/pkg/storageos/deploy.go index 739b0d343..b8a7e998a 100644 --- a/pkg/storageos/deploy.go +++ b/pkg/storageos/deploy.go @@ -40,44 +40,32 @@ const ( configmapName = "storageos-node-config" csiHelperName = "storageos-csi-helper" - // tlsSecretType = "kubernetes.io/tls" - // storageosSecretType = "kubernetes.io/storageos" - - defaultFSType = "ext4" - secretNamespaceKey = "adminSecretNamespace" - secretNameKey = "adminSecretName" - apiAddressKey = "apiAddress" - apiUsernameKey = "apiUsername" - apiPasswordKey = "apiPassword" - csiParameterPrefix = "csi.storage.k8s.io/" - csiProvisionUsernameKey = "csiProvisionUsername" - csiProvisionPasswordKey = "csiProvisionPassword" - csiControllerPublishUsernameKey = "csiControllerPublishUsername" - csiControllerPublishPasswordKey = "csiControllerPublishPassword" - csiNodePublishUsernameKey = "csiNodePublishUsername" - csiNodePublishPasswordKey = "csiNodePublishPassword" - csiControllerExpandUsernameKey = "csiControllerExpandUsername" - csiControllerExpandPasswordKey = "csiControllerExpandPassword" - fsType = "fsType" - csiV0ProvisionerSecretNameKey = "csiProvisionerSecretName" - csiV0ProvisionerSecretNamespaceKey = "csiProvisionerSecretNamespace" - csiV0ControllerPublishSecretNameKey = "csiControllerPublishSecretName" - csiV0ControllerPublishSecretNamespaceKey = "csiControllerPublishSecretNamespace" - csiV0NodePublishSecretNameKey = "csiNodePublishSecretName" - csiV0NodePublishSecretNamespaceKey = "csiNodePublishSecretNamespace" - csiV1FSType = csiParameterPrefix + "fstype" - csiV1ProvisionerSecretNameKey = csiParameterPrefix + "provisioner-secret-name" - csiV1ProvisionerSecretNamespaceKey = csiParameterPrefix + "provisioner-secret-namespace" - csiV1ControllerPublishSecretNameKey = csiParameterPrefix + "controller-publish-secret-name" - csiV1ControllerPublishSecretNamespaceKey = csiParameterPrefix + "controller-publish-secret-namespace" - csiV1NodePublishSecretNameKey = csiParameterPrefix + "node-publish-secret-name" - csiV1NodePublishSecretNamespaceKey = csiParameterPrefix + "node-publish-secret-namespace" - csiV1ControllerExpandSecretNameKey = csiParameterPrefix + "controller-expand-secret-name" - csiV1ControllerExpandSecretnamespaceKey = csiParameterPrefix + "controller-expand-secret-namespace" - tlsCertKey = "tls.crt" - tlsKeyKey = "tls.key" - credUsernameKey = "username" - credPasswordKey = "password" + defaultFSType = "ext4" + apiAddressKey = "apiAddress" + apiUsernameKey = "apiUsername" + apiPasswordKey = "apiPassword" + csiParameterPrefix = "csi.storage.k8s.io/" + csiProvisionUsernameKey = "csiProvisionUsername" + csiProvisionPasswordKey = "csiProvisionPassword" + csiControllerPublishUsernameKey = "csiControllerPublishUsername" + csiControllerPublishPasswordKey = "csiControllerPublishPassword" + csiNodePublishUsernameKey = "csiNodePublishUsername" + csiNodePublishPasswordKey = "csiNodePublishPassword" + csiControllerExpandUsernameKey = "csiControllerExpandUsername" + csiControllerExpandPasswordKey = "csiControllerExpandPassword" + csiFSType = csiParameterPrefix + "fstype" + csiProvisionerSecretNameKey = csiParameterPrefix + "provisioner-secret-name" + csiProvisionerSecretNamespaceKey = csiParameterPrefix + "provisioner-secret-namespace" + csiControllerPublishSecretNameKey = csiParameterPrefix + "controller-publish-secret-name" + csiControllerPublishSecretNamespaceKey = csiParameterPrefix + "controller-publish-secret-namespace" + csiNodePublishSecretNameKey = csiParameterPrefix + "node-publish-secret-name" + csiNodePublishSecretNamespaceKey = csiParameterPrefix + "node-publish-secret-namespace" + csiControllerExpandSecretNameKey = csiParameterPrefix + "controller-expand-secret-name" + csiControllerExpandSecretnamespaceKey = csiParameterPrefix + "controller-expand-secret-namespace" + tlsCertKey = "tls.crt" + tlsKeyKey = "tls.key" + credUsernameKey = "username" + credPasswordKey = "password" defaultUsername = "storageos" defaultPassword = "storageos" @@ -116,14 +104,6 @@ func (s *Deployment) Deploy() error { return err } - if err := s.createClusterRoleForNFS(); err != nil { - return err - } - - if err := s.createClusterRoleBindingForNFS(); err != nil { - return err - } - if err := s.createClusterRoleForInit(); err != nil { return err } @@ -297,24 +277,6 @@ func (s *Deployment) addNodeContainerResources(nodeContainer *corev1.Container) } } -// kubeletPluginsWatcherSupported checks if the given version of k8s supports -// KubeletPluginsWatcher. This is used to change the CSI driver registry setup -// based on the kubernetes cluster setup. -func kubeletPluginsWatcherSupported(version string) bool { - // Supported if v1.12.0 or above. - return versionSupported(version, "1.12.0") -} - -// CSIV1Supported returns true for k8s versions that support CSI v1. -func CSIV1Supported(version string) bool { - return versionSupported(version, "1.13.0") -} - -// CSIExternalAttacherV2Supported returns true for k8s 1.14+. -func CSIExternalAttacherV2Supported(version string) bool { - return versionSupported(version, "1.14.0") -} - // CSIExternalResizerSupported returns true for k8s 1.16+. func CSIExternalResizerSupported(version string) bool { return versionSupported(version, "1.16.0") diff --git a/pkg/storageos/deploy_test.go b/pkg/storageos/deploy_test.go index c9cb0235f..562bb5bef 100644 --- a/pkg/storageos/deploy_test.go +++ b/pkg/storageos/deploy_test.go @@ -162,7 +162,6 @@ func TestCreateDaemonSet(t *testing.T) { testcases := []struct { name string spec api.StorageOSClusterSpec - wantEnableCSI bool wantSharedDir string wantDisableTelemetry bool wantDisableFencing bool @@ -174,22 +173,31 @@ func TestCreateDaemonSet(t *testing.T) { wantPasswordEnvVar string }{ { - name: "legacy-daemonset", + name: "daemonset", spec: api.StorageOSClusterSpec{ SecretRefName: "foo-secret", SecretRefNamespace: "default", + CSI: api.StorageOSClusterCSI{ + Enable: true, + }, }, }, { - name: "csi-daemonset", + name: "csi enabled by default", + spec: api.StorageOSClusterSpec{ + SecretRefName: "foo-secret", + SecretRefNamespace: "default", + }, + }, + { + name: "csi can't be disabled", spec: api.StorageOSClusterSpec{ SecretRefName: "foo-secret", SecretRefNamespace: "default", CSI: api.StorageOSClusterCSI{ - Enable: true, + Enable: false, }, }, - wantEnableCSI: true, }, { name: "shared-dir", @@ -254,123 +262,115 @@ func TestCreateDaemonSet(t *testing.T) { } for _, tc := range testcases { - // Create fake client with pre-existing resources. - c := fake.NewFakeClientWithScheme(testScheme, etcdSecret) + var tc = tc - stosCluster.Spec = tc.spec - deploy, err := setupFakeDeploymentWithClientAndCluster(c, stosCluster) - if err != nil { - t.Fatalf("failed to create deployment: %v", err) - } - if err := deploy.createDaemonSet(); err != nil { - t.Fatal("failed to create daemonset", err) - } + t.Run(tc.name, func(t *testing.T) { + // Create fake client with pre-existing resources. + c := fake.NewFakeClientWithScheme(testScheme, etcdSecret) - nsName := types.NamespacedName{ - Name: DaemonSetName, - Namespace: defaultNS, - } - createdDaemonset := &appsv1.DaemonSet{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "apps/v1", - Kind: "DaemonSet", - }, - ObjectMeta: metav1.ObjectMeta{ + stosCluster.Spec = tc.spec + deploy, err := setupFakeDeploymentWithClientAndCluster(c, stosCluster) + if err != nil { + t.Fatalf("failed to create deployment: %v", err) + } + if err := deploy.createDaemonSet(); err != nil { + t.Fatal("failed to create daemonset", err) + } + + nsName := types.NamespacedName{ Name: DaemonSetName, Namespace: defaultNS, - }, - } - if err := c.Get(context.Background(), nsName, createdDaemonset); err != nil { - t.Fatal("failed to get the created object", err) - } - - // Check if default container for logging is set. - got, ok := createdDaemonset.Spec.Template.Annotations[DefaultLogsContainerAnnotationName] - if !ok { - t.Errorf("expected annotation %q not set", DefaultLogsContainerAnnotationName) - } - if ok && got != NodeContainerName { - t.Errorf("expected annotation %q set to %q, want %q", DefaultLogsContainerAnnotationName, got, NodeContainerName) - } - - if tc.wantEnableCSI { - if len(createdDaemonset.Spec.Template.Spec.Containers) != 2 { - t.Errorf("unexpected number of containers in daemonset:\n\t(WNT) %d\n\t(GOT): %d", len(createdDaemonset.Spec.Template.Spec.Containers), 2) } - } else { - if len(createdDaemonset.Spec.Template.Spec.Containers) != 1 { - t.Errorf("unexpected number of containers in daemonset:\n\t(WNT) %d\n\t(GOT): %d", len(createdDaemonset.Spec.Template.Spec.Containers), 1) + createdDaemonset := &appsv1.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "DaemonSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: DaemonSetName, + Namespace: defaultNS, + }, + } + if err := c.Get(context.Background(), nsName, createdDaemonset); err != nil { + t.Fatal("failed to get the created object", err) } - } - if tc.wantSharedDir != "" { - sharedDirVolFound := false - for _, vol := range createdDaemonset.Spec.Template.Spec.Volumes { - if vol.Name == "shared" { - sharedDirVolFound = true - if vol.HostPath.Path != tc.wantSharedDir { - t.Errorf("unexpected sharedDir path:\n\t(WNT) %s\n\t(GOT) %s", tc.wantSharedDir, vol.HostPath.Path) - } - break + if len(createdDaemonset.Spec.Template.Spec.Containers) != 3 { + for _, c := range createdDaemonset.Spec.Template.Spec.Containers { + fmt.Printf("XXX: %s: %s\n", c.Name, c.Image) } + t.Errorf("unexpected number of containers in daemonset:\n\t(WNT) %d\n\t(GOT): %d", 3, len(createdDaemonset.Spec.Template.Spec.Containers)) } - if !sharedDirVolFound { - t.Errorf("expected shared volume, but not found") - } - } - if tc.wantTLSEtcd { - // Check if the TLS certs volume exists in the spec. - volumeFound := false - for _, vol := range createdDaemonset.Spec.Template.Spec.Volumes { - if vol.Name == tlsEtcdCertsVolume { - volumeFound = true + if tc.wantSharedDir != "" { + sharedDirVolFound := false + for _, vol := range createdDaemonset.Spec.Template.Spec.Volumes { + if vol.Name == "shared" { + sharedDirVolFound = true + if vol.HostPath.Path != tc.wantSharedDir { + t.Errorf("unexpected sharedDir path:\n\t(WNT) %s\n\t(GOT) %s", tc.wantSharedDir, vol.HostPath.Path) + } + break + } + } + if !sharedDirVolFound { + t.Errorf("expected shared volume, but not found") } - } - if !volumeFound { - t.Error("TLS etcd certs volume not found in daemonset spec") } - // Check if TLS certs volume mount exists in the node container. - volumeMountFound := false - for _, volMount := range createdDaemonset.Spec.Template.Spec.Containers[0].VolumeMounts { - if volMount.Name == tlsEtcdCertsVolume && - volMount.MountPath == tlsEtcdRootPath { - volumeMountFound = true + if tc.wantTLSEtcd { + // Check if the TLS certs volume exists in the spec. + volumeFound := false + for _, vol := range createdDaemonset.Spec.Template.Spec.Volumes { + if vol.Name == tlsEtcdCertsVolume { + volumeFound = true + } + } + if !volumeFound { + t.Error("TLS etcd certs volume not found in daemonset spec") + } + + // Check if TLS certs volume mount exists in the node container. + volumeMountFound := false + for _, volMount := range createdDaemonset.Spec.Template.Spec.Containers[0].VolumeMounts { + if volMount.Name == tlsEtcdCertsVolume && + volMount.MountPath == tlsEtcdRootPath { + volumeMountFound = true + } + } + if !volumeMountFound { + t.Error("TLS etcd certs volume mount not found in the node container") } } - if !volumeMountFound { - t.Error("TLS etcd certs volume mount not found in the node container") - } - } - // Check the username and password env vars only when wanted username - // and passwords are provided. - if tc.wantUserNameEnvVar != "" && tc.wantPasswordEnvVar != "" { - // First container is the node container. - nodeEnvs := createdDaemonset.Spec.Template.Spec.Containers[0].Env + // Check the username and password env vars only when wanted username + // and passwords are provided. + if tc.wantUserNameEnvVar != "" && tc.wantPasswordEnvVar != "" { + // First container is the node container. + nodeEnvs := createdDaemonset.Spec.Template.Spec.Containers[0].Env - // 2nd env var is the username. - usernameEnvVar := nodeEnvs[1].Name - // 3rd env var is the password. - passwordEnvVar := nodeEnvs[2].Name + // 2nd env var is the username. + usernameEnvVar := nodeEnvs[1].Name + // 3rd env var is the password. + passwordEnvVar := nodeEnvs[2].Name - if usernameEnvVar != tc.wantUserNameEnvVar { - t.Errorf("unexpected username env var name:\n\t(WNT) %q\n\t(GOT) %q", tc.wantUserNameEnvVar, usernameEnvVar) - } + if usernameEnvVar != tc.wantUserNameEnvVar { + t.Errorf("unexpected username env var name:\n\t(WNT) %q\n\t(GOT) %q", tc.wantUserNameEnvVar, usernameEnvVar) + } - if passwordEnvVar != tc.wantPasswordEnvVar { - t.Errorf("unexpected password env var name:\n\t(WNT) %q\n\t(GOT) %q", tc.wantPasswordEnvVar, passwordEnvVar) + if passwordEnvVar != tc.wantPasswordEnvVar { + t.Errorf("unexpected password env var name:\n\t(WNT) %q\n\t(GOT) %q", tc.wantPasswordEnvVar, passwordEnvVar) + } } - } - stosCluster.Spec = api.StorageOSClusterSpec{} - if err := c.Delete(context.Background(), createdDaemonset); err != nil { - t.Error(err) - } - if err := c.Get(context.Background(), nsName, createdDaemonset); err == nil { - t.Fatal("failed to delete the created object", err) - } + stosCluster.Spec = api.StorageOSClusterSpec{} + if err := c.Delete(context.Background(), createdDaemonset); err != nil { + t.Error(err) + } + if err := c.Get(context.Background(), nsName, createdDaemonset); err == nil { + t.Fatal("failed to delete the created object", err) + } + }) } } @@ -598,7 +598,7 @@ func TestCreateAPIManager(t *testing.T) { func TestDeployCSI(t *testing.T) { const ( kubeletPluginsWatcherDriverRegArgsCount = 3 - containersCount = 2 + containersCount = 3 // node, cs-driver-registrar, csi-liveness-probe volumesCount = 10 //Includes ConfigMap volume ) @@ -619,31 +619,24 @@ func TestDeployCSI(t *testing.T) { } testCases := []struct { - name string - k8sVersion string - supportsKubeletPluginsWatcher bool + name string + k8sVersion string }{ { name: "empty", k8sVersion: "", }, { - name: "1.9.0", - k8sVersion: "1.9.0", - }, - { - name: "1.11.0", - k8sVersion: "1.11.0", + name: "1.15.0", + k8sVersion: "1.15.0", }, { - name: "1.12.0", - k8sVersion: "1.12.0", - supportsKubeletPluginsWatcher: true, + name: "1.20.0", + k8sVersion: "1.20.0", }, { - name: "1.12.2", - k8sVersion: "1.12.2", - supportsKubeletPluginsWatcher: true, + name: "1.20.2", + k8sVersion: "1.20.2", }, { name: "1.9.1+a0ce1bc657", @@ -697,20 +690,6 @@ func TestDeployCSI(t *testing.T) { if len(createdDaemonset.Spec.Template.Spec.Volumes) != volumesCount { t.Errorf("unexpected number of volumes in the DaemonSet:\n\t(GOT) %d\n\t(WNT) %d", len(createdDaemonset.Spec.Template.Spec.Volumes), volumesCount) } - - // KubeletPluginsWatcher support is only on k8s 1.12.0+. - if kubeletPluginsWatcherSupported(tc.k8sVersion) != tc.supportsKubeletPluginsWatcher { - t.Errorf("expected KubeletPluginsWatcherSupported to be %t", tc.supportsKubeletPluginsWatcher) - } - - // When KubeletPluginsWatcher is supported, some extra arguments are - // passed to set the proper registration mode. - if kubeletPluginsWatcherSupported(tc.k8sVersion) { - driverReg := createdDaemonset.Spec.Template.Spec.Containers[1] - if len(driverReg.Args) != kubeletPluginsWatcherDriverRegArgsCount { - t.Errorf("unexpected number of args for DriverRegistration container:\n\t(GOT) %d\n\t(WNT) %d", len(driverReg.Args), kubeletPluginsWatcherDriverRegArgsCount) - } - } }) } } @@ -1858,34 +1837,26 @@ func TestContainerImageSelection(t *testing.T) { csiExternalAttacherImage = "CSIExternalAttacher" csiLivenessProbeImage = "CSILivenessProbe" kubeSchedulerImage = "KubeScheduler" - nfsImage = "NFS" ) // Given image name, cluster spec and k8s version, return the appropriate // image. getImage := func(name string, spec api.StorageOSClusterSpec, k8sVersion string) string { - csiV1Supported := CSIV1Supported(k8sVersion) - attacherV2Supported := CSIExternalAttacherV2Supported(k8sVersion) - switch name { case storageOSNodeImage: return spec.GetNodeContainerImage() case storageOSInitImage: return spec.GetInitContainerImage() - case csiClusterDriverRegistrarImage: - return spec.GetCSIClusterDriverRegistrarImage() case csiNodeDriverRegistrarImage: - return spec.GetCSINodeDriverRegistrarImage(csiV1Supported) + return spec.GetCSINodeDriverRegistrarImage() case csiExternalProvisionerImage: - return spec.GetCSIExternalProvisionerImage(csiV1Supported) + return spec.GetCSIExternalProvisionerImage() case csiExternalAttacherImage: - return spec.GetCSIExternalAttacherImage(csiV1Supported, attacherV2Supported) + return spec.GetCSIExternalAttacherImage() case csiLivenessProbeImage: return spec.GetCSILivenessProbeImage() case kubeSchedulerImage: return spec.GetKubeSchedulerImage(k8sVersion) - case nfsImage: - return spec.GetNFSServerImage() default: return "" } @@ -1901,105 +1872,77 @@ func TestContainerImageSelection(t *testing.T) { { name: "images from env var - k8s 1.13", envVars: map[string]string{ - image.StorageOSNodeImageEnvVar: "foo/node:1", - image.StorageOSInitImageEnvVar: "foo/init:1", - image.CSIv1ClusterDriverRegistrarImageEnvVar: "foo/cdr:1", - image.CSIv1NodeDriverRegistrarImageEnvVar: "foo/ndr:1", - image.CSIv1ExternalProvisionerImageEnvVar: "foo/ep:1", + image.StorageOSNodeImageEnvVar: "foo/node:1", + image.StorageOSInitImageEnvVar: "foo/init:1", + image.CSINodeDriverRegistrarImageEnvVar: "foo/ndr:1", + image.CSIExternalProvisionerImageEnvVar: "foo/ep:1", // k8s 1.13 supports CSI external attacher v1 only. - image.CSIv1ExternalAttacherImageEnvVar: "foo/ea:1", - image.CSIv1LivenessProbeImageEnvVar: "foo/lp:1", - image.KubeSchedulerImageEnvVar: "foo/ks:1", - image.NFSImageEnvVar: "foo/nfs:1", + image.CSIExternalAttacherImageEnvVar: "foo/ea:1", + image.CSILivenessProbeImageEnvVar: "foo/lp:1", + image.KubeSchedulerImageEnvVar: "foo/ks:1", }, k8sVersion: "1.13.0", wantImages: map[string]string{ - storageOSNodeImage: "foo/node:1", - storageOSInitImage: "foo/init:1", - csiClusterDriverRegistrarImage: "foo/cdr:1", - csiNodeDriverRegistrarImage: "foo/ndr:1", - csiExternalProvisionerImage: "foo/ep:1", - csiExternalAttacherImage: "foo/ea:1", - csiLivenessProbeImage: "foo/lp:1", - kubeSchedulerImage: "foo/ks:1", - nfsImage: "foo/nfs:1", + storageOSNodeImage: "foo/node:1", + storageOSInitImage: "foo/init:1", + csiNodeDriverRegistrarImage: "foo/ndr:1", + csiExternalProvisionerImage: "foo/ep:1", + csiExternalAttacherImage: "foo/ea:1", + csiLivenessProbeImage: "foo/lp:1", + kubeSchedulerImage: "foo/ks:1", }, }, { name: "images override from cluster spec - k8s 1.13", envVars: map[string]string{ - image.StorageOSNodeImageEnvVar: "foo/node:1", - image.StorageOSInitImageEnvVar: "foo/init:1", - image.CSIv1ClusterDriverRegistrarImageEnvVar: "foo/cdr:1", - image.CSIv1NodeDriverRegistrarImageEnvVar: "foo/ndr:1", - image.CSIv1ExternalProvisionerImageEnvVar: "foo/ep:1", + image.StorageOSNodeImageEnvVar: "foo/node:1", + image.StorageOSInitImageEnvVar: "foo/init:1", + image.CSINodeDriverRegistrarImageEnvVar: "foo/ndr:1", + image.CSIExternalProvisionerImageEnvVar: "foo/ep:1", // k8s 1.13 supports CSI external attacher v1 only. - image.CSIv1ExternalAttacherImageEnvVar: "foo/ea:1", - image.CSIv1LivenessProbeImageEnvVar: "foo/lp:1", - image.KubeSchedulerImageEnvVar: "foo/ks:1", - image.NFSImageEnvVar: "foo/nfs:1", + image.CSIExternalAttacherImageEnvVar: "foo/ea:1", + image.CSILivenessProbeImageEnvVar: "foo/lp:1", + image.KubeSchedulerImageEnvVar: "foo/ks:1", }, clusterSpec: api.StorageOSClusterSpec{ Images: api.ContainerImages{ - NodeContainer: "zoo/node:1", - InitContainer: "zoo/init:1", - CSIClusterDriverRegistrarContainer: "zoo/cdr:1", - CSINodeDriverRegistrarContainer: "zoo/ndr:1", - CSIExternalProvisionerContainer: "zoo/ep:1", - CSIExternalAttacherContainer: "zoo/ea:1", - CSILivenessProbeContainer: "zoo/lp:1", - KubeSchedulerContainer: "zoo/ks:1", - NFSContainer: "zoo/nfs:1", + NodeContainer: "zoo/node:1", + InitContainer: "zoo/init:1", + CSINodeDriverRegistrarContainer: "zoo/ndr:1", + CSIExternalProvisionerContainer: "zoo/ep:1", + CSIExternalAttacherContainer: "zoo/ea:1", + CSILivenessProbeContainer: "zoo/lp:1", + KubeSchedulerContainer: "zoo/ks:1", }, }, k8sVersion: "1.13.0", wantImages: map[string]string{ - storageOSNodeImage: "zoo/node:1", - storageOSInitImage: "zoo/init:1", - csiClusterDriverRegistrarImage: "zoo/cdr:1", - csiNodeDriverRegistrarImage: "zoo/ndr:1", - csiExternalProvisionerImage: "zoo/ep:1", - csiExternalAttacherImage: "zoo/ea:1", - csiLivenessProbeImage: "zoo/lp:1", - kubeSchedulerImage: "zoo/ks:1", - nfsImage: "zoo/nfs:1", + storageOSNodeImage: "zoo/node:1", + storageOSInitImage: "zoo/init:1", + csiNodeDriverRegistrarImage: "zoo/ndr:1", + csiExternalProvisionerImage: "zoo/ep:1", + csiExternalAttacherImage: "zoo/ea:1", + csiLivenessProbeImage: "zoo/lp:1", + kubeSchedulerImage: "zoo/ks:1", }, }, { name: "no env vars, no overrides, fallback images - k8s 1.13, node v2", k8sVersion: "1.13.0", wantImages: map[string]string{ - storageOSNodeImage: image.DefaultNodeContainerImage, - storageOSInitImage: image.DefaultInitContainerImage, - csiClusterDriverRegistrarImage: image.CSIv1ClusterDriverRegistrarContainerImage, - csiNodeDriverRegistrarImage: image.CSIv1NodeDriverRegistrarContainerImage, - csiExternalProvisionerImage: image.CSIv1ExternalProvisionerContainerImageV2, - csiExternalAttacherImage: image.CSIv1ExternalAttacherContainerImage, - csiLivenessProbeImage: image.CSIv1LivenessProbeContainerImage, - kubeSchedulerImage: fmt.Sprintf("%s:%s", image.DefaultKubeSchedulerContainerRegistry, "v1.13.0"), - nfsImage: image.DefaultNFSContainerImage, - }, - }, - { - name: "env var images - k8s 1.12 - CSIv0", - envVars: map[string]string{ - image.CSIv0DriverRegistrarImageEnvVar: "foo/dr:1", - image.CSIv0ExternalProvisionerImageEnvVar: "foo/ep:1", - image.CSIv0ExternalAttacherImageEnvVar: "foo/ea:1", - }, - k8sVersion: "1.12.0", - // Only relevant images. - // Use CSI v0 helper images. - wantImages: map[string]string{ - csiNodeDriverRegistrarImage: "foo/dr:1", - csiExternalProvisionerImage: "foo/ep:1", - csiExternalAttacherImage: "foo/ea:1", + storageOSNodeImage: image.DefaultNodeContainerImage, + storageOSInitImage: image.DefaultInitContainerImage, + csiNodeDriverRegistrarImage: image.CSINodeDriverRegistrarContainerImage, + csiExternalProvisionerImage: image.CSIExternalProvisionerContainerImage, + csiExternalAttacherImage: image.CSIExternalAttacherContainerImage, + csiLivenessProbeImage: image.CSILivenessProbeContainerImage, + kubeSchedulerImage: fmt.Sprintf("%s:%s", image.DefaultKubeSchedulerContainerRegistry, "v1.13.0"), }, }, { name: "env var images - k8s 1.14 - CSIv1", envVars: map[string]string{ - image.CSIv1ExternalAttacherv2ImageEnvVar: "foo/ea:2", + image.CSIExternalAttacherImageEnvVar: "foo/ea:2", }, k8sVersion: "1.14.0", // Only relevant images. diff --git a/pkg/storageos/podspec.go b/pkg/storageos/podspec.go index 8e0ef8474..66a32b461 100644 --- a/pkg/storageos/podspec.go +++ b/pkg/storageos/podspec.go @@ -59,162 +59,151 @@ func (s *Deployment) addCSI(podSpec *corev1.PodSpec) { nodeContainer := &podSpec.Containers[0] - // Add CSI specific configurations if enabled. - if s.stos.Spec.CSI.Enable { - vols := []corev1.Volume{ - { - Name: "registrar-socket-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: s.stos.Spec.GetCSIRegistrarSocketDir(), - Type: &hostpathDirOrCreate, - }, + vols := []corev1.Volume{ + { + Name: "registrar-socket-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: s.stos.Spec.GetCSIRegistrarSocketDir(), + Type: &hostpathDirOrCreate, }, }, - { - Name: "kubelet-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: s.stos.Spec.GetCSIKubeletDir(), - Type: &hostpathDir, - }, + }, + { + Name: "kubelet-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: s.stos.Spec.GetCSIKubeletDir(), + Type: &hostpathDir, }, }, - { - Name: "plugin-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: s.stos.Spec.GetCSIPluginDir(CSIV1Supported(s.k8sVersion)), - Type: &hostpathDirOrCreate, - }, + }, + { + Name: "plugin-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: s.stos.Spec.GetCSIPluginDir(), + Type: &hostpathDirOrCreate, }, }, - { - Name: "device-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: s.stos.Spec.GetCSIDeviceDir(), - Type: &hostpathDir, - }, + }, + { + Name: "device-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: s.stos.Spec.GetCSIDeviceDir(), + Type: &hostpathDir, }, }, - { - Name: "registration-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: s.stos.Spec.GetCSIRegistrationDir(CSIV1Supported(s.k8sVersion)), - Type: &hostpathDir, - }, + }, + { + Name: "registration-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: s.stos.Spec.GetCSIRegistrationDir(), + Type: &hostpathDir, }, }, - } + }, + } - podSpec.Volumes = append(podSpec.Volumes, vols...) + podSpec.Volumes = append(podSpec.Volumes, vols...) - volMnts := []corev1.VolumeMount{ - { - Name: "kubelet-dir", - MountPath: s.stos.Spec.GetCSIKubeletDir(), - MountPropagation: &mountPropagationBidirectional, - }, - { - Name: "device-dir", - MountPath: s.stos.Spec.GetCSIDeviceDir(), - }, - } - // Only add a mount for the plugin-dir if it's not under the kubelet-dir - // mount path, which is now the k8s default. Overlapping mounts will - // cause unmount issues when the container restarts, leaving entries in - // /proc/mounts. - if !strings.HasPrefix(s.stos.Spec.GetCSIPluginDir(CSIV1Supported(s.k8sVersion)), s.stos.Spec.GetCSIKubeletDir()) { - volMnts = append(volMnts, corev1.VolumeMount{ - Name: "plugin-dir", - MountPath: s.stos.Spec.GetCSIPluginDir(CSIV1Supported(s.k8sVersion)), - }) - } + volMnts := []corev1.VolumeMount{ + { + Name: "kubelet-dir", + MountPath: s.stos.Spec.GetCSIKubeletDir(), + MountPropagation: &mountPropagationBidirectional, + }, + { + Name: "device-dir", + MountPath: s.stos.Spec.GetCSIDeviceDir(), + }, + } + // Only add a mount for the plugin-dir if it's not under the kubelet-dir + // mount path, which is now the k8s default. Overlapping mounts will + // cause unmount issues when the container restarts, leaving entries in + // /proc/mounts. + if !strings.HasPrefix(s.stos.Spec.GetCSIPluginDir(), s.stos.Spec.GetCSIKubeletDir()) { + volMnts = append(volMnts, corev1.VolumeMount{ + Name: "plugin-dir", + MountPath: s.stos.Spec.GetCSIPluginDir(), + }) + } - // Append volume mounts to the first container, the only container is the node container, at this point. - nodeContainer.VolumeMounts = append(nodeContainer.VolumeMounts, volMnts...) + // Append volume mounts to the first container, the only container is the node container, at this point. + nodeContainer.VolumeMounts = append(nodeContainer.VolumeMounts, volMnts...) - driverReg := corev1.Container{ - Image: s.stos.Spec.GetCSINodeDriverRegistrarImage(CSIV1Supported(s.k8sVersion)), - Name: "csi-driver-registrar", - ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{ - "--v=5", - "--csi-address=$(ADDRESS)", + driverReg := corev1.Container{ + Image: s.stos.Spec.GetCSINodeDriverRegistrarImage(), + Name: "csi-driver-registrar", + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{ + "--v=5", + "--csi-address=$(ADDRESS)", + "--kubelet-registration-path=%s", s.stos.Spec.GetCSIKubeletRegistrationPath(), + }, + Env: []corev1.EnvVar{ + { + Name: addressEnvVar, + Value: "/csi/csi.sock", }, - Env: []corev1.EnvVar{ - { - Name: addressEnvVar, - Value: "/csi/csi.sock", - }, - { - Name: kubeNodeNameEnvVar, - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - APIVersion: "v1", - FieldPath: "spec.nodeName", - }, + { + Name: kubeNodeNameEnvVar, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "spec.nodeName", }, }, }, - SecurityContext: &corev1.SecurityContext{ - Privileged: &privileged, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: &privileged, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "plugin-dir", + MountPath: "/csi", }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "plugin-dir", - MountPath: "/csi", - }, - { - Name: "registrar-socket-dir", - MountPath: "/var/lib/csi/sockets/", - }, - { - Name: "registration-dir", - MountPath: "/registration", - }, + { + Name: "registrar-socket-dir", + MountPath: "/var/lib/csi/sockets/", }, - } + { + Name: "registration-dir", + MountPath: "/registration", + }, + }, + } - // Add extra flags to activate node-register mode if kubelet plugins - // watcher is supported. - if kubeletPluginsWatcherSupported(s.k8sVersion) { - driverReg.Args = append( - driverReg.Args, - fmt.Sprintf("--kubelet-registration-path=%s", s.stos.Spec.GetCSIKubeletRegistrationPath(CSIV1Supported(s.k8sVersion)))) - } - podSpec.Containers = append(podSpec.Containers, driverReg) + podSpec.Containers = append(podSpec.Containers, driverReg) - if CSIV1Supported(s.k8sVersion) { - livenessProbe := corev1.Container{ - Image: s.stos.Spec.GetCSILivenessProbeImage(), - Name: "csi-liveness-probe", - ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{ - "--csi-address=$(ADDRESS)", - "--connection-timeout=3s", - }, - Env: []corev1.EnvVar{ - { - Name: addressEnvVar, - Value: "/csi/csi.sock", - }, - }, - SecurityContext: &corev1.SecurityContext{ - Privileged: &privileged, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "plugin-dir", - MountPath: "/csi", - }, - }, - } - podSpec.Containers = append(podSpec.Containers, livenessProbe) - } + livenessProbe := corev1.Container{ + Image: s.stos.Spec.GetCSILivenessProbeImage(), + Name: "csi-liveness-probe", + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{ + "--csi-address=$(ADDRESS)", + "--connection-timeout=3s", + }, + Env: []corev1.EnvVar{ + { + Name: addressEnvVar, + Value: "/csi/csi.sock", + }, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: &privileged, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "plugin-dir", + MountPath: "/csi", + }, + }, } + podSpec.Containers = append(podSpec.Containers, livenessProbe) } // addNodeAffinity adds node affinity to the given pod spec from the cluster diff --git a/pkg/storageos/rbac.go b/pkg/storageos/rbac.go index 1f08f278f..4262ebcf0 100644 --- a/pkg/storageos/rbac.go +++ b/pkg/storageos/rbac.go @@ -35,9 +35,6 @@ const ( FencingClusterRoleName = "storageos:pod-fencer" FencingClusterBindingName = "storageos:pod-fencer" - NFSClusterRoleName = "storageos:nfs-provisioner" - NFSClusterBindingName = "storageos:nfs-provisioner" - SchedulerClusterRoleName = "storageos:scheduler-extender" SchedulerClusterBindingName = "storageos:scheduler-extender" @@ -174,17 +171,6 @@ func (s *Deployment) createClusterRoleForFencing() error { return s.k8sResourceManager.ClusterRole(FencingClusterRoleName, nil, rules).Create() } -func (s *Deployment) createClusterRoleForNFS() error { - rules := []rbacv1.PolicyRule{ - { - APIGroups: []string{"storageos.com"}, - Resources: []string{"nfsservers"}, - Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"}, - }, - } - return s.k8sResourceManager.ClusterRole(NFSClusterRoleName, nil, rules).Create() -} - func (s *Deployment) createClusterRoleForDriverRegistrar() error { rules := []rbacv1.PolicyRule{ { @@ -408,22 +394,6 @@ func (s *Deployment) createClusterRoleBindingForFencing() error { return s.k8sResourceManager.ClusterRoleBinding(FencingClusterBindingName, nil, subjects, roleRef).Create() } -func (s *Deployment) createClusterRoleBindingForNFS() error { - subjects := []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: DaemonsetSA, - Namespace: s.stos.Spec.GetResourceNS(), - }, - } - roleRef := &rbacv1.RoleRef{ - Kind: "ClusterRole", - Name: NFSClusterRoleName, - APIGroup: "rbac.authorization.k8s.io", - } - return s.k8sResourceManager.ClusterRoleBinding(NFSClusterBindingName, nil, subjects, roleRef).Create() -} - func (s *Deployment) createClusterRoleBindingForDriverRegistrar() error { subjects := []rbacv1.Subject{ { diff --git a/pkg/storageos/storageclass.go b/pkg/storageos/storageclass.go index d2d6aa290..d71f9cfd7 100644 --- a/pkg/storageos/storageclass.go +++ b/pkg/storageos/storageclass.go @@ -5,50 +5,26 @@ func (s *Deployment) createStorageClass() error { "pool": "default", } - if s.stos.Spec.CSI.Enable { - // Add CSI creds secrets in parameters. - if CSIV1Supported(s.k8sVersion) { - // New CSI secret parameter keys were introduced in CSI v1. - parameters[csiV1FSType] = defaultFSType - if s.stos.Spec.CSI.EnableProvisionCreds { - parameters[csiV1ProvisionerSecretNameKey] = csiProvisionerSecretName - parameters[csiV1ProvisionerSecretNamespaceKey] = s.stos.Spec.GetResourceNS() - } - if s.stos.Spec.CSI.EnableControllerPublishCreds { - parameters[csiV1ControllerPublishSecretNameKey] = csiControllerPublishSecretName - parameters[csiV1ControllerPublishSecretNamespaceKey] = s.stos.Spec.GetResourceNS() - } - if s.stos.Spec.CSI.EnableNodePublishCreds { - parameters[csiV1NodePublishSecretNameKey] = csiNodePublishSecretName - parameters[csiV1NodePublishSecretNamespaceKey] = s.stos.Spec.GetResourceNS() - } - // Add expand parameters only if it's supported. - if CSIExternalResizerSupported(s.k8sVersion) { - if s.stos.Spec.CSI.EnableControllerExpandCreds { - parameters[csiV1ControllerExpandSecretNameKey] = csiControllerExpandSecretName - parameters[csiV1ControllerExpandSecretnamespaceKey] = s.stos.Spec.GetResourceNS() - } - } - } else { - parameters[fsType] = defaultFSType - if s.stos.Spec.CSI.EnableProvisionCreds { - parameters[csiV0ProvisionerSecretNameKey] = csiProvisionerSecretName - parameters[csiV0ProvisionerSecretNamespaceKey] = s.stos.Spec.GetResourceNS() - } - if s.stos.Spec.CSI.EnableControllerPublishCreds { - parameters[csiV0ControllerPublishSecretNameKey] = csiControllerPublishSecretName - parameters[csiV0ControllerPublishSecretNamespaceKey] = s.stos.Spec.GetResourceNS() - } - if s.stos.Spec.CSI.EnableNodePublishCreds { - parameters[csiV0NodePublishSecretNameKey] = csiNodePublishSecretName - parameters[csiV0NodePublishSecretNamespaceKey] = s.stos.Spec.GetResourceNS() - } + // Add CSI creds secrets in parameters. + parameters[csiFSType] = defaultFSType + if s.stos.Spec.CSI.EnableProvisionCreds { + parameters[csiProvisionerSecretNameKey] = csiProvisionerSecretName + parameters[csiProvisionerSecretNamespaceKey] = s.stos.Spec.GetResourceNS() + } + if s.stos.Spec.CSI.EnableControllerPublishCreds { + parameters[csiControllerPublishSecretNameKey] = csiControllerPublishSecretName + parameters[csiControllerPublishSecretNamespaceKey] = s.stos.Spec.GetResourceNS() + } + if s.stos.Spec.CSI.EnableNodePublishCreds { + parameters[csiNodePublishSecretNameKey] = csiNodePublishSecretName + parameters[csiNodePublishSecretNamespaceKey] = s.stos.Spec.GetResourceNS() + } + // Add expand parameters only if it's supported. + if CSIExternalResizerSupported(s.k8sVersion) { + if s.stos.Spec.CSI.EnableControllerExpandCreds { + parameters[csiControllerExpandSecretNameKey] = csiControllerExpandSecretName + parameters[csiControllerExpandSecretnamespaceKey] = s.stos.Spec.GetResourceNS() } - } else { - parameters[fsType] = defaultFSType - // Add StorageOS admin secrets name and namespace. - parameters[secretNamespaceKey] = s.stos.Spec.SecretRefNamespace - parameters[secretNameKey] = s.stos.Spec.SecretRefName } return s.k8sResourceManager.StorageClass(s.stos.Spec.GetStorageClassName(), nil, StorageOSProvisionerName, parameters).Create() diff --git a/test/e2e/clusterCSINodeV2_test.go b/test/e2e/clusterCSINodeV2_test.go index 3bffdff57..5e1de5106 100644 --- a/test/e2e/clusterCSINodeV2_test.go +++ b/test/e2e/clusterCSINodeV2_test.go @@ -4,7 +4,6 @@ package e2e import ( "context" - "strings" "testing" framework "github.com/operator-framework/operator-sdk/pkg/test" @@ -78,23 +77,10 @@ func TestClusterCSINodeV2(t *testing.T) { t.Fatalf("failed to get storageos-daemonset: %v", err) } - info, err := f.KubeClient.Discovery().ServerVersion() - if err != nil { - t.Fatalf("failed to get version info: %v", err) - } - - version := strings.TrimLeft(info.String(), "v") - t.Run("CSIHelperCountTest", func(t *testing.T) { //Check the number of containers in daemonset pod spec. - if deploy.CSIV1Supported(version) { - if len(daemonset.Spec.Template.Spec.Containers) != 3 { - t.Errorf("unexpected number of daemonset pod containers:\n\t(GOT) %d\n\t(WNT) %d", len(daemonset.Spec.Template.Spec.Containers), 3) - } - } else { - if len(daemonset.Spec.Template.Spec.Containers) != 2 { - t.Errorf("unexpected number of daemonset pod containers:\n\t(GOT) %d\n\t(WNT) %d", len(daemonset.Spec.Template.Spec.Containers), 2) - } + if len(daemonset.Spec.Template.Spec.Containers) != 3 { + t.Errorf("unexpected number of daemonset pod containers:\n\t(GOT) %d\n\t(WNT) %d", len(daemonset.Spec.Template.Spec.Containers), 3) } }) From 96e024ef4211aaad639a829090122fdafa7b86f0 Mon Sep 17 00:00:00 2001 From: Simon Croome Date: Wed, 13 Jan 2021 14:41:56 +0000 Subject: [PATCH 2/5] Block startup on k8s 1.14 and earlier --- .../storageoscluster_controller.go | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/pkg/controller/storageoscluster/storageoscluster_controller.go b/pkg/controller/storageoscluster/storageoscluster_controller.go index 72cf772cf..8265b475c 100644 --- a/pkg/controller/storageoscluster/storageoscluster_controller.go +++ b/pkg/controller/storageoscluster/storageoscluster_controller.go @@ -22,6 +22,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" + "github.com/blang/semver" "github.com/storageos/cluster-operator/internal/pkg/storageoscluster" storageosv1 "github.com/storageos/cluster-operator/pkg/apis/storageos/v1" "github.com/storageos/cluster-operator/pkg/storageos" @@ -31,11 +32,20 @@ import ( var log = logf.Log.WithName("storageos.cluster") const ( - clusterFinalizer = "finalizer.storageoscluster.storageos.com" + // KubernetesMinimumVersion is the minimum version of Kubernetes required + // for the controller to operate in. + KubernetesMinimumVersion = "1.15.0" + clusterFinalizer = "finalizer.storageoscluster.storageos.com" reconcilePeriodSeconds = 15 ) +var ( + // ErrKubernetesTooOld is returned if the Kubernetes environment does not + // meet the minimum version required. + ErrKubernetesTooOld = fmt.Errorf("kubernetes %s or above required", KubernetesMinimumVersion) +) + // Add creates a new StorageOSCluster Controller and adds it to the Manager. The Manager will set fields on the Controller // and Start it when the Manager is Started. func Add(mgr manager.Manager) error { @@ -46,6 +56,19 @@ func Add(mgr manager.Manager) error { if err != nil { return err } + supported, err := versionSupported(version, KubernetesMinimumVersion) + if err != nil { + // If the check failed, default to assuming support. This will ensure + // that strange versions (perhaps due to custom builds) will still work. + log.WithValues("k8s", version).Info("Unable to determine kubernetes version. Skipping compatibility check.") + supported = true + } + if !supported { + // Returning an error here will trigger a fatal error and block startup. + // Make sure discovered version is logged. + log.WithValues("k8s", version).Error(err, "Compatibility check failed") + return ErrKubernetesTooOld + } log.WithValues("k8s", version).Info("Adding cluster controller") @@ -541,3 +564,23 @@ func remove(list []string, s string) []string { } return list } + +// versionSupported takes two versions, current version (haveVersion) and a +// minimum requirement version (wantVersion) and checks if the current version +// is supported by comparing it with the minimum requirement. +func versionSupported(haveVersion, wantVersion string) (bool, error) { + supportedVersion, err := semver.Parse(wantVersion) + if err != nil { + return false, err + } + + currentVersion, err := semver.Parse(haveVersion) + if err != nil { + return false, err + } + + if currentVersion.Compare(supportedVersion) >= 0 { + return true, nil + } + return false, nil +} From b52d3fbe3cc2e3e8a5e601294e3ebd87fc67023d Mon Sep 17 00:00:00 2001 From: Simon Croome Date: Thu, 14 Jan 2021 12:54:27 +0000 Subject: [PATCH 3/5] Fix e2e test --- test/e2e/clusterCSINodeV2_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/clusterCSINodeV2_test.go b/test/e2e/clusterCSINodeV2_test.go index 5e1de5106..7f38b7d3d 100644 --- a/test/e2e/clusterCSINodeV2_test.go +++ b/test/e2e/clusterCSINodeV2_test.go @@ -77,8 +77,8 @@ func TestClusterCSINodeV2(t *testing.T) { t.Fatalf("failed to get storageos-daemonset: %v", err) } + //Check the number of containers in daemonset pod spec. t.Run("CSIHelperCountTest", func(t *testing.T) { - //Check the number of containers in daemonset pod spec. if len(daemonset.Spec.Template.Spec.Containers) != 3 { t.Errorf("unexpected number of daemonset pod containers:\n\t(GOT) %d\n\t(WNT) %d", len(daemonset.Spec.Template.Spec.Containers), 3) } From 582adfbf4a0a604f3dceefffb8a560a0bdd87abb Mon Sep 17 00:00:00 2001 From: avestuk Date: Thu, 14 Jan 2021 18:24:15 +0000 Subject: [PATCH 4/5] Remove some other v1.14 specific logic (#298) * Remove some other v1.14 specific logic * Remove unused func --- test/e2e/util/admission_controller.go | 27 +------------ test/e2e/util/cluster.go | 57 +-------------------------- 2 files changed, 3 insertions(+), 81 deletions(-) diff --git a/test/e2e/util/admission_controller.go b/test/e2e/util/admission_controller.go index 714396d44..3faadb945 100644 --- a/test/e2e/util/admission_controller.go +++ b/test/e2e/util/admission_controller.go @@ -6,7 +6,6 @@ import ( "testing" "time" - "github.com/blang/semver" framework "github.com/operator-framework/operator-sdk/pkg/test" corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" @@ -31,33 +30,9 @@ const ( ) // PodSchedulerAdmissionControllerTest checks if the pod scheduler mutating -// admission controller mutates the scheduler name of a pod by creates a pvc +// admission controller mutates the scheduler name of a pod by creating a pvc // backed by StorageOS and a pod that uses the PVC. -// NOTE: This test has a minimum k8s version requirement. func PodSchedulerAdmissionControllerTest(t *testing.T, ctx *framework.Context) { - k8sVerMajor := 1 - k8sVerMinor := 13 - k8sVerPatch := 0 - // Minimum version of k8s required to run this test. - minVersion := semver.Version{ - Major: uint64(k8sVerMajor), - Minor: uint64(k8sVerMinor), - Patch: uint64(k8sVerPatch), - } - - // Check the k8s version before running this test. Admission controller - // does not works on openshift 3.11 (k8s 1.11). - featureSupported, err := featureSupportAvailable(minVersion) - if err != nil { - t.Errorf("failed to check platform support for admission controller test: %v", err) - return - } - - // Skip if the feature is not supported. - if !featureSupported { - return - } - scName1 := "sc1" scName2 := "sc2" diff --git a/test/e2e/util/cluster.go b/test/e2e/util/cluster.go index 87e67b189..c8df67742 100644 --- a/test/e2e/util/cluster.go +++ b/test/e2e/util/cluster.go @@ -8,7 +8,6 @@ import ( "testing" "time" - "github.com/blang/semver" monitoringv1 "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1" framework "github.com/operator-framework/operator-sdk/pkg/test" "github.com/operator-framework/operator-sdk/pkg/test/e2eutil" @@ -19,13 +18,11 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" - logf "sigs.k8s.io/controller-runtime/pkg/log" "github.com/storageos/cluster-operator/pkg/apis" storageos "github.com/storageos/cluster-operator/pkg/apis/storageos/v1" deploy "github.com/storageos/cluster-operator/pkg/storageos" "github.com/storageos/cluster-operator/pkg/util/k8s" - "github.com/storageos/cluster-operator/pkg/util/k8sutil" ) // Time constants. @@ -333,61 +330,11 @@ func StorageOSClusterCRAttributesTest(t *testing.T, crName string, crNamespace s } } -// featureSupportAvailable can be used by tests to check if the platform -// supports the test by passing a minimum version of k8s required to run the -// test. -func featureSupportAvailable(minVersion semver.Version) (bool, error) { - log := logf.Log.WithName("test.featureSupportAvailability") - k := k8sutil.NewK8SOps(framework.Global.KubeClient, log) - version, err := k.GetK8SVersion() - if err != nil { - return false, fmt.Errorf("failed to get k8s version: %v", err) - } - - currentVersion, err := semver.Parse(version) - if err != nil { - return false, fmt.Errorf("failed to parse k8s version: %v", err) - } - - if currentVersion.Compare(minVersion) >= 0 { - // This test is supported in this version of k8s. - return true, nil - } - - // Test is not supported in this version of k8s. Skip the test. - return false, nil -} - -// CSIDriverResourceTest checks if the CSIDriver resource is created. In k8s -// 1.14+, CSIDriver is created as part of the cluster deployment. +// CSIDriverResourceTest checks if the CSIDriver resource is created. func CSIDriverResourceTest(t *testing.T, driverName string) { - k8sVerMajor := 1 - k8sVerMinor := 14 - k8sVerPatch := 0 - - // Minimum version of k8s required to run this test. - minVersion := semver.Version{ - Major: uint64(k8sVerMajor), - Minor: uint64(k8sVerMinor), - Patch: uint64(k8sVerPatch), - } - - // Check the k8s version before running this test. CSIDriver built-in - // resource does not exists in openshift 3.11 (k8s 1.11). - featureSupported, err := featureSupportAvailable(minVersion) - if err != nil { - t.Errorf("failed to check platform support for CSIDriver test: %v", err) - return - } - - // Skip if the feature is not supported. - if !featureSupported { - return - } - f := framework.Global csiDriver := &storagev1beta1.CSIDriver{} - err = f.Client.Get(goctx.TODO(), types.NamespacedName{Name: driverName}, csiDriver) + err := f.Client.Get(goctx.TODO(), types.NamespacedName{Name: driverName}, csiDriver) if err != nil { t.Errorf("CSIDriver not found: %v", err) } From b505ddcf879f009c5f36d8b6e4f33ee9ad4cdf0d Mon Sep 17 00:00:00 2001 From: Simon Croome Date: Tue, 19 Jan 2021 18:00:04 +0000 Subject: [PATCH 5/5] Rename cluster test --- test/e2e/{clusterCSINodeV2_test.go => cluster_test.go} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename test/e2e/{clusterCSINodeV2_test.go => cluster_test.go} (96%) diff --git a/test/e2e/clusterCSINodeV2_test.go b/test/e2e/cluster_test.go similarity index 96% rename from test/e2e/clusterCSINodeV2_test.go rename to test/e2e/cluster_test.go index 7f38b7d3d..259f77684 100644 --- a/test/e2e/clusterCSINodeV2_test.go +++ b/test/e2e/cluster_test.go @@ -16,8 +16,8 @@ import ( testutil "github.com/storageos/cluster-operator/test/e2e/util" ) -// TestClusterCSIDeployment test the CSI helper deployment as Deployment. -func TestClusterCSINodeV2(t *testing.T) { +// TestCluster tests the cluster deployment. +func TestCluster(t *testing.T) { ctx := framework.NewTestCtx(t) defer ctx.Cleanup() resourceNS := "kube-system"