diff --git a/hatchery/config.go b/hatchery/config.go index e10e87a..28437a9 100644 --- a/hatchery/config.go +++ b/hatchery/config.go @@ -43,6 +43,19 @@ type S3Config struct { PrefixBase string `json:"prefixBase"` // e.g. "" (we’ll compute from userName) } +// Configuration for FSx for Lustre shared software library +type FSxConfig struct { + FsxID string `json:"fsxId"` // e.g. "fs-0db570e0cb6e6f5af" + DnsName string `json:"dnsName"` // e.g. "fs-0db570e0cb6e6f5af.fsx.us-east-1.amazonaws.com" + MountName string `json:"mountName"` // e.g. "nyj6tamv" + + // Optional: defaults to "1200Gi" if not specified + Storage string `json:"storage"` // e.g. "1200Gi" + + // Optional: where the volume is mounted inside the pod + MountPath string `json:"mountPath"` // e.g. "/apps" +} + // LicenseInfo contains configuration for Gen3 supplied licenses. type LicenseInfo struct { Enabled bool `json:"enabled"` @@ -152,6 +165,7 @@ type HatcheryConfig struct { MoreConfigs []AppConfigInfo `json:"more-configs"` PrismaConfig PrismaConfig `json:"prisma"` S3Config S3Config `json:"s3-config"` + FSxConfig FSxConfig `json:fsx-config` NextflowGlobalConfig NextflowGlobalConfig `json:"nextflow-global"` Pricing Pricing `json:"pricing"` } diff --git a/hatchery/pods.go b/hatchery/pods.go index 3aeca5e..51cd118 100644 --- a/hatchery/pods.go +++ b/hatchery/pods.go @@ -386,82 +386,52 @@ func userToResourceName(userName string, resourceType string) string { return fmt.Sprintf("%s-%s", resourceType, safeUserName) } -func s3NamesForUser(userName string, namespace string) (pvName, pvcName, volumeHandle string) { - base := userToResourceName(userName, "s3") // e.g. john-doe-s3 - return base + "-" + namespace + "-pv", base + "-pvc", base + "-" + namespace // unique handle -} - -func s3PrefixForUser(userName string) string { - // If you prefer purely the username: return fmt.Sprintf("%s/", userName) - // Using resource-safe version tends to be nicer: - return fmt.Sprintf("%s/", userToResourceName(userName, "")) -} - -func ensureS3PVandPVC( +func ensureSharedSoftwareLibrary( ctx context.Context, podClient corev1.CoreV1Interface, namespace string, - userName string, - bucket string, - region string, -) (pvcName string, err error) { - - pvName, pvcName, volumeHandle := s3NamesForUser(userName, namespace) - // Commented out for now as we don't need a prefix, this bucket will be mounted at / for all users, but read only. - // This will be used for the "software-repository" feature - // prefix := s3PrefixForUser(userName) - // prefix := "/" - - // ----- Ensure PV exists (cluster-scoped) ----- - // Try GET; if not found, create. - if _, errGet := podClient.PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}); errGet != nil { + cfg FSxConfig, +) error { + + // Use the namespaced, pv name (PV is a cluster resource) + pvName := "sl-" + namespace + pvcName := "software-library-pvc" + + // 1. Ensure the Singleton PV exists + _, err := podClient.PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) + if err != nil { + Config.Logger.Printf("Creating shared Lustre PV: %s", pvName) pv := &k8sv1.PersistentVolume{ - ObjectMeta: metav1.ObjectMeta{ - Name: pvName, - }, + ObjectMeta: metav1.ObjectMeta{Name: pvName}, Spec: k8sv1.PersistentVolumeSpec{ - Capacity: k8sv1.ResourceList{ - k8sv1.ResourceStorage: resource.MustParse("1200Gi"), // ignored by S3 CSI, but required - }, - AccessModes: []k8sv1.PersistentVolumeAccessMode{k8sv1.ReadWriteMany}, - StorageClassName: "", // static provisioning - ClaimRef: &k8sv1.ObjectReference{ - Namespace: namespace, - Name: pvcName, - }, - MountOptions: []string{ - "allow-delete", - "allow-other", - "gid=100", - "file-mode=555", - "dir-mode=555", - fmt.Sprintf("region %s", region), - // fmt.Sprintf("prefix %s", prefix), - // Caches directory listings and file stats for 1 hour (fixes the 77x walk/stat slowdown) - "metadata-ttl=3600", - }, + Capacity: k8sv1.ResourceList{k8sv1.ResourceStorage: resource.MustParse("1200Gi")}, + VolumeMode: (*k8sv1.PersistentVolumeMode)(aws.String(string(k8sv1.PersistentVolumeFilesystem))), + AccessModes: []k8sv1.PersistentVolumeAccessMode{k8sv1.ReadWriteMany}, + PersistentVolumeReclaimPolicy: k8sv1.PersistentVolumeReclaimRetain, + StorageClassName: "", // Static binding PersistentVolumeSource: k8sv1.PersistentVolumeSource{ CSI: &k8sv1.CSIPersistentVolumeSource{ - Driver: "s3.csi.aws.com", - VolumeHandle: volumeHandle, // must be unique + Driver: "fsx.csi.aws.com", + VolumeHandle: cfg.FsxID, VolumeAttributes: map[string]string{ - "bucketName": bucket, - // Enables safe file caching using an ephemeral emptyDir on the K8s node - // and hard-caps the cache at 10 Gigabytes to prevent disk pressure - "cacheEmptyDirSizeLimit": "10Gi", + "mountname": cfg.MountName, + "dnsname": cfg.DnsName, }, }, }, }, } - if _, errCreatePV := podClient.PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}); errCreatePV != nil { - return "", fmt.Errorf("failed to create S3 PV %s: %w", pvName, errCreatePV) + _, err = podClient.PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}) + if err != nil { + return err } } - // ----- Ensure PVC exists (namespaced) ----- - if _, errGet := podClient.PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}); errGet != nil { - empty := "" // PVC.StorageClassName is *string + // 2. Ensure the Singleton PVC exists in the shared namespace + _, err = podClient.PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil { + Config.Logger.Printf("Creating shared Lustre PVC: %s in namespace: %s", pvcName, namespace) + emptySC := "" pvc := &k8sv1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: pvcName, @@ -469,63 +439,43 @@ func ensureS3PVandPVC( }, Spec: k8sv1.PersistentVolumeClaimSpec{ AccessModes: []k8sv1.PersistentVolumeAccessMode{k8sv1.ReadWriteMany}, - StorageClassName: &empty, + StorageClassName: &emptySC, Resources: k8sv1.VolumeResourceRequirements{ - Requests: k8sv1.ResourceList{ - k8sv1.ResourceStorage: resource.MustParse("1200Gi"), - }, + Requests: k8sv1.ResourceList{k8sv1.ResourceStorage: resource.MustParse("1200Gi")}, }, VolumeName: pvName, }, } - if _, errCreatePVC := podClient.PersistentVolumeClaims(namespace).Create(ctx, pvc, metav1.CreateOptions{}); errCreatePVC != nil { - return "", fmt.Errorf("failed to create S3 PVC %s: %w", pvcName, errCreatePVC) + _, err = podClient.PersistentVolumeClaims(namespace).Create(ctx, pvc, metav1.CreateOptions{}) + if err != nil { + return err } } - return pvcName, nil + return nil } -// Adds the S3 volume + mount to the pod in-place. -// Call this after buildPod(), before creating it. -func addS3VolumeToPod(pod *k8sv1.Pod, pvcName string) { - // 1) Add Volume - hasVolume := false - for _, v := range pod.Spec.Volumes { - if v.Name == "s3-volume" { - hasVolume = true - break - } - } - if !hasVolume { - pod.Spec.Volumes = append(pod.Spec.Volumes, k8sv1.Volume{ - Name: "s3-volume", - VolumeSource: k8sv1.VolumeSource{ - PersistentVolumeClaim: &k8sv1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvcName, - // ReadOnly: false, // optional - }, +func injectSoftwareLibrary(pod *k8sv1.Pod) { + volumeName := "software-library" + pvcName := "software-library-pvc" + + // Add Volume to Pod Spec if not present + pod.Spec.Volumes = append(pod.Spec.Volumes, k8sv1.Volume{ + Name: volumeName, + VolumeSource: k8sv1.VolumeSource{ + PersistentVolumeClaim: &k8sv1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvcName, + ReadOnly: true, // Highly recommended for a shared software library }, - }) - } + }, + }) - // 2) Add VolumeMount (/apps) to every container (or just the main one) - for ci := range pod.Spec.Containers { - c := &pod.Spec.Containers[ci] - alreadyMounted := false - for _, m := range c.VolumeMounts { - if m.Name == "s3-volume" { - alreadyMounted = true - break - } - } - if !alreadyMounted { - c.VolumeMounts = append(c.VolumeMounts, k8sv1.VolumeMount{ - Name: "s3-volume", - // TODO: Read this path from config - MountPath: "/apps", - }) - } + // Mount to all containers + for i := range pod.Spec.Containers { + pod.Spec.Containers[i].VolumeMounts = append(pod.Spec.Containers[i].VolumeMounts, k8sv1.VolumeMount{ + Name: volumeName, + MountPath: Config.Config.FSxConfig.MountPath, + }) } } @@ -880,22 +830,20 @@ var createLocalK8sPod = func(ctx context.Context, hash string, userName string, Config.Logger.Panicf("Error in createLocalK8sPod: %v", err) return err } - // ensure S3 PV/PVC (dynamic per user) and wire into pod - if Config.Config.S3Config.BucketName != "" && Config.Config.S3Config.Region != "" { - Config.Logger.Print("Mounting S3 bucket as well..") - s3PVCName, err := ensureS3PVandPVC( + + if Config.Config.FSxConfig.FsxID != "" { + err := ensureSharedSoftwareLibrary( ctx, podClient, Config.Config.UserNamespace, - userName, - Config.Config.S3Config.BucketName, - Config.Config.S3Config.Region, + Config.Config.FSxConfig, ) if err != nil { - Config.Logger.Printf("Failed ensuring S3 PV/PVC for user %s: %v", userName, err) - return err + return fmt.Errorf("failed to setup software library: %w", err) } - addS3VolumeToPod(pod, s3PVCName) + + // Wire it into the pod spec we just built + injectSoftwareLibrary(pod) } // a null image indicates a dockstore app - always mount user volume