From 3e494a4f48c993d34826e22d962f45aae92bf390 Mon Sep 17 00:00:00 2001 From: Jawad Date: Mon, 27 Apr 2026 11:05:59 -0500 Subject: [PATCH 1/2] Add FSxConfig for shared software library integration --- hatchery/config.go | 14 ++++ hatchery/pods.go | 179 ++++++++++++++++----------------------------- 2 files changed, 77 insertions(+), 116 deletions(-) diff --git a/hatchery/config.go b/hatchery/config.go index e10e87a..28437a9 100644 --- a/hatchery/config.go +++ b/hatchery/config.go @@ -43,6 +43,19 @@ type S3Config struct { PrefixBase string `json:"prefixBase"` // e.g. "" (we’ll compute from userName) } +// Configuration for FSx for Lustre shared software library +type FSxConfig struct { + FsxID string `json:"fsxId"` // e.g. "fs-0db570e0cb6e6f5af" + DnsName string `json:"dnsName"` // e.g. "fs-0db570e0cb6e6f5af.fsx.us-east-1.amazonaws.com" + MountName string `json:"mountName"` // e.g. "nyj6tamv" + + // Optional: defaults to "1200Gi" if not specified + Storage string `json:"storage"` // e.g. "1200Gi" + + // Optional: where the volume is mounted inside the pod + MountPath string `json:"mountPath"` // e.g. "/apps" +} + // LicenseInfo contains configuration for Gen3 supplied licenses. type LicenseInfo struct { Enabled bool `json:"enabled"` @@ -152,6 +165,7 @@ type HatcheryConfig struct { MoreConfigs []AppConfigInfo `json:"more-configs"` PrismaConfig PrismaConfig `json:"prisma"` S3Config S3Config `json:"s3-config"` + FSxConfig FSxConfig `json:fsx-config` NextflowGlobalConfig NextflowGlobalConfig `json:"nextflow-global"` Pricing Pricing `json:"pricing"` } diff --git a/hatchery/pods.go b/hatchery/pods.go index 3aeca5e..bc4ad90 100644 --- a/hatchery/pods.go +++ b/hatchery/pods.go @@ -386,146 +386,95 @@ func userToResourceName(userName string, resourceType string) string { return fmt.Sprintf("%s-%s", resourceType, safeUserName) } -func s3NamesForUser(userName string, namespace string) (pvName, pvcName, volumeHandle string) { - base := userToResourceName(userName, "s3") // e.g. john-doe-s3 - return base + "-" + namespace + "-pv", base + "-pvc", base + "-" + namespace // unique handle -} - -func s3PrefixForUser(userName string) string { - // If you prefer purely the username: return fmt.Sprintf("%s/", userName) - // Using resource-safe version tends to be nicer: - return fmt.Sprintf("%s/", userToResourceName(userName, "")) -} +const ( + SoftwareLibraryPVName = "software-library-lustre-pv" + SoftwareLibraryPVCName = "software-library-lustre-pvc" +) -func ensureS3PVandPVC( +func ensureSharedSoftwareLibrary( ctx context.Context, podClient corev1.CoreV1Interface, namespace string, - userName string, - bucket string, - region string, -) (pvcName string, err error) { - - pvName, pvcName, volumeHandle := s3NamesForUser(userName, namespace) - // Commented out for now as we don't need a prefix, this bucket will be mounted at / for all users, but read only. - // This will be used for the "software-repository" feature - // prefix := s3PrefixForUser(userName) - // prefix := "/" - - // ----- Ensure PV exists (cluster-scoped) ----- - // Try GET; if not found, create. - if _, errGet := podClient.PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}); errGet != nil { + cfg FSxConfig, +) error { + // 1. Ensure the Singleton PV exists + _, err := podClient.PersistentVolumes().Get(ctx, SoftwareLibraryPVName, metav1.GetOptions{}) + if err != nil { + Config.Logger.Printf("Creating shared Lustre PV: %s", SoftwareLibraryPVName) pv := &k8sv1.PersistentVolume{ - ObjectMeta: metav1.ObjectMeta{ - Name: pvName, - }, + ObjectMeta: metav1.ObjectMeta{Name: SoftwareLibraryPVName}, Spec: k8sv1.PersistentVolumeSpec{ - Capacity: k8sv1.ResourceList{ - k8sv1.ResourceStorage: resource.MustParse("1200Gi"), // ignored by S3 CSI, but required - }, - AccessModes: []k8sv1.PersistentVolumeAccessMode{k8sv1.ReadWriteMany}, - StorageClassName: "", // static provisioning - ClaimRef: &k8sv1.ObjectReference{ - Namespace: namespace, - Name: pvcName, - }, - MountOptions: []string{ - "allow-delete", - "allow-other", - "gid=100", - "file-mode=555", - "dir-mode=555", - fmt.Sprintf("region %s", region), - // fmt.Sprintf("prefix %s", prefix), - // Caches directory listings and file stats for 1 hour (fixes the 77x walk/stat slowdown) - "metadata-ttl=3600", - }, + Capacity: k8sv1.ResourceList{k8sv1.ResourceStorage: resource.MustParse("1200Gi")}, + VolumeMode: (*k8sv1.PersistentVolumeMode)(aws.String(string(k8sv1.PersistentVolumeFilesystem))), + AccessModes: []k8sv1.PersistentVolumeAccessMode{k8sv1.ReadWriteMany}, + PersistentVolumeReclaimPolicy: k8sv1.PersistentVolumeReclaimRetain, + StorageClassName: "", // Static binding PersistentVolumeSource: k8sv1.PersistentVolumeSource{ CSI: &k8sv1.CSIPersistentVolumeSource{ - Driver: "s3.csi.aws.com", - VolumeHandle: volumeHandle, // must be unique + Driver: "fsx.csi.aws.com", + VolumeHandle: cfg.FsxID, VolumeAttributes: map[string]string{ - "bucketName": bucket, - // Enables safe file caching using an ephemeral emptyDir on the K8s node - // and hard-caps the cache at 10 Gigabytes to prevent disk pressure - "cacheEmptyDirSizeLimit": "10Gi", + "mountname": cfg.MountName, + "dnsname": cfg.DnsName, }, }, }, }, } - if _, errCreatePV := podClient.PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}); errCreatePV != nil { - return "", fmt.Errorf("failed to create S3 PV %s: %w", pvName, errCreatePV) + _, err = podClient.PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}) + if err != nil { + return err } } - // ----- Ensure PVC exists (namespaced) ----- - if _, errGet := podClient.PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}); errGet != nil { - empty := "" // PVC.StorageClassName is *string + // 2. Ensure the Singleton PVC exists in the shared namespace + _, err = podClient.PersistentVolumeClaims(namespace).Get(ctx, SoftwareLibraryPVCName, metav1.GetOptions{}) + if err != nil { + Config.Logger.Printf("Creating shared Lustre PVC: %s in namespace: %s", SoftwareLibraryPVCName, namespace) + emptySC := "" pvc := &k8sv1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ - Name: pvcName, + Name: SoftwareLibraryPVCName, Namespace: namespace, }, Spec: k8sv1.PersistentVolumeClaimSpec{ AccessModes: []k8sv1.PersistentVolumeAccessMode{k8sv1.ReadWriteMany}, - StorageClassName: &empty, + StorageClassName: &emptySC, Resources: k8sv1.VolumeResourceRequirements{ - Requests: k8sv1.ResourceList{ - k8sv1.ResourceStorage: resource.MustParse("1200Gi"), - }, + Requests: k8sv1.ResourceList{k8sv1.ResourceStorage: resource.MustParse("1200Gi")}, }, - VolumeName: pvName, + VolumeName: SoftwareLibraryPVName, }, } - if _, errCreatePVC := podClient.PersistentVolumeClaims(namespace).Create(ctx, pvc, metav1.CreateOptions{}); errCreatePVC != nil { - return "", fmt.Errorf("failed to create S3 PVC %s: %w", pvcName, errCreatePVC) + _, err = podClient.PersistentVolumeClaims(namespace).Create(ctx, pvc, metav1.CreateOptions{}) + if err != nil { + return err } } - return pvcName, nil + return nil } -// Adds the S3 volume + mount to the pod in-place. -// Call this after buildPod(), before creating it. -func addS3VolumeToPod(pod *k8sv1.Pod, pvcName string) { - // 1) Add Volume - hasVolume := false - for _, v := range pod.Spec.Volumes { - if v.Name == "s3-volume" { - hasVolume = true - break - } - } - if !hasVolume { - pod.Spec.Volumes = append(pod.Spec.Volumes, k8sv1.Volume{ - Name: "s3-volume", - VolumeSource: k8sv1.VolumeSource{ - PersistentVolumeClaim: &k8sv1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvcName, - // ReadOnly: false, // optional - }, +func injectSoftwareLibrary(pod *k8sv1.Pod) { + volumeName := "software-library" + + // Add Volume to Pod Spec if not present + pod.Spec.Volumes = append(pod.Spec.Volumes, k8sv1.Volume{ + Name: volumeName, + VolumeSource: k8sv1.VolumeSource{ + PersistentVolumeClaim: &k8sv1.PersistentVolumeClaimVolumeSource{ + ClaimName: SoftwareLibraryPVCName, + ReadOnly: true, // Highly recommended for a shared software library }, - }) - } + }, + }) - // 2) Add VolumeMount (/apps) to every container (or just the main one) - for ci := range pod.Spec.Containers { - c := &pod.Spec.Containers[ci] - alreadyMounted := false - for _, m := range c.VolumeMounts { - if m.Name == "s3-volume" { - alreadyMounted = true - break - } - } - if !alreadyMounted { - c.VolumeMounts = append(c.VolumeMounts, k8sv1.VolumeMount{ - Name: "s3-volume", - // TODO: Read this path from config - MountPath: "/apps", - }) - } + // Mount to all containers + for i := range pod.Spec.Containers { + pod.Spec.Containers[i].VolumeMounts = append(pod.Spec.Containers[i].VolumeMounts, k8sv1.VolumeMount{ + Name: volumeName, + MountPath: Config.Config.FSxConfig.MountPath, + }) } } @@ -880,22 +829,20 @@ var createLocalK8sPod = func(ctx context.Context, hash string, userName string, Config.Logger.Panicf("Error in createLocalK8sPod: %v", err) return err } - // ensure S3 PV/PVC (dynamic per user) and wire into pod - if Config.Config.S3Config.BucketName != "" && Config.Config.S3Config.Region != "" { - Config.Logger.Print("Mounting S3 bucket as well..") - s3PVCName, err := ensureS3PVandPVC( + + if Config.Config.FSxConfig.FsxID != "" { + err := ensureSharedSoftwareLibrary( ctx, podClient, Config.Config.UserNamespace, - userName, - Config.Config.S3Config.BucketName, - Config.Config.S3Config.Region, + Config.Config.FSxConfig, ) if err != nil { - Config.Logger.Printf("Failed ensuring S3 PV/PVC for user %s: %v", userName, err) - return err + return fmt.Errorf("failed to setup software library: %w", err) } - addS3VolumeToPod(pod, s3PVCName) + + // Wire it into the pod spec we just built + injectSoftwareLibrary(pod) } // a null image indicates a dockstore app - always mount user volume From 01125cc141cc2bba855505452a33fe25b7b551b1 Mon Sep 17 00:00:00 2001 From: Jawad Date: Mon, 27 Apr 2026 11:47:02 -0500 Subject: [PATCH 2/2] Refactor ensureSharedSoftwareLibrary to use namespaced PV name and update PVC references --- hatchery/pods.go | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/hatchery/pods.go b/hatchery/pods.go index bc4ad90..51cd118 100644 --- a/hatchery/pods.go +++ b/hatchery/pods.go @@ -386,23 +386,23 @@ func userToResourceName(userName string, resourceType string) string { return fmt.Sprintf("%s-%s", resourceType, safeUserName) } -const ( - SoftwareLibraryPVName = "software-library-lustre-pv" - SoftwareLibraryPVCName = "software-library-lustre-pvc" -) - func ensureSharedSoftwareLibrary( ctx context.Context, podClient corev1.CoreV1Interface, namespace string, cfg FSxConfig, ) error { + + // Use the namespaced, pv name (PV is a cluster resource) + pvName := "sl-" + namespace + pvcName := "software-library-pvc" + // 1. Ensure the Singleton PV exists - _, err := podClient.PersistentVolumes().Get(ctx, SoftwareLibraryPVName, metav1.GetOptions{}) + _, err := podClient.PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) if err != nil { - Config.Logger.Printf("Creating shared Lustre PV: %s", SoftwareLibraryPVName) + Config.Logger.Printf("Creating shared Lustre PV: %s", pvName) pv := &k8sv1.PersistentVolume{ - ObjectMeta: metav1.ObjectMeta{Name: SoftwareLibraryPVName}, + ObjectMeta: metav1.ObjectMeta{Name: pvName}, Spec: k8sv1.PersistentVolumeSpec{ Capacity: k8sv1.ResourceList{k8sv1.ResourceStorage: resource.MustParse("1200Gi")}, VolumeMode: (*k8sv1.PersistentVolumeMode)(aws.String(string(k8sv1.PersistentVolumeFilesystem))), @@ -428,13 +428,13 @@ func ensureSharedSoftwareLibrary( } // 2. Ensure the Singleton PVC exists in the shared namespace - _, err = podClient.PersistentVolumeClaims(namespace).Get(ctx, SoftwareLibraryPVCName, metav1.GetOptions{}) + _, err = podClient.PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}) if err != nil { - Config.Logger.Printf("Creating shared Lustre PVC: %s in namespace: %s", SoftwareLibraryPVCName, namespace) + Config.Logger.Printf("Creating shared Lustre PVC: %s in namespace: %s", pvcName, namespace) emptySC := "" pvc := &k8sv1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ - Name: SoftwareLibraryPVCName, + Name: pvcName, Namespace: namespace, }, Spec: k8sv1.PersistentVolumeClaimSpec{ @@ -443,7 +443,7 @@ func ensureSharedSoftwareLibrary( Resources: k8sv1.VolumeResourceRequirements{ Requests: k8sv1.ResourceList{k8sv1.ResourceStorage: resource.MustParse("1200Gi")}, }, - VolumeName: SoftwareLibraryPVName, + VolumeName: pvName, }, } _, err = podClient.PersistentVolumeClaims(namespace).Create(ctx, pvc, metav1.CreateOptions{}) @@ -457,13 +457,14 @@ func ensureSharedSoftwareLibrary( func injectSoftwareLibrary(pod *k8sv1.Pod) { volumeName := "software-library" + pvcName := "software-library-pvc" // Add Volume to Pod Spec if not present pod.Spec.Volumes = append(pod.Spec.Volumes, k8sv1.Volume{ Name: volumeName, VolumeSource: k8sv1.VolumeSource{ PersistentVolumeClaim: &k8sv1.PersistentVolumeClaimVolumeSource{ - ClaimName: SoftwareLibraryPVCName, + ClaimName: pvcName, ReadOnly: true, // Highly recommended for a shared software library }, },