Skip to content

Add FSxConfig for shared software library integration #149

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jawadqur wants to merge 2 commits into
base: feat/s3-mount
Choose a base branch
from
Open

Add FSxConfig for shared software library integration #149

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3e494a4
Add FSxConfig for shared software library integration
jawadqur Apr 27, 2026
01125cc
Refactor ensureSharedSoftwareLibrary to use namespaced PV name and up…
jawadqur Apr 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions hatchery/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,19 @@ type S3Config struct {
PrefixBase string `json:"prefixBase"` // e.g. "" (we’ll compute from userName)
}

// Configuration for FSx for Lustre shared software library
type FSxConfig struct {
FsxID string `json:"fsxId"` // e.g. "fs-0db570e0cb6e6f5af"
DnsName string `json:"dnsName"` // e.g. "fs-0db570e0cb6e6f5af.fsx.us-east-1.amazonaws.com"
MountName string `json:"mountName"` // e.g. "nyj6tamv"

// Optional: defaults to "1200Gi" if not specified
Storage string `json:"storage"` // e.g. "1200Gi"

// Optional: where the volume is mounted inside the pod
MountPath string `json:"mountPath"` // e.g. "/apps"
}

// LicenseInfo contains configuration for Gen3 supplied licenses.
type LicenseInfo struct {
Enabled bool `json:"enabled"`
Expand Down Expand Up @@ -152,6 +165,7 @@ type HatcheryConfig struct {
MoreConfigs []AppConfigInfo `json:"more-configs"`
PrismaConfig PrismaConfig `json:"prisma"`
S3Config S3Config `json:"s3-config"`
FSxConfig FSxConfig `json:fsx-config`
NextflowGlobalConfig NextflowGlobalConfig `json:"nextflow-global"`
Pricing Pricing `json:"pricing"`
}
Expand Down
178 changes: 63 additions & 115 deletions hatchery/pods.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -386,146 +386,96 @@ func userToResourceName(userName string, resourceType string) string {
return fmt.Sprintf("%s-%s", resourceType, safeUserName)
}

func s3NamesForUser(userName string, namespace string) (pvName, pvcName, volumeHandle string) {
base := userToResourceName(userName, "s3") // e.g. john-doe-s3
return base + "-" + namespace + "-pv", base + "-pvc", base + "-" + namespace // unique handle
}

func s3PrefixForUser(userName string) string {
// If you prefer purely the username: return fmt.Sprintf("%s/", userName)
// Using resource-safe version tends to be nicer:
return fmt.Sprintf("%s/", userToResourceName(userName, ""))
}

func ensureS3PVandPVC(
func ensureSharedSoftwareLibrary(
ctx context.Context,
podClient corev1.CoreV1Interface,
namespace string,
userName string,
bucket string,
region string,
) (pvcName string, err error) {

pvName, pvcName, volumeHandle := s3NamesForUser(userName, namespace)
// Commented out for now as we don't need a prefix, this bucket will be mounted at / for all users, but read only.
// This will be used for the "software-repository" feature
// prefix := s3PrefixForUser(userName)
// prefix := "/"

// ----- Ensure PV exists (cluster-scoped) -----
// Try GET; if not found, create.
if _, errGet := podClient.PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}); errGet != nil {
cfg FSxConfig,
) error {

// Use the namespaced, pv name (PV is a cluster resource)
pvName := "sl-" + namespace
pvcName := "software-library-pvc"

// 1. Ensure the Singleton PV exists
_, err := podClient.PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{})
if err != nil {
Config.Logger.Printf("Creating shared Lustre PV: %s", pvName)
pv := &k8sv1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: pvName,
},
ObjectMeta: metav1.ObjectMeta{Name: pvName},
Spec: k8sv1.PersistentVolumeSpec{
Capacity: k8sv1.ResourceList{
k8sv1.ResourceStorage: resource.MustParse("1200Gi"), // ignored by S3 CSI, but required
},
AccessModes: []k8sv1.PersistentVolumeAccessMode{k8sv1.ReadWriteMany},
StorageClassName: "", // static provisioning
ClaimRef: &k8sv1.ObjectReference{
Namespace: namespace,
Name: pvcName,
},
MountOptions: []string{
"allow-delete",
"allow-other",
"gid=100",
"file-mode=555",
"dir-mode=555",
fmt.Sprintf("region %s", region),
// fmt.Sprintf("prefix %s", prefix),
// Caches directory listings and file stats for 1 hour (fixes the 77x walk/stat slowdown)
"metadata-ttl=3600",
},
Capacity: k8sv1.ResourceList{k8sv1.ResourceStorage: resource.MustParse("1200Gi")},
VolumeMode: (*k8sv1.PersistentVolumeMode)(aws.String(string(k8sv1.PersistentVolumeFilesystem))),
AccessModes: []k8sv1.PersistentVolumeAccessMode{k8sv1.ReadWriteMany},
PersistentVolumeReclaimPolicy: k8sv1.PersistentVolumeReclaimRetain,
StorageClassName: "", // Static binding
PersistentVolumeSource: k8sv1.PersistentVolumeSource{
CSI: &k8sv1.CSIPersistentVolumeSource{
Driver: "s3.csi.aws.com",
VolumeHandle: volumeHandle, // must be unique
Driver: "fsx.csi.aws.com",
VolumeHandle: cfg.FsxID,
VolumeAttributes: map[string]string{
"bucketName": bucket,
// Enables safe file caching using an ephemeral emptyDir on the K8s node
// and hard-caps the cache at 10 Gigabytes to prevent disk pressure
"cacheEmptyDirSizeLimit": "10Gi",
"mountname": cfg.MountName,
"dnsname": cfg.DnsName,
},
},
},
},
}
if _, errCreatePV := podClient.PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{}); errCreatePV != nil {
return "", fmt.Errorf("failed to create S3 PV %s: %w", pvName, errCreatePV)
_, err = podClient.PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{})
if err != nil {
return err
}
}

// ----- Ensure PVC exists (namespaced) -----
if _, errGet := podClient.PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{}); errGet != nil {
empty := "" // PVC.StorageClassName is *string
// 2. Ensure the Singleton PVC exists in the shared namespace
_, err = podClient.PersistentVolumeClaims(namespace).Get(ctx, pvcName, metav1.GetOptions{})
if err != nil {
Config.Logger.Printf("Creating shared Lustre PVC: %s in namespace: %s", pvcName, namespace)
emptySC := ""
pvc := &k8sv1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: pvcName,
Namespace: namespace,
},
Spec: k8sv1.PersistentVolumeClaimSpec{
AccessModes: []k8sv1.PersistentVolumeAccessMode{k8sv1.ReadWriteMany},
StorageClassName: &empty,
StorageClassName: &emptySC,
Resources: k8sv1.VolumeResourceRequirements{
Requests: k8sv1.ResourceList{
k8sv1.ResourceStorage: resource.MustParse("1200Gi"),
},
Requests: k8sv1.ResourceList{k8sv1.ResourceStorage: resource.MustParse("1200Gi")},
},
VolumeName: pvName,
},
}
if _, errCreatePVC := podClient.PersistentVolumeClaims(namespace).Create(ctx, pvc, metav1.CreateOptions{}); errCreatePVC != nil {
return "", fmt.Errorf("failed to create S3 PVC %s: %w", pvcName, errCreatePVC)
_, err = podClient.PersistentVolumeClaims(namespace).Create(ctx, pvc, metav1.CreateOptions{})
if err != nil {
return err
}
}

return pvcName, nil
return nil
}

// Adds the S3 volume + mount to the pod in-place.
// Call this after buildPod(), before creating it.
func addS3VolumeToPod(pod *k8sv1.Pod, pvcName string) {
// 1) Add Volume
hasVolume := false
for _, v := range pod.Spec.Volumes {
if v.Name == "s3-volume" {
hasVolume = true
break
}
}
if !hasVolume {
pod.Spec.Volumes = append(pod.Spec.Volumes, k8sv1.Volume{
Name: "s3-volume",
VolumeSource: k8sv1.VolumeSource{
PersistentVolumeClaim: &k8sv1.PersistentVolumeClaimVolumeSource{
ClaimName: pvcName,
// ReadOnly: false, // optional
},
func injectSoftwareLibrary(pod *k8sv1.Pod) {
volumeName := "software-library"
pvcName := "software-library-pvc"

// Add Volume to Pod Spec if not present
pod.Spec.Volumes = append(pod.Spec.Volumes, k8sv1.Volume{
Name: volumeName,
VolumeSource: k8sv1.VolumeSource{
PersistentVolumeClaim: &k8sv1.PersistentVolumeClaimVolumeSource{
ClaimName: pvcName,
ReadOnly: true, // Highly recommended for a shared software library
},
})
}
},
})

// 2) Add VolumeMount (/apps) to every container (or just the main one)
for ci := range pod.Spec.Containers {
c := &pod.Spec.Containers[ci]
alreadyMounted := false
for _, m := range c.VolumeMounts {
if m.Name == "s3-volume" {
alreadyMounted = true
break
}
}
if !alreadyMounted {
c.VolumeMounts = append(c.VolumeMounts, k8sv1.VolumeMount{
Name: "s3-volume",
// TODO: Read this path from config
MountPath: "/apps",
})
}
// Mount to all containers
for i := range pod.Spec.Containers {
pod.Spec.Containers[i].VolumeMounts = append(pod.Spec.Containers[i].VolumeMounts, k8sv1.VolumeMount{
Name: volumeName,
MountPath: Config.Config.FSxConfig.MountPath,
})
}
}

Expand Down Expand Up @@ -880,22 +830,20 @@ var createLocalK8sPod = func(ctx context.Context, hash string, userName string,
Config.Logger.Panicf("Error in createLocalK8sPod: %v", err)
return err
}
// ensure S3 PV/PVC (dynamic per user) and wire into pod
if Config.Config.S3Config.BucketName != "" && Config.Config.S3Config.Region != "" {
Config.Logger.Print("Mounting S3 bucket as well..")
s3PVCName, err := ensureS3PVandPVC(

if Config.Config.FSxConfig.FsxID != "" {
err := ensureSharedSoftwareLibrary(
ctx,
podClient,
Config.Config.UserNamespace,
userName,
Config.Config.S3Config.BucketName,
Config.Config.S3Config.Region,
Config.Config.FSxConfig,
)
if err != nil {
Config.Logger.Printf("Failed ensuring S3 PV/PVC for user %s: %v", userName, err)
return err
return fmt.Errorf("failed to setup software library: %w", err)
}
addS3VolumeToPod(pod, s3PVCName)

// Wire it into the pod spec we just built
injectSoftwareLibrary(pod)
}

// a null image indicates a dockstore app - always mount user volume
Expand Down
Loading