From fae33630c2c9f84f8611e740c3cf9057ea9b4641 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Tue, 7 Apr 2026 13:37:18 -0400 Subject: [PATCH 01/55] Replace GKE/Helm Galaxy path with GCE VM-based deployment --- .../init-resources/galaxy-user-data.sh | 138 +++++ .../main/resources/init-resources/startup.sh | 4 + http/src/main/resources/reference.conf | 52 ++ .../workbench/leonardo/config/Config.scala | 18 +- .../leonardo/config/KubernetesAppConfig.scala | 15 +- .../leonardo/dao/HttpJupyterDAO.scala | 2 +- .../monitor/LeoPubsubMessageSubscriber.scala | 7 +- .../leonardo/util/BuildHelmChartValues.scala | 107 ---- .../leonardo/util/GKEInterpreter.scala | 576 ++++++++++-------- .../util/BuildHelmChartValuesSpec.scala | 172 ------ 10 files changed, 544 insertions(+), 547 deletions(-) create mode 100644 http/src/main/resources/init-resources/galaxy-user-data.sh diff --git a/http/src/main/resources/init-resources/galaxy-user-data.sh b/http/src/main/resources/init-resources/galaxy-user-data.sh new file mode 100644 index 0000000000..3b7fbb640b --- /dev/null +++ b/http/src/main/resources/init-resources/galaxy-user-data.sh @@ -0,0 +1,138 @@ +# Sourced from https://github.com/galaxyproject/galaxy-k8s-boot/blob/dev/bin/user_data.sh +# When updating this file, sync it manually from that repository and verify the changes. +#cloud-config +write_files: + - path: /usr/local/bin/galaxy_bootstrap.sh + permissions: '0755' + owner: root:root + content: | + #!/bin/bash + + echo "[$(date)] - Starting galaxy_bootstrap script..." + + # 1. Setup persistent disk if available + DISK_DEVICE="/dev/disk/by-id/google-galaxy-data" + if [ -b "$DISK_DEVICE" ]; then + echo "[$(date)] - Found persistent disk at $DISK_DEVICE" + + # Check if disk is already formatted + if ! blkid "$DISK_DEVICE" > /dev/null 2>&1; then + echo "[$(date)] - Formatting disk $DISK_DEVICE with ext4" + mkfs -t ext4 "$DISK_DEVICE" + else + echo "[$(date)] - Disk $DISK_DEVICE is already formatted" + fi + + # Create mount point and mount + mkdir -p /mnt/block_storage + mount "$DISK_DEVICE" /mnt/block_storage + + # Add to fstab for persistent mounting across reboots + DISK_UUID=$(blkid -s UUID -o value "$DISK_DEVICE") + if [ -n "$DISK_UUID" ] && ! grep -q "$DISK_UUID" /etc/fstab; then + echo "UUID=$DISK_UUID /mnt/block_storage ext4 defaults 0 2" >> /etc/fstab + fi + + # Set proper ownership + chown debian:debian /mnt/block_storage + echo "[$(date)] - Persistent disk mounted at /mnt/block_storage" + else + echo "[$(date)] - No persistent disk found at $DISK_DEVICE. Galaxy will use ephemeral storage." + fi + + # 2. Setup PostgreSQL disk if available + POSTGRES_DISK_DEVICE="/dev/disk/by-id/google-galaxy-postgres-data" + if [ -b "$POSTGRES_DISK_DEVICE" ]; then + echo "[$(date)] - Found PostgreSQL disk at $POSTGRES_DISK_DEVICE" + + # Check if disk is already formatted + if ! blkid "$POSTGRES_DISK_DEVICE" > /dev/null 2>&1; then + echo "[$(date)] - Formatting PostgreSQL disk $POSTGRES_DISK_DEVICE with ext4" + mkfs -t ext4 "$POSTGRES_DISK_DEVICE" + else + echo "[$(date)] - PostgreSQL disk $POSTGRES_DISK_DEVICE is already formatted" + fi + + # Create mount point and mount + mkdir -p /mnt/postgres_storage + mount "$POSTGRES_DISK_DEVICE" /mnt/postgres_storage + + # Add to fstab for persistent mounting across reboots + POSTGRES_DISK_UUID=$(blkid -s UUID -o value "$POSTGRES_DISK_DEVICE") + if [ -n "$POSTGRES_DISK_UUID" ] && ! grep -q "$POSTGRES_DISK_UUID" /etc/fstab; then + echo "UUID=$POSTGRES_DISK_UUID /mnt/postgres_storage ext4 defaults 0 2" >> /etc/fstab + fi + + # Set proper ownership + chown debian:debian /mnt/postgres_storage + echo "[$(date)] - PostgreSQL disk mounted at /mnt/postgres_storage" + else + echo "[$(date)] - No PostgreSQL disk found at $POSTGRES_DISK_DEVICE. PostgreSQL will use ephemeral storage." + fi + + # 3. Run ansible-pull + sudo -u debian bash -c ' + export HOME=/home/debian + HOST_IP=$(curl -s ifconfig.me) + + PV_SIZE=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/persistent-volume-size" -H "Metadata-Flavor: Google" 2>/dev/null) + if [ -z "$PV_SIZE" ]; then + echo "[$(date)] - persistent-volume-size metadata not found or empty, using default." + PV_SIZE="139Gi" + fi + echo "[$(date)] - NFS storage size for Galaxy: ${PV_SIZE}" + + # Add restore_galaxy if enabled + RESTORE_GALAXY=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/restore_galaxy" -H "Metadata-Flavor: Google" 2>/dev/null || echo "false") + + GCP_BATCH_SERVICE_ACCOUNT_EMAIL=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/gcp_batch_service_account_email" -H "Metadata-Flavor: Google" 2>/dev/null || echo "galaxy-batch-runner@anvil-and-terra-development.iam.gserviceaccount.com") + echo "[$(date)] - GCP Batch service account email: ${GCP_BATCH_SERVICE_ACCOUNT_EMAIL}" + + GIT_REPO=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/git-repo" -H "Metadata-Flavor: Google" 2>/dev/null || echo "https://github.com/galaxyproject/galaxy-k8s-boot.git") + GIT_BRANCH=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/git-branch" -H "Metadata-Flavor: Google" 2>/dev/null || echo "master") + + PULL_ARGS=( + -U "${GIT_REPO}" + -C "${GIT_BRANCH}" + -d /home/debian/ansible + -i /tmp/ansible-inventory/localhost + --accept-host-key + --limit 127.0.0.1 + --extra-vars "gcp_batch_service_account_email=${GCP_BATCH_SERVICE_ACCOUNT_EMAIL}" + ) + + if [ "$RESTORE_GALAXY" = "true" ]; then + PULL_ARGS+=(--extra-vars "restore_galaxy=true") + echo "[$(date)] - Galaxy Restore Mode: Enabled" + else + echo "[$(date)] - Galaxy Restore Mode: Disabled" + fi + + PULL_ARGS+=(playbook.yml) + + mkdir -p /tmp/ansible-inventory + cat > /tmp/ansible-inventory/localhost << EOF + [vm] + 127.0.0.1 ansible_connection=local ansible_python_interpreter="/usr/bin/python3" + + [all:vars] + ansible_user="debian" + rke2_token="defaultSecret12345" + rke2_additional_sans=["${HOST_IP}"] + rke2_debug=true + nfs_size="${PV_SIZE}" + galaxy_persistence_size="${PV_SIZE}" + galaxy_db_password="gxy-db-password" + galaxy_user="dev@galaxyproject.org" + EOF + + echo "[$(date)] - Inventory file created at /tmp/ansible-inventory/localhost; running ansible-pull..." + echo "[$(date)] - Running: ANSIBLE_CALLBACKS_ENABLED=profile_tasks ANSIBLE_HOST_PATTERN_MISMATCH=ignore ansible-pull ${PULL_ARGS[@]}" + + ANSIBLE_CALLBACKS_ENABLED=profile_tasks ANSIBLE_HOST_PATTERN_MISMATCH=ignore ansible-pull "${PULL_ARGS[@]}" + ' + + echo "[$(date)] - Bootstrap script completed." + +runcmd: + - /usr/local/bin/galaxy_bootstrap.sh diff --git a/http/src/main/resources/init-resources/startup.sh b/http/src/main/resources/init-resources/startup.sh index 324d6783e8..c28163ce25 100644 --- a/http/src/main/resources/init-resources/startup.sh +++ b/http/src/main/resources/init-resources/startup.sh @@ -122,6 +122,10 @@ function failScriptIfError() { function validateCert() { certFileDirectory=$1 + ## Only the master node has certs; worker nodes in multi-node Dataproc clusters do not. + if [ ! -f "${certFileDirectory}/jupyter-server.crt" ]; then + return 0 + fi ## This helps when we need to rotate certs. notAfter=`openssl x509 -enddate -noout -in ${certFileDirectory}/jupyter-server.crt` # output should be something like `notAfter=Jul 4 20:31:52 2026 GMT` diff --git a/http/src/main/resources/reference.conf b/http/src/main/resources/reference.conf index 09bca16223..7cb3d0892b 100644 --- a/http/src/main/resources/reference.conf +++ b/http/src/main/resources/reference.conf @@ -210,6 +210,44 @@ vpc { northamerica-northeast2 = "10.26.0.0/20" } firewallsToAdd = [ + # Allows Galaxy VM traffic on port 80 (nginx ingress on hostNetwork) + { + name-prefix = "leonardo-allow-http" + sourceRanges = { + us-central1 = ["0.0.0.0/0"] + northamerica-northeast1 = ["0.0.0.0/0"] + southamerica-east1 = ["0.0.0.0/0"] + us-east1 = ["0.0.0.0/0"] + us-east4 = ["0.0.0.0/0"] + us-west1 = ["0.0.0.0/0"] + us-west2 = ["0.0.0.0/0"] + us-west3 = ["0.0.0.0/0"] + us-west4 = ["0.0.0.0/0"] + europe-central2 = ["0.0.0.0/0"] + europe-north1 = ["0.0.0.0/0"] + europe-west1 = ["0.0.0.0/0"] + europe-west2 = ["0.0.0.0/0"] + europe-west3 = ["0.0.0.0/0"] + europe-west4 = ["0.0.0.0/0"] + europe-west6 = ["0.0.0.0/0"] + asia-east1 = ["0.0.0.0/0"] + asia-east2 = ["0.0.0.0/0"] + asia-northeast1 = ["0.0.0.0/0"] + asia-northeast2 = ["0.0.0.0/0"] + asia-northeast3 = ["0.0.0.0/0"] + asia-south1 = ["0.0.0.0/0"] + asia-southeast1 = ["0.0.0.0/0"] + asia-southeast2 = ["0.0.0.0/0"] + australia-southeast1 = ["0.0.0.0/0"] + northamerica-northeast2 = ["0.0.0.0/0"] + } + allowed = [ + { + protocol = "tcp" + port = "80" + } + ] + }, # Allows Leonardo proxy traffic on port 443 { name-prefix = "leonardo-allow-https" @@ -395,6 +433,20 @@ groups { } } +galaxyVm { + # Debian 12 image — galaxy-k8s-boot bootstrap requires a standard Debian VM, not COS + sourceImage = "projects/debian-cloud/global/images/family/debian-12" + machineType = "n1-highmem-8" + bootDiskSizeGb = 50 + postgresDiskSizeGb = 10 + # Suffix appended to the NFS disk name to derive the postgres disk name. + # Must match the value used in LeoPubsubMessageSubscriber (galaxyDisk.postgresDiskNameSuffix). + postgresDiskNameSuffix = ${gke.galaxyDisk.postgresDiskNameSuffix} + gcpBatchServiceAccountEmail = "" + gitRepo = "https://github.com/galaxyproject/galaxy-k8s-boot.git" + gitBranch = "master" +} + gke { cluster { location = "us-central1-a", diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/Config.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/Config.scala index 45a3e26f38..42c7a53737 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/Config.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/Config.scala @@ -131,6 +131,19 @@ object Config { ) } + implicit private val galaxyVmConfigReader: ValueReader[GalaxyVmConfig] = ValueReader.relative { config => + GalaxyVmConfig( + config.as[GceCustomImage]("sourceImage"), + config.as[MachineTypeName]("machineType"), + config.as[DiskSize]("bootDiskSizeGb"), + config.as[DiskSize]("postgresDiskSizeGb"), + config.as[String]("postgresDiskNameSuffix"), + config.as[String]("gcpBatchServiceAccountEmail"), + config.as[String]("gitRepo"), + config.as[String]("gitBranch") + ) + } + implicit private val gceConfigReader: ValueReader[GceConfig] = ValueReader.relative { config => GceConfig( config.as[GceCustomImage]("customGceImage"), @@ -497,6 +510,7 @@ object Config { val googleGroupsConfig = config.as[GoogleGroupsConfig]("groups") val dataprocConfig = config.as[DataprocConfig]("dataproc") + val galaxyVmConfig = config.as[GalaxyVmConfig]("galaxyVm") val gceConfig = config.as[GceConfig]("gce") val imageConfig = config.as[ImageConfig]("image") val prometheusConfig = config.as[PrometheusConfig]("prometheus") @@ -901,13 +915,13 @@ object Config { vpcConfig.networkTag, org.broadinstitute.dsde.workbench.leonardo.http.ConfigReader.appConfig.terraAppSetupChart, gkeIngressConfig, - gkeGalaxyAppConfig, gkeCromwellAppConfig, gkeCustomAppConfig, gkeAllowedAppConfig, appMonitorConfig, gkeClusterConfig, proxyConfig, - gkeGalaxyDiskConfig + gkeGalaxyDiskConfig, + galaxyVmConfig ) } diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/KubernetesAppConfig.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/KubernetesAppConfig.scala index 2b3fc4a587..53e2562db6 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/KubernetesAppConfig.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/KubernetesAppConfig.scala @@ -1,7 +1,9 @@ package org.broadinstitute.dsde.workbench.leonardo.config -import org.broadinstitute.dsde.workbench.google2.KubernetesSerializableName.ServiceAccountName +import org.broadinstitute.dsde.workbench.google2.{KubernetesSerializableName, MachineTypeName} +import KubernetesSerializableName.ServiceAccountName import org.broadinstitute.dsde.workbench.leonardo.AppType._ +import org.broadinstitute.dsde.workbench.leonardo.CustomImage.GceCustomImage import org.broadinstitute.dsde.workbench.leonardo._ import org.broadinstitute.dsp.{ChartName, ChartVersion} @@ -92,6 +94,17 @@ final case class CustomAppConfig(chartName: ChartName, val appType: AppType = AppType.Custom } +final case class GalaxyVmConfig( + sourceImage: GceCustomImage, + machineType: MachineTypeName, + bootDiskSizeGb: DiskSize, + postgresDiskSizeGb: DiskSize, + postgresDiskNameSuffix: String, + gcpBatchServiceAccountEmail: String, + gitRepo: String, + gitBranch: String +) + final case class ContainerRegistryUsername(asString: String) extends AnyVal final case class ContainerRegistryPassword(asString: String) extends AnyVal final case class ContainerRegistryCredentials(username: ContainerRegistryUsername, password: ContainerRegistryPassword) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpJupyterDAO.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpJupyterDAO.scala index 9bb06e9548..0c55f089db 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpJupyterDAO.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpJupyterDAO.scala @@ -36,7 +36,7 @@ class HttpJupyterDAO[F[_]](val runtimeDnsCache: RuntimeDnsCache[F], client: Clie headers = Headers.empty ) ) - .handleError(_ => false) + .handleErrorWith(e => logger.warn(e)(s"isProxyAvailable failed for ${cloudContext}/${runtimeName}").as(false)) case _ => F.pure(false) } } yield res diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriber.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriber.scala index 8c33c93579..97a6c984bd 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriber.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriber.scala @@ -999,10 +999,15 @@ class LeoPubsubMessageSubscriber[F[_]]( } yield res } else F.unit + // Galaxy uses VM-based deployment: no GKE cluster or nodepool is created. + // The VM is launched inside createAndPollApp instead. + effectiveClusterOrNodepoolOp = + if (msg.appType == AppType.Galaxy) F.unit else createClusterOrNodepoolOp + // build asynchronous task task = for { // parallelize disk creation and cluster/nodepool monitoring - _ <- List(createDiskOp, createSecondDiskOp, createClusterOrNodepoolOp).parSequence_ + _ <- List(createDiskOp, createSecondDiskOp, effectiveClusterOrNodepoolOp).parSequence_ // create and monitor app _ <- getGkeAlgFromRegistry() diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/BuildHelmChartValues.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/BuildHelmChartValues.scala index 7fa725c8cf..3c54efd044 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/BuildHelmChartValues.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/BuildHelmChartValues.scala @@ -1,10 +1,8 @@ package org.broadinstitute.dsde.workbench.leonardo package util -import org.broadinstitute.dsde.workbench.google2.DiskName import org.broadinstitute.dsde.workbench.google2.GKEModels.NodepoolName import org.broadinstitute.dsde.workbench.google2.KubernetesSerializableName.{NamespaceName, ServiceAccountName} -import org.broadinstitute.dsde.workbench.leonardo.AppRestore.GalaxyRestore import org.broadinstitute.dsde.workbench.leonardo.Autopilot import org.broadinstitute.dsde.workbench.leonardo.dao.CustomAppService import org.broadinstitute.dsde.workbench.leonardo.http.kubernetesProxyHost @@ -15,111 +13,6 @@ import org.broadinstitute.dsp.Release import java.nio.charset.StandardCharsets private[leonardo] object BuildHelmChartValues { - def buildGalaxyChartOverrideValuesString(config: GKEInterpreterConfig, - appName: AppName, - release: Release, - cluster: KubernetesCluster, - nodepoolName: NodepoolName, - userEmail: WorkbenchEmail, - customEnvironmentVariables: Map[String, String], - ksa: ServiceAccountName, - namespaceName: NamespaceName, - nfsDisk: PersistentDisk, - postgresDiskName: DiskName, - machineType: AppMachineType, - galaxyRestore: Option[GalaxyRestore] - ): List[String] = { - val k8sProxyHost = kubernetesProxyHost(cluster, config.proxyConfig.proxyDomain).address - val leoProxyhost = config.proxyConfig.getProxyServerHostName - val ingressPath = s"/proxy/google/v1/apps/${cluster.cloudContext.asString}/${appName.value}/galaxy" - val workspaceName = customEnvironmentVariables.getOrElse("WORKSPACE_NAME", "") - val workspaceNamespace = customEnvironmentVariables.getOrElse("WORKSPACE_NAMESPACE", "") - // Machine type info - val maxLimitMemory = machineType.memorySizeInGb - val maxLimitCpu = machineType.numOfCpus - val maxRequestMemory = maxLimitMemory - 22 - val maxRequestCpu = maxLimitCpu - 6 - - // Custom EV configs - val configs = customEnvironmentVariables.toList.zipWithIndex.flatMap { case ((k, v), i) => - List( - raw"""configs.$k=$v""", - raw"""extraEnv[$i].name=$k""", - raw"""extraEnv[$i].valueFrom.configMapKeyRef.name=${release.asString}-galaxykubeman-configs""", - raw"""extraEnv[$i].valueFrom.configMapKeyRef.key=$k""" - ) - } - - val galaxyRestoreSettings = galaxyRestore.fold(List.empty[String])(g => - List( - raw"""restore.persistence.nfs.galaxy.pvcID=${g.galaxyPvcId.asString}""", - raw"""galaxy.persistence.existingClaim=${release.asString}-galaxy-galaxy-pvc""" - ) - ) - // Using the string interpolator raw""" since the chart keys include quotes to escape Helm - // value override special characters such as '.' - // https://helm.sh/docs/intro/using_helm/#the-format-and-limitations-of---set - List( - // Storage class configs - raw"""nfs.storageClass.name=nfs-${release.asString}""", - raw"""galaxy.persistence.storageClass=nfs-${release.asString}""", - // Node selector config: this ensures the app is run on the user's nodepool - raw"""galaxy.nodeSelector.cloud\.google\.com/gke-nodepool=${nodepoolName.value}""", - raw"""nfs.nodeSelector.cloud\.google\.com/gke-nodepool=${nodepoolName.value}""", - raw"""galaxy.configs.job_conf\.yml.runners.k8s.k8s_node_selector=cloud.google.com/gke-nodepool: ${nodepoolName.value}""", - raw"""galaxy.postgresql.master.nodeSelector.cloud\.google\.com/gke-nodepool=${nodepoolName.value}""", - // Ingress configs - raw"""galaxy.ingress.path=${ingressPath}""", - raw"""galaxy.ingress.annotations.nginx\.ingress\.kubernetes\.io/proxy-redirect-from=https://${k8sProxyHost}""", - raw"""galaxy.ingress.annotations.nginx\.ingress\.kubernetes\.io/proxy-redirect-to=${leoProxyhost}""", - raw"""galaxy.ingress.hosts[0].host=${k8sProxyHost}""", - raw"""galaxy.ingress.hosts[0].paths[0].path=${ingressPath}""", - raw"""galaxy.ingress.tls[0].hosts[0]=${k8sProxyHost}""", - raw"""galaxy.ingress.tls[0].secretName=tls-secret""", - // CVMFS configs - raw"""cvmfs.cvmfscsi.cache.alien.pvc.storageClass=nfs-${release.asString}""", - raw"""cvmfs.cvmfscsi.cache.alien.pvc.name=cvmfs-alien-cache""", - // Galaxy configs - raw"""galaxy.configs.galaxy\.yml.galaxy.single_user=${userEmail.value}""", - raw"""galaxy.configs.galaxy\.yml.galaxy.admin_users=${userEmail.value}""", - raw"""galaxy.terra.launch.workspace=${workspaceName}""", - raw"""galaxy.terra.launch.namespace=${workspaceNamespace}""", - raw"""galaxy.terra.launch.apiURL=${config.galaxyAppConfig.orchUrl.value}""", - raw"""galaxy.terra.launch.drsURL=${config.galaxyAppConfig.drsUrl.value}""", - // Tusd ingress configs - raw"""galaxy.tusd.ingress.hosts[0].host=${k8sProxyHost}""", - raw"""galaxy.tusd.ingress.hosts[0].paths[0].path=${ingressPath}/api/upload/resumable_upload""", - raw"""galaxy.tusd.ingress.tls[0].hosts[0]=${k8sProxyHost}""", - raw"""galaxy.tusd.ingress.tls[0].secretName=tls-secret""", - // Set RabbitMQ storage class - raw"""galaxy.rabbitmq.persistence.storageClassName=nfs-${release.asString}""", - // Set Machine Type specs - raw"""galaxy.jobs.maxLimits.memory=${maxLimitMemory}""", - raw"""galaxy.jobs.maxLimits.cpu=${maxLimitCpu}""", - raw"""galaxy.jobs.maxRequests.memory=${maxRequestMemory}""", - raw"""galaxy.jobs.maxRequests.cpu=${maxRequestCpu}""", - raw"""galaxy.jobs.rules.tpv_rules_local\.yml.destinations.k8s.max_mem=${maxRequestMemory}""", - raw"""galaxy.jobs.rules.tpv_rules_local\.yml.destinations.k8s.max_cores=${maxRequestCpu}""", - // RBAC configs - raw"""galaxy.serviceAccount.create=false""", - raw"""galaxy.serviceAccount.name=${ksa.value}""", - raw"""rbac.serviceAccount=${ksa.value}""", - // Persistence configs - raw"""persistence.nfs.name=${namespaceName.value}-${config.galaxyDiskConfig.nfsPersistenceName}""", - raw"""persistence.nfs.persistentVolume.extraSpec.gcePersistentDisk.pdName=${nfsDisk.name.value}""", - raw"""persistence.nfs.size=${nfsDisk.size.gb.toString}Gi""", - raw"""persistence.postgres.name=${namespaceName.value}-${config.galaxyDiskConfig.postgresPersistenceName}""", - raw"""galaxy.postgresql.galaxyDatabasePassword=${config.galaxyAppConfig.postgresPassword.value}""", - raw"""persistence.postgres.persistentVolume.extraSpec.gcePersistentDisk.pdName=${postgresDiskName.value}""", - raw"""persistence.postgres.size=${config.galaxyDiskConfig.postgresDiskSizeGB.gb.toString}Gi""", - raw"""nfs.persistence.existingClaim=${namespaceName.value}-${config.galaxyDiskConfig.nfsPersistenceName}-pvc""", - raw"""nfs.persistence.size=${nfsDisk.size.gb.toString}Gi""", - raw"""galaxy.postgresql.persistence.existingClaim=${namespaceName.value}-${config.galaxyDiskConfig.postgresPersistenceName}-pvc""", - // Note Galaxy pvc claim is the nfs disk size minus 50G - raw"""galaxy.persistence.size=${(nfsDisk.size.gb - 50).toString}Gi""" - ) ++ configs ++ galaxyRestoreSettings - } - def buildCromwellAppChartOverrideValuesString(config: GKEInterpreterConfig, appName: AppName, cluster: KubernetesCluster, diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala index f3d609f228..b42d783d04 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala @@ -6,7 +6,17 @@ import cats.effect.Async import cats.mtl.Ask import cats.syntax.all._ import com.google.auth.oauth2.GoogleCredentials -import com.google.cloud.compute.v1.Disk +import com.google.cloud.compute.v1.{ + AccessConfig, + AttachedDisk, + AttachedDiskInitializeParams, + Instance, + Items, + Metadata, + NetworkInterface, + ServiceAccount, + Tags +} import com.google.container.v1._ import fs2.io.file.Files import org.broadinstitute.dsde.workbench.DoneCheckable @@ -27,26 +37,27 @@ import org.broadinstitute.dsde.workbench.google2.{ streamFUntilDone, streamUntilDoneOrTimeout, tracedRetryF, - DiskName, GoogleComputeService, GoogleDiskService, GoogleResourceService, KubernetesClusterNotFoundException, - PvName, + NetworkName, + RegionName, + SubnetworkName, ZoneName } +import org.broadinstitute.dsde.workbench.util2.InstanceName import org.broadinstitute.dsde.workbench.leonardo.dao.{AppDAO, AppDescriptorDAO} import org.broadinstitute.dsde.workbench.leonardo.db._ import org.broadinstitute.dsde.workbench.leonardo.http._ import org.broadinstitute.dsde.workbench.leonardo.http.service.AppNotFoundException +import org.broadinstitute.dsde.workbench.leonardo.dao.google.{buildMachineTypeUri, buildSubnetworkUri} import org.broadinstitute.dsde.workbench.leonardo.util.BuildHelmChartValues.{ buildAllowedAppChartOverrideValuesString, buildCromwellAppChartOverrideValuesString, - buildCustomChartOverrideValuesString, - buildGalaxyChartOverrideValuesString + buildCustomChartOverrideValuesString } import org.broadinstitute.dsde.workbench.leonardo.model.LeoException -import org.broadinstitute.dsde.workbench.leonardo.monitor.PubsubHandleMessageError.PubsubKubernetesError import org.broadinstitute.dsde.workbench.leonardo.util.GKEAlgebra._ import org.broadinstitute.dsde.workbench.model.google.{GcsBucketName, GoogleProject} import org.broadinstitute.dsde.workbench.model.{IP, TraceId, WorkbenchEmail} @@ -387,21 +398,52 @@ class GKEInterpreter[F[_]]( ) ) app = dbApp.app - namespaceName = app.appResources.namespace dbCluster = dbApp.cluster - gkeClusterId = dbCluster.getClusterId googleProject = params.googleProject - // TODO: This DB query might not be needed if it makes sense to add diskId in App model (will revisit in next PR) diskOpt <- appQuery.getDiskId(app.id).transaction diskId <- F.fromOption(diskOpt, DiskNotFoundForAppException(app.id, ctx.traceId)) - // Create namespace and secrets + _ <- logger.info(ctx.loggingCtx)(s"Begin App(${app.appName.value}) Creation.") + + nfsDisk <- F.fromOption( + dbApp.app.appResources.disk, + AppCreationException(s"NFS disk not found in DB for app ${app.appName.value} | trace id: ${ctx.traceId}") + ) + + // Galaxy uses a VM-based deployment; all other app types use the GKE/Helm path. + _ <- app.appType match { + case AppType.Galaxy => + installGalaxyVm(dbCluster, app, nfsDisk, googleProject) >> + persistentDiskQuery.updateLastUsedBy(diskId, app.id).transaction.void + + case _ => + createAndPollAppViaHelm(params, dbApp, app, dbCluster, nfsDisk, diskId, googleProject, ctx) + } + _ <- logger.info(ctx.loggingCtx)( - s"Begin App(${app.appName.value}) Creation." + s"Finished app creation for app ${app.appName.value}" ) - // Create KSA + readyTime <- F.realTimeInstant + _ <- appUsageQuery.recordStart(params.appId, readyTime) + _ <- appQuery.updateStatus(params.appId, AppStatus.Running).transaction + } yield () + + // GKE/Helm path for non-Galaxy app types (Cromwell, Allowed, Custom). + private def createAndPollAppViaHelm( + params: CreateAppParams, + dbApp: GetAppResult, + app: App, + dbCluster: KubernetesCluster, + nfsDisk: PersistentDisk, + diskId: DiskId, + googleProject: GoogleProject, + ctx: AppContext + )(implicit ev: Ask[F, AppContext]): F[Unit] = { + val namespaceName = app.appResources.namespace + val gkeClusterId = dbCluster.getClusterId + for { ksaName <- F.fromOption( app.appResources.kubernetesServiceAccountName, AppCreationException( @@ -424,11 +466,6 @@ class GKEInterpreter[F[_]]( ) } - nfsDisk <- F.fromOption( - dbApp.app.appResources.disk, - AppCreationException(s"NFS disk not found in DB for app ${app.appName.value} | trace id: ${ctx.traceId}") - ) - helmAuthContext <- getHelmAuthContext(googleCluster, dbCluster, namespaceName) _ <- helmClient @@ -442,12 +479,8 @@ class GKEInterpreter[F[_]]( true ) .run(helmAuthContext) - // update KSA in DB _ <- appQuery.updateKubernetesServiceAccount(app.id, ksaName).transaction - // Associate GSA to newly created KSA - // This string is constructed based on Google requirements to associate a GSA to a KSA - // (https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity#creating_a_relationship_between_ksas_and_gsas) ksaToGsa = s"${googleProject.value}.svc.id.goog[${namespaceName.value}/${ksaName.value}]" call = F.fromFuture( F.delay( @@ -458,49 +491,15 @@ class GKEInterpreter[F[_]]( ) ) ) - retryConfig = RetryPredicates.retryConfigWithPredicates( - when409 - ) + retryConfig = RetryPredicates.retryConfigWithPredicates(when409) _ <- tracedRetryF(retryConfig)( call, s"googleIamDAO.addIamPolicyBindingOnServiceAccount for GSA ${gsa.value} & KSA ${ksaName.value}" ).compile.lastOrError - // TODO: validate app release is the same as restore release - appRestore: Option[AppRestore] <- persistentDiskQuery.getAppDiskRestore(diskId).transaction - galaxyRestore: Option[AppRestore.GalaxyRestore] = appRestore.flatMap { - case a: AppRestore.GalaxyRestore => Some(a) - case _: AppRestore.Other => None - } - nodepool = if (app.autopilot.isDefined) None else Some(dbApp.nodepool.nodepoolName) - // helm install and wait + _ <- app.appType match { - case AppType.Galaxy => - for { - machineType <- F.fromOption( - params.appMachineType, - new LeoException( - s"can't find machine config for ${googleProject.value}/${app.appName.value}. This should never happen", - traceId = Some(ctx.traceId) - ) - ) - _ <- installGalaxy( - helmAuthContext, - app.appName, - app.release, - app.chart, - dbCluster, - dbApp.nodepool.nodepoolName, // https://broadworkbench.atlassian.net/browse/IA-4987 - namespaceName, - app.auditInfo.creator, - app.customEnvironmentVariables, - ksaName, - nfsDisk, - machineType, - galaxyRestore - ) - } yield () case AppType.Cromwell => installCromwellApp( helmAuthContext, @@ -551,60 +550,15 @@ class GKEInterpreter[F[_]]( F.raiseError(AppCreationException(s"App type ${app.appType} not supported on GCP")) } - _ <- logger.info(ctx.loggingCtx)( - s"Finished app creation for app ${app.appName.value} in cluster ${gkeClusterId.toString}" - ) - _ <- app.appType match { - case AppType.Galaxy => - if (galaxyRestore.isDefined) persistentDiskQuery.updateLastUsedBy(diskId, app.id).transaction.void - else - for { - pvcs <- kubeService.listPersistentVolumeClaims(gkeClusterId, - KubernetesNamespace(app.appResources.namespace) - ) - - _ <- pvcs - // We added an extra -galaxy here: https://github.com/galaxyproject/galaxykubeman-helm/blob/f7f27be74c213deda3ae53122[…]959c96480bb21f/galaxykubeman/templates/config-setup-galaxy.yaml - .find(pvc => pvc.getMetadata.getName == s"${app.release.asString}-galaxy-galaxy-pvc") - .fold( - F.raiseError[Unit]( - PubsubKubernetesError(AppError("Fail to retrieve pvc ids", - ctx.now, - ErrorAction.CreateApp, - ErrorSource.App, - None, - Some(ctx.traceId) - ), - Some(app.id), - false, - None, - None, - None - ) - ) - ) { galaxyPvc => - val galaxyDiskRestore = AppRestore.GalaxyRestore( - PvcId(galaxyPvc.getMetadata.getUid), - app.id - ) - persistentDiskQuery - .updateGalaxyDiskRestore(diskId, galaxyDiskRestore) - .transaction - .void - } - } yield () case AppType.Cromwell => persistentDiskQuery.updateLastUsedBy(diskId, app.id).transaction case AppType.Allowed => persistentDiskQuery.updateLastUsedBy(diskId, app.id).transaction case AppType.Custom => F.unit case _ => F.raiseError(AppCreationException(s"App type ${app.appType} not supported on GCP")) } - - readyTime <- F.realTimeInstant - _ <- appUsageQuery.recordStart(params.appId, readyTime) - _ <- appQuery.updateStatus(params.appId, AppStatus.Running).transaction } yield () + } override def deleteAndPollCluster(params: DeleteClusterParams)(implicit ev: Ask[F, AppContext]): F[Unit] = for { @@ -725,77 +679,86 @@ class GKEInterpreter[F[_]]( // Resolve the cluster in Google googleClusterOpt <- gkeService.getCluster(gkeClusterId) - _ <- googleClusterOpt - .traverse { googleCluster => - val uninstallCharts = for { - helmAuthContext <- getHelmAuthContext(googleCluster, dbCluster, namespaceName) - - _ <- logger.info(ctx.loggingCtx)( - s"Uninstalling release ${app.release.asString} for ${app.appType.toString} app ${app.appName.value} in cluster ${dbCluster.getClusterId.toString}" + _ <- app.appType match { + case AppType.Galaxy => + // Galaxy runs on a VM: delete the GCE instance. Disks are retained for persistence. + for { + gp <- F.fromOption( + LeoLenses.cloudContextToGoogleProject.get(dbCluster.cloudContext), + new RuntimeException("Galaxy app cloud context should be a google project") ) - - // helm uninstall the app chart and wait - _ <- helmClient - .uninstall(app.release, config.galaxyAppConfig.uninstallKeepHistory) - .run(helmAuthContext) - - last <- streamFUntilDone( - kubeService.listPodStatus(dbCluster.getClusterId, KubernetesNamespace(namespaceName)), - config.monitorConfig.deleteApp.maxAttempts, - config.monitorConfig.deleteApp.interval - ).compile.lastOrError - - _ <- - if (!podDoneCheckable.isDone(last)) { - val msg = - s"Helm deletion has failed or timed out for app ${app.appName.value} in cluster ${dbCluster.getClusterId.toString}. The following pods are not in a terminal state: ${last - .filterNot(isPodDone) - .map(_.name.value) - .mkString(", ")}" - logger.error(ctx.loggingCtx)(msg) >> - F.raiseError[Unit](AppDeletionException(msg)) - } else F.unit - - // helm uninstall the setup chart - _ <- helmClient - .uninstall( - getTerraAppSetupChartReleaseName(app.release), - config.galaxyAppConfig.uninstallKeepHistory - ) - .run(helmAuthContext) + instanceName = InstanceName(s"galaxy-${app.appName.value}") + zone = dbApp.app.appResources.disk.map(_.zone).getOrElse(ZoneName(config.clusterConfig.location.value)) + _ <- computeService + .deleteInstance(gp, zone, instanceName) + .void + .handleErrorWith { e => + logger.warn(ctx.loggingCtx)( + s"Failed to delete Galaxy VM ${instanceName.value}: ${e.getMessage}. Continuing with app deletion." + ) + } } yield () - uninstallCharts.handleErrorWith { e => - logger.info(ctx.loggingCtx)( - s"Uninstalling release ${app.release.asString} for ${app.appType.toString} app ${app.appName.value} in cluster ${dbCluster.getClusterId.toString} failed with error ${e.getMessage}" - ) - } - } - - // delete the namespace only after the helm uninstall completes - _ <- kubeService.deleteNamespace(dbApp.cluster.getClusterId, - KubernetesNamespace(dbApp.app.appResources.namespace) - ) + case _ => + // GKE/Helm path for all other app types + googleClusterOpt + .traverse { googleCluster => + val uninstallCharts = for { + helmAuthContext <- getHelmAuthContext(googleCluster, dbCluster, namespaceName) + + _ <- logger.info(ctx.loggingCtx)( + s"Uninstalling release ${app.release.asString} for ${app.appType.toString} app ${app.appName.value} in cluster ${dbCluster.getClusterId.toString}" + ) - fa = kubeService - .namespaceExists(dbApp.cluster.getClusterId, KubernetesNamespace(dbApp.app.appResources.namespace)) - .map(!_) // mapping to inverse because booleanDoneCheckable defines `Done` when it becomes `true`...In this case, the namespace will exists for a while, and eventually becomes non-existent + _ <- helmClient + .uninstall(app.release, true) + .run(helmAuthContext) + + last <- streamFUntilDone( + kubeService.listPodStatus(dbCluster.getClusterId, KubernetesNamespace(namespaceName)), + config.monitorConfig.deleteApp.maxAttempts, + config.monitorConfig.deleteApp.interval + ).compile.lastOrError + + _ <- + if (!podDoneCheckable.isDone(last)) { + val msg = + s"Helm deletion has failed or timed out for app ${app.appName.value} in cluster ${dbCluster.getClusterId.toString}. The following pods are not in a terminal state: ${last + .filterNot(isPodDone) + .map(_.name.value) + .mkString(", ")}" + logger.error(ctx.loggingCtx)(msg) >> + F.raiseError[Unit](AppDeletionException(msg)) + } else F.unit + + _ <- helmClient + .uninstall(getTerraAppSetupChartReleaseName(app.release), true) + .run(helmAuthContext) + } yield () + + uninstallCharts.handleErrorWith { e => + logger.info(ctx.loggingCtx)( + s"Uninstalling release ${app.release.asString} for ${app.appType.toString} app ${app.appName.value} in cluster ${dbCluster.getClusterId.toString} failed with error ${e.getMessage}" + ) + } + } + .void >> + kubeService + .deleteNamespace(dbApp.cluster.getClusterId, KubernetesNamespace(dbApp.app.appResources.namespace)) >> + streamUntilDoneOrTimeout( + kubeService + .namespaceExists(dbApp.cluster.getClusterId, KubernetesNamespace(dbApp.app.appResources.namespace)) + .map(!_), + 60, + 5 seconds, + "delete namespace timed out" + ) + } - _ <- streamUntilDoneOrTimeout(fa, 60, 5 seconds, "delete namespace timed out") _ <- logger.info(ctx.loggingCtx)( - s"Delete app operation has finished for app ${app.appName.value} in cluster ${gkeClusterId.toString}" + s"Delete app operation has finished for app ${app.appName.value}" ) - appRestore: Option[AppRestore.GalaxyRestore] = dbApp.app.appResources.disk.flatMap(_.appRestore).flatMap { - case a: AppRestore.GalaxyRestore => Some(a) - case _: AppRestore.Other => None - } - _ <- appRestore.traverse { restore => - for { - _ <- kubeService.deletePv(dbCluster.getClusterId, PvName(s"pvc-${restore.galaxyPvcId.asString}")) - } yield () - } - _ <- if (!params.errorAfterDelete) { F.unit @@ -1016,29 +979,6 @@ class GKEInterpreter[F[_]]( } } yield () - private[leonardo] def getGalaxyPostgresDisk(diskName: DiskName, - namespaceName: NamespaceName, - project: GoogleProject, - zone: ZoneName - )(implicit traceId: Ask[F, AppContext]): F[Option[Disk]] = - for { - postgresDiskOpt <- googleDiskService - .getDisk( - project, - zone, - getGalaxyPostgresDiskName(diskName, config.galaxyDiskConfig.postgresDiskNameSuffix) - ) - res <- postgresDiskOpt match { - case Some(disk) => F.pure(Some(disk)) - case None => - googleDiskService.getDisk( - project, - zone, - getOldStyleGalaxyPostgresDiskName(namespaceName, config.galaxyDiskConfig.postgresDiskNameSuffix) - ) - } - } yield res - private[util] def installNginx(dbCluster: KubernetesCluster, googleCluster: Cluster)(implicit ev: Ask[F, AppContext] ): F[IP] = @@ -1085,94 +1025,204 @@ class GKEInterpreter[F[_]]( ) } yield loadBalancerIp - private[util] def installGalaxy(helmAuthContext: AuthContext, - appName: AppName, - release: Release, - chart: Chart, - dbCluster: KubernetesCluster, - nodepoolName: NodepoolName, - namespaceName: NamespaceName, - userEmail: WorkbenchEmail, - customEnvironmentVariables: Map[String, String], - kubernetesServiceAccount: ServiceAccountName, - nfsDisk: PersistentDisk, - machineType: AppMachineType, - galaxyRestore: Option[AppRestore.GalaxyRestore] - )(implicit - ev: Ask[F, AppContext] - ): F[Unit] = + private[util] def installGalaxyVm( + dbCluster: KubernetesCluster, + app: App, + nfsDisk: PersistentDisk, + googleProject: GoogleProject + )(implicit ev: Ask[F, AppContext]): F[Unit] = for { ctx <- ev.ask _ <- logger.info(ctx.loggingCtx)( - s"Installing helm chart $chart for app ${appName.value} in cluster ${dbCluster.getClusterId.toString}" - ) - googleProject <- F.fromOption( - LeoLenses.cloudContextToGoogleProject.get(nfsDisk.cloudContext), - new RuntimeException("this should never happen. Galaxy disk's cloud context should be a google project") + s"Installing Galaxy VM for app ${app.appName.value} in project ${googleProject.value}" ) - postgresDiskNameOpt <- for { - disk <- getGalaxyPostgresDisk(nfsDisk.name, namespaceName, googleProject, nfsDisk.zone) - } yield disk.map(x => DiskName(x.getName)) - postgresDiskName <- F.fromOption( - postgresDiskNameOpt, - AppCreationException(s"No postgres disk found in google for app ${appName.value} ", traceId = Some(ctx.traceId)) + zoneParam = nfsDisk.zone + regionParam = RegionName(zoneParam.value.dropRight(2)) + + // Set up VPC and firewall + (network, subnetwork) <- vpcAlg.setUpProjectNetworkAndFirewalls( + SetUpProjectNetworkParams(googleProject, regionParam) ) - chartValues = buildGalaxyChartOverrideValuesString( - config, - appName, - release, - dbCluster, - nodepoolName, - userEmail, - customEnvironmentVariables, - kubernetesServiceAccount, - namespaceName, - nfsDisk, - postgresDiskName, - machineType, - galaxyRestore + // Load cloud-config content bundled from galaxy-k8s-boot bin/user_data.sh. + // Intentionally not fetched at runtime to avoid unexpected production changes. + // To update, sync manually from https://github.com/galaxyproject/galaxy-k8s-boot/blob/dev/bin/user_data.sh + userDataContent = scala.io.Source + .fromResource("init-resources/galaxy-user-data.sh") + .getLines() + .toList + .mkString("\n") + + // Derive postgres disk name using the same naming convention as the subscriber + postgresDiskName = GKEAlgebra.getGalaxyPostgresDiskName(nfsDisk.name, config.galaxyDiskConfig.postgresDiskNameSuffix) + + // Persistent-volume-size passed to ansible-pull (leave ~11 GiB for filesystem overhead on 150 GB disk) + pvSizeGi = math.max(1, nfsDisk.size.gb - 11) + pvSize = s"${pvSizeGi}Gi" + + // GCP Batch SA: prefer value from customEnvironmentVariables, fall back to config default + gcpBatchSa = app.customEnvironmentVariables.getOrElse( + "gcp_batch_service_account_email", + config.galaxyVmConfig.gcpBatchServiceAccountEmail ) + // restore_galaxy flag + restoreGalaxy = app.customEnvironmentVariables.getOrElse("restore_galaxy", "false") + + // Disks + bootDisk = AttachedDisk + .newBuilder() + .setBoot(true) + .setAutoDelete(true) + .setInitializeParams( + AttachedDiskInitializeParams + .newBuilder() + .setSourceImage(config.galaxyVmConfig.sourceImage.asString) + .setDiskSizeGb(config.galaxyVmConfig.bootDiskSizeGb.gb) + .putAllLabels(Map("leonardo" -> "true").asJava) + .build() + ) + .build() + + // Galaxy data disk — device name must match what the bootstrap script expects + dataDisk = AttachedDisk + .newBuilder() + .setBoot(false) + .setDeviceName("galaxy-data") + .setAutoDelete(false) + .setInitializeParams( + AttachedDiskInitializeParams + .newBuilder() + .setDiskName(nfsDisk.name.value) + .setDiskSizeGb(nfsDisk.size.gb) + .setDiskType(nfsDisk.diskType.googleString(googleProject, zoneParam)) + .putAllLabels(Map("leonardo" -> "true").asJava) + .build() + ) + .build() + + // PostgreSQL disk — device name must match what the bootstrap script expects + postgresDisk = AttachedDisk + .newBuilder() + .setBoot(false) + .setDeviceName("galaxy-postgres-data") + .setAutoDelete(false) + .setInitializeParams( + AttachedDiskInitializeParams + .newBuilder() + .setDiskName(postgresDiskName.value) + .setDiskSizeGb(config.galaxyVmConfig.postgresDiskSizeGb.gb) + .putAllLabels(Map("leonardo" -> "true").asJava) + .build() + ) + .build() + + // Network interface with external IP + networkInterface = NetworkInterface + .newBuilder() + .setSubnetwork( + buildSubnetworkUri(googleProject, regionParam, subnetwork) + ) + .addAccessConfigs(AccessConfig.newBuilder().setName("Leonardo Galaxy VM external IP").build()) + .build() + + instanceName = InstanceName(s"galaxy-${app.appName.value}") + + instance = Instance + .newBuilder() + .setName(instanceName.value) + .setDescription("Leonardo Galaxy VM") + .setTags(Tags.newBuilder().addItems(config.vpcNetworkTag.value).build()) + .setMachineType(buildMachineTypeUri(zoneParam, config.galaxyVmConfig.machineType)) + .addNetworkInterfaces(networkInterface) + .addAllDisks(List(bootDisk, dataDisk, postgresDisk).asJava) + .addServiceAccounts( + ServiceAccount + .newBuilder() + .setEmail(app.googleServiceAccount.value) + .addAllScopes( + List( + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/logging.write" + ).asJava + ) + .build() + ) + .setMetadata( + Metadata + .newBuilder() + .addItems(Items.newBuilder().setKey("user-data").setValue(userDataContent).build()) + .addItems(Items.newBuilder().setKey("google-logging-enabled").setValue("true").build()) + .addItems(Items.newBuilder().setKey("gcp_batch_service_account_email").setValue(gcpBatchSa).build()) + .addItems(Items.newBuilder().setKey("persistent-volume-size").setValue(pvSize).build()) + .addItems(Items.newBuilder().setKey("restore_galaxy").setValue(restoreGalaxy).build()) + .addItems(Items.newBuilder().setKey("git-repo").setValue(config.galaxyVmConfig.gitRepo).build()) + .addItems(Items.newBuilder().setKey("git-branch").setValue(config.galaxyVmConfig.gitBranch).build()) + .addItems(Items.newBuilder().setKey("gcp-region").setValue(regionParam.value).build()) + .addItems(Items.newBuilder().setKey("gcp-network").setValue(network.value).build()) + .addItems(Items.newBuilder().setKey("gcp-subnet").setValue(subnetwork.value).build()) + .build() + ) + .putAllLabels(Map("leonardo" -> "true").asJava) + .build() + + _ <- computeService.createInstance(googleProject, zoneParam, instance) + _ <- logger.info(ctx.loggingCtx)( - s"Chart override values are: ${chartValues.map(s => - if (s.contains("galaxyDatabasePassword")) "persistence.postgres.galaxyDatabasePassword=" - else s - )}" + s"Galaxy VM instance ${instanceName.value} submitted for project ${googleProject.value}; polling for external IP" ) - // Invoke helm - helmInstall = helmClient - .installChart( - release, - chart.name, - chart.version, - org.broadinstitute.dsp.Values(chartValues.mkString(",")), - false + // Poll until the instance has an external IP, then store it as the cluster's load balancer IP + // so that KubernetesDnsCache can resolve the proxy host. + externalIpOpt <- streamFUntilDone( + computeService.getInstance(googleProject, zoneParam, instanceName).map { instanceOpt => + instanceOpt.flatMap { inst => + import scala.jdk.CollectionConverters._ + for { + iface <- Option(inst.getNetworkInterfacesList).flatMap(_.asScala.headOption) + cfg <- Option(iface.getAccessConfigsList).flatMap(_.asScala.headOption) + natIp <- Option(cfg.getNatIP).filter(_.nonEmpty) + } yield IP(natIp) + } + }, + config.monitorConfig.createApp.maxAttempts, + config.monitorConfig.createApp.interval + ).compile.lastOrError + + externalIp <- F.fromOption( + externalIpOpt, + AppCreationException( + s"Galaxy VM ${instanceName.value} did not obtain an external IP after ${config.monitorConfig.createApp.interruptAfter}", + traceId = Some(ctx.traceId) ) - .run(helmAuthContext) + ) - // Currently we always retry. - // The main failure mode here is helm install, which does not have easily interpretable error codes - retryConfig = RetryPredicates.retryAllConfig - _ <- tracedRetryF(retryConfig)( - helmInstall, - s"helm install for app ${appName.value} in project ${dbCluster.cloudContext.asString}" - ).compile.lastOrError + _ <- logger.info(ctx.loggingCtx)( + s"Galaxy VM ${instanceName.value} has external IP ${externalIp.asString}; storing in cluster async fields" + ) - googleProject <- F.fromOption( - LeoLenses.cloudContextToGoogleProject.get(dbCluster.cloudContext), - new RuntimeException("trying to create a non google runtime in GKEInterpreter. This should never happen") + // Store the VM's external IP as the cluster load balancer IP consumed by KubernetesDnsCache + _ <- kubernetesClusterQuery + .updateAsyncFields( + dbCluster.id, + KubernetesClusterAsyncFields( + externalIp, + IP(""), + NetworkFields(NetworkName(""), SubnetworkName(""), IpRange("")) + ) + ) + .transaction + _ <- kubernetesClusterQuery.updateStatus(dbCluster.id, KubernetesClusterStatus.Running).transaction + + _ <- logger.info(ctx.loggingCtx)( + s"Polling Galaxy readiness for app ${app.appName.value} at ${externalIp.asString}:80" ) - // Poll galaxy until it starts up - // TODO potentially add other status checks for pod readiness, beyond just HTTP polling the galaxy-web service - // Wait a bit before starting polling for the app status check as the certificates might not be quite ready yet - // This seems to only impact galaxy, See https://broadworkbench.atlassian.net/browse/IA-4551 - _ <- F.sleep(60 seconds) + + // Wait for Galaxy's nginx ingress to respond on port 80 isDone <- streamFUntilDone( - appDao.isProxyAvailable(googleProject, appName, ServiceName("galaxy"), ctx.traceId), + appDao.isProxyAvailable(googleProject, app.appName, ServiceName("galaxy"), ctx.traceId), config.monitorConfig.createApp.maxAttempts, config.monitorConfig.createApp.interval ).interruptAfter(config.monitorConfig.createApp.interruptAfter).compile.lastOrError @@ -1180,9 +1230,9 @@ class GKEInterpreter[F[_]]( _ <- if (!isDone) { val msg = - s"Galaxy installation has failed or timed out for app ${appName.value} in cluster ${dbCluster.getClusterId.toString}" + s"Galaxy VM installation has failed or timed out for app ${app.appName.value} in project ${googleProject.value}" logger.error(ctx.loggingCtx)(msg) >> - F.raiseError[Unit](AppCreationException(msg)) + F.raiseError[Unit](AppCreationException(msg, traceId = Some(ctx.traceId))) } else F.unit } yield () @@ -1833,14 +1883,14 @@ final case class GKEInterpreterConfig(leoUrlBase: URL, vpcNetworkTag: NetworkTag, terraAppSetupChartConfig: TerraAppSetupChartConfig, ingressConfig: KubernetesIngressConfig, - galaxyAppConfig: GalaxyAppConfig, cromwellAppConfig: CromwellAppConfig, customAppConfig: CustomAppConfig, allowedAppConfig: AllowedAppConfig, monitorConfig: AppMonitorConfig, clusterConfig: KubernetesClusterConfig, proxyConfig: ProxyConfig, - galaxyDiskConfig: GalaxyDiskConfig + galaxyDiskConfig: GalaxyDiskConfig, + galaxyVmConfig: GalaxyVmConfig ) final case class TerraAppSetupChartConfig( diff --git a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/util/BuildHelmChartValuesSpec.scala b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/util/BuildHelmChartValuesSpec.scala index 6ea504a907..a408cd4559 100644 --- a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/util/BuildHelmChartValuesSpec.scala +++ b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/util/BuildHelmChartValuesSpec.scala @@ -4,7 +4,6 @@ package util import org.broadinstitute.dsde.workbench.google2.DiskName import org.broadinstitute.dsde.workbench.google2.GKEModels.NodepoolName import org.broadinstitute.dsde.workbench.google2.KubernetesSerializableName.{NamespaceName, ServiceAccountName} -import org.broadinstitute.dsde.workbench.leonardo.AppRestore.GalaxyRestore import org.broadinstitute.dsde.workbench.leonardo.CommonTestData.{makePersistentDisk, userEmail, userEmail2} import org.broadinstitute.dsde.workbench.leonardo.KubernetesTestData.{makeCustomAppService, makeKubeCluster} import org.broadinstitute.dsde.workbench.leonardo.config.Config @@ -16,177 +15,6 @@ import org.scalatest.flatspec.AnyFlatSpecLike class BuildHelmChartValuesSpec extends AnyFlatSpecLike with LeonardoTestSuite { - it should "build Galaxy override values string" in { - val savedCluster1 = makeKubeCluster(1) - val savedDisk1 = makePersistentDisk(Some(DiskName("disk1")), Some(FormattedBy.Galaxy)) - val res = buildGalaxyChartOverrideValuesString( - Config.gkeInterpConfig, - AppName("app1"), - Release("app1-galaxy-rls"), - savedCluster1, - NodepoolName("pool1"), - userEmail, - Map("WORKSPACE_NAME" -> "test-workspace", - "WORKSPACE_BUCKET" -> "gs://test-bucket", - "WORKSPACE_NAMESPACE" -> "dsp-leo-test1" - ), - ServiceAccountName("app1-galaxy-ksa"), - NamespaceName("ns"), - savedDisk1, - DiskName("disk1-gxy-postres-disk"), - AppMachineType(23, 7), - None - ) - - res.mkString( - "," - ) shouldBe - """nfs.storageClass.name=nfs-app1-galaxy-rls,""" + - """galaxy.persistence.storageClass=nfs-app1-galaxy-rls,""" + - """galaxy.nodeSelector.cloud\.google\.com/gke-nodepool=pool1,""" + - """nfs.nodeSelector.cloud\.google\.com/gke-nodepool=pool1,""" + - """galaxy.configs.job_conf\.yml.runners.k8s.k8s_node_selector=cloud.google.com/gke-nodepool: pool1,""" + - """galaxy.postgresql.master.nodeSelector.cloud\.google\.com/gke-nodepool=pool1,""" + - """galaxy.ingress.path=/proxy/google/v1/apps/dsp-leo-test1/app1/galaxy,""" + - """galaxy.ingress.annotations.nginx\.ingress\.kubernetes\.io/proxy-redirect-from=https://1455694897.jupyter.firecloud.org,""" + - """galaxy.ingress.annotations.nginx\.ingress\.kubernetes\.io/proxy-redirect-to=https://leo,""" + - """galaxy.ingress.hosts[0].host=1455694897.jupyter.firecloud.org,""" + - """galaxy.ingress.hosts[0].paths[0].path=/proxy/google/v1/apps/dsp-leo-test1/app1/galaxy,""" + - """galaxy.ingress.tls[0].hosts[0]=1455694897.jupyter.firecloud.org,""" + - """galaxy.ingress.tls[0].secretName=tls-secret,""" + - """cvmfs.cvmfscsi.cache.alien.pvc.storageClass=nfs-app1-galaxy-rls,""" + - """cvmfs.cvmfscsi.cache.alien.pvc.name=cvmfs-alien-cache,""" + - """galaxy.configs.galaxy\.yml.galaxy.single_user=user1@example.com,""" + - """galaxy.configs.galaxy\.yml.galaxy.admin_users=user1@example.com,""" + - """galaxy.terra.launch.workspace=test-workspace,""" + - """galaxy.terra.launch.namespace=dsp-leo-test1,""" + - """galaxy.terra.launch.apiURL=https://firecloud-orchestration.dsde-dev.broadinstitute.org/api/,""" + - """galaxy.terra.launch.drsURL=https://drshub.dsde-dev.broadinstitute.org/api/v4/drs/resolve,""" + - """galaxy.tusd.ingress.hosts[0].host=1455694897.jupyter.firecloud.org,""" + - """galaxy.tusd.ingress.hosts[0].paths[0].path=/proxy/google/v1/apps/dsp-leo-test1/app1/galaxy/api/upload/resumable_upload,""" + - """galaxy.tusd.ingress.tls[0].hosts[0]=1455694897.jupyter.firecloud.org,""" + - """galaxy.tusd.ingress.tls[0].secretName=tls-secret,""" + - """galaxy.rabbitmq.persistence.storageClassName=nfs-app1-galaxy-rls,""" + - """galaxy.jobs.maxLimits.memory=23,""" + - """galaxy.jobs.maxLimits.cpu=7,""" + - """galaxy.jobs.maxRequests.memory=1,""" + - """galaxy.jobs.maxRequests.cpu=1,""" + - """galaxy.jobs.rules.tpv_rules_local\.yml.destinations.k8s.max_mem=1,""" + - """galaxy.jobs.rules.tpv_rules_local\.yml.destinations.k8s.max_cores=1,""" + - """galaxy.serviceAccount.create=false,""" + - """galaxy.serviceAccount.name=app1-galaxy-ksa,""" + - """rbac.serviceAccount=app1-galaxy-ksa,persistence.nfs.name=ns-nfs-disk,""" + - """persistence.nfs.persistentVolume.extraSpec.gcePersistentDisk.pdName=disk1,""" + - """persistence.nfs.size=250Gi,""" + - """persistence.postgres.name=ns-postgres-disk,""" + - """galaxy.postgresql.galaxyDatabasePassword=replace-me,""" + - """persistence.postgres.persistentVolume.extraSpec.gcePersistentDisk.pdName=disk1-gxy-postres-disk,""" + - """persistence.postgres.size=10Gi,""" + - """nfs.persistence.existingClaim=ns-nfs-disk-pvc,""" + - """nfs.persistence.size=250Gi,""" + - """galaxy.postgresql.persistence.existingClaim=ns-postgres-disk-pvc,""" + - """galaxy.persistence.size=200Gi,""" + - """configs.WORKSPACE_NAME=test-workspace,""" + - """extraEnv[0].name=WORKSPACE_NAME,extraEnv[0].valueFrom.configMapKeyRef.name=app1-galaxy-rls-galaxykubeman-configs,""" + - """extraEnv[0].valueFrom.configMapKeyRef.key=WORKSPACE_NAME,""" + - """configs.WORKSPACE_BUCKET=gs://test-bucket,""" + - """extraEnv[1].name=WORKSPACE_BUCKET,""" + - """extraEnv[1].valueFrom.configMapKeyRef.name=app1-galaxy-rls-galaxykubeman-configs,""" + - """extraEnv[1].valueFrom.configMapKeyRef.key=WORKSPACE_BUCKET,""" + - """configs.WORKSPACE_NAMESPACE=dsp-leo-test1,""" + - """extraEnv[2].name=WORKSPACE_NAMESPACE,""" + - """extraEnv[2].valueFrom.configMapKeyRef.name=app1-galaxy-rls-galaxykubeman-configs,""" + - """extraEnv[2].valueFrom.configMapKeyRef.key=WORKSPACE_NAMESPACE""" - } - - it should "build Galaxy override values string with restore info" in { - val savedCluster1 = makeKubeCluster(1) - val savedDisk1 = makePersistentDisk(Some(DiskName("disk1")), Some(FormattedBy.Galaxy)) - val result = - buildGalaxyChartOverrideValuesString( - Config.gkeInterpConfig, - AppName("app1"), - Release("app1-galaxy-rls"), - savedCluster1, - NodepoolName("pool1"), - userEmail, - Map("WORKSPACE_NAME" -> "test-workspace", - "WORKSPACE_BUCKET" -> "gs://test-bucket", - "WORKSPACE_NAMESPACE" -> "dsp-leo-test1" - ), - ServiceAccountName("app1-galaxy-ksa"), - NamespaceName("ns"), - savedDisk1, - DiskName("disk1-gxy-postres"), - AppMachineType(23, 7), - Some( - GalaxyRestore(PvcId("galaxy-pvc-id"), AppId(123)) - ) - ) - result.mkString( - "," - ) shouldBe - """nfs.storageClass.name=nfs-app1-galaxy-rls,""" + - """galaxy.persistence.storageClass=nfs-app1-galaxy-rls,""" + - """galaxy.nodeSelector.cloud\.google\.com/gke-nodepool=pool1,""" + - """nfs.nodeSelector.cloud\.google\.com/gke-nodepool=pool1,""" + - """galaxy.configs.job_conf\.yml.runners.k8s.k8s_node_selector=cloud.google.com/gke-nodepool: pool1,""" + - """galaxy.postgresql.master.nodeSelector.cloud\.google\.com/gke-nodepool=pool1,""" + - """galaxy.ingress.path=/proxy/google/v1/apps/dsp-leo-test1/app1/galaxy,""" + - """galaxy.ingress.annotations.nginx\.ingress\.kubernetes\.io/proxy-redirect-from=https://1455694897.jupyter.firecloud.org,""" + - """galaxy.ingress.annotations.nginx\.ingress\.kubernetes\.io/proxy-redirect-to=https://leo,""" + - """galaxy.ingress.hosts[0].host=1455694897.jupyter.firecloud.org,""" + - """galaxy.ingress.hosts[0].paths[0].path=/proxy/google/v1/apps/dsp-leo-test1/app1/galaxy,""" + - """galaxy.ingress.tls[0].hosts[0]=1455694897.jupyter.firecloud.org,""" + - """galaxy.ingress.tls[0].secretName=tls-secret,""" + - """cvmfs.cvmfscsi.cache.alien.pvc.storageClass=nfs-app1-galaxy-rls,""" + - """cvmfs.cvmfscsi.cache.alien.pvc.name=cvmfs-alien-cache,""" + - """galaxy.configs.galaxy\.yml.galaxy.single_user=user1@example.com,""" + - """galaxy.configs.galaxy\.yml.galaxy.admin_users=user1@example.com,""" + - """galaxy.terra.launch.workspace=test-workspace,""" + - """galaxy.terra.launch.namespace=dsp-leo-test1,""" + - """galaxy.terra.launch.apiURL=https://firecloud-orchestration.dsde-dev.broadinstitute.org/api/,""" + - """galaxy.terra.launch.drsURL=https://drshub.dsde-dev.broadinstitute.org/api/v4/drs/resolve,""" + - """galaxy.tusd.ingress.hosts[0].host=1455694897.jupyter.firecloud.org,""" + - """galaxy.tusd.ingress.hosts[0].paths[0].path=/proxy/google/v1/apps/dsp-leo-test1/app1/galaxy/api/upload/resumable_upload,""" + - """galaxy.tusd.ingress.tls[0].hosts[0]=1455694897.jupyter.firecloud.org,""" + - """galaxy.tusd.ingress.tls[0].secretName=tls-secret,""" + - """galaxy.rabbitmq.persistence.storageClassName=nfs-app1-galaxy-rls,""" + - """galaxy.jobs.maxLimits.memory=23,""" + - """galaxy.jobs.maxLimits.cpu=7,""" + - """galaxy.jobs.maxRequests.memory=1,""" + - """galaxy.jobs.maxRequests.cpu=1,""" + - """galaxy.jobs.rules.tpv_rules_local\.yml.destinations.k8s.max_mem=1,""" + - """galaxy.jobs.rules.tpv_rules_local\.yml.destinations.k8s.max_cores=1,""" + - """galaxy.serviceAccount.create=false,""" + - """galaxy.serviceAccount.name=app1-galaxy-ksa,""" + - """rbac.serviceAccount=app1-galaxy-ksa,""" + - """persistence.nfs.name=ns-nfs-disk,""" + - """persistence.nfs.persistentVolume.extraSpec.gcePersistentDisk.pdName=disk1,persistence.nfs.size=250Gi,""" + - """persistence.postgres.name=ns-postgres-disk,""" + - """galaxy.postgresql.galaxyDatabasePassword=replace-me,""" + - """persistence.postgres.persistentVolume.extraSpec.gcePersistentDisk.pdName=disk1-gxy-postres,""" + - """persistence.postgres.size=10Gi,""" + - """nfs.persistence.existingClaim=ns-nfs-disk-pvc,""" + - """nfs.persistence.size=250Gi,""" + - """galaxy.postgresql.persistence.existingClaim=ns-postgres-disk-pvc,""" + - """galaxy.persistence.size=200Gi,""" + - """configs.WORKSPACE_NAME=test-workspace,""" + - """extraEnv[0].name=WORKSPACE_NAME,""" + - """extraEnv[0].valueFrom.configMapKeyRef.name=app1-galaxy-rls-galaxykubeman-configs,""" + - """extraEnv[0].valueFrom.configMapKeyRef.key=WORKSPACE_NAME,""" + - """configs.WORKSPACE_BUCKET=gs://test-bucket,""" + - """extraEnv[1].name=WORKSPACE_BUCKET,""" + - """extraEnv[1].valueFrom.configMapKeyRef.name=app1-galaxy-rls-galaxykubeman-configs,""" + - """extraEnv[1].valueFrom.configMapKeyRef.key=WORKSPACE_BUCKET,""" + - """configs.WORKSPACE_NAMESPACE=dsp-leo-test1,""" + - """extraEnv[2].name=WORKSPACE_NAMESPACE,""" + - """extraEnv[2].valueFrom.configMapKeyRef.name=app1-galaxy-rls-galaxykubeman-configs,""" + - """extraEnv[2].valueFrom.configMapKeyRef.key=WORKSPACE_NAMESPACE,""" + - """restore.persistence.nfs.galaxy.pvcID=galaxy-pvc-id,""" + - """galaxy.persistence.existingClaim=app1-galaxy-rls-galaxy-galaxy-pvc""".stripMargin - } - it should "build Cromwell override values string" in { val savedCluster1 = makeKubeCluster(1) val savedDisk1 = makePersistentDisk(Some(DiskName("disk1"))) From 6dbafe6db539e98d790dd267389647dfad5f1757 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Tue, 7 Apr 2026 13:51:53 -0400 Subject: [PATCH 02/55] fix format --- .../leonardo/dao/HttpJupyterDAO.scala | 4 +- .../leonardo/util/GKEInterpreter.scala | 88 +++++++++---------- 2 files changed, 47 insertions(+), 45 deletions(-) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpJupyterDAO.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpJupyterDAO.scala index 0c55f089db..4d09a08109 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpJupyterDAO.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpJupyterDAO.scala @@ -36,7 +36,9 @@ class HttpJupyterDAO[F[_]](val runtimeDnsCache: RuntimeDnsCache[F], client: Clie headers = Headers.empty ) ) - .handleErrorWith(e => logger.warn(e)(s"isProxyAvailable failed for ${cloudContext}/${runtimeName}").as(false)) + .handleErrorWith(e => + logger.warn(e)(s"isProxyAvailable failed for ${cloudContext}/${runtimeName}").as(false) + ) case _ => F.pure(false) } } yield res diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala index b42d783d04..782f4adabe 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala @@ -701,48 +701,46 @@ class GKEInterpreter[F[_]]( case _ => // GKE/Helm path for all other app types - googleClusterOpt - .traverse { googleCluster => - val uninstallCharts = for { - helmAuthContext <- getHelmAuthContext(googleCluster, dbCluster, namespaceName) + googleClusterOpt.traverse { googleCluster => + val uninstallCharts = for { + helmAuthContext <- getHelmAuthContext(googleCluster, dbCluster, namespaceName) - _ <- logger.info(ctx.loggingCtx)( - s"Uninstalling release ${app.release.asString} for ${app.appType.toString} app ${app.appName.value} in cluster ${dbCluster.getClusterId.toString}" - ) + _ <- logger.info(ctx.loggingCtx)( + s"Uninstalling release ${app.release.asString} for ${app.appType.toString} app ${app.appName.value} in cluster ${dbCluster.getClusterId.toString}" + ) - _ <- helmClient - .uninstall(app.release, true) - .run(helmAuthContext) - - last <- streamFUntilDone( - kubeService.listPodStatus(dbCluster.getClusterId, KubernetesNamespace(namespaceName)), - config.monitorConfig.deleteApp.maxAttempts, - config.monitorConfig.deleteApp.interval - ).compile.lastOrError - - _ <- - if (!podDoneCheckable.isDone(last)) { - val msg = - s"Helm deletion has failed or timed out for app ${app.appName.value} in cluster ${dbCluster.getClusterId.toString}. The following pods are not in a terminal state: ${last - .filterNot(isPodDone) - .map(_.name.value) - .mkString(", ")}" - logger.error(ctx.loggingCtx)(msg) >> - F.raiseError[Unit](AppDeletionException(msg)) - } else F.unit - - _ <- helmClient - .uninstall(getTerraAppSetupChartReleaseName(app.release), true) - .run(helmAuthContext) - } yield () - - uninstallCharts.handleErrorWith { e => - logger.info(ctx.loggingCtx)( - s"Uninstalling release ${app.release.asString} for ${app.appType.toString} app ${app.appName.value} in cluster ${dbCluster.getClusterId.toString} failed with error ${e.getMessage}" - ) - } + _ <- helmClient + .uninstall(app.release, true) + .run(helmAuthContext) + + last <- streamFUntilDone( + kubeService.listPodStatus(dbCluster.getClusterId, KubernetesNamespace(namespaceName)), + config.monitorConfig.deleteApp.maxAttempts, + config.monitorConfig.deleteApp.interval + ).compile.lastOrError + + _ <- + if (!podDoneCheckable.isDone(last)) { + val msg = + s"Helm deletion has failed or timed out for app ${app.appName.value} in cluster ${dbCluster.getClusterId.toString}. The following pods are not in a terminal state: ${last + .filterNot(isPodDone) + .map(_.name.value) + .mkString(", ")}" + logger.error(ctx.loggingCtx)(msg) >> + F.raiseError[Unit](AppDeletionException(msg)) + } else F.unit + + _ <- helmClient + .uninstall(getTerraAppSetupChartReleaseName(app.release), true) + .run(helmAuthContext) + } yield () + + uninstallCharts.handleErrorWith { e => + logger.info(ctx.loggingCtx)( + s"Uninstalling release ${app.release.asString} for ${app.appType.toString} app ${app.appName.value} in cluster ${dbCluster.getClusterId.toString} failed with error ${e.getMessage}" + ) } - .void >> + }.void >> kubeService .deleteNamespace(dbApp.cluster.getClusterId, KubernetesNamespace(dbApp.app.appResources.namespace)) >> streamUntilDoneOrTimeout( @@ -1056,11 +1054,13 @@ class GKEInterpreter[F[_]]( .mkString("\n") // Derive postgres disk name using the same naming convention as the subscriber - postgresDiskName = GKEAlgebra.getGalaxyPostgresDiskName(nfsDisk.name, config.galaxyDiskConfig.postgresDiskNameSuffix) + postgresDiskName = GKEAlgebra.getGalaxyPostgresDiskName(nfsDisk.name, + config.galaxyDiskConfig.postgresDiskNameSuffix + ) // Persistent-volume-size passed to ansible-pull (leave ~11 GiB for filesystem overhead on 150 GB disk) pvSizeGi = math.max(1, nfsDisk.size.gb - 11) - pvSize = s"${pvSizeGi}Gi" + pvSize = s"${pvSizeGi}Gi" // GCP Batch SA: prefer value from customEnvironmentVariables, fall back to config default gcpBatchSa = app.customEnvironmentVariables.getOrElse( @@ -1181,9 +1181,9 @@ class GKEInterpreter[F[_]]( instanceOpt.flatMap { inst => import scala.jdk.CollectionConverters._ for { - iface <- Option(inst.getNetworkInterfacesList).flatMap(_.asScala.headOption) - cfg <- Option(iface.getAccessConfigsList).flatMap(_.asScala.headOption) - natIp <- Option(cfg.getNatIP).filter(_.nonEmpty) + iface <- Option(inst.getNetworkInterfacesList).flatMap(_.asScala.headOption) + cfg <- Option(iface.getAccessConfigsList).flatMap(_.asScala.headOption) + natIp <- Option(cfg.getNatIP).filter(_.nonEmpty) } yield IP(natIp) } }, From 27d5780252b32f4442fc573ac9264552ab17d228 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Tue, 7 Apr 2026 14:41:45 -0400 Subject: [PATCH 03/55] Grant pet SA GCP Batch IAM roles at Galaxy VM creation time At Galaxy VM creation, grant the pet SA roles/batch.jobsEditor on the user's project so it can submit and monitor GCP Batch jobs. When the Batch SA lives in the same project, also grant serviceAccountUser on it; cross-project Batch SAs must have that binding configured externally. Co-Authored-By: Claude Sonnet 4.6 --- .../leonardo/util/GKEInterpreter.scala | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala index 782f4adabe..335e88fb24 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala @@ -60,6 +60,7 @@ import org.broadinstitute.dsde.workbench.leonardo.util.BuildHelmChartValues.{ import org.broadinstitute.dsde.workbench.leonardo.model.LeoException import org.broadinstitute.dsde.workbench.leonardo.util.GKEAlgebra._ import org.broadinstitute.dsde.workbench.model.google.{GcsBucketName, GoogleProject} +import org.broadinstitute.dsde.workbench.model.google.iam.IamMemberTypes import org.broadinstitute.dsde.workbench.model.{IP, TraceId, WorkbenchEmail} import org.broadinstitute.dsde.workbench.openTelemetry.OpenTelemetryMetrics import org.broadinstitute.dsp._ @@ -1170,6 +1171,45 @@ class GKEInterpreter[F[_]]( _ <- computeService.createInstance(googleProject, zoneParam, instance) + // Grant the pet SA permission to submit and monitor GCP Batch jobs in this project. + // Galaxy uses the VM's attached SA (pet SA) to call the Batch API. + _ <- { + val call = F.fromFuture( + F.delay( + googleIamDAO + .addRoles(googleProject, app.googleServiceAccount, IamMemberTypes.ServiceAccount, Set("roles/batch.jobsEditor")) + .void + ) + ) + val retryConfig = RetryPredicates.retryConfigWithPredicates(when409) + tracedRetryF(retryConfig)( + call, + s"googleIamDAO.addRoles(batch.jobsEditor) for pet SA ${app.googleServiceAccount.value} in project ${googleProject.value}" + ).compile.lastOrError + } + + // Grant the pet SA serviceAccountUser on the Batch SA so it can specify it as the job runner identity. + // Only attempted when the Batch SA lives in the same project as the user (i.e. not a shared platform SA). + // For cross-project Batch SAs, this binding must be set up externally (e.g. via Terraform). + gcpBatchSaProject = GoogleProject(gcpBatchSa.split("@").lastOption.getOrElse("").replace(".iam.gserviceaccount.com", "")) + _ <- + if (gcpBatchSaProject == googleProject) + F.fromFuture( + F.delay( + googleIamDAO.addIamPolicyBindingOnServiceAccount( + googleProject, + WorkbenchEmail(gcpBatchSa), + app.googleServiceAccount, + Set("roles/iam.serviceAccountUser") + ) + ) + ) + else + logger.info(ctx.loggingCtx)( + s"Batch SA $gcpBatchSa is in a different project ($gcpBatchSaProject) than ${googleProject.value}; " + + s"skipping serviceAccountUser binding — must be configured externally" + ) + _ <- logger.info(ctx.loggingCtx)( s"Galaxy VM instance ${instanceName.value} submitted for project ${googleProject.value}; polling for external IP" ) From d5d00974cab00941b94d03b6b4f6a66fc89d7614 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Tue, 7 Apr 2026 16:09:40 -0400 Subject: [PATCH 04/55] add SA permissions and fix unit tests --- .../leonardo/util/GKEInterpreter.scala | 10 ++- .../LeoPubsubMessageSubscriberSpec.scala | 87 +++++++++++-------- .../leonardo/util/VPCInterpreterSpec.scala | 10 ++- 3 files changed, 68 insertions(+), 39 deletions(-) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala index 335e88fb24..b0f52276c1 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala @@ -1177,7 +1177,11 @@ class GKEInterpreter[F[_]]( val call = F.fromFuture( F.delay( googleIamDAO - .addRoles(googleProject, app.googleServiceAccount, IamMemberTypes.ServiceAccount, Set("roles/batch.jobsEditor")) + .addRoles(googleProject, + app.googleServiceAccount, + IamMemberTypes.ServiceAccount, + Set("roles/batch.jobsEditor") + ) .void ) ) @@ -1191,7 +1195,9 @@ class GKEInterpreter[F[_]]( // Grant the pet SA serviceAccountUser on the Batch SA so it can specify it as the job runner identity. // Only attempted when the Batch SA lives in the same project as the user (i.e. not a shared platform SA). // For cross-project Batch SAs, this binding must be set up externally (e.g. via Terraform). - gcpBatchSaProject = GoogleProject(gcpBatchSa.split("@").lastOption.getOrElse("").replace(".iam.gserviceaccount.com", "")) + gcpBatchSaProject = GoogleProject( + gcpBatchSa.split("@").lastOption.getOrElse("").replace(".iam.gserviceaccount.com", "") + ) _ <- if (gcpBatchSaProject == googleProject) F.fromFuture( diff --git a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriberSpec.scala b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriberSpec.scala index d932498dc3..d7f351488d 100644 --- a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriberSpec.scala +++ b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriberSpec.scala @@ -10,7 +10,7 @@ import cats.mtl.Ask import cats.syntax.all._ import com.github.benmanes.caffeine.cache.Caffeine import com.google.api.gax.longrunning.OperationFuture -import com.google.cloud.compute.v1.{Disk, Operation} +import com.google.cloud.compute.v1.{AccessConfig, Disk, Instance, NetworkInterface, Operation} import com.google.protobuf.Timestamp import fs2.Stream import org.broadinstitute.dsde.workbench.google.GoogleStorageDAO @@ -21,13 +21,16 @@ import org.broadinstitute.dsde.workbench.google2.mock.{MockKubernetesService => import org.broadinstitute.dsde.workbench.google2.{ DiskName, GKEModels, + GoogleComputeService, GoogleDiskService, GoogleStorageService, KubernetesModels, MachineTypeName, - RegionName, + NetworkName, + SubnetworkName, ZoneName } +import org.broadinstitute.dsde.workbench.util2.InstanceName import org.broadinstitute.dsde.workbench.leonardo.AppRestore.GalaxyRestore import org.broadinstitute.dsde.workbench.leonardo.AsyncTaskProcessor.Task import org.broadinstitute.dsde.workbench.leonardo.CommonTestData._ @@ -97,6 +100,24 @@ class LeoPubsubMessageSubscriberSpec ): Future[Unit] = Future.successful(()) } val iamDAO = new MockGoogleIamDAO + + // Returns a GCE instance with external IP "1.2.3.4" so Galaxy VM IP polling succeeds in tests. + val galaxyComputeService: GoogleComputeService[IO] = new FakeGoogleComputeService { + override def getInstance(project: GoogleProject, zone: ZoneName, instanceName: InstanceName)(implicit + ev: Ask[IO, TraceId] + ): IO[Option[Instance]] = { + val inst = Instance + .newBuilder() + .addNetworkInterfaces( + NetworkInterface + .newBuilder() + .addAccessConfigs(AccessConfig.newBuilder().setNatIP("1.2.3.4").build()) + .build() + ) + .build() + IO.pure(Some(inst)) + } + } val resourceService = new FakeGoogleResourceService { override def getProjectNumber(project: GoogleProject)(implicit ev: Ask[IO, TraceId]): IO[Option[Long]] = IO(Some(1L)) @@ -903,32 +924,29 @@ class LeoPubsubMessageSubscriberSpec getDisk = getDiskOpt.get appRestore <- persistentDiskQuery.getAppDiskRestore(savedApp1.appResources.disk.get.id).transaction galaxyRestore = appRestore.map(_.asInstanceOf[GalaxyRestore]) - ipRange = Config.vpcConfig.subnetworkRegionIpRangeMap - .getOrElse(RegionName("us-central1"), throw new Exception(s"Unsupported Region us-central1")) } yield { getCluster.status shouldBe KubernetesClusterStatus.Running getCluster.nodepools.size shouldBe 2 - getCluster.nodepools.filter(_.isDefault).head.status shouldBe NodepoolStatus.Running + // Galaxy VM path does not create/poll GKE nodepools — their status stays Unspecified + getCluster.nodepools.filter(_.isDefault).head.status shouldBe NodepoolStatus.Unspecified getApp.app.errors shouldBe List.empty getApp.app.status shouldBe AppStatus.Running getApp.app.appResources.kubernetesServiceAccountName shouldBe Some( ServiceAccountName("gxy-ksa") ) getApp.cluster.status shouldBe KubernetesClusterStatus.Running - getApp.nodepool.status shouldBe NodepoolStatus.Running + // Galaxy VM path does not create/poll GKE nodepools — their status stays Unspecified + getApp.nodepool.status shouldBe NodepoolStatus.Unspecified + // Galaxy VM path stores external IP as loadBalancerIp; network fields are not populated getApp.cluster.asyncFields shouldBe Some( KubernetesClusterAsyncFields(IP("1.2.3.4"), - IP("0.0.0.0"), - NetworkFields(Config.vpcConfig.networkName, - Config.vpcConfig.subnetworkName, - ipRange - ) + IP(""), + NetworkFields(NetworkName(""), SubnetworkName(""), IpRange("")) ) ) getDisk.status shouldBe DiskStatus.Ready - galaxyRestore shouldBe Some( - GalaxyRestore(PvcId(s"nfs-pvc-id1"), getApp.app.id) - ) + // Galaxy VM path does not use PVCs — no GalaxyRestore is recorded + galaxyRestore shouldBe None } implicit val gkeAlg: GKEAlgebra[IO] = makeGKEInterp(nodepoolLock, List(savedApp1.release)) @@ -1062,25 +1080,22 @@ class LeoPubsubMessageSubscriberSpec .transaction getApp1 = getAppOpt1.get getApp2 = getAppOpt2.get - ipRange = Config.vpcConfig.subnetworkRegionIpRangeMap - .getOrElse(RegionName("us-central1"), throw new Exception(s"Unsupported Region us-central1")) } yield { getApp1.cluster.status shouldBe KubernetesClusterStatus.Running getApp2.cluster.status shouldBe KubernetesClusterStatus.Running - getApp1.nodepool.status shouldBe NodepoolStatus.Running - getApp2.nodepool.status shouldBe NodepoolStatus.Running + // Galaxy VM path does not create/poll GKE nodepools — their status stays Unspecified + getApp1.nodepool.status shouldBe NodepoolStatus.Unspecified + getApp2.nodepool.status shouldBe NodepoolStatus.Unspecified getApp1.app.errors shouldBe List() getApp1.app.status shouldBe AppStatus.Running getApp1.app.appResources.kubernetesServiceAccountName shouldBe Some( ServiceAccountName("gxy-ksa") ) + // Galaxy VM path stores external IP as loadBalancerIp; network fields are not populated getApp1.cluster.asyncFields shouldBe Some( KubernetesClusterAsyncFields(IP("1.2.3.4"), - IP("0.0.0.0"), - NetworkFields(Config.vpcConfig.networkName, - Config.vpcConfig.subnetworkName, - ipRange - ) + IP(""), + NetworkFields(NetworkName(""), SubnetworkName(""), IpRange("")) ) ) getApp2.app.errors shouldBe List() @@ -1298,7 +1313,9 @@ class LeoPubsubMessageSubscriberSpec it should "handle an error in delete app" in isolatedDbTest { val savedCluster1 = makeKubeCluster(1).save() val savedNodepool1 = makeNodepool(1, savedCluster1.id).save() - val savedApp1 = makeApp(1, savedNodepool1.id).save() + // Use Cromwell (GKE/Helm path) so the deleteNamespace error triggers AppStatus.Error. + // Galaxy now uses the VM path which swallows deleteInstance errors gracefully. + val savedApp1 = makeApp(1, savedNodepool1.id, appType = AppType.Cromwell).save() val mockAckConsumer = mock[AckHandler] val assertions = for { @@ -1676,7 +1693,9 @@ class LeoPubsubMessageSubscriberSpec savedApp1.appName, None, Map.empty, - AppType.Galaxy, + // Use Cromwell so the GKE cluster creation path is taken and mockGKEService.createCluster can throw. + // Galaxy skips cluster creation (VM path), so the mock would have no effect. + AppType.Cromwell, savedApp1.appResources.namespace, None, Some(tr), @@ -1821,26 +1840,24 @@ class LeoPubsubMessageSubscriberSpec getApp = getAppOpt.get getDiskOpt <- persistentDiskQuery.getById(savedApp1.appResources.disk.get.id).transaction getDisk = getDiskOpt.get - ipRange = Config.vpcConfig.subnetworkRegionIpRangeMap - .getOrElse(RegionName("us-central1"), throw new Exception(s"Unsupported Region us-central1")) } yield { getCluster.status shouldBe KubernetesClusterStatus.Running getCluster.nodepools.size shouldBe 2 - getCluster.nodepools.filter(_.isDefault).head.status shouldBe NodepoolStatus.Running + // Galaxy VM path does not create/poll GKE nodepools — their status stays Unspecified + getCluster.nodepools.filter(_.isDefault).head.status shouldBe NodepoolStatus.Unspecified getApp.app.errors shouldBe List() getApp.app.status shouldBe AppStatus.Running getApp.app.appResources.kubernetesServiceAccountName shouldBe Some( ServiceAccountName("gxy-ksa") ) getApp.cluster.status shouldBe KubernetesClusterStatus.Running - getApp.nodepool.status shouldBe NodepoolStatus.Running + // Galaxy VM path does not create/poll GKE nodepools — their status stays Unspecified + getApp.nodepool.status shouldBe NodepoolStatus.Unspecified + // Galaxy VM path stores external IP as loadBalancerIp; network fields are not populated getApp.cluster.asyncFields shouldBe Some( KubernetesClusterAsyncFields(IP("1.2.3.4"), - IP("0.0.0.0"), - NetworkFields(Config.vpcConfig.networkName, - Config.vpcConfig.subnetworkName, - ipRange - ) + IP(""), + NetworkFields(NetworkName(""), SubnetworkName(""), IpRange("")) ) ) getDisk.status shouldBe DiskStatus.Ready @@ -2024,7 +2041,7 @@ class LeoPubsubMessageSubscriberSpec MockAppDescriptorDAO, lock, resourceService, - FakeGoogleComputeService + galaxyComputeService ) def makeLeoSubscriber( diff --git a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/util/VPCInterpreterSpec.scala b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/util/VPCInterpreterSpec.scala index c6ad02ccc5..5b6a8b2190 100644 --- a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/util/VPCInterpreterSpec.scala +++ b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/util/VPCInterpreterSpec.scala @@ -87,7 +87,7 @@ class VPCInterpreterSpec extends AnyFlatSpecLike with LeonardoTestSuite { SetUpProjectFirewallsParams(project, vpcConfig.networkName, RegionName("us-central1"), Map.empty) ) .unsafeRunSync() - computeService.firewallMap.size shouldBe 4 + computeService.firewallMap.size shouldBe 5 vpcConfig.firewallsToAdd.foreach { fwConfig => val fw = computeService.firewallMap.get(FirewallRuleName(s"${fwConfig.namePrefix}-us-central1")) fw shouldBe defined @@ -155,6 +155,12 @@ class VPCInterpreterSpec extends AnyFlatSpecLike with LeonardoTestSuite { ) val test = new VPCInterpreter(Config.vpcInterpreterConfig, stubResourceService(Map.empty), computeService) + val expectedHttpFirewallRules = FirewallRuleConfig( + "leonardo-allow-http", + None, + allSupportedRegions.map(r => r -> List(IpRange("0.0.0.0/0"))).toMap, + List(Allowed("tcp", Some("80"))) + ) test .firewallRulesToAdd( Map( @@ -162,7 +168,7 @@ class VPCInterpreterSpec extends AnyFlatSpecLike with LeonardoTestSuite { "leonardo-allow-https-firewall-name" -> "leonardo-ssl" ) ) - .toSet shouldBe Set(expectedSshFirewallRules, expectedIapFirewallRules) + .toSet shouldBe Set(expectedHttpFirewallRules, expectedSshFirewallRules, expectedIapFirewallRules) } private def stubResourceService(labels: Map[String, String]): FakeGoogleResourceService = From 9a69ccc8bae956ccfcfca182cc6d9dfaf7b42a00 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Wed, 8 Apr 2026 10:30:46 -0400 Subject: [PATCH 05/55] fix the last two unit tests --- .../leonardo/util/GKEInterpreter.scala | 54 +++++++++++-------- .../LeoPubsubMessageSubscriberSpec.scala | 7 +-- 2 files changed, 36 insertions(+), 25 deletions(-) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala index b0f52276c1..75dadd0df1 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala @@ -401,34 +401,44 @@ class GKEInterpreter[F[_]]( app = dbApp.app dbCluster = dbApp.cluster googleProject = params.googleProject + // Idempotency: if the app is already Running, skip creation to avoid double-recording usage + _ <- + if (app.status == AppStatus.Running) + logger.info(ctx.loggingCtx)( + s"App ${app.appName.value} is already Running, skipping creation (idempotent)" + ) + else + for { + diskOpt <- appQuery.getDiskId(app.id).transaction + diskId <- F.fromOption(diskOpt, DiskNotFoundForAppException(app.id, ctx.traceId)) - diskOpt <- appQuery.getDiskId(app.id).transaction - diskId <- F.fromOption(diskOpt, DiskNotFoundForAppException(app.id, ctx.traceId)) - - _ <- logger.info(ctx.loggingCtx)(s"Begin App(${app.appName.value}) Creation.") + _ <- logger.info(ctx.loggingCtx)(s"Begin App(${app.appName.value}) Creation.") - nfsDisk <- F.fromOption( - dbApp.app.appResources.disk, - AppCreationException(s"NFS disk not found in DB for app ${app.appName.value} | trace id: ${ctx.traceId}") - ) + nfsDisk <- F.fromOption( + dbApp.app.appResources.disk, + AppCreationException( + s"NFS disk not found in DB for app ${app.appName.value} | trace id: ${ctx.traceId}" + ) + ) - // Galaxy uses a VM-based deployment; all other app types use the GKE/Helm path. - _ <- app.appType match { - case AppType.Galaxy => - installGalaxyVm(dbCluster, app, nfsDisk, googleProject) >> - persistentDiskQuery.updateLastUsedBy(diskId, app.id).transaction.void + // Galaxy uses a VM-based deployment; all other app types use the GKE/Helm path. + _ <- app.appType match { + case AppType.Galaxy => + installGalaxyVm(dbCluster, app, nfsDisk, googleProject) >> + persistentDiskQuery.updateLastUsedBy(diskId, app.id).transaction.void - case _ => - createAndPollAppViaHelm(params, dbApp, app, dbCluster, nfsDisk, diskId, googleProject, ctx) - } + case _ => + createAndPollAppViaHelm(params, dbApp, app, dbCluster, nfsDisk, diskId, googleProject, ctx) + } - _ <- logger.info(ctx.loggingCtx)( - s"Finished app creation for app ${app.appName.value}" - ) + _ <- logger.info(ctx.loggingCtx)( + s"Finished app creation for app ${app.appName.value}" + ) - readyTime <- F.realTimeInstant - _ <- appUsageQuery.recordStart(params.appId, readyTime) - _ <- appQuery.updateStatus(params.appId, AppStatus.Running).transaction + readyTime <- F.realTimeInstant + _ <- appUsageQuery.recordStart(params.appId, readyTime) + _ <- appQuery.updateStatus(params.appId, AppStatus.Running).transaction + } yield () } yield () // GKE/Helm path for non-Galaxy app types (Cromwell, Allowed, Custom). diff --git a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriberSpec.scala b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriberSpec.scala index d7f351488d..dd542cbd3a 100644 --- a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriberSpec.scala +++ b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriberSpec.scala @@ -1449,7 +1449,7 @@ class LeoPubsubMessageSubscriberSpec val savedNodepool1 = makeNodepool(1, savedCluster1.id).save() val disk = makePersistentDisk(None).save().unsafeRunSync()(cats.effect.unsafe.IORuntime.global) - val makeApp1 = makeApp(1, savedNodepool1.id) + val makeApp1 = makeApp(1, savedNodepool1.id, appType = AppType.Cromwell) val savedApp1 = makeApp1 .copy(appResources = makeApp1.appResources.copy( @@ -1540,7 +1540,7 @@ class LeoPubsubMessageSubscriberSpec savedApp1.appName, Some(disk.id), Map.empty, - AppType.Galaxy, + AppType.Cromwell, savedApp1.appResources.namespace, None, Some(tr), @@ -1889,7 +1889,8 @@ class LeoPubsubMessageSubscriberSpec false, Some(GcsBucketName("fc-bucket")) ) - asyncTaskProcessor = AsyncTaskProcessor(AsyncTaskProcessor.Config(10, 10), queue) + // maxConcurrentTasks=1 ensures tasks run sequentially so the idempotency check fires for the 2nd task + asyncTaskProcessor = AsyncTaskProcessor(AsyncTaskProcessor.Config(10, 1), queue) // send message twice _ <- leoSubscriber.handleCreateAppMessage(msg) _ <- leoSubscriber.handleCreateAppMessage(msg) From 049f46686b98330cfc8764dff22a4e21ae270971 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Fri, 10 Apr 2026 11:17:28 -0400 Subject: [PATCH 06/55] handle Batch SA creation and fix restore Galaxy --- http/src/main/resources/reference.conf | 1 - .../workbench/leonardo/config/Config.scala | 1 - .../leonardo/config/KubernetesAppConfig.scala | 1 - .../monitor/LeoPubsubMessageSubscriber.scala | 11 +- .../workbench/leonardo/util/GKEAlgebra.scala | 3 +- .../leonardo/util/GKEInterpreter.scala | 107 +++++++++++++----- 6 files changed, 89 insertions(+), 35 deletions(-) diff --git a/http/src/main/resources/reference.conf b/http/src/main/resources/reference.conf index 7cb3d0892b..369a2fbbc7 100644 --- a/http/src/main/resources/reference.conf +++ b/http/src/main/resources/reference.conf @@ -442,7 +442,6 @@ galaxyVm { # Suffix appended to the NFS disk name to derive the postgres disk name. # Must match the value used in LeoPubsubMessageSubscriber (galaxyDisk.postgresDiskNameSuffix). postgresDiskNameSuffix = ${gke.galaxyDisk.postgresDiskNameSuffix} - gcpBatchServiceAccountEmail = "" gitRepo = "https://github.com/galaxyproject/galaxy-k8s-boot.git" gitBranch = "master" } diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/Config.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/Config.scala index 42c7a53737..08eb73020d 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/Config.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/Config.scala @@ -138,7 +138,6 @@ object Config { config.as[DiskSize]("bootDiskSizeGb"), config.as[DiskSize]("postgresDiskSizeGb"), config.as[String]("postgresDiskNameSuffix"), - config.as[String]("gcpBatchServiceAccountEmail"), config.as[String]("gitRepo"), config.as[String]("gitBranch") ) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/KubernetesAppConfig.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/KubernetesAppConfig.scala index 53e2562db6..bd6712279f 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/KubernetesAppConfig.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/KubernetesAppConfig.scala @@ -100,7 +100,6 @@ final case class GalaxyVmConfig( bootDiskSizeGb: DiskSize, postgresDiskSizeGb: DiskSize, postgresDiskNameSuffix: String, - gcpBatchServiceAccountEmail: String, gitRepo: String, gitBranch: String ) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriber.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriber.scala index 97a6c984bd..fb71a5603d 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriber.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriber.scala @@ -980,9 +980,14 @@ class LeoPubsubMessageSubscriber[F[_]]( ) .void - // create second Galaxy disk asynchronously + // Restore mode: disk already exists (no new disk to create) for a Galaxy app. + // In this case we skip creating new disks and pass restore=true so the VM + // runs the Ansible restore playbook instead of a fresh install. + restore = msg.appType == AppType.Galaxy && msg.createDisk.isEmpty + + // create second Galaxy disk asynchronously (only for fresh installs) createSecondDiskOp = - if (msg.appType == AppType.Galaxy && disk.isDefined) { + if (msg.appType == AppType.Galaxy && disk.isDefined && !restore) { val d = disk.get // it's safe to do `.get` here because we've verified for { res <- createGalaxyPostgresDiskOnlyInGoogle(msg.project, ZoneName("us-central1-a"), msg.appName, d.name) @@ -1012,7 +1017,7 @@ class LeoPubsubMessageSubscriber[F[_]]( // create and monitor app _ <- getGkeAlgFromRegistry() .createAndPollApp( - CreateAppParams(msg.appId, msg.project, msg.appName, msg.machineType, msg.bucketNameToMount) + CreateAppParams(msg.appId, msg.project, msg.appName, msg.machineType, msg.bucketNameToMount, restore) ) .onError { case e => cleanUpAfterCreateAppError(msg.appId, msg.appName, msg.project, msg.createDisk, e) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEAlgebra.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEAlgebra.scala index 22108113e1..9a81c53064 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEAlgebra.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEAlgebra.scala @@ -125,7 +125,8 @@ final case class CreateAppParams(appId: AppId, googleProject: GoogleProject, appName: AppName, appMachineType: Option[AppMachineType], - bucketNameToMount: Option[GcsBucketName] + bucketNameToMount: Option[GcsBucketName], + restore: Boolean = false ) final case class DeleteClusterParams(clusterId: KubernetesClusterLeoId, googleProject: GoogleProject) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala index 75dadd0df1..4bb7017ccd 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala @@ -8,8 +8,10 @@ import cats.syntax.all._ import com.google.auth.oauth2.GoogleCredentials import com.google.cloud.compute.v1.{ AccessConfig, + Allowed, AttachedDisk, AttachedDiskInitializeParams, + Firewall, Instance, Items, Metadata, @@ -37,6 +39,7 @@ import org.broadinstitute.dsde.workbench.google2.{ streamFUntilDone, streamUntilDoneOrTimeout, tracedRetryF, + FirewallRuleName, GoogleComputeService, GoogleDiskService, GoogleResourceService, @@ -59,7 +62,12 @@ import org.broadinstitute.dsde.workbench.leonardo.util.BuildHelmChartValues.{ } import org.broadinstitute.dsde.workbench.leonardo.model.LeoException import org.broadinstitute.dsde.workbench.leonardo.util.GKEAlgebra._ -import org.broadinstitute.dsde.workbench.model.google.{GcsBucketName, GoogleProject} +import org.broadinstitute.dsde.workbench.model.google.{ + GcsBucketName, + GoogleProject, + ServiceAccountDisplayName, + ServiceAccountName +} import org.broadinstitute.dsde.workbench.model.google.iam.IamMemberTypes import org.broadinstitute.dsde.workbench.model.{IP, TraceId, WorkbenchEmail} import org.broadinstitute.dsde.workbench.openTelemetry.OpenTelemetryMetrics @@ -424,7 +432,7 @@ class GKEInterpreter[F[_]]( // Galaxy uses a VM-based deployment; all other app types use the GKE/Helm path. _ <- app.appType match { case AppType.Galaxy => - installGalaxyVm(dbCluster, app, nfsDisk, googleProject) >> + installGalaxyVm(dbCluster, app, nfsDisk, googleProject, params.restore) >> persistentDiskQuery.updateLastUsedBy(diskId, app.id).transaction.void case _ => @@ -1038,7 +1046,8 @@ class GKEInterpreter[F[_]]( dbCluster: KubernetesCluster, app: App, nfsDisk: PersistentDisk, - googleProject: GoogleProject + googleProject: GoogleProject, + restore: Boolean )(implicit ev: Ask[F, AppContext]): F[Unit] = for { ctx <- ev.ask @@ -1073,16 +1082,23 @@ class GKEInterpreter[F[_]]( pvSizeGi = math.max(1, nfsDisk.size.gb - 11) pvSize = s"${pvSizeGi}Gi" - // GCP Batch SA: prefer value from customEnvironmentVariables, fall back to config default - gcpBatchSa = app.customEnvironmentVariables.getOrElse( - "gcp_batch_service_account_email", - config.galaxyVmConfig.gcpBatchServiceAccountEmail - ) - - // restore_galaxy flag - restoreGalaxy = app.customEnvironmentVariables.getOrElse("restore_galaxy", "false") + // Get or create the galaxy-batch-runner SA in the user's project. + gcpBatchSa <- F + .fromFuture( + F.delay( + googleIamDAO.getOrCreateServiceAccount( + googleProject, + ServiceAccountName("galaxy-batch-runner"), + ServiceAccountDisplayName("Galaxy Batch Runner"), + executionContext + ) + ) + ) + .map(sa => sa.email.value) - // Disks + // Disks — data and postgres disks are always pre-existing by the time this method runs + // (created by createDiskOp / createSecondDiskOp, or retained from a previous app). + // Use setSource to attach existing disks; only the boot disk is created fresh. bootDisk = AttachedDisk .newBuilder() .setBoot(true) @@ -1103,14 +1119,8 @@ class GKEInterpreter[F[_]]( .setBoot(false) .setDeviceName("galaxy-data") .setAutoDelete(false) - .setInitializeParams( - AttachedDiskInitializeParams - .newBuilder() - .setDiskName(nfsDisk.name.value) - .setDiskSizeGb(nfsDisk.size.gb) - .setDiskType(nfsDisk.diskType.googleString(googleProject, zoneParam)) - .putAllLabels(Map("leonardo" -> "true").asJava) - .build() + .setSource( + s"projects/${googleProject.value}/zones/${zoneParam.value}/disks/${nfsDisk.name.value}" ) .build() @@ -1120,13 +1130,8 @@ class GKEInterpreter[F[_]]( .setBoot(false) .setDeviceName("galaxy-postgres-data") .setAutoDelete(false) - .setInitializeParams( - AttachedDiskInitializeParams - .newBuilder() - .setDiskName(postgresDiskName.value) - .setDiskSizeGb(config.galaxyVmConfig.postgresDiskSizeGb.gb) - .putAllLabels(Map("leonardo" -> "true").asJava) - .build() + .setSource( + s"projects/${googleProject.value}/zones/${zoneParam.value}/disks/${postgresDiskName.value}" ) .build() @@ -1168,7 +1173,7 @@ class GKEInterpreter[F[_]]( .addItems(Items.newBuilder().setKey("google-logging-enabled").setValue("true").build()) .addItems(Items.newBuilder().setKey("gcp_batch_service_account_email").setValue(gcpBatchSa).build()) .addItems(Items.newBuilder().setKey("persistent-volume-size").setValue(pvSize).build()) - .addItems(Items.newBuilder().setKey("restore_galaxy").setValue(restoreGalaxy).build()) + .addItems(Items.newBuilder().setKey("restore_galaxy").setValue(restore.toString).build()) .addItems(Items.newBuilder().setKey("git-repo").setValue(config.galaxyVmConfig.gitRepo).build()) .addItems(Items.newBuilder().setKey("git-branch").setValue(config.galaxyVmConfig.gitBranch).build()) .addItems(Items.newBuilder().setKey("gcp-region").setValue(regionParam.value).build()) @@ -1226,6 +1231,52 @@ class GKEInterpreter[F[_]]( s"skipping serviceAccountUser binding — must be configured externally" ) + // Grant the Batch SA the project-level roles it needs to run jobs and attach a service account to Batch VMs. + // See https://github.com/galaxyproject/galaxy-k8s-boot?tab=readme-ov-file#prerequisites + _ <- { + val call = F.fromFuture( + F.delay( + googleIamDAO + .addRoles(googleProject, + WorkbenchEmail(gcpBatchSa), + IamMemberTypes.ServiceAccount, + Set("roles/batch.jobsEditor", "roles/iam.serviceAccountUser") + ) + .void + ) + ) + val retryConfig = RetryPredicates.retryConfigWithPredicates(when409) + tracedRetryF(retryConfig)( + call, + s"googleIamDAO.addRoles(batch.jobsEditor, iam.serviceAccountUser) for Batch SA $gcpBatchSa in project ${googleProject.value}" + ).compile.lastOrError + } + + // Create an NFS firewall rule so GCP Batch VMs can reach the Galaxy VM's NFS server. + // Idempotent: skipped if the rule already exists. + nfsFwName = FirewallRuleName("leonardo-galaxy-allow-nfs-for-batch") + nfsFwExists <- computeService.getFirewallRule(googleProject, nfsFwName) + _ <- + if (nfsFwExists.isEmpty) { + val nfsFirewall = Firewall + .newBuilder() + .setName(nfsFwName.value) + .setNetwork(s"projects/${googleProject.value}/global/networks/${network.value}") + .addSourceRanges("10.0.0.0/8") + .addTargetTags(config.vpcNetworkTag.value) + .addAllowed(Allowed.newBuilder().setIPProtocol("tcp").addPorts("2049").build()) + .addAllowed(Allowed.newBuilder().setIPProtocol("udp").addPorts("2049").build()) + .addAllowed(Allowed.newBuilder().setIPProtocol("tcp").addPorts("111").build()) + .addAllowed(Allowed.newBuilder().setIPProtocol("udp").addPorts("111").build()) + .build() + computeService + .addFirewallRule(googleProject, nfsFirewall) + .flatMap(op => F.blocking(op.get()).void) + } else + logger.info(ctx.loggingCtx)( + s"NFS firewall rule ${nfsFwName.value} already exists, skipping creation" + ) + _ <- logger.info(ctx.loggingCtx)( s"Galaxy VM instance ${instanceName.value} submitted for project ${googleProject.value}; polling for external IP" ) From 606e510a108549f605b189f829a0e3502a3a3916 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Fri, 10 Apr 2026 11:42:46 -0400 Subject: [PATCH 07/55] fix compiler issues --- .../workbench/leonardo/util/GKEInterpreter.scala | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala index 4bb7017ccd..719b57386b 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala @@ -62,12 +62,7 @@ import org.broadinstitute.dsde.workbench.leonardo.util.BuildHelmChartValues.{ } import org.broadinstitute.dsde.workbench.leonardo.model.LeoException import org.broadinstitute.dsde.workbench.leonardo.util.GKEAlgebra._ -import org.broadinstitute.dsde.workbench.model.google.{ - GcsBucketName, - GoogleProject, - ServiceAccountDisplayName, - ServiceAccountName -} +import org.broadinstitute.dsde.workbench.model.google.{GcsBucketName, GoogleProject, ServiceAccountDisplayName} import org.broadinstitute.dsde.workbench.model.google.iam.IamMemberTypes import org.broadinstitute.dsde.workbench.model.{IP, TraceId, WorkbenchEmail} import org.broadinstitute.dsde.workbench.openTelemetry.OpenTelemetryMetrics @@ -1088,9 +1083,8 @@ class GKEInterpreter[F[_]]( F.delay( googleIamDAO.getOrCreateServiceAccount( googleProject, - ServiceAccountName("galaxy-batch-runner"), - ServiceAccountDisplayName("Galaxy Batch Runner"), - executionContext + org.broadinstitute.dsde.workbench.model.google.ServiceAccountName("galaxy-batch-runner"), + ServiceAccountDisplayName("Galaxy Batch Runner") ) ) ) From 339ca5d0a8e0f5186feffdab5c016a85ea67c21a Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Mon, 13 Apr 2026 08:55:22 -0400 Subject: [PATCH 08/55] Fix Leo proxy for Galaxy VM: use HTTP on port 80 via VM internal IP --- .../workbench/leonardo/dao/HttpAppDAO.scala | 2 +- .../workbench/leonardo/dao/ProxyDAO.scala | 5 +- .../leonardo/dns/KubernetesDnsCache.scala | 8 +- .../leonardo/http/service/ProxyService.scala | 86 +++++++++++++------ .../leonardo/util/GKEInterpreter.scala | 31 ++++--- .../LeoPubsubMessageSubscriberSpec.scala | 15 ++-- 6 files changed, 97 insertions(+), 50 deletions(-) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpAppDAO.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpAppDAO.scala index 84a04b45da..e846792f11 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpAppDAO.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpAppDAO.scala @@ -23,7 +23,7 @@ class HttpAppDAO[F[_]: Async](kubernetesDnsCache: KubernetesDnsCache[F], client: traceId: TraceId ): F[Boolean] = Proxy.getAppTargetHost[F](kubernetesDnsCache, CloudContext.Gcp(googleProject), appName) flatMap { - case HostReady(targetHost, _, _) => + case HostReady(targetHost, _, _, _) => val serviceUrl = serviceName match { case ServiceName("welder-service") => s"https://${targetHost.address}/proxy/google/v1/apps/${googleProject.value}/${appName.value}/${serviceName.value}/status/" diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/ProxyDAO.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/ProxyDAO.scala index 6f6251a07d..ee1b07ddb2 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/ProxyDAO.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/ProxyDAO.scala @@ -11,7 +11,10 @@ object HostStatus { final case object HostNotFound extends HostStatus final case object HostNotReady extends HostStatus final case object HostPaused extends HostStatus - final case class HostReady(hostname: Host, path: String, cloudProvider: CloudProvider) extends HostStatus { + // useHttp = true means the proxy connects to the backend via plain HTTP (port 80) instead of + // HTTPS (proxyConfig.proxyPort). Used for Galaxy VM apps whose nginx serves HTTP only. + final case class HostReady(hostname: Host, path: String, cloudProvider: CloudProvider, useHttp: Boolean = false) + extends HostStatus { def toUri: Uri = Uri.unsafeFromString(s"https://${hostname.address()}/proxy/${path}") def toNotebooksUri: Uri = Uri.unsafeFromString(s"https://${hostname.address()}/notebooks/${path}") diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dns/KubernetesDnsCache.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dns/KubernetesDnsCache.scala index 6f7ff4783c..4d286c7f4b 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dns/KubernetesDnsCache.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dns/KubernetesDnsCache.scala @@ -7,6 +7,7 @@ import org.broadinstitute.dsde.workbench.leonardo.dao.HostStatus import org.broadinstitute.dsde.workbench.leonardo.dao.HostStatus.{HostNotFound, HostNotReady, HostReady} import org.broadinstitute.dsde.workbench.leonardo.db.{DbReference, KubernetesServiceDbQueries} import org.broadinstitute.dsde.workbench.leonardo.http.{kubernetesProxyHost, GetAppResult} +import org.broadinstitute.dsde.workbench.leonardo.AppType.Galaxy import org.broadinstitute.dsde.workbench.leonardo.{AppName, CloudContext, CloudProvider} import org.broadinstitute.dsde.workbench.model.IP import org.broadinstitute.dsde.workbench.openTelemetry.OpenTelemetryMetrics @@ -48,8 +49,13 @@ final class KubernetesDnsCache[F[_]: Logger: OpenTelemetryMetrics]( case None => F.pure[HostStatus](HostNotReady) case Some(ip) => val h = kubernetesProxyHost(appResult.cluster, proxyConfig.proxyDomain) + // Galaxy VM apps serve HTTP on port 80. The proxy should connect via plain HTTP, + // and we map the fake hostname to the VM's internal IP (stored in loadBalancerIp). + val isGalaxyVm = appResult.app.appType == Galaxy hostToIpMapping .getAndUpdate(_ + (h.address -> ip)) - .as[HostStatus](HostReady(h, "", CloudProvider.Gcp)) // TODO: update this once we start support AKS + .as[HostStatus]( + HostReady(h, "", CloudProvider.Gcp, useHttp = isGalaxyVm) + ) // TODO: update this once we start support AKS } } diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/ProxyService.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/ProxyService.scala index 5d989946bd..0ba9da08e0 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/ProxyService.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/ProxyService.scala @@ -256,7 +256,7 @@ class ProxyService( hostStatus <- getRuntimeTargetHost(cloudContext, runtimeName) _ <- hostStatus match { - case HostReady(_, _, _) => + case HostReady(_, _, _, _) => dateAccessUpdaterQueue.offer( UpdateDateAccessedMessage(UpdateTarget.Runtime(runtimeName), cloudContext, ctx.now) ) @@ -350,7 +350,7 @@ class ProxyService( } else IO.unit hostStatus <- getAppTargetHost(cloudContext, appName) _ <- hostStatus match { - case HostReady(_, _, _) => + case HostReady(_, _, _, _) => dateAccessUpdaterQueue.offer(UpdateDateAccessedMessage(UpdateTarget.App(appName), cloudContext, ctx.now)) case _ => IO.unit } @@ -376,16 +376,16 @@ class ProxyService( for { ctx <- ev.ask[AppContext] res <- hostContext.status match { - case HostReady(targetHost, _, _) => + case HostReady(targetHost, _, _, useHttp) => // If this is a WebSocket request (e.g. wss://leo:8080/...) then akka-http injects a // virtual UpgradeToWebSocket header which contains facilities to handle the WebSocket data. // The presence of this header distinguishes WebSocket from http requests. val res = for { response <- request.attribute(AttributeKeys.webSocketUpgrade) match { case Some(upgrade) => - IO.fromFuture(IO(handleWebSocketRequest(targetHost, request, upgrade))) + IO.fromFuture(IO(handleWebSocketRequest(targetHost, request, upgrade, useHttp))) case None => - IO.fromFuture(IO(handleHttpRequest(targetHost, request))) + IO.fromFuture(IO(handleHttpRequest(targetHost, request, useHttp))) } r <- if (response.status.isFailure()) @@ -418,9 +418,10 @@ class ProxyService( } } yield res - private def handleHttpRequest(targetHost: Host, request: HttpRequest): Future[HttpResponse] = { - logger.debug(s"Opening https connection to ${targetHost.address}:${proxyConfig.proxyPort}") - + private def handleHttpRequest(targetHost: Host, + request: HttpRequest, + useHttp: Boolean = false + ): Future[HttpResponse] = { // A note on akka-http philosophy: // The Akka HTTP server is implemented on top of Streams and makes heavy use of it. Requests come // in as a Source[HttpRequest] and responses are returned as a Sink[HttpResponse]. The transformation @@ -429,12 +430,24 @@ class ProxyService( // Initializes a Flow representing a prospective connection to the given endpoint. The connection // is not made until a Source and Sink are plugged into the Flow (i.e. it is materialized). - val flow = Http() - .connectionTo(targetHost.address) - .toPort(proxyConfig.proxyPort) - .withCustomHttpsConnectionContext(httpsConnectionContext) - .withClientConnectionSettings(clientConnectionSettings) - .https() + // Galaxy VM apps use plain HTTP on port 80; all other backends use HTTPS on proxyConfig.proxyPort. + val flow = + if (useHttp) { + logger.debug(s"Opening http connection to ${targetHost.address}:80") + Http() + .connectionTo(targetHost.address) + .toPort(80) + .withClientConnectionSettings(clientConnectionSettings) + .http() + } else { + logger.debug(s"Opening https connection to ${targetHost.address}:${proxyConfig.proxyPort}") + Http() + .connectionTo(targetHost.address) + .toPort(proxyConfig.proxyPort) + .withCustomHttpsConnectionContext(httpsConnectionContext) + .withClientConnectionSettings(clientConnectionSettings) + .https() + } // Now build a Source[Request] out of the original HttpRequest. We need to make some modifications // to the original request in order for the proxy to work: @@ -492,7 +505,8 @@ class ProxyService( private def handleWebSocketRequest(targetHost: Host, request: HttpRequest, - upgrade: WebSocketUpgrade + upgrade: WebSocketUpgrade, + useHttp: Boolean = false ): Future[HttpResponse] = { logger.info(s"Opening websocket connection to ${targetHost.address}") @@ -509,19 +523,35 @@ class ProxyService( // Make a single WebSocketRequest to the notebook server, passing in our Flow. This returns a Future[WebSocketUpgradeResponse]. // Keep our publisher/subscriber (e.g. sink/source) for use later. These are returned because we specified Keep.both above. // Note that we are rewriting the paths for any requests that are routed to /proxy/*/*/jupyter/ - val (responseFuture, (publisher, subscriber)) = Http().singleWebSocketRequest( - WebSocketRequest( - request.uri.copy(path = rewriteJupyterPath(request.uri.path), - authority = request.uri.authority.copy(host = targetHost, port = proxyConfig.proxyPort), - scheme = "wss" - ), - extraHeaders = filterHeaders(request.headers), - upgrade.requestedProtocols.headOption - ), - flow, - httpsConnectionContext, - settings = clientConnectionSettings - ) + // Galaxy VM apps use ws:// on port 80; all other backends use wss:// on proxyConfig.proxyPort. + val (responseFuture, (publisher, subscriber)) = + if (useHttp) + Http().singleWebSocketRequest( + WebSocketRequest( + request.uri.copy(path = rewriteJupyterPath(request.uri.path), + authority = request.uri.authority.copy(host = targetHost, port = 80), + scheme = "ws" + ), + extraHeaders = filterHeaders(request.headers), + upgrade.requestedProtocols.headOption + ), + flow, + settings = clientConnectionSettings + ) + else + Http().singleWebSocketRequest( + WebSocketRequest( + request.uri.copy(path = rewriteJupyterPath(request.uri.path), + authority = request.uri.authority.copy(host = targetHost, port = proxyConfig.proxyPort), + scheme = "wss" + ), + extraHeaders = filterHeaders(request.headers), + upgrade.requestedProtocols.headOption + ), + flow, + httpsConnectionContext, + settings = clientConnectionSettings + ) // If we got a valid WebSocketUpgradeResponse, call handleMessages with our publisher/subscriber, which are // already materialized from the HttpRequest. diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala index 719b57386b..0e9992a673 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala @@ -1275,41 +1275,45 @@ class GKEInterpreter[F[_]]( s"Galaxy VM instance ${instanceName.value} submitted for project ${googleProject.value}; polling for external IP" ) - // Poll until the instance has an external IP, then store it as the cluster's load balancer IP - // so that KubernetesDnsCache can resolve the proxy host. - externalIpOpt <- streamFUntilDone( + // Poll until the instance has both internal and external IPs assigned. + // We store the internal IP as the proxy backend (KubernetesDnsCache loadBalancerIp) so that + // the Leo proxy connects to the VM over the internal VPC network using plain HTTP. + // The external IP is only used for the readiness health check (TCP to port 80). + ipPairOpt <- streamFUntilDone( computeService.getInstance(googleProject, zoneParam, instanceName).map { instanceOpt => instanceOpt.flatMap { inst => import scala.jdk.CollectionConverters._ for { iface <- Option(inst.getNetworkInterfacesList).flatMap(_.asScala.headOption) + internalIp = IP(iface.getNetworkIP) cfg <- Option(iface.getAccessConfigsList).flatMap(_.asScala.headOption) natIp <- Option(cfg.getNatIP).filter(_.nonEmpty) - } yield IP(natIp) + } yield (internalIp, IP(natIp)) } }, config.monitorConfig.createApp.maxAttempts, config.monitorConfig.createApp.interval ).compile.lastOrError - externalIp <- F.fromOption( - externalIpOpt, + (internalIp, externalIp) <- F.fromOption( + ipPairOpt, AppCreationException( - s"Galaxy VM ${instanceName.value} did not obtain an external IP after ${config.monitorConfig.createApp.interruptAfter}", + s"Galaxy VM ${instanceName.value} did not obtain an IP after ${config.monitorConfig.createApp.interruptAfter}", traceId = Some(ctx.traceId) ) ) _ <- logger.info(ctx.loggingCtx)( - s"Galaxy VM ${instanceName.value} has external IP ${externalIp.asString}; storing in cluster async fields" + s"Galaxy VM ${instanceName.value} has internal IP ${internalIp.asString} / external IP ${externalIp.asString}; storing internal IP in cluster async fields" ) - // Store the VM's external IP as the cluster load balancer IP consumed by KubernetesDnsCache + // Store the VM's internal IP as the cluster load balancer IP consumed by KubernetesDnsCache. + // The proxy will connect to this IP via HTTP on port 80 (Galaxy VM serves HTTP, not HTTPS). _ <- kubernetesClusterQuery .updateAsyncFields( dbCluster.id, KubernetesClusterAsyncFields( - externalIp, + internalIp, IP(""), NetworkFields(NetworkName(""), SubnetworkName(""), IpRange("")) ) @@ -1318,10 +1322,13 @@ class GKEInterpreter[F[_]]( _ <- kubernetesClusterQuery.updateStatus(dbCluster.id, KubernetesClusterStatus.Running).transaction _ <- logger.info(ctx.loggingCtx)( - s"Polling Galaxy readiness for app ${app.appName.value} at ${externalIp.asString}:80" + s"Polling Galaxy readiness for app ${app.appName.value} via proxy (backend: ${internalIp.asString}:80)" ) - // Wait for Galaxy's nginx ingress to respond on port 80 + // Wait for Galaxy's nginx to respond. + // Uses isProxyAvailable which routes through the Leo proxy. The proxy now connects to the + // VM's internal IP on port 80 (plain HTTP) because KubernetesDnsCache sets useHttp=true for + // Galaxy apps and stores the internal IP as loadBalancerIp. isDone <- streamFUntilDone( appDao.isProxyAvailable(googleProject, app.appName, ServiceName("galaxy"), ctx.traceId), config.monitorConfig.createApp.maxAttempts, diff --git a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriberSpec.scala b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriberSpec.scala index dd542cbd3a..3b368a36a8 100644 --- a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriberSpec.scala +++ b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriberSpec.scala @@ -101,7 +101,7 @@ class LeoPubsubMessageSubscriberSpec } val iamDAO = new MockGoogleIamDAO - // Returns a GCE instance with external IP "1.2.3.4" so Galaxy VM IP polling succeeds in tests. + // Returns a GCE instance with internal IP "10.0.0.1" and external IP "1.2.3.4" so Galaxy VM IP polling succeeds in tests. val galaxyComputeService: GoogleComputeService[IO] = new FakeGoogleComputeService { override def getInstance(project: GoogleProject, zone: ZoneName, instanceName: InstanceName)(implicit ev: Ask[IO, TraceId] @@ -111,6 +111,7 @@ class LeoPubsubMessageSubscriberSpec .addNetworkInterfaces( NetworkInterface .newBuilder() + .setNetworkIP("10.0.0.1") .addAccessConfigs(AccessConfig.newBuilder().setNatIP("1.2.3.4").build()) .build() ) @@ -937,9 +938,9 @@ class LeoPubsubMessageSubscriberSpec getApp.cluster.status shouldBe KubernetesClusterStatus.Running // Galaxy VM path does not create/poll GKE nodepools — their status stays Unspecified getApp.nodepool.status shouldBe NodepoolStatus.Unspecified - // Galaxy VM path stores external IP as loadBalancerIp; network fields are not populated + // Galaxy VM path stores internal IP as loadBalancerIp (proxy routes to VM via HTTP on port 80); network fields are not populated getApp.cluster.asyncFields shouldBe Some( - KubernetesClusterAsyncFields(IP("1.2.3.4"), + KubernetesClusterAsyncFields(IP("10.0.0.1"), IP(""), NetworkFields(NetworkName(""), SubnetworkName(""), IpRange("")) ) @@ -1091,9 +1092,9 @@ class LeoPubsubMessageSubscriberSpec getApp1.app.appResources.kubernetesServiceAccountName shouldBe Some( ServiceAccountName("gxy-ksa") ) - // Galaxy VM path stores external IP as loadBalancerIp; network fields are not populated + // Galaxy VM path stores internal IP as loadBalancerIp (proxy routes to VM via HTTP on port 80); network fields are not populated getApp1.cluster.asyncFields shouldBe Some( - KubernetesClusterAsyncFields(IP("1.2.3.4"), + KubernetesClusterAsyncFields(IP("10.0.0.1"), IP(""), NetworkFields(NetworkName(""), SubnetworkName(""), IpRange("")) ) @@ -1853,9 +1854,9 @@ class LeoPubsubMessageSubscriberSpec getApp.cluster.status shouldBe KubernetesClusterStatus.Running // Galaxy VM path does not create/poll GKE nodepools — their status stays Unspecified getApp.nodepool.status shouldBe NodepoolStatus.Unspecified - // Galaxy VM path stores external IP as loadBalancerIp; network fields are not populated + // Galaxy VM path stores internal IP as loadBalancerIp (proxy routes to VM via HTTP on port 80); network fields are not populated getApp.cluster.asyncFields shouldBe Some( - KubernetesClusterAsyncFields(IP("1.2.3.4"), + KubernetesClusterAsyncFields(IP("10.0.0.1"), IP(""), NetworkFields(NetworkName(""), SubnetworkName(""), IpRange("")) ) From cb1187837c29096e0aad29ccf5b4e67509f2a540 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Mon, 13 Apr 2026 09:39:49 -0400 Subject: [PATCH 09/55] Fix Galaxy VM readiness check: use direct HTTP to VM internal IP --- .../dsde/workbench/leonardo/dao/HttpAppDAO.scala | 16 ++++++++++++++++ .../workbench/leonardo/util/GKEInterpreter.scala | 8 ++++---- .../dsde/workbench/leonardo/dao/MockAppDAO.scala | 5 ++++- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpAppDAO.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpAppDAO.scala index e846792f11..7fc78868ef 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpAppDAO.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpAppDAO.scala @@ -44,6 +44,20 @@ class HttpAppDAO[F[_]: Async](kubernetesDnsCache: KubernetesDnsCache[F], client: ) case _ => Async[F].pure(false) // Update once we support Relay for apps } + + def isVmReachable(ip: org.broadinstitute.dsde.workbench.model.IP, port: Int, traceId: TraceId): F[Boolean] = + client + .status( + Request[F]( + method = Method.GET, + uri = Uri.unsafeFromString(s"http://${ip.asString}:${port}/"), + headers = Headers(Header.Raw(CIString("X-Request-ID"), traceId.asString)) + ) + ) + .map(status => status.code < 500) + .handleErrorWith(t => + logger.error(Map("traceId" -> traceId.asString), t)("Fail to check if VM is reachable").as(false) + ) } trait AppDAO[F[_]] { @@ -52,4 +66,6 @@ trait AppDAO[F[_]] { serviceName: ServiceName, traceId: TraceId ): F[Boolean] + + def isVmReachable(ip: org.broadinstitute.dsde.workbench.model.IP, port: Int, traceId: TraceId): F[Boolean] } diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala index 0e9992a673..40fc8c437e 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala @@ -1326,11 +1326,11 @@ class GKEInterpreter[F[_]]( ) // Wait for Galaxy's nginx to respond. - // Uses isProxyAvailable which routes through the Leo proxy. The proxy now connects to the - // VM's internal IP on port 80 (plain HTTP) because KubernetesDnsCache sets useHttp=true for - // Galaxy apps and stores the internal IP as loadBalancerIp. + // Uses a direct HTTP check to the VM's internal IP on port 80, bypassing the Leo proxy + // hostname chain (which would require the proxy wildcard DNS to be reachable from within + // the Leo pod — unreliable in BEE environments due to hairpin NAT). isDone <- streamFUntilDone( - appDao.isProxyAvailable(googleProject, app.appName, ServiceName("galaxy"), ctx.traceId), + appDao.isVmReachable(internalIp, 80, ctx.traceId), config.monitorConfig.createApp.maxAttempts, config.monitorConfig.createApp.interval ).interruptAfter(config.monitorConfig.createApp.interruptAfter).compile.lastOrError diff --git a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/dao/MockAppDAO.scala b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/dao/MockAppDAO.scala index 8ee7c2c2cb..647f3cc77f 100644 --- a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/dao/MockAppDAO.scala +++ b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/dao/MockAppDAO.scala @@ -3,7 +3,7 @@ package org.broadinstitute.dsde.workbench.leonardo.dao import cats.effect.IO import org.broadinstitute.dsde.workbench.google2.KubernetesSerializableName.ServiceName import org.broadinstitute.dsde.workbench.leonardo.AppName -import org.broadinstitute.dsde.workbench.model.TraceId +import org.broadinstitute.dsde.workbench.model.{IP, TraceId} import org.broadinstitute.dsde.workbench.model.google.GoogleProject class MockAppDAO(isUp: Boolean = true) extends AppDAO[IO] { @@ -13,5 +13,8 @@ class MockAppDAO(isUp: Boolean = true) extends AppDAO[IO] { traceId: TraceId ): IO[Boolean] = IO.pure(isUp) + + override def isVmReachable(ip: IP, port: Int, traceId: TraceId): IO[Boolean] = + IO.pure(isUp) } object MockAppDAO extends MockAppDAO(isUp = true) From ce1b4e157254d4fcb826410008e0af77dc01f494 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Mon, 13 Apr 2026 12:46:58 -0400 Subject: [PATCH 10/55] Fix Galaxy VM proxy: use external IP instead of internal IP --- .../leonardo/dns/KubernetesDnsCache.scala | 3 ++- .../workbench/leonardo/util/GKEInterpreter.scala | 16 ++++++++++------ .../monitor/LeoPubsubMessageSubscriberSpec.scala | 12 ++++++------ 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dns/KubernetesDnsCache.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dns/KubernetesDnsCache.scala index 4d286c7f4b..a0bc7196fb 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dns/KubernetesDnsCache.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dns/KubernetesDnsCache.scala @@ -50,7 +50,8 @@ final class KubernetesDnsCache[F[_]: Logger: OpenTelemetryMetrics]( case Some(ip) => val h = kubernetesProxyHost(appResult.cluster, proxyConfig.proxyDomain) // Galaxy VM apps serve HTTP on port 80. The proxy should connect via plain HTTP, - // and we map the fake hostname to the VM's internal IP (stored in loadBalancerIp). + // and we map the fake hostname to the VM's external IP (stored in loadBalancerIp). + // External IP is used because Leo's pod is in a different VPC from the user's workspace project. val isGalaxyVm = appResult.app.appType == Galaxy hostToIpMapping .getAndUpdate(_ + (h.address -> ip)) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala index 40fc8c437e..322931d48b 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala @@ -1304,16 +1304,20 @@ class GKEInterpreter[F[_]]( ) _ <- logger.info(ctx.loggingCtx)( - s"Galaxy VM ${instanceName.value} has internal IP ${internalIp.asString} / external IP ${externalIp.asString}; storing internal IP in cluster async fields" + s"Galaxy VM ${instanceName.value} has internal IP ${internalIp.asString} / external IP ${externalIp.asString}; storing external IP in cluster async fields for proxy access" ) - // Store the VM's internal IP as the cluster load balancer IP consumed by KubernetesDnsCache. + // Store the VM's external IP as the cluster load balancer IP consumed by KubernetesDnsCache. // The proxy will connect to this IP via HTTP on port 80 (Galaxy VM serves HTTP, not HTTPS). + // We use the external IP because Leo's GKE cluster is in Leo's GCP project while the Galaxy VM + // is in the user's workspace project — the two VPCs are not peered, so the internal IP is + // not routable from Leo's pod. The leonardo-allow-http firewall rule (0.0.0.0/0 → port 80, + // targeting VMs with the "leonardo" tag) allows Leo to reach the VM on its external IP. _ <- kubernetesClusterQuery .updateAsyncFields( dbCluster.id, KubernetesClusterAsyncFields( - internalIp, + externalIp, IP(""), NetworkFields(NetworkName(""), SubnetworkName(""), IpRange("")) ) @@ -1322,15 +1326,15 @@ class GKEInterpreter[F[_]]( _ <- kubernetesClusterQuery.updateStatus(dbCluster.id, KubernetesClusterStatus.Running).transaction _ <- logger.info(ctx.loggingCtx)( - s"Polling Galaxy readiness for app ${app.appName.value} via proxy (backend: ${internalIp.asString}:80)" + s"Polling Galaxy readiness for app ${app.appName.value} via proxy (backend: ${externalIp.asString}:80)" ) // Wait for Galaxy's nginx to respond. - // Uses a direct HTTP check to the VM's internal IP on port 80, bypassing the Leo proxy + // Uses a direct HTTP check to the VM's external IP on port 80, bypassing the Leo proxy // hostname chain (which would require the proxy wildcard DNS to be reachable from within // the Leo pod — unreliable in BEE environments due to hairpin NAT). isDone <- streamFUntilDone( - appDao.isVmReachable(internalIp, 80, ctx.traceId), + appDao.isVmReachable(externalIp, 80, ctx.traceId), config.monitorConfig.createApp.maxAttempts, config.monitorConfig.createApp.interval ).interruptAfter(config.monitorConfig.createApp.interruptAfter).compile.lastOrError diff --git a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriberSpec.scala b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriberSpec.scala index 3b368a36a8..3da67b1856 100644 --- a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriberSpec.scala +++ b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriberSpec.scala @@ -938,9 +938,9 @@ class LeoPubsubMessageSubscriberSpec getApp.cluster.status shouldBe KubernetesClusterStatus.Running // Galaxy VM path does not create/poll GKE nodepools — their status stays Unspecified getApp.nodepool.status shouldBe NodepoolStatus.Unspecified - // Galaxy VM path stores internal IP as loadBalancerIp (proxy routes to VM via HTTP on port 80); network fields are not populated + // Galaxy VM path stores external IP as loadBalancerIp (proxy uses external IP because Leo VPC ≠ user VPC); network fields are not populated getApp.cluster.asyncFields shouldBe Some( - KubernetesClusterAsyncFields(IP("10.0.0.1"), + KubernetesClusterAsyncFields(IP("1.2.3.4"), IP(""), NetworkFields(NetworkName(""), SubnetworkName(""), IpRange("")) ) @@ -1092,9 +1092,9 @@ class LeoPubsubMessageSubscriberSpec getApp1.app.appResources.kubernetesServiceAccountName shouldBe Some( ServiceAccountName("gxy-ksa") ) - // Galaxy VM path stores internal IP as loadBalancerIp (proxy routes to VM via HTTP on port 80); network fields are not populated + // Galaxy VM path stores external IP as loadBalancerIp (proxy uses external IP because Leo VPC ≠ user VPC); network fields are not populated getApp1.cluster.asyncFields shouldBe Some( - KubernetesClusterAsyncFields(IP("10.0.0.1"), + KubernetesClusterAsyncFields(IP("1.2.3.4"), IP(""), NetworkFields(NetworkName(""), SubnetworkName(""), IpRange("")) ) @@ -1854,9 +1854,9 @@ class LeoPubsubMessageSubscriberSpec getApp.cluster.status shouldBe KubernetesClusterStatus.Running // Galaxy VM path does not create/poll GKE nodepools — their status stays Unspecified getApp.nodepool.status shouldBe NodepoolStatus.Unspecified - // Galaxy VM path stores internal IP as loadBalancerIp (proxy routes to VM via HTTP on port 80); network fields are not populated + // Galaxy VM path stores external IP as loadBalancerIp (proxy uses external IP because Leo VPC ≠ user VPC); network fields are not populated getApp.cluster.asyncFields shouldBe Some( - KubernetesClusterAsyncFields(IP("10.0.0.1"), + KubernetesClusterAsyncFields(IP("1.2.3.4"), IP(""), NetworkFields(NetworkName(""), SubnetworkName(""), IpRange("")) ) From 6fc11f35a63bde71e8d46cf508ec62b8e553275d Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Mon, 13 Apr 2026 16:20:20 -0400 Subject: [PATCH 11/55] Fix Galaxy VM bootstrap: use galaxy-k8s-boot image --- http/src/main/resources/init-resources/galaxy-user-data.sh | 2 +- http/src/main/resources/reference.conf | 6 ++++-- .../dsde/workbench/leonardo/util/GKEInterpreter.scala | 4 +++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/http/src/main/resources/init-resources/galaxy-user-data.sh b/http/src/main/resources/init-resources/galaxy-user-data.sh index 3b7fbb640b..fb3306e6c9 100644 --- a/http/src/main/resources/init-resources/galaxy-user-data.sh +++ b/http/src/main/resources/init-resources/galaxy-user-data.sh @@ -1,6 +1,6 @@ +#cloud-config # Sourced from https://github.com/galaxyproject/galaxy-k8s-boot/blob/dev/bin/user_data.sh # When updating this file, sync it manually from that repository and verify the changes. -#cloud-config write_files: - path: /usr/local/bin/galaxy_bootstrap.sh permissions: '0755' diff --git a/http/src/main/resources/reference.conf b/http/src/main/resources/reference.conf index 369a2fbbc7..b9574a08da 100644 --- a/http/src/main/resources/reference.conf +++ b/http/src/main/resources/reference.conf @@ -434,8 +434,10 @@ groups { } galaxyVm { - # Debian 12 image — galaxy-k8s-boot bootstrap requires a standard Debian VM, not COS - sourceImage = "projects/debian-cloud/global/images/family/debian-12" + # Pre-built galaxy-k8s-boot image with all dependencies (Ansible, RKE2, etc.) pre-installed. + # Has cloud-init, which processes the "user-data" metadata key on first boot. + # Source: https://github.com/galaxyproject/galaxy-k8s-boot (image built by the Galaxy team) + sourceImage = "projects/anvil-and-terra-development/global/images/galaxy-k8s-boot-v2026-02-25" machineType = "n1-highmem-8" bootDiskSizeGb = 50 postgresDiskSizeGb = 10 diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala index 322931d48b..73b8751fa5 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala @@ -1060,7 +1060,9 @@ class GKEInterpreter[F[_]]( ) // Load cloud-config content bundled from galaxy-k8s-boot bin/user_data.sh. - // Intentionally not fetched at runtime to avoid unexpected production changes. + // Passed as the "user-data" metadata key, processed by cloud-init on first boot only. + // The galaxy-k8s-boot custom image has cloud-init pre-installed; "#cloud-config" must be + // the first line for cloud-init to recognise the file format. // To update, sync manually from https://github.com/galaxyproject/galaxy-k8s-boot/blob/dev/bin/user_data.sh userDataContent = scala.io.Source .fromResource("init-resources/galaxy-user-data.sh") From 6149be932fef2968976bfe3023413884ec2c7d25 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Tue, 14 Apr 2026 10:15:25 -0400 Subject: [PATCH 12/55] increase boot disk size to Galaxy VM disk image reqs --- http/src/main/resources/reference.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/src/main/resources/reference.conf b/http/src/main/resources/reference.conf index b9574a08da..665d985473 100644 --- a/http/src/main/resources/reference.conf +++ b/http/src/main/resources/reference.conf @@ -439,7 +439,7 @@ galaxyVm { # Source: https://github.com/galaxyproject/galaxy-k8s-boot (image built by the Galaxy team) sourceImage = "projects/anvil-and-terra-development/global/images/galaxy-k8s-boot-v2026-02-25" machineType = "n1-highmem-8" - bootDiskSizeGb = 50 + bootDiskSizeGb = 100 postgresDiskSizeGb = 10 # Suffix appended to the NFS disk name to derive the postgres disk name. # Must match the value used in LeoPubsubMessageSubscriber (galaxyDisk.postgresDiskNameSuffix). From 30169a2e3fdc105f9f1e29c12d56d2bee1e37e8d Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Tue, 14 Apr 2026 16:34:14 -0400 Subject: [PATCH 13/55] mark cluster as deleted and wait for iam to propagate --- .../dsde/workbench/leonardo/util/GKEInterpreter.scala | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala index 73b8751fa5..8e110f2cb1 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala @@ -711,6 +711,10 @@ class GKEInterpreter[F[_]]( s"Failed to delete Galaxy VM ${instanceName.value}: ${e.getMessage}. Continuing with app deletion." ) } + // Mark the cluster DB record as deleted so future app creation in this project is not blocked. + // For Galaxy, the "cluster" is a pure DB abstraction (no real GKE cluster); it must be + // cleaned up here because no separate cluster-deletion pubsub message is sent. + _ <- kubernetesClusterQuery.markAsDeleted(dbCluster.id, ctx.now).transaction } yield () case _ => @@ -1196,7 +1200,7 @@ class GKEInterpreter[F[_]]( .void ) ) - val retryConfig = RetryPredicates.retryConfigWithPredicates(when409) + val retryConfig = RetryPredicates.retryConfigWithPredicates(when409, whenGroupDoesNotExist) tracedRetryF(retryConfig)( call, s"googleIamDAO.addRoles(batch.jobsEditor) for pet SA ${app.googleServiceAccount.value} in project ${googleProject.value}" @@ -1241,7 +1245,7 @@ class GKEInterpreter[F[_]]( .void ) ) - val retryConfig = RetryPredicates.retryConfigWithPredicates(when409) + val retryConfig = RetryPredicates.retryConfigWithPredicates(when409, whenGroupDoesNotExist) tracedRetryF(retryConfig)( call, s"googleIamDAO.addRoles(batch.jobsEditor, iam.serviceAccountUser) for Batch SA $gcpBatchSa in project ${googleProject.value}" From 2a992dd7cb436f593f7f9f0b21b2fd0520b24548 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Wed, 15 Apr 2026 09:59:20 -0400 Subject: [PATCH 14/55] fix Gb to Gib conversion --- .../dsde/workbench/leonardo/util/GKEInterpreter.scala | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala index 8e110f2cb1..c823e255ae 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala @@ -1079,8 +1079,13 @@ class GKEInterpreter[F[_]]( config.galaxyDiskConfig.postgresDiskNameSuffix ) - // Persistent-volume-size passed to ansible-pull (leave ~11 GiB for filesystem overhead on 150 GB disk) - pvSizeGi = math.max(1, nfsDisk.size.gb - 11) + // Persistent-volume-size passed to ansible-pull. + // nfsDisk.size.gb is in decimal GB; convert to binary GiB before subtracting filesystem overhead. + // Example: a 500 GB disk = (500 * 10^9) / 2^30 ≈ 465 GiB, so we request 465 - 11 = 454 GiB. + // Using raw gb - 11 (treating GB as GiB) overestimates by ~23 GiB on a 500 GB disk and + // causes the NFS provisioner to fail with "insufficient available space". + diskSizeGiB = (nfsDisk.size.gb.toLong * 1000L * 1000L * 1000L) / (1024L * 1024L * 1024L) + pvSizeGi = math.max(1, diskSizeGiB - 11) pvSize = s"${pvSizeGi}Gi" // Get or create the galaxy-batch-runner SA in the user's project. From 3298f65c74a61cb1ae0ce729a2afc1f175a948b1 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Wed, 15 Apr 2026 12:15:45 -0400 Subject: [PATCH 15/55] strip leo proxy prefix --- .../leonardo/http/service/ProxyService.scala | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/ProxyService.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/ProxyService.scala index 0ba9da08e0..841e75f7d0 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/ProxyService.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/ProxyService.scala @@ -355,7 +355,20 @@ class ProxyService( case _ => IO.unit } hostContext = HostContext(hostStatus, s"${cloudContext.asString}/${appName.value}/${serviceName.value}") - r <- proxyInternal(hostContext, request) + // Galaxy VM apps serve at /, but Leo forwards the full proxy path + // (e.g. /proxy/google/v1/apps/{project}/{app}/galaxy). Strip the Leo + // prefix so Galaxy's nginx sees requests rooted at /. + adjustedRequest = hostStatus match { + case HostReady(_, _, _, useHttp) if useHttp => + val prefix = s"/proxy/google/v1/apps/${cloudContext.asString}/${appName.value}/${serviceName.value}" + val stripped = request.uri.path.toString.stripPrefix(prefix) match { + case "" | "/" => "/" + case p => p + } + request.withUri(request.uri.withPath(Uri.Path(stripped))) + case _ => request + } + r <- proxyInternal(hostContext, adjustedRequest) appType <- appQuery.getAppType(appName).transaction result = if (r.status.isSuccess()) "success" else "failure" _ <- metrics.incrementCounter( From f5d6f65a3882343cfc4edfc1d6f261b4e2821359 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Wed, 15 Apr 2026 14:28:17 -0400 Subject: [PATCH 16/55] pass the galaxy_url_prefix --- .../resources/init-resources/galaxy-user-data.sh | 12 ++++++++++++ .../workbench/leonardo/util/GKEInterpreter.scala | 14 ++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/http/src/main/resources/init-resources/galaxy-user-data.sh b/http/src/main/resources/init-resources/galaxy-user-data.sh index fb3306e6c9..c21013f438 100644 --- a/http/src/main/resources/init-resources/galaxy-user-data.sh +++ b/http/src/main/resources/init-resources/galaxy-user-data.sh @@ -88,6 +88,13 @@ write_files: GCP_BATCH_SERVICE_ACCOUNT_EMAIL=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/gcp_batch_service_account_email" -H "Metadata-Flavor: Google" 2>/dev/null || echo "galaxy-batch-runner@anvil-and-terra-development.iam.gserviceaccount.com") echo "[$(date)] - GCP Batch service account email: ${GCP_BATCH_SERVICE_ACCOUNT_EMAIL}" + # Leo proxy path prefix for this Galaxy app (e.g. /proxy/google/v1/apps/{project}/{appName}/galaxy). + # Passed to ansible as galaxy_url_prefix so Galaxy generates correct absolute links (JS/CSS/API) + # that include the full proxy path. Without this Galaxy emits links rooted at / which the + # browser resolves against Leo's host and gets 404s → blank page. + GALAXY_URL_PREFIX=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/galaxy-url-prefix" -H "Metadata-Flavor: Google" 2>/dev/null || echo "") + echo "[$(date)] - Galaxy URL prefix: ${GALAXY_URL_PREFIX}" + GIT_REPO=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/git-repo" -H "Metadata-Flavor: Google" 2>/dev/null || echo "https://github.com/galaxyproject/galaxy-k8s-boot.git") GIT_BRANCH=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/git-branch" -H "Metadata-Flavor: Google" 2>/dev/null || echo "master") @@ -108,6 +115,11 @@ write_files: echo "[$(date)] - Galaxy Restore Mode: Disabled" fi + if [ -n "$GALAXY_URL_PREFIX" ]; then + PULL_ARGS+=(--extra-vars "galaxy_url_prefix=${GALAXY_URL_PREFIX}") + echo "[$(date)] - Galaxy URL prefix passed to ansible: ${GALAXY_URL_PREFIX}" + fi + PULL_ARGS+=(playbook.yml) mkdir -p /tmp/ansible-inventory diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala index c823e255ae..1901685b67 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala @@ -1184,6 +1184,20 @@ class GKEInterpreter[F[_]]( .addItems(Items.newBuilder().setKey("gcp-region").setValue(regionParam.value).build()) .addItems(Items.newBuilder().setKey("gcp-network").setValue(network.value).build()) .addItems(Items.newBuilder().setKey("gcp-subnet").setValue(subnetwork.value).build()) + // Galaxy needs to know its public URL prefix so it generates correct absolute links + // (JS, CSS, API calls) that include the full Leo proxy path. + // galaxy-k8s-boot's ansible playbook must accept galaxy_url_prefix and set it in + // Galaxy's helm values (galaxy.yml). Without this, Galaxy generates links rooted at / + // which the browser resolves against Leo's host and gets 404s → blank page. + .addItems( + Items + .newBuilder() + .setKey("galaxy-url-prefix") + .setValue( + s"/proxy/google/v1/apps/${googleProject.value}/${app.appName.value}/galaxy" + ) + .build() + ) .build() ) .putAllLabels(Map("leonardo" -> "true").asJava) From 09b21d03a6ed90b441bd3ba97ec4a79f45c47dec Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Fri, 8 May 2026 13:21:29 -0400 Subject: [PATCH 17/55] use new proxy path env variable --- .../init-resources/galaxy-user-data.sh | 2 +- .../leonardo/http/service/ProxyService.scala | 19 +++++-------------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/http/src/main/resources/init-resources/galaxy-user-data.sh b/http/src/main/resources/init-resources/galaxy-user-data.sh index c21013f438..1aa42cb6c8 100644 --- a/http/src/main/resources/init-resources/galaxy-user-data.sh +++ b/http/src/main/resources/init-resources/galaxy-user-data.sh @@ -116,7 +116,7 @@ write_files: fi if [ -n "$GALAXY_URL_PREFIX" ]; then - PULL_ARGS+=(--extra-vars "galaxy_url_prefix=${GALAXY_URL_PREFIX}") + PULL_ARGS+=(--extra-vars "galaxy_prefix=${GALAXY_URL_PREFIX}") echo "[$(date)] - Galaxy URL prefix passed to ansible: ${GALAXY_URL_PREFIX}" fi diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/ProxyService.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/ProxyService.scala index 841e75f7d0..5f407ad424 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/ProxyService.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/ProxyService.scala @@ -355,20 +355,11 @@ class ProxyService( case _ => IO.unit } hostContext = HostContext(hostStatus, s"${cloudContext.asString}/${appName.value}/${serviceName.value}") - // Galaxy VM apps serve at /, but Leo forwards the full proxy path - // (e.g. /proxy/google/v1/apps/{project}/{app}/galaxy). Strip the Leo - // prefix so Galaxy's nginx sees requests rooted at /. - adjustedRequest = hostStatus match { - case HostReady(_, _, _, useHttp) if useHttp => - val prefix = s"/proxy/google/v1/apps/${cloudContext.asString}/${appName.value}/${serviceName.value}" - val stripped = request.uri.path.toString.stripPrefix(prefix) match { - case "" | "/" => "/" - case p => p - } - request.withUri(request.uri.withPath(Uri.Path(stripped))) - case _ => request - } - r <- proxyInternal(hostContext, adjustedRequest) + // Galaxy VM apps: forward the full Leo proxy path to the VM unchanged. + // galaxy-k8s-boot configures nginx location blocks and galaxy_url_prefix + // using the ingress.path value (= the Leo proxy prefix), so the VM expects + // to receive the full path including the prefix. + r <- proxyInternal(hostContext, request) appType <- appQuery.getAppType(appName).transaction result = if (r.status.isSuccess()) "success" else "failure" _ <- metrics.incrementCounter( From 5e088d02c5d45148e688954a42dd02c92844b22c Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Fri, 12 Jun 2026 13:59:41 -0400 Subject: [PATCH 18/55] Address review findings: galaxy_user email, Source leak, anvil branch, gcpBatchSaProject parsing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Pass app.auditInfo.creator as galaxy-user-email GCE metadata so the actual workspace user (not dev@galaxyproject.org) becomes the Galaxy admin - Fix scala.io.Source resource leak in installGalaxyVm using scala.util.Using - Update sourceImage to galaxy-k8s-boot-v2026-06-10 and gitBranch to "anvil" - Fix HOST_IP to use GCE metadata server instead of external ifconfig.me - Fix gcpBatchSaProject SA email parsing: lift(1) + stripSuffix instead of lastOption + replace to avoid matching suffix in unexpected positions - Correct stale comments: galaxy_url_prefix → galaxy_prefix, dev → anvil branch, wrong "internal IP" comment corrected to "external IP" Co-Authored-By: Claude Sonnet 4.6 --- .../init-resources/galaxy-user-data.sh | 16 +++++----- http/src/main/resources/reference.conf | 4 +-- .../leonardo/util/GKEInterpreter.scala | 29 +++++++++---------- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/http/src/main/resources/init-resources/galaxy-user-data.sh b/http/src/main/resources/init-resources/galaxy-user-data.sh index 1aa42cb6c8..1a106b66b3 100644 --- a/http/src/main/resources/init-resources/galaxy-user-data.sh +++ b/http/src/main/resources/init-resources/galaxy-user-data.sh @@ -1,5 +1,5 @@ #cloud-config -# Sourced from https://github.com/galaxyproject/galaxy-k8s-boot/blob/dev/bin/user_data.sh +# Sourced from https://github.com/galaxyproject/galaxy-k8s-boot/blob/anvil/bin/user_data.sh # When updating this file, sync it manually from that repository and verify the changes. write_files: - path: /usr/local/bin/galaxy_bootstrap.sh @@ -73,7 +73,7 @@ write_files: # 3. Run ansible-pull sudo -u debian bash -c ' export HOME=/home/debian - HOST_IP=$(curl -s ifconfig.me) + HOST_IP=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip" -H "Metadata-Flavor: Google" 2>/dev/null || curl -s ifconfig.me) PV_SIZE=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/persistent-volume-size" -H "Metadata-Flavor: Google" 2>/dev/null) if [ -z "$PV_SIZE" ]; then @@ -89,14 +89,16 @@ write_files: echo "[$(date)] - GCP Batch service account email: ${GCP_BATCH_SERVICE_ACCOUNT_EMAIL}" # Leo proxy path prefix for this Galaxy app (e.g. /proxy/google/v1/apps/{project}/{appName}/galaxy). - # Passed to ansible as galaxy_url_prefix so Galaxy generates correct absolute links (JS/CSS/API) - # that include the full proxy path. Without this Galaxy emits links rooted at / which the - # browser resolves against Leo's host and gets 404s → blank page. + # Passed to ansible as galaxy_prefix so Galaxy's nginx ingress is configured at the correct subpath. + # Without this Galaxy generates links rooted at / which the browser resolves against Leo's host → 404s. GALAXY_URL_PREFIX=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/galaxy-url-prefix" -H "Metadata-Flavor: Google" 2>/dev/null || echo "") echo "[$(date)] - Galaxy URL prefix: ${GALAXY_URL_PREFIX}" + GALAXY_USER=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/galaxy-user-email" -H "Metadata-Flavor: Google" 2>/dev/null || echo "") + echo "[$(date)] - Galaxy user email: ${GALAXY_USER}" + GIT_REPO=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/git-repo" -H "Metadata-Flavor: Google" 2>/dev/null || echo "https://github.com/galaxyproject/galaxy-k8s-boot.git") - GIT_BRANCH=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/git-branch" -H "Metadata-Flavor: Google" 2>/dev/null || echo "master") + GIT_BRANCH=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/git-branch" -H "Metadata-Flavor: Google" 2>/dev/null || echo "anvil") PULL_ARGS=( -U "${GIT_REPO}" @@ -135,7 +137,7 @@ write_files: nfs_size="${PV_SIZE}" galaxy_persistence_size="${PV_SIZE}" galaxy_db_password="gxy-db-password" - galaxy_user="dev@galaxyproject.org" + galaxy_user="${GALAXY_USER}" EOF echo "[$(date)] - Inventory file created at /tmp/ansible-inventory/localhost; running ansible-pull..." diff --git a/http/src/main/resources/reference.conf b/http/src/main/resources/reference.conf index 665d985473..898674850b 100644 --- a/http/src/main/resources/reference.conf +++ b/http/src/main/resources/reference.conf @@ -437,7 +437,7 @@ galaxyVm { # Pre-built galaxy-k8s-boot image with all dependencies (Ansible, RKE2, etc.) pre-installed. # Has cloud-init, which processes the "user-data" metadata key on first boot. # Source: https://github.com/galaxyproject/galaxy-k8s-boot (image built by the Galaxy team) - sourceImage = "projects/anvil-and-terra-development/global/images/galaxy-k8s-boot-v2026-02-25" + sourceImage = "projects/anvil-and-terra-development/global/images/galaxy-k8s-boot-v2026-06-10" machineType = "n1-highmem-8" bootDiskSizeGb = 100 postgresDiskSizeGb = 10 @@ -445,7 +445,7 @@ galaxyVm { # Must match the value used in LeoPubsubMessageSubscriber (galaxyDisk.postgresDiskNameSuffix). postgresDiskNameSuffix = ${gke.galaxyDisk.postgresDiskNameSuffix} gitRepo = "https://github.com/galaxyproject/galaxy-k8s-boot.git" - gitBranch = "master" + gitBranch = "anvil" } gke { diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala index 1901685b67..01726fae37 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala @@ -1067,12 +1067,10 @@ class GKEInterpreter[F[_]]( // Passed as the "user-data" metadata key, processed by cloud-init on first boot only. // The galaxy-k8s-boot custom image has cloud-init pre-installed; "#cloud-config" must be // the first line for cloud-init to recognise the file format. - // To update, sync manually from https://github.com/galaxyproject/galaxy-k8s-boot/blob/dev/bin/user_data.sh - userDataContent = scala.io.Source - .fromResource("init-resources/galaxy-user-data.sh") - .getLines() - .toList - .mkString("\n") + // To update, sync manually from https://github.com/galaxyproject/galaxy-k8s-boot/blob/anvil/bin/user_data.sh + userDataContent <- F.fromTry( + scala.util.Using(scala.io.Source.fromResource("init-resources/galaxy-user-data.sh"))(_.mkString) + ) // Derive postgres disk name using the same naming convention as the subscriber postgresDiskName = GKEAlgebra.getGalaxyPostgresDiskName(nfsDisk.name, @@ -1184,10 +1182,12 @@ class GKEInterpreter[F[_]]( .addItems(Items.newBuilder().setKey("gcp-region").setValue(regionParam.value).build()) .addItems(Items.newBuilder().setKey("gcp-network").setValue(network.value).build()) .addItems(Items.newBuilder().setKey("gcp-subnet").setValue(subnetwork.value).build()) - // Galaxy needs to know its public URL prefix so it generates correct absolute links - // (JS, CSS, API calls) that include the full Leo proxy path. - // galaxy-k8s-boot's ansible playbook must accept galaxy_url_prefix and set it in - // Galaxy's helm values (galaxy.yml). Without this, Galaxy generates links rooted at / + // Galaxy admin user email — used by the post-install job to create the initial Galaxy admin. + .addItems( + Items.newBuilder().setKey("galaxy-user-email").setValue(app.auditInfo.creator.value).build() + ) + // Leo proxy path prefix passed to ansible-pull as galaxy_prefix so Galaxy's nginx ingress + // is configured at the correct subpath. Without this, Galaxy generates links rooted at / // which the browser resolves against Leo's host and gets 404s → blank page. .addItems( Items @@ -1230,7 +1230,7 @@ class GKEInterpreter[F[_]]( // Only attempted when the Batch SA lives in the same project as the user (i.e. not a shared platform SA). // For cross-project Batch SAs, this binding must be set up externally (e.g. via Terraform). gcpBatchSaProject = GoogleProject( - gcpBatchSa.split("@").lastOption.getOrElse("").replace(".iam.gserviceaccount.com", "") + gcpBatchSa.split("@").lift(1).map(_.stripSuffix(".iam.gserviceaccount.com")).getOrElse("") ) _ <- if (gcpBatchSaProject == googleProject) @@ -1300,10 +1300,9 @@ class GKEInterpreter[F[_]]( s"Galaxy VM instance ${instanceName.value} submitted for project ${googleProject.value}; polling for external IP" ) - // Poll until the instance has both internal and external IPs assigned. - // We store the internal IP as the proxy backend (KubernetesDnsCache loadBalancerIp) so that - // the Leo proxy connects to the VM over the internal VPC network using plain HTTP. - // The external IP is only used for the readiness health check (TCP to port 80). + // Poll until the instance has an external IP assigned (needed for both proxy routing and readiness check). + // We store the external IP because Leo's GKE cluster and the Galaxy VM are in different GCP projects + // whose VPCs are not peered, making the internal IP unreachable from Leo's pod. ipPairOpt <- streamFUntilDone( computeService.getInstance(googleProject, zoneParam, instanceName).map { instanceOpt => instanceOpt.flatMap { inst => From 4aae1c17780eb9abd455e260dfeee2911f5dd88b Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Fri, 12 Jun 2026 16:12:37 -0400 Subject: [PATCH 19/55] Switch Galaxy bootstrap from cloud-init user-data to GCE startup-script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pre-baked galaxy-k8s-boot image carries cloud-init state from its build, so cloud-init treats new VM launches as subsequent boots and skips runcmd — causing "No startup scripts to run" in Guest Agent logs and a VM that never bootstraps Galaxy. Fix: pass the bootstrap script as the "startup-script" metadata key instead of "user-data". The GCE Guest Agent always executes startup-script on boot, regardless of cloud-init state. galaxy-user-data.sh is reformatted from cloud-config YAML to a plain bash script. The sudo -u debian block now uses a single-quoted heredoc delimiter (<<'DEBIAN_EOF') to avoid apostrophes in comments breaking shell quoting. Co-Authored-By: Claude Sonnet 4.6 --- .../init-resources/galaxy-user-data.sh | 296 +++++++++--------- .../leonardo/util/GKEInterpreter.scala | 11 +- 2 files changed, 151 insertions(+), 156 deletions(-) diff --git a/http/src/main/resources/init-resources/galaxy-user-data.sh b/http/src/main/resources/init-resources/galaxy-user-data.sh index 1a106b66b3..f6b6ee6662 100644 --- a/http/src/main/resources/init-resources/galaxy-user-data.sh +++ b/http/src/main/resources/init-resources/galaxy-user-data.sh @@ -1,152 +1,148 @@ -#cloud-config +#!/bin/bash # Sourced from https://github.com/galaxyproject/galaxy-k8s-boot/blob/anvil/bin/user_data.sh # When updating this file, sync it manually from that repository and verify the changes. -write_files: - - path: /usr/local/bin/galaxy_bootstrap.sh - permissions: '0755' - owner: root:root - content: | - #!/bin/bash - - echo "[$(date)] - Starting galaxy_bootstrap script..." - - # 1. Setup persistent disk if available - DISK_DEVICE="/dev/disk/by-id/google-galaxy-data" - if [ -b "$DISK_DEVICE" ]; then - echo "[$(date)] - Found persistent disk at $DISK_DEVICE" - - # Check if disk is already formatted - if ! blkid "$DISK_DEVICE" > /dev/null 2>&1; then - echo "[$(date)] - Formatting disk $DISK_DEVICE with ext4" - mkfs -t ext4 "$DISK_DEVICE" - else - echo "[$(date)] - Disk $DISK_DEVICE is already formatted" - fi - - # Create mount point and mount - mkdir -p /mnt/block_storage - mount "$DISK_DEVICE" /mnt/block_storage - - # Add to fstab for persistent mounting across reboots - DISK_UUID=$(blkid -s UUID -o value "$DISK_DEVICE") - if [ -n "$DISK_UUID" ] && ! grep -q "$DISK_UUID" /etc/fstab; then - echo "UUID=$DISK_UUID /mnt/block_storage ext4 defaults 0 2" >> /etc/fstab - fi - - # Set proper ownership - chown debian:debian /mnt/block_storage - echo "[$(date)] - Persistent disk mounted at /mnt/block_storage" - else - echo "[$(date)] - No persistent disk found at $DISK_DEVICE. Galaxy will use ephemeral storage." - fi - - # 2. Setup PostgreSQL disk if available - POSTGRES_DISK_DEVICE="/dev/disk/by-id/google-galaxy-postgres-data" - if [ -b "$POSTGRES_DISK_DEVICE" ]; then - echo "[$(date)] - Found PostgreSQL disk at $POSTGRES_DISK_DEVICE" - - # Check if disk is already formatted - if ! blkid "$POSTGRES_DISK_DEVICE" > /dev/null 2>&1; then - echo "[$(date)] - Formatting PostgreSQL disk $POSTGRES_DISK_DEVICE with ext4" - mkfs -t ext4 "$POSTGRES_DISK_DEVICE" - else - echo "[$(date)] - PostgreSQL disk $POSTGRES_DISK_DEVICE is already formatted" - fi - - # Create mount point and mount - mkdir -p /mnt/postgres_storage - mount "$POSTGRES_DISK_DEVICE" /mnt/postgres_storage - - # Add to fstab for persistent mounting across reboots - POSTGRES_DISK_UUID=$(blkid -s UUID -o value "$POSTGRES_DISK_DEVICE") - if [ -n "$POSTGRES_DISK_UUID" ] && ! grep -q "$POSTGRES_DISK_UUID" /etc/fstab; then - echo "UUID=$POSTGRES_DISK_UUID /mnt/postgres_storage ext4 defaults 0 2" >> /etc/fstab - fi - - # Set proper ownership - chown debian:debian /mnt/postgres_storage - echo "[$(date)] - PostgreSQL disk mounted at /mnt/postgres_storage" - else - echo "[$(date)] - No PostgreSQL disk found at $POSTGRES_DISK_DEVICE. PostgreSQL will use ephemeral storage." - fi - - # 3. Run ansible-pull - sudo -u debian bash -c ' - export HOME=/home/debian - HOST_IP=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip" -H "Metadata-Flavor: Google" 2>/dev/null || curl -s ifconfig.me) - - PV_SIZE=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/persistent-volume-size" -H "Metadata-Flavor: Google" 2>/dev/null) - if [ -z "$PV_SIZE" ]; then - echo "[$(date)] - persistent-volume-size metadata not found or empty, using default." - PV_SIZE="139Gi" - fi - echo "[$(date)] - NFS storage size for Galaxy: ${PV_SIZE}" - - # Add restore_galaxy if enabled - RESTORE_GALAXY=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/restore_galaxy" -H "Metadata-Flavor: Google" 2>/dev/null || echo "false") - - GCP_BATCH_SERVICE_ACCOUNT_EMAIL=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/gcp_batch_service_account_email" -H "Metadata-Flavor: Google" 2>/dev/null || echo "galaxy-batch-runner@anvil-and-terra-development.iam.gserviceaccount.com") - echo "[$(date)] - GCP Batch service account email: ${GCP_BATCH_SERVICE_ACCOUNT_EMAIL}" - - # Leo proxy path prefix for this Galaxy app (e.g. /proxy/google/v1/apps/{project}/{appName}/galaxy). - # Passed to ansible as galaxy_prefix so Galaxy's nginx ingress is configured at the correct subpath. - # Without this Galaxy generates links rooted at / which the browser resolves against Leo's host → 404s. - GALAXY_URL_PREFIX=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/galaxy-url-prefix" -H "Metadata-Flavor: Google" 2>/dev/null || echo "") - echo "[$(date)] - Galaxy URL prefix: ${GALAXY_URL_PREFIX}" - - GALAXY_USER=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/galaxy-user-email" -H "Metadata-Flavor: Google" 2>/dev/null || echo "") - echo "[$(date)] - Galaxy user email: ${GALAXY_USER}" - - GIT_REPO=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/git-repo" -H "Metadata-Flavor: Google" 2>/dev/null || echo "https://github.com/galaxyproject/galaxy-k8s-boot.git") - GIT_BRANCH=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/git-branch" -H "Metadata-Flavor: Google" 2>/dev/null || echo "anvil") - - PULL_ARGS=( - -U "${GIT_REPO}" - -C "${GIT_BRANCH}" - -d /home/debian/ansible - -i /tmp/ansible-inventory/localhost - --accept-host-key - --limit 127.0.0.1 - --extra-vars "gcp_batch_service_account_email=${GCP_BATCH_SERVICE_ACCOUNT_EMAIL}" - ) - - if [ "$RESTORE_GALAXY" = "true" ]; then - PULL_ARGS+=(--extra-vars "restore_galaxy=true") - echo "[$(date)] - Galaxy Restore Mode: Enabled" - else - echo "[$(date)] - Galaxy Restore Mode: Disabled" - fi - - if [ -n "$GALAXY_URL_PREFIX" ]; then - PULL_ARGS+=(--extra-vars "galaxy_prefix=${GALAXY_URL_PREFIX}") - echo "[$(date)] - Galaxy URL prefix passed to ansible: ${GALAXY_URL_PREFIX}" - fi - - PULL_ARGS+=(playbook.yml) - - mkdir -p /tmp/ansible-inventory - cat > /tmp/ansible-inventory/localhost << EOF - [vm] - 127.0.0.1 ansible_connection=local ansible_python_interpreter="/usr/bin/python3" - - [all:vars] - ansible_user="debian" - rke2_token="defaultSecret12345" - rke2_additional_sans=["${HOST_IP}"] - rke2_debug=true - nfs_size="${PV_SIZE}" - galaxy_persistence_size="${PV_SIZE}" - galaxy_db_password="gxy-db-password" - galaxy_user="${GALAXY_USER}" - EOF - - echo "[$(date)] - Inventory file created at /tmp/ansible-inventory/localhost; running ansible-pull..." - echo "[$(date)] - Running: ANSIBLE_CALLBACKS_ENABLED=profile_tasks ANSIBLE_HOST_PATTERN_MISMATCH=ignore ansible-pull ${PULL_ARGS[@]}" - - ANSIBLE_CALLBACKS_ENABLED=profile_tasks ANSIBLE_HOST_PATTERN_MISMATCH=ignore ansible-pull "${PULL_ARGS[@]}" - ' - - echo "[$(date)] - Bootstrap script completed." - -runcmd: - - /usr/local/bin/galaxy_bootstrap.sh +# Passed to the GCE instance as the "startup-script" metadata key so the Guest Agent +# executes it on every boot (cloud-init user-data is skipped on pre-baked images). + +echo "[$(date)] - Starting galaxy_bootstrap script..." + +# 1. Setup persistent disk if available +DISK_DEVICE="/dev/disk/by-id/google-galaxy-data" +if [ -b "$DISK_DEVICE" ]; then + echo "[$(date)] - Found persistent disk at $DISK_DEVICE" + + if ! blkid "$DISK_DEVICE" > /dev/null 2>&1; then + echo "[$(date)] - Formatting disk $DISK_DEVICE with ext4" + mkfs -t ext4 "$DISK_DEVICE" + else + echo "[$(date)] - Disk $DISK_DEVICE is already formatted" + fi + + mkdir -p /mnt/block_storage + mount "$DISK_DEVICE" /mnt/block_storage + + DISK_UUID=$(blkid -s UUID -o value "$DISK_DEVICE") + if [ -n "$DISK_UUID" ] && ! grep -q "$DISK_UUID" /etc/fstab; then + echo "UUID=$DISK_UUID /mnt/block_storage ext4 defaults 0 2" >> /etc/fstab + fi + + chown debian:debian /mnt/block_storage + echo "[$(date)] - Persistent disk mounted at /mnt/block_storage" +else + echo "[$(date)] - No persistent disk found at $DISK_DEVICE. Galaxy will use ephemeral storage." +fi + +# 2. Setup PostgreSQL disk if available +POSTGRES_DISK_DEVICE="/dev/disk/by-id/google-galaxy-postgres-data" +if [ -b "$POSTGRES_DISK_DEVICE" ]; then + echo "[$(date)] - Found PostgreSQL disk at $POSTGRES_DISK_DEVICE" + + if ! blkid "$POSTGRES_DISK_DEVICE" > /dev/null 2>&1; then + echo "[$(date)] - Formatting PostgreSQL disk $POSTGRES_DISK_DEVICE with ext4" + mkfs -t ext4 "$POSTGRES_DISK_DEVICE" + else + echo "[$(date)] - PostgreSQL disk $POSTGRES_DISK_DEVICE is already formatted" + fi + + mkdir -p /mnt/postgres_storage + mount "$POSTGRES_DISK_DEVICE" /mnt/postgres_storage + + POSTGRES_DISK_UUID=$(blkid -s UUID -o value "$POSTGRES_DISK_DEVICE") + if [ -n "$POSTGRES_DISK_UUID" ] && ! grep -q "$POSTGRES_DISK_UUID" /etc/fstab; then + echo "UUID=$POSTGRES_DISK_UUID /mnt/postgres_storage ext4 defaults 0 2" >> /etc/fstab + fi + + chown debian:debian /mnt/postgres_storage + echo "[$(date)] - PostgreSQL disk mounted at /mnt/postgres_storage" +else + echo "[$(date)] - No PostgreSQL disk found at $POSTGRES_DISK_DEVICE. PostgreSQL will use ephemeral storage." +fi + +# 3. Run ansible-pull as the debian user. +# Single-quoted heredoc delimiter means the outer (root) shell does not expand any +# variables — all expansion happens inside the debian bash session. +sudo -u debian bash <<'DEBIAN_EOF' +export HOME=/home/debian + +HOST_IP=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip" \ + -H "Metadata-Flavor: Google" 2>/dev/null || curl -s ifconfig.me) + +PV_SIZE=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/persistent-volume-size" \ + -H "Metadata-Flavor: Google" 2>/dev/null) +if [ -z "$PV_SIZE" ]; then + echo "[$(date)] - persistent-volume-size metadata not found or empty, using default." + PV_SIZE="139Gi" +fi +echo "[$(date)] - NFS storage size for Galaxy: ${PV_SIZE}" + +RESTORE_GALAXY=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/restore_galaxy" \ + -H "Metadata-Flavor: Google" 2>/dev/null || echo "false") + +GCP_BATCH_SERVICE_ACCOUNT_EMAIL=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/gcp_batch_service_account_email" \ + -H "Metadata-Flavor: Google" 2>/dev/null || echo "galaxy-batch-runner@anvil-and-terra-development.iam.gserviceaccount.com") +echo "[$(date)] - GCP Batch service account email: ${GCP_BATCH_SERVICE_ACCOUNT_EMAIL}" + +# Leo proxy path prefix (e.g. /proxy/google/v1/apps/{project}/{appName}/galaxy). +# Passed to ansible-pull as galaxy_prefix so Galaxy nginx ingress is configured at +# the correct subpath. Without this Galaxy generates links rooted at / which the +# browser resolves against Leo host and gets 404s. +GALAXY_URL_PREFIX=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/galaxy-url-prefix" \ + -H "Metadata-Flavor: Google" 2>/dev/null || echo "") +echo "[$(date)] - Galaxy URL prefix: ${GALAXY_URL_PREFIX}" + +GALAXY_USER=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/galaxy-user-email" \ + -H "Metadata-Flavor: Google" 2>/dev/null || echo "") +echo "[$(date)] - Galaxy user email: ${GALAXY_USER}" + +GIT_REPO=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/git-repo" \ + -H "Metadata-Flavor: Google" 2>/dev/null || echo "https://github.com/galaxyproject/galaxy-k8s-boot.git") +GIT_BRANCH=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/git-branch" \ + -H "Metadata-Flavor: Google" 2>/dev/null || echo "anvil") + +PULL_ARGS=( + -U "${GIT_REPO}" + -C "${GIT_BRANCH}" + -d /home/debian/ansible + -i /tmp/ansible-inventory/localhost + --accept-host-key + --limit 127.0.0.1 + --extra-vars "gcp_batch_service_account_email=${GCP_BATCH_SERVICE_ACCOUNT_EMAIL}" +) + +if [ "$RESTORE_GALAXY" = "true" ]; then + PULL_ARGS+=(--extra-vars "restore_galaxy=true") + echo "[$(date)] - Galaxy Restore Mode: Enabled" +else + echo "[$(date)] - Galaxy Restore Mode: Disabled" +fi + +if [ -n "$GALAXY_URL_PREFIX" ]; then + PULL_ARGS+=(--extra-vars "galaxy_prefix=${GALAXY_URL_PREFIX}") + echo "[$(date)] - Galaxy URL prefix passed to ansible: ${GALAXY_URL_PREFIX}" +fi + +PULL_ARGS+=(playbook.yml) + +mkdir -p /tmp/ansible-inventory +cat > /tmp/ansible-inventory/localhost << EOF +[vm] +127.0.0.1 ansible_connection=local ansible_python_interpreter="/usr/bin/python3" + +[all:vars] +ansible_user="debian" +rke2_token="defaultSecret12345" +rke2_additional_sans=["${HOST_IP}"] +rke2_debug=true +nfs_size="${PV_SIZE}" +galaxy_persistence_size="${PV_SIZE}" +galaxy_db_password="gxy-db-password" +galaxy_user="${GALAXY_USER}" +EOF + +echo "[$(date)] - Inventory file created at /tmp/ansible-inventory/localhost; running ansible-pull..." +echo "[$(date)] - Running: ANSIBLE_CALLBACKS_ENABLED=profile_tasks ANSIBLE_HOST_PATTERN_MISMATCH=ignore ansible-pull ${PULL_ARGS[@]}" + +ANSIBLE_CALLBACKS_ENABLED=profile_tasks ANSIBLE_HOST_PATTERN_MISMATCH=ignore ansible-pull "${PULL_ARGS[@]}" +DEBIAN_EOF + +echo "[$(date)] - Bootstrap script completed." diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala index 01726fae37..eff5f12e03 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala @@ -1063,12 +1063,11 @@ class GKEInterpreter[F[_]]( SetUpProjectNetworkParams(googleProject, regionParam) ) - // Load cloud-config content bundled from galaxy-k8s-boot bin/user_data.sh. - // Passed as the "user-data" metadata key, processed by cloud-init on first boot only. - // The galaxy-k8s-boot custom image has cloud-init pre-installed; "#cloud-config" must be - // the first line for cloud-init to recognise the file format. + // Load the startup-script passed to the GCE Guest Agent via the "startup-script" metadata key. + // The Guest Agent executes it on every boot, unlike cloud-init user-data which the pre-baked + // galaxy-k8s-boot image treats as already-run and skips. // To update, sync manually from https://github.com/galaxyproject/galaxy-k8s-boot/blob/anvil/bin/user_data.sh - userDataContent <- F.fromTry( + startupScriptContent <- F.fromTry( scala.util.Using(scala.io.Source.fromResource("init-resources/galaxy-user-data.sh"))(_.mkString) ) @@ -1172,7 +1171,7 @@ class GKEInterpreter[F[_]]( .setMetadata( Metadata .newBuilder() - .addItems(Items.newBuilder().setKey("user-data").setValue(userDataContent).build()) + .addItems(Items.newBuilder().setKey("startup-script").setValue(startupScriptContent).build()) .addItems(Items.newBuilder().setKey("google-logging-enabled").setValue("true").build()) .addItems(Items.newBuilder().setKey("gcp_batch_service_account_email").setValue(gcpBatchSa).build()) .addItems(Items.newBuilder().setKey("persistent-volume-size").setValue(pvSize).build()) From 0dd49b4350c43e12c51768b6cd69bfcf2e229c8f Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Fri, 12 Jun 2026 17:04:16 -0400 Subject: [PATCH 20/55] Fix ansible inventory group name: [vm] -> [vms] to match hosts: vms in playbook.yml Co-Authored-By: Claude Sonnet 4.6 --- http/src/main/resources/init-resources/galaxy-user-data.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/src/main/resources/init-resources/galaxy-user-data.sh b/http/src/main/resources/init-resources/galaxy-user-data.sh index f6b6ee6662..f02fc4d562 100644 --- a/http/src/main/resources/init-resources/galaxy-user-data.sh +++ b/http/src/main/resources/init-resources/galaxy-user-data.sh @@ -125,7 +125,7 @@ PULL_ARGS+=(playbook.yml) mkdir -p /tmp/ansible-inventory cat > /tmp/ansible-inventory/localhost << EOF -[vm] +[vms] 127.0.0.1 ansible_connection=local ansible_python_interpreter="/usr/bin/python3" [all:vars] From cc14cc3ec6f8b420976d7b03144ab3e225b7a34b Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Mon, 15 Jun 2026 09:52:09 -0400 Subject: [PATCH 21/55] Revert sourceImage to v2026-02-25: v2026-06-10 has Helm incompatibility with anvil playbook (helm list --all) Co-Authored-By: Claude Sonnet 4.6 --- http/src/main/resources/reference.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/src/main/resources/reference.conf b/http/src/main/resources/reference.conf index 898674850b..369f1a4aa4 100644 --- a/http/src/main/resources/reference.conf +++ b/http/src/main/resources/reference.conf @@ -437,7 +437,7 @@ galaxyVm { # Pre-built galaxy-k8s-boot image with all dependencies (Ansible, RKE2, etc.) pre-installed. # Has cloud-init, which processes the "user-data" metadata key on first boot. # Source: https://github.com/galaxyproject/galaxy-k8s-boot (image built by the Galaxy team) - sourceImage = "projects/anvil-and-terra-development/global/images/galaxy-k8s-boot-v2026-06-10" + sourceImage = "projects/anvil-and-terra-development/global/images/galaxy-k8s-boot-v2026-02-25" machineType = "n1-highmem-8" bootDiskSizeGb = 100 postgresDiskSizeGb = 10 From 455bbc0c7b4a74a7283dffddcffa177bec198a6d Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Mon, 15 Jun 2026 11:18:46 -0400 Subject: [PATCH 22/55] Fix Galaxy blank page: set galaxy_url_prefix and improve Running health check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs found during BEE testing: 1. Blank page on Galaxy load: the anvil playbook's galaxy_prefix only configures the nginx ingress path, not Galaxy's own galaxy_url_prefix in galaxy.yml. Without it Galaxy generates /static/... links that resolve without the Leo proxy prefix, leaving a blank page. Fixed by passing galaxy_helm_extra_sets via an extra-vars YAML file so Helm also sets configs.galaxy\.yml.galaxy_url_prefix. 2. App marked Running too early: isVmReachable was polling GET / which returns 404 from nginx (no ingress rule at root) — 404 < 500 = true — so Leo marked the app Running before Galaxy pods were ready. Changed to poll the galaxy prefix path and require status < 400 so 404 (no ingress yet) and 502 (pods starting) both keep polling. Co-Authored-By: Claude Sonnet 4.6 --- .../resources/init-resources/galaxy-user-data.sh | 11 +++++++++++ .../dsde/workbench/leonardo/dao/HttpAppDAO.scala | 16 ++++++++++++---- .../workbench/leonardo/util/GKEInterpreter.scala | 15 +++++++-------- .../dsde/workbench/leonardo/dao/MockAppDAO.scala | 2 +- 4 files changed, 31 insertions(+), 13 deletions(-) diff --git a/http/src/main/resources/init-resources/galaxy-user-data.sh b/http/src/main/resources/init-resources/galaxy-user-data.sh index f02fc4d562..b2fad17ce2 100644 --- a/http/src/main/resources/init-resources/galaxy-user-data.sh +++ b/http/src/main/resources/init-resources/galaxy-user-data.sh @@ -118,6 +118,17 @@ fi if [ -n "$GALAXY_URL_PREFIX" ]; then PULL_ARGS+=(--extra-vars "galaxy_prefix=${GALAXY_URL_PREFIX}") + # Also configure Galaxy's own galaxy_url_prefix so it generates correct subpath links. + # galaxy_prefix only sets the nginx ingress path; without galaxy_url_prefix in galaxy.yml, + # Galaxy generates /static/... links that the browser resolves without the prefix → blank page. + # galaxy_helm_extra_sets is a variable the anvil playbook merges into Helm set_values. + # The YAML file avoids shell-escaping the nested key (configs.galaxy\.yml.*). + mkdir -p /tmp/ansible-extra-vars + cat > /tmp/ansible-extra-vars/galaxy_prefix.yml << EOF +galaxy_helm_extra_sets: + - value: "configs.galaxy\\.yml.galaxy_url_prefix=${GALAXY_URL_PREFIX}" +EOF + PULL_ARGS+=(--extra-vars "@/tmp/ansible-extra-vars/galaxy_prefix.yml") echo "[$(date)] - Galaxy URL prefix passed to ansible: ${GALAXY_URL_PREFIX}" fi diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpAppDAO.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpAppDAO.scala index 7fc78868ef..29a83f38ce 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpAppDAO.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/HttpAppDAO.scala @@ -45,16 +45,20 @@ class HttpAppDAO[F[_]: Async](kubernetesDnsCache: KubernetesDnsCache[F], client: case _ => Async[F].pure(false) // Update once we support Relay for apps } - def isVmReachable(ip: org.broadinstitute.dsde.workbench.model.IP, port: Int, traceId: TraceId): F[Boolean] = + def isVmReachable(ip: org.broadinstitute.dsde.workbench.model.IP, + port: Int, + traceId: TraceId, + path: String = "/" + ): F[Boolean] = client .status( Request[F]( method = Method.GET, - uri = Uri.unsafeFromString(s"http://${ip.asString}:${port}/"), + uri = Uri.unsafeFromString(s"http://${ip.asString}:${port}${path}"), headers = Headers(Header.Raw(CIString("X-Request-ID"), traceId.asString)) ) ) - .map(status => status.code < 500) + .map(status => status.code < 400) .handleErrorWith(t => logger.error(Map("traceId" -> traceId.asString), t)("Fail to check if VM is reachable").as(false) ) @@ -67,5 +71,9 @@ trait AppDAO[F[_]] { traceId: TraceId ): F[Boolean] - def isVmReachable(ip: org.broadinstitute.dsde.workbench.model.IP, port: Int, traceId: TraceId): F[Boolean] + def isVmReachable(ip: org.broadinstitute.dsde.workbench.model.IP, + port: Int, + traceId: TraceId, + path: String = "/" + ): F[Boolean] } diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala index eff5f12e03..0fb9cd8a37 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala @@ -1098,6 +1098,8 @@ class GKEInterpreter[F[_]]( ) .map(sa => sa.email.value) + galaxyUrlPrefix = s"/proxy/google/v1/apps/${googleProject.value}/${app.appName.value}/galaxy" + // Disks — data and postgres disks are always pre-existing by the time this method runs // (created by createDiskOp / createSecondDiskOp, or retained from a previous app). // Use setSource to attach existing disks; only the boot disk is created fresh. @@ -1192,9 +1194,7 @@ class GKEInterpreter[F[_]]( Items .newBuilder() .setKey("galaxy-url-prefix") - .setValue( - s"/proxy/google/v1/apps/${googleProject.value}/${app.appName.value}/galaxy" - ) + .setValue(galaxyUrlPrefix) .build() ) .build() @@ -1352,12 +1352,11 @@ class GKEInterpreter[F[_]]( s"Polling Galaxy readiness for app ${app.appName.value} via proxy (backend: ${externalIp.asString}:80)" ) - // Wait for Galaxy's nginx to respond. - // Uses a direct HTTP check to the VM's external IP on port 80, bypassing the Leo proxy - // hostname chain (which would require the proxy wildcard DNS to be reachable from within - // the Leo pod — unreliable in BEE environments due to hairpin NAT). + // Wait for Galaxy to be fully ready: poll the actual galaxy prefix path, not just /. + // nginx returns 404 for / (no ingress rule at root) and 502 for the galaxy path while + // pods are starting — both are < 400 = false. Only returns true once Galaxy responds 200. isDone <- streamFUntilDone( - appDao.isVmReachable(externalIp, 80, ctx.traceId), + appDao.isVmReachable(externalIp, 80, ctx.traceId, galaxyUrlPrefix), config.monitorConfig.createApp.maxAttempts, config.monitorConfig.createApp.interval ).interruptAfter(config.monitorConfig.createApp.interruptAfter).compile.lastOrError diff --git a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/dao/MockAppDAO.scala b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/dao/MockAppDAO.scala index 647f3cc77f..bba7d6bc8b 100644 --- a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/dao/MockAppDAO.scala +++ b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/dao/MockAppDAO.scala @@ -14,7 +14,7 @@ class MockAppDAO(isUp: Boolean = true) extends AppDAO[IO] { ): IO[Boolean] = IO.pure(isUp) - override def isVmReachable(ip: IP, port: Int, traceId: TraceId): IO[Boolean] = + override def isVmReachable(ip: IP, port: Int, traceId: TraceId, path: String = "/"): IO[Boolean] = IO.pure(isUp) } object MockAppDAO extends MockAppDAO(isUp = true) From ad5e132fe7815f557ddd51339043a11d41f754b0 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Mon, 15 Jun 2026 12:13:49 -0400 Subject: [PATCH 23/55] Fix YAML syntax in galaxy_prefix.yml: use single quotes so backslash is literal YAML double-quoted strings treat \ as escape, making \.yml invalid. Single-quoted YAML scalars treat backslash as literal, which is what Helm needs for the dotted key configs.galaxy\.yml.galaxy_url_prefix. Co-Authored-By: Claude Sonnet 4.6 --- http/src/main/resources/init-resources/galaxy-user-data.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/src/main/resources/init-resources/galaxy-user-data.sh b/http/src/main/resources/init-resources/galaxy-user-data.sh index b2fad17ce2..1795a60bbc 100644 --- a/http/src/main/resources/init-resources/galaxy-user-data.sh +++ b/http/src/main/resources/init-resources/galaxy-user-data.sh @@ -126,7 +126,7 @@ if [ -n "$GALAXY_URL_PREFIX" ]; then mkdir -p /tmp/ansible-extra-vars cat > /tmp/ansible-extra-vars/galaxy_prefix.yml << EOF galaxy_helm_extra_sets: - - value: "configs.galaxy\\.yml.galaxy_url_prefix=${GALAXY_URL_PREFIX}" + - value: 'configs.galaxy\.yml.galaxy_url_prefix=${GALAXY_URL_PREFIX}' EOF PULL_ARGS+=(--extra-vars "@/tmp/ansible-extra-vars/galaxy_prefix.yml") echo "[$(date)] - Galaxy URL prefix passed to ansible: ${GALAXY_URL_PREFIX}" From 9bec16beec5c65e47d7554ca49e548fd77f4509a Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Mon, 15 Jun 2026 15:23:31 -0400 Subject: [PATCH 24/55] Fix galaxy_url_prefix Helm key: add missing .galaxy. nesting The cloudve/galaxy chart stores Galaxy config under configs.galaxy.yml.galaxy.* so the correct key is configs.galaxy\.yml.galaxy.galaxy_url_prefix, not configs.galaxy\.yml.galaxy_url_prefix. The previous key wrote the value at the wrong level of galaxy.yml where Galaxy doesn't read it. Co-Authored-By: Claude Sonnet 4.6 --- http/src/main/resources/init-resources/galaxy-user-data.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/src/main/resources/init-resources/galaxy-user-data.sh b/http/src/main/resources/init-resources/galaxy-user-data.sh index 1795a60bbc..66059dd8a3 100644 --- a/http/src/main/resources/init-resources/galaxy-user-data.sh +++ b/http/src/main/resources/init-resources/galaxy-user-data.sh @@ -126,7 +126,7 @@ if [ -n "$GALAXY_URL_PREFIX" ]; then mkdir -p /tmp/ansible-extra-vars cat > /tmp/ansible-extra-vars/galaxy_prefix.yml << EOF galaxy_helm_extra_sets: - - value: 'configs.galaxy\.yml.galaxy_url_prefix=${GALAXY_URL_PREFIX}' + - value: 'configs.galaxy\.yml.galaxy.galaxy_url_prefix=${GALAXY_URL_PREFIX}' EOF PULL_ARGS+=(--extra-vars "@/tmp/ansible-extra-vars/galaxy_prefix.yml") echo "[$(date)] - Galaxy URL prefix passed to ansible: ${GALAXY_URL_PREFIX}" From b43c0cea146e7ab25a212220ab1cd6e184eb72ff Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Tue, 16 Jun 2026 12:05:22 -0400 Subject: [PATCH 25/55] Fix Galaxy blank page: strip Authorization header and improve readiness probe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three changes: - ProxyService: add Authorization to HeadersToFilter so Leo no longer forwards the Terra Bearer JWT to backends. Galaxy 23.1+ treats any Authorization: Bearer value as a Galaxy API key; the Terra JWT fails validation and every API call returns 400, causing a blank page. Leo is the auth boundary — backends should not receive user tokens. - GKEInterpreter: poll /api/version instead of the bare galaxy prefix path for the VM readiness check. The bare path can return a 301 redirect (nginx trailing-slash normalisation) before Galaxy's Python backend is ready, satisfying < 400 and marking Running too early. - galaxy-user-data.sh: remove the galaxy_helm_extra_sets extra-vars block. The CloudVE chart already auto-wires galaxy_url_prefix from ingress.path via double-tpl, so the override is redundant. It also risks breaking restore mode: the playbook's set_fact overrides galaxy_helm_extra_sets with PVC values for restore, and if Ansible extra-vars win precedence, restore mode fails to apply existingClaim. Co-Authored-By: Claude Sonnet 4.6 --- .../main/resources/init-resources/galaxy-user-data.sh | 11 ----------- .../leonardo/http/service/ProxyService.scala | 7 ++++++- .../dsde/workbench/leonardo/util/GKEInterpreter.scala | 9 +++++---- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/http/src/main/resources/init-resources/galaxy-user-data.sh b/http/src/main/resources/init-resources/galaxy-user-data.sh index 66059dd8a3..f02fc4d562 100644 --- a/http/src/main/resources/init-resources/galaxy-user-data.sh +++ b/http/src/main/resources/init-resources/galaxy-user-data.sh @@ -118,17 +118,6 @@ fi if [ -n "$GALAXY_URL_PREFIX" ]; then PULL_ARGS+=(--extra-vars "galaxy_prefix=${GALAXY_URL_PREFIX}") - # Also configure Galaxy's own galaxy_url_prefix so it generates correct subpath links. - # galaxy_prefix only sets the nginx ingress path; without galaxy_url_prefix in galaxy.yml, - # Galaxy generates /static/... links that the browser resolves without the prefix → blank page. - # galaxy_helm_extra_sets is a variable the anvil playbook merges into Helm set_values. - # The YAML file avoids shell-escaping the nested key (configs.galaxy\.yml.*). - mkdir -p /tmp/ansible-extra-vars - cat > /tmp/ansible-extra-vars/galaxy_prefix.yml << EOF -galaxy_helm_extra_sets: - - value: 'configs.galaxy\.yml.galaxy.galaxy_url_prefix=${GALAXY_URL_PREFIX}' -EOF - PULL_ARGS+=(--extra-vars "@/tmp/ansible-extra-vars/galaxy_prefix.yml") echo "[$(date)] - Galaxy URL prefix passed to ansible: ${GALAXY_URL_PREFIX}" fi diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/ProxyService.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/ProxyService.scala index 5f407ad424..e567fc11b7 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/ProxyService.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/ProxyService.scala @@ -587,7 +587,12 @@ class ProxyService( "Sec-WebSocket-Protocol", "UpgradeToWebSocket", "Upgrade", - "Connection" + "Connection", + // Strip the user's Leo/Terra bearer token: backends must not receive it. + // Galaxy 23.1+ treats any Authorization: Bearer value as a Galaxy API key; + // forwarding the Terra JWT causes Galaxy to return 400 for every API call. + // Leo is the auth boundary — backends authenticate via their own mechanisms. + "Authorization" ).map(_.toLowerCase) } diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala index 0fb9cd8a37..caf2cbba2f 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala @@ -1352,11 +1352,12 @@ class GKEInterpreter[F[_]]( s"Polling Galaxy readiness for app ${app.appName.value} via proxy (backend: ${externalIp.asString}:80)" ) - // Wait for Galaxy to be fully ready: poll the actual galaxy prefix path, not just /. - // nginx returns 404 for / (no ingress rule at root) and 502 for the galaxy path while - // pods are starting — both are < 400 = false. Only returns true once Galaxy responds 200. + // Poll /api/version rather than the bare prefix path: nginx can return a 301 redirect + // (trailing-slash normalisation) for the bare path while Galaxy pods are still starting, + // which would satisfy < 400 and mark the app Running too early. /api/version requires + // Galaxy's Python API to be fully initialised and only returns 200 at that point. isDone <- streamFUntilDone( - appDao.isVmReachable(externalIp, 80, ctx.traceId, galaxyUrlPrefix), + appDao.isVmReachable(externalIp, 80, ctx.traceId, s"$galaxyUrlPrefix/api/version"), config.monitorConfig.createApp.maxAttempts, config.monitorConfig.createApp.interval ).interruptAfter(config.monitorConfig.createApp.interruptAfter).compile.lastOrError From 2a97bf95f0f5925f9614a65b781e0fc1f4195857 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Tue, 16 Jun 2026 14:39:53 -0400 Subject: [PATCH 26/55] Fix Galaxy VM restore: save lastUsedBy on keep-disk delete and accept AppRestore.Other for Galaxy When a VM-based Galaxy app is deleted with disks kept, Leo never called updateLastUsedBy/updateGalaxyDiskRestore so the disk's appRestore was always None. On re-create, LeoAppServiceInterp threw "no restore info found in DB". Three coordinated fixes: - LeoPubsubMessageSubscriber.deleteApp: call updateLastUsedBy for the data disk whenever the disk is being kept (msg.diskId is absent); works for both GKE and VM Galaxy. - PersistentDiskComponent: when formattedBy=Galaxy but galaxyPvcId is null (VM path has no Kubernetes PVC), map to AppRestore.Other(lastUsedBy) instead of producing None. - LeoAppServiceInterp: accept AppRestore.Other for Galaxy in the restore success match so VM apps that have AppRestore.Other (rather than GalaxyRestore) can be re-created. Co-Authored-By: Claude Sonnet 4.6 --- .../workbench/leonardo/db/PersistentDiskComponent.scala | 6 +++++- .../leonardo/http/service/LeoAppServiceInterp.scala | 1 + .../leonardo/monitor/LeoPubsubMessageSubscriber.scala | 9 +++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/db/PersistentDiskComponent.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/db/PersistentDiskComponent.scala index c3e82ee8af..ef91f3762f 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/db/PersistentDiskComponent.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/db/PersistentDiskComponent.scala @@ -113,7 +113,11 @@ class PersistentDiskTable(tag: Tag) extends Table[PersistentDiskRecord](tag, "PE formattedBy, formattedBy.flatMap { case FormattedBy.Galaxy => - (galaxyPvcId, lastUsedBy).mapN((gp, lb) => GalaxyRestore(gp, lb)) + // GKE-based Galaxy stores a PVC ID; VM-based Galaxy has no PVC so falls back to Other. + galaxyPvcId match { + case Some(pvcId) => lastUsedBy.map(lb => GalaxyRestore(pvcId, lb)) + case None => lastUsedBy.map(Other) + } case FormattedBy.Cromwell => lastUsedBy.map(Other) case FormattedBy.Allowed => lastUsedBy.map(Other) case FormattedBy.GCE | FormattedBy.Custom => None diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/LeoAppServiceInterp.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/LeoAppServiceInterp.scala index dd60eba7c5..11b958adf5 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/LeoAppServiceInterp.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/LeoAppServiceInterp.scala @@ -828,6 +828,7 @@ final class LeoAppServiceInterp[F[_]: Parallel](config: AppServiceConfig, } else { (diskResult.disk.formattedBy, diskResult.disk.appRestore) match { case (Some(FormattedBy.Galaxy), Some(GalaxyRestore(_, _))) | + (Some(FormattedBy.Galaxy), Some(AppRestore.Other(_))) | (Some(FormattedBy.Cromwell), Some(AppRestore.Other(_))) | (Some(FormattedBy.Allowed), Some(AppRestore.Other(_))) => val lastUsedBy = diskResult.disk.appRestore.get.lastUsedBy diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriber.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriber.scala index fb71a5603d..403899d919 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriber.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriber.scala @@ -1167,6 +1167,15 @@ class LeoPubsubMessageSubscriber[F[_]]( ) } + // When keeping the disk (diskId absent), record which app last used it so the next + // create-app call can restore from it. For VM-based Galaxy there is no PVC, so we + // only save lastUsedBy; PersistentDiskComponent maps this to AppRestore.Other. + _ <- if (msg.diskId.isEmpty) + dbApp.app.appResources.disk.traverse_ { disk => + persistentDiskQuery.updateLastUsedBy(disk.id, msg.appId).transaction + } + else F.unit + // detach/delete disk when we need to delete disk _ <- msg.diskId.traverse_ { diskId => // we now use the detach timestamp recorded prior to helm uninstall so we can observe when galaxy actually 'detaches' the disk from google's perspective From 74d24020b671870dd239be3c85de6faf3bcccebf Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Tue, 16 Jun 2026 16:49:04 -0400 Subject: [PATCH 27/55] format fix --- .../leonardo/monitor/LeoPubsubMessageSubscriber.scala | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriber.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriber.scala index 403899d919..eb1a28514e 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriber.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriber.scala @@ -1170,11 +1170,12 @@ class LeoPubsubMessageSubscriber[F[_]]( // When keeping the disk (diskId absent), record which app last used it so the next // create-app call can restore from it. For VM-based Galaxy there is no PVC, so we // only save lastUsedBy; PersistentDiskComponent maps this to AppRestore.Other. - _ <- if (msg.diskId.isEmpty) - dbApp.app.appResources.disk.traverse_ { disk => - persistentDiskQuery.updateLastUsedBy(disk.id, msg.appId).transaction - } - else F.unit + _ <- + if (msg.diskId.isEmpty) + dbApp.app.appResources.disk.traverse_ { disk => + persistentDiskQuery.updateLastUsedBy(disk.id, msg.appId).transaction + } + else F.unit // detach/delete disk when we need to delete disk _ <- msg.diskId.traverse_ { diskId => From 2b662945369b12478cbd8fee1f338b77173bd052 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Wed, 17 Jun 2026 09:30:18 -0400 Subject: [PATCH 28/55] Fix test: Galaxy VM create sets lastUsedBy, appRestore is now AppRestore.Other not None GKEInterpreter.createAndPollApp already calls updateLastUsedBy after installGalaxyVm (line 431), so the keep-disk delete path does not need to set it separately. The prior commit's PersistentDiskComponent change now correctly maps a Galaxy disk with lastUsedBy set but no galaxyPvcId to AppRestore.Other instead of None. That exposed a fragile test assertion that assumed appRestore would be None and used an unsafe asInstanceOf[GalaxyRestore] cast. Updated assertion to expect Some(AppRestore.Other(appId)). Co-Authored-By: Claude Sonnet 4.6 --- .../leonardo/monitor/LeoPubsubMessageSubscriber.scala | 10 ---------- .../monitor/LeoPubsubMessageSubscriberSpec.scala | 8 ++++---- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriber.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriber.scala index eb1a28514e..fb71a5603d 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriber.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriber.scala @@ -1167,16 +1167,6 @@ class LeoPubsubMessageSubscriber[F[_]]( ) } - // When keeping the disk (diskId absent), record which app last used it so the next - // create-app call can restore from it. For VM-based Galaxy there is no PVC, so we - // only save lastUsedBy; PersistentDiskComponent maps this to AppRestore.Other. - _ <- - if (msg.diskId.isEmpty) - dbApp.app.appResources.disk.traverse_ { disk => - persistentDiskQuery.updateLastUsedBy(disk.id, msg.appId).transaction - } - else F.unit - // detach/delete disk when we need to delete disk _ <- msg.diskId.traverse_ { diskId => // we now use the detach timestamp recorded prior to helm uninstall so we can observe when galaxy actually 'detaches' the disk from google's perspective diff --git a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriberSpec.scala b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriberSpec.scala index 3da67b1856..b99360ea9e 100644 --- a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriberSpec.scala +++ b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/monitor/LeoPubsubMessageSubscriberSpec.scala @@ -31,7 +31,7 @@ import org.broadinstitute.dsde.workbench.google2.{ ZoneName } import org.broadinstitute.dsde.workbench.util2.InstanceName -import org.broadinstitute.dsde.workbench.leonardo.AppRestore.GalaxyRestore +import org.broadinstitute.dsde.workbench.leonardo.AppRestore import org.broadinstitute.dsde.workbench.leonardo.AsyncTaskProcessor.Task import org.broadinstitute.dsde.workbench.leonardo.CommonTestData._ import org.broadinstitute.dsde.workbench.leonardo.KubernetesTestData.{ @@ -924,7 +924,6 @@ class LeoPubsubMessageSubscriberSpec getDiskOpt <- persistentDiskQuery.getById(savedApp1.appResources.disk.get.id).transaction getDisk = getDiskOpt.get appRestore <- persistentDiskQuery.getAppDiskRestore(savedApp1.appResources.disk.get.id).transaction - galaxyRestore = appRestore.map(_.asInstanceOf[GalaxyRestore]) } yield { getCluster.status shouldBe KubernetesClusterStatus.Running getCluster.nodepools.size shouldBe 2 @@ -946,8 +945,9 @@ class LeoPubsubMessageSubscriberSpec ) ) getDisk.status shouldBe DiskStatus.Ready - // Galaxy VM path does not use PVCs — no GalaxyRestore is recorded - galaxyRestore shouldBe None + // Galaxy VM path: GKEInterpreter.createAndPollApp calls updateLastUsedBy, so appRestore is + // AppRestore.Other (no PVC ID — VM-based Galaxy has no Kubernetes PVC). + appRestore shouldBe Some(AppRestore.Other(savedApp1.id)) } implicit val gkeAlg: GKEAlgebra[IO] = makeGKEInterp(nodepoolLock, List(savedApp1.release)) From a1876272e18329df5c343b79f23e2854f0003949 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Fri, 17 Jul 2026 10:58:56 -0400 Subject: [PATCH 29/55] Update Galaxy VM source image to v2026-06-30 Picks up rke2 v1.36.2+rke2r1, Helm v4.2.2, ingress-nginx 4.13.2, and Galaxy Helm chart 6.8.1 baked into the new galaxy-k8s-boot image. Co-Authored-By: Claude Sonnet 4.6 --- http/src/main/resources/reference.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/src/main/resources/reference.conf b/http/src/main/resources/reference.conf index 369f1a4aa4..e32b894443 100644 --- a/http/src/main/resources/reference.conf +++ b/http/src/main/resources/reference.conf @@ -437,7 +437,7 @@ galaxyVm { # Pre-built galaxy-k8s-boot image with all dependencies (Ansible, RKE2, etc.) pre-installed. # Has cloud-init, which processes the "user-data" metadata key on first boot. # Source: https://github.com/galaxyproject/galaxy-k8s-boot (image built by the Galaxy team) - sourceImage = "projects/anvil-and-terra-development/global/images/galaxy-k8s-boot-v2026-02-25" + sourceImage = "projects/anvil-and-terra-development/global/images/galaxy-k8s-boot-v2026-06-30" machineType = "n1-highmem-8" bootDiskSizeGb = 100 postgresDiskSizeGb = 10 From 88c8a3a49caf880befae50bf8d7cb5636e64c378 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Thu, 30 Jul 2026 11:01:09 -0400 Subject: [PATCH 30/55] fix(access-controller): give each Galaxy VM app its own Leo cluster record MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When two Galaxy VM apps share a project, they reused the same Leo cluster record via `saveOrGetClusterForApp`. When the second VM started, it called `updateAsyncFields` and overwrote the shared cluster's `loadBalancerIp` with its own external IP. Since `kubernetesProxyHost` is keyed on cluster ID, both apps produced the same proxy hostname, causing all requests to be routed to the second user's VM (resulting in 404/401 for the first user). Fix: Galaxy apps now always create a fresh Leo cluster record via `saveNewClusterForApp`. Each VM gets a unique cluster ID → unique proxy hostname → no IP collision in `hostToIpMapping`. GKE-based apps (Cromwell, Allowed, Custom) continue to share a cluster per project. Co-Authored-By: Claude Sonnet 4.6 --- .../leonardo/db/KubernetesServiceDbQueries.scala | 13 +++++++++++++ .../http/service/LeoAppServiceInterp.scala | 14 +++++++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/db/KubernetesServiceDbQueries.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/db/KubernetesServiceDbQueries.scala index 36c3b0f161..e0c9def918 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/db/KubernetesServiceDbQueries.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/db/KubernetesServiceDbQueries.scala @@ -116,6 +116,19 @@ object KubernetesServiceDbQueries { } } yield eitherClusterOrError + /** + * Always persists a new KubernetesCluster record (never reuses an existing one). + * Used for Galaxy VM apps: each VM needs its own cluster record to store a unique + * loadBalancerIp, so that multiple Galaxy VMs in the same project don't overwrite + * each other's IP in the cluster table. + */ + def saveNewClusterForApp( + saveKubernetesCluster: SaveKubernetesCluster + )(implicit ec: ExecutionContext): DBIO[ClusterDoesNotExist] = + kubernetesClusterQuery + .save(saveKubernetesCluster) + .map(c => ClusterDoesNotExist(c, DefaultNodepool.fromNodepool(c.nodepools.head))) + /** * Gets an active app by name and cloud context. */ diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/LeoAppServiceInterp.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/LeoAppServiceInterp.scala index 11b958adf5..f980b05888 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/LeoAppServiceInterp.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/LeoAppServiceInterp.scala @@ -170,9 +170,17 @@ final class LeoAppServiceInterp[F[_]: Parallel](config: AppServiceConfig, getSavableCluster(userEmail, cloudContext, req.autopilot.isDefined, ctx.now) ) - saveClusterResult <- KubernetesServiceDbQueries - .saveOrGetClusterForApp(saveCluster, ctx.traceId) - .transaction(isolationLevel = TransactionIsolation.Serializable) + // Galaxy VM apps each get their own Leo cluster record. If multiple Galaxy apps shared + // a cluster, the last VM to start would overwrite the cluster's loadBalancerIp, routing + // all users' proxy requests to the same (wrong) VM. GKE-based apps continue to share a + // cluster per project as before. + saveClusterResult <- + if (req.appType == AppType.Galaxy) + KubernetesServiceDbQueries.saveNewClusterForApp(saveCluster).transaction + else + KubernetesServiceDbQueries + .saveOrGetClusterForApp(saveCluster, ctx.traceId) + .transaction(isolationLevel = TransactionIsolation.Serializable) // TODO Remove the block below to allow app creation on a new cluster when the existing cluster is in Error status _ <- if (saveClusterResult.minimalCluster.status == KubernetesClusterStatus.Error) From 0a0c2febeae2b1f5c3dde27b91a7668744760528 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Thu, 30 Jul 2026 12:40:47 -0400 Subject: [PATCH 31/55] fix(access-controller): allow multiple Galaxy VM cluster records per project Drop IDX_KUBERNETES_CLUSTER_UNIQUE_V2 to permit more than one active KubernetesCluster row per cloud context. Galaxy VM apps need their own cluster record so each VM stores its external IP independently via updateAsyncFields; without the constraint drop, saveNewClusterForApp throws SQLIntegrityConstraintViolationException when a second Galaxy app is created in the same project. Also add an early disk-attachment check in createApp before the cluster record is saved, so DiskAlreadyAttachedException is still raised (rather than the constraint violation) when the same disk is reused across apps. Co-Authored-By: Claude Sonnet 4.6 --- .../leonardo/liquibase/changelog.xml | 1 + ...0730_allow_multiple_galaxy_vm_clusters.xml | 12 ++++++++++++ .../http/service/LeoAppServiceInterp.scala | 19 +++++++++++++++++++ 3 files changed, 32 insertions(+) create mode 100644 http/src/main/resources/org/broadinstitute/dsde/workbench/leonardo/liquibase/changesets/20260730_allow_multiple_galaxy_vm_clusters.xml diff --git a/http/src/main/resources/org/broadinstitute/dsde/workbench/leonardo/liquibase/changelog.xml b/http/src/main/resources/org/broadinstitute/dsde/workbench/leonardo/liquibase/changelog.xml index e7c117f1a2..620f67bbe6 100644 --- a/http/src/main/resources/org/broadinstitute/dsde/workbench/leonardo/liquibase/changelog.xml +++ b/http/src/main/resources/org/broadinstitute/dsde/workbench/leonardo/liquibase/changelog.xml @@ -125,4 +125,5 @@ + diff --git a/http/src/main/resources/org/broadinstitute/dsde/workbench/leonardo/liquibase/changesets/20260730_allow_multiple_galaxy_vm_clusters.xml b/http/src/main/resources/org/broadinstitute/dsde/workbench/leonardo/liquibase/changesets/20260730_allow_multiple_galaxy_vm_clusters.xml new file mode 100644 index 0000000000..a96483ca20 --- /dev/null +++ b/http/src/main/resources/org/broadinstitute/dsde/workbench/leonardo/liquibase/changesets/20260730_allow_multiple_galaxy_vm_clusters.xml @@ -0,0 +1,12 @@ + + + + + + + diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/LeoAppServiceInterp.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/LeoAppServiceInterp.scala index f980b05888..38c15bdc89 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/LeoAppServiceInterp.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/LeoAppServiceInterp.scala @@ -166,6 +166,25 @@ final class LeoAppServiceInterp[F[_]: Parallel](config: AppServiceConfig, None, getAppSamPolicyMap(userEmail, leoEmail, req.accessScope) ) + + // For Galaxy VM apps, check disk attachment before creating a cluster record. Without this + // early check, saveNewClusterForApp would be attempted first; if it succeeded we'd then + // get DiskAlreadyAttachedException from processPersistentDiskRequest, leaving an orphaned + // cluster record. Doing the check here keeps the error path clean. + _ <- if (req.appType == AppType.Galaxy) { + req.diskConfig.flatTraverse { diskReq => + persistentDiskQuery.getActiveByName(cloudContext, diskReq.name).transaction + }.flatMap { + case Some(pd) => + appQuery.isDiskAttached(pd.id).transaction.flatMap { isAttached => + if (isAttached) + F.raiseError[Unit](DiskAlreadyAttachedException(cloudContext, pd.name, ctx.traceId)) + else F.unit + } + case None => F.unit + } + } else F.unit + saveCluster <- F.fromEither( getSavableCluster(userEmail, cloudContext, req.autopilot.isDefined, ctx.now) ) From 6793c51df0f63ff6d9ff4aaef38d178135d34939 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Thu, 30 Jul 2026 12:58:07 -0400 Subject: [PATCH 32/55] style(access-controller): apply scalafmt formatting Co-Authored-By: Claude Sonnet 4.6 --- .../http/service/LeoAppServiceInterp.scala | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/LeoAppServiceInterp.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/LeoAppServiceInterp.scala index 38c15bdc89..fbf78a245c 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/LeoAppServiceInterp.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/LeoAppServiceInterp.scala @@ -171,19 +171,22 @@ final class LeoAppServiceInterp[F[_]: Parallel](config: AppServiceConfig, // early check, saveNewClusterForApp would be attempted first; if it succeeded we'd then // get DiskAlreadyAttachedException from processPersistentDiskRequest, leaving an orphaned // cluster record. Doing the check here keeps the error path clean. - _ <- if (req.appType == AppType.Galaxy) { - req.diskConfig.flatTraverse { diskReq => - persistentDiskQuery.getActiveByName(cloudContext, diskReq.name).transaction - }.flatMap { - case Some(pd) => - appQuery.isDiskAttached(pd.id).transaction.flatMap { isAttached => - if (isAttached) - F.raiseError[Unit](DiskAlreadyAttachedException(cloudContext, pd.name, ctx.traceId)) - else F.unit + _ <- + if (req.appType == AppType.Galaxy) { + req.diskConfig + .flatTraverse { diskReq => + persistentDiskQuery.getActiveByName(cloudContext, diskReq.name).transaction } - case None => F.unit - } - } else F.unit + .flatMap { + case Some(pd) => + appQuery.isDiskAttached(pd.id).transaction.flatMap { isAttached => + if (isAttached) + F.raiseError[Unit](DiskAlreadyAttachedException(cloudContext, pd.name, ctx.traceId)) + else F.unit + } + case None => F.unit + } + } else F.unit saveCluster <- F.fromEither( getSavableCluster(userEmail, cloudContext, req.autopilot.isDefined, ctx.now) From ed0aef136a43827bc7f30061a2151b11776cda6a Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Thu, 30 Jul 2026 13:22:58 -0400 Subject: [PATCH 33/55] test(access-controller): remove unique-constraint test for Galaxy VM cluster support The IDX_KUBERNETES_CLUSTER_UNIQUE_V2 constraint was dropped to allow multiple active cluster records per cloud context for Galaxy VM apps. Co-Authored-By: Claude Sonnet 4.6 --- .../leonardo/db/KubernetesClusterComponentSpec.scala | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/db/KubernetesClusterComponentSpec.scala b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/db/KubernetesClusterComponentSpec.scala index d68914b368..7223e44a1c 100644 --- a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/db/KubernetesClusterComponentSpec.scala +++ b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/db/KubernetesClusterComponentSpec.scala @@ -74,16 +74,6 @@ class KubernetesClusterComponentSpec extends AnyFlatSpecLike with TestComponent savedCluster1.nodepools.size shouldBe 1 } - it should "prevent duplicate (googleProject, destroyedDate) kubernetes clusters" in isolatedDbTest { - val cluster1 = makeKubeCluster(1) - - cluster1.save() - val caught = the[java.sql.SQLIntegrityConstraintViolationException] thrownBy { - cluster1.save() - } - caught.getMessage should include("IDX_KUBERNETES_CLUSTER_UNIQUE") - } - it should "update async fields" in isolatedDbTest { val savedCluster1 = makeKubeCluster(1).save() From 19650481467e2a001695b139205e6cb4a3d364ca Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Fri, 31 Jul 2026 12:33:03 -0400 Subject: [PATCH 34/55] feat(access-controller): pass Terra workspace context to Galaxy VM ansible playbook Supplies terra_workspace, terra_namespace, terra_drs_url, and terra_api_url as ansible extra-vars so galaxy-k8s-boot can configure Galaxy's Terra integration (required by cloudve/galaxy Helm chart 6.8.2). Co-Authored-By: Claude Sonnet 4.6 --- .../resources/init-resources/galaxy-user-data.sh | 13 +++++++++++++ http/src/main/resources/reference.conf | 2 ++ .../dsde/workbench/leonardo/config/Config.scala | 4 +++- .../leonardo/config/KubernetesAppConfig.scala | 4 +++- .../workbench/leonardo/util/GKEInterpreter.scala | 10 ++++++++++ 5 files changed, 31 insertions(+), 2 deletions(-) diff --git a/http/src/main/resources/init-resources/galaxy-user-data.sh b/http/src/main/resources/init-resources/galaxy-user-data.sh index f02fc4d562..b0af66901c 100644 --- a/http/src/main/resources/init-resources/galaxy-user-data.sh +++ b/http/src/main/resources/init-resources/galaxy-user-data.sh @@ -82,6 +82,15 @@ GCP_BATCH_SERVICE_ACCOUNT_EMAIL=$(curl -s -f "http://metadata.google.internal/co -H "Metadata-Flavor: Google" 2>/dev/null || echo "galaxy-batch-runner@anvil-and-terra-development.iam.gserviceaccount.com") echo "[$(date)] - GCP Batch service account email: ${GCP_BATCH_SERVICE_ACCOUNT_EMAIL}" +TERRA_WORKSPACE=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/terra-workspace" \ + -H "Metadata-Flavor: Google" 2>/dev/null || echo "") +TERRA_NAMESPACE=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/terra-namespace" \ + -H "Metadata-Flavor: Google" 2>/dev/null || echo "") +TERRA_DRS_URL=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/terra-drs-url" \ + -H "Metadata-Flavor: Google" 2>/dev/null || echo "") +TERRA_API_URL=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/terra-api-url" \ + -H "Metadata-Flavor: Google" 2>/dev/null || echo "") + # Leo proxy path prefix (e.g. /proxy/google/v1/apps/{project}/{appName}/galaxy). # Passed to ansible-pull as galaxy_prefix so Galaxy nginx ingress is configured at # the correct subpath. Without this Galaxy generates links rooted at / which the @@ -107,6 +116,10 @@ PULL_ARGS=( --accept-host-key --limit 127.0.0.1 --extra-vars "gcp_batch_service_account_email=${GCP_BATCH_SERVICE_ACCOUNT_EMAIL}" + --extra-vars "terra_workspace=${TERRA_WORKSPACE}" + --extra-vars "terra_namespace=${TERRA_NAMESPACE}" + --extra-vars "terra_drs_url=${TERRA_DRS_URL}" + --extra-vars "terra_api_url=${TERRA_API_URL}" ) if [ "$RESTORE_GALAXY" = "true" ]; then diff --git a/http/src/main/resources/reference.conf b/http/src/main/resources/reference.conf index e32b894443..042acd965e 100644 --- a/http/src/main/resources/reference.conf +++ b/http/src/main/resources/reference.conf @@ -446,6 +446,8 @@ galaxyVm { postgresDiskNameSuffix = ${gke.galaxyDisk.postgresDiskNameSuffix} gitRepo = "https://github.com/galaxyproject/galaxy-k8s-boot.git" gitBranch = "anvil" + orchUrl = "https://firecloud-orchestration.dsde-dev.broadinstitute.org/api/" + drsUrl = ${drs.url} } gke { diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/Config.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/Config.scala index 08eb73020d..b32efa8989 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/Config.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/Config.scala @@ -139,7 +139,9 @@ object Config { config.as[DiskSize]("postgresDiskSizeGb"), config.as[String]("postgresDiskNameSuffix"), config.as[String]("gitRepo"), - config.as[String]("gitBranch") + config.as[String]("gitBranch"), + config.as[String]("orchUrl"), + config.as[String]("drsUrl") ) } diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/KubernetesAppConfig.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/KubernetesAppConfig.scala index bd6712279f..8dc3d54a70 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/KubernetesAppConfig.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/KubernetesAppConfig.scala @@ -101,7 +101,9 @@ final case class GalaxyVmConfig( postgresDiskSizeGb: DiskSize, postgresDiskNameSuffix: String, gitRepo: String, - gitBranch: String + gitBranch: String, + orchUrl: String, + drsUrl: String ) final case class ContainerRegistryUsername(asString: String) extends AnyVal diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala index caf2cbba2f..17071ccb1c 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala @@ -1183,6 +1183,16 @@ class GKEInterpreter[F[_]]( .addItems(Items.newBuilder().setKey("gcp-region").setValue(regionParam.value).build()) .addItems(Items.newBuilder().setKey("gcp-network").setValue(network.value).build()) .addItems(Items.newBuilder().setKey("gcp-subnet").setValue(subnetwork.value).build()) + .addItems( + Items + .newBuilder() + .setKey("terra-workspace") + .setValue(app.workspaceId.map(_.value.toString).getOrElse("")) + .build() + ) + .addItems(Items.newBuilder().setKey("terra-namespace").setValue(googleProject.value).build()) + .addItems(Items.newBuilder().setKey("terra-drs-url").setValue(config.galaxyVmConfig.drsUrl).build()) + .addItems(Items.newBuilder().setKey("terra-api-url").setValue(config.galaxyVmConfig.orchUrl).build()) // Galaxy admin user email — used by the post-install job to create the initial Galaxy admin. .addItems( Items.newBuilder().setKey("galaxy-user-email").setValue(app.auditInfo.creator.value).build() From df8ea17fd4c2cbaa6569f1bb162cfa07e469f7d5 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Fri, 31 Jul 2026 15:39:05 -0400 Subject: [PATCH 35/55] fix(access-controller): inject X-Forwarded-Proto: https for Galaxy VM proxy hops Leo terminates TLS and speaks plain HTTP to Galaxy VM ingress-nginx. Without this header, ingress-nginx (use-forwarded-headers: false by default) stamps X-Forwarded-Proto: http on requests to tusd. tusd runs with -behind-proxy and trusts that header, so it generates Location: http://... for TUS upload sessions. Browsers block the subsequent PATCH as mixed content, silently dropping uploads. Adding X-Forwarded-Proto: https only for useHttp=true (Galaxy VM) backends; all other backends use HTTPS natively and are unaffected. The Galaxy team also needs to enable use-forwarded-headers: true in the ingress-nginx configmap so the header reaches tusd, and to update anvil branch to galaxy Helm chart 6.8.2 which fixes the tusd Ingress routing (6.8.1 had an annotation that caused ingress-nginx to silently discard the tusd Ingress). Co-Authored-By: Claude Sonnet 4.6 --- .../leonardo/http/service/ProxyService.scala | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/ProxyService.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/ProxyService.scala index e567fc11b7..68bbf00785 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/ProxyService.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/ProxyService.scala @@ -5,7 +5,7 @@ package service import akka.actor.ActorSystem import akka.http.scaladsl.model.Uri.Host import akka.http.scaladsl.model._ -import akka.http.scaladsl.model.headers.{`Content-Disposition`, OAuth2BearerToken} +import akka.http.scaladsl.model.headers.{`Content-Disposition`, OAuth2BearerToken, RawHeader} import akka.http.scaladsl.model.ws._ import akka.http.scaladsl.settings.ClientConnectionSettings import akka.http.scaladsl.unmarshalling.Unmarshal @@ -459,8 +459,14 @@ class ProxyService( // Rewrite the path if it is proxy/*/*/jupyter/, otherwise pass it through as is (see rewriteJupyterPath) val rewrittenPath = rewriteJupyterPath(request.uri.path) - // 1. filter out headers not needed for the backend server - val newHeaders = filterHeaders(request.headers) + // 1. filter out headers not needed for the backend server, then inject X-Forwarded-Proto. + // Galaxy VM backends receive plain HTTP from Leo (TLS is terminated here), so we + // must declare the original scheme so tusd (running with -behind-proxy) generates + // Location: https://... rather than http://, which browsers block as mixed content. + val filteredHeaders = filterHeaders(request.headers) + val newHeaders = + if (useHttp) filteredHeaders :+ RawHeader("X-Forwarded-Proto", "https") + else filteredHeaders // 2. strip out Uri.Authority: val newUri = Uri(path = rewrittenPath, queryString = request.uri.rawQueryString) // 3. build a new HttpRequest From 4de192f35ffea6a3b60c0f5f70b379a3a6709181 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Mon, 3 Aug 2026 09:26:06 -0400 Subject: [PATCH 36/55] fix(access-controller): enable ingress-nginx use-forwarded-headers for Galaxy VM Leo terminates TLS and speaks plain HTTP to the Galaxy VM. ingress-nginx must be told to trust Leo's X-Forwarded-Proto header (now set to "https" by ProxyService) rather than deriving the scheme from its own plaintext connection, so tusd receives X-Forwarded-Proto: https and generates Location: https://... for TUS upload sessions. The galaxy-k8s-boot anvil branch exposes this as ingress_use_forwarded_headers (default false for non-Leo deployments); we set it to true via ansible-pull extra-vars. Co-Authored-By: Claude Sonnet 4.6 --- http/src/main/resources/init-resources/galaxy-user-data.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/http/src/main/resources/init-resources/galaxy-user-data.sh b/http/src/main/resources/init-resources/galaxy-user-data.sh index b0af66901c..57c7432fc7 100644 --- a/http/src/main/resources/init-resources/galaxy-user-data.sh +++ b/http/src/main/resources/init-resources/galaxy-user-data.sh @@ -120,6 +120,7 @@ PULL_ARGS=( --extra-vars "terra_namespace=${TERRA_NAMESPACE}" --extra-vars "terra_drs_url=${TERRA_DRS_URL}" --extra-vars "terra_api_url=${TERRA_API_URL}" + --extra-vars "ingress_use_forwarded_headers=true" ) if [ "$RESTORE_GALAXY" = "true" ]; then From 5ec07bed8a1ba9c4b2d5a2b17f2f0444711e7610 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Mon, 3 Aug 2026 13:21:59 -0400 Subject: [PATCH 37/55] fix(access-controller): rewrite http Location headers to https for Galaxy VM responses Leo terminates TLS and speaks plain HTTP to the Galaxy VM, so tusd's `-behind-proxy` flag generates `Location: http://...` URLs for TUS uploads. The browser blocks the subsequent PATCH as mixed content from an https page. Rewrite Location scheme to https for all useHttp=true (Galaxy VM) backends in the proxy response path. Co-Authored-By: Claude Sonnet 4.6 --- .../leonardo/http/service/ProxyService.scala | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/ProxyService.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/ProxyService.scala index 68bbf00785..5768b18c45 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/ProxyService.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/ProxyService.scala @@ -5,7 +5,7 @@ package service import akka.actor.ActorSystem import akka.http.scaladsl.model.Uri.Host import akka.http.scaladsl.model._ -import akka.http.scaladsl.model.headers.{`Content-Disposition`, OAuth2BearerToken, RawHeader} +import akka.http.scaladsl.model.headers.{`Content-Disposition`, Location, OAuth2BearerToken, RawHeader} import akka.http.scaladsl.model.ws._ import akka.http.scaladsl.settings.ClientConnectionSettings import akka.http.scaladsl.unmarshalling.Unmarshal @@ -485,11 +485,25 @@ class ProxyService( Source .single(newRequest) .via(flow) + .map(if (useHttp) fixLocationScheme else identity) .map(fixContentDisposition) .runWith(Sink.head) .flatMap(_.toStrict(requestTimeout)) } + // Galaxy VM backends speak HTTP to Leo, which terminates TLS for the client. Any + // absolute URL they return must use https:// so the browser can follow it from an + // https:// page without a mixed-content block. The primary case is the TUS upload + // Location header generated by tusd with -behind-proxy. + private def fixLocationScheme(httpResponse: HttpResponse): HttpResponse = + httpResponse.header[Location] match { + case Some(loc) if loc.uri.scheme == "http" => + val newHeaders = + httpResponse.headers.filterNot(_.isInstanceOf[Location]) :+ Location(loc.uri.withScheme("https")) + httpResponse.withHeaders(newHeaders) + case _ => httpResponse + } + // This is our current workaround for a bug that causes notebooks to download with "utf-8''" prepended to the file name // This is due to an akka-http bug currently being worked on here: https://github.com/playframework/playframework/issues/7719 // For now, for each response that has a Content-Disposition header with a 'filename' in the header params, we remove any "utf-8''". From 042aca3248a0c0993256e73f4ac230b6ebc0762a Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Mon, 3 Aug 2026 13:23:56 -0400 Subject: [PATCH 38/55] fix(access-controller): pass GCP project ID to Galaxy VM ansible for Batch jobs GCP Batch jobs were 403-ing because galaxy-k8s-boot values.yml hardcodes project_id: anvil-and-terra-development. Pass the actual VM project (already available as terra-namespace metadata) to ansible-pull as gcp_project_id so galaxy-k8s-boot can use it to set the correct project. The galaxy-k8s-boot anvil branch also needs to accept this variable and use it to override project_id in the Helm values. Co-Authored-By: Claude Sonnet 4.6 --- http/src/main/resources/init-resources/galaxy-user-data.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/http/src/main/resources/init-resources/galaxy-user-data.sh b/http/src/main/resources/init-resources/galaxy-user-data.sh index 57c7432fc7..f6e9c2c55f 100644 --- a/http/src/main/resources/init-resources/galaxy-user-data.sh +++ b/http/src/main/resources/init-resources/galaxy-user-data.sh @@ -121,6 +121,7 @@ PULL_ARGS=( --extra-vars "terra_drs_url=${TERRA_DRS_URL}" --extra-vars "terra_api_url=${TERRA_API_URL}" --extra-vars "ingress_use_forwarded_headers=true" + --extra-vars "gcp_project_id=${TERRA_NAMESPACE}" ) if [ "$RESTORE_GALAXY" = "true" ]; then From c8aaee9199544f1d731c444bf2fe46a88cef4e53 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Mon, 3 Aug 2026 13:52:28 -0400 Subject: [PATCH 39/55] fix(access-controller): forward workspace name to Galaxy VM for Terra file source label Terra passes WORKSPACE_NAME in customEnvironmentVariables (already used for disk restore validation). Forward it to the VM as terra-workspace-name metadata and pass to ansible-pull as terra_workspace_name so galaxy-k8s-boot can populate the workspace: field in the anvil file source config. Without this the field is empty, causing Galaxy to show "Unlabeled Rfs File Source" and fail to list workspace files. galaxy-k8s-boot also needs a matching change to use terra_workspace_name for the workspace: field. Co-Authored-By: Claude Sonnet 4.6 --- .../dsde/workbench/leonardo/util/GKEInterpreter.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala index 17071ccb1c..e8b58ee6a0 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala @@ -1187,7 +1187,7 @@ class GKEInterpreter[F[_]]( Items .newBuilder() .setKey("terra-workspace") - .setValue(app.workspaceId.map(_.value.toString).getOrElse("")) + .setValue(app.customEnvironmentVariables.getOrElse(WORKSPACE_NAME_KEY, "")) .build() ) .addItems(Items.newBuilder().setKey("terra-namespace").setValue(googleProject.value).build()) From a8475474f1fb4c1a32a9f326c1107df4ee29a9cd Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Mon, 3 Aug 2026 15:01:43 -0400 Subject: [PATCH 40/55] fix(access-controller): wire ORCH_URL env var to galaxyVm.orchUrl config ORCH_URL was already mapped to gke.galaxyApp.orchUrl but not to galaxyVm.orchUrl, so the Galaxy VM Terra file source api_url always defaulted to the dsde-dev hardcoded value in reference.conf. No helmfile change needed; terra-helmfile already sets ORCH_URL per environment. Co-Authored-By: Claude Sonnet 4.6 --- http/src/main/resources/leo.conf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/http/src/main/resources/leo.conf b/http/src/main/resources/leo.conf index 0727a64f07..b303020dc9 100644 --- a/http/src/main/resources/leo.conf +++ b/http/src/main/resources/leo.conf @@ -23,6 +23,10 @@ gce { } } +galaxyVm { + orchUrl = ${?ORCH_URL} +} + gke { cluster { version = ${?KUBERNETES_VERSION} From 399a43ca1f8ab952fb591dc75244e446735a8f00 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Tue, 4 Aug 2026 10:18:44 -0400 Subject: [PATCH 41/55] fix(access-controller): allow Galaxy app recreate when prior disk has no restore info A Galaxy VM disk gets formattedBy=Galaxy when created but lastUsedBy is only written after a successful install. If the previous app errored during provisioning, the disk has no data and should be reusable for a fresh install rather than returning a 500. Co-Authored-By: Claude Sonnet 4.6 --- .../leonardo/http/service/LeoAppServiceInterp.scala | 8 ++++++-- .../leonardo/http/service/AppServiceInterpSpec.scala | 6 ++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/LeoAppServiceInterp.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/LeoAppServiceInterp.scala index fbf78a245c..dc520a842c 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/LeoAppServiceInterp.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/LeoAppServiceInterp.scala @@ -879,8 +879,12 @@ final class LeoAppServiceInterp[F[_]: Parallel](config: AppServiceConfig, ) } } yield lastUsed.some - case (Some(FormattedBy.Galaxy), None) | (Some(FormattedBy.Cromwell), None) | - (Some(FormattedBy.Allowed), None) => + // Galaxy VM writes restore info only after a successful install. If an app + // failed during creation before that point, the disk has no data and can + // be reused as a fresh install. + case (Some(FormattedBy.Galaxy), None) => + F.pure(none[LastUsedApp]) + case (Some(FormattedBy.Cromwell), None) | (Some(FormattedBy.Allowed), None) => F.raiseError[Option[LastUsedApp]]( new LeoException(s"Existing ${diskResult.disk.id} found, but no restore info found in DB", traceId = Some(ctx.traceId) diff --git a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/AppServiceInterpSpec.scala b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/AppServiceInterpSpec.scala index 6ea9223615..e11f873171 100644 --- a/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/AppServiceInterpSpec.scala +++ b/http/src/test/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/AppServiceInterpSpec.scala @@ -635,7 +635,9 @@ class AppServiceInterpTest extends AnyFlatSpec with AppServiceInterpSpec with Le publisherQueue.tryTake.unsafeRunSync()(cats.effect.unsafe.IORuntime.global).get shouldBe a[DeleteAppMessage] } - it should "error creating an app with an existing disk if no restore info found" in isolatedDbTest { + it should "allow creating a Galaxy app with an existing disk that has no restore info (failed provisioning case)" in isolatedDbTest { + // Galaxy writes restore info only after a successful install. If the previous app + // failed during creation, the disk has no data and should be reusable as a fresh install. val disk = makePersistentDisk(None, formattedBy = Some(FormattedBy.Galaxy)) .copy(cloudContext = cloudContextGcp) .save() @@ -652,7 +654,7 @@ class AppServiceInterpTest extends AnyFlatSpec with AppServiceInterpSpec with Le .attempt .unsafeRunSync()(cats.effect.unsafe.IORuntime.global) - res.swap.toOption.get.getMessage shouldBe s"Existing ${disk.id} found, but no restore info found in DB" + res shouldBe a[Right[_, _]] } it should "error on creation of a galaxy app without a disk" in isolatedDbTest { From b509b6dce106f5e8cf58c3b42924f9f29b24b4e7 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Tue, 4 Aug 2026 15:09:19 -0400 Subject: [PATCH 42/55] fix(access-controller): pass correct Terra billing namespace and GCP project to Galaxy VM terra-namespace was being set to the GCP project (e.g. terra-quality-50d1be3e) instead of the Terra billing namespace (e.g. broad-dsp-liz). The startup script was using terra-namespace for both gcp_project_id and terra_namespace in ansible, conflating two distinct values. Fix: terra-namespace now carries the Terra billing namespace (from WORKSPACE_NAMESPACE custom env var). A new gcp-project-id metadata key carries the GCP project, and the startup script reads it separately for gcp_project_id. This fixes the "Problem listing file source path gxfiles://terra-launch-workspace/" error in Galaxy. Co-Authored-By: Claude Sonnet 4.6 --- .../main/resources/init-resources/galaxy-user-data.sh | 4 +++- .../dsde/workbench/leonardo/http/package.scala | 1 + .../dsde/workbench/leonardo/util/GKEInterpreter.scala | 9 ++++++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/http/src/main/resources/init-resources/galaxy-user-data.sh b/http/src/main/resources/init-resources/galaxy-user-data.sh index f6e9c2c55f..1ac1e9051b 100644 --- a/http/src/main/resources/init-resources/galaxy-user-data.sh +++ b/http/src/main/resources/init-resources/galaxy-user-data.sh @@ -86,6 +86,8 @@ TERRA_WORKSPACE=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1 -H "Metadata-Flavor: Google" 2>/dev/null || echo "") TERRA_NAMESPACE=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/terra-namespace" \ -H "Metadata-Flavor: Google" 2>/dev/null || echo "") +GCP_PROJECT_ID=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/gcp-project-id" \ + -H "Metadata-Flavor: Google" 2>/dev/null || echo "") TERRA_DRS_URL=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/terra-drs-url" \ -H "Metadata-Flavor: Google" 2>/dev/null || echo "") TERRA_API_URL=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/terra-api-url" \ @@ -121,7 +123,7 @@ PULL_ARGS=( --extra-vars "terra_drs_url=${TERRA_DRS_URL}" --extra-vars "terra_api_url=${TERRA_API_URL}" --extra-vars "ingress_use_forwarded_headers=true" - --extra-vars "gcp_project_id=${TERRA_NAMESPACE}" + --extra-vars "gcp_project_id=${GCP_PROJECT_ID}" ) if [ "$RESTORE_GALAXY" = "true" ]; then diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/package.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/package.scala index 41dee5ec42..d14ac37a87 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/package.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/package.scala @@ -34,6 +34,7 @@ package object http { val creatorOnlyValue = SamRole.Creator.asString val bucketPathMaxLength = 1024 val WORKSPACE_NAME_KEY = "WORKSPACE_NAME" + val WORKSPACE_NAMESPACE_KEY = "WORKSPACE_NAMESPACE" implicit val errorReportSource: ErrorReportSource = ErrorReportSource("leonardo") implicit def dbioToIO[A](dbio: DBIO[A]): DBIOOps[A] = new DBIOOps(dbio) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala index e8b58ee6a0..1ee5433952 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala @@ -1190,7 +1190,14 @@ class GKEInterpreter[F[_]]( .setValue(app.customEnvironmentVariables.getOrElse(WORKSPACE_NAME_KEY, "")) .build() ) - .addItems(Items.newBuilder().setKey("terra-namespace").setValue(googleProject.value).build()) + .addItems( + Items + .newBuilder() + .setKey("terra-namespace") + .setValue(app.customEnvironmentVariables.getOrElse(WORKSPACE_NAMESPACE_KEY, "")) + .build() + ) + .addItems(Items.newBuilder().setKey("gcp-project-id").setValue(googleProject.value).build()) .addItems(Items.newBuilder().setKey("terra-drs-url").setValue(config.galaxyVmConfig.drsUrl).build()) .addItems(Items.newBuilder().setKey("terra-api-url").setValue(config.galaxyVmConfig.orchUrl).build()) // Galaxy admin user email — used by the post-install job to create the initial Galaxy admin. From c4c4c683b387cffdbd3a11cc5cb9799e7fe34e8f Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Wed, 5 Aug 2026 09:17:47 -0400 Subject: [PATCH 43/55] fix(access-controller): pass galaxy_infrastructure_url so Galaxy generates https:// links Without this, Galaxy infers http:// from its internal Leo connection and generates http:// absolute URLs (e.g. history export links). Browsers refuse to send the Secure-flagged LeoToken cookie on http:// requests, causing 401s. Co-Authored-By: Claude Sonnet 4.6 --- .../resources/init-resources/galaxy-user-data.sh | 13 +++++++++++++ .../workbench/leonardo/util/GKEInterpreter.scala | 11 +++++++++++ 2 files changed, 24 insertions(+) diff --git a/http/src/main/resources/init-resources/galaxy-user-data.sh b/http/src/main/resources/init-resources/galaxy-user-data.sh index 1ac1e9051b..6f296be8cf 100644 --- a/http/src/main/resources/init-resources/galaxy-user-data.sh +++ b/http/src/main/resources/init-resources/galaxy-user-data.sh @@ -101,6 +101,14 @@ GALAXY_URL_PREFIX=$(curl -s -f "http://metadata.google.internal/computeMetadata/ -H "Metadata-Flavor: Google" 2>/dev/null || echo "") echo "[$(date)] - Galaxy URL prefix: ${GALAXY_URL_PREFIX}" +GALAXY_PROXY_BASE_URL=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/galaxy-proxy-base-url" \ + -H "Metadata-Flavor: Google" 2>/dev/null || echo "") +# Full HTTPS URL Galaxy uses to generate absolute links (history exports, API callbacks, etc.). +# Without this Galaxy infers http:// from its internal connection and produces links that +# browsers refuse to send the Secure LeoToken cookie on. +GALAXY_INFRASTRUCTURE_URL="${GALAXY_PROXY_BASE_URL}${GALAXY_URL_PREFIX}" +echo "[$(date)] - Galaxy infrastructure URL: ${GALAXY_INFRASTRUCTURE_URL}" + GALAXY_USER=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/galaxy-user-email" \ -H "Metadata-Flavor: Google" 2>/dev/null || echo "") echo "[$(date)] - Galaxy user email: ${GALAXY_USER}" @@ -138,6 +146,11 @@ if [ -n "$GALAXY_URL_PREFIX" ]; then echo "[$(date)] - Galaxy URL prefix passed to ansible: ${GALAXY_URL_PREFIX}" fi +if [ -n "$GALAXY_INFRASTRUCTURE_URL" ]; then + PULL_ARGS+=(--extra-vars "galaxy_infrastructure_url=${GALAXY_INFRASTRUCTURE_URL}") + echo "[$(date)] - Galaxy infrastructure URL passed to ansible: ${GALAXY_INFRASTRUCTURE_URL}" +fi + PULL_ARGS+=(playbook.yml) mkdir -p /tmp/ansible-inventory diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala index 1ee5433952..838dd67229 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala @@ -1214,6 +1214,17 @@ class GKEInterpreter[F[_]]( .setValue(galaxyUrlPrefix) .build() ) + // Full HTTPS base URL of the Leo proxy (scheme + host). Combined with galaxy-url-prefix + // in the startup script to form galaxy_infrastructure_url, which tells Galaxy to generate + // https:// absolute links. Without this Galaxy uses its internal http:// connection and + // produces http:// links (e.g. history export URLs) that browsers reject for the Secure cookie. + .addItems( + Items + .newBuilder() + .setKey("galaxy-proxy-base-url") + .setValue(config.leoUrlBase.toString.stripSuffix("/")) + .build() + ) .build() ) .putAllLabels(Map("leonardo" -> "true").asJava) From 4270d1709b8ed7d34399ecef4ba364be1584745a Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Wed, 5 Aug 2026 12:08:40 -0400 Subject: [PATCH 44/55] fix(access-controller): accept Origin header as Referer fallback for viz plugin requests Galaxy's visualization plugin loader (analysis.bundled.js) fetches index.js and index.css without a Referer header, causing Leo's checkReferer to 401 before the LeoToken cookie is ever checked. The Origin header is present on these requests and provides equivalent CSRF protection (browsers set it; HTML forms cannot forge it). Co-Authored-By: Claude Sonnet 4.6 --- .../leonardo/http/api/ProxyRoutes.scala | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/api/ProxyRoutes.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/api/ProxyRoutes.scala index f4e4e8cc5b..380cb5fd4f 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/api/ProxyRoutes.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/api/ProxyRoutes.scala @@ -293,7 +293,24 @@ class ProxyRoutes(proxyService: ProxyService, corsSupport: CorsSupport, refererC logRequestPath.tflatMap(_ => failWith(AuthenticationError())) } case None => - logger.info(s"Referer header is missing") - logRequestPath.tflatMap(_ => failWith(AuthenticationError())) + // Referer is absent — check the Origin header as a fallback. Browsers send Origin + // on fetch/module-import requests (e.g. Galaxy visualization plugin assets loaded + // by analysis.bundled.js) even when Referer is stripped by the referrer policy. + // Origin provides equivalent CSRF protection: it cannot be forged by HTML forms + // and is set to the initiating document's origin by the browser. + optionalHeaderValueByType(`Origin`) flatMap { + case Some(origin) => + val hasValidOrigin = origin.origins.exists { o => + refererConfig.validHosts.contains(o.host.toString()) || refererConfig.validHosts.contains("*") + } + if (hasValidOrigin) pass + else { + logger.info(s"Referer header is missing and Origin ${origin.value} is not allowed") + logRequestPath.tflatMap(_ => failWith(AuthenticationError())) + } + case None => + logger.info(s"Referer header is missing") + logRequestPath.tflatMap(_ => failWith(AuthenticationError())) + } } } From b242aac15fe8457d408c441509d503f1a98da14b Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Wed, 5 Aug 2026 12:13:34 -0400 Subject: [PATCH 45/55] fix(access-controller): pass VPC network/subnet to Galaxy ansible for GCP Batch GCP Batch jobs fail with CODE_GCE_RESOURCE_NOT_FOUND because galaxy-k8s-boot hardcodes network/subnet to 'default', which doesn't exist in Terra projects. Read gcp-network and gcp-subnet from VM instance metadata (already set by Leo) and forward as gcp_batch_network and gcp_batch_subnet extra-vars so galaxy-k8s-boot can configure the Batch runner to use the correct VPC. Co-Authored-By: Claude Sonnet 4.6 --- .../resources/init-resources/galaxy-user-data.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/http/src/main/resources/init-resources/galaxy-user-data.sh b/http/src/main/resources/init-resources/galaxy-user-data.sh index 6f296be8cf..db5d45d98e 100644 --- a/http/src/main/resources/init-resources/galaxy-user-data.sh +++ b/http/src/main/resources/init-resources/galaxy-user-data.sh @@ -88,6 +88,11 @@ TERRA_NAMESPACE=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1 -H "Metadata-Flavor: Google" 2>/dev/null || echo "") GCP_PROJECT_ID=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/gcp-project-id" \ -H "Metadata-Flavor: Google" 2>/dev/null || echo "") +GCP_NETWORK=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/gcp-network" \ + -H "Metadata-Flavor: Google" 2>/dev/null || echo "") +GCP_SUBNET=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/gcp-subnet" \ + -H "Metadata-Flavor: Google" 2>/dev/null || echo "") +echo "[$(date)] - GCP network: ${GCP_NETWORK}, subnet: ${GCP_SUBNET}" TERRA_DRS_URL=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/terra-drs-url" \ -H "Metadata-Flavor: Google" 2>/dev/null || echo "") TERRA_API_URL=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/terra-api-url" \ @@ -134,6 +139,16 @@ PULL_ARGS=( --extra-vars "gcp_project_id=${GCP_PROJECT_ID}" ) +if [ -n "$GCP_NETWORK" ]; then + PULL_ARGS+=(--extra-vars "gcp_batch_network=${GCP_NETWORK}") + echo "[$(date)] - GCP Batch network passed to ansible: ${GCP_NETWORK}" +fi + +if [ -n "$GCP_SUBNET" ]; then + PULL_ARGS+=(--extra-vars "gcp_batch_subnet=${GCP_SUBNET}") + echo "[$(date)] - GCP Batch subnet passed to ansible: ${GCP_SUBNET}" +fi + if [ "$RESTORE_GALAXY" = "true" ]; then PULL_ARGS+=(--extra-vars "restore_galaxy=true") echo "[$(date)] - Galaxy Restore Mode: Enabled" From ec1ed643737836f32bb2f69b1b62e62b01028901 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Wed, 5 Aug 2026 12:34:04 -0400 Subject: [PATCH 46/55] fix(access-controller): add userinfo scopes to Galaxy GCE VM service account Without `userinfo.email` and `userinfo.profile` scopes, the VM pet SA token cannot be validated by Terra's Sam (Google's userinfo endpoint won't return the caller's email), causing all `anvilfs` calls to Rawls/Orchestration to fail with 401 Unauthorized. Co-Authored-By: Claude Sonnet 4.6 --- .../dsde/workbench/leonardo/util/GKEInterpreter.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala index 838dd67229..1aae89fb96 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala @@ -1165,7 +1165,9 @@ class GKEInterpreter[F[_]]( .addAllScopes( List( "https://www.googleapis.com/auth/cloud-platform", - "https://www.googleapis.com/auth/logging.write" + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/userinfo.profile" ).asJava ) .build() From 3acc4d4198c8325733d621017ba5e5c383e9cfd8 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Wed, 5 Aug 2026 14:54:52 -0400 Subject: [PATCH 47/55] fix(access-controller): pass GCP Batch boot image as VM metadata to ansible GCP Batch was looking for the boot image by name in the job's project (terra-quality-ff0f1af3), but the image lives in anvil-and-terra-development. Passes the full image resource URL from Leo config as `gcp-batch-boot-image` VM metadata so galaxy-k8s-boot can supply it to the Batch runner config. Co-Authored-By: Claude Sonnet 4.6 --- .../src/main/resources/init-resources/galaxy-user-data.sh | 8 ++++++++ .../dsde/workbench/leonardo/util/GKEInterpreter.scala | 1 + 2 files changed, 9 insertions(+) diff --git a/http/src/main/resources/init-resources/galaxy-user-data.sh b/http/src/main/resources/init-resources/galaxy-user-data.sh index db5d45d98e..c5d860b1a4 100644 --- a/http/src/main/resources/init-resources/galaxy-user-data.sh +++ b/http/src/main/resources/init-resources/galaxy-user-data.sh @@ -93,6 +93,9 @@ GCP_NETWORK=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/ins GCP_SUBNET=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/gcp-subnet" \ -H "Metadata-Flavor: Google" 2>/dev/null || echo "") echo "[$(date)] - GCP network: ${GCP_NETWORK}, subnet: ${GCP_SUBNET}" +GCP_BATCH_BOOT_IMAGE=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/gcp-batch-boot-image" \ + -H "Metadata-Flavor: Google" 2>/dev/null || echo "") +echo "[$(date)] - GCP Batch boot image: ${GCP_BATCH_BOOT_IMAGE}" TERRA_DRS_URL=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/terra-drs-url" \ -H "Metadata-Flavor: Google" 2>/dev/null || echo "") TERRA_API_URL=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/terra-api-url" \ @@ -149,6 +152,11 @@ if [ -n "$GCP_SUBNET" ]; then echo "[$(date)] - GCP Batch subnet passed to ansible: ${GCP_SUBNET}" fi +if [ -n "$GCP_BATCH_BOOT_IMAGE" ]; then + PULL_ARGS+=(--extra-vars "gcp_batch_boot_image=${GCP_BATCH_BOOT_IMAGE}") + echo "[$(date)] - GCP Batch boot image passed to ansible: ${GCP_BATCH_BOOT_IMAGE}" +fi + if [ "$RESTORE_GALAXY" = "true" ]; then PULL_ARGS+=(--extra-vars "restore_galaxy=true") echo "[$(date)] - Galaxy Restore Mode: Enabled" diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala index 1aae89fb96..23efe8b95d 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala @@ -1185,6 +1185,7 @@ class GKEInterpreter[F[_]]( .addItems(Items.newBuilder().setKey("gcp-region").setValue(regionParam.value).build()) .addItems(Items.newBuilder().setKey("gcp-network").setValue(network.value).build()) .addItems(Items.newBuilder().setKey("gcp-subnet").setValue(subnetwork.value).build()) + .addItems(Items.newBuilder().setKey("gcp-batch-boot-image").setValue(config.galaxyVmConfig.sourceImage.asString).build()) .addItems( Items .newBuilder() From 0aa3deaf1fc1f00df159c518148c99773f22b2ee Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Wed, 5 Aug 2026 15:02:39 -0400 Subject: [PATCH 48/55] fix(access-controller): scalafmt Co-Authored-By: Claude Sonnet 4.6 --- .../dsde/workbench/leonardo/util/GKEInterpreter.scala | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala index 23efe8b95d..3193b9ce6c 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala @@ -1185,7 +1185,13 @@ class GKEInterpreter[F[_]]( .addItems(Items.newBuilder().setKey("gcp-region").setValue(regionParam.value).build()) .addItems(Items.newBuilder().setKey("gcp-network").setValue(network.value).build()) .addItems(Items.newBuilder().setKey("gcp-subnet").setValue(subnetwork.value).build()) - .addItems(Items.newBuilder().setKey("gcp-batch-boot-image").setValue(config.galaxyVmConfig.sourceImage.asString).build()) + .addItems( + Items + .newBuilder() + .setKey("gcp-batch-boot-image") + .setValue(config.galaxyVmConfig.sourceImage.asString) + .build() + ) .addItems( Items .newBuilder() From 661b35a06ca93fd4833324e3e8b0fddf8d356e12 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Wed, 5 Aug 2026 15:51:57 -0400 Subject: [PATCH 49/55] revert(access-controller): remove gcp-batch-boot-image metadata Galaxy team is fixing the image path regression in galaxy-k8s-boot directly; passing it from Leo is not needed. Co-Authored-By: Claude Sonnet 4.6 --- .../dsde/workbench/leonardo/util/GKEInterpreter.scala | 7 ------- 1 file changed, 7 deletions(-) diff --git a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala index 3193b9ce6c..1aae89fb96 100644 --- a/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala +++ b/http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/util/GKEInterpreter.scala @@ -1185,13 +1185,6 @@ class GKEInterpreter[F[_]]( .addItems(Items.newBuilder().setKey("gcp-region").setValue(regionParam.value).build()) .addItems(Items.newBuilder().setKey("gcp-network").setValue(network.value).build()) .addItems(Items.newBuilder().setKey("gcp-subnet").setValue(subnetwork.value).build()) - .addItems( - Items - .newBuilder() - .setKey("gcp-batch-boot-image") - .setValue(config.galaxyVmConfig.sourceImage.asString) - .build() - ) .addItems( Items .newBuilder() From 4706f1048e1901c9db7f3d34b4a8b79d7bf70be2 Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Wed, 5 Aug 2026 15:58:43 -0400 Subject: [PATCH 50/55] fix(access-controller): quote terra_workspace/namespace as JSON extra-vars Ansible splits --extra-vars key=value on whitespace, so workspace names with spaces (e.g. "Galaxy Testing Party - 080426") were silently truncated to the first word. JSON format preserves the full value. Co-Authored-By: Claude Sonnet 4.6 --- http/src/main/resources/init-resources/galaxy-user-data.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/http/src/main/resources/init-resources/galaxy-user-data.sh b/http/src/main/resources/init-resources/galaxy-user-data.sh index c5d860b1a4..a36e840dd9 100644 --- a/http/src/main/resources/init-resources/galaxy-user-data.sh +++ b/http/src/main/resources/init-resources/galaxy-user-data.sh @@ -134,8 +134,7 @@ PULL_ARGS=( --accept-host-key --limit 127.0.0.1 --extra-vars "gcp_batch_service_account_email=${GCP_BATCH_SERVICE_ACCOUNT_EMAIL}" - --extra-vars "terra_workspace=${TERRA_WORKSPACE}" - --extra-vars "terra_namespace=${TERRA_NAMESPACE}" + --extra-vars "{\"terra_workspace\": \"${TERRA_WORKSPACE}\", \"terra_namespace\": \"${TERRA_NAMESPACE}\"}" --extra-vars "terra_drs_url=${TERRA_DRS_URL}" --extra-vars "terra_api_url=${TERRA_API_URL}" --extra-vars "ingress_use_forwarded_headers=true" From 334d3b0112b681c1deff10833a9aad3c6bf87f7a Mon Sep 17 00:00:00 2001 From: Liz Baldo Date: Thu, 6 Aug 2026 12:16:37 -0400 Subject: [PATCH 51/55] fix(access-controller): allow same-origin requests with no Referer or Origin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Galaxy visualization plugin assets (CSS, JS loaded by index.js) are fetched via no-cors /