CTM-397: Replace GKE/Helm Galaxy path with GCE VM-based deployment

http/src/main/resources/init-resources/galaxy-user-data.sh

@@ -0,0 +1,200 @@
+#!/bin/bash
+# Sourced from https://github.com/galaxyproject/galaxy-k8s-boot/blob/anvil/bin/user_data.sh
+# When updating this file, sync it manually from that repository and verify the changes.
+# Passed to the GCE instance as the "startup-script" metadata key so the Guest Agent
+# executes it on every boot (cloud-init user-data is skipped on pre-baked images).
+
+echo "[$(date)] - Starting galaxy_bootstrap script..."
+
+# 1. Setup persistent disk if available
+DISK_DEVICE="/dev/disk/by-id/google-galaxy-data"
+if [ -b "$DISK_DEVICE" ]; then
+  echo "[$(date)] - Found persistent disk at $DISK_DEVICE"
+
+  if ! blkid "$DISK_DEVICE" > /dev/null 2>&1; then
+    echo "[$(date)] - Formatting disk $DISK_DEVICE with ext4"
+    mkfs -t ext4 "$DISK_DEVICE"
+  else
+    echo "[$(date)] - Disk $DISK_DEVICE is already formatted"
+  fi
+
+  mkdir -p /mnt/block_storage
+  mount "$DISK_DEVICE" /mnt/block_storage
+
+  DISK_UUID=$(blkid -s UUID -o value "$DISK_DEVICE")
+  if [ -n "$DISK_UUID" ] && ! grep -q "$DISK_UUID" /etc/fstab; then
+    echo "UUID=$DISK_UUID /mnt/block_storage ext4 defaults 0 2" >> /etc/fstab
+  fi
+
+  chown debian:debian /mnt/block_storage
+  echo "[$(date)] - Persistent disk mounted at /mnt/block_storage"
+else
+  echo "[$(date)] - No persistent disk found at $DISK_DEVICE. Galaxy will use ephemeral storage."
+fi
+
+# 2. Setup PostgreSQL disk if available
+POSTGRES_DISK_DEVICE="/dev/disk/by-id/google-galaxy-postgres-data"
+if [ -b "$POSTGRES_DISK_DEVICE" ]; then
+  echo "[$(date)] - Found PostgreSQL disk at $POSTGRES_DISK_DEVICE"
+
+  if ! blkid "$POSTGRES_DISK_DEVICE" > /dev/null 2>&1; then
+    echo "[$(date)] - Formatting PostgreSQL disk $POSTGRES_DISK_DEVICE with ext4"
+    mkfs -t ext4 "$POSTGRES_DISK_DEVICE"
+  else
+    echo "[$(date)] - PostgreSQL disk $POSTGRES_DISK_DEVICE is already formatted"
+  fi
+
+  mkdir -p /mnt/postgres_storage
+  mount "$POSTGRES_DISK_DEVICE" /mnt/postgres_storage
+
+  POSTGRES_DISK_UUID=$(blkid -s UUID -o value "$POSTGRES_DISK_DEVICE")
+  if [ -n "$POSTGRES_DISK_UUID" ] && ! grep -q "$POSTGRES_DISK_UUID" /etc/fstab; then
+    echo "UUID=$POSTGRES_DISK_UUID /mnt/postgres_storage ext4 defaults 0 2" >> /etc/fstab
+  fi
+
+  chown debian:debian /mnt/postgres_storage
+  echo "[$(date)] - PostgreSQL disk mounted at /mnt/postgres_storage"
+else
+  echo "[$(date)] - No PostgreSQL disk found at $POSTGRES_DISK_DEVICE. PostgreSQL will use ephemeral storage."
+fi
+
+# 3. Run ansible-pull as the debian user.
+# Single-quoted heredoc delimiter means the outer (root) shell does not expand any
+# variables — all expansion happens inside the debian bash session.
+sudo -u debian bash <<'DEBIAN_EOF'
+export HOME=/home/debian
+
+HOST_IP=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip" \
+  -H "Metadata-Flavor: Google" 2>/dev/null || curl -s ifconfig.me)
+
+PV_SIZE=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/persistent-volume-size" \
+  -H "Metadata-Flavor: Google" 2>/dev/null)
+if [ -z "$PV_SIZE" ]; then
+    echo "[$(date)] - persistent-volume-size metadata not found or empty, using default."
+    PV_SIZE="139Gi"
+fi
+echo "[$(date)] - NFS storage size for Galaxy: ${PV_SIZE}"
+
+RESTORE_GALAXY=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/restore_galaxy" \
+  -H "Metadata-Flavor: Google" 2>/dev/null || echo "false")
+
+GCP_BATCH_SERVICE_ACCOUNT_EMAIL=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/gcp_batch_service_account_email" \
+  -H "Metadata-Flavor: Google" 2>/dev/null || echo "galaxy-batch-runner@anvil-and-terra-development.iam.gserviceaccount.com")
+echo "[$(date)] - GCP Batch service account email: ${GCP_BATCH_SERVICE_ACCOUNT_EMAIL}"
+
+TERRA_WORKSPACE=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/terra-workspace" \
+  -H "Metadata-Flavor: Google" 2>/dev/null || echo "")
+TERRA_NAMESPACE=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/terra-namespace" \
+  -H "Metadata-Flavor: Google" 2>/dev/null || echo "")
+GCP_PROJECT_ID=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/gcp-project-id" \
+  -H "Metadata-Flavor: Google" 2>/dev/null || echo "")
+GCP_NETWORK=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/gcp-network" \
+  -H "Metadata-Flavor: Google" 2>/dev/null || echo "")
+GCP_SUBNET=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/gcp-subnet" \
+  -H "Metadata-Flavor: Google" 2>/dev/null || echo "")
+echo "[$(date)] - GCP network: ${GCP_NETWORK}, subnet: ${GCP_SUBNET}"
+GCP_BATCH_BOOT_IMAGE=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/gcp-batch-boot-image" \
+  -H "Metadata-Flavor: Google" 2>/dev/null || echo "")
+echo "[$(date)] - GCP Batch boot image: ${GCP_BATCH_BOOT_IMAGE}"
+TERRA_DRS_URL=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/terra-drs-url" \
+  -H "Metadata-Flavor: Google" 2>/dev/null || echo "")
+TERRA_API_URL=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/terra-api-url" \
+  -H "Metadata-Flavor: Google" 2>/dev/null || echo "")
+
+# Leo proxy path prefix (e.g. /proxy/google/v1/apps/{project}/{appName}/galaxy).
+# Passed to ansible-pull as galaxy_prefix so Galaxy nginx ingress is configured at
+# the correct subpath. Without this Galaxy generates links rooted at / which the
+# browser resolves against Leo host and gets 404s.
+GALAXY_URL_PREFIX=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/galaxy-url-prefix" \
+  -H "Metadata-Flavor: Google" 2>/dev/null || echo "")
+echo "[$(date)] - Galaxy URL prefix: ${GALAXY_URL_PREFIX}"
+
+GALAXY_PROXY_BASE_URL=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/galaxy-proxy-base-url" \
+  -H "Metadata-Flavor: Google" 2>/dev/null || echo "")
+# Full HTTPS URL Galaxy uses to generate absolute links (history exports, API callbacks, etc.).
+# Without this Galaxy infers http:// from its internal connection and produces links that
+# browsers refuse to send the Secure LeoToken cookie on.
+GALAXY_INFRASTRUCTURE_URL="${GALAXY_PROXY_BASE_URL}${GALAXY_URL_PREFIX}"
+echo "[$(date)] - Galaxy infrastructure URL: ${GALAXY_INFRASTRUCTURE_URL}"
+
+GALAXY_USER=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/galaxy-user-email" \
+  -H "Metadata-Flavor: Google" 2>/dev/null || echo "")
+echo "[$(date)] - Galaxy user email: ${GALAXY_USER}"
+
+GIT_REPO=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/git-repo" \
+  -H "Metadata-Flavor: Google" 2>/dev/null || echo "https://github.com/galaxyproject/galaxy-k8s-boot.git")
+GIT_BRANCH=$(curl -s -f "http://metadata.google.internal/computeMetadata/v1/instance/attributes/git-branch" \
+  -H "Metadata-Flavor: Google" 2>/dev/null || echo "anvil")
+
+PULL_ARGS=(
+  -U "${GIT_REPO}"
+  -C "${GIT_BRANCH}"
+  -d /home/debian/ansible
+  -i /tmp/ansible-inventory/localhost
+  --accept-host-key
+  --limit 127.0.0.1
+  --extra-vars "gcp_batch_service_account_email=${GCP_BATCH_SERVICE_ACCOUNT_EMAIL}"
+  --extra-vars "{\"terra_workspace\": \"${TERRA_WORKSPACE}\", \"terra_namespace\": \"${TERRA_NAMESPACE}\"}"
+  --extra-vars "terra_drs_url=${TERRA_DRS_URL}"
+  --extra-vars "terra_api_url=${TERRA_API_URL}"
+  --extra-vars "ingress_use_forwarded_headers=true"
+  --extra-vars "gcp_project_id=${GCP_PROJECT_ID}"
+)
+
+if [ -n "$GCP_NETWORK" ]; then
+    PULL_ARGS+=(--extra-vars "gcp_batch_network=${GCP_NETWORK}")
+    echo "[$(date)] - GCP Batch network passed to ansible: ${GCP_NETWORK}"
+fi
+
+if [ -n "$GCP_SUBNET" ]; then
+    PULL_ARGS+=(--extra-vars "gcp_batch_subnet=${GCP_SUBNET}")
+    echo "[$(date)] - GCP Batch subnet passed to ansible: ${GCP_SUBNET}"
+fi
+
+if [ -n "$GCP_BATCH_BOOT_IMAGE" ]; then
+    PULL_ARGS+=(--extra-vars "gcp_batch_boot_image=${GCP_BATCH_BOOT_IMAGE}")
+    echo "[$(date)] - GCP Batch boot image passed to ansible: ${GCP_BATCH_BOOT_IMAGE}"
+fi
+
+if [ "$RESTORE_GALAXY" = "true" ]; then
+    PULL_ARGS+=(--extra-vars "restore_galaxy=true")
+    echo "[$(date)] - Galaxy Restore Mode: Enabled"
+else
+    echo "[$(date)] - Galaxy Restore Mode: Disabled"
+fi
+
+if [ -n "$GALAXY_URL_PREFIX" ]; then
+    PULL_ARGS+=(--extra-vars "galaxy_prefix=${GALAXY_URL_PREFIX}")
+    echo "[$(date)] - Galaxy URL prefix passed to ansible: ${GALAXY_URL_PREFIX}"
+fi
+
+if [ -n "$GALAXY_INFRASTRUCTURE_URL" ]; then
+    PULL_ARGS+=(--extra-vars "galaxy_infrastructure_url=${GALAXY_INFRASTRUCTURE_URL}")
+    echo "[$(date)] - Galaxy infrastructure URL passed to ansible: ${GALAXY_INFRASTRUCTURE_URL}"
+fi
+
+PULL_ARGS+=(playbook.yml)
+
+mkdir -p /tmp/ansible-inventory
+cat > /tmp/ansible-inventory/localhost << EOF
+[vms]
+127.0.0.1 ansible_connection=local ansible_python_interpreter="/usr/bin/python3"
+
+[all:vars]
+ansible_user="debian"
+rke2_token="defaultSecret12345"
+rke2_additional_sans=["${HOST_IP}"]
+rke2_debug=true
+nfs_size="${PV_SIZE}"
+galaxy_persistence_size="${PV_SIZE}"
+galaxy_db_password="gxy-db-password"
+galaxy_user="${GALAXY_USER}"
+EOF
+
+echo "[$(date)] - Inventory file created at /tmp/ansible-inventory/localhost; running ansible-pull..."
+echo "[$(date)] - Running: ANSIBLE_CALLBACKS_ENABLED=profile_tasks ANSIBLE_HOST_PATTERN_MISMATCH=ignore ansible-pull ${PULL_ARGS[@]}"
+
+ANSIBLE_CALLBACKS_ENABLED=profile_tasks ANSIBLE_HOST_PATTERN_MISMATCH=ignore ansible-pull "${PULL_ARGS[@]}"
+DEBIAN_EOF
+
+echo "[$(date)] - Bootstrap script completed."

http/src/main/resources/init-resources/startup.sh

@@ -122,6 +122,10 @@ function failScriptIfError() {
 
 function validateCert() {
   certFileDirectory=$1
+  ## Only the master node has certs; worker nodes in multi-node Dataproc clusters do not.
+  if [ ! -f "${certFileDirectory}/jupyter-server.crt" ]; then
+    return 0
+  fi
   ## This helps when we need to rotate certs.
   notAfter=`openssl x509 -enddate -noout -in ${certFileDirectory}/jupyter-server.crt` # output should be something like `notAfter=Jul  4 20:31:52 2026 GMT`
 

http/src/main/resources/leo.conf

@@ -23,6 +23,10 @@ gce {
   }
 }
 
+galaxyVm {
+  orchUrl = ${?ORCH_URL}
+}
+
 gke {
   cluster {
     version = ${?KUBERNETES_VERSION}

http/src/main/resources/org/broadinstitute/dsde/workbench/leonardo/liquibase/changelog.xml

@@ -125,4 +125,5 @@
     <include file="changesets/20251024_delete_app_controlled_resource_table.xml" relativeToChangelogFile="true" />
     <include file="changesets/20251024_delete_runtime_controlled_resource_table.xml" relativeToChangelogFile="true" />
     <include file="changesets/20251024_delete_update_app_log_table.xml" relativeToChangelogFile="true" />
+    <include file="changesets/20260730_allow_multiple_galaxy_vm_clusters.xml" relativeToChangelogFile="true" />
 </databaseChangeLog>

http/src/main/resources/org/broadinstitute/dsde/workbench/leonardo/liquibase/changesets/20260730_allow_multiple_galaxy_vm_clusters.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog logicalFilePath="leonardo" xmlns="http://www.liquibase.org/xml/ns/dbchangelog" xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.4.xsd">
+    <changeSet logicalFilePath="leonardo" author="lbaldo" id="lbaldo_20260730_allow_multiple_galaxy_vm_clusters">
+        <!-- Galaxy VM-based deployment creates one Leo KubernetesCluster record per app (rather than
+             one per project) so that each app can store its own VM's external IP via updateAsyncFields
+             without overwriting another app's IP. Drop the project-level uniqueness constraint so that
+             multiple active cluster records can coexist in the same cloud context.
+             Application-level Serializable-isolation logic in saveOrGetClusterForApp continues to
+             enforce single-cluster-per-project semantics for GKE-based apps. -->
+        <dropUniqueConstraint tableName="KUBERNETES_CLUSTER" constraintName="IDX_KUBERNETES_CLUSTER_UNIQUE_V2"/>
+    </changeSet>
+</databaseChangeLog>

http/src/main/resources/reference.conf

@@ -210,6 +210,44 @@ vpc {
     northamerica-northeast2 = "10.26.0.0/20"
   }
   firewallsToAdd = [
+    # Allows Galaxy VM traffic on port 80 (nginx ingress on hostNetwork)
+    {
+      name-prefix = "leonardo-allow-http"
+      sourceRanges = {
+        us-central1 = ["0.0.0.0/0"]
+        northamerica-northeast1 = ["0.0.0.0/0"]
+        southamerica-east1 = ["0.0.0.0/0"]
+        us-east1 = ["0.0.0.0/0"]
+        us-east4 = ["0.0.0.0/0"]
+        us-west1 = ["0.0.0.0/0"]
+        us-west2 = ["0.0.0.0/0"]
+        us-west3 = ["0.0.0.0/0"]
+        us-west4 = ["0.0.0.0/0"]
+        europe-central2 = ["0.0.0.0/0"]
+        europe-north1 = ["0.0.0.0/0"]
+        europe-west1 = ["0.0.0.0/0"]
+        europe-west2 = ["0.0.0.0/0"]
+        europe-west3 = ["0.0.0.0/0"]
+        europe-west4 = ["0.0.0.0/0"]
+        europe-west6 = ["0.0.0.0/0"]
+        asia-east1 = ["0.0.0.0/0"]
+        asia-east2 = ["0.0.0.0/0"]
+        asia-northeast1 = ["0.0.0.0/0"]
+        asia-northeast2 = ["0.0.0.0/0"]
+        asia-northeast3 = ["0.0.0.0/0"]
+        asia-south1 = ["0.0.0.0/0"]
+        asia-southeast1 = ["0.0.0.0/0"]
+        asia-southeast2 = ["0.0.0.0/0"]
+        australia-southeast1 = ["0.0.0.0/0"]
+        northamerica-northeast2 = ["0.0.0.0/0"]
+      }
+      allowed = [
+        {
+          protocol = "tcp"
+          port = "80"
+        }
+      ]
+    },
     # Allows Leonardo proxy traffic on port 443
     {
       name-prefix = "leonardo-allow-https"
@@ -395,6 +433,23 @@ groups {
   }
 }
 
+galaxyVm {
+  # Pre-built galaxy-k8s-boot image with all dependencies (Ansible, RKE2, etc.) pre-installed.
+  # Has cloud-init, which processes the "user-data" metadata key on first boot.
+  # Source: https://github.com/galaxyproject/galaxy-k8s-boot (image built by the Galaxy team)
+  sourceImage = "projects/anvil-and-terra-development/global/images/galaxy-k8s-boot-v2026-06-30"
+  machineType = "n1-highmem-8"
+  bootDiskSizeGb = 100
+  postgresDiskSizeGb = 10
+  # Suffix appended to the NFS disk name to derive the postgres disk name.
+  # Must match the value used in LeoPubsubMessageSubscriber (galaxyDisk.postgresDiskNameSuffix).
+  postgresDiskNameSuffix = ${gke.galaxyDisk.postgresDiskNameSuffix}
+  gitRepo = "https://github.com/galaxyproject/galaxy-k8s-boot.git"
+  gitBranch = "anvil"
+  orchUrl = "https://firecloud-orchestration.dsde-dev.broadinstitute.org/api/"
+  drsUrl = ${drs.url}
+}
+
 gke {
   cluster {
     location = "us-central1-a",

http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/Config.scala

@@ -131,6 +131,20 @@ object Config {
     )
   }
 
+  implicit private val galaxyVmConfigReader: ValueReader[GalaxyVmConfig] = ValueReader.relative { config =>
+    GalaxyVmConfig(
+      config.as[GceCustomImage]("sourceImage"),
+      config.as[MachineTypeName]("machineType"),
+      config.as[DiskSize]("bootDiskSizeGb"),
+      config.as[DiskSize]("postgresDiskSizeGb"),
+      config.as[String]("postgresDiskNameSuffix"),
+      config.as[String]("gitRepo"),
+      config.as[String]("gitBranch"),
+      config.as[String]("orchUrl"),
+      config.as[String]("drsUrl")
+    )
+  }
+
   implicit private val gceConfigReader: ValueReader[GceConfig] = ValueReader.relative { config =>
     GceConfig(
       config.as[GceCustomImage]("customGceImage"),
@@ -497,6 +511,7 @@ object Config {
   val googleGroupsConfig = config.as[GoogleGroupsConfig]("groups")
 
   val dataprocConfig = config.as[DataprocConfig]("dataproc")
+  val galaxyVmConfig = config.as[GalaxyVmConfig]("galaxyVm")
   val gceConfig = config.as[GceConfig]("gce")
   val imageConfig = config.as[ImageConfig]("image")
   val prometheusConfig = config.as[PrometheusConfig]("prometheus")
@@ -901,13 +916,13 @@ object Config {
       vpcConfig.networkTag,
       org.broadinstitute.dsde.workbench.leonardo.http.ConfigReader.appConfig.terraAppSetupChart,
       gkeIngressConfig,
-      gkeGalaxyAppConfig,
       gkeCromwellAppConfig,
       gkeCustomAppConfig,
       gkeAllowedAppConfig,
       appMonitorConfig,
       gkeClusterConfig,
       proxyConfig,
-      gkeGalaxyDiskConfig
+      gkeGalaxyDiskConfig,
+      galaxyVmConfig
     )
 }

http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/config/KubernetesAppConfig.scala

@@ -1,7 +1,9 @@
 package org.broadinstitute.dsde.workbench.leonardo.config
 
-import org.broadinstitute.dsde.workbench.google2.KubernetesSerializableName.ServiceAccountName
+import org.broadinstitute.dsde.workbench.google2.{KubernetesSerializableName, MachineTypeName}
+import KubernetesSerializableName.ServiceAccountName
 import org.broadinstitute.dsde.workbench.leonardo.AppType._
+import org.broadinstitute.dsde.workbench.leonardo.CustomImage.GceCustomImage
 import org.broadinstitute.dsde.workbench.leonardo._
 import org.broadinstitute.dsp.{ChartName, ChartVersion}
 
@@ -92,6 +94,18 @@ final case class CustomAppConfig(chartName: ChartName,
   val appType: AppType = AppType.Custom
 }
 
+final case class GalaxyVmConfig(
+  sourceImage: GceCustomImage,
+  machineType: MachineTypeName,
+  bootDiskSizeGb: DiskSize,
+  postgresDiskSizeGb: DiskSize,
+  postgresDiskNameSuffix: String,
+  gitRepo: String,
+  gitBranch: String,
+  orchUrl: String,
+  drsUrl: String
+)
+
 final case class ContainerRegistryUsername(asString: String) extends AnyVal
 final case class ContainerRegistryPassword(asString: String) extends AnyVal
 final case class ContainerRegistryCredentials(username: ContainerRegistryUsername, password: ContainerRegistryPassword)