IDP Arc portal integration with OSB

applications/accounts-api/Dockerfile

@@ -13,5 +13,3 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 COPY backend /usr/src/app
 
 RUN pip3 install -e .
-
-ENTRYPOINT gunicorn --workers=$WORKERS --bind=0.0.0.0:$PORT $MODULE_NAME.__main__:app

applications/accounts-api/backend/requirements.txt

@@ -1,11 +1,9 @@
-connexion[swagger-ui] >= 2.6.0; python_version>="3.6"
-# 2.3 is the last version that supports python 3.4-3.5
-connexion[swagger-ui] <= 2.3.0; python_version=="3.5" or python_version=="3.4"
-# connexion requires werkzeug but connexion < 2.4.0 does not install werkzeug
-# we must peg werkzeug versions below to fix connexion
-# https://github.com/zalando/connexion/pull/1044
-werkzeug == 2.2.3; python_version=="3.5" or python_version=="3.4"
-swagger-ui-bundle >= 0.0.2
-python_dateutil >= 2.6.0
-setuptools >= 21.0.0
-cloudharness
+connexion[swagger-ui,flask,uvicorn]>=3.0.0,<4.0.0
+swagger-ui-bundle>=1.1.0
+python_dateutil>=2.9.0
+setuptools>=21.0.0
+uvicorn
+# Following some unnecessary requirements to make sure they can be installed
+psycopg2-binary
+sqlalchemy<2.0.0
+scipy

applications/accounts-api/backend/setup.py

@@ -14,10 +14,10 @@
 # http://pypi.python.org/pypi/setuptools
 
 REQUIRES = [
-    "connexion>=2.0.2",
-    "swagger-ui-bundle>=0.0.2",
-    "python_dateutil>=2.6.0",
-    "cloudharness"
+    "connexion[swagger-ui,flask,uvicorn]>=3.0.0,<4.0.0",
+    "Flask",
+    "swagger-ui-bundle",
+    "cloudharness",
 ]
 
 setup(

applications/accounts/deploy/resources/realm.json

@@ -322,6 +322,34 @@
                     "offline_access",
                     "microprofile-jwt"
                 ]
+            },
+            {
+                "clientId": "idp-arc",
+                "name": "idp-arc",
+                "enabled": true,
+                "publicClient": true,
+                "standardFlowEnabled": true,
+                "implicitFlowEnabled": false,
+                "directAccessGrantsEnabled": false,
+                "serviceAccountsEnabled": false,
+                "protocol": "openid-connect",
+                "rootUrl": "{{ .Values.idpArc.rootUrl }}",
+                "redirectUris": [
+                    "{{ .Values.idpArc.rootUrl }}/*"
+                ],
+                "webOrigins": [
+                    "{{ .Values.idpArc.rootUrl }}"
+                ],
+                "attributes": {
+                    "pkce.code.challenge.method": "S256"
+                },
+                "fullScopeAllowed": true,
+                "defaultClientScopes": [
+                    "web-origins", "profile", "roles", "email"
+                ],
+                "optionalClientScopes": [
+                    "offline_access", "{{ .Values.apps.accounts.admin.role }}-scope"
+                ]
             }
         ],
         "clientScopes": [

applications/jupyterlab/Dockerfile

@@ -77,15 +77,15 @@ USER root
 RUN conda install conda-forge::nest-simulator
 ENV NEST_HOME=/opt/conda
 
-USER jovyan
-
-#### Install Python packages
+#### Install Python packages (as root to avoid permission issues with conda-managed packages)
 
 RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip && \ 
     pip install backports.tarfile>=1.2 # temp fix for error: ImportError: cannot import name 'tarfile' from 'backports'
 RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\ 
     pip install -r requirements.txt --upgrade --no-cache-dir
 
+USER jovyan
+
 #  Compile NEURON mod files for PyNN
 RUN cd /opt/conda/lib/python3.12/site-packages/pyNN/neuron/nmodl && nrnivmodl
 

applications/jupyterlab/requirements.txt

@@ -3,7 +3,7 @@ pyelectro
 neurotune
 
 ####  NEURON & NetPyNE
-neuron
+neuron<9
 
 # Install specific version of NetPyNE
 git+https://github.com/Neurosim-lab/netpyne.git@osbv2#egg=netpyne

applications/netpyne/Dockerfile

@@ -1,6 +1,6 @@
 
 
-FROM node:18 AS jsbuild
+FROM node:16-bullseye AS jsbuild
 ARG APP_DIR=dependencies/NetPyNE-UI
 ENV FOLDER=netpyne
 

applications/netpyne/deploy/values.yaml

@@ -1,7 +1,5 @@
 harness:
   subdomain: netpyne
-  secured: true
-  uri_role_mapping: []
   service:
     auto: false
     port: 80

applications/nfsserver/deploy/values-dev.yaml

@@ -1,4 +1,4 @@
 nfs:
   useDNS: true
 server:
-  diskSize: 80Gi
+  diskSize: 160Gi

applications/nfsserver/deploy/values.yaml

@@ -2,7 +2,7 @@
 harness:
   deployment:
     auto: false
-    image: gcr.io/metacellllc/osb/nfsserver:0.7.6
+    image: gcr.io/metacellllc/cloud-harness/nfsserver:featurech-261
 # nfs server pvc disk size (/exports)
 server:
   diskSize: 240Gi

applications/osb-portal/src/components/workspace/WorkspaceFrame.tsx

@@ -3,13 +3,18 @@ import { useParams } from "react-router-dom";
 import { AnyAction, Dispatch } from "redux";
 import { PayloadAction } from "@reduxjs/toolkit";
 
+import Box from "@mui/material/Box";
+import Button from "@mui/material/Button";
+import Typography from "@mui/material/Typography";
+
 import {
   Workspace,
   WorkspaceResource,
   OSBApplications,
   ResourceStatus,
 } from "../../types/workspace";
 import { UserInfo } from "../../types/user";
+import workspaceService from "../../service/WorkspaceService";
 import WorkspaceResourceService from "../../service/WorkspaceResourceService";
 import { getBaseDomain, getApplicationDomain } from "../../utils";
 
@@ -30,6 +35,8 @@ export const WorkspaceFrame = (props: {
   currentResource: WorkspaceResource;
 }) => {
   const [frameUrl, setFrameUrl] = React.useState(null);
+  const [frameError, setFrameError] = React.useState(false);
+  const [retryToken, setRetryToken] = React.useState(0);
   const { app } = useParams<{ app: string }>();
 
   const { user, workspace, dispatch, currentResource } = props;
@@ -57,7 +64,7 @@ export const WorkspaceFrame = (props: {
       "workspace-frame"
     ) as HTMLIFrameElement;
     const messageListener = (message: MessageEvent<PayloadAction>) => {
-      if (message.source !== iFrame.contentWindow) {
+      if (!iFrame || message.source !== iFrame.contentWindow) {
         return;
       }
       console.debug("Message", message);
@@ -99,24 +106,92 @@ export const WorkspaceFrame = (props: {
   }, [openResource]);
 
   React.useEffect(() => {
-    const applicationDomain = getApplicationDomain(application);
-    const domain = getBaseDomain();
-
-    const userParam = user == null ? "" : `${user.id}`;
-    const type = application.subdomain.slice(0, 4);
-    document.cookie = `workspaceId=${workspace.id};path=/;domain=${domain}`;
-    if(applicationDomain) {
-      setFrameUrl(`//${applicationDomain}/hub/spawn/${userParam}/${workspace.id}${type}${document.location.search ?? ''}`);
-    } else {
-      setFrameUrl(`/testapp?workspaceId=${workspace.id}&type=${type}&user=${userParam}&application=${application.name}`);
-    }
+    let cancelled = false;
+    setFrameError(false);
+    setFrameUrl(null);
+
+    const spawn = async () => {
+      // Make sure the workspace volume is created and ready (bound) before we
+      // spawn the iframe, so the app pod does not fail to mount it.
+      try {
+        await workspaceService.ensureWorkspaceReady(workspace.id);
+      } catch (e) {
+        if (e?.volumeNotReady) {
+          // Transient: the storage is genuinely not bound yet. Surface a
+          // retryable error instead of spawning a pod that would fail to mount.
+          console.error("Workspace volume not ready, not spawning iframe", e);
+          if (!cancelled) {
+            setFrameError(true);
+          }
+          return;
+        }
+        // Any other failure (auth, network, 5xx) is unrelated to volume
+        // readiness; don't block workspace access — fall through and spawn the
+        // iframe as we did before this readiness check existed.
+        console.warn("Workspace readiness check failed, spawning anyway", e);
+      }
+      if (cancelled) {
+        return;
+      }
 
-  }, [application, user, workspace.id]);
+      const applicationDomain = getApplicationDomain(application);
+      const domain = getBaseDomain();
+
+      const userParam = user == null ? "" : `${user.id}`;
+      const type = application.subdomain.slice(0, 4);
+      document.cookie = `workspaceId=${workspace.id};path=/;domain=${domain}`;
+      if (applicationDomain) {
+        setFrameUrl(`//${applicationDomain}/hub/spawn/${userParam}/${workspace.id}${type}${document.location.search ?? ''}`);
+      } else {
+        setFrameUrl(`/testapp?workspaceId=${workspace.id}&type=${type}&user=${userParam}&application=${application.name}`);
+      }
+    };
+
+    spawn();
+
+    return () => {
+      cancelled = true;
+    };
+    // Depend on the primitive inputs the effect actually uses, not the
+    // `application`/`user` object identities — those are rebuilt on unrelated
+    // redux updates and would otherwise reset frameUrl to null (remounting the
+    // iframe via key={frameUrl}) and refire the /open fetch on every re-render.
+  }, [application?.subdomain, application?.name, user?.id, workspace.id, retryToken]);
 
   if (!workspace) {
     return null;
   }
 
+  if (frameError) {
+    return (
+      <Box
+        display="flex"
+        flexDirection="column"
+        alignItems="center"
+        justifyContent="center"
+        flex={1}
+        gap={2}
+        p={4}
+        textAlign="center"
+      >
+        <Typography variant="h6">
+          The workspace is temporarily unavailable
+        </Typography>
+        <Typography variant="body1" color="textSecondary">
+          The workspace storage is not ready yet. This is usually temporary —
+          please try again in a few seconds.
+        </Typography>
+        <Button
+          variant="contained"
+          color="primary"
+          onClick={() => setRetryToken((token) => token + 1)}
+        >
+          Retry
+        </Button>
+      </Box>
+    );
+  }
+
   return (
     <iframe
       id="workspace-frame"

applications/osb-portal/src/service/WorkspaceService.tsx

@@ -63,6 +63,37 @@ class WorkspaceService {
     return ws;
   }
 
+  /**
+   * Ensure the workspace volume exists and is ready (bound) before the app
+   * iframe is spawned. The backend blocks up to 5s waiting for the PVC; if it
+   * is still not ready it answers 503, which we surface via `volumeNotReady` so
+   * the caller can show a temporary, retryable error instead of spawning the
+   * iframe. Only a 503 is retryable — other statuses are reported as-is so the
+   * caller can tell a transient storage hiccup from a real auth/not-found error.
+   *
+   * NOTE: this uses fetch rather than `this.workspacesApi` because the generated
+   * client does not yet expose the `/workspace/{id}/open` operation; switch to
+   * the generated client once the apiclient is regenerated from the OpenAPI
+   * spec. The bearer token is taken from the same `accessToken` the client uses.
+   */
+  async ensureWorkspaceReady(id: number): Promise<void> {
+    const response = await fetch(`${workspacesApiUri}/workspace/${id}/open`, {
+      headers: this.accessToken
+        ? { Authorization: `Bearer ${this.accessToken}` }
+        : {},
+    });
+    if (response.ok) {
+      return;
+    }
+    const error: any = new Error(
+      `Could not open workspace ${id} (status ${response.status})`
+    );
+    error.status = response.status;
+    // 503 => volume not ready yet: a transient, retryable server error.
+    error.volumeNotReady = response.status === 503;
+    throw error;
+  }
+
   async refreshResources(selectedWorkspaceId: any) {
     return this.workspacesApi.workspacesControllersWorkspaceControllerImportResources(
       { id: selectedWorkspaceId, inlineObject: {} }

applications/workspaces/deploy/values-dev.yaml

@@ -5,14 +5,13 @@ harness:
     resources:
       requests:
         memory: 128Mi
-        cpu: 10m
+        cpu: 50m
   deployment:
     resources:
       requests:
-        memory: 128Mi
-        cpu: 10m
-      limits:
         memory: 512Mi
-        cpu: 1500m
+        cpu: 100m
+      limits:
+        memory: 750Mi
 
 

applications/workspaces/deploy/values.yaml

@@ -11,10 +11,26 @@ harness:
     path: /api/live
     initialDelaySeconds: 5
     periodSeconds: 15
+    # 1s (the k8s default) is too tight: under node CPU contention even the
+    # trivial /api/live handler can take >1s, which liveness-killed the pod.
+    timeoutSeconds: 5
+    failureThreshold: 6
   readinessProbe:
     path: /api/ready
     initialDelaySeconds: 10
     periodSeconds: 15
+    timeoutSeconds: 5
+  # The server runs 4 gunicorn workers loading a heavy connexion/SQLAlchemy app
+  # and can take well over the ~50s liveness window to start serving — more so
+  # under the CPU contention of a spot-node reschedule storm. Without a startup
+  # probe, liveness kills it mid-boot and it crash-loops. Give it up to ~300s
+  # (30 x 10s) to come up before liveness/readiness apply.
+  startupProbe:
+    path: /api/live
+    initialDelaySeconds: 10
+    periodSeconds: 10
+    failureThreshold: 30
+    timeoutSeconds: 5
   deployment:
     auto: true
     port: 8080
@@ -24,11 +40,12 @@ harness:
       mountpath: /usr/src/app/workspaces/static/workspaces
     resources:
       requests:
-        memory: 128Mi
-        cpu: 20m
-      limits:
         memory: 512Mi
-        cpu: 1500m
+        cpu: 100m
+      limits:
+        # Raised 512Mi -> 1Gi: the app's steady-state RSS sits at ~490Mi, so a
+        # 512Mi limit caused recurring OOMKills (exit 137) under load.
+        memory: 1Gi
   quotas:
     # sets the max number of workspaces per user
     quota-ws-max: 5