From 6825295800596f3fb8938d0beb6ed349dc64493d Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Tue, 11 Aug 2026 16:53:26 +0100 Subject: [PATCH 1/2] fix(spawner): derive co-location affinity from the pod's username label kubespawner 7 computes the hub.jupyter.org/username label with a different slug truncation than {username} template expansion, so templating the value into the required pod affinity made it unsatisfiable for usernames that get a hashed slug (emails): every spawn failed with FailedScheduling on all nodes. Build the affinity in a modify_pod_hook from the pod's own label so the values match under any slug scheme. --- config/jupyterhub/01-spawner.py | 61 ++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/config/jupyterhub/01-spawner.py b/config/jupyterhub/01-spawner.py index 10a3297..ded64cb 100644 --- a/config/jupyterhub/01-spawner.py +++ b/config/jupyterhub/01-spawner.py @@ -13,6 +13,7 @@ from urllib.request import Request, urlopen import escapism +from kubernetes_asyncio.client.models import V1Affinity, V1PodAffinity from kubernetes_asyncio.client.rest import ApiException from kubespawner.objects import make_pvc from z2jh import get_config @@ -117,11 +118,6 @@ "jupyterhub-singleuser", ] -# affinity — co-locate all pods for the same user on the same node. -# hcloud-volumes is ReadWriteOnce — only one node can mount it at a time. -# Pod affinity ensures jhub-apps app pods land on the same node as the -# user's JupyterLab pod so the shared home PVC can be mounted by all of them. -# # securityContext.fsGroup: 100 — GID 100 (users group) as the pod's fsGroup. # Kubernetes adds GID 100 as a supplemental group and chgrps mounted volumes # to 100 at pod start. This ensures shared dirs (created by init container as @@ -138,24 +134,6 @@ # any securityContext we set here REPLACES the one fs_gid would produce, so # both fields must live in this dict together. c.KubeSpawner.extra_pod_config = { - "affinity": { - "podAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": [ - { - "labelSelector": { - "matchExpressions": [ - { - "key": "hub.jupyter.org/username", - "operator": "In", - "values": ["{username}"], - } - ] - }, - "topologyKey": "kubernetes.io/hostname", - } - ] - } - }, "securityContext": { "fsGroup": 100, "fsGroupChangePolicy": "OnRootMismatch", @@ -163,6 +141,43 @@ } +# Co-locate all pods for the same user on the same node. The home PVC is +# ReadWriteOnce, so only one node can mount it at a time; pod affinity makes +# jhub-apps app pods land on the same node as the user's JupyterLab pod. +# +# The affinity value is copied from the pod's own hub.jupyter.org/username +# label instead of templating "{username}" into extra_pod_config: kubespawner +# computes that label with a different slug truncation than {username} +# template expansion (kubespawner 7 safe-slug scheme, usernames that need a +# hash suffix, e.g. emails), and a value that differs from the label makes +# the pod's required affinity unsatisfiable, so it never schedules. +def _colocate_user_pods(spawner, pod): + username_label = (pod.metadata.labels or {}).get("hub.jupyter.org/username") + if not username_label: + return pod + term = { + "labelSelector": { + "matchExpressions": [ + { + "key": "hub.jupyter.org/username", + "operator": "In", + "values": [username_label], + } + ] + }, + "topologyKey": "kubernetes.io/hostname", + } + if pod.spec.affinity is None: + pod.spec.affinity = V1Affinity() + pod.spec.affinity.pod_affinity = V1PodAffinity( + required_during_scheduling_ignored_during_execution=[term] + ) + return pod + + +c.KubeSpawner.modify_pod_hook = _colocate_user_pods + + # --------------------------------------------------------------------------- # Shared storage (RWX PVC for group directories) # --------------------------------------------------------------------------- From 464021ccb7de63d0db39d54fa95eea81fc2ec4d5 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Tue, 11 Aug 2026 19:54:04 +0100 Subject: [PATCH 2/2] docs: add User Pod Scheduling admin page Covers why home volumes are ReadWriteOnce, the co-location pod affinity that RWO requires, and how to debug spawns that time out unscheduled. --- docs/astro.config.mjs | 6 + docs/src/content/docs/user-pod-scheduling.md | 127 +++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 docs/src/content/docs/user-pod-scheduling.md diff --git a/docs/astro.config.mjs b/docs/astro.config.mjs index 0c54baf..b199eda 100644 --- a/docs/astro.config.mjs +++ b/docs/astro.config.mjs @@ -33,6 +33,12 @@ export default defineConfig({ { label: 'Shared Storage', slug: 'shared-storage' }, ], }, + { + label: 'Administration', + items: [ + { label: 'User Pod Scheduling', slug: 'user-pod-scheduling' }, + ], + }, { label: 'Reference', items: [ diff --git a/docs/src/content/docs/user-pod-scheduling.md b/docs/src/content/docs/user-pod-scheduling.md new file mode 100644 index 0000000..fc99dd0 --- /dev/null +++ b/docs/src/content/docs/user-pod-scheduling.md @@ -0,0 +1,127 @@ +--- +title: User Pod Scheduling +description: Why all of a user's pods run on one node, how the co-location affinity works, and how to debug spawns that time out unscheduled. +--- + +The Data Science Pack constrains all pods belonging to a user, the +JupyterLab server and any jhub-apps application servers, to run on a +single node. This page +explains why that constraint exists, how it is enforced through pod +affinity, and how to diagnose spawn failures caused by scheduling. + +## Motivation + +The Data Science Pack provisions one home volume per user +(`claim-{username}`) with the +`ReadWriteOnce` [access mode](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes), +which restricts attachment to a single node at a time. + +`ReadWriteMany` is available to the chart and already in use for the +`/shared/` directories, provided by a cluster RWX `StorageClass` +such as Longhorn (see [Shared Storage](/shared-storage/)). Home +directories deliberately do not use it, for performance. + +Every `ReadWriteMany` implementation is network file storage. Home +directory workloads (conda and pixi environments, version control, +notebook autosave) consist largely of metadata operations on many small +files; a single Python environment holds tens of thousands of them; +over a network filesystem each operation incurs a round trip, degrading +environment creation and interactive latency by orders of magnitude +compared to node-attached block storage. The `/shared` directories pay +this cost because there is no alternative for them: they must be writable +by many users' pods across nodes concurrently, which only network file +storage provides. A home volume is accessed by a single user's pods, so +it can avoid the cost by keeping those pods on one node. + +The consequence of `ReadWriteOnce` home volumes is a scheduling +constraint: all pods belonging to a user must run on the node to which the +user's home volume is attached. The remainder of this page describes how +the chart enforces that constraint and how to diagnose failures related to +it. + +## Problem + +A user may run multiple pods concurrently: the JupyterLab server and any +number of jhub-apps application servers, all of which mount the same home +volume. If the scheduler places two of these pods on different nodes, the +second pod remains in `ContainerCreating` with a `Multi-Attach` error, +because the volume is already attached to the first node. + +```mermaid +flowchart TB + subgraph node1["Node 1"] + lab["JupyterLab pod"] + end + subgraph node2["Node 2"] + app["jhub-apps pod"] + end + home[("home volume\nclaim-user (RWO)")] + home -->|attached| lab + home -.-|"Multi-Attach error"| app +``` + +## Solution + +Every user pod is created with a required inter-pod affinity term that +restricts scheduling to nodes running a pod labeled +`hub.jupyter.org/username=`. For the first pod, the term is +satisfied by the pod's own label: the Kubernetes scheduler permits a pod +to fulfill its own required affinity when the required value matches one +of the pod's labels. Subsequent pods for the same user are placed on the +same node. + +```mermaid +flowchart TB + subgraph node1["Node 1"] + lab["JupyterLab pod"] + app["jhub-apps pod"] + end + subgraph node2["Node 2"] + other["another user's pods"] + end + home[("home volume\nclaim-user (RWO)")] + shared[("shared volume\n/shared (RWX)")] + lab ---|"pod affinity:\nsame node"| app + home -->|attached to one node| lab + home --> app + shared --> lab + shared --> app + shared --> other +``` + +The user's pods share one node, so the `ReadWriteOnce` home volume serves +all of them. The `ReadWriteMany` shared volume is unaffected by node +placement and mounts into every user's pods on every node. + +The affinity term is constructed by a `modify_pod_hook` in +`config/jupyterhub/01-spawner.py`, which copies the value of the pod's own +`hub.jupyter.org/username` label into the term. The value must not be +produced with the `{username}` template instead: kubespawner derives the +label and the template expansion with different slug rules, so for +usernames that require escaping (such as email addresses) the two values +diverge. A diverged value cannot be satisfied by any node, and every spawn +for the affected user fails at scheduling until the spawn timeout expires. + +## Debugging spawn timeouts + +JupyterHub deletes the pod when the spawn timeout expires, but the +namespace events outlive it: + +```bash +kubectl get events -n --sort-by=.lastTimestamp | grep +``` + +| Event | Cause | +|---|---| +| `didn't match pod affinity rules` (all nodes) | The value in `spec.affinity.podAffinity` does not match the pod's `metadata.labels."hub.jupyter.org/username"`. This cannot occur while the `modify_pod_hook` is intact; it indicates the hook was removed or the value is being templated again. Compare the two fields with the command below while the pod is still `Pending`. | +| `Multi-Attach error` | The previous node still holds the volume attachment, typically after a node failure or while a detach is still in progress, or the co-location affinity has been removed. Check `kubectl get volumeattachment` for the volume, then verify the hook in the hub ConfigMap. | +| `FailedAttachVolume` (volume never attaches) | The storage backend could not provision or place the volume. This is a cluster storage issue, not a chart issue; inspect the volume's status with the tooling of the installed storage layer. | + +To compare the username label with the affinity value on a `Pending` pod: + +```bash +kubectl get pod -n \ + -o jsonpath='{.metadata.labels.hub\.jupyter\.org/username}{"\n"}{.spec.affinity.podAffinity}' +``` + +The two values must be identical.