sleep3r · Oct 13, 2026 · Oct 13, 2026
@@ -109,6 +109,10 @@ KERNEL_ENV=base
 # it from the machine: its cores and memory divided by a new submission's
 # default cores and memory. The panel's Resources tab overrides this.
 # COMPETITION_SLOTS=auto
+# Entries of KERNEL_GPUS given to competition submissions instead of rooms:
+# COMPETITION_GPUS=
+# Submissions sharing one such card (1-8), each held to its share of memory:
+# COMPETITION_GPU_PER_DEVICE=1
 # Local Docker room network only; broker jobs use Kubernetes NetworkPolicy.
 # KERNEL_NETWORK=colloq
 # Every room container runs hardened: uid 1000, no capabilities,

@@ -663,11 +663,14 @@ TAG = colloq-vast:dev
 
 .PHONY: vast-vm vast-image vast-image-run vast-image-stop
 
-vast-vm: ## Rent a Vast VM that boots the colloq-vast image. HOST=name GPU="RTX 3070" CPU=48 RAM=96 OFFER=number INSTANCE=id
+vast-vm: ## Rent a Vast VM that boots the colloq-vast image. HOST=name GPU="RTX 3070" GPUS=2 GPU_RAM=24 CPU=48 RAM=96 DISK=100 MAX_PRICE=1.5 OFFER=number INSTANCE=id
 	@# Over the API, because only a rent request with "vm": true makes a VM;
 	@# the console's template starts even vastai/kvm as a Docker instance. The
-	@# on-start is backups/vast-onstart.sh (ONSTART=… for another one).
-	@NAME="$(NAME)" HOST="$(HOST)" GPU="$(GPU)" CPU="$(CPU)" RAM="$(RAM)" OFFER="$(OFFER)" INSTANCE="$(INSTANCE)" ./scripts/vast-vm.sh
+	@# on-start is backups/vast-onstart.sh (ONSTART=… for another one). Empty
+	@# variables mean "not asked for": DISK and MAX_PRICE then come from .env,
+	@# as before, and GPUS and GPU_RAM filter nothing.
+	@NAME="$(NAME)" HOST="$(HOST)" GPU="$(GPU)" GPUS="$(GPUS)" GPU_RAM="$(GPU_RAM)" CPU="$(CPU)" RAM="$(RAM)" \
+	  DISK="$(DISK)" MAX_PRICE="$(MAX_PRICE)" OFFER="$(OFFER)" INSTANCE="$(INSTANCE)" ./scripts/vast-vm.sh
 
 vast-image: ## Build the colloq-vast image. TAG=name:tag PLATFORM=linux/amd64
 	@# The version comes from the root package.json (its only source, see

@@ -108,9 +108,10 @@ Room kernel images install Python packages from PyPI at build time:
 
 - `kernel/requirements.txt`: Jupyter Server, ipykernel, NumPy, pandas,
   Matplotlib, Plotly, scikit-learn, requests and Black;
-- `kernel/environments/*.txt`, for example seaborn, PyTorch (the CPU build in
-  `cv`, the CUDA build in `base-gpu`), torchvision, timm, OpenCV and
-  Transformers.
+- `kernel/environments/*.txt`, for example seaborn, CatBoost, LightGBM and
+  XGBoost (`kaggle-base`), PyTorch and torchvision (the CUDA build in
+  `base-gpu`), and OpenCV and timm (`kaggle-gpu`).
 
 Each package keeps its own license inside the image. The CUDA build of PyTorch
 brings NVIDIA CUDA libraries, which are covered by NVIDIA's license terms.
+XGBoost on Linux also installs NVIDIA's NCCL package (`nvidia-nccl-cu12`).
@@ -22,16 +22,16 @@
  * WHERE THE ENVIRONMENTS LIE is the main decision of the group, and it is
  * this.
  *
- *   · The ones shipped with the product (base, base-gpu, cv, gpu) are in
- *     <app>/kernel/environments and are only read: for an installed colloq
- *     that is site-packages, and it cannot be written to in any sense (see
- *     env.ts · ownEnvDir).
+ *   · The ones shipped with the product (base, kaggle-base, base-gpu,
+ *     kaggle-gpu) are in <app>/kernel/environments and are only read: for
+ *     an installed colloq that is site-packages, and it cannot be written to
+ *     in any sense (see env.ts · ownEnvDir).
  *   · One's own are in <home>/environments, next to .env and data/. In the
  *     repository that is the very same kernel/environments, and nothing
  *     changes there.
  *   · The list is the union of the two directories; one's own name overrides
- *     a shipped one, because redefining cv for one's course is a legitimate
- *     wish.
+ *     a shipped one, because redefining kaggle-base for one's course is a
+ *     legitimate wish.
  *
  * Exactly one function turns a name into a path, envFile(); everything else
  * calls it. A second answer to the question "where does cv lie" would split

@@ -297,12 +297,12 @@ export function createEnv(opts: EnvOptions = {}): Env {
      * The second directory of environments, and it is created exactly where
      * the first one cannot be written to.
      *
-     * The environments that arrived with the product (base, base-gpu, cv, gpu)
-     * lie in <app>/kernel/environments. For an installed colloq that is
-     * site-packages: the directory is overwritten entirely by the next
-     * `pip install -U`, and on many machines is not writable at all. Creating
-     * one's own environment there means either a permission refusal or losing
-     * the file on the very first update.
+     * The environments that arrived with the product (base, kaggle-base,
+     * base-gpu, kaggle-gpu) lie in <app>/kernel/environments. For an
+     * installed colloq that is site-packages: the directory is overwritten
+     * entirely by the next `pip install -U`, and on many machines is not
+     * writable at all. Creating one's own environment there means either a
+     * permission refusal or losing the file on the very first update.
      *
      * So one's own lives in the state directory, next to .env and data/, where
      * nobody will overwrite it: <home>/environments.

@@ -48,7 +48,7 @@ namespace fails on those names instead of sharing rooms with the first.
 | 8 | Secrets from Vault through External Secrets, deployment through ArgoCD. The chart never needs a random value at render time: every secret can come from an existing Secret. | `config.existingSecret`, `secrets.*`, `metrics.existingSecret` |
 | 9 | Prometheus Operator (optional ServiceMonitor), logs from stdout. | `metrics.*`, `config.logFormat` |
 | 10 | Backups by volume snapshots (Velero, CSI) plus consistent database snapshots the app writes into its data volume. | `config.dbSnapshotHours`, `config.dbSnapshotKeep` |
-| 11 | GPU nodes optional: runtime class, node selector and tolerations of GPU rooms are values. | `gpu.*` |
+| 11 | GPU nodes optional: runtime class, node selector and tolerations of GPU rooms are values, and so is the extended resource a GPU competition's submissions request; the device plugin and its sharing (time-slicing, MPS, MIG) are the platform's. | `gpu.*` |
 
 ## Storage: ReadWriteOnce or ReadWriteMany
 
@@ -73,6 +73,89 @@ room and personal-notebook Pods mount the workspace, competition Pods the data.
 `accessMode` to that claim's mode. Claims are annotated to survive
 `helm uninstall` and the deletion of an ArgoCD application.
 
+## GPU for competition submissions
+
+Off by default. A teacher may ask for a GPU per competition (on a catalog
+environment with `gpu: true`); with `gpu.competitions.resource` set, the
+notebook step of each submission then requests `gpu.competitions.count` units of
+that extended resource (requests equal limits) and runs with the GPU rooms'
+`gpu.runtimeClassName`, `gpu.nodeSelector` and `gpu.tolerations`, and with a
+memory-backed `/dev/shm` of half its memory, at most `gpu.competitions.shmMb`
+(1024 MiB), for PyTorch's DataLoader workers. The metric step never gets a GPU:
+it mounts the hidden answers. Everything else on a GPU environment's image (the metric, package
+preparation, a competition that asked for no GPU) runs on the CPU with
+`NVIDIA_VISIBLE_DEVICES=void`, so an image built from `nvidia/cuda` gets no card
+on a node whose default runtime is NVIDIA's. While the resource is empty the app
+refuses submissions to a competition that asks for a GPU and says why; it never
+runs them on the CPU instead.
+
+What one unit is, the platform's NVIDIA device plugin decides (directly, or
+through the GPU operator's `devicePlugin.config`), not the chart:
+
+- **Time-slicing** (`sharing.timeSlicing.resources: [{name: nvidia.com/gpu, replicas: 4}]`):
+  every card is offered four times and four submissions share it in time, with
+  no memory or fault isolation between them. With `renameByDefault: true` the
+  replicas are `nvidia.com/gpu.shared`: set that name here.
+- **MPS** (`sharing.mps`, device plugin 0.15 or newer): replicas as above, each
+  limited to an equal part of the card's memory. **Set `gpu.competitions.shmMb: 0`.**
+  The plugin mounts its MPS daemon's shared memory at the container's `/dev/shm`,
+  and the kubelet mounts a Pod's own volumes after the plugin's: a submission's
+  own `/dev/shm` would hide the daemon's, CUDA could not reach the MPS server,
+  and every GPU notebook would train on the CPU or fail
+  ([NVIDIA/k8s-device-plugin#467](https://github.com/NVIDIA/k8s-device-plugin/issues/467)).
+  With 0 the `/dev/shm` is the daemon's, shared by the node's MPS clients and
+  sized by it. Rooms keep their own 1Gi `/dev/shm`, so keep them off MPS nodes:
+  rename the MPS replicas (`renameByDefault: true`, `nvidia.com/gpu.shared`)
+  and set that name here.
+- **MIG** (A100, A30, H100 and newer): slices isolated in hardware. The `mixed`
+  strategy names them by profile, e.g. `nvidia.com/mig-1g.10gb`; `single` keeps
+  `nvidia.com/gpu`.
+
+Rooms always request one `nvidia.com/gpu`. Where the cards a class uses must
+never be shared with submissions, give submissions a name of their own (a MIG
+profile, or time-sliced replicas renamed to `nvidia.com/gpu.shared` on nodes
+labelled for that device-plugin configuration). A count above 1 makes sense for
+whole cards and MIG slices; time-sliced replicas of one card add no compute
+(`failRequestsGreaterThanOne: true` makes the plugin refuse such requests).
+
+**ReadWriteOnce storage pins every competition Pod to the app's node**
+(`RUNTIME_COLOCATE_WITH_APP=1`, above), GPU submissions included: the units
+must be on that node, so the app must run on a GPU node (`app.nodeSelector`,
+`app.tolerations`), or the data claim must be ReadWriteMany. Otherwise no node
+satisfies both, and every GPU submission waits in the queue "for resources" for
+as long as nobody looks. The chart sees no nodes and cannot check this; after
+the install, run a competition's baseline check on a GPU environment.
+
+**How many at once: `gpu.competitions.slots`.** The app's queue keeps that many
+GPU submissions at the broker and lets CPU work (other competitions, rescores,
+baseline checks) go past the GPU submissions that wait for their turn. Set it
+to how many submissions the resource can serve at once, `count` units each,
+after what GPU rooms take when they request the same resource. Unset, the queue
+sends one at a time. Too high costs throughput, not results: a GPU submission
+the scheduler cannot place holds a general slot for 30 seconds and then waits
+in the queue again.
+
+**When Kubernetes refuses a GPU submission's Pod.** A ResourceQuota that names
+the resource (`requests.nvidia.com/gpu: "2"`) bounds GPU rooms and GPU
+submissions together, and the API server refuses a Pod over it. That refusal,
+a missing RuntimeClass (`gpu.runtimeClassName`; `nvidia` is created by the GPU
+operator, and absent without it), or an admission policy that turns down the GPU
+parts of the Pod sends the submission back to the queue: it is retried a few
+seconds to a minute later, and its teacher sees the cause next to it. A CPU job
+refused the same way still fails, as it always did. Keep
+`gpu.competitions.slots` × `count` within such a quota, less what GPU rooms
+hold.
+
+**Image pull: `gpu.competitions.pullSeconds`.** GPU catalog images are 10 to
+13 GB, and every job of a GPU environment runs one: the notebook on a GPU node,
+the metric and package preparation on any node of the students' pool. A node
+that has never run the image pulls it after the Pod is created, and the broker
+waits up to `pullSeconds` (900) for that before the job's own time starts; the
+Pod's `activeDeadlineSeconds` grows by as much. A job that started still gets
+exactly its own limit. To spare the first class the wait, pull the GPU images
+onto those nodes beforehand (for instance, a DaemonSet on those nodes whose init
+containers are the GPU images, each running `true`).
+
 ## Secrets, Vault and ArgoCD
 
 | Secret | Keys | Mounted by | Replace with |
@@ -292,9 +375,14 @@ out so the app's default applies. The app reads the ConfigMap, then the Secret
 | `rooms.priorityClassName` | `""` | `RUNTIME_PRIORITY_CLASS` (every broker-created Pod) |
 | `rooms.podLabels` | `{}` | `RUNTIME_POD_LABELS` (JSON, with `app.kubernetes.io/name`, `part-of` and `commonLabels`; never `instance`, which ArgoCD's label tracking would take for its own and prune) |
 | `rooms.podAnnotations` | `{}` | `RUNTIME_POD_ANNOTATIONS` (JSON) |
-| `gpu.runtimeClassName` | `nvidia` | `RUNTIME_GPU_RUNTIME_CLASS` (`""`: none) |
-| `gpu.nodeSelector` | `{}` | `RUNTIME_GPU_NODE_SELECTOR` (JSON) |
-| `gpu.tolerations` | `[]` | `RUNTIME_GPU_TOLERATIONS` (JSON) |
+| `gpu.runtimeClassName` | `nvidia` | `RUNTIME_GPU_RUNTIME_CLASS` (`""`: none); also a competition notebook given a GPU |
+| `gpu.nodeSelector` | `{}` | `RUNTIME_GPU_NODE_SELECTOR` (JSON); also a competition notebook given a GPU |
+| `gpu.tolerations` | `[]` | `RUNTIME_GPU_TOLERATIONS` (JSON); also a competition notebook given a GPU |
+| `gpu.competitions.resource` | `""` (off) | `RUNTIME_COMPETITION_GPU_RESOURCE`, rendered only when set: the extended resource a GPU competition's notebook step requests, e.g. `nvidia.com/gpu`, `nvidia.com/mig-1g.10gb` (see *GPU for competition submissions*) |
+| `gpu.competitions.count` | `1` | `RUNTIME_COMPETITION_GPU_COUNT`, with the resource: units per submission, 1..8 |
+| `gpu.competitions.slots` | `null` (one at a time) | `RUNTIME_COMPETITION_GPU_SLOTS`, with the resource, rendered only when set: GPU submissions the cluster runs at once, 1..256 |
+| `gpu.competitions.shmMb` | `1024` | `RUNTIME_COMPETITION_GPU_SHM_MB`, with the resource: the ceiling of a GPU submission's `/dev/shm`, MiB (also at most half its memory); `0`: none, for MPS |
+| `gpu.competitions.pullSeconds` | `900` | `RUNTIME_COMPETITION_GPU_PULL_SECONDS`: the image pull allowance of every Pod of a GPU catalog image, 0..3600 |
 
 Set `rooms.maxMemory` on a cluster of unequal nodes: without it the broker takes
 its own node's memory as every room's ceiling. A room reserves its memory and

@@ -131,6 +131,24 @@ spec:
             - name: RUNTIME_GPU_TOLERATIONS
               value: {{ toJson . | quote }}
             {{- end }}
+            {{- /* With the resource or not at all: without one competition jobs get no GPU, and the count, the slots and /dev/shm mean nothing. */}}
+            {{- with .Values.gpu.competitions.resource }}
+            - name: RUNTIME_COMPETITION_GPU_RESOURCE
+              value: {{ . | quote }}
+            - name: RUNTIME_COMPETITION_GPU_COUNT
+              value: {{ $.Values.gpu.competitions.count | int | toString | quote }}
+            {{- /* null is "not said": never an empty variable, which the broker reads as an unparseable number. */}}
+            {{- with $.Values.gpu.competitions.slots }}
+            - name: RUNTIME_COMPETITION_GPU_SLOTS
+              value: {{ . | int | toString | quote }}
+            {{- end }}
+            {{- /* 0 is a setting of its own (no /dev/shm of ours, for MPS), so it is passed, not dropped. */}}
+            - name: RUNTIME_COMPETITION_GPU_SHM_MB
+              value: {{ $.Values.gpu.competitions.shmMb | int | toString | quote }}
+            {{- end }}
+            {{- /* Every Pod of a GPU catalog image, with a GPU or without one. */}}
+            - name: RUNTIME_COMPETITION_GPU_PULL_SECONDS
+              value: {{ .Values.gpu.competitions.pullSeconds | int | toString | quote }}
             {{- with .Values.rooms.priorityClassName }}
             - name: RUNTIME_PRIORITY_CLASS
               value: {{ . | quote }}

@@ -76,6 +76,30 @@ behind a green sync.
 {{- fail "rooms.cpu: at most 64 cores" -}}
 {{- end -}}
 
+{{- /* A GPU for competition submissions: Kubernetes' rule for an extended resource, as the broker checks it (runtime/src/pod-policy.ts · parseExtendedResource). */ -}}
+{{- with $values.gpu.competitions.resource -}}
+{{- if or (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/[A-Za-z0-9]([-A-Za-z0-9_.]{0,61}[A-Za-z0-9])?$" .)) (contains "kubernetes.io/" .) (hasPrefix "requests." .) -}}
+{{- fail (printf "gpu.competitions.resource %q: the extended resource your GPU device plugin offers, e.g. nvidia.com/gpu or nvidia.com/mig-1g.10gb" .) -}}
+{{- end -}}
+{{- $count := int $values.gpu.competitions.count -}}
+{{- if or (lt $count 1) (gt $count 8) -}}
+{{- fail (printf "gpu.competitions.count %v: 1 to 8 units of %s per submission" $values.gpu.competitions.count .) -}}
+{{- end -}}
+{{- with $values.gpu.competitions.slots -}}
+{{- if or (lt (int .) 1) (gt (int .) 256) -}}
+{{- fail (printf "gpu.competitions.slots %v: 1 to 256 GPU submissions at once, or null" .) -}}
+{{- end -}}
+{{- end -}}
+{{- $shm := int $values.gpu.competitions.shmMb -}}
+{{- if or (lt $shm 0) (gt $shm 65536) -}}
+{{- fail (printf "gpu.competitions.shmMb %v: 0 to 65536 MiB (0: no /dev/shm of ours, for MPS)" $values.gpu.competitions.shmMb) -}}
+{{- end -}}
+{{- end -}}
+{{- $pull := int $values.gpu.competitions.pullSeconds -}}
+{{- if or (lt $pull 0) (gt $pull 3600) -}}
+{{- fail (printf "gpu.competitions.pullSeconds %v: 0 to 3600 seconds" $values.gpu.competitions.pullSeconds) -}}
+{{- end -}}
+
 {{- /* Keys that belong to the chart and the broker. */ -}}
 {{- range $key, $_ := $values.commonLabels -}}
 {{- if or (hasPrefix "app.kubernetes.io/" $key) (hasPrefix "colloq.dev/" $key) (hasPrefix "helm.sh/" $key) -}}

@@ -284,7 +284,18 @@
       "properties": {
         "runtimeClassName": { "type": "string" },
         "nodeSelector": { "$ref": "#/definitions/stringMap" },
-        "tolerations": { "$ref": "#/definitions/objectList" }
+        "tolerations": { "$ref": "#/definitions/objectList" },
+        "competitions": {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "resource": { "type": "string", "pattern": "^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/[A-Za-z0-9]([-A-Za-z0-9_.]{0,61}[A-Za-z0-9])?)?$" },
+            "count": { "type": "integer", "minimum": 1, "maximum": 8 },
+            "slots": { "type": ["integer", "null"], "minimum": 1, "maximum": 256 },
+            "shmMb": { "type": "integer", "minimum": 0, "maximum": 65536 },
+            "pullSeconds": { "type": "integer", "minimum": 0, "maximum": 3600 }
+          }
+        }
       }
     },
     "persistence": {