[POC]: feat: add Kueue prebuilt workload backend

docs/community/kueue-tas-backend-slides/demo/grove-kind-topology.yaml

@@ -0,0 +1,11 @@
+# Grove source-of-truth topology for the CPU-only kind TAS smoke test.
+apiVersion: grove.io/v1alpha1
+kind: ClusterTopologyBinding
+metadata:
+  name: grove-kind-topology
+spec:
+  levels:
+  - domain: rack
+    key: topology.ai-dynamo.io/rack
+  - domain: host
+    key: kubernetes.io/hostname

docs/community/kueue-tas-backend-slides/demo/grove-kueue-pcsg.yaml

@@ -0,0 +1,57 @@
+# Grove PodCliqueSet with a PodCliqueScalingGroup (PCSG).
+# POC assumption: PCSG.minAvailable == PCSG.replicas, so the whole gang is
+# all-or-nothing. Grove builds one prebuilt Kueue Workload per PodGang and
+# omits minCount for the PCSG podSets.
+# Apply after kueue-kind-queue.yaml and grove-kind-topology.yaml.
+apiVersion: grove.io/v1alpha1
+kind: PodCliqueSet
+metadata:
+  name: grove-kueue-pcsg
+  labels:
+    app: grove-kueue-pcsg
+    kueue.x-k8s.io/queue-name: grove-poc
+spec:
+  replicas: 1
+  template:
+    topologyConstraint:
+      topologyName: grove-kind-topology
+      pack:
+        required: rack
+    podCliqueScalingGroups:
+    - name: decode
+      replicas: 2
+      minAvailable: 2
+      cliqueNames:
+      - decode-leader
+      - decode-worker
+    cliques:
+    - name: decode-leader
+      spec:
+        roleName: decode-leader
+        replicas: 1
+        minAvailable: 1
+        podSpec:
+          schedulerName: kueue
+          terminationGracePeriodSeconds: 5
+          containers:
+          - name: pause
+            image: registry.k8s.io/pause:3.9
+            resources:
+              requests:
+                cpu: 50m
+                memory: 32Mi
+    - name: decode-worker
+      spec:
+        roleName: decode-worker
+        replicas: 2
+        minAvailable: 2
+        podSpec:
+          schedulerName: kueue
+          terminationGracePeriodSeconds: 5
+          containers:
+          - name: pause
+            image: registry.k8s.io/pause:3.9
+            resources:
+              requests:
+                cpu: 50m
+                memory: 32Mi

docs/community/kueue-tas-backend-slides/demo/grove-kueue-simple-mincount.yaml

@@ -0,0 +1,34 @@
+# Grove PodCliqueSet with minCount semantics and no PodCliqueScalingGroup.
+# Grove's Kueue backend builds one prebuilt Kueue Workload per PodGang, with a
+# podSet whose count == clique replicas and minCount == clique minAvailable.
+# Apply after kueue-kind-queue.yaml and grove-kind-topology.yaml.
+apiVersion: grove.io/v1alpha1
+kind: PodCliqueSet
+metadata:
+  name: grove-kueue-mincount
+  labels:
+    app: grove-kueue-mincount
+    kueue.x-k8s.io/queue-name: grove-poc
+spec:
+  replicas: 1
+  template:
+    topologyConstraint:
+      topologyName: grove-kind-topology
+      pack:
+        required: rack
+    cliques:
+    - name: frontend
+      spec:
+        roleName: frontend
+        replicas: 4
+        minAvailable: 2
+        podSpec:
+          schedulerName: kueue
+          terminationGracePeriodSeconds: 5
+          containers:
+          - name: pause
+            image: registry.k8s.io/pause:3.9
+            resources:
+              requests:
+                cpu: 100m
+                memory: 64Mi

docs/community/kueue-tas-backend-slides/demo/kueue-kind-queue.yaml

@@ -0,0 +1,44 @@
+# CPU-only Kueue queue setup for kind smoke tests.
+# This is intentionally not the slide-deck GPU/TAS platform example.
+apiVersion: kueue.x-k8s.io/v1beta2
+kind: ResourceFlavor
+metadata:
+  name: kind-cpu
+spec:
+  nodeLabels:
+    topology.ai-dynamo.io/tas-node: "true"
+  topologyName: grove-kind-topology
+---
+apiVersion: kueue.x-k8s.io/v1beta2
+kind: ClusterQueue
+metadata:
+  name: kind-cpu
+spec:
+  namespaceSelector: {}
+  resourceGroups:
+  - coveredResources:
+    - cpu
+    - memory
+    flavors:
+    - name: kind-cpu
+      resources:
+      - name: cpu
+        nominalQuota: "4"
+      - name: memory
+        nominalQuota: 8Gi
+---
+apiVersion: kueue.x-k8s.io/v1beta2
+kind: LocalQueue
+metadata:
+  name: production-gpu
+  namespace: default
+spec:
+  clusterQueue: kind-cpu
+---
+apiVersion: kueue.x-k8s.io/v1beta2
+kind: LocalQueue
+metadata:
+  name: grove-poc
+  namespace: default
+spec:
+  clusterQueue: kind-cpu

docs/community/kueue-tas-backend-slides/demo/validate-grove-kueue-mincount.sh

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Validates the Grove Kueue "simple PCS with minCount" path: Grove builds one
+# prebuilt Kueue Workload per PodGang with a podSet whose count == clique
+# replicas and minCount == clique minAvailable, and stamps the Pods with
+# kueue.x-k8s.io/prebuilt-workload-name.
+
+KUBECTL="${KUBECTL:-kubectl}"
+NAMESPACE="${NAMESPACE:-default}"
+TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-120}"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+MANIFEST="${MANIFEST:-${SCRIPT_DIR}/grove-kueue-simple-mincount.yaml}"
+PCS_NAME="grove-kueue-mincount"
+PODGANG_NAME="${PCS_NAME}-0"
+
+cleanup() {
+  "${KUBECTL}" delete -f "${MANIFEST}" --ignore-not-found >/dev/null 2>&1 || true
+  "${KUBECTL}" -n "${NAMESPACE}" delete workload "${PODGANG_NAME}" --ignore-not-found >/dev/null 2>&1 || true
+}
+
+wait_for_jsonpath() {
+  local name="$1"
+  local jsonpath="$2"
+  local expected="$3"
+  local timeout_seconds="${4:-120}"
+  local deadline=$((SECONDS + timeout_seconds))
+
+  while (( SECONDS < deadline )); do
+    local actual
+    actual="$("${KUBECTL}" -n "${NAMESPACE}" get "${name}" -o "jsonpath=${jsonpath}" 2>/dev/null || true)"
+    if [[ "${actual}" == "${expected}" ]]; then
+      return 0
+    fi
+    sleep 2
+  done
+
+  echo "Timed out waiting for ${name} jsonpath ${jsonpath} to equal ${expected}" >&2
+  return 1
+}
+
+cleanup
+
+"${KUBECTL}" apply -f "${MANIFEST}" >/dev/null
+
+# Grove builds one prebuilt Workload named after the PodGang.
+wait_for_jsonpath \
+  "workload/${PODGANG_NAME}" \
+  "{.metadata.name}" \
+  "${PODGANG_NAME}" \
+  "${TIMEOUT_SECONDS}"
+
+wait_for_jsonpath "workload/${PODGANG_NAME}" "{.spec.podSets[0].count}" "4" "${TIMEOUT_SECONDS}"
+wait_for_jsonpath "workload/${PODGANG_NAME}" "{.spec.podSets[0].minCount}" "2" "${TIMEOUT_SECONDS}"
+
+# Pods reference the prebuilt Workload.
+pod="$("${KUBECTL}" -n "${NAMESPACE}" get pods \
+  -l "app.kubernetes.io/part-of=${PCS_NAME}" \
+  -o "jsonpath={.items[0].metadata.name}")"
+if [[ -z "${pod}" ]]; then
+  echo "Expected at least one Pod for PodCliqueSet ${PCS_NAME}" >&2
+  exit 1
+fi
+
+prebuilt="$("${KUBECTL}" -n "${NAMESPACE}" get pod "${pod}" \
+  -o "jsonpath={.metadata.labels.kueue\\.x-k8s\\.io/prebuilt-workload-name}")"
+if [[ "${prebuilt}" != "${PODGANG_NAME}" ]]; then
+  echo "Expected Pod ${pod} to carry kueue.x-k8s.io/prebuilt-workload-name=${PODGANG_NAME}, got '${prebuilt}'" >&2
+  exit 1
+fi
+
+echo "PASS: Grove created one prebuilt Workload named ${PODGANG_NAME}."
+echo "PASS: the Workload podSet has count=4 and minCount=2."
+echo "PASS: Pods reference the prebuilt Workload via kueue.x-k8s.io/prebuilt-workload-name."
+echo
+echo "Inspect:"
+echo "  ${KUBECTL} -n ${NAMESPACE} get workload ${PODGANG_NAME} -o yaml"
+echo "  ${KUBECTL} -n ${NAMESPACE} get pods -l app.kubernetes.io/part-of=${PCS_NAME} -o wide"
+echo
+echo "Cleanup:"
+echo "  ${KUBECTL} delete -f ${MANIFEST} --ignore-not-found"
+echo "  ${KUBECTL} -n ${NAMESPACE} delete workload ${PODGANG_NAME} --ignore-not-found"

docs/community/kueue-tas-backend-slides/demo/validate-grove-kueue-pcsg.sh

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Validates the Grove Kueue "PCS with PCSG" path. Grove pre-builds one Kueue
+# Workload per PodGang for every PodCliqueSet. PodCliqueScalingGroup cliques are
+# all-or-nothing: their podSets use minCount == count (POC assumption
+# PCSG.minAvailable == PCSG.replicas). Pods reference the prebuilt Workload.
+
+KUBECTL="${KUBECTL:-kubectl}"
+NAMESPACE="${NAMESPACE:-default}"
+TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-120}"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+MANIFEST="${MANIFEST:-${SCRIPT_DIR}/grove-kueue-pcsg.yaml}"
+PCS_NAME="grove-kueue-pcsg"
+PODGANG_NAME="${PCS_NAME}-0"
+# decode-leader(1) + decode-worker(2) per PCSG replica, times 2 replicas = 6.
+EXPECTED_TOTAL="6"
+
+cleanup() {
+  "${KUBECTL}" delete -f "${MANIFEST}" --ignore-not-found >/dev/null 2>&1 || true
+  "${KUBECTL}" -n "${NAMESPACE}" delete workload "${PODGANG_NAME}" --ignore-not-found >/dev/null 2>&1 || true
+}
+
+wait_for_jsonpath() {
+  local name="$1"
+  local jsonpath="$2"
+  local expected="$3"
+  local timeout_seconds="${4:-120}"
+  local deadline=$((SECONDS + timeout_seconds))
+
+  while (( SECONDS < deadline )); do
+    local actual
+    actual="$("${KUBECTL}" -n "${NAMESPACE}" get "${name}" -o "jsonpath=${jsonpath}" 2>/dev/null || true)"
+    if [[ "${actual}" == "${expected}" ]]; then
+      return 0
+    fi
+    sleep 2
+  done
+
+  echo "Timed out waiting for ${name} jsonpath ${jsonpath} to equal ${expected}" >&2
+  return 1
+}
+
+cleanup
+
+"${KUBECTL}" apply -f "${MANIFEST}" >/dev/null
+
+# Grove builds one prebuilt Workload named after the PodGang, even for PCSG.
+wait_for_jsonpath \
+  "workload/${PODGANG_NAME}" \
+  "{.metadata.name}" \
+  "${PODGANG_NAME}" \
+  "${TIMEOUT_SECONDS}"
+
+# Every PCSG podSet is all-or-nothing: minCount is omitted (Kueue defaults it to count, and Kueue rejects
+# Workloads where more than one podSet sets minCount).
+min_counts="$("${KUBECTL}" -n "${NAMESPACE}" get "workload/${PODGANG_NAME}" \
+  -o "jsonpath={range .spec.podSets[*]}{.minCount}{' '}{end}" | tr -d ' ')"
+if [[ -n "${min_counts}" ]]; then
+  echo "Expected no PCSG podSet to set minCount (all-or-nothing), got minCounts='${min_counts}'" >&2
+  exit 1
+fi
+
+# Pods reference the prebuilt Workload.
+pod="$("${KUBECTL}" -n "${NAMESPACE}" get pods \
+  -l "app.kubernetes.io/part-of=${PCS_NAME}" \
+  -o "jsonpath={.items[0].metadata.name}")"
+if [[ -z "${pod}" ]]; then
+  echo "Expected at least one Pod for PodCliqueSet ${PCS_NAME}" >&2
+  exit 1
+fi
+
+group="$("${KUBECTL}" -n "${NAMESPACE}" get pod "${pod}" \
+  -o "jsonpath={.metadata.labels.kueue\\.x-k8s\\.io/pod-group-name}")"
+if [[ "${group}" != "${PODGANG_NAME}" ]]; then
+  echo "Expected Pod ${pod} to use kueue.x-k8s.io/pod-group-name=${PODGANG_NAME}, got '${group}'" >&2
+  exit 1
+fi
+
+prebuilt="$("${KUBECTL}" -n "${NAMESPACE}" get pod "${pod}" \
+  -o "jsonpath={.metadata.labels.kueue\\.x-k8s\\.io/prebuilt-workload-name}")"
+if [[ "${prebuilt}" != "${PODGANG_NAME}" ]]; then
+  echo "Expected Pod ${pod} to carry kueue.x-k8s.io/prebuilt-workload-name=${PODGANG_NAME}, got '${prebuilt}'" >&2
+  exit 1
+fi
+
+total="$("${KUBECTL}" -n "${NAMESPACE}" get pod "${pod}" \
+  -o "jsonpath={.metadata.annotations.kueue\\.x-k8s\\.io/pod-group-total-count}")"
+if [[ "${total}" != "${EXPECTED_TOTAL}" ]]; then
+  echo "Expected pod-group-total-count=${EXPECTED_TOTAL}, got '${total}'" >&2
+  exit 1
+fi
+
+echo "PASS: Grove created one prebuilt Workload named ${PODGANG_NAME} for the PCSG PodCliqueSet."
+echo "PASS: every PCSG podSet is all-or-nothing (minCount omitted; Kueue allows minCount on at most one podSet)."
+echo "PASS: Pods reference the prebuilt Workload and share pod-group-name=${PODGANG_NAME} (total ${EXPECTED_TOTAL})."
+echo
+echo "Inspect:"
+echo "  ${KUBECTL} -n ${NAMESPACE} get workload ${PODGANG_NAME} -o yaml"
+echo "  ${KUBECTL} -n ${NAMESPACE} get pods -l app.kubernetes.io/part-of=${PCS_NAME} -o wide"
+echo
+echo "Cleanup:"
+echo "  ${KUBECTL} delete -f ${MANIFEST} --ignore-not-found"
+echo "  ${KUBECTL} -n ${NAMESPACE} delete workload ${PODGANG_NAME} --ignore-not-found"

operator/api/config/v1alpha1/types.go

@@ -56,6 +56,8 @@ type SchedulerName string
 const (
 	// SchedulerNameKai is the KAI scheduler backend.
 	SchedulerNameKai SchedulerName = "kai-scheduler"
+	// SchedulerNameKueue is the Kueue plain Pod integration backend.
+	SchedulerNameKueue SchedulerName = "kueue"
 	// SchedulerNameKube is the profile name for the Kubernetes default scheduler in OperatorConfiguration.
 	SchedulerNameKube SchedulerName = "default-scheduler"
 	// SchedulerNameVolcano is the Volcano scheduler backend. It supports gang scheduling via Volcano PodGroup.
@@ -68,6 +70,7 @@ var (
 	// SupportedSchedulerNames is the list of profile names allowed in scheduler.profiles[].name.
 	SupportedSchedulerNames = []SchedulerName{
 		SchedulerNameKai,
+		SchedulerNameKueue,
 		SchedulerNameKube,
 		SchedulerNameVolcano,
 		SchedulerNameLPX,
@@ -79,7 +82,7 @@ type SchedulerConfiguration struct {
 	// Profiles is the list of scheduler profiles. Each profile has a backend name and an optional config.
 	// The default-scheduler backend is always enabled to ensure that the kubernetes default scheduler is always enabled and supported.
 	// Use profile name "default-scheduler" to configure or set it as default.
-	// Valid profile names: "default-scheduler", "kai-scheduler", "volcano", "lpx-scheduler".
+	// Valid profile names: "default-scheduler", "kai-scheduler", "kueue", "volcano", "lpx-scheduler".
 	// Use defaultProfileName to designate the default backend.
 	// +optional
 	Profiles []SchedulerProfile `json:"profiles,omitempty"`
@@ -95,7 +98,7 @@ type SchedulerProfile struct {
 	// For the Kubernetes default scheduler use the standard "default-scheduler".
 	// Ensure that the name chosen is a valid scheduler name. The name will also be directly set in `Pod.Spec.SchedulerName`.
 	// +kubebuilder:validation:Required
-	// +kubebuilder:validation:Enum=kai-scheduler;default-scheduler;volcano;lpx-scheduler
+	// +kubebuilder:validation:Enum=kai-scheduler;kueue;default-scheduler;volcano;lpx-scheduler
 	Name SchedulerName `json:"name"`
 
 	// Config holds backend-specific options. The operator unmarshals it into the config type for this backend (see backend config types).
@@ -108,6 +111,16 @@ type KaiSchedulerConfiguration struct {
 	// Reserved for future kai-scheduler-specific options.
 }
 
+// KueueSchedulerConfiguration defines the configuration for the kueue backend.
+type KueueSchedulerConfiguration struct {
+	// RequiredTopologyKey is stamped as kueue.x-k8s.io/podset-required-topology when set.
+	// +optional
+	RequiredTopologyKey string `json:"requiredTopologyKey,omitempty"`
+	// UnderlyingSchedulerName is the schedulerName used after Kueue admits the Pods.
+	// +optional
+	UnderlyingSchedulerName string `json:"underlyingSchedulerName,omitempty"`
+}
+
 // KubeSchedulerConfig holds the configuration for the default scheduler.
 // Used when unmarshalling SchedulerProfile.Config for default-scheduler.
 type KubeSchedulerConfig struct {