feat: allow disjoint schedulers

operator/e2e/tests/scheduler_test.go

@@ -0,0 +1,81 @@
+//go:build e2e
+
+// /*
+// Copyright 2026 The Grove Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// */
+
+package tests
+
+import (
+	"context"
+	"testing"
+
+	apicommon "github.com/ai-dynamo/grove/operator/api/common"
+	configv1alpha1 "github.com/ai-dynamo/grove/operator/api/config/v1alpha1"
+	"github.com/ai-dynamo/grove/operator/e2e/testctx"
+)
+
+// Test_SR1_DisjointPodCliqueSchedulers verifies that each PodClique in a single
+// PodCliqueSet can select a different enabled scheduler backend. Besides
+// admission, this covers per-clique Pod preparation and successful scheduling.
+func Test_SR1_DisjointPodCliqueSchedulers(t *testing.T) {
+	const (
+		workloadName = "disjoint-schedulers"
+		expectedPods = 2
+	)
+
+	ctx := context.Background()
+	tc, cleanup := testctx.PrepareTest(ctx, t, expectedPods,
+		testctx.WithWorkload(&testctx.WorkloadConfig{
+			Name:         workloadName,
+			YAMLPath:     "../yaml/disjoint-schedulers.yaml",
+			Namespace:    "default",
+			ExpectedPods: expectedPods,
+		}),
+	)
+	defer cleanup()
+
+	pods, err := tc.DeployAndVerifyWorkload()
+	if err != nil {
+		t.Fatalf("PodCliqueSet with disjoint scheduler names was not admitted and reconciled: %v", err)
+	}
+
+	wantSchedulerByClique := map[string]string{
+		workloadName + "-0-kube-worker": string(configv1alpha1.SchedulerNameKube),
+		workloadName + "-0-kai-worker":  string(configv1alpha1.SchedulerNameKai),
+	}
+	for _, pod := range pods.Items {
+		cliqueName := pod.Labels[apicommon.LabelPodClique]
+		wantScheduler, ok := wantSchedulerByClique[cliqueName]
+		if !ok {
+			t.Errorf("pod %s has unexpected PodClique label %q", pod.Name, cliqueName)
+			continue
+		}
+		if pod.Spec.SchedulerName != wantScheduler {
+			t.Errorf("pod %s from PodClique %s uses scheduler %q, want %q", pod.Name, cliqueName, pod.Spec.SchedulerName, wantScheduler)
+		}
+		delete(wantSchedulerByClique, cliqueName)
+	}
+	for cliqueName := range wantSchedulerByClique {
+		t.Errorf("no pod was created for PodClique %s", cliqueName)
+	}
+	if t.Failed() {
+		return
+	}
+
+	if err := tc.WaitForPods(expectedPods); err != nil {
+		t.Fatalf("pods using disjoint scheduler names did not become ready: %v", err)
+	}
+}

operator/e2e/yaml/disjoint-schedulers.yaml

@@ -0,0 +1,72 @@
+# PodCliqueSet whose PodCliques select different enabled scheduler backends.
+---
+apiVersion: grove.io/v1alpha1
+kind: PodCliqueSet
+metadata:
+  name: disjoint-schedulers
+  labels:
+    app: disjoint-schedulers
+spec:
+  replicas: 1
+  template:
+    cliques:
+      - name: kube-worker
+        spec:
+          roleName: kube-worker
+          replicas: 1
+          minAvailable: 1
+          podSpec:
+            terminationGracePeriodSeconds: 5
+            schedulerName: default-scheduler
+            affinity:
+              nodeAffinity:
+                requiredDuringSchedulingIgnoredDuringExecution:
+                  nodeSelectorTerms:
+                    - matchExpressions:
+                        - key: node_role.e2e.grove.nvidia.com
+                          operator: In
+                          values:
+                            - agent
+            tolerations:
+              - key: node_role.e2e.grove.nvidia.com
+                operator: Equal
+                value: agent
+                effect: NoSchedule
+            containers:
+              - name: kube-worker
+                image: registry:5001/busybox:latest
+                command: ["sleep", "infinity"]
+                resources:
+                  requests:
+                    memory: 80Mi
+      - name: kai-worker
+        labels:
+          kai.scheduler/queue: test
+        spec:
+          roleName: kai-worker
+          replicas: 1
+          minAvailable: 1
+          podSpec:
+            terminationGracePeriodSeconds: 5
+            schedulerName: kai-scheduler
+            affinity:
+              nodeAffinity:
+                requiredDuringSchedulingIgnoredDuringExecution:
+                  nodeSelectorTerms:
+                    - matchExpressions:
+                        - key: node_role.e2e.grove.nvidia.com
+                          operator: In
+                          values:
+                            - agent
+            tolerations:
+              - key: node_role.e2e.grove.nvidia.com
+                operator: Equal
+                value: agent
+                effect: NoSchedule
+            containers:
+              - name: kai-worker
+                image: registry:5001/busybox:latest
+                command: ["sleep", "infinity"]
+                resources:
+                  requests:
+                    memory: 80Mi

operator/internal/controller/common/component/utils/scheduler.go

@@ -0,0 +1,76 @@
+// /*
+// Copyright 2026 The Grove Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// */
+
+package utils
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"maps"
+	"slices"
+	"strings"
+
+	grovecorev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1"
+	"github.com/ai-dynamo/grove/operator/internal/scheduler"
+)
+
+const maxLabelValueLength = 63
+
+// ResolveSchedulerName resolves an empty scheduler name to the configured
+// default and otherwise returns the registered backend's canonical name.
+func ResolveSchedulerName(schedRegistry scheduler.Registry, schedulerName string) string {
+	if schedRegistry == nil {
+		return schedulerName
+	}
+	if backend := schedRegistry.GetOrDefault(schedulerName); backend != nil {
+		return backend.Name()
+	}
+	return schedulerName
+}
+
+// SchedulerNamesForPodCliqueSet returns the distinct resolved scheduler names
+// selected by the PodClique templates in stable order.
+func SchedulerNamesForPodCliqueSet(pcs *grovecorev1alpha1.PodCliqueSet, schedRegistry scheduler.Registry) []string {
+	names := make(map[string]struct{}, len(pcs.Spec.Template.Cliques))
+	for _, clique := range pcs.Spec.Template.Cliques {
+		if clique == nil {
+			continue
+		}
+		name := ResolveSchedulerName(schedRegistry, clique.Spec.PodSpec.SchedulerName)
+		names[name] = struct{}{}
+	}
+	return slices.Sorted(maps.Keys(names))
+}
+
+// GenerateSchedulerScopedPodGangName returns the legacy PodGang name for a
+// single-scheduler PCS and a scheduler-qualified name for a disjoint PCS.
+func GenerateSchedulerScopedPodGangName(baseName string, pcs *grovecorev1alpha1.PodCliqueSet, schedulerName string, schedRegistry scheduler.Registry) string {
+	if len(SchedulerNamesForPodCliqueSet(pcs, schedRegistry)) <= 1 {
+		return baseName
+	}
+
+	resolvedSchedulerName := ResolveSchedulerName(schedRegistry, schedulerName)
+	candidate := fmt.Sprintf("%s-%s", baseName, resolvedSchedulerName)
+	if len(candidate) <= maxLabelValueLength {
+		return candidate
+	}
+
+	nameHash := sha256.Sum256([]byte(candidate))
+	hashSuffix := fmt.Sprintf("-%x", nameHash[:4])
+	maxBaseNameLength := maxLabelValueLength - len(hashSuffix)
+	trimmedBaseName := strings.TrimRight(baseName[:min(len(baseName), maxBaseNameLength)], "-.")
+	return trimmedBaseName + hashSuffix
+}

operator/internal/controller/common/component/utils/scheduler_test.go

@@ -0,0 +1,81 @@
+// /*
+// Copyright 2026 The Grove Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// */
+
+package utils
+
+import (
+	"strings"
+	"testing"
+
+	configv1alpha1 "github.com/ai-dynamo/grove/operator/api/config/v1alpha1"
+	grovecorev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1"
+	"github.com/ai-dynamo/grove/operator/internal/scheduler"
+	testutils "github.com/ai-dynamo/grove/operator/test/utils"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+)
+
+func TestGenerateSchedulerScopedPodGangName(t *testing.T) {
+	registry := &testutils.FakeSchedulerRegistry{
+		Backends: map[string]scheduler.Backend{
+			string(configv1alpha1.SchedulerNameKube): testutils.NewFakeSchedulerBackend(string(configv1alpha1.SchedulerNameKube)),
+			string(configv1alpha1.SchedulerNameKai):  testutils.NewFakeSchedulerBackend(string(configv1alpha1.SchedulerNameKai)),
+		},
+		DefaultBackend: string(configv1alpha1.SchedulerNameKube),
+	}
+
+	t.Run("preserves legacy name for a single scheduler", func(t *testing.T) {
+		pcs := podCliqueSetWithSchedulers("", string(configv1alpha1.SchedulerNameKube))
+		assert.Equal(t, "workload-0", GenerateSchedulerScopedPodGangName("workload-0", pcs, "", registry))
+	})
+
+	t.Run("qualifies each scheduler in a disjoint PCS", func(t *testing.T) {
+		pcs := podCliqueSetWithSchedulers("", string(configv1alpha1.SchedulerNameKai))
+		assert.Equal(t, "workload-0-default-scheduler", GenerateSchedulerScopedPodGangName("workload-0", pcs, "", registry))
+		assert.Equal(t, "workload-0-kai-scheduler", GenerateSchedulerScopedPodGangName("workload-0", pcs, string(configv1alpha1.SchedulerNameKai), registry))
+	})
+
+	t.Run("keeps label values within the Kubernetes limit", func(t *testing.T) {
+		pcs := podCliqueSetWithSchedulers("", string(configv1alpha1.SchedulerNameKai))
+		name := GenerateSchedulerScopedPodGangName(strings.Repeat("a", 60), pcs, string(configv1alpha1.SchedulerNameKai), registry)
+		require.Len(t, name, maxLabelValueLength)
+		assert.Regexp(t, `^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`, name)
+	})
+
+	t.Run("hashes a short base name when the scheduler suffix exceeds the limit", func(t *testing.T) {
+		pcs := podCliqueSetWithSchedulers("", string(configv1alpha1.SchedulerNameKai))
+		baseName := strings.Repeat("a", 44) + "-0"
+
+		name := GenerateSchedulerScopedPodGangName(baseName, pcs, "", registry)
+
+		require.Len(t, name, len(baseName)+9)
+		assert.Regexp(t, `^`+baseName+`-[a-f0-9]{8}$`, name)
+	})
+}
+
+func podCliqueSetWithSchedulers(schedulerNames ...string) *grovecorev1alpha1.PodCliqueSet {
+	pcs := &grovecorev1alpha1.PodCliqueSet{}
+	for _, schedulerName := range schedulerNames {
+		pcs.Spec.Template.Cliques = append(pcs.Spec.Template.Cliques, &grovecorev1alpha1.PodCliqueTemplateSpec{
+			Spec: grovecorev1alpha1.PodCliqueSpec{
+				PodSpec: corev1.PodSpec{SchedulerName: schedulerName},
+			},
+		})
+	}
+	return pcs
+}

operator/internal/controller/podcliquescalinggroup/components/podclique/podclique.go

@@ -33,6 +33,7 @@ import (
 	componentutils "github.com/ai-dynamo/grove/operator/internal/controller/common/component/utils"
 	groveerr "github.com/ai-dynamo/grove/operator/internal/errors"
 	"github.com/ai-dynamo/grove/operator/internal/mnnvl"
+	"github.com/ai-dynamo/grove/operator/internal/scheduler"
 	"github.com/ai-dynamo/grove/operator/internal/utils"
 	k8sutils "github.com/ai-dynamo/grove/operator/internal/utils/kubernetes"
 
@@ -74,14 +75,16 @@ type _resource struct {
 	client        client.Client
 	scheme        *runtime.Scheme
 	eventRecorder record.EventRecorder
+	schedRegistry scheduler.Registry
 }
 
 // New creates a new PodClique operator for managing PodClique resources within PodCliqueScalingGroups
-func New(client client.Client, scheme *runtime.Scheme, eventRecorder record.EventRecorder) component.Operator[grovecorev1alpha1.PodCliqueScalingGroup] {
+func New(client client.Client, scheme *runtime.Scheme, eventRecorder record.EventRecorder, schedRegistry scheduler.Registry) component.Operator[grovecorev1alpha1.PodCliqueScalingGroup] {
 	return &_resource{
 		client:        client,
 		scheme:        scheme,
 		eventRecorder: eventRecorder,
+		schedRegistry: schedRegistry,
 	}
 }
 
@@ -313,9 +316,16 @@ func (r _resource) buildResource(logger logr.Logger, pcs *grovecorev1alpha1.PodC
 		return err
 	}
 
-	podGangName := apicommon.GeneratePodGangNameForPodCliqueOwnedByPCSG(pcs, pcsReplicaIndex, pcsg, pcsgReplicaIndex)
+	basePodGangName := apicommon.GeneratePodGangNameForPodCliqueOwnedByPCSG(pcs, pcsReplicaIndex, pcsg, pcsgReplicaIndex)
+	podGangName := componentutils.GenerateSchedulerScopedPodGangName(basePodGangName, pcs, pclqTemplateSpec.Spec.PodSpec.SchedulerName, r.schedRegistry)
+	scopedBasePodGangName := componentutils.GenerateSchedulerScopedPodGangName(
+		apicommon.GenerateBasePodGangName(apicommon.ResourceNameReplica{Name: pcs.Name, Replica: pcsReplicaIndex}),
+		pcs,
+		pclqTemplateSpec.Spec.PodSpec.SchedulerName,
+		r.schedRegistry,
+	)
 
-	pclq.Labels = getLabels(pcs, pcsReplicaIndex, pcsg, pcsgReplicaIndex, pclqObjectKey, pclqTemplateSpec, podGangName)
+	pclq.Labels = getLabels(pcs, pcsReplicaIndex, pcsg, pcsgReplicaIndex, pclqObjectKey, pclqTemplateSpec, podGangName, scopedBasePodGangName)
 	pclq.Annotations = maps.Clone(pclqTemplateSpec.Annotations)
 	// PodGang owns topology selection; do not propagate a template topology annotation to PodClique pods.
 	delete(pclq.Annotations, apiconstants.AnnotationTopologyName)
@@ -469,7 +479,7 @@ func getPodCliqueSelectorLabels(pcsgObjectMeta metav1.ObjectMeta) map[string]str
 }
 
 // getLabels constructs the complete set of labels for a PodClique including Grove-specific, component, and template labels
-func getLabels(pcs *grovecorev1alpha1.PodCliqueSet, pcsReplicaIndex int, pcsg *grovecorev1alpha1.PodCliqueScalingGroup, pcsgReplicaIndex int, pclqObjectKey client.ObjectKey, pclqTemplateSpec *grovecorev1alpha1.PodCliqueTemplateSpec, podGangName string) map[string]string {
+func getLabels(pcs *grovecorev1alpha1.PodCliqueSet, pcsReplicaIndex int, pcsg *grovecorev1alpha1.PodCliqueScalingGroup, pcsgReplicaIndex int, pclqObjectKey client.ObjectKey, pclqTemplateSpec *grovecorev1alpha1.PodCliqueTemplateSpec, podGangName, basePodGangName string) map[string]string {
 	pclqComponentLabels := map[string]string{
 		apicommon.LabelAppNameKey:                        pclqObjectKey.Name,
 		apicommon.LabelComponentKey:                      apicommon.LabelComponentNamePodCliqueScalingGroupPodClique,
@@ -481,9 +491,6 @@ func getLabels(pcs *grovecorev1alpha1.PodCliqueSet, pcsReplicaIndex int, pcsg *g
 	}
 
 	// Add base-podgang label for scaled PodGang pods (beyond minAvailable)
-	basePodGangName := apicommon.GenerateBasePodGangName(
-		apicommon.ResourceNameReplica{Name: pcs.Name, Replica: pcsReplicaIndex},
-	)
 	if podGangName != basePodGangName {
 		// This pod belongs to a scaled PodGang - add the base PodGang label
 		pclqComponentLabels[apicommon.LabelBasePodGang] = basePodGangName

operator/internal/controller/podcliquescalinggroup/components/podclique/podclique_test.go

@@ -54,7 +54,7 @@ func TestNew(t *testing.T) {
 	client := fake.NewClientBuilder().WithScheme(scheme).Build()
 	eventRecorder := &record.FakeRecorder{}
 
-	operator := New(client, scheme, eventRecorder)
+	operator := New(client, scheme, eventRecorder, testutils.NewDefaultFakeRegistry())
 
 	assert.NotNil(t, operator)
 	resource, ok := operator.(*_resource)