diff --git a/docs/user/container-images.md b/docs/user/container-images.md index cf9055c90..7fe975e97 100644 --- a/docs/user/container-images.md +++ b/docs/user/container-images.md @@ -20,7 +20,7 @@ A machine-readable **CycloneDX 1.6 JSON** companion to this page is produced by ## Summary - Components: **44** -- Unique images: **99** +- Unique images: **103** - Distinct registries: **11** Registries: `602401143452.dkr.ecr.us-west-2.amazonaws.com`, `cr.agentgateway.dev`, `docker.io`, `gcr.io`, `ghcr.io`, `gke.gcr.io`, `nvcr.io`, `public.ecr.aws`, `quay.io`, `registry.k8s.io`, `us-docker.pkg.dev` @@ -56,7 +56,7 @@ _Rendering fidelity:_ `catalog-parity: charts are rendered with the shared recip | kueue | helm | kueue | 0.18.2 | 1 | | mariadb-operator | helm | mariadb-operator | 26.6.0 | 1 | | mariadb-operator-crds | helm | mariadb-operator-crds | 26.6.0 | 0 | -| network-operator | helm | nvidia/network-operator | 26.4.1 | 5 | +| network-operator | helm | nvidia/network-operator | 26.4.1 | 9 | | network-operator-ocp | manifest | — | — | 0 | | network-operator-ocp-olm | manifest | — | — | 0 | | nfd | helm | node-feature-discovery | 0.19.0 | 1 | @@ -240,6 +240,10 @@ _No images extracted._ ### network-operator - `docker.io/library/busybox:1.38.0@sha256:dc2d74b28e4cf8984fa52af1f39bc7c3d9c73760b41a74d629f5d11b1ab28616` +- `ghcr.io/k8snetworkplumbingwg/multus-cni:v4.2.1` +- `ghcr.io/k8snetworkplumbingwg/plugins:v1.6.2-update.1` +- `ghcr.io/k8snetworkplumbingwg/sriov-network-device-plugin:v3.9.0` +- `ghcr.io/mellanox/nvidia-k8s-ipam:v0.2.0` - `nvcr.io/nvidia/cloud-native/network-operator:v26.4.1` - `nvcr.io/nvidia/doca/doca_telemetry:1.22.5-doca3.1.0-host` - `nvcr.io/nvidia/mellanox/doca-driver:doca3.2.0-25.10-1.2.8.0-2` diff --git a/pkg/bundler/testdata/stock_render_golden.yaml b/pkg/bundler/testdata/stock_render_golden.yaml index d7380d12c..680f35ec9 100644 --- a/pkg/bundler/testdata/stock_render_golden.yaml +++ b/pkg/bundler/testdata/stock_render_golden.yaml @@ -17,7 +17,7 @@ gb200-eks-ubuntu-inference-dynamo: e6c0d3837b99ad66586170448dcb45a9918acd4ae2422 gb200-eks-ubuntu-training-kubeflow: ba426f4ef5cbf4694240672e03bd9e4ca8db611c905271019597945b09ed0d36 gb200-eks-ubuntu-training-slurm: a9261f38f32d264191d271266c683d25bdd9a0a9e5ed874724a51a3b59e30675 gb200-oke-ubuntu-inference-dynamo: 354f1904b1c09f9216f29786cbcdb2c4238b0b6e807244b6b6c8ddbc1087c900 -gb200-oke-ubuntu-training-kubeflow: fae79d77329020822102973f9ba01049077bb51ab44b043f2e5c0c80ba3ee248 +gb200-oke-ubuntu-training-kubeflow: baf8f6ecd08e6ba9ddd94fd9557eb7472c6822f31d407de788e4dd6a7a166106 gb300-any: a707d3e822046e6cfbed77deecf18e7017dd827f026d23fa6128a8ae038eaf66 gb300-eks-ubuntu-inference-dynamo: 98c6b918c3a8c2e8ca1f2254957a219296add77494b1a8613b31b9ab91b782da gb300-eks-ubuntu-training-kubeflow: d8f9462fe98c6b1224e92fc16e3f3e1c8f35f433e659599ea0c252c9cd4f9a56 @@ -41,7 +41,7 @@ h200-eks-inference: dfba31bc08abd37181aad00bf68cc9983b82f1abe9172700d2d184c48f1a h200-eks-training: 44bfaf2d63b2e21797095a5b2467294eb4f3ee6b148314a0a96d08054e06bc7b l40s-any: ec32a4b689457a781f99fe75cad4182c20ee8b22c90fa665bce52ebdc6971213 l40s-oke-inference: 15e05a20bee6cd7e4b760d26c480f87a706fc8c176cb0ab8a91fa95dbfafdd29 -l40s-oke-training: 4ee65b66ec8d0967094ab91b29efa12b2062cac25b9f2ca7e36a00ecb0639056 +l40s-oke-training: 29cd0d200f6139d888c0359614a4b169e5363da9f489a481fb94e2659a92dc6e monitoring-hpa: a0f8fbabea4a7cb67c0c57317ef17cd8bc18c1f801f84cd752bb14df8f7e8055 ocp-inference-nim: 016fc59c13e901de5556f09f3cb3d5a90f91b9826d34f126fde1e066b311605f ocp-training: 652f7ccdf52009adafe993da111857aa67390678f382b4c0caf0aecd0d613635 diff --git a/pkg/recipe/performance_goals_oke_test.go b/pkg/recipe/performance_goals_oke_test.go index b6e8e77d8..18a32d742 100644 --- a/pkg/recipe/performance_goals_oke_test.go +++ b/pkg/recipe/performance_goals_oke_test.go @@ -34,22 +34,25 @@ func TestOKEPerformanceGoalsFollowTrainingInferencePattern(t *testing.T) { }{ { name: "gb200-oke-training", - wantChecks: []string{"nccl-all-reduce-bw-nvls"}, + wantChecks: []string{"nccl-all-reduce-bw-net", "nccl-all-reduce-bw-nvls"}, wantConstraints: map[string]string{ + "nccl-all-reduce-bw-net": ">= 40", "nccl-all-reduce-bw-nvls": ">= 500", }, }, { name: "gb200-oke-ubuntu-training", - wantChecks: []string{"nccl-all-reduce-bw-nvls"}, + wantChecks: []string{"nccl-all-reduce-bw-net", "nccl-all-reduce-bw-nvls"}, wantConstraints: map[string]string{ + "nccl-all-reduce-bw-net": ">= 40", "nccl-all-reduce-bw-nvls": ">= 500", }, }, { name: "gb200-oke-ubuntu-training-kubeflow", - wantChecks: []string{"nccl-all-reduce-bw-nvls"}, + wantChecks: []string{"nccl-all-reduce-bw-net", "nccl-all-reduce-bw-nvls"}, wantConstraints: map[string]string{ + "nccl-all-reduce-bw-net": ">= 40", "nccl-all-reduce-bw-nvls": ">= 500", }, }, diff --git a/pkg/recipe/testdata/catalog_parity_golden.yaml b/pkg/recipe/testdata/catalog_parity_golden.yaml index f3569018f..2db49033c 100644 --- a/pkg/recipe/testdata/catalog_parity_golden.yaml +++ b/pkg/recipe/testdata/catalog_parity_golden.yaml @@ -17,7 +17,7 @@ gb200-eks-ubuntu-inference-dynamo: 540292f9c9c4c0a59bfd8ab7cd03738d32a96fe380bab gb200-eks-ubuntu-training-kubeflow: bf2be525e4bf0ec8fa98988f6d01939dac21cabe06cf2526cd42488b9961ee2d gb200-eks-ubuntu-training-slurm: 2ed987ee1adf483fa04ceebf6ac6f5433db69eb65a138085744e3a4f11efd68e gb200-oke-ubuntu-inference-dynamo: 55b2a4514e5f774d819d02c2370254152ba1febc59947e6ec74d89a7c07d59d1 -gb200-oke-ubuntu-training-kubeflow: 29c49c4ec3bc40c711b40fae53d7be9a0261ff1ca7ad5b971878192bc5a0085d +gb200-oke-ubuntu-training-kubeflow: 1623885a9abede27ce49d8113323a76a0df3e75bb941edf6daa74854dcb48d00 gb300-any: 86219d16100b8708cf7fa8f24d7ca5992afd8677be5195e5f2637592ff00be31 gb300-eks-ubuntu-inference-dynamo: c30b877b0670396b1f6182e0fe7e491ccd1ea4b4a55379c53f9e22d51adcf40b gb300-eks-ubuntu-training-kubeflow: f80319d540d3f814edf72a896a90c02e1f6eca85590d224e2ddb84a92d9b50ce @@ -41,7 +41,7 @@ h200-eks-inference: e99c57d64be024d5180e70c58be17f4d3dfc08ac3b730840d924de807783 h200-eks-training: 61d684ba87b90e60dd27bfba231c811f020e4577127df1ca30505920c1efc37b l40s-any: 0210e041b3ffc84e6b7414ba40e5156496ccd61253483074fbc27a9e09957ba0 l40s-oke-inference: b4f3de89404d53779a87d24cd601775487f39124d4e84d8139f9a91e7d80dcd9 -l40s-oke-training: 2c7bdcb28b68f12075f05104ee2d77919f7492b79f67432fe0de90fbbb8b995e +l40s-oke-training: d7fae25290a9e2782f094ceacd06a3b68e95c7a5c22c849bd6396b32db91ae7f monitoring-hpa: 6ddb7b4bc162aa46e2b4e565853c83ca4e9b89cf49ebcc5fa66e6f1bc7b66502 ocp-inference-nim: 85d8abf6b1c516fbfbc6285367ef55f0a4c25b1d0b93e99d94ca1e146f9be46e ocp-training: 335a9679e6de0b0e643deb55f34507220b378f3965a119fdb72d9d0c9adad147 diff --git a/recipes/components/network-operator/manifests/nic-cluster-policy-oke-gb200.yaml b/recipes/components/network-operator/manifests/nic-cluster-policy-oke-gb200.yaml new file mode 100644 index 000000000..66161dbca --- /dev/null +++ b/recipes/components/network-operator/manifests/nic-cluster-policy-oke-gb200.yaml @@ -0,0 +1,48 @@ +# NicClusterPolicy for GB200 OKE (OCI) — rdmaSharedDevicePlugin over InfiniBand. +# +# Mirrors the AOR OCI GB200 config validated on gb200-ew. No ofedDriver (host +# MOFED), no SR-IOV: the NVL72 east-west fabric is IB on the rdma0-3 netdevs +# (oci_hpc.rdma_device_names_mode=2 kernel cmdline names them deterministically). +# +# The IB devices are advertised as nvidia.com/mlnxnics — the same resource +# name the L40S SR-IOV path uses, so workloads request RDMA uniformly +# across OKE fabrics. +apiVersion: mellanox.com/v1alpha1 +kind: NicClusterPolicy +metadata: + name: nic-cluster-policy + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "5" + helm.sh/hook-delete-policy: before-hook-creation + labels: + app.kubernetes.io/managed-by: {{ .Release.Service }} + helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +spec: + rdmaSharedDevicePlugin: + image: k8s-rdma-shared-dev-plugin + repository: nvcr.io/nvidia/mellanox + version: network-operator-v26.4.1 + config: | + { + "configList": [ + { + "resourcePrefix": "nvidia.com", + "resourceName": "mlnxnics", + "rdmaHcaMax": 63, + "selectors": { + "linkTypes": ["infiniband"], + "ifNames": ["rdma0", "rdma1", "rdma2", "rdma3"] + } + } + ] + } + deploymentTolerations: + - key: CriticalAddonsOnly + operator: Exists + tolerations: + # RDMA DaemonSets must land on tainted GPU nodes. + - key: nvidia.com/gpu + operator: Exists + - key: CriticalAddonsOnly + operator: Exists diff --git a/recipes/components/network-operator/manifests/nic-cluster-policy-oke-l40s.yaml b/recipes/components/network-operator/manifests/nic-cluster-policy-oke-l40s.yaml new file mode 100644 index 000000000..0a78aacca --- /dev/null +++ b/recipes/components/network-operator/manifests/nic-cluster-policy-oke-l40s.yaml @@ -0,0 +1,73 @@ +# NicClusterPolicy for L40S OKE (OCI) SR-IOV RoCE. +# +# The network-operator Helm chart installs the operator + CRD but does not template +# a NicClusterPolicy CR (values-oke-l40s.yaml sets deployCR: false). This manifest +# creates it so the operator reconciles the RoCE fabric stack. Hand-rendered from +# AOR's network-operator/nicclusterpolicy.yaml.tmpl (provider: oci branch, with +# network.type == roce → nvIpam + secondaryNetwork included). +# +# OCI specifics (vs Forge IB): NO ofedDriver — OCI nodes carry host MOFED, consumed +# by the GPU Operator driver via driver.rdma.useHostMofed (l40s-oke-ubuntu leaf). One +# sriovDevicePlugin resource, nvidia.com/mlnxnics, selecting the OCI ConnectX VF +# device IDs (101a = ConnectX-5 Ex VF, 101e = mlx5Gen VF). RoCE also needs nv-ipam +# (VF IP allocation) + secondaryNetwork/multus (attach the VF into workload pods). +# vendor 15b3 = Mellanox. +apiVersion: mellanox.com/v1alpha1 +kind: NicClusterPolicy +metadata: + name: nic-cluster-policy + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "5" + helm.sh/hook-delete-policy: before-hook-creation + labels: + app.kubernetes.io/managed-by: {{ .Release.Service }} + helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +spec: + # RoCE: allocate IPs for the RDMA VFs and wire them into pods via multus. + nvIpam: + image: nvidia-k8s-ipam + repository: ghcr.io/mellanox + version: v0.2.0 + enableWebhook: false + containerResources: + - name: nv-ipam-node + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "1" + memory: 2Gi + secondaryNetwork: + cniPlugins: + image: plugins + repository: ghcr.io/k8snetworkplumbingwg + version: v1.6.2-update.1 + multus: + image: multus-cni + repository: ghcr.io/k8snetworkplumbingwg + version: v4.2.1 + sriovDevicePlugin: + image: sriov-network-device-plugin + repository: ghcr.io/k8snetworkplumbingwg + version: v3.9.0 + config: | + { + "resourceList": [ + { + "resourcePrefix": "nvidia.com", + "resourceName": "mlnxnics", + "selectors": {"isRdma":true,"vendors":["15b3"],"devices":["101a","101e"]} + } + ] + } + # Operator DaemonSet placement: system/monitoring nodes only (matches AOR). + deploymentTolerations: + - key: CriticalAddonsOnly + operator: Exists + tolerations: + # RDMA DaemonSets must land on tainted GPU nodes. + - key: nvidia.com/gpu + operator: Exists + - key: CriticalAddonsOnly + operator: Exists diff --git a/recipes/components/network-operator/values-oke-gb200.yaml b/recipes/components/network-operator/values-oke-gb200.yaml new file mode 100644 index 000000000..6211096f4 --- /dev/null +++ b/recipes/components/network-operator/values-oke-gb200.yaml @@ -0,0 +1,25 @@ +# network-operator Helm values for GB200 OKE (OCI) InfiniBand. +# +# OCI GB200 NVL72 model (vs L40S RoCE / Forge IB): NO ofedDriver — nodes carry host +# MOFED — and no SR-IOV/nv-ipam/multus either. East-west is InfiniBand (rdma0-3), +# served by rdmaSharedDevicePlugin from the post-install NicClusterPolicy manifest, +# NOT the chart. deployCR off so that manifest CR is authoritative. +# nfd.enabled: false — GPU Operator's NFD is used; no second NFD. +deployCR: false +nvIpam: + enabled: false +secondaryNetwork: + deploy: false +nfd: + enabled: false +operator: + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: 500m + memory: 2Gi + # Operator placement comes from the bundler's system-node scheduling + # injection (registry nodeScheduling: operator.nodeSelector / + # operator.tolerations) — no hardcoded affinity here. diff --git a/recipes/components/network-operator/values-oke-l40s.yaml b/recipes/components/network-operator/values-oke-l40s.yaml new file mode 100644 index 000000000..109a16ac9 --- /dev/null +++ b/recipes/components/network-operator/values-oke-l40s.yaml @@ -0,0 +1,34 @@ +# network-operator Helm values for L40S OKE (OCI) SR-IOV RoCE. +# Hand-rendered from AOR's network-operator/values.yaml.tmpl (provider: oci) + +# nicclusterpolicy.yaml.tmpl (oci branch, network.type == roce). +# +# OCI model (vs Forge IB / Mistral DOCA): NO ofedDriver — OCI bare-metal nodes carry +# host MOFED, so the GPU Operator uses it via driver.rdma.useHostMofed (set on the +# l40s-oke-ubuntu leaf). network-operator's job here is the SR-IOV VF device plugin +# (advertises nvidia.com/mlnxnics RDMA VFs) plus nv-ipam + secondaryNetwork (multus) +# for RoCE — all supplied by the post-install NicClusterPolicy manifest, NOT the chart. +# +# deployCR/nvIpam/secondaryNetwork: AICR's wrapper defaults are on (deployCR: true, +# nvIpam.enabled: true, secondaryNetwork.deploy: true) — they template the wrapper's +# own NicClusterPolicy. Turn deployCR off so our manifest CR is authoritative (it is +# the only place the OCI VF selectors 101a/101e can be expressed); the operator +# reconciles nv-ipam + secondaryNetwork + sriovDevicePlugin from that CR regardless. +# nfd.enabled: false — GPU Operator's NFD is used; no second NFD. +deployCR: false +nvIpam: + enabled: false +secondaryNetwork: + deploy: false +nfd: + enabled: false +operator: + resources: + limits: + cpu: "1" + memory: 2Gi + requests: + cpu: 500m + memory: 2Gi + # Operator placement comes from the bundler's system-node scheduling + # injection (registry nodeScheduling: operator.nodeSelector / + # operator.tolerations) — no hardcoded affinity here. diff --git a/recipes/manifest_images_test.go b/recipes/manifest_images_test.go index 23461a37e..1d45458c8 100644 --- a/recipes/manifest_images_test.go +++ b/recipes/manifest_images_test.go @@ -106,6 +106,12 @@ var imageDigestExemptions = map[string]string{ // registry to pin a digest against, the "digest" differs per COS build, // and the ref must never be mirrored or pulled. Issue #1716. "cos-nvidia-installer:fixed": "COS-node-local preloaded image (imagePullPolicy: Never); no registry digest exists and it must not be mirrored; issue #1716", + // NicClusterPolicy (network-operator OKE): same repository/image/version + // triplet schema as the AKS entries above — no digest field in the CRD. + "ghcr.io/mellanox/nvidia-k8s-ipam:v0.2.0": "NicClusterPolicy CRD does not accept image digests; tracked via #745 and Mellanox/network-operator#2555", + "ghcr.io/k8snetworkplumbingwg/multus-cni:v4.2.1": "NicClusterPolicy CRD does not accept image digests; tracked via #745 and Mellanox/network-operator#2555", + "ghcr.io/k8snetworkplumbingwg/plugins:v1.6.2-update.1": "NicClusterPolicy CRD does not accept image digests; tracked via #745 and Mellanox/network-operator#2555", + "ghcr.io/k8snetworkplumbingwg/sriov-network-device-plugin:v3.9.0": "NicClusterPolicy CRD does not accept image digests; tracked via #745 and Mellanox/network-operator#2555", } // TestComponentManifestImagesAreDigestPinned asserts that every image diff --git a/recipes/overlays/gb200-oke-training.yaml b/recipes/overlays/gb200-oke-training.yaml index d9543c6b7..eeff79b0d 100644 --- a/recipes/overlays/gb200-oke-training.yaml +++ b/recipes/overlays/gb200-oke-training.yaml @@ -39,9 +39,15 @@ spec: value: ">= 1.34" componentRefs: - # GB200-specific GPU Operator overrides (inherits valuesFile from oke-training) + # GB200-specific GPU Operator overrides (inherits valuesFile from oke-training). + # kernel-module-params sets NVreg_GrdmaPciTopoCheckOverride=1, required + # for dma-buf attach over the IB fabric (GPUDirect RDMA without + # nvidia-peermem, whose chroot modprobe fails to build against the -64k + # Grace kernel). - name: gpu-operator type: Helm + preManifestFiles: + - components/gpu-operator/manifests/kernel-module-params.yaml dependencyRefs: - nfd - cert-manager @@ -49,6 +55,9 @@ spec: overrides: gdrcopy: enabled: true + driver: + kernelModuleConfig: + name: nvidia-kernel-module-params - name: nfd type: Helm @@ -56,13 +65,33 @@ spec: topologyUpdater: enable: true + # InfiniBand east-west fabric (NVL72 rdma0-3). rdmaSharedDevicePlugin + # advertises the shared HCAs as nvidia.com/mlnxnics; no SR-IOV/nv-ipam + # (that is the L40S RoCE path) and no ofedDriver (OCI nodes carry host + # MOFED). NicClusterPolicy is manifest-supplied (chart deployCR off). + # Present in every gpuStack value; incompatible with Oracle's opt-in + # NvidiaNetworkOperator add-on. + - name: network-operator + type: Helm + valuesFile: components/network-operator/values-oke-gb200.yaml + manifestFiles: + - components/network-operator/manifests/nic-cluster-policy-oke-gb200.yaml + dependencyRefs: + - nfd + - cert-manager + validation: performance: - # NVLS runtime support is OKE-specific. NET/RDMA is intentionally left - # out until OCI-specific pod RDMA exposure is verified on the testbed. + # Both transport variants: NVLS (MNNVL across the NVL72 IMEX domain) + # and NET (the IB east-west fabric this leaf's NicClusterPolicy + # exposes — validated on a BM.GPU.GB200.4 NVL72 rack). Constraints + # match gb200-eks-training. checks: + - nccl-all-reduce-bw-net - nccl-all-reduce-bw-nvls constraints: + - name: nccl-all-reduce-bw-net + value: ">= 40" - name: nccl-all-reduce-bw-nvls value: ">= 500" conformance: diff --git a/recipes/overlays/l40s-oke-training.yaml b/recipes/overlays/l40s-oke-training.yaml index 6314819d2..e3c756e75 100644 --- a/recipes/overlays/l40s-oke-training.yaml +++ b/recipes/overlays/l40s-oke-training.yaml @@ -54,6 +54,26 @@ spec: topologyUpdater: enable: true + # RDMA fabric (RoCE over SR-IOV VFs). Every L40S OCI cluster runs RoCE; + # the SR-IOV VF device plugin advertises nvidia.com/mlnxnics RDMA VFs, + # with nv-ipam + multus attaching the VFs into workload pods. The + # NicClusterPolicy is supplied by the manifest (the chart's deployCR is + # off — the manifest is the only place the OCI VF selectors 101a/101e + # can be expressed). OCI nodes carry host MOFED, so there is no + # ofedDriver in any configuration. GPUDirect RDMA works via DMA-BUF; + # nvidia-peermem stays off (base default) — inert on this topology. + # Present in every gpuStack value: the fabric is orthogonal to GPU + # driver/plugin ownership. Incompatible with Oracle's opt-in + # NvidiaNetworkOperator add-on (two lifecycle managers, one release). + - name: network-operator + type: Helm + valuesFile: components/network-operator/values-oke-l40s.yaml + manifestFiles: + - components/network-operator/manifests/nic-cluster-policy-oke-l40s.yaml + dependencyRefs: + - nfd + - cert-manager + # Validation checks for L40S on OKE training workloads. # Defined at the intent layer (not OS-specific) so all OS variants inherit them. # diff --git a/validators/performance/nccl_all_reduce_bw_constraint.go b/validators/performance/nccl_all_reduce_bw_constraint.go index 92365f519..d50f49e26 100644 --- a/validators/performance/nccl_all_reduce_bw_constraint.go +++ b/validators/performance/nccl_all_reduce_bw_constraint.go @@ -247,6 +247,10 @@ var supportedNCCLCombinations = map[ncclVariant]map[recipe.CriteriaServiceType][ }, variantNET: { recipe.CriteriaServiceEKS: {recipe.CriteriaAcceleratorGB200}, + // OKE GB200 NVL72: IB east-west (rdma0-3) via the + // rdmaSharedDevicePlugin's nvidia.com/mlnxnics shared HCAs — + // see testdata/gb200/oke/runtime-net.yaml. + recipe.CriteriaServiceOKE: {recipe.CriteriaAcceleratorGB200}, }, variantNVLS: { recipe.CriteriaServiceEKS: {recipe.CriteriaAcceleratorGB200}, diff --git a/validators/performance/nccl_benchmark_profile_test.go b/validators/performance/nccl_benchmark_profile_test.go index bedc0f876..99044c2ac 100644 --- a/validators/performance/nccl_benchmark_profile_test.go +++ b/validators/performance/nccl_benchmark_profile_test.go @@ -156,7 +156,7 @@ func TestNCCLCombinationSupported(t *testing.T) { {"default B200 any", variantDefault, fabricEFA, target(recipe.CriteriaAcceleratorB200, recipe.CriteriaServiceAny), true}, {"default GB200 EKS not covered", variantDefault, fabricEFA, target(recipe.CriteriaAcceleratorGB200, recipe.CriteriaServiceEKS), false}, {"NET GB200 EKS", variantNET, fabricEFA, target(recipe.CriteriaAcceleratorGB200, recipe.CriteriaServiceEKS), true}, - {"NET GB200 OKE not covered", variantNET, fabricEFA, target(recipe.CriteriaAcceleratorGB200, recipe.CriteriaServiceOKE), false}, + {"NET GB200 OKE (IB via rdmaSharedDevicePlugin)", variantNET, fabricEFA, target(recipe.CriteriaAcceleratorGB200, recipe.CriteriaServiceOKE), true}, {"NVLS GB200 EKS", variantNVLS, fabricEFA, target(recipe.CriteriaAcceleratorGB200, recipe.CriteriaServiceEKS), true}, {"NVLS GB200 OKE", variantNVLS, fabricEFA, target(recipe.CriteriaAcceleratorGB200, recipe.CriteriaServiceOKE), true}, {"unknown service", variantNVLS, fabricEFA, target(recipe.CriteriaAcceleratorGB200, "custom-svc"), false}, diff --git a/validators/performance/testdata/gb200/oke/runtime-net.yaml b/validators/performance/testdata/gb200/oke/runtime-net.yaml new file mode 100644 index 000000000..7ae7aea32 --- /dev/null +++ b/validators/performance/testdata/gb200/oke/runtime-net.yaml @@ -0,0 +1,208 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NCCL all-reduce TrainingRuntime for GB200 on OKE, NET/IB-transport variant. +# +# Measures the InfiniBand east-west fabric (rdma0-3, served by the +# rdmaSharedDevicePlugin from the gb200-oke NicClusterPolicy) instead of +# NVLink: NVLS and MNNVL are forced off so NCCL falls back to its built-in +# IB transport over the nvidia.com/mlnxnics shared HCAs. NCCL_DEBUG stays +# INFO — verifyTransportFromLogs confirms the transport by parsing the +# "NCCL INFO Using network" banner from launcher logs. +# +# Composition: the gb200/oke runtime-nvls.yaml scaffold (pytorch image, +# mpirun ssh wiring) with the IMEX resource claims removed and the +# net-variant env from gb200/eks runtime-net.yaml, minus the EFA provider +# knobs. Node pods request nvidia.com/mlnxnics alongside GPUs. +# +# Cluster prerequisites: rdmaSharedDevicePlugin healthy (nvidia.com/mlnxnics +# allocatable on every GPU node — the gb200-oke leaves' NicClusterPolicy), +# and NVreg_GrdmaPciTopoCheckOverride=1 from the leaf's kernel-module-params +# ConfigMap (dma-buf attach over the IB fabric). + +apiVersion: trainer.kubeflow.org/v1alpha1 +kind: TrainingRuntime +metadata: + name: nccl-all-reduce-runtime + namespace: ${NAMESPACE} + labels: + trainer.kubeflow.org/framework: mpi +spec: + mlPolicy: + mpi: + mpiImplementation: OpenMPI + numProcPerNode: ${GPU_COUNT_PER_NODE} + runLauncherAsNode: false + sshAuthMountPath: /tmp/mpi-keys + template: + spec: + network: + enableDNSHostnames: true + publishNotReadyAddresses: true + replicatedJobs: + - name: launcher + replicas: 1 + template: + spec: + template: + spec: + tolerations: + - operator: Exists + initContainers: + - name: fix-ssh-perms + image: nvcr.io/nvidia/pytorch:25.06-py3 + command: + - /bin/sh + - -c + - | + mkdir -p /root/.ssh + cp /tmp/mpi-keys/id_rsa /root/.ssh/id_rsa + cp /tmp/mpi-keys/authorized_keys /root/.ssh/authorized_keys + chmod 700 /root/.ssh + chmod 600 /root/.ssh/id_rsa /root/.ssh/authorized_keys + volumeMounts: + - name: mpi-ssh-auth + mountPath: /tmp/mpi-keys + readOnly: true + - name: ssh-config + mountPath: /root/.ssh + containers: + - name: node + image: nvcr.io/nvidia/pytorch:25.06-py3 + env: + - name: LD_LIBRARY_PATH + value: "/usr/local/nvidia/lib64:/usr/local/cuda/lib64" + command: + - /usr/local/mpi/bin/mpirun + args: + - -np + - "${GPU_COUNT}" + - --allow-run-as-root + - --mca + - plm_rsh_args + - -o StrictHostKeyChecking=no -o ConnectionAttempts=10 + - --mca + - btl + - ^openib + - --mca + - btl_tcp_if_include + - eth0 + - --mca + - oob_tcp_if_include + - eth0 + - -x + - LD_LIBRARY_PATH + - -x + - NCCL_DEBUG=INFO + # Force NCCL onto the NET/IB transport: disable NVLS (NVLink + # SHARP) and MNNVL (multi-node NVLink) so traffic crosses the + # IB fabric; NCCL_NET_PLUGIN=none selects the built-in IB + # verbs transport over the shared mlx5 HCAs. + - -x + - NCCL_NVLS_ENABLE=0 + - -x + - NCCL_MNNVL_ENABLE=0 + - -x + - NCCL_NET_PLUGIN=none + - -x + - NCCL_SOCKET_IFNAME=eth0 + - -x + - NCCL_IGNORE_DISABLED_P2P=1 + - /usr/local/bin/${TEST_TYPE}_mpi + - -b + - ${MIN_MESSAGE_SIZE} + - -e + - ${MAX_MESSAGE_SIZE} + - -f + - "2" + - -g + - "1" + resources: + limits: + cpu: "2" + memory: 128Mi + volumeMounts: + - name: ssh-config + mountPath: /root/.ssh + volumes: + - name: ssh-config + emptyDir: {} + - name: node + template: + spec: + template: + spec: + tolerations: + - operator: Exists + initContainers: + - name: fix-ssh-perms + image: nvcr.io/nvidia/pytorch:25.06-py3 + command: + - /bin/sh + - -c + - | + apt-get update && + apt-get install -y --no-install-recommends openssh-server && + mkdir -p /var/run/sshd && + chmod 0755 /var/run/sshd && + mkdir -p /root/.ssh && + cp /tmp/mpi-keys/authorized_keys /root/.ssh/authorized_keys && + chmod 700 /root/.ssh && + chmod 600 /root/.ssh/authorized_keys + volumeMounts: + - name: mpi-ssh-auth + mountPath: /tmp/mpi-keys + readOnly: true + - name: ssh-config + mountPath: /root/.ssh + containers: + - name: node + image: nvcr.io/nvidia/pytorch:25.06-py3 + command: ["sh", "-c"] + args: + - | + apt-get update && + apt-get install -y --no-install-recommends openssh-server && + mkdir -p /var/run/sshd && + chmod 0755 /var/run/sshd && + mkdir -p /root/.ssh && + cp /tmp/mpi-keys/* /root/.ssh/ && + chmod 700 /root/.ssh && + chmod 600 /root/.ssh/authorized_keys && + /usr/sbin/sshd -De + resources: + limits: + nvidia.com/gpu: ${GPU_COUNT_PER_NODE} + nvidia.com/mlnxnics: "1" + requests: + nvidia.com/gpu: ${GPU_COUNT_PER_NODE} + nvidia.com/mlnxnics: "1" + securityContext: + capabilities: + add: ["IPC_LOCK"] + volumeMounts: + - name: ssh-config + mountPath: /root/.ssh + - name: dshm + mountPath: /dev/shm + volumes: + - name: ssh-config + emptyDir: {} + - name: dshm + emptyDir: + medium: Memory + successPolicy: + operator: All + targetReplicatedJobs: + - launcher