NVIDIA · atif1996 · Aug 24, 2026
@@ -20,7 +20,7 @@ A machine-readable **CycloneDX 1.6 JSON** companion to this page is produced by
 ## Summary
 
 - Components: **43**
-- Unique images: **97**
+- Unique images: **101**
 - Distinct registries: **11**
 
 Registries: `602401143452.dkr.ecr.us-west-2.amazonaws.com`, `cr.agentgateway.dev`, `docker.io`, `gcr.io`, `ghcr.io`, `gke.gcr.io`, `nvcr.io`, `public.ecr.aws`, `quay.io`, `registry.k8s.io`, `us-docker.pkg.dev`
@@ -55,7 +55,7 @@ _Rendering fidelity:_ `catalog-parity: charts are rendered with the shared recip
 | kueue | helm | kueue | 0.18.2 | 1 |
 | mariadb-operator | helm | mariadb-operator | 26.6.0 | 1 |
 | mariadb-operator-crds | helm | mariadb-operator-crds | 26.6.0 | 0 |
-| network-operator | helm | nvidia/network-operator | 26.4.1 | 5 |
+| network-operator | helm | nvidia/network-operator | 26.4.1 | 9 |
 | network-operator-ocp | manifest | — | — | 0 |
 | network-operator-ocp-olm | manifest | — | — | 0 |
 | nfd | helm | node-feature-discovery | 0.19.0 | 1 |
@@ -233,6 +233,10 @@ _No images extracted._
 ### network-operator
 
 - `docker.io/library/busybox:1.38.0@sha256:dc2d74b28e4cf8984fa52af1f39bc7c3d9c73760b41a74d629f5d11b1ab28616`
+- `ghcr.io/k8snetworkplumbingwg/multus-cni:v4.2.1`
+- `ghcr.io/k8snetworkplumbingwg/plugins:v1.6.2-update.1`
+- `ghcr.io/k8snetworkplumbingwg/sriov-network-device-plugin:v3.9.0`
+- `ghcr.io/mellanox/nvidia-k8s-ipam:v0.2.0`
 - `nvcr.io/nvidia/cloud-native/network-operator:v26.4.1`
 - `nvcr.io/nvidia/doca/doca_telemetry:1.22.5-doca3.1.0-host`
 - `nvcr.io/nvidia/mellanox/doca-driver:doca3.2.0-25.10-1.2.8.0-2`

@@ -17,7 +17,7 @@ gb200-eks-ubuntu-inference-dynamo: faed7a104d1bc1ee252d77b971df7b8afc71495eb2292
 gb200-eks-ubuntu-training-kubeflow: 5b393b53870521ec1ad82012080bdda259276b1fafafde18cc89f6d3422b612b
 gb200-eks-ubuntu-training-slurm: df81536ab91019c2cacb69ad747a3103a7829bb5b90c4f34395ec78e4c36b858
 gb200-oke-ubuntu-inference-dynamo: 89524fb57cca12f10bab5747bd7bf584c5a93e35fa29df6b2ff169303f345216
-gb200-oke-ubuntu-training-kubeflow: 606e95bf8e55120e2df56e7f6715e8f52d717198bb014dfccacd7a809a055dc1
+gb200-oke-ubuntu-training-kubeflow: 86c522cd4280fb25fef783d6a03c2ca05ce8ebcaf3472db4fdfd1ee950044340
 h100-aks-ubuntu-inference-dynamo: 966031acc3abedf7a0289530b9a37af29596d5896831030ff372f6bf1bdc4f6f
 h100-aks-ubuntu-training-kubeflow: e2ae0d0021d961ebef8e8d857aab32b9fe2015f5c42172b8ccfad1fee0fb9539
 h100-aks-ubuntu-training-slurm: 01507beb65f458fcb1bbf1eee688d92b199912fa7af108ea1e4d0727f6ce1a32
@@ -38,7 +38,7 @@ h200-eks-inference: e9c38a77ae6067ce8a8ffcffde56f87a0bfe3c311a73197d95766aa73c9e
 h200-eks-training: a7951808f09ed1aa30300b0f62954657df7063996fff4455d559d7e5b4f5ae0a
 l40s-any: 068ed1d8149883225e51e2554a2feb5632992aac90c07cc0dc458e7d7983d13f
 l40s-oke-inference: d396a6d8b01065a3f64b12ee4a2801b71d4a0031c6e5831000993da721342331
-l40s-oke-training: 2da8c3f72fe690d8ee6a3b01ae089b0627bb8d6505ec000333f7b35a5063f275
+l40s-oke-training: ee6caf6926b6ac1b1d83368d36e17bb4a8be7f1a3980cceaafe55979c8d6f6d6
 monitoring-hpa: 832b485a6dbd53b9cbb275305415ea77e9f2bef5ec5e70382330b9904571bc72
 ocp-inference-nim: bb00cdb191823b32da334bea70826c8a92c11b62d0e096918d43a8e1b043c361
 ocp-training: d7a213263630f2c25982d6f4a144df7d9d5784428d296ccda1b4dab5a42b98bb

@@ -34,22 +34,25 @@ func TestOKEPerformanceGoalsFollowTrainingInferencePattern(t *testing.T) {
 	}{
 		{
 			name:       "gb200-oke-training",
-			wantChecks: []string{"nccl-all-reduce-bw-nvls"},
+			wantChecks: []string{"nccl-all-reduce-bw-net", "nccl-all-reduce-bw-nvls"},
 			wantConstraints: map[string]string{
+				"nccl-all-reduce-bw-net":  ">= 40",
 				"nccl-all-reduce-bw-nvls": ">= 500",
 			},
 		},
 		{
 			name:       "gb200-oke-ubuntu-training",
-			wantChecks: []string{"nccl-all-reduce-bw-nvls"},
+			wantChecks: []string{"nccl-all-reduce-bw-net", "nccl-all-reduce-bw-nvls"},
 			wantConstraints: map[string]string{
+				"nccl-all-reduce-bw-net":  ">= 40",
 				"nccl-all-reduce-bw-nvls": ">= 500",
 			},
 		},
 		{
 			name:       "gb200-oke-ubuntu-training-kubeflow",
-			wantChecks: []string{"nccl-all-reduce-bw-nvls"},
+			wantChecks: []string{"nccl-all-reduce-bw-net", "nccl-all-reduce-bw-nvls"},
 			wantConstraints: map[string]string{
+				"nccl-all-reduce-bw-net":  ">= 40",
 				"nccl-all-reduce-bw-nvls": ">= 500",
 			},
 		},

@@ -17,7 +17,7 @@ gb200-eks-ubuntu-inference-dynamo: 9e6bf776ce05caaf8072cedeaa4cdb46110c91409b365
 gb200-eks-ubuntu-training-kubeflow: 8f680335b3480755ba9386922372422a74ed6b563eba868b952aed603056a6ed
 gb200-eks-ubuntu-training-slurm: 2f575bbbc126ae761d0333c133dae2dc068deba7289637b59d811c5ebeffcd45
 gb200-oke-ubuntu-inference-dynamo: abc880c4e0fcefcb5902ec34f94c5d3b421bc4eda9a5b33a4b5a29e3dc2e048d
-gb200-oke-ubuntu-training-kubeflow: 75ad2dd381a86a636525de7dadc61923e74ef90715dd417c191f1e8bf979b0cb
+gb200-oke-ubuntu-training-kubeflow: ed509ad8ccd4951fe0198b11033a8f312b41cd12fa1e312cfad42badcc50225f
 h100-aks-ubuntu-inference-dynamo: 623d87a7206ef064ec4bb088d082febeedbfbbfd3ba1e7a0edc30cd75e9f796d
 h100-aks-ubuntu-training-kubeflow: 4f557906f610b6c71eea7c48ab356a8b60b3e7da7a7a3537093f0633418c58ca
 h100-aks-ubuntu-training-slurm: 3918af534fd888957654dea0e89f08aea2fc31e57316773a37eff24b23c231cc
@@ -38,7 +38,7 @@ h200-eks-inference: cb67a2c82c7e5c4766ad74e2184d4714c226853d84272196369c007835d4
 h200-eks-training: f622509c221285e5fff866b6311b0f10c0be827d3de014b57b8e78e843b7dfca
 l40s-any: 594d6a6ad6b7a943e4c400fdd52f0f7fb4cddb59214812f23a3b158e49afc6a5
 l40s-oke-inference: 17053c54993f726338082dc541ae34a3308893450e582e2b479d79337cc4aa9f
-l40s-oke-training: 8db430710ae40b810c5ed356f8acc038af855616a0cd2402bfce2e0fdc07ea77
+l40s-oke-training: c2ad43da9182132a5a449d9ebae91f587b019f39a5aaf6296f79b26c3c759bcf
 monitoring-hpa: f281c5b4c34b0aa0a5505baf24378fdad7afb06269e939e9004793c240035cf3
 ocp-inference-nim: f57147ace807d49644443fdbed18bd6e6c20fa028f2144eb2d79989eb4271d5b
 ocp-training: d48c15a49e3c8b4f59aff6936c054bd714f54812141be7679961b7b669e3351d

@@ -0,0 +1,48 @@
+# NicClusterPolicy for GB200 OKE (OCI) — rdmaSharedDevicePlugin over InfiniBand.
+#
+# Mirrors the AOR OCI GB200 config validated on gb200-ew. No ofedDriver (host
+# MOFED), no SR-IOV: the NVL72 east-west fabric is IB on the rdma0-3 netdevs
+# (oci_hpc.rdma_device_names_mode=2 kernel cmdline names them deterministically).
+#
+# The IB devices are advertised as nvidia.com/mlnxnics — the same resource
+# name the L40S SR-IOV path uses, so workloads request RDMA uniformly
+# across OKE fabrics.
+apiVersion: mellanox.com/v1alpha1
+kind: NicClusterPolicy
+metadata:
+  name: nic-cluster-policy
+  annotations:
+    helm.sh/hook: post-install,post-upgrade
+    helm.sh/hook-weight: "5"
+    helm.sh/hook-delete-policy: before-hook-creation
+  labels:
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+spec:
+  rdmaSharedDevicePlugin:
+    image: k8s-rdma-shared-dev-plugin
+    repository: nvcr.io/nvidia/mellanox
+    version: network-operator-v26.4.1
+    config: |
+      {
+        "configList": [
+          {
+            "resourcePrefix": "nvidia.com",
+            "resourceName": "mlnxnics",
+            "rdmaHcaMax": 63,
+            "selectors": {
+              "linkTypes": ["infiniband"],
+              "ifNames": ["rdma0", "rdma1", "rdma2", "rdma3"]
+            }
+          }
+        ]
+      }
+  deploymentTolerations:
+    - key: CriticalAddonsOnly
+      operator: Exists
+  tolerations:
+    # RDMA DaemonSets must land on tainted GPU nodes.
+    - key: nvidia.com/gpu
+      operator: Exists
+    - key: CriticalAddonsOnly
+      operator: Exists
@@ -0,0 +1,73 @@
+# NicClusterPolicy for L40S OKE (OCI) SR-IOV RoCE.
+#
+# The network-operator Helm chart installs the operator + CRD but does not template
+# a NicClusterPolicy CR (values-oke-l40s.yaml sets deployCR: false). This manifest
+# creates it so the operator reconciles the RoCE fabric stack. Hand-rendered from
+# AOR's network-operator/nicclusterpolicy.yaml.tmpl (provider: oci branch, with
+# network.type == roce → nvIpam + secondaryNetwork included).
+#
+# OCI specifics (vs Forge IB): NO ofedDriver — OCI nodes carry host MOFED, consumed
+# by the GPU Operator driver via driver.rdma.useHostMofed (l40s-oke-ubuntu leaf). One
+# sriovDevicePlugin resource, nvidia.com/mlnxnics, selecting the OCI ConnectX VF
+# device IDs (101a = ConnectX-5 Ex VF, 101e = mlx5Gen VF). RoCE also needs nv-ipam
+# (VF IP allocation) + secondaryNetwork/multus (attach the VF into workload pods).
+# vendor 15b3 = Mellanox.
+apiVersion: mellanox.com/v1alpha1
+kind: NicClusterPolicy
+metadata:
+  name: nic-cluster-policy
+  annotations:
+    helm.sh/hook: post-install,post-upgrade
+    helm.sh/hook-weight: "5"
+    helm.sh/hook-delete-policy: before-hook-creation
+  labels:
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+spec:
+  # RoCE: allocate IPs for the RDMA VFs and wire them into pods via multus.
+  nvIpam:
+    image: nvidia-k8s-ipam
+    repository: ghcr.io/mellanox
+    version: v0.2.0
+    enableWebhook: false
+    containerResources:
+      - name: nv-ipam-node
+        requests:
+          cpu: 500m
+          memory: 1Gi
+        limits:
+          cpu: "1"
+          memory: 2Gi
+  secondaryNetwork:
+    cniPlugins:
+      image: plugins
+      repository: ghcr.io/k8snetworkplumbingwg
+      version: v1.6.2-update.1
+    multus:
+      image: multus-cni
+      repository: ghcr.io/k8snetworkplumbingwg
+      version: v4.2.1
+  sriovDevicePlugin:
+    image: sriov-network-device-plugin
+    repository: ghcr.io/k8snetworkplumbingwg
+    version: v3.9.0
+    config: |
+      {
+        "resourceList": [
+          {
+            "resourcePrefix": "nvidia.com",
+            "resourceName": "mlnxnics",
+            "selectors": {"isRdma":true,"vendors":["15b3"],"devices":["101a","101e"]}
+          }
+        ]
+      }
+  # Operator DaemonSet placement: system/monitoring nodes only (matches AOR).
+  deploymentTolerations:
+    - key: CriticalAddonsOnly
+      operator: Exists
+  tolerations:
+    # RDMA DaemonSets must land on tainted GPU nodes.
+    - key: nvidia.com/gpu
+      operator: Exists
+    - key: CriticalAddonsOnly
+      operator: Exists
@@ -0,0 +1,25 @@
+# network-operator Helm values for GB200 OKE (OCI) InfiniBand.
+#
+# OCI GB200 NVL72 model (vs L40S RoCE / Forge IB): NO ofedDriver — nodes carry host
+# MOFED — and no SR-IOV/nv-ipam/multus either. East-west is InfiniBand (rdma0-3),
+# served by rdmaSharedDevicePlugin from the post-install NicClusterPolicy manifest,
+# NOT the chart. deployCR off so that manifest CR is authoritative.
+# nfd.enabled: false — GPU Operator's NFD is used; no second NFD.
+deployCR: false
+nvIpam:
+  enabled: false
+secondaryNetwork:
+  deploy: false
+nfd:
+  enabled: false
+operator:
+  resources:
+    limits:
+      cpu: "1"
+      memory: 2Gi
+    requests:
+      cpu: 500m
+      memory: 2Gi
+  # Operator placement comes from the bundler's system-node scheduling
+  # injection (registry nodeScheduling: operator.nodeSelector /
+  # operator.tolerations) — no hardcoded affinity here.
@@ -0,0 +1,34 @@
+# network-operator Helm values for L40S OKE (OCI) SR-IOV RoCE.
+# Hand-rendered from AOR's network-operator/values.yaml.tmpl (provider: oci) +
+# nicclusterpolicy.yaml.tmpl (oci branch, network.type == roce).
+#
+# OCI model (vs Forge IB / Mistral DOCA): NO ofedDriver — OCI bare-metal nodes carry
+# host MOFED, so the GPU Operator uses it via driver.rdma.useHostMofed (set on the
+# l40s-oke-ubuntu leaf). network-operator's job here is the SR-IOV VF device plugin
+# (advertises nvidia.com/mlnxnics RDMA VFs) plus nv-ipam + secondaryNetwork (multus)
+# for RoCE — all supplied by the post-install NicClusterPolicy manifest, NOT the chart.
+#
+# deployCR/nvIpam/secondaryNetwork: AICR's wrapper defaults are on (deployCR: true,
+# nvIpam.enabled: true, secondaryNetwork.deploy: true) — they template the wrapper's
+# own NicClusterPolicy. Turn deployCR off so our manifest CR is authoritative (it is
+# the only place the OCI VF selectors 101a/101e can be expressed); the operator
+# reconciles nv-ipam + secondaryNetwork + sriovDevicePlugin from that CR regardless.
+# nfd.enabled: false — GPU Operator's NFD is used; no second NFD.
+deployCR: false
+nvIpam:
+  enabled: false
+secondaryNetwork:
+  deploy: false
+nfd:
+  enabled: false
+operator:
+  resources:
+    limits:
+      cpu: "1"
+      memory: 2Gi
+    requests:
+      cpu: 500m
+      memory: 2Gi
+  # Operator placement comes from the bundler's system-node scheduling
+  # injection (registry nodeScheduling: operator.nodeSelector /
+  # operator.tolerations) — no hardcoded affinity here.
@@ -100,6 +100,13 @@ var imageDigestExemptions = map[string]string{
 	// Skyhook Package `containerSHA` field (issue #1031), folded into the
 	// extracted image ref as `@sha256:...` by pkg/bom.ExtractImagesFromYAML.
 	"ghcr.io/nvidia/skyhook-packages/shellscript:1.1.1": "Skyhook Package CRD does not accept image digests; tracked via #745 and NVIDIA/nodewright#224",
+
+	// NicClusterPolicy (network-operator OKE): same repository/image/version
+	// triplet schema as the AKS entries above — no digest field in the CRD.
+	"ghcr.io/mellanox/nvidia-k8s-ipam:v0.2.0":                         "NicClusterPolicy CRD does not accept image digests; tracked via #745 and Mellanox/network-operator#2555",
+	"ghcr.io/k8snetworkplumbingwg/multus-cni:v4.2.1":                  "NicClusterPolicy CRD does not accept image digests; tracked via #745 and Mellanox/network-operator#2555",
+	"ghcr.io/k8snetworkplumbingwg/plugins:v1.6.2-update.1":            "NicClusterPolicy CRD does not accept image digests; tracked via #745 and Mellanox/network-operator#2555",
+	"ghcr.io/k8snetworkplumbingwg/sriov-network-device-plugin:v3.9.0": "NicClusterPolicy CRD does not accept image digests; tracked via #745 and Mellanox/network-operator#2555",
 }
 
 // TestComponentManifestImagesAreDigestPinned asserts that every image

@@ -39,30 +39,59 @@ spec:
       value: ">= 1.34"
 
   componentRefs:
-    # GB200-specific GPU Operator overrides (inherits valuesFile from oke-training)
+    # GB200-specific GPU Operator overrides (inherits valuesFile from oke-training).
+    # kernel-module-params sets NVreg_GrdmaPciTopoCheckOverride=1, required
+    # for dma-buf attach over the IB fabric (GPUDirect RDMA without
+    # nvidia-peermem, whose chroot modprobe fails to build against the -64k
+    # Grace kernel).
     - name: gpu-operator
       type: Helm
+      preManifestFiles:
+        - components/gpu-operator/manifests/kernel-module-params.yaml
       dependencyRefs:
         - nfd
         - cert-manager
         - kube-prometheus-stack
       overrides:
         gdrcopy:
           enabled: true
+        driver:
+          kernelModuleConfig:
+            name: nvidia-kernel-module-params
 
     - name: nfd
       type: Helm
       overrides:
         topologyUpdater:
           enable: true
 
+    # InfiniBand east-west fabric (NVL72 rdma0-3). rdmaSharedDevicePlugin
+    # advertises the shared HCAs as nvidia.com/mlnxnics; no SR-IOV/nv-ipam
+    # (that is the L40S RoCE path) and no ofedDriver (OCI nodes carry host
+    # MOFED). NicClusterPolicy is manifest-supplied (chart deployCR off).
+    # Present in every gpuStack value; incompatible with Oracle's opt-in
+    # NvidiaNetworkOperator add-on.
+    - name: network-operator
+      type: Helm
+      valuesFile: components/network-operator/values-oke-gb200.yaml
+      manifestFiles:
+        - components/network-operator/manifests/nic-cluster-policy-oke-gb200.yaml
+      dependencyRefs:
+        - nfd
+        - cert-manager
+
   validation:
     performance:
-      # NVLS runtime support is OKE-specific. NET/RDMA is intentionally left
-      # out until OCI-specific pod RDMA exposure is verified on the testbed.
+      # Both transport variants: NVLS (MNNVL across the NVL72 IMEX domain)
+      # and NET (the IB east-west fabric this leaf's NicClusterPolicy
+      # exposes — validated on a BM.GPU.GB200.4 NVL72 rack). Constraints
+      # match gb200-eks-training.
       checks:
+        - nccl-all-reduce-bw-net
         - nccl-all-reduce-bw-nvls
       constraints:
+        - name: nccl-all-reduce-bw-net
+          value: ">= 40"
         - name: nccl-all-reduce-bw-nvls
           value: ">= 500"
     conformance:

@@ -54,6 +54,26 @@ spec:
         topologyUpdater:
           enable: true
 
+    # RDMA fabric (RoCE over SR-IOV VFs). Every L40S OCI cluster runs RoCE;
+    # the SR-IOV VF device plugin advertises nvidia.com/mlnxnics RDMA VFs,
+    # with nv-ipam + multus attaching the VFs into workload pods. The
+    # NicClusterPolicy is supplied by the manifest (the chart's deployCR is
+    # off — the manifest is the only place the OCI VF selectors 101a/101e
+    # can be expressed). OCI nodes carry host MOFED, so there is no
+    # ofedDriver in any configuration. GPUDirect RDMA works via DMA-BUF;
+    # nvidia-peermem stays off (base default) — inert on this topology.
+    # Present in every gpuStack value: the fabric is orthogonal to GPU
+    # driver/plugin ownership. Incompatible with Oracle's opt-in
+    # NvidiaNetworkOperator add-on (two lifecycle managers, one release).
+    - name: network-operator
+      type: Helm
+      valuesFile: components/network-operator/values-oke-l40s.yaml
+      manifestFiles:
+        - components/network-operator/manifests/nic-cluster-policy-oke-l40s.yaml
+      dependencyRefs:
+        - nfd
+        - cert-manager
+
   # Validation checks for L40S on OKE training workloads.
   # Defined at the intent layer (not OS-specific) so all OS variants inherit them.
   #

@@ -247,6 +247,10 @@ var supportedNCCLCombinations = map[ncclVariant]map[recipe.CriteriaServiceType][
 	},
 	variantNET: {
 		recipe.CriteriaServiceEKS: {recipe.CriteriaAcceleratorGB200},
+		// OKE GB200 NVL72: IB east-west (rdma0-3) via the
+		// rdmaSharedDevicePlugin's nvidia.com/mlnxnics shared HCAs —
+		// see testdata/gb200/oke/runtime-net.yaml.
+		recipe.CriteriaServiceOKE: {recipe.CriteriaAcceleratorGB200},
 	},
 	variantNVLS: {
 		recipe.CriteriaServiceEKS: {recipe.CriteriaAcceleratorGB200},