From 6c9a276528dc75ba4c591b1708d665e27024527e Mon Sep 17 00:00:00 2001 From: Abigail McCarthy <20771501+a-mccarthy@users.noreply.github.com> Date: Tue, 2 Jun 2026 14:52:16 -0400 Subject: [PATCH 1/9] Add docs for 26.7.0 Signed-off-by: Abigail McCarthy <20771501+a-mccarthy@users.noreply.github.com> --- gpu-operator/release-notes.rst | 18 ------------------ gpu-operator/versions1.json | 4 ++++ repo.toml | 2 +- 3 files changed, 5 insertions(+), 19 deletions(-) diff --git a/gpu-operator/release-notes.rst b/gpu-operator/release-notes.rst index 394f7e2f9..5c2dec9b1 100644 --- a/gpu-operator/release-notes.rst +++ b/gpu-operator/release-notes.rst @@ -346,24 +346,6 @@ Known Issues Refer to the MIG Controller issue `#329 `_ for more information. -* After you delete the default NVIDIADriver custom resource, any custom NVIDIADriver - custom resources that you created might not become active automatically. - The custom resources remain in a pending state because the Operator controller - does not re-evaluate them after the conflicting default custom resource is removed. - - To work around this issue, restart the GPU Operator controller by deleting - the controller pod: - - .. code-block:: console - - $ kubectl delete pod -n gpu-operator -l app=gpu-operator - - Restarting the controller pod does not disrupt running GPU workloads or - driver pods on nodes. - - Refer to issue `#2259 `_ - for more information. - Removals and Deprecations ------------------------- diff --git a/gpu-operator/versions1.json b/gpu-operator/versions1.json index 27150f84d..508fbb389 100644 --- a/gpu-operator/versions1.json +++ b/gpu-operator/versions1.json @@ -1,6 +1,10 @@ [ { "preferred": "true", + "url": "../26.7", + "version": "26.7" + }, + { "url": "../26.3/", "version": "26.3" }, diff --git a/repo.toml b/repo.toml index 3420e5ed8..1ec47ba03 100644 --- a/repo.toml +++ b/repo.toml @@ -221,7 +221,7 @@ output_format = "linkcheck" docs_root = "${root}/openshift" project = "gpu-operator-openshift" name = "NVIDIA GPU Operator on Red Hat OpenShift Container Platform" -version = "26.3" # Check that this value matches maj.min version of repo_docs.projects.gpu-operator.version +version = "26.7" # Check that this value matches maj.min version of repo_docs.projects.gpu-operator.version copyright_start = 2020 sphinx_exclude_patterns = [ "get-entitlement.rst", From 503e1d9da20c2b79b6beff5d2f29dbe339119ab7 Mon Sep 17 00:00:00 2001 From: Abigail McCarthy <20771501+a-mccarthy@users.noreply.github.com> Date: Fri, 10 Jul 2026 12:21:57 -0400 Subject: [PATCH 2/9] add docs for 26.7.0 Signed-off-by: Abigail McCarthy <20771501+a-mccarthy@users.noreply.github.com> --- gpu-operator/release-notes.rst | 35 ++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/gpu-operator/release-notes.rst b/gpu-operator/release-notes.rst index 5c2dec9b1..aa1569fa0 100644 --- a/gpu-operator/release-notes.rst +++ b/gpu-operator/release-notes.rst @@ -33,6 +33,41 @@ Refer to the :ref:`GPU Operator Component Matrix` for a list of software compone ---- +.. _v26.7.0: + +26.7.0 +======= + +.. This 26.7.0 entry is in progress. Only changes confirmed against merged code are listed. + Pending items (NVIDIADriver CR migration, CDI/NRI default change, image hardening, + Ubuntu 26.04, new hardware, component versions) are added as they are verified on the + release branch. + +New Features +------------ + +* Added an ``upgradePolicy`` field to the NVIDIA driver custom resource definition (CRD). + You can now define a driver upgrade policy per NVIDIADriver custom resource. + When the field is unset, the driver-upgrade controller falls back to the default upgrade policy that is defined in the Helm chart values. + (`PR #2582 `__) + +* Added the ``hostPaths.kubeletRootDir`` Helm value to configure a custom kubelet root directory. + When left empty, the GPU Operator uses ``/var/lib/kubelet`` as the default path. + (`PR #1384 `__) + +* Added the ``dcgmExporter.serviceMonitor.scrapeTimeout`` Helm value to configure the scrape timeout for the DCGM Exporter ``ServiceMonitor``. + (`PR #2404 `__) + +Fixed Issues +------------ + +* Fixed an issue where custom NVIDIADriver custom resources could remain stuck and fail to reconcile after a conflicting default NVIDIADriver custom resource was deleted. + The GPU Operator now re-evaluates all NVIDIADriver custom resources whenever any NVIDIADriver custom resource changes. + (`PR #2258 `__) + + +---- + .. _v26.3.3: 26.3.3 From 74f3a12c1a0e55cd457a8df3bb5b3efb6e65bada Mon Sep 17 00:00:00 2001 From: Abigail McCarthy <20771501+a-mccarthy@users.noreply.github.com> Date: Fri, 10 Jul 2026 15:05:41 -0400 Subject: [PATCH 3/9] update versions Signed-off-by: Abigail McCarthy <20771501+a-mccarthy@users.noreply.github.com> --- openshift/versions1.json | 4 ++++ repo.toml | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/openshift/versions1.json b/openshift/versions1.json index a6293db72..cde5ec46a 100644 --- a/openshift/versions1.json +++ b/openshift/versions1.json @@ -1,6 +1,10 @@ [ { "preferred": "true", + "url": "../26.7/", + "version": "26.7" + }, + { "url": "../26.3/", "version": "26.3" }, diff --git a/repo.toml b/repo.toml index 1ec47ba03..619a661cd 100644 --- a/repo.toml +++ b/repo.toml @@ -171,8 +171,8 @@ output_format = "linkcheck" docs_root = "${root}/gpu-operator" project = "gpu-operator" name = "NVIDIA GPU Operator" -version = "26.3" # Update repo_docs.projects.openshift.version to match latest patch version maj.min.patch -source_substitutions = { minor_version = "26.3", version = "v26.3.3", recommended = "580.167.08", dra_version = "0.4.1" } +version = "26.7" # Update repo_docs.projects.openshift.version to match latest patch version maj.min.patch +source_substitutions = { minor_version = "26.7", version = "v26.7.0", recommended = "580.167.08", dra_version = "0.4.1" } copyright_start = 2020 sphinx_exclude_patterns = [ "life-cycle-policy.rst", From a4dd5291f970e95ae70dca83feb9ede4f703931e Mon Sep 17 00:00:00 2001 From: Abigail McCarthy <20771501+a-mccarthy@users.noreply.github.com> Date: Tue, 14 Jul 2026 12:08:42 -0400 Subject: [PATCH 4/9] Update release notes for codedrop 2 Signed-off-by: Abigail McCarthy <20771501+a-mccarthy@users.noreply.github.com> --- gpu-operator/cdi.rst | 92 ++++++++++++++++++++++++++++++++++ gpu-operator/release-notes.rst | 45 +++++++++++++++-- repo.toml | 2 +- 3 files changed, 133 insertions(+), 6 deletions(-) diff --git a/gpu-operator/cdi.rst b/gpu-operator/cdi.rst index acd15b8dc..b1bd35933 100644 --- a/gpu-operator/cdi.rst +++ b/gpu-operator/cdi.rst @@ -244,6 +244,98 @@ Disable the NRI Plugin by modifying the cluster policy: After disabling the NRI Plugin, the ``nvidia`` runtime class will be created. +.. _nri-cdi-devices: + +****************************************** +Requesting CDI Devices with the NRI Plugin +****************************************** + +When the NRI Plugin is enabled, a pod can request that the NRI Plugin inject specific CDI devices into one of its containers. +Add an annotation of the following form to the pod specification: + +.. code-block:: yaml + + metadata: + annotations: + nvidia.cdi.k8s.io/container.: "" + +Replace ```` with the name of the container in the pod that the device is injected into, and replace ```` with the CDI device to request. +The NRI Plugin supports two kinds of CDI devices: non-management devices and management devices. + +Requesting a Non-Management CDI Device +************************************** + +A non-management CDI device injects a single GPU, identified by its GPU UUID, into a container. +Pods can request a non-management CDI device from **any** namespace. + +Retrieve the GPU UUID from the ``/var/run/cdi/k8s.device-plugin.nvidia.com-gpu.json`` file on the GPU node. + +Request the device with an annotation of the form ``k8s.device-plugin.nvidia.com/gpu=``. +The following pod specification requests a specific GPU for the ``ubuntu`` container: + +.. code-block:: yaml + + apiVersion: v1 + kind: Pod + metadata: + name: ubuntu + labels: + app: ubuntu + annotations: + nvidia.cdi.k8s.io/container.ubuntu: "k8s.device-plugin.nvidia.com/gpu=GPU-1d7f7c5b-b2f6-148d-ff9a-d8bc4941885c" + spec: + containers: + - image: ubuntu + name: ubuntu + command: + - "sleep" + - "604800" + imagePullPolicy: IfNotPresent + restartPolicy: Always + +Requesting a Management CDI Device +********************************** + +A management CDI device provides management-level access to all GPUs on the node, bypassing GPU allocation through the Device Plugin or DRA Driver for GPUs. +This is the access that a GPU Management Container requires. +Request a management device with the ``management.nvidia.com/gpu=all`` annotation value. + +By default, only pods in the namespace where the GPU Operator is installed, which is the namespace where the NVIDIA Container Toolkit runs, are permitted to request management CDI devices. +To permit pods in other namespaces to request management CDI devices, add those namespaces to the ``NRI_MANAGEMENT_CDI_DEVICE_NAMESPACES`` environment variable on the NVIDIA Container Toolkit. +The value is a comma-separated list of namespaces. + +Set the environment variable through the ``toolkit.env`` Helm value at installation time, or by modifying the cluster policy on an existing installation: + +.. code-block:: console + + $ kubectl patch clusterpolicies.nvidia.com/cluster-policy --type='json' \ + -p='[{"op": "replace", "path": "/spec/toolkit/env", "value": [{"name": "NRI_MANAGEMENT_CDI_DEVICE_NAMESPACES", "value": "test-namespace"}]}]' + +The following pod specification, in ``test-namespace``, requests a management CDI device for the ``ctr`` container. +For this pod to start, ``test-namespace`` must be included in the ``NRI_MANAGEMENT_CDI_DEVICE_NAMESPACES`` environment variable. + +.. code-block:: yaml + + apiVersion: v1 + kind: Pod + metadata: + name: cdi-nri-test + namespace: test-namespace + labels: + app: cdi-nri-test + annotations: + nvidia.cdi.k8s.io/container.ctr: "management.nvidia.com/gpu=all" + spec: + containers: + - name: ctr + image: ubuntu + imagePullPolicy: IfNotPresent + command: + - bash + - -c + - sleep infinity + + ************ Known Issues ************ diff --git a/gpu-operator/release-notes.rst b/gpu-operator/release-notes.rst index aa1569fa0..e6659a23d 100644 --- a/gpu-operator/release-notes.rst +++ b/gpu-operator/release-notes.rst @@ -38,14 +38,28 @@ Refer to the :ref:`GPU Operator Component Matrix` for a list of software compone 26.7.0 ======= -.. This 26.7.0 entry is in progress. Only changes confirmed against merged code are listed. - Pending items (NVIDIADriver CR migration, CDI/NRI default change, image hardening, - Ubuntu 26.04, new hardware, component versions) are added as they are verified on the - release branch. - New Features ------------ +* Updated software component versions: + + - NVIDIA GPU Driver 595.71.05 + - NVIDIA Container Toolkit v1.20.0 + - NVIDIA DCGM 4.6.0-1 + - NVIDIA MIG Manager for Kubernetes v0.14.3 + - NVIDIA GDRCopy Driver v2.6 + - Node Feature Discovery v0.19.0 + +* Added a ``nvidia.com/gpu.deploy.client`` node label that lets the GPU Operator manage third-party GPU client pods during driver upgrades and MIG configuration changes. + Advanced users who run their own GPU client workloads that hold GPU device handles (for example, a standalone NVIDIA DRA driver) can add ``nvidia.com/gpu.deploy.client=true`` to the ``nodeSelector`` of the workload's DaemonSet, Deployment, StatefulSet, or Job. + The GPU Operator then automatically restarts these pods during a driver upgrade or a MIG configuration change, so the operation can proceed without manual pod eviction. + (`PR #2607 `__) + +* Added support for restarting driver pods in place during patch upgrades when the driver configuration is unchanged. + Previously, a chart upgrade that changed only cosmetic pod-template metadata, such as the ``helm.sh/chart`` label, evicted running GPU workloads and drained the node. + The driver-upgrade controller now compares the driver configuration digest between the running pod and the desired DaemonSet, and when they match, it cordons the node and restarts the driver pod in place without evicting workloads or draining the node. + (`PR #2527 `__) + * Added an ``upgradePolicy`` field to the NVIDIA driver custom resource definition (CRD). You can now define a driver upgrade policy per NVIDIADriver custom resource. When the field is unset, the driver-upgrade controller falls back to the default upgrade policy that is defined in the Helm chart values. @@ -58,6 +72,19 @@ New Features * Added the ``dcgmExporter.serviceMonitor.scrapeTimeout`` Helm value to configure the scrape timeout for the DCGM Exporter ``ServiceMonitor``. (`PR #2404 `__) +* Added support for configuring the namespace that the GPU Operator uses for its leader election ``Lease`` object. + When unset, the operator continues to use the namespace that it runs in. + (`PR #1333 `__) + +* Changed the RHEL-specific driver container image tags to use only the major RHEL version instead of the full ``major.minor`` version. + (`PR #2497 `__) + +* Added the ``NRI_MANAGEMENT_CDI_DEVICE_NAMESPACES`` environment variable for the NVIDIA Container Toolkit. + When the NRI Plugin is enabled, this variable holds the list of namespaces whose pods are permitted to request management CDI devices. + By default, only pods in the namespace where the GPU Operator is installed can request management CDI devices. + Set the variable through the ``toolkit.env`` Helm value or the ClusterPolicy custom resource. + Refer to :ref:`Requesting CDI Devices with the NRI Plugin ` for more information. + Fixed Issues ------------ @@ -65,6 +92,14 @@ Fixed Issues The GPU Operator now re-evaluates all NVIDIADriver custom resources whenever any NVIDIADriver custom resource changes. (`PR #2258 `__) +* Fixed an issue where the ``ClusterPolicy`` status could briefly report ``Ready`` during a driver upgrade before all driver pods were upgraded. + (`PR #2510 `__, `Issue #1567 `__) + +* Fixed an issue where the driver container's containerd configuration mounts were set up even when the NRI plugin was enabled. + On immutable hosts with read-only containerd paths, this prevented the NRI plugin pod from starting. + The GPU Operator no longer configures these mounts when NRI is enabled. + (`PR #2514 `__) + ---- diff --git a/repo.toml b/repo.toml index 619a661cd..69017b4dc 100644 --- a/repo.toml +++ b/repo.toml @@ -172,7 +172,7 @@ docs_root = "${root}/gpu-operator" project = "gpu-operator" name = "NVIDIA GPU Operator" version = "26.7" # Update repo_docs.projects.openshift.version to match latest patch version maj.min.patch -source_substitutions = { minor_version = "26.7", version = "v26.7.0", recommended = "580.167.08", dra_version = "0.4.1" } +source_substitutions = { minor_version = "26.7", version = "v26.7.0", recommended = "595.71.05", dra_version = "0.4.1" } copyright_start = 2020 sphinx_exclude_patterns = [ "life-cycle-policy.rst", From 8193e866d66347768a4355ff738ff1e9ee887999 Mon Sep 17 00:00:00 2001 From: Abigail McCarthy <20771501+a-mccarthy@users.noreply.github.com> Date: Fri, 24 Jul 2026 13:26:45 -0400 Subject: [PATCH 5/9] ideas for release Signed-off-by: Abigail McCarthy <20771501+a-mccarthy@users.noreply.github.com> --- gpu-operator/architecture.rst | 387 ++++ gpu-operator/clusterpolicy-reference.rst | 1903 +++++++++++++++++++ gpu-operator/getting-started.rst | 326 +--- gpu-operator/gpu-operator-dra.rst | 471 +++++ gpu-operator/gpucluster-reference.rst | 321 ++++ gpu-operator/helm-values-reference.rst | 398 ++++ gpu-operator/index.rst | 49 +- gpu-operator/install-gpu-operator-nvaie.rst | 4 - gpu-operator/install-paths.rst | 296 +++ gpu-operator/life-cycle-policy.rst | 69 +- gpu-operator/overview.rst | 9 +- gpu-operator/platform-support.rst | 1 + gpu-operator/prerequisites.rst | 114 ++ 13 files changed, 3979 insertions(+), 369 deletions(-) create mode 100644 gpu-operator/architecture.rst create mode 100644 gpu-operator/clusterpolicy-reference.rst create mode 100644 gpu-operator/gpu-operator-dra.rst create mode 100644 gpu-operator/gpucluster-reference.rst create mode 100644 gpu-operator/helm-values-reference.rst create mode 100644 gpu-operator/install-paths.rst create mode 100644 gpu-operator/prerequisites.rst diff --git a/gpu-operator/architecture.rst b/gpu-operator/architecture.rst new file mode 100644 index 000000000..987c9d72f --- /dev/null +++ b/gpu-operator/architecture.rst @@ -0,0 +1,387 @@ +.. license-header + SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +.. headings (h1/h2/h3/h4/h5) are # * = - + +.. _gpu-operator-control-plane-architecture: + +####################################### +GPU Operator Control Plane Architecture +####################################### + +The NVIDIA GPU Operator is a Kubernetes controller that translates a cluster-level GPU policy into +the node-level software stack required to run GPU workloads. +This page is for platform engineers who need to understand the control plane, operand dependencies, +and the path from GPU discovery to workload allocation. + +.. note:: + + This page describes the device-plugin-based control path implemented on the GPU Operator + ``main`` branch at the source baseline recorded at the end of this file. + DRA-based resource allocation uses a different architecture and is outside this page's scope. + +.. image:: graphics/gpu-operator-architecture.svg + :alt: GPU Operator control plane and operand architecture + +The GPU Operator manager watches custom resources and Nodes through the Kubernetes API. +It labels eligible nodes, creates or updates operand resources in a fixed order, and reports +reconciliation status. +Most operands run as DaemonSets on GPU nodes and are selected by labels that the Operator manages. + +******************* +Operator Components +******************* + +The GPU Operator manages a set of components that together provide the GPU software stack. +The following table lists the components, the category each belongs to, whether it is deployed by +default with the container workload configuration, and what it does. + +.. list-table:: GPU Operator components + :header-rows: 1 + :widths: 24 46 + + * - Component + - Default + - Description + * - GPU Operator controller + - Runs the reconcilers that watch the custom resources and Nodes and manage every operand. + * - Node Feature Discovery (NFD) + - Detects NVIDIA PCI devices and platform features and labels Nodes. + Installed as a subchart unless NFD is already present in the cluster. + * - NVIDIA driver + - Installs the kernel modules and user-space driver libraries on GPU nodes. + Can be disabled when the driver is preinstalled on the host. + * - NVIDIA Container Toolkit + - Configures the container runtime and Container Device Interface (CDI) so containers can + access GPUs. + * - NVIDIA Kubernetes Device Plugin + - Registers ``nvidia.com/gpu`` and MIG extended resources with kubelet. + * - MPS control daemon + - Supports Multi-Process Service (MPS) GPU sharing; deployed with the device plugin. + * - GPU Feature Discovery (GFD) + - Publishes GPU model, memory, and MIG capabilities as Node labels. + * - MIG Manager + - Applies the requested Multi-Instance GPU (MIG) geometry on MIG-capable nodes. + * - Operator Validator + - Runs driver, toolkit, CUDA, and device-plugin checks and gates dependent operands through + readiness files. + * - DCGM + - Standalone NVIDIA DCGM hostengine. + When disabled, DCGM Exporter uses its embedded engine. + * - DCGM Exporter + - Exports GPU health and utilization metrics for Prometheus. + * - Node Status Exporter + - Reports node-level GPU Operator state. + * - Sandbox Device Plugin + - Advertises passthrough GPUs to kubelet on KubeVirt or Kata nodes. + * - VFIO Manager + - Binds GPUs to the ``vfio-pci`` driver for VFIO passthrough. + * - vGPU Manager + - Installs the NVIDIA vGPU host driver on vGPU nodes. + * - vGPU Device Manager + - Creates and manages vGPU devices on the node. + * - Sandbox Validator + - Validates the sandboxed and virtualized workload stack. + * - Confidential Computing Manager + - Sets confidential computing mode on GPUs for Kata Containers and Confidential Containers. + +.. [#mig] MIG Manager is deployed on MIG-capable nodes that use the container workload + configuration. + +The remaining sections describe how these components are organized into layers, the order in which +they are reconciled, and the topologies in which they are deployed. + +**************** +Component Layers +**************** + +API and desired-state layer +=========================== + +The API layer stores the configuration that the Operator reconciles: + +* ``ClusterPolicy`` is the primary, cluster-scoped API. + It configures the driver, container toolkit, device plugin, telemetry, validation, and optional + workload stacks. + Only one ``ClusterPolicy`` is active in a cluster. +* ``GPUCluster`` is a cluster-scoped API for configuring the DRA Driver for NVIDIA GPUs. + It configures the DRA Driver for NVIDIA GPUs, including the DRA Driver for NVIDIA GPUs DaemonSet. + Only one ``GPUCluster`` is active in a cluster. +* ``NVIDIADriver`` is an optional, cluster-scoped API for assigning different driver configurations + to different node pools. + When this mode is enabled, the ``NVIDIADriver`` controller manages driver DaemonSets instead of + the ``ClusterPolicy`` controller. +* Kubernetes ``Node`` objects carry discovery, placement, workload, and upgrade state as labels + and annotations. + +For the fields available in these APIs, refer to :ref:`clusterpolicy-reference`, +:ref:`gpucluster-reference`, and +:doc:`NVIDIA GPU Driver Custom Resource Definition `. + +Control plane +============= + +The Operator runs as a controller-runtime manager in the Operator namespace. +The manager starts four reconcilers: + +* **ClusterPolicy reconciler** — renders and reconciles the operand resources that implement the + active policy. +* **Node labeling reconciler** — converts hardware discovery and workload configuration into + ``nvidia.com/gpu.present`` and ``nvidia.com/gpu.deploy.*`` placement labels. +* **NVIDIADriver reconciler** — manages driver DaemonSets and node ownership when the NVIDIA driver + custom resource mode is enabled. +* **Upgrade reconciler** — coordinates driver rollout state and the handling of GPU workloads during + an automatic driver upgrade. + +The reconcilers use the Kubernetes API as the shared state store. +They watch custom resources, Nodes, and owned DaemonSets rather than communicating directly with +each other. + +Discovery and placement layer +============================= + +Node Feature Discovery (NFD) identifies NVIDIA PCI devices and publishes feature labels on Nodes. +The Node labeling reconciler uses those labels to set ``nvidia.com/gpu.present=true`` and the +per-operand deployment labels. + +This separation has two effects: + +* hardware discovery remains independent from operand lifecycle management; and +* each DaemonSet can use a narrow node selector such as + ``nvidia.com/gpu.deploy.device-plugin=true``. + +The user-controlled ``nvidia.com/gpu.workload.config`` label selects the effective workload stack +for a node. +The default stack supports container workloads. +Optional stacks support GPU passthrough, NVIDIA vGPU, and sandboxed workloads. + +Node operand layer +================== + +The default container workload path includes the following operands: + +* **NVIDIA driver** — installs or exposes the kernel modules and user-space driver libraries. + The driver can instead be preinstalled on the host. +* **NVIDIA Container Toolkit** — configures the container runtime and Container Device Interface + (CDI) support. +* **Operator Validator** — checks driver, toolkit, CUDA, and device-plugin readiness. + Other operands use its readiness files to avoid starting before their dependencies are usable. +* **NVIDIA Kubernetes Device Plugin** — registers GPU and MIG extended resources with kubelet. +* **GPU Feature Discovery (GFD)** — publishes GPU capabilities and topology as Node labels. +* **MIG Manager** — applies the requested Multi-Instance GPU configuration on supported nodes. +* **DCGM and DCGM Exporter** — provide GPU health data and Prometheus metrics. + +Optional operands add node-status reporting, vGPU management, VFIO management, device plugins for +sandboxed workloads, and confidential-computing support. + +**************************** +Desired-State Reconciliation +**************************** + +The ``ClusterPolicy`` reconciler processes operand states in a fixed sequence. +The important dependency order is: + +#. prerequisites and Operator metrics; +#. driver; +#. container toolkit; +#. validation; +#. device plugin and MPS control daemon; +#. DCGM and DCGM Exporter; +#. GFD; +#. MIG Manager and node-status exporter; and +#. optional vGPU, VFIO, sandbox, and confidential-computing operands. + +For each state, the Operator loads the embedded manifests, applies policy-specific transformations, +and creates or updates Kubernetes objects. +Objects within a state are processed in a stable order so that service accounts and RBAC exist +before the workloads that use them. + +An operand state can be disabled by policy or skipped when another component owns that function. +For example, the ``ClusterPolicy`` driver state is skipped when ``NVIDIADriver`` custom resources +manage the driver. + +The Operator records the aggregate result in ``ClusterPolicy.status``. +If an operand is not ready, reconciliation is retried. +A missing NFD label set or a cluster with no GPU nodes is reported as a condition rather than +treated as a terminal controller failure. + +*********************** +GPU Workload Data Flow +*********************** + +For the default device-plugin model, a GPU request follows this path: + +#. NFD detects NVIDIA hardware and labels the Node. +#. The Node labeling reconciler adds the Operator's GPU and operand placement labels. +#. The driver DaemonSet makes the GPU available to the host operating system. +#. The Container Toolkit configures the container runtime and CDI integration. +#. The validator records that the required host components are ready. +#. The NVIDIA Kubernetes Device Plugin registers resources such as ``nvidia.com/gpu`` with kubelet. +#. A workload requests an NVIDIA extended resource in its Pod specification. +#. The Kubernetes scheduler selects a Node with available GPU capacity. +#. Kubelet calls the device plugin to allocate the device, and the runtime or CDI injects the + required device nodes and configuration into the container. + +MIG changes the resources advertised by the device plugin, but it does not change this +control-plane flow. +For MIG configuration concepts, refer to :ref:`install-gpu-operator-mig`. + +********************* +Deployment Topologies +********************* + +The Operator supports several deployment topologies that combine the components in different ways. +The following table compares them. +The sections that follow describe each topology in more detail. + +.. list-table:: Deployment topology comparison + :header-rows: 1 + :widths: 26 30 44 + + * - Topology + - Driver management + - Operands and typical use + * - Operator-managed stack (default) + - ``ClusterPolicy`` manages the driver. + - Deploys the full container operand set on NFD-discovered nodes. + Use when the Operator should own the complete GPU software lifecycle. + * - Preinstalled host components + - Driver, toolkit, or both installed outside the Operator; the matching state is disabled. + - The Operator still manages the device plugin, validation, telemetry, and other enabled + operands. + Use when the platform or node image already provides host software. + * - Multiple driver pools + - ``NVIDIADriver`` custom resources, one per node pool, selected by node selector. + - The Operator creates driver DaemonSets per operating system and, for precompiled drivers, + per kernel version. + Use when node pools require different driver types or versions. + * - Sandboxed and virtualized workloads + - Data-center driver or vGPU host driver, depending on the workload configuration. + - Replaces the container operands with passthrough, vGPU, Kata, or confidential-computing + operands. + Use for virtual machine or sandboxed GPU workloads. + +Operator-managed stack +====================== + +This is the default topology. +``ClusterPolicy`` manages the driver and the remaining operands on nodes discovered by NFD. +Use this topology when the Operator should own the complete GPU software lifecycle. + +Preinstalled host components +============================ + +The driver, the Container Toolkit, or both can be installed outside the Operator. +The corresponding policy state is disabled, while the Operator continues to manage the +device plugin, validation, telemetry, and other enabled operands. +Use this topology when the platform or image lifecycle already owns host software. + +Multiple driver pools +===================== + +The NVIDIA driver custom resource mode assigns each GPU node to an ``NVIDIADriver`` resource by +node selector. +The controller creates the driver DaemonSets needed for the matching operating system and, for +precompiled drivers, kernel versions. +Use this topology when node pools require different driver types or versions. + +Sandboxed and virtualized workloads +=================================== + +Optional policy settings replace or extend the default container workload labels with operands for +GPU passthrough, NVIDIA vGPU, Kata Containers, or confidential containers. +These operands remain under the same reconciliation and node-labeling control plane, but require +additional runtime and platform integration. + +****************** +Integration Points +****************** + +Node Feature Discovery +====================== + +NFD is the hardware-discovery input to the architecture. +Without the NVIDIA PCI labels that NFD publishes, the Operator cannot classify GPU nodes or place +the node-level operands. + +Container runtimes +================== + +The Container Toolkit integrates with containerd, CRI-O, or Docker. +CDI is enabled by default and provides a runtime-independent device injection path. +RuntimeClass and NVIDIA Runtime Injection (NRI) options support alternate runtime configurations. + +Prometheus +========== + +DCGM Exporter exposes GPU telemetry. +When the Prometheus Operator integration is enabled, the GPU Operator creates the corresponding +ServiceMonitor and PrometheusRule resources. +The Operator also publishes reconciliation, GPU-node, and driver-upgrade metrics. + +OpenShift +========= + +On Red Hat OpenShift, the Operator detects platform APIs and can integrate with Security Context +Constraints, the Driver Toolkit, cluster proxy settings, and user-workload monitoring. + +************************** +Operational Considerations +************************** + +Observability +============= + +Use the following signals together: + +* ``ClusterPolicy.status`` and ``NVIDIADriver.status`` report reconciliation state and conditions. +* Controller logs identify the operand state and Kubernetes object that failed to reconcile. +* Operator metrics report reconciliation status, GPU-node counts, and upgrade activity. +* DCGM Exporter reports GPU health and utilization. +* The optional node-status exporter reports node-level GPU state. + +Resilience and upgrades +======================= + +Reconciliation is idempotent: the Operator continuously compares desired and observed state and +retries resources that are not ready. +Optional leader election prevents multiple Operator replicas from acting as the active controller. + +The upgrade reconciler coordinates automatic driver rollout through node upgrade-state labels. +It can drain GPU workloads before a driver pod is replaced and returns the node to service after +validation. +For configuration and limitations, refer to :ref:`gpu-driver-upgrades`. + +Security boundary +================= + +Several node-level operands require privileged containers, host PID or IPC access, host-path +mounts, and permission to load kernel modules or restart the container runtime. +Restrict access to the Operator namespace and its custom resources to cluster administrators. +For details, refer to :doc:`Security Considerations `. + +******* +Related +******* + +* :ref:`operator-install-guide` +* :ref:`clusterpolicy-reference` +* :ref:`gpu-driver-upgrades` +* :ref:`install-gpu-operator-mig` +* :doc:`Troubleshooting the NVIDIA GPU Operator ` + +.. Source baseline: NVIDIA/gpu-operator origin/main at + 57752060b8cd83ffa4a54a58b2de093e48f8bb5e (2026-07-20). diff --git a/gpu-operator/clusterpolicy-reference.rst b/gpu-operator/clusterpolicy-reference.rst new file mode 100644 index 000000000..f33e7521b --- /dev/null +++ b/gpu-operator/clusterpolicy-reference.rst @@ -0,0 +1,1903 @@ + +.. license-header + SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +.. headings (h1/h2/h3/h4/h5) are # * = - ^ + +.. _clusterpolicy-reference: + +############################################## +ClusterPolicy Custom Resource Reference +############################################## + +The ``ClusterPolicy`` custom resource definition (CRD) is the primary configuration object for the NVIDIA GPU Operator. +A single ``ClusterPolicy`` instance in the cluster describes the desired state of every GPU Operator component—which operands to enable, +which container images to use, and how each component should be configured. + +The GPU Operator watches the ``ClusterPolicy`` object and reconciles all managed DaemonSets and Deployments to match the desired state. + +.. note:: + + The NVIDIA GPU Operator also provides a separate ``NVIDIADriver`` custom resource for fine-grained, per-node driver configuration. + When you enable the NVIDIA Driver CRD (``spec.driver.useNvidiaDriverCRD: true``), driver configuration is taken from ``NVIDIADriver`` objects rather than from the ``driver`` section of ``ClusterPolicy``. + For more information, refer to :doc:`GPU Driver CRD `. + +********************************* +ClusterPolicy Resource Structure +********************************* + +``ClusterPolicy`` is a cluster-scoped resource in the ``nvidia.com/v1`` API group. + +.. code-block:: yaml + + apiVersion: nvidia.com/v1 + kind: ClusterPolicy + metadata: + name: gpu-cluster-policy + spec: + operator: {} + daemonsets: {} + driver: {} + toolkit: {} + devicePlugin: {} + dcgmExporter: {} + dcgm: {} + nodeStatusExporter: {} + gfd: {} + mig: {} + migManager: {} + validator: {} + gds: {} + gdrcopy: {} + sandboxWorkloads: {} + vfioManager: {} + sandboxDevicePlugin: {} + vgpuManager: {} + vgpuDeviceManager: {} + cdi: {} + kataManager: {} + ccManager: {} + hostPaths: {} + kataSandboxDevicePlugin: {} + + +**Top-Level Spec Fields** + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Required + + * - ``operator`` + - Configuration options for the GPU Operator itself. + - Yes + + * - ``daemonsets`` + - Common settings applied to all DaemonSets managed by the Operator. + - Yes + + * - ``driver`` + - Configuration for deploying and managing the NVIDIA GPU Driver. + - Yes + + * - ``toolkit`` + - Configuration for the NVIDIA Container Toolkit. + - Yes + + * - ``devicePlugin`` + - Configuration for the NVIDIA Kubernetes Device Plugin. + - Yes + + * - ``dcgmExporter`` + - Configuration for the NVIDIA DCGM Exporter (GPU telemetry). + - Yes + + * - ``dcgm`` + - Configuration for deploying NVIDIA DCGM host engine as a standalone pod. + - Yes + + * - ``nodeStatusExporter`` + - Configuration for the Node Status Exporter. + - Yes + + * - ``gfd`` + - Configuration for the NVIDIA GPU Feature Discovery (GFD) plugin. + - Yes + + * - ``mig`` + - MIG (Multi-Instance GPU) strategy configuration. + - No + + * - ``migManager`` + - Configuration for the NVIDIA MIG Manager. + - No + + * - ``validator`` + - Configuration for the NVIDIA GPU Operator Validator. + - No + + * - ``gds`` + - Configuration for NVIDIA GPUDirect Storage (GDS). Experimental. + - No + + * - ``gdrcopy`` + - Configuration for the NVIDIA GDRCopy driver. + - No + + * - ``sandboxWorkloads`` + - Configuration for sandbox workloads (KubeVirt, Kata Containers). + - No + + * - ``vfioManager`` + - Configuration for the VFIO-PCI Manager (used with VM passthrough workloads). + - No + + * - ``sandboxDevicePlugin`` + - Configuration for the NVIDIA KubeVirt GPU Device Plugin. + - No + + * - ``vgpuManager`` + - Configuration for the NVIDIA vGPU Manager. + - No + + * - ``vgpuDeviceManager`` + - Configuration for the NVIDIA vGPU Device Manager. + - No + + * - ``cdi`` + - Configuration for the Container Device Interface (CDI). + - No + + * - ``kataManager`` + - Configuration for the Kata Manager (Kata Containers support). + - No + + * - ``ccManager`` + - Configuration for the NVIDIA Confidential Computing Manager. + - No + + * - ``hostPaths`` + - Custom host filesystem paths used by GPU Operator components. + - No + + * - ``kataSandboxDevicePlugin`` + - Configuration for the NVIDIA Kata Sandbox Device Plugin. + - No + + * - ``psa`` + - Configuration for PodSecurityAdmission. + - No + + * - ``psp`` + - Deprecated. PodSecurityPolicy configuration (no longer supported). + - No + + +**Common Component Fields** + +Most component sections share the following fields. +Component-specific fields are documented in each section below. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``enabled`` + - Whether the Operator deploys this component. + Set to ``false`` to disable a component without removing the rest of the GPU Operator configuration. + - Varies by component. See individual sections. + + * - ``repository`` + - Container registry and path for the component image. + Override this when using a private registry or air-gapped environment. + - Component-specific NGC registry path. + + * - ``image`` + - Container image name (without tag). + - Component-specific. + + * - ``version`` + - Image tag or version string. + - Defined by the Operator release. Refer to the :ref:`operator-component-matrix`. + + * - ``imagePullPolicy`` + - Kubernetes `image pull policy `_. + Accepted values: ``Always``, ``IfNotPresent``, ``Never``. + - ``IfNotPresent`` + + * - ``imagePullSecrets`` + - List of Kubernetes Secret names that contain credentials for pulling images from a private registry. + - None + + * - ``resources`` + - CPU and memory resource requests and limits for component pods. + Follows the standard Kubernetes ``ResourceRequirements`` structure. + - None (uses Kubernetes defaults) + + * - ``args`` + - Additional command-line arguments to pass to the component container. + - None + + * - ``env`` + - List of environment variables (``name``/``value`` pairs) to set in the component container. + - None + + * - ``hostNetwork`` + - When ``true``, the component pod runs in the host network namespace. + Required in some environments for proper network access. + - ``false`` + + +************************* +spec.operator +************************* + +Configures operator-level settings. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``runtimeClass`` + - Specifies the Kubernetes RuntimeClass used for GPU-enabled pods. + - ``nvidia`` + + * - ``labels`` + - Map of additional labels to add to all GPU Operator managed pods. + - None + + * - ``annotations`` + - Map of additional annotations to add to all GPU Operator managed pods. + - None + + * - ``use_ocp_driver_toolkit`` + - On OpenShift, when set to ``true``, the Operator uses the DriverToolkit image to build and install driver kernel modules. + - ``false`` + + * - ``defaultRuntime`` + - Deprecated. The container runtime is now detected automatically at runtime. + - N/A + + * - ``initContainer`` + - Deprecated. Configuration for the init container image used with all components. + - N/A + + +************************* +spec.daemonsets +************************* + +Applies common settings to all DaemonSets managed by the GPU Operator. +These settings serve as defaults and can be supplemented by per-component configuration. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``labels`` + - Map of additional labels to apply to all GPU Operator DaemonSet pods. + - None + + * - ``annotations`` + - Map of additional annotations to apply to all GPU Operator DaemonSet pods. + - None + + * - ``tolerations`` + - List of Kubernetes tolerations to apply to all DaemonSet pods. + Refer to the `Kubernetes tolerations documentation `_. + - None + + * - ``priorityClassName`` + - Priority class for all DaemonSet pods. + - None + + * - ``updateStrategy`` + - DaemonSet update strategy. Accepted values: ``RollingUpdate``, ``OnDelete``. + - ``RollingUpdate`` + + * - ``rollingUpdate.maxUnavailable`` + - For ``RollingUpdate``, the maximum number of nodes that can simultaneously have DaemonSet pods updated. + Accepts an absolute number or a percentage string (for example, ``"25%"``). + - ``1`` + + * - ``podSecurityContext`` + - Pod-level security context applied as defaults to all DaemonSet pods. + Follows the standard Kubernetes ``PodSecurityContext`` structure. + - None + + +************************* +spec.driver +************************* + +Controls NVIDIA GPU Driver deployment. +When ``useNvidiaDriverCRD`` is ``false`` (the default), the Operator manages a single driver DaemonSet +for the entire cluster based on this configuration. +When ``useNvidiaDriverCRD`` is ``true``, driver configuration is instead sourced from ``NVIDIADriver`` custom resources. +Refer to :doc:`GPU Driver CRD ` for details. + +.. list-table:: + :header-rows: 1 + :widths: 30 50 20 + + * - Field + - Description + - Default + + * - ``enabled`` + - When ``true``, the Operator deploys the NVIDIA GPU Driver as a container. + Set to ``false`` on systems that have a pre-installed GPU driver. + - ``true`` + + * - ``useNvidiaDriverCRD`` + - When ``true``, driver management uses ``NVIDIADriver`` custom resources instead of this section. + Refer to :doc:`GPU Driver CRD `. + - ``false`` + + * - ``usePrecompiled`` + - When ``true``, the Operator uses driver containers with pre-compiled kernel modules. + Refer to :doc:`precompiled-drivers` for supported operating systems and limitations. + Set ``version`` to a driver branch (for example, ``"580"``), not a full version. + This field is immutable when using the ``NVIDIADriver`` CRD. + - ``false`` + + * - ``kernelModuleType`` + - Specifies the GPU kernel module type. Accepted values: + + - ``auto`` — The Operator selects open or proprietary modules based on the GPU and driver branch. + Supported with driver 570.86.15 or later; 550 and 535 branch drivers do not support ``auto``. + - ``open`` — Uses NVIDIA Open GPU Kernel Modules (OpenRM). + - ``proprietary`` — Uses the proprietary NVIDIA kernel module. + - ``auto`` + + * - ``useOpenKernelModules`` + - Deprecated as of v25.3.0. Use ``kernelModuleType`` instead. + - N/A + + * - ``repository`` + - Registry path for the driver container. + - ``nvcr.io/nvidia`` + + * - ``image`` + - Driver container image name. + - ``driver`` + + * - ``version`` + - GPU driver version to install. + For a standard driver, specify the full version such as ``580.126.20``. + For precompiled drivers, specify the branch such as ``580``. + Refer to the :ref:`operator-component-matrix` for supported versions. + - Version defined by the Operator release. + + * - ``imagePullPolicy`` + - Image pull policy for the driver container. + - ``IfNotPresent`` + + * - ``imagePullSecrets`` + - Image pull secrets for the driver container registry. + - None + + * - ``hostNetwork`` + - When ``true``, the driver pod runs in the host network namespace. + - ``false`` + + * - ``secretEnv`` + - Name of a Kubernetes Secret containing environment variables to pass to the driver container. + A common use case is providing an Ubuntu Pro token for government-ready deployments. + Refer to :doc:`install-gpu-operator-gov-ready`. + - None + + * - ``startupProbe`` + - Startup probe settings for the driver container. + By default, the driver container waits 60 seconds before probing with ``nvidia-smi``. + Increase ``timeoutSeconds`` if ``nvidia-smi`` is slow in your environment. + See :ref:`ContainerProbeSpec fields `. + - ``initialDelaySeconds: 60, timeoutSeconds: 60`` + + * - ``livenessProbe`` + - Liveness probe settings for the driver container. + See :ref:`ContainerProbeSpec fields `. + - None + + * - ``readinessProbe`` + - Readiness probe settings for the driver container. + See :ref:`ContainerProbeSpec fields `. + - None + + * - ``rdma.enabled`` + - When ``true``, the driver pod builds and loads the ``nvidia-peermem`` kernel module to support legacy GPUDirect RDMA. + Refer to :doc:`gpu-operator-rdma` to determine whether you need this or can use the DMA-BUF approach instead. + - ``false`` + + * - ``rdma.useHostMofed`` + - When ``true``, indicates that MLNX_OFED (MOFED) drivers are pre-installed on the host. + Used together with ``rdma.enabled``. + - ``false`` + + * - ``upgradePolicy`` + - Automatic driver upgrade policy. + When configured, the Operator can perform rolling upgrades of driver pods without manual intervention. + Follows the ``DriverUpgradePolicySpec`` from the ``k8s-operator-libs`` library. + Refer to the `NVIDIA k8s-operator-libs documentation `__ for field details. + - None (upgrades must be triggered manually) + + * - ``manager`` + - Configuration for the NVIDIA Driver Manager init container, which prepares nodes for driver installation. + See :ref:`DriverManagerSpec fields `. + - Managed by the Operator. + + * - ``resources`` + - Resource requests and limits for the driver container. + - None + + * - ``args`` + - Additional arguments for the driver container. + - None + + * - ``env`` + - Environment variables for the driver container. + - None + + * - ``repoConfig`` + - Custom apt/yum repository configuration for the driver container. + Specify a ConfigMap name that the Operator mounts into the driver container for custom package sources. + Refer to :doc:`install-gpu-operator-air-gapped` for air-gapped use cases. + - None + + * - ``certConfig`` + - Custom certificate configuration for the driver container. + Specify a ConfigMap name containing custom CA certificates. + - None + + * - ``licensingConfig`` + - vGPU license configuration. See :ref:`DriverLicensingConfigSpec fields `. + Required when deploying with NVIDIA vGPU. + - None + + * - ``virtualTopology`` + - Virtual topology daemon configuration for NVIDIA vGPU drivers. + Specify a ConfigMap name containing the ``nvidia-topologyd.conf`` configuration file. + - None + + * - ``kernelModuleConfig`` + - Custom kernel module parameters for the NVIDIA driver. + Specify a ConfigMap name containing kernel module configuration. + Refer to :doc:`custom-driver-params`. + - None + + +.. _driver-probe-spec: + +ContainerProbeSpec Fields +========================== + +Used by ``startupProbe``, ``livenessProbe``, and ``readinessProbe`` in the ``driver`` section. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``initialDelaySeconds`` + - Seconds to wait after the container starts before initiating probes. + - ``0`` + + * - ``timeoutSeconds`` + - Seconds after which a probe times out. Minimum value: ``1``. + - ``1`` + + * - ``periodSeconds`` + - How often (in seconds) to perform the probe. Minimum value: ``1``. + - ``10`` + + * - ``successThreshold`` + - Minimum consecutive successes for the probe to be considered successful. Minimum: ``1``. + - ``1`` + + * - ``failureThreshold`` + - Minimum consecutive failures before the probe is considered failed. Minimum: ``1``. + - ``3`` + + +.. _driver-manager-spec: + +DriverManagerSpec Fields +========================= + +Configures the Driver Manager init container that runs before the driver container to manage driver lifecycle (for example, unloading existing modules before an upgrade). + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``repository`` + - Registry path for the Driver Manager image. + - Operator default. + + * - ``image`` + - Driver Manager image name. + - ``k8s-driver-manager`` + + * - ``version`` + - Driver Manager image version. + Refer to the :ref:`operator-component-matrix`. + - Version defined by the Operator release. + + * - ``imagePullPolicy`` + - Image pull policy for the Driver Manager image. + - ``IfNotPresent`` + + * - ``imagePullSecrets`` + - Image pull secrets for the Driver Manager registry. + - None + + * - ``env`` + - Environment variables to set in the Driver Manager init container. + - None + + +.. _driver-licensing-spec: + +DriverLicensingConfigSpec Fields +================================== + +Required for NVIDIA vGPU deployments to configure the license server. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``secretName`` + - Name of a Kubernetes Secret containing the NLS (NVIDIA Licensing System) client configuration token. + Refer to :doc:`install-gpu-operator-vgpu`. + - None + + * - ``configMapName`` + - Deprecated. Use ``secretName`` instead. + - None + + * - ``nlsEnabled`` + - When ``true``, uses the NVIDIA Licensing System (NLS) for license management. + - ``false`` + + +************************* +spec.toolkit +************************* + +Controls deployment of the NVIDIA Container Toolkit, which configures the container runtime +(containerd, CRI-O, or Docker) to support GPU-accelerated containers. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``enabled`` + - When ``true``, deploys the NVIDIA Container Toolkit. + Set to ``false`` on systems where the NVIDIA Container Toolkit is pre-installed. + - ``true`` + + * - ``repository`` + - Registry path for the Container Toolkit image. + - ``nvcr.io/nvidia/k8s`` + + * - ``image`` + - Container Toolkit image name. + - ``container-toolkit`` + + * - ``version`` + - Container Toolkit image version. + Refer to the :ref:`operator-component-matrix`. + - Version defined by the Operator release. + + * - ``imagePullPolicy`` + - Image pull policy. + - ``IfNotPresent`` + + * - ``imagePullSecrets`` + - Image pull secrets. + - None + + * - ``installDir`` + - Host path where the NVIDIA Container Toolkit installs its binaries. + - ``/usr/local/nvidia`` + + * - ``hostNetwork`` + - When ``true``, the Container Toolkit pod uses the host network namespace. + - ``false`` + + * - ``resources`` + - Resource requests and limits. + - None + + * - ``args`` + - Additional arguments for the Container Toolkit container. + - None + + * - ``env`` + - Environment variables for the Container Toolkit container. + Used to configure non-default container runtime socket paths or configuration locations. + Refer to :doc:`getting-started` for examples. + - None + + +************************* +spec.devicePlugin +************************* + +Controls deployment of the NVIDIA Kubernetes Device Plugin, which advertises GPU resources to the Kubernetes scheduler. +For configuration options such as time-slicing, MPS, or MIG strategy, the Device Plugin is configured +through a ConfigMap. Refer to :doc:`gpu-sharing` and :doc:`gpu-operator-mig`. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``enabled`` + - When ``true``, deploys the NVIDIA Kubernetes Device Plugin. + - ``true`` + + * - ``repository`` + - Registry path for the Device Plugin image. + - ``nvcr.io/nvidia`` + + * - ``image`` + - Device Plugin image name. + - ``k8s-device-plugin`` + + * - ``version`` + - Device Plugin image version. + Refer to the :ref:`operator-component-matrix`. + - Version defined by the Operator release. + + * - ``imagePullPolicy`` + - Image pull policy. + - ``IfNotPresent`` + + * - ``imagePullSecrets`` + - Image pull secrets. + - None + + * - ``hostNetwork`` + - When ``true``, the Device Plugin pod uses the host network namespace. + - ``false`` + + * - ``resources`` + - Resource requests and limits. + - None + + * - ``args`` + - Additional arguments for the Device Plugin container. + - None + + * - ``env`` + - Environment variables for the Device Plugin container. + - None + + * - ``config.name`` + - Name of the ConfigMap containing the Device Plugin configuration. + The ConfigMap can contain multiple named configurations for different node types. + Refer to :doc:`gpu-sharing` for configuring time-slicing and MPS. + - None + + * - ``config.default`` + - The default configuration name within the ConfigMap to use when no specific configuration is selected. + - None + + * - ``mps.root`` + - Host path to use as the MPS (Multi-Process Service) root directory. + Relevant when using the Device Plugin in MPS mode. + - ``/run/nvidia/mps`` + + +************************* +spec.dcgmExporter +************************* + +Controls deployment of the NVIDIA DCGM Exporter, which exposes GPU metrics in Prometheus format. +Refer to the `DCGM Exporter documentation `_ for details on available metrics. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``enabled`` + - When ``true``, deploys the NVIDIA DCGM Exporter. + - ``true`` + + * - ``repository`` + - Registry path for the DCGM Exporter image. + - ``nvcr.io/nvidia/k8s`` + + * - ``image`` + - DCGM Exporter image name. + - ``dcgm-exporter`` + + * - ``version`` + - DCGM Exporter image version. + Refer to the :ref:`operator-component-matrix`. + - Version defined by the Operator release. + + * - ``imagePullPolicy`` + - Image pull policy. + - ``IfNotPresent`` + + * - ``imagePullSecrets`` + - Image pull secrets. + - None + + * - ``hostNetwork`` + - When ``true``, the DCGM Exporter pod exposes its metrics port on the host network namespace. + - ``false`` + + * - ``hostPID`` + - When ``true``, the DCGM Exporter pod can access the host's PID namespace. + Required for some per-process GPU metrics. + - ``false`` + + * - ``resources`` + - Resource requests and limits. + - None + + * - ``args`` + - Additional arguments for the DCGM Exporter container. + - None + + * - ``env`` + - Environment variables for the DCGM Exporter container. + - None + + * - ``config.name`` + - Name of the ConfigMap containing a custom ``dcgm-metrics.csv`` file. + Use this to override the default set of metrics collected by DCGM Exporter. + - None (uses default metric set) + + * - ``service.type`` + - Kubernetes Service type for the DCGM Exporter Service. + Refer to the `Kubernetes Service documentation `_. + - ``ClusterIP`` + + * - ``service.internalTrafficPolicy`` + - `Internal traffic policy `_ for the DCGM Exporter Service. + Use ``Local`` to limit metrics scraping to only the local node's DCGM Exporter. + - ``Cluster`` + + * - ``serviceMonitor.enabled`` + - When ``true``, deploys a Prometheus Operator ``ServiceMonitor`` resource for automatic metrics discovery. + Requires the Prometheus Operator to be installed in the cluster. + - ``false`` + + * - ``serviceMonitor.interval`` + - How frequently Prometheus scrapes DCGM Exporter. + If not specified, Prometheus uses its global scrape interval. + Supported units: ``y``, ``w``, ``d``, ``h``, ``m``, ``s``, ``ms``. + - Prometheus global scrape interval + + * - ``serviceMonitor.honorLabels`` + - When ``true``, the metric's own labels take precedence on collision with target labels. + - ``false`` + + * - ``serviceMonitor.additionalLabels`` + - Map of additional labels to add to the ``ServiceMonitor`` resource. + Use these to match the label selectors of your Prometheus instance. + - None + + * - ``serviceMonitor.relabelings`` + - List of Prometheus relabel configurations to rewrite labels on metric sets. + Follows the Prometheus Operator ``RelabelConfig`` structure. + - None + + * - ``hpcJobMapping.enabled`` + - When ``true``, enables HPC job mapping for DCGM Exporter. + Allows correlating GPU metrics with HPC workload manager job IDs. + - ``false`` + + * - ``hpcJobMapping.directory`` + - Host directory path where HPC job mapping files are written by the workload manager. + - ``/var/lib/dcgm-exporter/job-mapping`` + + +************************* +spec.dcgm +************************* + +Controls deployment of NVIDIA DCGM host engine as a standalone pod. +By default, DCGM runs as a sidecar in the DCGM Exporter pod. +Enabling the standalone DCGM pod can improve performance when multiple consumers +need to connect to the same DCGM host engine. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``enabled`` + - When ``true``, deploys NVIDIA DCGM host engine as a standalone pod. + - ``true`` + + * - ``repository`` + - Registry path for the DCGM image. + - ``nvcr.io/nvidia/cloud-native`` + + * - ``image`` + - DCGM image name. + - ``dcgm`` + + * - ``version`` + - DCGM image version. + Refer to the :ref:`operator-component-matrix`. + - Version defined by the Operator release. + + * - ``imagePullPolicy`` + - Image pull policy. + - ``IfNotPresent`` + + * - ``imagePullSecrets`` + - Image pull secrets. + - None + + * - ``hostNetwork`` + - When ``true``, the DCGM pod uses the host network namespace. + - ``false`` + + * - ``hostPort`` + - Deprecated. Host port previously used for the DCGM engine. + - N/A + + * - ``resources`` + - Resource requests and limits. + - None + + * - ``args`` + - Additional arguments. + - None + + * - ``env`` + - Environment variables. + - None + + +************************* +spec.nodeStatusExporter +************************* + +Controls deployment of the Node Status Exporter, which reports GPU node status information to the Kubernetes API. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``enabled`` + - When ``true``, deploys the Node Status Exporter. + - ``false`` + + * - ``repository`` + - Registry path for the Node Status Exporter image. + - ``nvcr.io/nvidia/cloud-native`` + + * - ``image`` + - Node Status Exporter image name. + - ``gpu-operator-validator`` + + * - ``version`` + - Image version. + Refer to the :ref:`operator-component-matrix`. + - Version defined by the Operator release. + + * - ``imagePullPolicy`` + - Image pull policy. + - ``IfNotPresent`` + + * - ``imagePullSecrets`` + - Image pull secrets. + - None + + * - ``hostNetwork`` + - When ``true``, the Node Status Exporter pod uses the host network namespace. + - ``false`` + + * - ``resources`` + - Resource requests and limits. + - None + + * - ``args`` + - Additional arguments. + - None + + * - ``env`` + - Environment variables. + - None + + +************************* +spec.gfd +************************* + +Controls deployment of the NVIDIA GPU Feature Discovery (GFD) plugin, which labels Kubernetes nodes with GPU hardware properties +(such as GPU model, driver version, CUDA version, and MIG configuration). +Node Feature Discovery (NFD) must be deployed in the cluster for GFD to function. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``enabled`` + - When ``true``, deploys GPU Feature Discovery. + - ``true`` + + * - ``repository`` + - Registry path for the GFD image. + - ``nvcr.io/nvidia`` + + * - ``image`` + - GFD image name. + - ``k8s-device-plugin`` + + * - ``version`` + - GFD image version. + Refer to the :ref:`operator-component-matrix`. + - Version defined by the Operator release. + + * - ``imagePullPolicy`` + - Image pull policy. + - ``IfNotPresent`` + + * - ``imagePullSecrets`` + - Image pull secrets. + - None + + * - ``hostNetwork`` + - When ``true``, the GFD pod uses the host network namespace. + - ``false`` + + * - ``resources`` + - Resource requests and limits. + - None + + * - ``args`` + - Additional arguments. + - None + + * - ``env`` + - Environment variables. + - None + + +************************* +spec.mig +************************* + +Configures the MIG (Multi-Instance GPU) strategy. +Refer to :doc:`gpu-operator-mig` for complete MIG configuration guidance. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``strategy`` + - MIG strategy to use on MIG-capable GPUs (such as A100, H100). + Accepted values: + + - ``none`` — MIG is not enabled. All GPU memory is exposed as a single resource. + - ``single`` — All MIG instances on a node share the same MIG geometry. + Only one ``nvidia.com/gpu`` resource type is advertised. + - ``mixed`` — Nodes can have different MIG instance types. + Individual MIG instance types are advertised as separate resources. + - ``none`` + + +************************* +spec.migManager +************************* + +Controls deployment of the NVIDIA MIG Manager, which watches for MIG geometry changes +and automatically reconfigures nodes. The MIG Manager runs only on MIG-capable nodes. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``enabled`` + - When ``true``, deploys the NVIDIA MIG Manager. + The MIG Manager only schedules on nodes with MIG-capable GPUs. + - ``true`` + + * - ``repository`` + - Registry path for the MIG Manager image. + - ``nvcr.io/nvidia/cloud-native`` + + * - ``image`` + - MIG Manager image name. + - ``k8s-mig-manager`` + + * - ``version`` + - MIG Manager image version. + Refer to the :ref:`operator-component-matrix`. + - Version defined by the Operator release. + + * - ``imagePullPolicy`` + - Image pull policy. + - ``IfNotPresent`` + + * - ``imagePullSecrets`` + - Image pull secrets. + - None + + * - ``hostNetwork`` + - When ``true``, the MIG Manager pod uses the host network namespace. + - ``false`` + + * - ``resources`` + - Resource requests and limits. + - None + + * - ``args`` + - Additional arguments. + - None + + * - ``env`` + - Environment variables. + - None + + * - ``config.name`` + - Name of the ConfigMap containing the ``mig-parted`` MIG partition configuration. + If not specified, MIG configuration is dynamically generated from the node's hardware. + - None + + * - ``config.default`` + - Default MIG configuration to apply when a node has no ``nvidia.com/mig.config`` label. + Accepted values: ``all-disabled``. + - ``all-disabled`` + + * - ``gpuClientsConfig.name`` + - Name of the ConfigMap listing GPU client processes that the MIG Manager should stop before reconfiguring MIG geometry. + - None + + +************************* +spec.validator +************************* + +Controls deployment of the NVIDIA GPU Operator Validator, which runs a series of tests +to confirm that each GPU Operator component is functioning correctly on each node. +The Validator runs as a DaemonSet and reports pass/fail status. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``repository`` + - Registry path for the Validator image. + - ``nvcr.io/nvidia/cloud-native`` + + * - ``image`` + - Validator image name. + - ``gpu-operator-validator`` + + * - ``version`` + - Validator image version. + Refer to the :ref:`operator-component-matrix`. + - Version defined by the Operator release. + + * - ``imagePullPolicy`` + - Image pull policy. + - ``IfNotPresent`` + + * - ``imagePullSecrets`` + - Image pull secrets. + - None + + * - ``hostNetwork`` + - When ``true``, the Validator pod uses the host network namespace. + - ``false`` + + * - ``resources`` + - Resource requests and limits. + - None + + * - ``args`` + - Additional arguments. + - None + + * - ``env`` + - Environment variables. + - None + + * - ``plugin.env`` + - Environment variables specific to the Device Plugin validation step. + - None + + * - ``toolkit.env`` + - Environment variables specific to the Container Toolkit validation step. + - None + + * - ``driver.env`` + - Environment variables specific to the driver validation step. + - None + + * - ``cuda.env`` + - Environment variables specific to the CUDA workload validation step. + - None + + * - ``vfioPCI.env`` + - Environment variables specific to the VFIO-PCI device validation step. + - None + + * - ``vgpuManager.env`` + - Environment variables specific to the vGPU Manager validation step. + - None + + * - ``vgpuDevices.env`` + - Environment variables specific to the vGPU device validation step. + - None + + +************************* +spec.gds +************************* + +Controls deployment of the NVIDIA GPUDirect Storage (GDS) driver. +GDS enables direct DMA transfers between GPU memory and storage without CPU involvement. +This feature is experimental. + +When using the ``NVIDIADriver`` CRD, configure GDS within the ``NVIDIADriver`` spec instead. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``enabled`` + - When ``true``, enables GPUDirect Storage. + - ``false`` + + * - ``repository`` + - Registry path for the GDS driver image. + - ``nvcr.io/nvidia/cloud-native`` + + * - ``image`` + - GDS driver image name. + - ``nvidia-fs`` + + * - ``version`` + - GDS driver image version. + Refer to the :ref:`operator-component-matrix`. + - Version defined by the Operator release. + + * - ``imagePullPolicy`` + - Image pull policy. + - ``IfNotPresent`` + + * - ``imagePullSecrets`` + - Image pull secrets. + - None + + * - ``args`` + - Additional arguments. + - None + + * - ``env`` + - Environment variables. + - None + + +************************* +spec.gdrcopy +************************* + +Controls deployment of the NVIDIA GDRCopy driver (``gdrdrv`` kernel module), which enables +low-latency GPU-to-CPU memory copies using GPUDirect RDMA. +When using ``ClusterPolicy``-managed drivers, the GDRCopy driver runs as a sidecar in the driver pod. + +When using the ``NVIDIADriver`` CRD, configure GDRCopy within the ``NVIDIADriver`` spec instead. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``enabled`` + - When ``true``, deploys the GDRCopy driver. + - ``false`` + + * - ``repository`` + - Registry path for the GDRCopy driver image. + - ``nvcr.io/nvidia/cloud-native`` + + * - ``image`` + - GDRCopy driver image name. + - ``gdrdrv`` + + * - ``version`` + - GDRCopy driver image version. + Refer to the :ref:`operator-component-matrix`. + - Version defined by the Operator release. + + * - ``imagePullPolicy`` + - Image pull policy. + - ``IfNotPresent`` + + * - ``imagePullSecrets`` + - Image pull secrets. + - None + + * - ``args`` + - Additional arguments. + - None + + * - ``env`` + - Environment variables. + - None + + +************************* +spec.sandboxWorkloads +************************* + +Controls support for sandbox-based GPU workloads such as KubeVirt virtual machines or Kata Containers. +When enabled, the Operator deploys additional components including the VFIO Manager, vGPU Manager (if applicable), +and sandbox-specific device plugins. + +Refer to :doc:`gpu-operator-kubevirt` and :doc:`deploy-kata-containers` for deployment guides. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``enabled`` + - When ``true``, enables sandbox workload support and deploys the additional required components. + - ``false`` + + * - ``defaultWorkload`` + - Default GPU workload type to configure on worker nodes. + Accepted values: + + - ``container`` — Standard GPU containers. + - ``vm-passthrough`` — GPU passthrough to virtual machines using VFIO-PCI. + - ``vm-vgpu`` — vGPU virtualization. + - ``container`` + + * - ``mode`` + - Sandbox mode for sandboxed workloads. + Accepted values: + + - ``kubevirt`` — KubeVirt-based virtual machines. + - ``kata`` — Kata Containers-based sandboxes. + - ``kubevirt`` + + +************************* +spec.vfioManager +************************* + +Controls deployment of the VFIO-PCI Manager, which binds GPU devices to the ``vfio-pci`` +kernel driver to enable PCI passthrough to virtual machines. +Only deployed when ``sandboxWorkloads.enabled`` is ``true`` and ``sandboxWorkloads.mode`` is ``kubevirt``. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``enabled`` + - When ``true``, deploys the VFIO Manager. + - ``false`` + + * - ``repository`` + - Registry path for the VFIO Manager image. + - ``nvcr.io/nvidia/cloud-native`` + + * - ``image`` + - VFIO Manager image name. + - ``vfio-manager`` + + * - ``version`` + - VFIO Manager image version. + - Version defined by the Operator release. + + * - ``imagePullPolicy`` + - Image pull policy. + - ``IfNotPresent`` + + * - ``imagePullSecrets`` + - Image pull secrets. + - None + + * - ``hostNetwork`` + - When ``true``, the VFIO Manager pod uses the host network namespace. + - ``false`` + + * - ``resources`` + - Resource requests and limits. + - None + + * - ``args`` + - Additional arguments. + - None + + * - ``env`` + - Environment variables. + - None + + * - ``driverManager`` + - Driver Manager configuration for the VFIO Manager. + See :ref:`DriverManagerSpec fields `. + - Managed by the Operator. + + +************************* +spec.sandboxDevicePlugin +************************* + +Controls deployment of the NVIDIA KubeVirt GPU Device Plugin, which advertises vGPU and GPU passthrough +devices to the Kubernetes scheduler for use by KubeVirt virtual machines. +Only deployed when ``sandboxWorkloads.enabled`` is ``true``. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``enabled`` + - When ``true``, deploys the KubeVirt GPU Device Plugin. + - ``false`` + + * - ``repository`` + - Registry path for the Sandbox Device Plugin image. + - ``nvcr.io/nvidia`` + + * - ``image`` + - Sandbox Device Plugin image name. + - ``kubevirt-gpu-device-plugin`` + + * - ``version`` + - Image version. + Refer to the :ref:`operator-component-matrix`. + - Version defined by the Operator release. + + * - ``imagePullPolicy`` + - Image pull policy. + - ``IfNotPresent`` + + * - ``imagePullSecrets`` + - Image pull secrets. + - None + + * - ``hostNetwork`` + - When ``true``, the Sandbox Device Plugin pod uses the host network namespace. + - ``false`` + + * - ``resources`` + - Resource requests and limits. + - None + + * - ``args`` + - Additional arguments. + - None + + * - ``env`` + - Environment variables. + - None + + +************************* +spec.vgpuManager +************************* + +Controls deployment of the NVIDIA vGPU Manager (vGPU host driver), which runs on the hypervisor node +and manages vGPU instances for virtual machines. +Only deployed when ``sandboxWorkloads.enabled`` is ``true``. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``enabled`` + - When ``true``, deploys the NVIDIA vGPU Manager. + - ``false`` + + * - ``repository`` + - Registry path for the vGPU Manager image. + - Must be specified; images are not on NGC. + + * - ``image`` + - vGPU Manager image name. + - Must be specified. + + * - ``version`` + - vGPU Manager image version. Must match the NVIDIA vGPU software version in use. + - Must be specified. + + * - ``imagePullPolicy`` + - Image pull policy. + - ``IfNotPresent`` + + * - ``imagePullSecrets`` + - Image pull secrets. Required if images are on a private registry. + - None + + * - ``hostNetwork`` + - When ``true``, the vGPU Manager pod uses the host network namespace. + - ``false`` + + * - ``resources`` + - Resource requests and limits. + - None + + * - ``args`` + - Additional arguments. + - None + + * - ``env`` + - Environment variables. + - None + + * - ``kernelModuleConfig`` + - Custom kernel module parameters for the vGPU Manager. + - None + + * - ``driverManager`` + - Driver Manager configuration. + See :ref:`DriverManagerSpec fields `. + - Managed by the Operator. + + +************************* +spec.vgpuDeviceManager +************************* + +Controls deployment of the NVIDIA vGPU Device Manager, which manages vGPU device creation +on virtualization hosts. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``enabled`` + - When ``true``, deploys the NVIDIA vGPU Device Manager. + - ``false`` + + * - ``repository`` + - Registry path for the vGPU Device Manager image. + - ``nvcr.io/nvidia/cloud-native`` + + * - ``image`` + - vGPU Device Manager image name. + - ``vgpu-device-manager`` + + * - ``version`` + - vGPU Device Manager image version. + Refer to the :ref:`operator-component-matrix`. + - Version defined by the Operator release. + + * - ``imagePullPolicy`` + - Image pull policy. + - ``IfNotPresent`` + + * - ``imagePullSecrets`` + - Image pull secrets. + - None + + * - ``hostNetwork`` + - When ``true``, the vGPU Device Manager pod uses the host network namespace. + - ``false`` + + * - ``resources`` + - Resource requests and limits. + - None + + * - ``args`` + - Additional arguments. + - None + + * - ``env`` + - Environment variables. + - None + + * - ``config.name`` + - Name of the ConfigMap containing the vGPU device configuration. + The ConfigMap can contain multiple named configurations. + - None + + * - ``config.default`` + - Default configuration name within the ConfigMap. + - ``default`` + + +************************* +spec.cdi +************************* + +Configures how the Container Device Interface (CDI) is used in the cluster. +CDI is the default and recommended mechanism for exposing GPU devices to containers. +Refer to :doc:`cdi` for more information. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``enabled`` + - When ``true``, the Container Toolkit uses CDI as the mechanism for making GPUs accessible to containers. + - ``true`` + + * - ``nriPluginEnabled`` + - When ``true``, deploys an NRI (Node Resource Interface) plugin as an additional mechanism + for injecting CDI devices into GPU management containers. + When enabled, you do not need to configure ``toolkit.env`` options for CDI injection. + - ``false`` + + * - ``default`` + - Deprecated as of v25.10.0. CDI is enabled by default and this field is ignored. + - N/A + + +************************* +spec.kataManager +************************* + +Controls deployment of the NVIDIA Kata Manager, which prepares NVIDIA-specific Kata runtime classes +on nodes for use with Kata Containers-based sandbox workloads. +Only deployed when ``sandboxWorkloads.enabled`` is ``true`` and ``sandboxWorkloads.mode`` is ``kata``. + +Refer to :doc:`deploy-kata-containers` for deployment guidance. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``enabled`` + - When ``true``, deploys the Kata Manager. + - ``false`` + + * - ``config`` + - Kata Manager configuration specifying which kata runtime artifacts to use. + Follows the ``Config`` structure from the `k8s-kata-manager `_ repository. + - None + + * - ``repository`` + - Registry path for the Kata Manager image. + - ``nvcr.io/nvidia/cloud-native`` + + * - ``image`` + - Kata Manager image name. + - ``k8s-kata-manager`` + + * - ``version`` + - Kata Manager image version. + - Version defined by the Operator release. + + * - ``imagePullPolicy`` + - Image pull policy. + - ``IfNotPresent`` + + * - ``imagePullSecrets`` + - Image pull secrets. + - None + + * - ``hostNetwork`` + - When ``true``, the Kata Manager pod uses the host network namespace. + - ``false`` + + * - ``resources`` + - Resource requests and limits. + - None + + * - ``args`` + - Additional arguments. + - None + + * - ``env`` + - Environment variables. + - None + + +************************* +spec.ccManager +************************* + +Controls deployment of the NVIDIA Confidential Computing Manager, which configures +the Confidential Computing (CC) mode on compatible GPUs (H100 and later). + +Refer to :doc:`confidential-containers-deploy` for deployment guidance. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``enabled`` + - When ``true``, deploys the Confidential Computing Manager. + - ``false`` + + * - ``defaultMode`` + - Default Confidential Computing mode to apply to all CC-capable GPUs on each node. + Accepted values: + + - ``off`` — CC mode disabled. + - ``on`` — CC mode enabled. + - ``devtools`` — CC mode enabled with DevTools access for debugging. + - None + + * - ``repository`` + - Registry path for the CC Manager image. + - ``nvcr.io/nvidia/cloud-native`` + + * - ``image`` + - CC Manager image name. + - ``k8s-cc-manager`` + + * - ``version`` + - CC Manager image version. + Refer to the :ref:`operator-component-matrix`. + - Version defined by the Operator release. + + * - ``imagePullPolicy`` + - Image pull policy. + - ``IfNotPresent`` + + * - ``imagePullSecrets`` + - Image pull secrets. + - None + + * - ``hostNetwork`` + - When ``true``, the CC Manager pod uses the host network namespace. + - ``false`` + + * - ``resources`` + - Resource requests and limits. + - None + + * - ``args`` + - Additional arguments. + - None + + * - ``env`` + - Environment variables. + - None + + +***************************** +spec.kataSandboxDevicePlugin +***************************** + +Controls deployment of the NVIDIA Kata Sandbox Device Plugin, which advertises GPU devices +to the Kubernetes scheduler for use with Kata Containers-based sandbox workloads. +Deployed when ``sandboxWorkloads.enabled`` is ``true`` and ``sandboxWorkloads.mode`` is ``kata``. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``enabled`` + - When ``true``, deploys the Kata Sandbox Device Plugin. + - ``false`` + + * - ``repository`` + - Registry path for the Kata Sandbox Device Plugin image. + - ``nvcr.io/nvidia`` + + * - ``image`` + - Kata Sandbox Device Plugin image name. + - ``kata-gpu-sandbox-device-plugin`` + + * - ``version`` + - Image version. + Refer to the :ref:`operator-component-matrix`. + - Version defined by the Operator release. + + * - ``imagePullPolicy`` + - Image pull policy. + - ``IfNotPresent`` + + * - ``imagePullSecrets`` + - Image pull secrets. + - None + + * - ``hostNetwork`` + - When ``true``, the Kata Sandbox Device Plugin pod uses the host network namespace. + - ``false`` + + * - ``resources`` + - Resource requests and limits. + - None + + * - ``args`` + - Additional arguments. + - None + + * - ``env`` + - Environment variables. + - None + + +************************* +spec.hostPaths +************************* + +Defines custom host filesystem paths needed by GPU Operator components. +Override these only when your nodes use non-standard filesystem layouts. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``rootFS`` + - Path to the root filesystem of the host. + Must be a chroot-able filesystem. Used by components that interact with the host OS directly + (for example, MIG Manager and Container Toolkit when stopping or restarting systemd services). + - ``/`` + + * - ``driverInstallDir`` + - Root directory where the GPU driver files (libraries, executables, configuration) are installed. + Override when using a custom driver install path. + - ``/run/nvidia/driver`` + + +************************* +spec.psa +************************* + +Configures PodSecurityAdmission for GPU Operator pods. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``enabled`` + - When ``true``, enables PodSecurityAdmission configuration for all GPU Operator pods. + - ``false`` + + +***************************** +ClusterPolicy Status Fields +***************************** + +The Operator sets the following fields in ``ClusterPolicy.status``. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Values + + * - ``state`` + - Overall state of the ClusterPolicy. + - ``ready``, ``notReady``, ``ignored`` + + * - ``namespace`` + - Namespace in which the GPU Operator is installed. + - Operator namespace string. + + * - ``conditions`` + - List of Kubernetes conditions describing the detailed state of the ClusterPolicy. + Each condition has a ``Type``, ``Status``, ``Reason``, and ``Message``. + - Standard Kubernetes condition list. + +To check the ClusterPolicy status: + +.. code-block:: console + + $ kubectl get clusterpolicy gpu-cluster-policy -o jsonpath='{.status}' diff --git a/gpu-operator/getting-started.rst b/gpu-operator/getting-started.rst index 87acbe3b3..a118092e7 100644 --- a/gpu-operator/getting-started.rst +++ b/gpu-operator/getting-started.rst @@ -31,68 +31,20 @@ Installing the NVIDIA GPU Operator The current patch release of this version of the NVIDIA GPU Operator is ``${version}``. -.. admonition:: Red Hat OpenShift Container Platform Install - :class: tip +Before installing the GPU Operator, refer to the :doc:`prerequisites` page to make sure your cluster is configured correctly. - For installation on Red Hat OpenShift Container Platform, refer to :external+ocp:doc:`steps-overview`. +.. seealso:: -************* -Prerequisites -************* + Not sure which install guide fits your environment? Refer to :ref:`install-paths`. -#. You have the ``kubectl`` and ``helm`` CLIs available on a client machine. - - You can run the following commands to install the Helm CLI: - - .. code-block:: console - - $ curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 \ - && chmod 700 get_helm.sh \ - && ./get_helm.sh - -#. If you are planning to use ClusterPolicy for driver configuration, all worker nodes or node groups to run GPU workloads in the Kubernetes cluster must run the same operating system version to use the NVIDIA GPU Driver container. - Alternatively, if you pre-install the NVIDIA GPU Driver on the nodes, then you can run different operating systems. - - For worker nodes or node groups that run CPU workloads only, the nodes can run any operating system because - the GPU Operator does not perform any configuration or management of nodes for CPU-only workloads. - - If you are planning to use NVIDIA GPU Driver Custom Resource Definition, you can use a mix of operating system versions on CPU and GPU nodes. Refer to the :doc:`NVIDIA GPU Driver Custom Resource Definition ` page for more information. - -#. Nodes must be configured with a container engine such as CRI-O or containerd. - -#. If your cluster uses Pod Security Admission (PSA) to restrict the behavior of pods, - label the namespace for the Operator to set the enforcement policy to privileged: - - .. code-block:: console - - $ kubectl create ns gpu-operator - $ kubectl label --overwrite ns gpu-operator pod-security.kubernetes.io/enforce=privileged - -#. Node Feature Discovery (NFD) is a dependency for the Operator on each node. - By default, NFD master and worker are automatically deployed by the Operator. - If NFD is already running in the cluster, then you must disable deploying NFD when you install the Operator. - - One way to determine if NFD is already running in the cluster is to check for an NFD label on your nodes - (requires `jq `__): - - .. code-block:: console - - $ kubectl get nodes -o json | jq '.items[].metadata.labels | keys | any(startswith("feature.node.kubernetes.io"))' - - If the command output is ``true``, then NFD is already running in the cluster. - -********* -Procedure -********* +********** +QuickStart +********** A default installation deploys the NVIDIA GPU driver, NVIDIA Container Toolkit, NVIDIA Device Plugin, DCGM Exporter, and MIG Manager as pods on every GPU worker node. Use ``--set`` options to customize the deployment for your environment. -.. tip:: - - For installation on Red Hat OpenShift Container Platform, - refer to :external+ocp:doc:`steps-overview`. #. Add the NVIDIA Helm repository: @@ -336,249 +288,43 @@ The notebook should now be accessible from your browser at this URL: `http://your-machine-ip:30001/?token=3660c9ee9b225458faaf853200bc512ff2206f635ab2b1d9 `_. -.. _gpu-operator-helm-chart-options: -.. _chart-customization-options: -.. _chart customization options: +*************************** +Common Deployment Scenarios +*************************** -********************************** -Common Chart Customization Options -********************************** +The following common deployment scenarios and sample commands apply best to +bare metal hosts or virtual machines with GPU passthrough. -The following options are available when using the Helm chart. -These options can be used with ``--set`` when installing with Helm. +Installing with DRA Driver for NVIDIA GPUs +========================================== -The following table identifies the most frequently used options. -To view all the options, run ``helm show values nvidia/gpu-operator``. +.. note:: -.. list-table:: - :widths: 20 50 30 - :header-rows: 1 + Deploying and managing the DRA Driver for NVIDIA GPUs through the ``GPUCluster`` custom resource is in Technology + Preview and is supported only for greenfield (new) deployments. + Configuration options may change in future releases. + Migrating an existing ``ClusterPolicy`` deployment to ``GPUCluster`` in place is not supported. + Do not use ``ClusterPolicy`` and ``GPUCluster`` as GPU resource management models in the same cluster. - * - Parameter - - Description - - Default - - * - ``ccManager.enabled`` - - When set to ``true``, the Operator deploys NVIDIA Confidential Computing Manager for Kubernetes. - - ``false`` - - * - ``cdi.enabled`` - - When set to ``true`` (default), the Container Device Interface (CDI) will be used for - injecting GPUs into workload containers. - The Operator will no longer configure the ``nvidia`` runtime class as the default runtime handler. - Instead, native-CDI support in container runtimes like containerd or cri-o will be leveraged for injecting GPUs into workload containers. - Refer to the :doc:`cdi` page for more information. - - ``true`` - - * - ``cdi.nriPluginEnabled`` - - When set to ``true``, the Node Resource Interface (NRI) Plugin will be used for injecting GPUs into workload containers. - - In NRI Plugin mode, the NVIDIA Container Toolkit will no longer modify the runtime config. - This feature requires containerd v1.7.30, v2.1.x, or v2.2.x, or cri-o v1.34 or later. - Refer to the :doc:`cdi` page for more information. - - ``false`` - - * - ``cdi.default`` (Deprecated) - - This field is deprecated as of v25.10.0 and will be ignored. - The ``cdi.enabled`` field is set to ``true`` by default in versions 25.10.0 and later. - When set to ``true``, the container runtime uses CDI to perform device injection by default. - - ``false`` - - * - ``daemonsets.annotations`` - - Map of custom annotations to add to all GPU Operator managed pods. - - ``{}`` - - * - ``daemonsets.labels`` - - Map of custom labels to add to all GPU Operator managed pods. - - ``{}`` - - * - ``dcgmExporter.enabled`` - - By default, the Operator gathers GPU telemetry in Kubernetes using `DCGM Exporter `_. - Set this value to ``false`` to disable it. - Available values are ``true`` (default) or ``false``. - - ``true`` - - * - ``dcgmExporter.service.internalTrafficPolicy`` - - Specifies the `internalTrafficPolicy `_ for the DCGM Exporter service. - Available values are ``Cluster`` (default) or ``Local``. - - ``Cluster`` - - * - ``dcgmExporter.hostNetwork`` - - When set to ``true``, the DCGM Exporter exposes a metric port on the host's network namespace. - - ``false`` - - * - ``dcgmExporter.annotations`` - - Map of custom annotations to add to the DCGM Exporter DaemonSet. - - ``{}`` - - * - ``dcgmExporter.enablePodLabels`` - - When set to ``true``, Kubernetes pod labels are added as Prometheus label dimensions on the GPU metrics. - Enabling this option causes the Operator to provision a cluster-scoped ClusterRole and ClusterRoleBinding - (``nvidia-dcgm-exporter-read-pods``) that grants the DCGM Exporter service account ``get``, ``list``, and ``watch`` access to pods. - Use ``dcgmExporter.podLabelAllowlistRegex`` to limit which labels are emitted. - - ``false`` - - * - ``dcgmExporter.enablePodUID`` - - When set to ``true``, the Kubernetes pod UID is added as a Prometheus label dimension on the GPU metrics. - Like ``dcgmExporter.enablePodLabels``, this provisions a cluster-scoped ClusterRole and ClusterRoleBinding that grants the DCGM Exporter - service account ``get``, ``list``, and ``watch`` access to pods. - - ``false`` - - * - ``dcgmExporter.podLabelAllowlistRegex`` - - List of regular expressions that filter which pod labels are emitted as Prometheus dimensions when ``dcgmExporter.enablePodLabels`` is ``true``. - NVIDIA recommends configuring this allowlist in clusters with many pod labels to reduce Prometheus cardinality. - - ``none`` - - * - ``devicePlugin.config`` - - Specifies the configuration for the NVIDIA Device Plugin as a config map. - - In most cases, this field is configured after installing the Operator, such as - to configure :doc:`gpu-sharing`. - - ``{}`` - - * - ``driver.enabled`` - - By default, the Operator deploys NVIDIA drivers as a container on the system. - Set this value to ``false`` when using the Operator on systems with pre-installed drivers. - - ``true`` - - * - ``driver.image`` - - Name of the NVIDIA Driver Container image to use. - - ``driver`` - - * - ``driver.imagePullSecrets`` - - List of the image pull secret used for pulling the driver container image from the registry. - - None - - * - ``driver.kernelModuleType`` - - Specifies the type of the NVIDIA GPU Kernel modules to use. - Valid values are ``auto`` (default), ``proprietary``, and ``open``. - - ``Auto`` means that the recommended kernel module type (open or proprietary) is chosen based on the GPU devices on the host and the driver branch used. - The ``auto`` option is only supported with the 570.86.15 and 570.124.06 or later driver containers. - 550 and 535 branch drivers do not yet support this mode. - ``Open`` means the open kernel module is used. - ``Proprietary`` means the proprietary module is used. - - ``auto`` - - * - ``driver.nvidiaDriverCRD.enabled`` - - When set to ``true``, the Operator deploys NVIDIA GPU Driver Custom Resource Definition. - Refer to the :doc:`NVIDIA GPU Driver Custom Resource Definition ` page for more information. - - ``false`` - - * - ``driver.repository`` - - The images are downloaded from NGC. Specify another image repository when using - custom driver images. - - ``nvcr.io/nvidia`` - - * - ``driver.rdma.enabled`` - - Controls whether the driver daemon set builds and loads the legacy ``nvidia-peermem`` kernel module. - - You might be able to use GPUDirect RDMA without enabling this option. - Refer to :doc:`gpu-operator-rdma` for information about whether you can use DMA-BUF or - you need to use legacy ``nvidia-peermem``. - - ``false`` - - * - ``driver.rdma.useHostMofed`` - - Indicate if MLNX_OFED (MOFED) drivers are pre-installed on the host. - - ``false`` - - * - ``driver.secretEnv`` - - The name of the secret to the driver container. - A common use case is to use this field to pass your Ubuntu Pro token secret if you are deploying the GPU Operator with government-ready components. Refer to :doc:`install-gpu-operator-gov-ready` for more information. - - None - - * - ``driver.startupProbe`` - - By default, the driver container has an initial delay of ``60s`` before starting liveness probes. - The probe runs the ``nvidia-smi`` command with a timeout duration of ``60s``. - You can increase the ``timeoutSeconds`` duration if the ``nvidia-smi`` command - runs slowly in your cluster. - - ``60s`` - - * - ``driver.useOpenKernelModules`` (Deprecated) - - This field is deprecated as of v25.3.0 and will be ignored. Use ``kernelModuleType`` instead. - When set to ``true``, the driver containers install the NVIDIA Open GPU Kernel module driver. - - ``false`` - - * - ``driver.usePrecompiled`` - - When set to ``true``, the Operator attempts to deploy driver containers that have - precompiled kernel drivers. - Refer to the :doc:`precompiled driver containers ` page for the supported operating systems. - - ``false`` - - * - ``driver.version`` - - Version of the NVIDIA datacenter driver supported by the Operator. - - If you set ``driver.usePrecompiled`` to ``true``, then set this field to - a driver branch, such as ``525``. - - Depends on the version of the Operator. Refer to the :ref:`GPU Operator Component Matrix` - for more information on supported drivers. - - * - ``gdrcopy.enabled`` - - Enables support for GDRCopy. - When set to ``true``, the GDRCopy Driver runs as a sidecar container in the GPU driver pod. - For information about GDRCopy, refer to the `gdrcopy `__ page. - - You can enable GDRCopy if you use the :doc:`gpu-driver-configuration`. - - ``false`` - - - * - ``mig.strategy`` - - Controls the strategy to be used with MIG on supported NVIDIA GPUs. Options - are either ``mixed`` or ``single``. - - ``single`` - - * - ``migManager.enabled`` - - The MIG manager watches for changes to the MIG geometry and applies reconfiguration as needed. By - default, the MIG manager only runs on nodes with GPUs that support MIG (such as the A100). - - ``true`` - - * - ``nfd.enabled`` - - Deploys Node Feature Discovery plugin as a daemonset. - Set this variable to ``false`` if NFD is already running in the cluster. - - ``true`` - - * - ``nfd.nodefeaturerules`` - - Installs node feature rules that are related to confidential computing. - NFD uses the rules to detect security features in CPUs and NVIDIA GPUs. - Set this variable to ``true`` when you configure the Operator for Confidential Containers. - - ``false`` - - * - ``operator.labels`` - - Map of custom labels to add to all GPU Operator managed pods. - - ``{}`` - - * - ``psp.enabled`` - - The GPU Operator deploys ``PodSecurityPolicies`` if enabled. - - ``false`` - - * - ``sandboxWorkloads.enabled`` - - Specifies if sandbox containers are enabled. - - ``false`` - - * - ``sandboxWorkloads.defaultWorkload`` - - Specifies the default type of workload for the cluster, one of ``container``, ``vm-passthrough``, or ``vm-vgpu``. - - Setting ``vm-passthrough`` or ``vm-vgpu`` can be helpful if you plan to run all or mostly virtual machines in your cluster. - Refer to :doc:`KubeVirt `, :doc:`Kata Containers ` for more details on deploying different workload containers. - - ``container`` - - * - ``sandboxWorkloads.mode`` - - Specifies the sandbox mode to use when deploying sandbox workloads. - Accepted values are ``kubevirt`` (default) and ``kata``. - Refer to the :doc:`KubeVirt ` or the :doc:`Kata Containers ` pages for more information on using KubeVirt or Kata based workloads. - - ``kubevirt`` - * - ``toolkit.enabled`` - - By default, the Operator deploys the NVIDIA Container Toolkit (``nvidia-docker2`` stack) - as a container on the system. Set this value to ``false`` when using the Operator on systems - with pre-installed NVIDIA runtimes. - - ``true`` +If you want to use Kubernetes Dynamic Resource Allocation (DRA) to manage GPU resource allocation in your cluster, +install the GPU Operator with the ``GPUCluster`` custom resource enabled and ``DEFAULT_GPU_ALLOCATION_MODE`` set to +``dra``. +This deploys the GPU Operator with the components necessary for DRA, including the DRA Driver for NVIDIA GPUs. -*************************** -Common Deployment Scenarios -*************************** +.. code-block:: console -The following common deployment scenarios and sample commands apply best to -bare metal hosts or virtual machines with GPU passthrough. + $ helm upgrade --install gpu-operator nvidia/gpu-operator \ + --version=${version} \ + --create-namespace \ + --namespace gpu-operator \ + --set gpuCluster.enabled=true \ + --set driver.nvidiaDriverCRD.enabled=true \ + --set operator.env[0].name=DEFAULT_GPU_ALLOCATION_MODE \ + --set operator.env[0].value=dra + +The ``gpuCluster.enabled=true`` flag creates the default ``GPUCluster`` resource. +The ``driver.nvidiaDriverCRD.enabled=true`` flag creates the ``NVIDIADriver`` custom resource to manage the NVIDIA GPU driver. If you are planning to use pre-installed drivers, set this flag to ``false`` and include the ``driver.enabled=false`` flag. +Setting the ``DEFAULT_GPU_ALLOCATION_MODE`` environment variable to ``dra`` ensures that GPU nodes are labeled for DRA components. Specifying the Operator Namespace ================================= diff --git a/gpu-operator/gpu-operator-dra.rst b/gpu-operator/gpu-operator-dra.rst new file mode 100644 index 000000000..5b798bf05 --- /dev/null +++ b/gpu-operator/gpu-operator-dra.rst @@ -0,0 +1,471 @@ +.. license-header + SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +.. headings (h1/h2/h3/h4/h5) are # * = - + +.. _gpu-operator-dra: + +########################################### +Deploying the GPU Operator with DRA Support +########################################### + +Dynamic Resource Allocation (DRA) is a Kubernetes API for flexibly requesting, configuring, and sharing specialized +devices such as GPUs. +Starting with GPU Operator ${version}, the GPU Operator can deploy and manage the `DRA Driver for NVIDIA GPUs +`__ (v${dra_version}) directly, as a native operand, through the +``GPUCluster`` custom resource. + +This page describes the architecture of the DRA-based software stack, how to install the GPU Operator with DRA +support enabled, and the current limitations of the integration. + +.. note:: + + Deploying and managing the DRA Driver for NVIDIA GPUs through the ``GPUCluster`` custom resource is in Technology + Preview. + The ``GPUCluster`` API is served under ``nvidia.com/v1alpha1`` and is subject to change in future releases. + Only greenfield (new) deployments are supported. + Migrating an existing ``ClusterPolicy`` deployment to ``GPUCluster`` in place is not supported. + + If you want to install the fully-supported DRA Driver for NVIDIA GPUs as a standalone Helm chart alongside the GPU + Operator instead of having the Operator manage it, refer to + :doc:`DRA Driver for NVIDIA GPUs `. + +Before you begin, it is recommended that you are familiar with the following: + +* `Upstream Kubernetes DRA documentation `_. +* `DRA Driver for NVIDIA GPUs documentation `__. + +******** +Overview +******** + +The GPU Operator supports two GPU resource management models. +The ``ClusterPolicy`` custom resource configures the device-plugin model and deploys the NVIDIA Kubernetes Device +Plugin along with the other operands needed to manage GPU resources. + +For a greenfield DRA deployment, the cluster-scoped ``GPUCluster`` custom resource configures the operands needed to +use the DRA Driver for NVIDIA GPUs to allocate GPUs. +Use one GPU resource management model in a cluster; deploying both ``ClusterPolicy`` and ``GPUCluster`` as active +models is not supported. + +With the DRA Driver for NVIDIA GPUs, your Kubernetes workloads can allocate and consume two types of resources: + +* GPU allocation for controlled sharing and dynamic reconfiguration of GPUs. + This replaces the traditional GPU allocation method that is provided by the NVIDIA Kubernetes Device Plugin. +* ComputeDomains, an abstraction for secure `Multi-Node NVLink (MNNVL) + `_ on NVIDIA GB200 and similar systems. + +You can use these features independently or together in the same cluster. + +Unlike ``ClusterPolicy``, the ``GPUCluster`` resource does not manage the NVIDIA GPU driver or a device plugin. +The GPU driver must be either pre-installed on the host or managed by an ``NVIDIADriver`` custom resource, and GPUs are +surfaced to workloads through DRA. + +The following table summarizes the differences between the two stacks. + +.. list-table:: + :header-rows: 1 + :widths: 34 33 33 + + * - Component + - Device-plugin model (``ClusterPolicy``) + - DRA model (``GPUCluster``) + * - GPU allocation mechanism + - NVIDIA Kubernetes Device Plugin (extended resources) + - DRA Driver for NVIDIA GPUs (ResourceClaims) + * - NVIDIA GPU driver + - Managed by ``ClusterPolicy`` or ``NVIDIADriver`` + - Pre-installed or managed by ``NVIDIADriver`` (not managed by ``GPUCluster``) + * - NVIDIA Container Toolkit + - Deployed + - Not deployed (workloads use CDI through DRA) + * - GPU Feature Discovery (GFD) + - Deployed + - Not deployed by ``GPUCluster`` + * - DCGM and DCGM Exporter + - Deployed + - Deployed + * - MIG Manager + - Deployed + - Not deployed + * - Validation + - ``nvidia-operator-validator`` + - ``nvidia-dra-validator`` (validates through a DRA ResourceClaim) + +************ +Architecture +************ + +The DRA model +============= + +The DRA model is configured through the ``GPUCluster`` custom resource. +When you install the GPU Operator with DRA support enabled, the Helm chart creates a default ``GPUCluster`` resource +named ``gpu-cluster``, which the GPU Operator watches and reconciles into the following operands: + +* **DRA Driver for NVIDIA GPUs** — a kubelet-plugin DaemonSet and, when ComputeDomains are enabled, a controller + Deployment. +* **DCGM Exporter** — enabled by default, for GPU telemetry. +* **DCGM** — the standalone NVIDIA DCGM hostengine, disabled by default (DCGM Exporter uses its embedded + ``nv-hostengine`` when standalone DCGM is disabled). +* **DRA validator** — validates that the DRA driver can allocate a GPU on each node. + +``GPUCluster`` is a singleton, cluster-scoped resource in the ``nvidia.com/v1alpha1`` API group. +For the full list of ``GPUCluster`` fields, its singleton behavior, and status values, refer to the +:doc:`GPUCluster Custom Resource Reference `. + +How the DRA driver works +======================== + +The DRA Driver for NVIDIA GPUs runs as a kubelet-plugin DaemonSet on each GPU node. +The kubelet-plugin pod is composed of the following: + +* An ``dra-driver-validator`` init container that confirms an NVIDIA GPU driver is installed and ready before the + driver starts. + The validator probes for a driver that is pre-installed on the host and, if none is found, for a + driver installed by the GPU Operator, and it writes the resulting driver root paths to a file that the + kubelet-plugin containers read on startup. +* A ``gpus`` container (always deployed) that advertises GPU devices for allocation. +* A ``compute-domains`` container (deployed when ComputeDomains are enabled) that advertises ComputeDomain devices. + +When the driver is running, it publishes the GPUs on each node as ``ResourceSlice`` objects and registers a set of +``DeviceClass`` objects that workloads reference in their claims: + +.. list-table:: + :header-rows: 1 + :widths: 45 55 + + * - DeviceClass + - Purpose + * - ``gpu.nvidia.com`` + - Full GPU allocation. + * - ``mig.nvidia.com`` + - Multi-Instance GPU (MIG) device allocation. + * - ``vfio.gpu.nvidia.com`` + - GPUs bound to ``vfio-pci`` for passthrough (for example, to virtual machines). + * - ``compute-domain-daemon.nvidia.com`` + - ComputeDomain daemon devices (created when ComputeDomains are enabled). + * - ``compute-domain-default-channel.nvidia.com`` + - ComputeDomain channel devices (created when ComputeDomains are enabled). + +Claiming GPUs for workloads +=========================== + +With DRA, a workload requests a GPU by referencing a ``ResourceClaim`` or ``ResourceClaimTemplate`` that targets one +of the NVIDIA ``DeviceClass`` objects, rather than requesting an extended resource such as ``nvidia.com/gpu``. + +The following example allocates a single full GPU to a pod: + +.. code-block:: yaml + + apiVersion: resource.k8s.io/v1 + kind: ResourceClaimTemplate + metadata: + name: single-gpu + spec: + spec: + devices: + requests: + - name: gpu + exactly: + deviceClassName: gpu.nvidia.com + allocationMode: ExactCount + count: 1 + --- + apiVersion: v1 + kind: Pod + metadata: + name: gpu-workload + spec: + restartPolicy: Never + resourceClaims: + - name: gpu + resourceClaimTemplateName: single-gpu + containers: + - name: workload + image: nvcr.io/nvidia/cuda:12.6.2-base-ubi9 + command: ["nvidia-smi", "-L"] + resources: + claims: + - name: gpu + +For GPU resource management, DRA enables scenarios that are difficult to express with extended resources, including: + +* Requesting GPUs with specific attributes (for example, a model, memory capacity, or a MIG profile) through CEL + selectors in the claim. +* Controlled sharing of a single GPU across multiple containers or pods. +* Allocating MIG devices dynamically through the ``mig.nvidia.com`` DeviceClass. + +.. note:: + + To keep existing workloads that request GPUs through the extended resource API (for example, ``nvidia.com/gpu: 1``) + working while the DRA driver is used, enable the `DRAExtendedResource + `_ feature + gate. This feature gate is enabled by default in Kubernetes v1.36 and later. + +For detailed workload examples, including MIG and ComputeDomain workloads, refer to the `DRA Driver for NVIDIA GPUs +documentation `__. + +Management access to GPUs +========================= + +Operands such as DCGM Exporter and the DRA validator need administrative access to GPUs. +Instead of relying on the ``nvidia`` runtime class, these operands request access through a DRA ``ResourceClaim`` that +sets ``adminAccess: true``. +Administrative access requires the GPU Operator namespace to carry the ``resource.kubernetes.io/admin-access: "true"`` +label; the GPU Operator applies this label automatically during reconciliation. + +.. note:: + + The ``adminAccess`` field is gated by the ``DRAAdminAccess`` feature gate, which is beta and enabled by default + starting in Kubernetes v1.34. + +**************************** +Limitations and Restrictions +**************************** + +Before you enable DRA support, review the following limitations. + +* **Technology preview.** The DRA Driver for NVIDIA GPUs itself is fully supported; deploying and managing it through + the ``GPUCluster`` resource is a technology preview feature. The ``nvidia.com/v1alpha1`` API is subject to change. +* **New installations only.** Only greenfield (new) installations are supported. There is no supported procedure for + migrating an existing device-plugin (``ClusterPolicy``) installation to the DRA model in place. +* **Kubernetes version.** DRA must be served by the cluster. Kubernetes v1.34.2 or later is required. If the + ``resource.k8s.io`` DeviceClass API is not served, the Helm install fails with a validation error. +* **Driver management.** ``GPUCluster`` does not manage the GPU driver or a device plugin. The driver must be + pre-installed on the host or managed by an ``NVIDIADriver`` custom resource. +* **Driver auto-upgrade.** Automatic driver upgrades are not yet supported for DRA nodes. When drivers are managed by + an ``NVIDIADriver`` resource that serves DRA nodes, disable automatic upgrades by setting + ``spec.driver.upgradePolicy.autoUpgrade: false``. +* **GPU Feature Discovery.** GFD is not deployed by ``GPUCluster``. +* **ComputeDomains hardware.** ComputeDomains require NVIDIA Grace Blackwell systems with Multi-Node NVLink, such as + NVIDIA HGX GB200 NVL72 or NVIDIA HGX GB300 NVL72. + + +.. _dra-deployment-scenarios: + +**************** +Deployment Model +**************** + +The ``GPUCluster`` resource supports a dedicated, greenfield DRA deployment. +Every GPU node is served by the DRA model. +Do not use ``ClusterPolicy`` and ``GPUCluster`` as GPU resource management models in the same cluster. +In-place migration from an existing device-plugin deployment is not supported. + +Install with ``gpuCluster.enabled=true`` and set ``DEFAULT_GPU_ALLOCATION_MODE`` to ``dra``. +Setting ``DEFAULT_GPU_ALLOCATION_MODE`` to ``dra`` ensures that every GPU node is labeled for, and served by, the DRA +model. + +Refer to :ref:`Install ` for the commands. + +Operator-managed or pre-installed driver +========================================= + +The ``GPUCluster`` resource does not manage the NVIDIA GPU driver, so you choose how the driver is provided on your +GPU nodes: + +* **Operator-managed driver** — the GPU Operator installs and manages the driver through the ``NVIDIADriver`` custom + resource (``driver.nvidiaDriverCRD.enabled=true``). +* **Pre-installed driver** — the NVIDIA GPU driver is already installed on each GPU node + (``driver.enabled=false``). + +Both options are shown as tabs in :ref:`Install `. + +.. _dra-install: + +******* +Install +******* + +This section covers a fresh install of the GPU Operator with DRA support enabled. + +#. Add the NVIDIA Helm repository: + + .. code-block:: console + + $ helm repo add nvidia https://helm.ngc.nvidia.com/nvidia \ + && helm repo update + +#. Install the GPU Operator with the ``GPUCluster`` resource enabled and ``DEFAULT_GPU_ALLOCATION_MODE`` set to ``dra`` + so that GPU nodes are served by the DRA model. + + Select the tab that matches how the GPU driver is managed on your cluster. + + .. tab-set:: + + .. tab-item:: Operator-managed driver + + Use this option to let the GPU Operator install and manage the driver through the ``NVIDIADriver`` custom + resource. + + .. code-block:: console + + $ helm upgrade --install gpu-operator nvidia/gpu-operator \ + --version=${version} \ + --create-namespace \ + --namespace gpu-operator \ + --set gpuCluster.enabled=true \ + --set driver.nvidiaDriverCRD.enabled=true \ + --set operator.env[0].name=DEFAULT_GPU_ALLOCATION_MODE \ + --set operator.env[0].value=dra + + .. tab-item:: Pre-installed driver + + Use this option when the NVIDIA GPU driver is already installed on each GPU node. + + .. code-block:: console + + $ helm upgrade --install gpu-operator nvidia/gpu-operator \ + --version=${version} \ + --create-namespace \ + --namespace gpu-operator \ + --set gpuCluster.enabled=true \ + --set driver.enabled=false \ + --set operator.env[0].name=DEFAULT_GPU_ALLOCATION_MODE \ + --set operator.env[0].value=dra + + The ``gpuCluster.enabled=true`` flag creates the default ``GPUCluster`` resource and enables the DRA model. + Setting ``DEFAULT_GPU_ALLOCATION_MODE`` to ``dra`` ensures that GPU nodes are labeled for, and served by, the DRA + model. + + To customize the DRA driver or the other operands, refer to + :ref:`Helm Configuration Reference `. + +********************* +Validate Installation +********************* + +#. Confirm that the ``GPUCluster`` resource reports a ``ready`` state: + + .. code-block:: console + + $ kubectl get gpucluster + + *Example Output* + + .. code-block:: output + + NAME STATUS AGE + gpu-cluster ready 3m12s + +#. Confirm that the DRA model operands are running: + + .. code-block:: console + + $ kubectl get pods -n gpu-operator + + *Example Output* + + .. code-block:: output + + NAME READY STATUS RESTARTS AGE + gpu-operator-... 1/1 Running 0 4m + nvidia-dra-driver-controller-... 1/1 Running 0 3m + nvidia-dra-driver-kubelet-plugin-... 2/2 Running 0 3m + nvidia-dra-validator-... 1/1 Running 0 2m + nvidia-dcgm-exporter-... 1/1 Running 0 2m + + The kubelet-plugin pod runs two containers when both GPU allocation and ComputeDomains are enabled — one for GPU + resources (``gpus``) and one for ComputeDomain resources (``compute-domains``) — so it shows ``2/2``. + If ComputeDomains are disabled, the kubelet-plugin pod shows ``1/1`` and the ``nvidia-dra-driver-controller`` + Deployment is not present. + The ``nvidia-dra-validator`` pod becomes ready only after the DRA driver successfully allocates a GPU on the node. + +#. Verify that the NVIDIA DeviceClasses are available: + + .. code-block:: console + + $ kubectl get deviceclass + + *Example Output* + + .. code-block:: output + + NAME AGE + compute-domain-daemon.nvidia.com 3m + compute-domain-default-channel.nvidia.com 3m + gpu.nvidia.com 3m + mig.nvidia.com 3m + vfio.gpu.nvidia.com 3m + +#. Verify that the driver publishes GPUs as ``ResourceSlice`` objects: + + .. code-block:: console + + $ kubectl get resourceslices + + Each GPU node should have one or more ``ResourceSlice`` objects that describe its GPUs. + +.. _dra-helm-reference: + +**************************** +Helm Configuration Reference +**************************** + +The following Helm values enable and configure the DRA integration. +The ``gpuCluster.*`` values populate the default ``GPUCluster`` resource that the chart creates; each maps to a field +documented in the :doc:`GPUCluster Custom Resource Reference `. + +.. list-table:: + :header-rows: 1 + :widths: 40 15 45 + + * - Helm value + - Default + - Description + * - ``gpuCluster.enabled`` + - ``false`` + - Enables DRA support by creating the default ``GPUCluster`` resource. + * - ``operator.env[*]`` (``DEFAULT_GPU_ALLOCATION_MODE``) + - unset + - Set to ``dra`` for a ``GPUCluster`` deployment so newly-added GPU nodes use the DRA model. + * - ``gpuCluster.draDriver.*`` + - See reference + - Configures the DRA driver image and its GPU allocation (``gpus``) and ComputeDomains capabilities. Populates + ``spec.draDriver`` of the ``GPUCluster`` resource. + +The DRA model also reuses the following top-level Helm values, which are shared with the device-plugin model and are +applied to the operands that ``GPUCluster`` deploys. These populate the corresponding sections of the ``GPUCluster`` +resource. + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Helm value + - Description + * - ``dcgmExporter.*`` + - Configuration for DCGM Exporter (enabled by default). + * - ``dcgm.*`` + - Configuration for the standalone DCGM hostengine (disabled by default). + * - ``hostPaths.*`` + - Host paths for the root filesystem, driver install directory, and kubelet root directory. + * - ``daemonsets.*`` + - Common labels, annotations, priority class, tolerations, and update strategy for the deployed workloads. + +For the full set of configurable ``GPUCluster`` fields, refer to the +:doc:`GPUCluster Custom Resource Reference `. + +************************ +Additional Documentation +************************ + +For more information about the DRA Driver for NVIDIA GPUs, refer to the following resources: + +* :doc:`DRA Driver for NVIDIA GPUs ` — installing the DRA driver as a standalone Helm chart. +* `DRA Driver for NVIDIA GPUs documentation `__ +* `Upstream Kubernetes DRA documentation `_ diff --git a/gpu-operator/gpucluster-reference.rst b/gpu-operator/gpucluster-reference.rst new file mode 100644 index 000000000..4f4aacb0f --- /dev/null +++ b/gpu-operator/gpucluster-reference.rst @@ -0,0 +1,321 @@ +.. license-header + SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +.. headings (h1/h2/h3/h4/h5) are # * = - ^ + +.. _gpucluster-reference: + +####################################### +GPUCluster Custom Resource Reference +####################################### + +The ``GPUCluster`` custom resource definition (CRD) configures the Dynamic Resource Allocation (DRA) GPU resource +management model that the NVIDIA GPU Operator manages. +A single ``GPUCluster`` instance in the cluster describes the desired state of the DRA operands—the +`DRA Driver for NVIDIA GPUs `__, DCGM, and DCGM Exporter—including +which container images to use and how each component is configured. + +The GPU Operator watches the ``GPUCluster`` object and reconciles the managed DaemonSets and Deployments to match the +desired state. + +Unlike :doc:`ClusterPolicy `, ``GPUCluster`` does not manage the NVIDIA GPU driver or the +NVIDIA Kubernetes Device Plugin. +The GPU driver must be pre-installed on the host or managed by an ``NVIDIADriver`` custom resource, and GPUs are +surfaced to workloads through DRA rather than through extended resources. +For a conceptual overview of the DRA stack, installation steps, and deployment scenarios, refer to +:doc:`Deploying the GPU Operator with DRA Support `. + +.. note:: + + Deploying and managing the DRA Driver for NVIDIA GPUs through the ``GPUCluster`` custom resource is in Technology + Preview. The ``GPUCluster`` API is served under ``nvidia.com/v1alpha1`` and is subject to change in future releases. + ``GPUCluster`` is supported only for greenfield (new) deployments. + Migrating an existing ``ClusterPolicy`` deployment to ``GPUCluster`` in place is not supported. + Do not use ``ClusterPolicy`` and ``GPUCluster`` as GPU resource management models in the same cluster. + +******************************* +Singleton Behavior +******************************* + +``GPUCluster`` is a *singleton* resource: the GPU Operator reconciles only one instance in the cluster. +The GPU Operator Helm chart creates a default instance named ``gpu-cluster`` when DRA support is enabled +(``gpuCluster.enabled=true``). + +The first ``GPUCluster`` object that the controller observes becomes the active instance ("first-wins"). +Any additional ``GPUCluster`` objects are not reconciled and are marked with the ``ignored`` state in their status. +To change the configuration, edit the active instance rather than creating a second one. + +******************************* +GPUCluster Resource Structure +******************************* + +``GPUCluster`` is a cluster-scoped resource in the ``nvidia.com/v1alpha1`` API group. +Its short name is ``gc``. + +.. code-block:: yaml + + apiVersion: nvidia.com/v1alpha1 + kind: GPUCluster + metadata: + name: gpu-cluster + spec: + draDriver: {} + dcgm: {} + dcgmExporter: {} + hostPaths: {} + daemonsets: {} + +**Top-Level Spec Fields** + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Required + + * - ``draDriver`` + - Configuration for the DRA Driver for NVIDIA GPUs, including the GPU allocation (``gpus``) and ComputeDomains + capabilities. + - Yes + + * - ``dcgm`` + - Configuration for the standalone NVIDIA DCGM hostengine. Disabled by default. When disabled, DCGM Exporter uses + its embedded ``nv-hostengine``. + - No + + * - ``dcgmExporter`` + - Configuration for the NVIDIA DCGM Exporter (GPU telemetry). Enabled by default. + - No + + * - ``hostPaths`` + - Custom host filesystem paths used by the deployed components. + - No + + * - ``daemonsets`` + - Common settings applied to all DaemonSets managed by the ``GPUCluster`` controller. + - No + +******************************* +spec.draDriver +******************************* + +Configures the DRA Driver for NVIDIA GPUs. +There is no top-level ``enabled`` toggle: the GPU allocation (``gpus``) capability is always deployed, and the +ComputeDomains capability has its own ``enabled`` field. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``repository`` + - Container registry and path for the DRA driver image. + - ``registry.k8s.io/dra-driver-nvidia`` + + * - ``image`` + - DRA driver image name (without tag). + - ``dra-driver-nvidia-gpu`` + + * - ``version`` + - DRA driver image tag or version string. + - Defined by the Operator release. Refer to the :ref:`operator-component-matrix`. + + * - ``imagePullPolicy`` + - Kubernetes `image pull policy `_. + Accepted values: ``Always``, ``IfNotPresent``, ``Never``. + - ``IfNotPresent`` + + * - ``imagePullSecrets`` + - List of Kubernetes Secret names that contain credentials for pulling the DRA driver image from a private + registry. + - None + + * - ``featureGates`` + - Map of DRA driver feature gate names to booleans. Rendered as the ``FEATURE_GATES`` environment variable on the + DRA driver containers, for example ``{MPSSupport: true}``. + - None + + * - ``gpus`` + - Configuration for the ``gpu.nvidia.com`` (GPU allocation) capability. See :ref:`gpucluster-dradriver-gpus`. + - N/A + + * - ``computeDomains`` + - Configuration for the ComputeDomains (Multi-Node NVLink) capability. See + :ref:`gpucluster-dradriver-computedomains`. + - N/A + +.. _gpucluster-dradriver-gpus: + +spec.draDriver.gpus +=================== + +Configures the ``gpu.nvidia.com`` capability, which maps onto the ``gpus`` container of the kubelet-plugin DaemonSet. +This capability is always deployed. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``kubeletPlugin`` + - Configuration for the ``gpus`` kubelet-plugin container. See :ref:`gpucluster-dradriver-kubeletplugin`. + - N/A + +.. _gpucluster-dradriver-computedomains: + +spec.draDriver.computeDomains +============================= + +Configures the ComputeDomains capability, an abstraction for secure `Multi-Node NVLink (MNNVL) +`_ on NVIDIA GB200 and similar systems. +The ``kubeletPlugin`` maps onto the ``compute-domains`` container of the kubelet-plugin DaemonSet, and the +``controller`` is a separate Deployment. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``enabled`` + - Whether the ComputeDomains capability is deployed. When enabled, the DRA driver deploys the + ``compute-domains`` kubelet-plugin container and the compute-domain controller Deployment. + - ``true`` + + * - ``controller`` + - Overrides for the compute-domain controller Deployment. Accepts ``env`` and ``resources``. + - N/A + + * - ``kubeletPlugin`` + - Configuration for the ``compute-domains`` kubelet-plugin container. See + :ref:`gpucluster-dradriver-kubeletplugin`. + - N/A + +.. _gpucluster-dradriver-kubeletplugin: + +kubelet-plugin fields +===================== + +The ``gpus.kubeletPlugin`` and ``computeDomains.kubeletPlugin`` blocks map onto the two containers of a single +kubelet-plugin DaemonSet and accept the following fields. +Scheduling is opinionated and is not configurable. + +.. list-table:: + :header-rows: 1 + :widths: 25 55 20 + + * - Field + - Description + - Default + + * - ``env`` + - List of environment variables (``name``/``value`` pairs) to set in the container. + - None + + * - ``resources`` + - CPU and memory resource requests and limits for the container. Follows the standard Kubernetes + ``ResourceRequirements`` structure. + - None + + * - ``healthcheckPort`` + - Port for a gRPC health service checked by a liveness probe. Set to a negative value to disable the service and + the probe. + - Component default + +The ``computeDomains.controller`` Deployment accepts ``env`` and ``resources`` with the same meanings. + +******************************* +spec.dcgm +******************************* + +Configures the standalone NVIDIA DCGM hostengine. +This component is **disabled by default**; when it is disabled, DCGM Exporter uses its embedded ``nv-hostengine``. + +The ``dcgm`` section uses the same fields as the ``dcgm`` section of ``ClusterPolicy``. +Refer to ``spec.dcgm`` in the :doc:`ClusterPolicy Custom Resource Reference ` for the full +list of fields. + +******************************* +spec.dcgmExporter +******************************* + +Configures the NVIDIA DCGM Exporter for GPU telemetry. +This component is **enabled by default**. + +The ``dcgmExporter`` section uses the same fields as the ``dcgmExporter`` section of ``ClusterPolicy``. +Refer to ``spec.dcgmExporter`` in the :doc:`ClusterPolicy Custom Resource Reference ` for the +full list of fields. + +******************************* +spec.hostPaths +******************************* + +Configures custom host filesystem paths used by the deployed components, such as the root filesystem, the driver +install directory, and the kubelet root directory. + +The ``hostPaths`` section uses the same fields as the ``hostPaths`` section of ``ClusterPolicy``. +Refer to ``spec.hostPaths`` in the :doc:`ClusterPolicy Custom Resource Reference ` for the +full list of fields. + +******************************* +spec.daemonsets +******************************* + +Applies common settings—such as labels, annotations, priority class, tolerations, and update strategy—to all +DaemonSets that the ``GPUCluster`` controller manages. + +The ``daemonsets`` section uses the same fields as the ``daemonsets`` section of ``ClusterPolicy``. +Refer to ``spec.daemonsets`` in the :doc:`ClusterPolicy Custom Resource Reference ` for the +full list of fields. + +******************************* +Status +******************************* + +The GPU Operator reports the observed state of the active ``GPUCluster`` instance in its ``status`` field. +The ``status.state`` value is shown in the ``Status`` column of ``kubectl get gpucluster``. + +.. list-table:: + :header-rows: 1 + :widths: 20 80 + + * - State + - Description + + * - ``ready`` + - All enabled operands are deployed and healthy. + + * - ``notReady`` + - One or more operands are not yet deployed or are not healthy. + + * - ``disabled`` + - The ``GPUCluster`` instance is present but its operands are disabled. + + * - ``ignored`` + - A duplicate ``GPUCluster`` instance that the singleton controller does not reconcile. + +The ``status`` field also reports the ``namespace`` in which the operator and operands are installed and a list of +``conditions`` that represent the current state of the resource. diff --git a/gpu-operator/helm-values-reference.rst b/gpu-operator/helm-values-reference.rst new file mode 100644 index 000000000..f4a60b8e1 --- /dev/null +++ b/gpu-operator/helm-values-reference.rst @@ -0,0 +1,398 @@ +.. license-header + SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +.. headings (h1/h2/h3/h4/h5) are # * = - ^ + +.. _helm-values-reference: + +########################### +Helm Chart Values Reference +########################### + +The NVIDIA GPU Operator is installed and configured with a Helm chart. +This page is a reference for the most frequently used chart values, grouped by component. + +You can set values on the command line with ``--set =`` when you install or upgrade the chart, or provide +a values file with ``-f values.yaml``. +To view the complete set of values for a specific chart version, run: + +.. code-block:: console + + $ helm show values nvidia/gpu-operator --version=${version} + +.. note:: + + Many chart values populate fields of the ``ClusterPolicy`` or ``GPUCluster`` custom resources that the chart + creates. For the complete custom resource schemas, refer to the + :doc:`ClusterPolicy Custom Resource Reference ` and the + :doc:`GPUCluster Custom Resource Reference `. + +******************************* +Operator and Global Settings +******************************* + +.. list-table:: + :widths: 25 50 25 + :header-rows: 1 + + * - Parameter + - Description + - Default + + * - ``operator.labels`` + - Map of custom labels to add to all GPU Operator managed pods. + - ``{}`` + + * - ``operator.env`` + - List of environment variables (``name``/``value`` pairs) set on the GPU Operator. + For a ``GPUCluster`` deployment, set ``DEFAULT_GPU_ALLOCATION_MODE`` to ``dra`` so newly-added GPU nodes use the + DRA model. Refer to :ref:`Deployment Model `. + - ``[]`` + + * - ``daemonsets.labels`` + - Map of custom labels to add to all GPU Operator managed DaemonSet pods. + - ``{}`` + + * - ``daemonsets.annotations`` + - Map of custom annotations to add to all GPU Operator managed DaemonSet pods. + - ``{}`` + + * - ``psa.enabled`` + - When set to ``true``, the Operator configures Pod Security Admission labels on its namespace. + - ``false`` + + * - ``psp.enabled`` (Deprecated) + - Deploys ``PodSecurityPolicies`` when enabled. PodSecurityPolicy is no longer supported in current Kubernetes + releases. + - ``false`` + +***************** +NVIDIA GPU Driver +***************** + +These values configure the ``driver`` section of ``ClusterPolicy``. +For per-node driver configuration through the ``NVIDIADriver`` custom resource, refer to +:doc:`GPU Driver CRD `. + +.. list-table:: + :widths: 25 50 25 + :header-rows: 1 + + * - Parameter + - Description + - Default + + * - ``driver.enabled`` + - By default, the Operator deploys NVIDIA drivers as a container on the system. + Set this value to ``false`` when using the Operator on systems with pre-installed drivers. + - ``true`` + + * - ``driver.nvidiaDriverCRD.enabled`` + - When set to ``true``, the Operator deploys and uses the NVIDIA GPU Driver custom resource definition. + Refer to the :doc:`NVIDIA GPU Driver Custom Resource Definition ` page for more + information. + - ``false`` + + * - ``driver.repository`` + - Image repository for the driver container. Specify another image repository when using custom driver images. + - ``nvcr.io/nvidia`` + + * - ``driver.image`` + - Name of the NVIDIA driver container image to use. + - ``driver`` + + * - ``driver.version`` + - Version of the NVIDIA datacenter driver supported by the Operator. + If you set ``driver.usePrecompiled`` to ``true``, set this field to a driver branch, such as ``535``. + - Depends on the Operator version. Refer to the :ref:`operator-component-matrix`. + + * - ``driver.imagePullSecrets`` + - List of image pull secrets used for pulling the driver container image from the registry. + - None + + * - ``driver.kernelModuleType`` + - Type of NVIDIA GPU kernel modules to use. Valid values are ``auto`` (default), ``proprietary``, and ``open``. + ``auto`` selects the recommended module type based on the GPU devices and driver branch. + - ``auto`` + + * - ``driver.rdma.enabled`` + - Controls whether the driver DaemonSet builds and loads the legacy ``nvidia-peermem`` kernel module. + Refer to :doc:`gpu-operator-rdma` for guidance on whether you need it. + - ``false`` + + * - ``driver.rdma.useHostMofed`` + - Indicates that MLNX_OFED (MOFED) drivers are pre-installed on the host. + - ``false`` + + * - ``driver.secretEnv`` + - Name of a secret passed to the driver container. A common use is passing an Ubuntu Pro token secret for + government-ready components. Refer to :doc:`install-gpu-operator-gov-ready`. + - None + + * - ``driver.startupProbe`` + - Startup probe configuration for the driver container. By default the probe runs ``nvidia-smi`` with a ``60s`` + timeout after an initial delay. Increase ``timeoutSeconds`` if ``nvidia-smi`` runs slowly in your cluster. + - ``60s`` + + * - ``driver.usePrecompiled`` + - When set to ``true``, the Operator attempts to deploy driver containers with precompiled kernel drivers. + Refer to the :doc:`precompiled driver containers ` page for supported operating systems. + - ``false`` + + * - ``driver.useOpenKernelModules`` (Deprecated) + - Deprecated as of v25.3.0 and ignored. Use ``driver.kernelModuleType`` instead. + - ``false`` + +************************ +NVIDIA Container Toolkit +************************ + +.. list-table:: + :widths: 25 50 25 + :header-rows: 1 + + * - Parameter + - Description + - Default + + * - ``toolkit.enabled`` + - By default, the Operator deploys the NVIDIA Container Toolkit as a container on the system. + Set this value to ``false`` when using the Operator on systems with pre-installed NVIDIA runtimes. + - ``true`` + +************* +Device Plugin +************* + +.. list-table:: + :widths: 25 50 25 + :header-rows: 1 + + * - Parameter + - Description + - Default + + * - ``devicePlugin.config`` + - Specifies the configuration for the NVIDIA Device Plugin as a config map. In most cases this is configured + after installing the Operator, for example to configure :doc:`gpu-sharing`. + - ``{}`` + +*********** +CDI and NRI +*********** + +.. list-table:: + :widths: 25 50 25 + :header-rows: 1 + + * - Parameter + - Description + - Default + + * - ``cdi.enabled`` + - When set to ``true`` (default), the Container Device Interface (CDI) is used for injecting GPUs into workload + containers, and the Operator no longer configures the ``nvidia`` runtime class as the default runtime handler. + Refer to the :doc:`cdi` page for more information. + - ``true`` + + * - ``cdi.nriPluginEnabled`` + - When set to ``true``, the Node Resource Interface (NRI) Plugin is used for injecting GPUs into workload + containers, and the NVIDIA Container Toolkit no longer modifies the runtime configuration. + Requires containerd v1.7.30, v2.1.x, or v2.2.x, or CRI-O v1.34 or later. Refer to the :doc:`cdi` page. + - ``false`` + + * - ``cdi.default`` (Deprecated) + - Deprecated as of v25.10.0 and ignored. ``cdi.enabled`` defaults to ``true`` in v25.10.0 and later. + - ``false`` + +********************** +DCGM and DCGM Exporter +********************** + +.. list-table:: + :widths: 25 50 25 + :header-rows: 1 + + * - Parameter + - Description + - Default + + * - ``dcgm.enabled`` + - When set to ``true``, the Operator deploys a standalone NVIDIA DCGM hostengine. When ``false``, DCGM Exporter + uses its embedded ``nv-hostengine``. + - ``false`` + + * - ``dcgmExporter.enabled`` + - By default, the Operator gathers GPU telemetry using + `DCGM Exporter `_. + Set this value to ``false`` to disable it. + - ``true`` + + * - ``dcgmExporter.service.internalTrafficPolicy`` + - Specifies the + `internalTrafficPolicy `_ + for the DCGM Exporter service. Available values are ``Cluster`` (default) or ``Local``. + - ``Cluster`` + + * - ``dcgmExporter.hostNetwork`` + - When set to ``true``, DCGM Exporter exposes a metric port on the host's network namespace. + - ``false`` + + * - ``dcgmExporter.annotations`` + - Map of custom annotations to add to the DCGM Exporter DaemonSet. + - ``{}`` + + * - ``dcgmExporter.enablePodLabels`` + - When set to ``true``, Kubernetes pod labels are added as Prometheus label dimensions on GPU metrics. + This provisions a cluster-scoped ClusterRole and ClusterRoleBinding (``nvidia-dcgm-exporter-read-pods``) that + grants the DCGM Exporter service account ``get``, ``list``, and ``watch`` access to pods. + Use ``dcgmExporter.podLabelAllowlistRegex`` to limit which labels are emitted. + - ``false`` + + * - ``dcgmExporter.enablePodUID`` + - When set to ``true``, the Kubernetes pod UID is added as a Prometheus label dimension on GPU metrics. + Like ``dcgmExporter.enablePodLabels``, this provisions a cluster-scoped ClusterRole and ClusterRoleBinding. + - ``false`` + + * - ``dcgmExporter.podLabelAllowlistRegex`` + - List of regular expressions that filter which pod labels are emitted as Prometheus dimensions when + ``dcgmExporter.enablePodLabels`` is ``true``. NVIDIA recommends configuring this allowlist in clusters with + many pod labels to reduce Prometheus cardinality. + - None + +************************ +MIG (Multi-Instance GPU) +************************ + +.. list-table:: + :widths: 25 50 25 + :header-rows: 1 + + * - Parameter + - Description + - Default + + * - ``mig.strategy`` + - Controls the strategy used with MIG on supported NVIDIA GPUs. Options are ``single`` or ``mixed``. + - ``single`` + + * - ``migManager.enabled`` + - The MIG Manager watches for changes to the MIG geometry and applies reconfiguration as needed. By default it + only runs on nodes with GPUs that support MIG. + - ``true`` + +********************** +Node Feature Discovery +********************** + +.. list-table:: + :widths: 25 50 25 + :header-rows: 1 + + * - Parameter + - Description + - Default + + * - ``nfd.enabled`` + - Deploys the Node Feature Discovery (NFD) plugin as a DaemonSet. + Set this value to ``false`` if NFD is already running in the cluster. + - ``true`` + + * - ``nfd.nodefeaturerules`` + - Installs node feature rules related to confidential computing. NFD uses the rules to detect security features + in CPUs and NVIDIA GPUs. Set this value to ``true`` when configuring the Operator for Confidential Containers. + - ``false`` + +.. _helm-values-dra: + +************************ +DRA Support (GPUCluster) +************************ + +These values enable and configure the DRA GPU resource management model. +The ``gpuCluster.*`` values populate the ``GPUCluster`` custom resource; for the full field list, refer to the +:doc:`GPUCluster Custom Resource Reference ` and +:doc:`Deploying the GPU Operator with DRA Support `. + +.. list-table:: + :widths: 25 50 25 + :header-rows: 1 + + * - Parameter + - Description + - Default + + * - ``gpuCluster.enabled`` + - When set to ``true``, the Operator creates the default ``GPUCluster`` resource and enables the DRA model. + - ``false`` + + * - ``gpuCluster.draDriver.*`` + - Configures the DRA driver image and its GPU allocation (``gpus``) and ComputeDomains capabilities. + Populates ``spec.draDriver`` of the ``GPUCluster`` resource. + - See reference + + * - ``operator.env[*]`` (``DEFAULT_GPU_ALLOCATION_MODE``) + - Set to ``dra`` for a ``GPUCluster`` deployment so newly-added GPU nodes use the DRA model. + - unset + +***************** +Sandbox Workloads +***************** + +These values apply when running virtual machine workloads (KubeVirt) or Kata Containers. + +.. list-table:: + :widths: 25 50 25 + :header-rows: 1 + + * - Parameter + - Description + - Default + + * - ``sandboxWorkloads.enabled`` + - Specifies whether sandbox workloads are enabled. + - ``false`` + + * - ``sandboxWorkloads.defaultWorkload`` + - Default workload type for the cluster: ``container``, ``vm-passthrough``, or ``vm-vgpu``. + Refer to :doc:`KubeVirt ` and :doc:`Kata Containers `. + - ``container`` + + * - ``sandboxWorkloads.mode`` + - Sandbox mode used when deploying sandbox workloads. Accepted values are ``kubevirt`` (default) and ``kata``. + - ``kubevirt`` + +**************** +Other Components +**************** + +.. list-table:: + :widths: 25 50 25 + :header-rows: 1 + + * - Parameter + - Description + - Default + + * - ``ccManager.enabled`` + - When set to ``true``, the Operator deploys the NVIDIA Confidential Computing Manager for Kubernetes. + - ``false`` + + * - ``gdrcopy.enabled`` + - Enables support for GDRCopy. When set to ``true``, the GDRCopy driver runs as a sidecar container in the GPU + driver pod. For information about GDRCopy, refer to the + `gdrcopy `__ page. + - ``false`` diff --git a/gpu-operator/index.rst b/gpu-operator/index.rst index 010695806..271e832fa 100644 --- a/gpu-operator/index.rst +++ b/gpu-operator/index.rst @@ -17,29 +17,44 @@ .. headings # #, * *, =, -, ^, " .. toctree:: - :caption: NVIDIA GPU Operator + :caption: Overview :titlesonly: :hidden: About the Operator - Install - Upgrade - Uninstall + Architecture + Control Plane Architecture Platform Support Release Notes - Troubleshooting - gpu-driver-upgrades.rst - install-gpu-operator-vgpu.rst - install-gpu-operator-nvaie.rst - Security Considerations +.. toctree:: + :caption: Getting Started + :titlesonly: + :hidden: + + Install Overview + Prerequisites + Install with Helm + Install on Red Hat OpenShift Container Platform + Upgrade + gpu-driver-upgrades.rst + Uninstall +.. toctree:: + :caption: Liscensed Installs + :titlesonly: + :hidden: + + Install with NVIDIA vGPU + Install with NVIDIA AI Enterprise + Install Government Ready Components .. toctree:: :caption: Advanced Operator Configuration :titlesonly: :hidden: + Deploying with DRA Support DRA Driver for NVIDIA GPUs Multi-Instance GPU (MIG) Time-Slicing GPUs @@ -68,12 +83,6 @@ Air-Gapped Network Service Mesh -.. toctree:: - :titlesonly: - :hidden: - - - .. toctree:: :caption: Platform-Specific Configurations :titlesonly: @@ -85,5 +94,15 @@ NVIDIA GPU Operator on Red Hat OpenShift Container Platform Partner Validated +.. toctree:: + :caption: Reference + :titlesonly: + :hidden: + + ClusterPolicy Reference + GPUCluster Reference + NVIDIA GPU Driver Reference + Troubleshooting + Security Considerations .. include:: overview.rst diff --git a/gpu-operator/install-gpu-operator-nvaie.rst b/gpu-operator/install-gpu-operator-nvaie.rst index 0facd1377..b98ffd9c5 100644 --- a/gpu-operator/install-gpu-operator-nvaie.rst +++ b/gpu-operator/install-gpu-operator-nvaie.rst @@ -193,8 +193,4 @@ Use the ``--version=`` argument when installing with Helm. Related Information ******************* -.. toctree:: - - Government Ready - - `NVIDIA AI Enterprise `_ web page. diff --git a/gpu-operator/install-paths.rst b/gpu-operator/install-paths.rst new file mode 100644 index 000000000..567a7b6cc --- /dev/null +++ b/gpu-operator/install-paths.rst @@ -0,0 +1,296 @@ +.. license-header + SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +.. headings (h1/h2/h3/h4/h5) are # * = - + +.. _install-paths: +.. _gpu-operator-install-paths: + +================ +Install Overview +================ + +The NVIDIA GPU Operator supports many Kubernetes platforms, licensing models, and deployment configurations. +This page helps you choose the install guide that matches your environment. +Each linked page contains the step-by-step procedures; start here when you are not sure where to begin. + +Before you install, review :doc:`prerequisites` to confirm that your cluster, hardware, and tooling meet the requirements for your chosen path. + +.. admonition:: Red Hat OpenShift Container Platform + :class: tip + + If you run Red Hat OpenShift, install the GPU Operator through OperatorHub or the OpenShift CLI (OLM), not the Helm + quickstart in this book. Refer to :external+ocp:doc:`index` for OpenShift install, upgrade, and disconnected + deployment procedures. + + +GPU Resource Management +======================= + +The GPU Operator supports managing GPU resources on your cluster using the following methods: + +- NVIDIA Device Plugin through the ``ClusterPolicy`` custom resource. +- DRA Driver for NVIDIA GPUs through the ``GPUCluster`` custom resource. + +At install time, choose one GPU resource management model for the cluster. +Use either ``ClusterPolicy`` or ``GPUCluster``; using both in the same cluster is not supported. + +.. note:: + + Deploying the DRA model through the ``GPUCluster`` custom resource is in Technology Preview and is supported only + for greenfield (new) deployments. + The ``GPUCluster`` API is served under ``nvidia.com/v1alpha1`` and is subject to change in future releases. + Migrating an existing ``ClusterPolicy`` deployment to ``GPUCluster`` in place is not supported. + +.. list-table:: + :header-rows: 1 + :widths: 34 33 33 + + * - Component + - Device-plugin model (``ClusterPolicy``) + - DRA model (``GPUCluster``) + * - GPU allocation + - NVIDIA Kubernetes Device Plugin (extended resources) + - DRA Driver for NVIDIA GPUs (ResourceClaims) + * - NVIDIA GPU driver + - Managed by ``ClusterPolicy`` or ``NVIDIADriver`` + - Pre-installed or managed by ``NVIDIADriver`` (not managed by ``GPUCluster``) + * - Default install path + - :doc:`Install with Helm ` + - :doc:`Deploying the GPU Operator with DRA Support ` + +Do not deploy ``ClusterPolicy`` and ``GPUCluster`` as GPU resource management models in the same cluster. +Refer to :doc:`Deploying the GPU Operator with DRA Support ` for the supported greenfield DRA +deployment. + +For the fully supported standalone DRA Driver Helm chart (without Operator-managed ``GPUCluster``), refer to +:doc:`DRA Driver for NVIDIA GPUs `. + + +Driver Provisioning +=================== + +How the NVIDIA GPU driver is installed is independent of which platform or GPU resource management model you choose. + +.. list-table:: + :header-rows: 1 + :widths: 30 35 35 + + * - Model + - When to use + - Where to configure + * - Operator-managed (``ClusterPolicy``) + - Default for generic Kubernetes and most cloud installs + - :doc:`Install with Helm ` and :ref:`common deployment scenarios` + * - ``NVIDIADriver`` custom resource + - Per-node or mixed-OS driver management; use this for Operator-managed drivers with the DRA model + - :doc:`GPU Driver CRD ` + * - Pre-installed on the host + - Driver already present on GPU nodes (``driver.enabled=false``) + - :doc:`Install with Helm `, :doc:`Microsoft AKS `, or + :doc:`Deploying the GPU Operator with DRA Support ` + + +NVIDIA Licenses +=============== + +If your deployment requires a licensed NVIDIA product or a sovereign-cloud configuration, use one of these guides +instead of the default Helm install. + +.. grid:: 1 1 2 2 + :gutter: 3 + + .. grid-item-card:: :octicon:`briefcase;1.5em;sd-mr-1` NVIDIA AI Enterprise + :link: install-gpu-operator-nvaie + :link-type: doc + + Install with NVAIE licensing using the NGC Bash script (vGPU guest) or standard Helm (datacenter driver). + +++ + :bdg-secondary:`NVAIE` + + .. grid-item-card:: :octicon:`device-desktop;1.5em;sd-mr-1` NVIDIA vGPU + :link: install-gpu-operator-vgpu + :link-type: doc + + Build a custom driver image and install with vGPU licensing secrets. + +++ + :bdg-secondary:`vGPU` + + .. grid-item-card:: :octicon:`shield;1.5em;sd-mr-1` Government Ready Components + :link: install-gpu-operator-gov-ready + :link-type: doc + + Deploy STIG- and FIPS-hardened containers for sovereign and FedRAMP environments. + Requires NVAIE licensing. + +++ + :bdg-secondary:`gov-ready` + + +Platform +======== + +Select the guide for your Kubernetes distribution or cloud platform. + +.. grid:: 1 1 2 2 + :gutter: 3 + + .. grid-item-card:: :octicon:`server;1.5em;sd-mr-1` Upstream Kubernetes or Bare Metal + :link: getting-started + :link-type: doc + + Install with Helm on upstream Kubernetes, bare-metal hosts, or VMs with GPU passthrough. This is the default path + for most self-managed clusters. + +++ + :bdg-secondary:`Helm` :bdg-secondary:`default` + + .. grid-item-card:: :octicon:`cloud;1.5em;sd-mr-1` Amazon EKS + :link: amazon-eks + :link-type: doc + + Prepare Ubuntu node groups and choose between the default EKS GPU stack and Operator-managed drivers. + +++ + :bdg-secondary:`EKS` + + .. grid-item-card:: :octicon:`cloud;1.5em;sd-mr-1` Azure AKS + :link: microsoft-aks + :link-type: doc + + Compare AKS GPU options and install the Operator with ``--skip-gpu-driver-install`` or pre-installed drivers. + +++ + :bdg-secondary:`AKS` + + .. grid-item-card:: :octicon:`cloud;1.5em;sd-mr-1` Google GKE + :link: google-gke + :link-type: doc + + Choose between the Google driver installer plus the Operator, or full Operator driver management on Ubuntu nodes. + +++ + :bdg-secondary:`GKE` + + .. grid-item-card:: :octicon:`package;1.5em;sd-mr-1` Red Hat OpenShift + :link: https://docs.nvidia.com/datacenter/cloud-native/openshift/latest/index.html + :link-type: url + + Install through OperatorHub or ``oc`` using the OpenShift documentation book. + +++ + :bdg-secondary:`OLM` :bdg-secondary:`OpenShift` + + .. grid-item-card:: :octicon:`checklist;1.5em;sd-mr-1` Partner Validated Platforms + :link: https://docs.nvidia.com/datacenter/cloud-native/partner-validated/latest/index.html + :link-type: url + + Validated configurations for partner Kubernetes distributions such as Mirantis MKE and SUSE RKE2. + +++ + :bdg-secondary:`partner` + + .. grid-item-card:: :octicon:`globe;1.5em;sd-mr-1` Google Cloud Anthos + :link: https://docs.nvidia.com/datacenter/cloud-native/edge/latest/anthos-guide.html + :link-type: url + + Deploy on Anthos clusters on bare metal or VMware vSphere. + +++ + :bdg-secondary:`Anthos` + +.. _install-paths-network: + +Specialized Network Environments +================================ + +These guides apply **in addition to** your primary install path when the cluster has network or mesh constraints. + +.. grid:: 1 1 3 3 + :gutter: 3 + + .. grid-item-card:: :octicon:`globe;1.5em;sd-mr-1` HTTP Proxy + :link: install-gpu-operator-proxy + :link-type: doc + + Configure the driver container to reach external endpoints through a proxy. + +++ + :bdg-secondary:`proxy` + + .. grid-item-card:: :octicon:`lock;1.5em;sd-mr-1` Air-Gapped Network + :link: install-gpu-operator-air-gapped + :link-type: doc + + Mirror images and package repositories for disconnected clusters. + +++ + :bdg-secondary:`air-gap` + + .. grid-item-card:: :octicon:`git-branch;1.5em;sd-mr-1` Service Mesh + :link: install-gpu-operator-service-mesh + :link-type: doc + + Disable sidecar injection on the ``gpu-operator`` namespace before installing. + +++ + :bdg-secondary:`Istio` :bdg-secondary:`Linkerd` + +---- + +Suggested Paths +=============== + +If you are new to the GPU Operator, follow one of these common sequences. + +.. mermaid:: + + flowchart LR + subgraph bareMetal["Bare Metal / Generic K8s"] + P1[Prerequisites] --> H1[Install with Helm] + end + + subgraph cloud["Cloud Managed K8s"] + P2[Prerequisites] --> C1[Platform Guide] + C1 --> H2[Install with Helm] + end + + subgraph ocp["OpenShift"] + O1[OpenShift Book] + end + + subgraph dra["DRA Greenfield"] + P3[Prerequisites] --> D1[DRA Install Guide] + end + + H1 --> V[Verify Installation] + H2 --> V + O1 --> V + D1 --> V + +**Common journeys:** + +- **Bare metal or generic Kubernetes**: :doc:`prerequisites` → :doc:`Install with Helm ` → verify +- **Amazon EKS, Azure AKS, or Google GKE**: :doc:`prerequisites` → platform guide → :doc:`Install with Helm + ` → verify +- **Red Hat OpenShift**: :external+ocp:doc:`index` (OperatorHub or ``oc`` install) +- **DRA-native cluster (Technology Preview)**: :doc:`prerequisites` → :doc:`Deploying the GPU Operator with DRA Support + ` → verify + +If your cluster uses a restricted network, add the matching guide from :ref:`install-paths-network` before or during +install. + +---- + +Next Steps +========== + +After you choose a path: + +#. Review :doc:`prerequisites` for your platform, license, and GPU resource management model. +#. Follow the install guide for your chosen path. +#. Verify the installation using the steps in that guide. +#. Configure workloads (MIG, time-slicing, RDMA, and others) from the Advanced Operator Configuration section in the + sidebar. diff --git a/gpu-operator/life-cycle-policy.rst b/gpu-operator/life-cycle-policy.rst index 682d24c8f..e918f154e 100644 --- a/gpu-operator/life-cycle-policy.rst +++ b/gpu-operator/life-cycle-policy.rst @@ -88,12 +88,9 @@ Refer to :ref:`Upgrading the NVIDIA GPU Operator` for more information. :header-rows: 2 * - :rspan:`1` Component - - :cspan:`3` GPU Operator Version + - GPU Operator Version - * - v26.3.0 - - v26.3.1 - - v26.3.2 - - v26.3.3 + * - v26.7.0 * - NVIDIA GPU Driver |ki|_ - | `595.71.05 `_ @@ -104,99 +101,53 @@ Refer to :ref:`Upgrading the NVIDIA GPU Operator` for more information. | `580.159.03 `_ | `580.126.20 `_ (**D**) | `570.211.01 `_ - | `535.309.01 `_ - | `535.288.01 `_ - - | `595.71.05 `_ - | `595.58.03 `_ - | `590.48.01 `_ - | `580.167.08 `_ (**R**) - | `580.159.04 `_ - | `580.159.03 `_ - | `580.126.20 `_ (**D**) - | `570.211.01 `_ - | `535.309.01 `_ - | `535.288.01 `_ - - | `595.71.05 `_ - | `595.58.03 `_ - | `590.48.01 `_ - | `580.167.08 `_ (**R**) - | `580.159.04 `_ - | `580.159.03 `_ - | `580.126.20 `_ (**D**) - | `570.211.01 `_ - | `535.309.01 `_ - | `535.288.01 `_ - - | `595.71.05 `_ - | `595.58.03 `_ - | `590.48.01 `_ - | `580.167.08 `_ (**R**) - | `580.159.04 `_ - | `580.159.03 `_ - | `580.126.20 `_ (**D**) - | `570.211.01 `_ - | `535.309.01 `_ - | `535.288.01 `_ * - NVIDIA Driver Manager for Kubernetes - - :cspan:`1` `v0.10.0 `__ - :cspan:`1` `v0.11.0 `__ * - NVIDIA Container Toolkit - - :cspan:`1` `1.19.0 `__ - :cspan:`1` `1.19.1 `__ * - NVIDIA Kubernetes Device Plugin - - :cspan:`1` `0.19.0 `__ - - `0.19.2 `__ - `0.19.3 `__ * - DCGM Exporter - - :cspan:`1` `v4.5.1-4.8.0 `__ - :cspan:`1` `v4.5.3-4.8.2 `__ * - Node Feature Discovery - - :cspan:`3` `v0.18.3 `__ + - :cspan:`1` `v0.18.3 `__ * - | NVIDIA GPU Feature Discovery | for Kubernetes - - :cspan:`1` `0.19.0 `__ - - `0.19.2 `__ - `0.19.3 `__ * - NVIDIA MIG Manager for Kubernetes - - :cspan:`1` `0.14.0 `__ - :cspan:`1` `0.14.2 `__ * - DCGM - - :cspan:`3` `4.5.2-1 `__ + - :cspan:`1` `4.5.2-1 `__ * - Validator for NVIDIA GPU Operator - - v26.3.0 - - v26.3.1 - - v26.3.2 - v26.3.3 * - NVIDIA KubeVirt GPU Device Plugin - - :cspan:`3` `v1.5.0 `__ + - :cspan:`1` `v1.5.0 `__ * - NVIDIA vGPU Device Manager - - :cspan:`3` `v0.4.2 `__ + - :cspan:`1` `v0.4.2 `__ * - NVIDIA GDS Driver |gds|_ - - :cspan:`3` `2.27.3 `__ + - :cspan:`1` `2.27.3 `__ * - | NVIDIA Confidential Computing | Manager for Kubernetes - - `v0.3.0 `__ - - :cspan:`2` `v0.4.0 `__ + - :cspan:`1` `v0.4.0 `__ * - NVIDIA GDRCopy Driver - - `v2.5.1 `__ - - :cspan:`2` `v2.5.2 `__ + - :cspan:`1` `v2.5.2 `__ * - NVIDIA Kata Sandbox Device Plugin - - `v0.0.2 `__ - - :cspan:`2` `v0.0.3 `__ + - :cspan:`1` `v0.0.3 `__ .. _known-issue: diff --git a/gpu-operator/overview.rst b/gpu-operator/overview.rst index 8d2007e22..1c00442b1 100644 --- a/gpu-operator/overview.rst +++ b/gpu-operator/overview.rst @@ -47,7 +47,14 @@ Browse through the following documents for getting started, platform support and Getting Started --------------- -The :ref:`operator-install-guide` guide includes information on installing the GPU Operator in a Kubernetes cluster. +Use :ref:`install-paths` to choose the install guide for your platform, license, and GPU resource management model. +The :ref:`operator-install-guide` guide covers the default Helm install for generic Kubernetes clusters. + +Architecture +------------ + +Refer to :ref:`gpu-operator-architecture` to learn how the GPU Operator manages components through ``ClusterPolicy``, +``GPUCluster``, and ``NVIDIADriver`` custom resources. Release Notes --------------- diff --git a/gpu-operator/platform-support.rst b/gpu-operator/platform-support.rst index 970b7cd0c..1677044b2 100644 --- a/gpu-operator/platform-support.rst +++ b/gpu-operator/platform-support.rst @@ -493,6 +493,7 @@ See the :doc:`precompiled-drivers` page for more information about using precomp | | AWS, Oracle | | | +----------------------------+------------------------+----------------+---------------------+ +.. _supported-container-runtimes: **************************** Supported Container Runtimes diff --git a/gpu-operator/prerequisites.rst b/gpu-operator/prerequisites.rst new file mode 100644 index 000000000..a7576a380 --- /dev/null +++ b/gpu-operator/prerequisites.rst @@ -0,0 +1,114 @@ +.. license-header + SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +.. headings (h1/h2/h3/h4/h5) are # * = - + +============= +Prerequisites +============= + +The following are the prerequisites for deploying the NVIDIA GPU Operator. + + +Hardware And Operating System Requirements +****************************************** + +Your GPUs and cluster align with the :ref:`GPU Operator support matrix +` for hardware and operating system. + +Additional Operating System configurations depend on how you plan to manage the NVIDIA GPU Driver: + +* If you are planning to use NVIDIA GPU Driver Custom Resource Definition to manage drivers, you can use a mix of operating system versions on CPU and GPU nodes. Refer to the :doc:`NVIDIA GPU Driver Custom Resource Definition ` page for more information on using this custom resource. + +* If you are planning to use ClusterPolicy for driver configuration, all worker nodes or node groups that will run GPU workloads in the Kubernetes cluster must run the same operating system version to use the NVIDIA GPU Driver container. + +* If you are planning to pre-install the NVIDIA GPU Driver on your nodes, then you can run different operating systems on your nodes. + +* For worker nodes or node groups that run CPU workloads only, the nodes can run any operating system because the GPU Operator does not perform any configuration or management of nodes for CPU-only workloads. + + +Cluster Requirements +******************** + +* You have the ``kubectl`` and ``helm`` CLIs available on a client machine. + For supported Kubernetes versions refer to the :ref:`Container Platforms support matrix `. + + You can run the following commands to install the Helm CLI: + + .. code-block:: console + + $ curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 \ + && chmod 700 get_helm.sh \ + && ./get_helm.sh + +* Nodes must be configured with a container engine such as CRI-O or containerd. + For supported container runtime, refer to :ref:`Supported Container Runtimes `. + +* If your cluster uses Pod Security Admission (PSA) to restrict the behavior of pods, label the namespace for the Operator to set the enforcement policy to privileged: + + .. code-block:: console + + $ kubectl create ns gpu-operator + $ kubectl label --overwrite ns gpu-operator pod-security.kubernetes.io/enforce=privileged + +* Node Feature Discovery (NFD) is a dependency for the Operator on each node. + By default, NFD master and worker are automatically deployed by the Operator. + If NFD is already running in the cluster, then you must disable deploying NFD when you install the Operator. + + One way to determine if NFD is already running in the cluster is to check for an NFD label on your nodes (requires `jq `__): + + .. code-block:: console + + $ kubectl get nodes -o json | jq '.items[].metadata.labels | keys | any(startswith("feature.node.kubernetes.io"))' + + If the command output is ``true``, then NFD is already running in the cluster. + + +DRA Requirements +**************** + +If you are planning to use the DRA Driver for NVIDIA GPUs for GPU resource management, the following requirements must be met: + +* Use Kubernetes 1.34.2 or later. + +* If you plan to use pre-installed driver, the NVIDIA GPU driver must be version 580 or later. + +* For ComputeDomains, ensure the following: + + * NVIDIA Grace Blackwell GPUs with Multi-Node NVLink (MNNVL) are available on your cluster. + Refer to the `NVIDIA Multi-Node NVLink Systems documentation + `_ for details. + + * When using ComputeDomains with a pre-installed GPU driver: + + * The corresponding ``nvidia-imex-*`` packages are installed through your Linux distribution's package manager. + * The IMEX systemd service is disabled before installing the GPU Operator (on all GPU nodes). For example: + + .. code-block:: console + + $ systemctl disable --now nvidia-imex.service && systemctl mask nvidia-imex.service + + +********** +Next Steps +********** + +After verifying prerequisites, choose an install path: + +- :ref:`install-paths` — overview of platform, license, and enablement-stack options +- :doc:`getting-started` — default Helm install for generic Kubernetes +- :doc:`install-gpu-operator-nvaie` — NVIDIA AI Enterprise +- :doc:`install-gpu-operator-vgpu` — NVIDIA vGPU From 4b8853cddc9a38e95a0df3e1731b6428affcd442 Mon Sep 17 00:00:00 2001 From: Mike McKiernan Date: Fri, 31 Jul 2026 15:33:27 -0400 Subject: [PATCH 6/9] docs: Daemon set mounts for OL nodes Signed-off-by: Mike McKiernan --- gpu-operator/platform-support.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/gpu-operator/platform-support.rst b/gpu-operator/platform-support.rst index 1677044b2..d3a12699a 100644 --- a/gpu-operator/platform-support.rst +++ b/gpu-operator/platform-support.rst @@ -471,6 +471,21 @@ The GPU Operator has been validated in the following scenarios: - 1.32---1.36 - 1.32---1.36 +Starting with version 26.7.0, the Operator can render driver workloads for nodes when Node Feature Discovery +reports ``feature.node.kubernetes.io/system-os_release.ID=ol``. +The Operator combines ``ol`` with the major component of the ``VERSION_ID`` label to form the operating system tag. +For example, ``VERSION_ID=9.7`` produces ``ol9``. +For tag-based driver images, the Operator appends the operating system tag to the configured driver +version, so a version such as ``580.126.20`` resolves to an image tag that ends in ``580.126.20-ol9``. +The Operator also mounts custom yum repository configuration in ``/etc/yum.repos.d`` and custom certificate +configuration in ``/etc/pki/ca-trust/extracted/pem`` on these nodes. + +This behavior provides operating system tag selection and manifest rendering only. +The change does not add an NVIDIA-published Oracle Linux driver container image or qualify Oracle Linux for the validated operating +system and container runtime support matrices on this page. +You are responsible for building, validating, publishing, and maintaining a custom driver image with the tag that the Operator constructs. +NVIDIA provides limited support for custom driver container images. + .. _supported-precompiled-drivers: ***************************** From 8acadc537feb137b5d52caed65cc13cad72c9c83 Mon Sep 17 00:00:00 2001 From: Mike McKiernan Date: Fri, 31 Jul 2026 15:45:48 -0400 Subject: [PATCH 7/9] docs: DRA driver with OCP Signed-off-by: Mike McKiernan --- openshift/gpu-operator-dra.rst | 143 +++++++++++++++++++++++++++++++++ openshift/index.rst | 1 + 2 files changed, 144 insertions(+) create mode 100644 openshift/gpu-operator-dra.rst diff --git a/openshift/gpu-operator-dra.rst b/openshift/gpu-operator-dra.rst new file mode 100644 index 000000000..332bbd364 --- /dev/null +++ b/openshift/gpu-operator-dra.rst @@ -0,0 +1,143 @@ +.. _openshift-gpu-operator-dra: + +################################################ +Using Operator-Managed DRA on Red Hat OpenShift +################################################ + +The GPU Operator can deploy the DRA Driver for NVIDIA GPUs as an operand of the ``ClusterPolicy`` resource. +This page describes the OpenShift-specific prerequisites and security configuration for this deployment model. +For an overview of Dynamic Resource Allocation (DRA), configuration options, and sample workloads, refer to +:external+gpuop:doc:`gpu-operator-dra`. + +.. important:: + + This page documents the DRA driver operand that is configured with ``ClusterPolicy.spec.draDriver``. + This deployment model is distinct from the Technology Preview ``GPUCluster`` integration and from installing the + DRA Driver for NVIDIA GPUs as a standalone Helm chart. + Support information for one deployment model does not apply to the other deployment models. + +************* +Prerequisites +************* + +Before you enable the Operator-managed DRA driver, ensure that your environment meets the following requirements: + +* The OpenShift Container Platform and GPU Operator versions are listed as a supported combination in the + :external+gpuop:ref:`Container Platforms ` table. +* You have ``cluster-admin`` access. + This access is required because the Operator creates cluster roles, cluster role bindings, and DRA resources. +* The cluster serves the Kubernetes DRA API and includes the cluster-scoped ``DeviceClass`` resource. + Run the following command and confirm that ``deviceclasses`` is included in the output: + + .. code-block:: console + + $ oc api-resources --api-group=resource.k8s.io --namespaced=false + + The GPU Operator also checks for the ``DeviceClass`` resource. + If the resource is not available, a ``ClusterPolicy`` that enables DRA remains ``notReady`` and reports a + validation error. + +Review the following configuration restrictions: + +* ``draDriver.gpus.enabled`` and ``devicePlugin.enabled`` cannot both be ``true``. + Both components would otherwise advertise the same GPUs for allocation. +* ``draDriver.computeDomains.enabled`` can be used with the NVIDIA device plugin because ComputeDomain resources are + separate from GPU resources. +* ``sandboxWorkloads.enabled`` cannot be ``true`` when either DRA driver capability is enabled in the same + ``ClusterPolicy``. +* The integrated configuration does not provide a per-node transition between DRA GPU allocation and device-plugin + GPU allocation. + Configure the allocation method before scheduling GPU workloads. + +********** +Enable DRA +********** + +Follow :ref:`install-nvidiagpu` to install the GPU Operator. +When you create the ``ClusterPolicy`` instance, add one of the following configurations to its ``spec``. + +To use DRA for GPU allocation with resource claim objects, disable the NVIDIA device plugin and enable the GPU capability of the DRA driver: + +.. code-block:: yaml + + spec: + devicePlugin: + enabled: false + draDriver: + gpus: + enabled: true + +To use traditional `nvidia.com/gpu` GPU allocation and add compute domain functionality from DRA, leave the NVIDIA device plugin enabled and enable the ComputeDomains capability: + +.. code-block:: yaml + + spec: + devicePlugin: + enabled: true + draDriver: + computeDomains: + enabled: true + +You can enable both DRA capabilities in one ``ClusterPolicy``, but the NVIDIA device plugin must be disabled when +``draDriver.gpus.enabled`` is ``true``. + +After you apply the configuration, verify that the ``ClusterPolicy`` is ready: + +.. code-block:: console + + $ oc get clusterpolicy + +Verify that the DRA kubelet plugin pods are running: + +.. code-block:: console + + $ oc get pods -n nvidia-gpu-operator -l app=nvidia-dra-driver-kubelet-plugin + +Verify that the DRA resources are available: + +.. code-block:: console + + $ oc get deviceclasses,resourceslices + +If you installed the Operator in a different namespace, replace ``nvidia-gpu-operator`` in the preceding commands +with that namespace. + +*************************** +Security and SCC Management +*************************** + +The DRA kubelet plugin requires privileged access to GPU devices and to kubelet plugin directories on each GPU node. +On OpenShift, the GPU Operator detects the platform and creates the required cluster role bindings to the built-in +Security Context Constraints (SCC) roles. + +.. list-table:: OpenShift SCC bindings for Operator-managed DRA + :header-rows: 1 + :widths: 34 28 38 + + * - Service account + - SCC cluster role + - Purpose + * - ``nvidia-dra-driver-kubeletplugin`` + - ``system:openshift:scc:privileged`` + - Runs the DRA kubelet plugin containers with privileged access and the required host-path mounts. + * - ``compute-domain-daemon-service-account`` + - ``system:openshift:scc:anyuid`` + - Binds the ComputeDomain daemon service account to the ``anyuid`` SCC. + +The Operator sets the service account namespace in these bindings to the namespace where the Operator is installed. +Do not manually add the service accounts to the SCCs. + +You can verify that the Operator created the bindings: + +.. code-block:: console + + $ oc get clusterrolebinding nvidia-dra-driver-openshift-privileged-role-binding-kubeletplugin + +.. code-block:: console + + $ oc get clusterrolebinding compute-domain-daemon-openshift-anyuid-role-binding + +The GPU Operator service account receives RBAC permissions to create and reconcile ``DeviceClass`` resources. +The DRA service accounts receive the permissions they require for ``ResourceClaim``, ``ResourceClaimTemplate``, +``ResourceSlice``, ``ComputeDomain``, and ``ComputeDomainClique`` resources, as well as leases for controller leader +election. diff --git a/openshift/index.rst b/openshift/index.rst index 9fee76700..b1fbddcf5 100644 --- a/openshift/index.rst +++ b/openshift/index.rst @@ -10,6 +10,7 @@ NVIDIA GPU Operator on Red Hat OpenShift Container Platform Installation and Upgrade Overview NFD Operator Installation GPU Operator Installation + Operator-Managed DRA nvaie-with-ocp.rst mig-ocp.rst clean-up.rst From a4e64cc44a0517c5d1e22eb89730f04aa938fa68 Mon Sep 17 00:00:00 2001 From: Mike McKiernan Date: Fri, 31 Jul 2026 15:49:00 -0400 Subject: [PATCH 8/9] docs: Sundry bug fixes Signed-off-by: Mike McKiernan --- gpu-operator/release-notes.rst | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/gpu-operator/release-notes.rst b/gpu-operator/release-notes.rst index e6659a23d..984a03ac3 100644 --- a/gpu-operator/release-notes.rst +++ b/gpu-operator/release-notes.rst @@ -88,6 +88,37 @@ New Features Fixed Issues ------------ +* Fixed an issue where host driver validation could incorrectly report that ``nvidia-smi`` was missing when ``/usr/bin`` was an absolute symlink. The validator now resolves the symlink within the mounted host file system. (`PR #2464 `__, `Issue #1357 `__) * Fixed host driver and vGPU Manager validation so that the validator finds ``nvidia-smi`` in additional host locations, including ``/usr/sbin``, ``/opt/bin``, and the Windows Subsystem for Linux driver path. + Symlinks are resolved within the mounted host file system. + (`PR #2611 `__, `Issue #2506 `__) + +* Fixed an issue where vGPU Manager validation on non-SR-IOV GPUs could wait for unavailable SR-IOV virtual functions and time out. + The validator now waits for mediated-device parents on non-SR-IOV GPUs and for virtual functions on SR-IOV GPUs. + (`PR #2502 `__, `Issue #2365 `__) + +* Fixed an issue where the driver pod failed to start on kernels without ``CONFIG_MEMORY_HOTPLUG`` because the ``/sys/devices/system/memory/auto_online_blocks`` file was unavailable. + The driver pod now mounts the stable ``/sys/devices/system`` parent directory. + (`PR #2517 `__, `Issue #2463 `__) + +* Fixed an issue where the NVIDIA vGPU Device Manager did not start when the vGPU Manager driver was preinstalled on the host and ``driver.enabled=false``. + The readiness gate now accepts the status file from either a container-managed or host-installed vGPU Manager. + (`PR #2599 `__) + +* Fixed an issue where setting ``devicePlugin.config.create=true`` with empty configuration data could leave device plugin pods referencing a missing ConfigMap. + The Helm chart now rejects the configuration when ``devicePlugin.config.name`` or ``devicePlugin.config.data`` is empty. + (`PR #2642 `__, `Issue #2641 `__) + +* Fixed an issue where the ``ClusterPolicy`` status could fluctuate during an NVIDIADriver rolling upgrade. + The status now remains ``notReady`` while an NVIDIADriver-owned node has a pending, in-progress, or failed upgrade. + (`PR #2665 `__) + +* Fixed an issue where multiple NVIDIADriver custom resources with ``spec.default=true`` could incorrectly report a ``ready`` status. + The conflicting resources now report ``notReady`` with a ``ReconcileFailed`` condition until the conflict is resolved. + (`PR #2678 `__) + +* Updated the bundled vGPU Device Manager configuration for vGPU 20.0 so that PCI device ID ``0x318210DE`` maps to the ``B300X-269C`` profile. + (`PR #2625 `__, `Issue #2231 `__) + * Fixed an issue where custom NVIDIADriver custom resources could remain stuck and fail to reconcile after a conflicting default NVIDIADriver custom resource was deleted. The GPU Operator now re-evaluates all NVIDIADriver custom resources whenever any NVIDIADriver custom resource changes. (`PR #2258 `__) From 004f8eae00663bc0bcd1c488b1fdc5391a515594 Mon Sep 17 00:00:00 2001 From: Mike McKiernan Date: Fri, 31 Jul 2026 15:50:53 -0400 Subject: [PATCH 9/9] docs: Fix nav for arch Signed-off-by: Mike McKiernan --- gpu-operator/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gpu-operator/index.rst b/gpu-operator/index.rst index 271e832fa..2a269c3c0 100644 --- a/gpu-operator/index.rst +++ b/gpu-operator/index.rst @@ -23,7 +23,7 @@ About the Operator Architecture - Control Plane Architecture + Control Plane Architecture Platform Support Release Notes @@ -101,7 +101,7 @@ ClusterPolicy Reference GPUCluster Reference - NVIDIA GPU Driver Reference + NVIDIA GPU Driver Reference Troubleshooting Security Considerations