From d776b4461951b836573dbdaa1b106644a5a7d334 Mon Sep 17 00:00:00 2001 From: Simone Tiraboschi Date: Thu, 9 Jul 2026 18:00:03 +0200 Subject: [PATCH] fix(kubevirt): stop hardcoding evictionStrategy=External on KubeVirt VMs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #6380 unconditionally set evictionStrategy=External on all KubeVirt VMs created for HCP NodePools. This overrides the cluster-level default set by HCO (LiveMigrate on HA clusters), forcing CAPK to drain the guest node and delete the VMI on every infra node drain — even when the VM is live-migratable and KubeVirt could transparently migrate it with zero guest disruption. With External strategy, KubeVirt's evacuation controller skips the VM entirely (VMIMigratableOnEviction returns false for External), so no live migration is ever attempted. Instead, CAPK cordons and drains the guest node, deletes the VMI, and KubeVirt recreates it on another host. This is a full disruption to all workloads on the guest node. The External strategy is only appropriate for non-migratable VMs (e.g., with GPU passthrough or SR-IOV) where live migration is impossible and CAPK's drain-then-delete is the only graceful option. For the majority of VMs that are migratable, it is strictly worse than letting KubeVirt handle migration automatically via the cluster default. This commit removes the hardcoded External strategy and adds an optional EvictionStrategy field to KubevirtNodePoolPlatform. When not set, the cluster-level KubeVirt default applies. Users with non-migratable VMs can explicitly opt into External when needed. Fixes: https://redhat.atlassian.net/browse/OCPBUGS-98217 Signed-off-by: Simone Tiraboschi --- api/hypershift/v1beta1/kubevirt.go | 30 ++++ api/hypershift/v1beta1/nodepool_types_test.go | 132 +++++++++++++++++- .../AAA_ungated.yaml | 22 +++ .../GCPPlatform.yaml | 22 +++ .../OSStreams.yaml | 22 +++ .../OpenStack.yaml | 22 +++ .../v1beta1/kubevirtnodepoolplatform.go | 23 ++- .../nodepools-CustomNoUpgrade.crd.yaml | 22 +++ .../nodepools-Default.crd.yaml | 22 +++ .../nodepools-TechPreviewNoUpgrade.crd.yaml | 22 +++ docs/content/how-to/kubevirt/gpu-devices.md | 26 ++++ docs/content/reference/aggregated-docs.md | 79 +++++++++++ docs/content/reference/api.md | 53 +++++++ .../controllers/nodepool/kubevirt/kubevirt.go | 8 +- .../nodepool/kubevirt/kubevirt_test.go | 64 ++++++++- .../api/hypershift/v1beta1/kubevirt.go | 30 ++++ 16 files changed, 586 insertions(+), 13 deletions(-) diff --git a/api/hypershift/v1beta1/kubevirt.go b/api/hypershift/v1beta1/kubevirt.go index cc83e14d45bc..6f6207934d3a 100644 --- a/api/hypershift/v1beta1/kubevirt.go +++ b/api/hypershift/v1beta1/kubevirt.go @@ -143,6 +143,18 @@ const ( MultiQueueDisable MultiQueueSetting = "Disable" ) +// KubevirtEvictionStrategy defines the eviction behavior for KubeVirt VMs. +// +// +kubebuilder:validation:Enum=LiveMigrate;LiveMigrateIfPossible;External;None +type KubevirtEvictionStrategy string + +const ( + EvictionStrategyLiveMigrate KubevirtEvictionStrategy = "LiveMigrate" + EvictionStrategyLiveMigrateIfPossible KubevirtEvictionStrategy = "LiveMigrateIfPossible" + EvictionStrategyExternal KubevirtEvictionStrategy = "External" + EvictionStrategyNone KubevirtEvictionStrategy = "None" +) + // KubevirtNodePoolPlatform specifies the configuration of a NodePool when operating // on KubeVirt platform. type KubevirtNodePoolPlatform struct { @@ -191,6 +203,24 @@ type KubevirtNodePoolPlatform struct { // +optional // +kubebuilder:validation:MaxItems=10 KubevirtHostDevices []KubevirtHostDevice `json:"hostDevices,omitempty"` + + // evictionStrategy defines the eviction behavior for KubeVirt VMs during + // infrastructure node drain. If not set, the cluster-level default from the + // KubeVirt configuration applies (typically LiveMigrate on HA clusters when + // managed by HCO). + // + // - LiveMigrate: KubeVirt live-migrates the VM to another node (zero guest + // disruption). If the VM is not migratable, the node drain stalls. + // - LiveMigrateIfPossible: live-migrates if possible, otherwise allows the + // VMI pod to be killed (no graceful guest drain). + // - External: delegates eviction handling to CAPK, which drains the guest + // node before deleting the VMI. Use this for non-migratable VMs (e.g., + // with GPU passthrough or SR-IOV) that need graceful guest node drain. + // - None: the VM is killed immediately on eviction with no migration or + // graceful guest drain. This is the HCO default on single-node OpenShift. + // + // +optional + EvictionStrategy KubevirtEvictionStrategy `json:"evictionStrategy,omitempty"` } // KubevirtNetwork specifies the configuration for a virtual machine diff --git a/api/hypershift/v1beta1/nodepool_types_test.go b/api/hypershift/v1beta1/nodepool_types_test.go index d5f5a42c994d..b7e2cdd3b3cb 100644 --- a/api/hypershift/v1beta1/nodepool_types_test.go +++ b/api/hypershift/v1beta1/nodepool_types_test.go @@ -2,6 +2,7 @@ package v1beta1 import ( "encoding/json" + "reflect" "testing" "k8s.io/utils/ptr" @@ -21,9 +22,9 @@ func TestNodePoolAutoScalingSerializationCompatibility(t *testing.T) { name string // current is the N (current) version of the struct current NodePoolAutoScaling - // expectedJSON is the expected JSON output from marshalling current + // expectedJSON is the expected JSON output from marshaling current expectedJSON string - // nMinus1Result is the expected result when unmarshalling into the N-1 struct + // nMinus1Result is the expected result when unmarshaling into the N-1 struct nMinus1Result nodePoolAutoScalingNMinus1 }{ { @@ -93,3 +94,130 @@ func TestNodePoolAutoScalingSerializationCompatibility(t *testing.T) { }) } } + +func TestKubevirtEvictionStrategySerializationCompatibility(t *testing.T) { + tests := []struct { + name string + current KubevirtNodePoolPlatform + expectedJSON string + }{ + { + name: "When EvictionStrategy is not set it should be omitted from JSON", + current: KubevirtNodePoolPlatform{ + RootVolume: &KubevirtRootVolume{}, + }, + expectedJSON: `{"rootVolume":{}}`, + }, + { + name: "When EvictionStrategy is External it should be preserved", + current: KubevirtNodePoolPlatform{ + RootVolume: &KubevirtRootVolume{}, + EvictionStrategy: EvictionStrategyExternal, + }, + expectedJSON: `{"rootVolume":{},"evictionStrategy":"External"}`, + }, + { + name: "When EvictionStrategy is LiveMigrate it should be preserved", + current: KubevirtNodePoolPlatform{ + RootVolume: &KubevirtRootVolume{}, + EvictionStrategy: EvictionStrategyLiveMigrate, + }, + expectedJSON: `{"rootVolume":{},"evictionStrategy":"LiveMigrate"}`, + }, + { + name: "When EvictionStrategy is None it should be preserved", + current: KubevirtNodePoolPlatform{ + RootVolume: &KubevirtRootVolume{}, + EvictionStrategy: EvictionStrategyNone, + }, + expectedJSON: `{"rootVolume":{},"evictionStrategy":"None"}`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := json.Marshal(tt.current) + if err != nil { + t.Fatalf("failed to marshal current struct: %v", err) + } + if string(data) != tt.expectedJSON { + t.Errorf("unexpected JSON output: got %s, want %s", string(data), tt.expectedJSON) + } + + // Round-trip: unmarshal back into current type + var roundTripped KubevirtNodePoolPlatform + if err := json.Unmarshal(data, &roundTripped); err != nil { + t.Fatalf("failed to round-trip unmarshal: %v", err) + } + if roundTripped.EvictionStrategy != tt.current.EvictionStrategy { + t.Errorf("EvictionStrategy mismatch after round-trip: got %q, want %q", roundTripped.EvictionStrategy, tt.current.EvictionStrategy) + } + + // N-1 compatibility: verify current JSON can be parsed by code + // that doesn't know about evictionStrategy (unknown fields ignored) + var rawMap map[string]json.RawMessage + if err := json.Unmarshal(data, &rawMap); err != nil { + t.Fatalf("failed to unmarshal into raw map: %v", err) + } + + // N-1 JSON (without evictionStrategy) deserializes into current struct + delete(rawMap, "evictionStrategy") + nMinus1Data, err := json.Marshal(rawMap) + if err != nil { + t.Fatalf("failed to marshal N-1 map: %v", err) + } + var fromNMinus1 KubevirtNodePoolPlatform + if err := json.Unmarshal(nMinus1Data, &fromNMinus1); err != nil { + t.Fatalf("N failed to unmarshal JSON from N-1: %v", err) + } + if fromNMinus1.EvictionStrategy != "" { + t.Errorf("EvictionStrategy should be zero value when deserialized from N-1, got %q", fromNMinus1.EvictionStrategy) + } + }) + } +} + +func TestKubevirtEvictionStrategyEnumValues(t *testing.T) { + expected := []KubevirtEvictionStrategy{ + EvictionStrategyLiveMigrate, + EvictionStrategyLiveMigrateIfPossible, + EvictionStrategyExternal, + EvictionStrategyNone, + } + + for _, strategy := range expected { + t.Run("When EvictionStrategy is "+string(strategy)+" it should round-trip through JSON", func(t *testing.T) { + src := KubevirtNodePoolPlatform{ + RootVolume: &KubevirtRootVolume{}, + EvictionStrategy: strategy, + } + data, err := json.Marshal(src) + if err != nil { + t.Fatalf("failed to marshal: %v", err) + } + var dst KubevirtNodePoolPlatform + if err := json.Unmarshal(data, &dst); err != nil { + t.Fatalf("failed to unmarshal: %v", err) + } + if dst.EvictionStrategy != strategy { + t.Errorf("got %q, want %q", dst.EvictionStrategy, strategy) + } + }) + } +} + +func TestKubevirtNodePoolPlatformDeepCopyPreservesEvictionStrategy(t *testing.T) { + original := &KubevirtNodePoolPlatform{ + RootVolume: &KubevirtRootVolume{}, + EvictionStrategy: EvictionStrategyExternal, + } + copied := original.DeepCopy() + if !reflect.DeepEqual(original, copied) { + t.Errorf("DeepCopy mismatch: got %+v, want %+v", copied, original) + } + // Mutate copy, verify original unchanged + copied.EvictionStrategy = EvictionStrategyNone + if original.EvictionStrategy != EvictionStrategyExternal { + t.Error("mutating DeepCopy affected original") + } +} diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/nodepools.hypershift.openshift.io/AAA_ungated.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/nodepools.hypershift.openshift.io/AAA_ungated.yaml index 28b0b76bc4a9..f3d0d2c97968 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/nodepools.hypershift.openshift.io/AAA_ungated.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/nodepools.hypershift.openshift.io/AAA_ungated.yaml @@ -1129,6 +1129,28 @@ spec: - Guaranteed type: string type: object + evictionStrategy: + description: |- + evictionStrategy defines the eviction behavior for KubeVirt VMs during + infrastructure node drain. If not set, the cluster-level default from the + KubeVirt configuration applies (typically LiveMigrate on HA clusters when + managed by HCO). + + - LiveMigrate: KubeVirt live-migrates the VM to another node (zero guest + disruption). If the VM is not migratable, the node drain stalls. + - LiveMigrateIfPossible: live-migrates if possible, otherwise allows the + VMI pod to be killed (no graceful guest drain). + - External: delegates eviction handling to CAPK, which drains the guest + node before deleting the VMI. Use this for non-migratable VMs (e.g., + with GPU passthrough or SR-IOV) that need graceful guest node drain. + - None: the VM is killed immediately on eviction with no migration or + graceful guest drain. This is the HCO default on single-node OpenShift. + enum: + - LiveMigrate + - LiveMigrateIfPossible + - External + - None + type: string hostDevices: description: |- hostDevices specifies the host devices (e.g. GPU devices) to be passed diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/nodepools.hypershift.openshift.io/GCPPlatform.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/nodepools.hypershift.openshift.io/GCPPlatform.yaml index 2705760fb216..35c435aaa3dc 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/nodepools.hypershift.openshift.io/GCPPlatform.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/nodepools.hypershift.openshift.io/GCPPlatform.yaml @@ -1396,6 +1396,28 @@ spec: - Guaranteed type: string type: object + evictionStrategy: + description: |- + evictionStrategy defines the eviction behavior for KubeVirt VMs during + infrastructure node drain. If not set, the cluster-level default from the + KubeVirt configuration applies (typically LiveMigrate on HA clusters when + managed by HCO). + + - LiveMigrate: KubeVirt live-migrates the VM to another node (zero guest + disruption). If the VM is not migratable, the node drain stalls. + - LiveMigrateIfPossible: live-migrates if possible, otherwise allows the + VMI pod to be killed (no graceful guest drain). + - External: delegates eviction handling to CAPK, which drains the guest + node before deleting the VMI. Use this for non-migratable VMs (e.g., + with GPU passthrough or SR-IOV) that need graceful guest node drain. + - None: the VM is killed immediately on eviction with no migration or + graceful guest drain. This is the HCO default on single-node OpenShift. + enum: + - LiveMigrate + - LiveMigrateIfPossible + - External + - None + type: string hostDevices: description: |- hostDevices specifies the host devices (e.g. GPU devices) to be passed diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/nodepools.hypershift.openshift.io/OSStreams.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/nodepools.hypershift.openshift.io/OSStreams.yaml index f795a85dcc42..fa971e5d7210 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/nodepools.hypershift.openshift.io/OSStreams.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/nodepools.hypershift.openshift.io/OSStreams.yaml @@ -1162,6 +1162,28 @@ spec: - Guaranteed type: string type: object + evictionStrategy: + description: |- + evictionStrategy defines the eviction behavior for KubeVirt VMs during + infrastructure node drain. If not set, the cluster-level default from the + KubeVirt configuration applies (typically LiveMigrate on HA clusters when + managed by HCO). + + - LiveMigrate: KubeVirt live-migrates the VM to another node (zero guest + disruption). If the VM is not migratable, the node drain stalls. + - LiveMigrateIfPossible: live-migrates if possible, otherwise allows the + VMI pod to be killed (no graceful guest drain). + - External: delegates eviction handling to CAPK, which drains the guest + node before deleting the VMI. Use this for non-migratable VMs (e.g., + with GPU passthrough or SR-IOV) that need graceful guest node drain. + - None: the VM is killed immediately on eviction with no migration or + graceful guest drain. This is the HCO default on single-node OpenShift. + enum: + - LiveMigrate + - LiveMigrateIfPossible + - External + - None + type: string hostDevices: description: |- hostDevices specifies the host devices (e.g. GPU devices) to be passed diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/nodepools.hypershift.openshift.io/OpenStack.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/nodepools.hypershift.openshift.io/OpenStack.yaml index a7c77a9ab071..3648c36cc2ef 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/nodepools.hypershift.openshift.io/OpenStack.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/nodepools.hypershift.openshift.io/OpenStack.yaml @@ -1129,6 +1129,28 @@ spec: - Guaranteed type: string type: object + evictionStrategy: + description: |- + evictionStrategy defines the eviction behavior for KubeVirt VMs during + infrastructure node drain. If not set, the cluster-level default from the + KubeVirt configuration applies (typically LiveMigrate on HA clusters when + managed by HCO). + + - LiveMigrate: KubeVirt live-migrates the VM to another node (zero guest + disruption). If the VM is not migratable, the node drain stalls. + - LiveMigrateIfPossible: live-migrates if possible, otherwise allows the + VMI pod to be killed (no graceful guest drain). + - External: delegates eviction handling to CAPK, which drains the guest + node before deleting the VMI. Use this for non-migratable VMs (e.g., + with GPU passthrough or SR-IOV) that need graceful guest node drain. + - None: the VM is killed immediately on eviction with no migration or + graceful guest drain. This is the HCO default on single-node OpenShift. + enum: + - LiveMigrate + - LiveMigrateIfPossible + - External + - None + type: string hostDevices: description: |- hostDevices specifies the host devices (e.g. GPU devices) to be passed diff --git a/client/applyconfiguration/hypershift/v1beta1/kubevirtnodepoolplatform.go b/client/applyconfiguration/hypershift/v1beta1/kubevirtnodepoolplatform.go index c9648d9b9887..2a92aac32bed 100644 --- a/client/applyconfiguration/hypershift/v1beta1/kubevirtnodepoolplatform.go +++ b/client/applyconfiguration/hypershift/v1beta1/kubevirtnodepoolplatform.go @@ -24,13 +24,14 @@ import ( // KubevirtNodePoolPlatformApplyConfiguration represents a declarative configuration of the KubevirtNodePoolPlatform type for use // with apply. type KubevirtNodePoolPlatformApplyConfiguration struct { - RootVolume *KubevirtRootVolumeApplyConfiguration `json:"rootVolume,omitempty"` - Compute *KubevirtComputeApplyConfiguration `json:"compute,omitempty"` - NetworkInterfaceMultiQueue *hypershiftv1beta1.MultiQueueSetting `json:"networkInterfaceMultiqueue,omitempty"` - AdditionalNetworks []KubevirtNetworkApplyConfiguration `json:"additionalNetworks,omitempty"` - AttachDefaultNetwork *bool `json:"attachDefaultNetwork,omitempty"` - NodeSelector map[string]string `json:"nodeSelector,omitempty"` - KubevirtHostDevices []KubevirtHostDeviceApplyConfiguration `json:"hostDevices,omitempty"` + RootVolume *KubevirtRootVolumeApplyConfiguration `json:"rootVolume,omitempty"` + Compute *KubevirtComputeApplyConfiguration `json:"compute,omitempty"` + NetworkInterfaceMultiQueue *hypershiftv1beta1.MultiQueueSetting `json:"networkInterfaceMultiqueue,omitempty"` + AdditionalNetworks []KubevirtNetworkApplyConfiguration `json:"additionalNetworks,omitempty"` + AttachDefaultNetwork *bool `json:"attachDefaultNetwork,omitempty"` + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + KubevirtHostDevices []KubevirtHostDeviceApplyConfiguration `json:"hostDevices,omitempty"` + EvictionStrategy *hypershiftv1beta1.KubevirtEvictionStrategy `json:"evictionStrategy,omitempty"` } // KubevirtNodePoolPlatformApplyConfiguration constructs a declarative configuration of the KubevirtNodePoolPlatform type for use with @@ -110,3 +111,11 @@ func (b *KubevirtNodePoolPlatformApplyConfiguration) WithKubevirtHostDevices(val } return b } + +// WithEvictionStrategy sets the EvictionStrategy field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the EvictionStrategy field is set to the value of the last call. +func (b *KubevirtNodePoolPlatformApplyConfiguration) WithEvictionStrategy(value hypershiftv1beta1.KubevirtEvictionStrategy) *KubevirtNodePoolPlatformApplyConfiguration { + b.EvictionStrategy = &value + return b +} diff --git a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/nodepools-CustomNoUpgrade.crd.yaml b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/nodepools-CustomNoUpgrade.crd.yaml index 2fa0fb4be0b9..ecd375ae0c20 100644 --- a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/nodepools-CustomNoUpgrade.crd.yaml +++ b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/nodepools-CustomNoUpgrade.crd.yaml @@ -1432,6 +1432,28 @@ spec: - Guaranteed type: string type: object + evictionStrategy: + description: |- + evictionStrategy defines the eviction behavior for KubeVirt VMs during + infrastructure node drain. If not set, the cluster-level default from the + KubeVirt configuration applies (typically LiveMigrate on HA clusters when + managed by HCO). + + - LiveMigrate: KubeVirt live-migrates the VM to another node (zero guest + disruption). If the VM is not migratable, the node drain stalls. + - LiveMigrateIfPossible: live-migrates if possible, otherwise allows the + VMI pod to be killed (no graceful guest drain). + - External: delegates eviction handling to CAPK, which drains the guest + node before deleting the VMI. Use this for non-migratable VMs (e.g., + with GPU passthrough or SR-IOV) that need graceful guest node drain. + - None: the VM is killed immediately on eviction with no migration or + graceful guest drain. This is the HCO default on single-node OpenShift. + enum: + - LiveMigrate + - LiveMigrateIfPossible + - External + - None + type: string hostDevices: description: |- hostDevices specifies the host devices (e.g. GPU devices) to be passed diff --git a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/nodepools-Default.crd.yaml b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/nodepools-Default.crd.yaml index 0df741304192..24ec09cb913f 100644 --- a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/nodepools-Default.crd.yaml +++ b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/nodepools-Default.crd.yaml @@ -1132,6 +1132,28 @@ spec: - Guaranteed type: string type: object + evictionStrategy: + description: |- + evictionStrategy defines the eviction behavior for KubeVirt VMs during + infrastructure node drain. If not set, the cluster-level default from the + KubeVirt configuration applies (typically LiveMigrate on HA clusters when + managed by HCO). + + - LiveMigrate: KubeVirt live-migrates the VM to another node (zero guest + disruption). If the VM is not migratable, the node drain stalls. + - LiveMigrateIfPossible: live-migrates if possible, otherwise allows the + VMI pod to be killed (no graceful guest drain). + - External: delegates eviction handling to CAPK, which drains the guest + node before deleting the VMI. Use this for non-migratable VMs (e.g., + with GPU passthrough or SR-IOV) that need graceful guest node drain. + - None: the VM is killed immediately on eviction with no migration or + graceful guest drain. This is the HCO default on single-node OpenShift. + enum: + - LiveMigrate + - LiveMigrateIfPossible + - External + - None + type: string hostDevices: description: |- hostDevices specifies the host devices (e.g. GPU devices) to be passed diff --git a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/nodepools-TechPreviewNoUpgrade.crd.yaml b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/nodepools-TechPreviewNoUpgrade.crd.yaml index 8d13378de0e6..f1eed2a0447a 100644 --- a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/nodepools-TechPreviewNoUpgrade.crd.yaml +++ b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/nodepools-TechPreviewNoUpgrade.crd.yaml @@ -1432,6 +1432,28 @@ spec: - Guaranteed type: string type: object + evictionStrategy: + description: |- + evictionStrategy defines the eviction behavior for KubeVirt VMs during + infrastructure node drain. If not set, the cluster-level default from the + KubeVirt configuration applies (typically LiveMigrate on HA clusters when + managed by HCO). + + - LiveMigrate: KubeVirt live-migrates the VM to another node (zero guest + disruption). If the VM is not migratable, the node drain stalls. + - LiveMigrateIfPossible: live-migrates if possible, otherwise allows the + VMI pod to be killed (no graceful guest drain). + - External: delegates eviction handling to CAPK, which drains the guest + node before deleting the VMI. Use this for non-migratable VMs (e.g., + with GPU passthrough or SR-IOV) that need graceful guest node drain. + - None: the VM is killed immediately on eviction with no migration or + graceful guest drain. This is the HCO default on single-node OpenShift. + enum: + - LiveMigrate + - LiveMigrateIfPossible + - External + - None + type: string hostDevices: description: |- hostDevices specifies the host devices (e.g. GPU devices) to be passed diff --git a/docs/content/how-to/kubevirt/gpu-devices.md b/docs/content/how-to/kubevirt/gpu-devices.md index 400c4dd435d5..079e996abaca 100644 --- a/docs/content/how-to/kubevirt/gpu-devices.md +++ b/docs/content/how-to/kubevirt/gpu-devices.md @@ -78,6 +78,7 @@ spec: hostDevices: - count: 2 deviceName: nvidia-a100 + evictionStrategy: External networkInterfaceMultiqueue: Enable rootVolume: persistent: @@ -87,3 +88,28 @@ spec: replicas: 3 ``` +## Eviction Strategy for GPU NodePools + +VMs with GPU passthrough devices are not live-migratable. By default, when +no `evictionStrategy` is set on a NodePool, the cluster-level KubeVirt +default applies. On multi-node HA clusters managed by HCO this is +`LiveMigrate`; on single-node OpenShift (SNO) it is `None`. With +`LiveMigrate`, draining an infrastructure node that hosts a non-migratable +VM will stall indefinitely because KubeVirt cannot migrate the VM and +refuses to evict it. + +Setting `evictionStrategy: External` on GPU NodePools delegates eviction +handling to the Cluster API Provider for KubeVirt (CAPK), which gracefully +drains the guest node inside the hosted cluster before deleting the VM. +KubeVirt then recreates the VM on another available node. + +The supported values for `evictionStrategy` are: + +| Value | Behavior | +|-------|----------| +| *(not set)* | Uses the cluster-level KubeVirt default (`LiveMigrate` on HA clusters, `None` on single-node). Best for migratable VMs. | +| `LiveMigrate` | KubeVirt live-migrates the VM on node drain. If the VM is not migratable, the drain stalls. | +| `LiveMigrateIfPossible` | Live-migrates if possible, otherwise allows the VM to be killed without graceful guest drain. | +| `External` | CAPK drains the guest node, then deletes the VM. Recommended for non-migratable VMs (GPU, SR-IOV). | +| `None` | The VM is killed immediately on eviction with no migration or graceful drain. | + diff --git a/docs/content/reference/aggregated-docs.md b/docs/content/reference/aggregated-docs.md index 20af683cfd24..a62f1f8c4529 100644 --- a/docs/content/reference/aggregated-docs.md +++ b/docs/content/reference/aggregated-docs.md @@ -25727,6 +25727,7 @@ spec: hostDevices: - count: 2 deviceName: nvidia-a100 + evictionStrategy: External networkInterfaceMultiqueue: Enable rootVolume: persistent: @@ -25736,6 +25737,31 @@ spec: replicas: 3 ``` +## Eviction Strategy for GPU NodePools + +VMs with GPU passthrough devices are not live-migratable. By default, when +no `evictionStrategy` is set on a NodePool, the cluster-level KubeVirt +default applies. On multi-node HA clusters managed by HCO this is +`LiveMigrate`; on single-node OpenShift (SNO) it is `None`. With +`LiveMigrate`, draining an infrastructure node that hosts a non-migratable +VM will stall indefinitely because KubeVirt cannot migrate the VM and +refuses to evict it. + +Setting `evictionStrategy: External` on GPU NodePools delegates eviction +handling to the Cluster API Provider for KubeVirt (CAPK), which gracefully +drains the guest node inside the hosted cluster before deleting the VM. +KubeVirt then recreates the VM on another available node. + +The supported values for `evictionStrategy` are: + +| Value | Behavior | +|-------|----------| +| *(not set)* | Uses the cluster-level KubeVirt default (`LiveMigrate` on HA clusters, `None` on single-node). Best for migratable VMs. | +| `LiveMigrate` | KubeVirt live-migrates the VM on node drain. If the VM is not migratable, the drain stalls. | +| `LiveMigrateIfPossible` | Live-migrates if possible, otherwise allows the VM to be killed without graceful guest drain. | +| `External` | CAPK drains the guest node, then deletes the VM. Recommended for non-migratable VMs (GPU, SR-IOV). | +| `None` | The VM is killed immediately on eviction with no migration or graceful drain. | + --- @@ -49060,6 +49086,31 @@ string +###KubevirtEvictionStrategy { #hypershift.openshift.io/v1beta1.KubevirtEvictionStrategy } +

+(Appears on: +KubevirtNodePoolPlatform) +

+

+

KubevirtEvictionStrategy defines the eviction behavior for KubeVirt VMs.

+

+ + + + + + + + + + + + + + + + +
ValueDescription

"External"

"LiveMigrate"

"LiveMigrateIfPossible"

"None"

###KubevirtHostDevice { #hypershift.openshift.io/v1beta1.KubevirtHostDevice }

(Appears on: @@ -49301,6 +49352,34 @@ Selector which must match a node’s labels for the VM to be scheduled on th from the management cluster, to the nodepool nodes

+ + +evictionStrategy
+ + +KubevirtEvictionStrategy + + + + +(Optional) +

evictionStrategy defines the eviction behavior for KubeVirt VMs during +infrastructure node drain. If not set, the cluster-level default from the +KubeVirt configuration applies (typically LiveMigrate on HA clusters when +managed by HCO).

+ + + ###KubevirtPersistentVolume { #hypershift.openshift.io/v1beta1.KubevirtPersistentVolume } diff --git a/docs/content/reference/api.md b/docs/content/reference/api.md index 21327968cdaa..e8dc21cfe7ec 100644 --- a/docs/content/reference/api.md +++ b/docs/content/reference/api.md @@ -11918,6 +11918,31 @@ string +###KubevirtEvictionStrategy { #hypershift.openshift.io/v1beta1.KubevirtEvictionStrategy } +

+(Appears on: +KubevirtNodePoolPlatform) +

+

+

KubevirtEvictionStrategy defines the eviction behavior for KubeVirt VMs.

+

+ + + + + + + + + + + + + + + + +
ValueDescription

"External"

"LiveMigrate"

"LiveMigrateIfPossible"

"None"

###KubevirtHostDevice { #hypershift.openshift.io/v1beta1.KubevirtHostDevice }

(Appears on: @@ -12159,6 +12184,34 @@ Selector which must match a node’s labels for the VM to be scheduled on th from the management cluster, to the nodepool nodes

+ + +evictionStrategy
+ + +KubevirtEvictionStrategy + + + + +(Optional) +

evictionStrategy defines the eviction behavior for KubeVirt VMs during +infrastructure node drain. If not set, the cluster-level default from the +KubeVirt configuration applies (typically LiveMigrate on HA clusters when +managed by HCO).

+ + + ###KubevirtPersistentVolume { #hypershift.openshift.io/v1beta1.KubevirtPersistentVolume } diff --git a/hypershift-operator/controllers/nodepool/kubevirt/kubevirt.go b/hypershift-operator/controllers/nodepool/kubevirt/kubevirt.go index c5c087a6f3d9..89ad231af1c6 100644 --- a/hypershift-operator/controllers/nodepool/kubevirt/kubevirt.go +++ b/hypershift-operator/controllers/nodepool/kubevirt/kubevirt.go @@ -176,8 +176,7 @@ func virtualMachineTemplateBase(nodePool *hyperv1.NodePool, bootImage BootImage) Interfaces: virtualMachineInterfaces(kvPlatform), }, }, - EvictionStrategy: ptr.To(kubevirtv1.EvictionStrategyExternal), - Networks: virtualMachineNetworks(kvPlatform), + Networks: virtualMachineNetworks(kvPlatform), }, }, }, @@ -296,6 +295,11 @@ func virtualMachineTemplateBase(nodePool *hyperv1.NodePool, bootImage BootImage) template.Spec.Template.Spec.Domain.Devices.HostDevices = hostDevices } + if kvPlatform.EvictionStrategy != "" { + strategy := kubevirtv1.EvictionStrategy(string(kvPlatform.EvictionStrategy)) + template.Spec.Template.Spec.EvictionStrategy = &strategy + } + return template, nil } diff --git a/hypershift-operator/controllers/nodepool/kubevirt/kubevirt_test.go b/hypershift-operator/controllers/nodepool/kubevirt/kubevirt_test.go index 2084a6b65cc5..7fe449c7e6cd 100644 --- a/hypershift-operator/controllers/nodepool/kubevirt/kubevirt_test.go +++ b/hypershift-operator/controllers/nodepool/kubevirt/kubevirt_test.go @@ -537,6 +537,56 @@ func TestKubevirtMachineTemplate(t *testing.T) { }, }, }, + { + name: "Eviction strategy is set explicitly", + nodePool: &hyperv1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: poolName, + Namespace: namespace, + }, + Spec: hyperv1.NodePoolSpec{ + ClusterName: clusterName, + Replicas: nil, + Config: nil, + Management: hyperv1.NodePoolManagement{}, + AutoScaling: nil, + Platform: hyperv1.NodePoolPlatform{ + Type: hyperv1.KubevirtPlatform, + Kubevirt: generateKubevirtPlatform( + memoryNPOption("5Gi"), + coresNPOption(4), + imageNPOption("testimage"), + volumeNPOption("32Gi"), + evictionStrategyNPOption(hyperv1.EvictionStrategyExternal), + ), + }, + Release: hyperv1.Release{}, + }, + }, + hcluster: &hyperv1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-hostedcluster", + Namespace: "clusters", + }, + Spec: hyperv1.HostedClusterSpec{ + InfraID: "1234", + }, + }, + + expected: &capikubevirt.KubevirtMachineTemplateSpec{ + Template: capikubevirt.KubevirtMachineTemplateResource{ + Spec: capikubevirt.KubevirtMachineSpec{ + BootstrapCheckSpec: capikubevirt.VirtualMachineBootstrapCheckSpec{CheckStrategy: "none"}, + VirtualMachineTemplate: *generateNodeTemplate( + memoryTmpltOpt("5Gi"), + cpuTmpltOpt(4), + storageTmpltOpt("32Gi"), + evictionStrategyTmpltOpt(kubevirtv1.EvictionStrategyExternal), + ), + }, + }, + }, + }, { name: "Host Devices count has an invalid value", nodePool: &hyperv1.NodePool{ @@ -1290,6 +1340,12 @@ func hostDevicesOption(hostDevices []hyperv1.KubevirtHostDevice) nodePoolOption } } +func evictionStrategyNPOption(strategy hyperv1.KubevirtEvictionStrategy) nodePoolOption { + return func(kvNodePool *hyperv1.KubevirtNodePoolPlatform) { + kvNodePool.EvictionStrategy = strategy + } +} + func generateKubevirtPlatform(options ...nodePoolOption) *hyperv1.KubevirtNodePoolPlatform { exampleTemplate := &hyperv1.KubevirtNodePoolPlatform{} @@ -1383,6 +1439,12 @@ func addNetworkOpt(nw kubevirtv1.Network) nodeTemplateOption { } } +func evictionStrategyTmpltOpt(strategy kubevirtv1.EvictionStrategy) nodeTemplateOption { + return func(template *capikubevirt.VirtualMachineTemplateSpec) { + template.Spec.Template.Spec.EvictionStrategy = &strategy + } +} + func annotationsTmpltOpt(annotations map[string]string) nodeTemplateOption { return func(template *capikubevirt.VirtualMachineTemplateSpec) { template.Spec.Template.ObjectMeta.Annotations = annotations @@ -1454,8 +1516,6 @@ func generateNodeTemplate(options ...nodeTemplateOption) *capikubevirt.VirtualMa }, }, - EvictionStrategy: ptr.To(kubevirtv1.EvictionStrategyExternal), - Domain: kubevirtv1.DomainSpec{ Devices: kubevirtv1.Devices{ Disks: []kubevirtv1.Disk{ diff --git a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/kubevirt.go b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/kubevirt.go index cc83e14d45bc..6f6207934d3a 100644 --- a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/kubevirt.go +++ b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/kubevirt.go @@ -143,6 +143,18 @@ const ( MultiQueueDisable MultiQueueSetting = "Disable" ) +// KubevirtEvictionStrategy defines the eviction behavior for KubeVirt VMs. +// +// +kubebuilder:validation:Enum=LiveMigrate;LiveMigrateIfPossible;External;None +type KubevirtEvictionStrategy string + +const ( + EvictionStrategyLiveMigrate KubevirtEvictionStrategy = "LiveMigrate" + EvictionStrategyLiveMigrateIfPossible KubevirtEvictionStrategy = "LiveMigrateIfPossible" + EvictionStrategyExternal KubevirtEvictionStrategy = "External" + EvictionStrategyNone KubevirtEvictionStrategy = "None" +) + // KubevirtNodePoolPlatform specifies the configuration of a NodePool when operating // on KubeVirt platform. type KubevirtNodePoolPlatform struct { @@ -191,6 +203,24 @@ type KubevirtNodePoolPlatform struct { // +optional // +kubebuilder:validation:MaxItems=10 KubevirtHostDevices []KubevirtHostDevice `json:"hostDevices,omitempty"` + + // evictionStrategy defines the eviction behavior for KubeVirt VMs during + // infrastructure node drain. If not set, the cluster-level default from the + // KubeVirt configuration applies (typically LiveMigrate on HA clusters when + // managed by HCO). + // + // - LiveMigrate: KubeVirt live-migrates the VM to another node (zero guest + // disruption). If the VM is not migratable, the node drain stalls. + // - LiveMigrateIfPossible: live-migrates if possible, otherwise allows the + // VMI pod to be killed (no graceful guest drain). + // - External: delegates eviction handling to CAPK, which drains the guest + // node before deleting the VMI. Use this for non-migratable VMs (e.g., + // with GPU passthrough or SR-IOV) that need graceful guest node drain. + // - None: the VM is killed immediately on eviction with no migration or + // graceful guest drain. This is the HCO default on single-node OpenShift. + // + // +optional + EvictionStrategy KubevirtEvictionStrategy `json:"evictionStrategy,omitempty"` } // KubevirtNetwork specifies the configuration for a virtual machine