From c8f7d6d2273a8c35c7a47ed47adec38d5a570d0a Mon Sep 17 00:00:00 2001 From: svarnam Date: Thu, 11 Jun 2026 22:31:45 +0530 Subject: [PATCH] Add support for featureGates Signed-off-by: svarnam --- cmd/dra-example-kubeletplugin/main.go | 23 ++-- cmd/dra-example-kubeletplugin/state.go | 3 +- cmd/dra-example-webhook/main.go | 18 +-- .../dra-example-driver/templates/_helpers.tpl | 6 +- .../templates/controller-deployment.yaml | 4 +- .../templates/kubeletplugin.yaml | 10 +- .../templates/webhook-deployment.yaml | 10 +- .../dra-example-driver/values.schema.json | 10 +- .../helm/dra-example-driver/values.yaml | 20 +-- pkg/featuregates/featuregates.go | 93 ++++++++++++++ pkg/featuregates/featuregates_test.go | 117 ++++++++++++++++++ pkg/flags/logging.go | 19 +-- 12 files changed, 275 insertions(+), 58 deletions(-) create mode 100644 pkg/featuregates/featuregates.go create mode 100644 pkg/featuregates/featuregates_test.go diff --git a/cmd/dra-example-kubeletplugin/main.go b/cmd/dra-example-kubeletplugin/main.go index 07e683a1..181ab049 100644 --- a/cmd/dra-example-kubeletplugin/main.go +++ b/cmd/dra-example-kubeletplugin/main.go @@ -33,6 +33,7 @@ import ( "sigs.k8s.io/dra-example-driver/internal/profiles" "sigs.k8s.io/dra-example-driver/internal/profiles/gpu" + "sigs.k8s.io/dra-example-driver/pkg/featuregates" "sigs.k8s.io/dra-example-driver/pkg/flags" ) @@ -50,7 +51,6 @@ type Flags struct { kubeletRegistrarDirectoryPath string kubeletPluginsDirectoryPath string healthcheckPort int - profile string driverName string podUID string gpuPartitions int @@ -138,16 +138,9 @@ func newApp() *cli.App { Destination: &flags.healthcheckPort, EnvVars: []string{"HEALTHCHECK_PORT"}, }, - &cli.StringFlag{ - Name: "device-profile", - Usage: fmt.Sprintf("Name of the device profile. Valid values are %q.", validProfileNames), - Value: gpu.ProfileName, - Destination: &flags.profile, - EnvVars: []string{"DEVICE_PROFILE"}, - }, &cli.StringFlag{ Name: "driver-name", - Usage: "Name of the DRA driver. Its default is derived from the device profile.", + Usage: "Name of the DRA driver. Its default is derived from the active device profile.", Destination: &flags.driverName, EnvVars: []string{"DRIVER_NAME"}, }, @@ -200,13 +193,19 @@ func newApp() *cli.App { return fmt.Errorf("create client: %w", err) } + profile := featuregates.DeviceProfile() + if flags.driverName == "" { - flags.driverName = flags.profile + ".example.com" + flags.driverName = profile + ".example.com" } - newProfile, ok := validProfiles[flags.profile] + newProfile, ok := validProfiles[profile] if !ok { - return fmt.Errorf("invalid device profile %q, valid profiles are %q", flags.profile, validProfileNames) + var valid []string + for profileName := range validProfiles { + valid = append(valid, profileName) + } + return fmt.Errorf("invalid device profile %q, valid profiles are %q", profile, valid) } config := &Config{ diff --git a/cmd/dra-example-kubeletplugin/state.go b/cmd/dra-example-kubeletplugin/state.go index 81c08268..e46c7e0b 100644 --- a/cmd/dra-example-kubeletplugin/state.go +++ b/cmd/dra-example-kubeletplugin/state.go @@ -43,6 +43,7 @@ import ( checkpointinstall "sigs.k8s.io/dra-example-driver/internal/api/checkpoint/install" checkpointv1alpha1 "sigs.k8s.io/dra-example-driver/internal/api/checkpoint/v1" "sigs.k8s.io/dra-example-driver/internal/profiles" + "sigs.k8s.io/dra-example-driver/pkg/featuregates" ) type AllocatableDevices map[string]resourceapi.Device @@ -90,7 +91,7 @@ func NewDeviceState(config *Config) (*DeviceState, error) { return nil, fmt.Errorf("error enumerating all possible devices: %v", err) } - cdi, err := NewCDIHandler(config.flags.cdiRoot, config.flags.driverName, config.flags.profile) + cdi, err := NewCDIHandler(config.flags.cdiRoot, config.flags.driverName, featuregates.DeviceProfile()) if err != nil { return nil, fmt.Errorf("unable to create CDI handler: %v", err) } diff --git a/cmd/dra-example-webhook/main.go b/cmd/dra-example-webhook/main.go index 5ddb2b51..e72676bb 100644 --- a/cmd/dra-example-webhook/main.go +++ b/cmd/dra-example-webhook/main.go @@ -36,6 +36,7 @@ import ( "sigs.k8s.io/dra-example-driver/internal/profiles" "sigs.k8s.io/dra-example-driver/internal/profiles/gpu" + "sigs.k8s.io/dra-example-driver/pkg/featuregates" "sigs.k8s.io/dra-example-driver/pkg/flags" ) @@ -45,7 +46,6 @@ type Flags struct { certFile string keyFile string port int - profile string driverName string } @@ -85,16 +85,9 @@ func newApp() *cli.App { Value: 443, Destination: &flags.port, }, - &cli.StringFlag{ - Name: "device-profile", - Usage: fmt.Sprintf("Name of the device profile. Valid values are %q.", validProfiles), - Value: gpu.ProfileName, - Destination: &flags.profile, - EnvVars: []string{"DEVICE_PROFILE"}, - }, &cli.StringFlag{ Name: "driver-name", - Usage: "Name of the DRA driver. Its default is derived from the device profile.", + Usage: "Name of the DRA driver. Its default is derived from the active device profile.", Destination: &flags.driverName, EnvVars: []string{"DRIVER_NAME"}, }, @@ -114,17 +107,18 @@ func newApp() *cli.App { return flags.loggingConfig.Apply() }, Action: func(c *cli.Context) error { - configHandler, ok := validProfiles[flags.profile] + profile := featuregates.DeviceProfile() + configHandler, ok := validProfiles[profile] if !ok { var valid []string for profileName := range validProfiles { valid = append(valid, profileName) } - return fmt.Errorf("invalid device profile %q, valid profiles are %q", flags.profile, valid) + return fmt.Errorf("invalid device profile %q, valid profiles are %q", profile, valid) } if flags.driverName == "" { - flags.driverName = flags.profile + ".example.com" + flags.driverName = profile + ".example.com" } mux, err := newMux(configHandler, flags.driverName) diff --git a/deployments/helm/dra-example-driver/templates/_helpers.tpl b/deployments/helm/dra-example-driver/templates/_helpers.tpl index 6198ba65..0b28bdef 100644 --- a/deployments/helm/dra-example-driver/templates/_helpers.tpl +++ b/deployments/helm/dra-example-driver/templates/_helpers.tpl @@ -126,5 +126,9 @@ resource.k8s.io/v1beta1 The driver name. */}} {{- define "dra-example-driver.driverName" -}} -{{ default (print .Values.deviceProfile ".example.com") .Values.driverName }} +{{- if .Values.driverName -}} +{{- .Values.driverName -}} +{{- else -}} +gpu.example.com +{{- end -}} {{- end -}} diff --git a/deployments/helm/dra-example-driver/templates/controller-deployment.yaml b/deployments/helm/dra-example-driver/templates/controller-deployment.yaml index b4c9b3a9..e7bea922 100644 --- a/deployments/helm/dra-example-driver/templates/controller-deployment.yaml +++ b/deployments/helm/dra-example-driver/templates/controller-deployment.yaml @@ -31,7 +31,9 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} command: ["dra-example-controller"] args: - - --driver-name={{ include "dra-example-driver.driverName" . }} + {{- if .Values.driverName }} + - --driver-name={{ .Values.driverName }} + {{- end }} {{- range .Values.controller.plugins }} - --enable-plugin={{ . }} {{- end }} diff --git a/deployments/helm/dra-example-driver/templates/kubeletplugin.yaml b/deployments/helm/dra-example-driver/templates/kubeletplugin.yaml index 4a40b171..6e05816c 100644 --- a/deployments/helm/dra-example-driver/templates/kubeletplugin.yaml +++ b/deployments/helm/dra-example-driver/templates/kubeletplugin.yaml @@ -59,10 +59,14 @@ spec: periodSeconds: 10 {{- end }} env: + {{- if .Values.driverName }} - name: DRIVER_NAME - value: {{ include "dra-example-driver.driverName" . | quote }} - - name: DEVICE_PROFILE - value: {{ .Values.deviceProfile | quote }} + value: {{ .Values.driverName | quote }} + {{- end }} + {{- if .Values.featureGates }} + - name: FEATURE_GATES + value: "{{ range $key, $value := .Values.featureGates }}{{ $key }}={{ $value }},{{ end }}" + {{- end }} - name: CDI_ROOT value: /var/run/cdi - name: KUBELET_REGISTRAR_DIRECTORY_PATH diff --git a/deployments/helm/dra-example-driver/templates/webhook-deployment.yaml b/deployments/helm/dra-example-driver/templates/webhook-deployment.yaml index 920e879f..cb743618 100644 --- a/deployments/helm/dra-example-driver/templates/webhook-deployment.yaml +++ b/deployments/helm/dra-example-driver/templates/webhook-deployment.yaml @@ -43,8 +43,14 @@ spec: - --tls-cert-file=/cert/tls.crt - --tls-private-key-file=/cert/tls.key - --port={{ .Values.webhook.containerPort }} - - --device-profile={{ .Values.deviceProfile }} - - --driver-name={{ include "dra-example-driver.driverName" . }} + {{- if .Values.driverName }} + - --driver-name={{ .Values.driverName }} + {{- end }} + {{- if .Values.featureGates }} + env: + - name: FEATURE_GATES + value: "{{ range $key, $value := .Values.featureGates }}{{ $key }}={{ $value }},{{ end }}" + {{- end }} ports: - name: webhook containerPort: {{ .Values.webhook.containerPort }} diff --git a/deployments/helm/dra-example-driver/values.schema.json b/deployments/helm/dra-example-driver/values.schema.json index 3dff0a3a..09842ba5 100644 --- a/deployments/helm/dra-example-driver/values.schema.json +++ b/deployments/helm/dra-example-driver/values.schema.json @@ -1,11 +1,11 @@ { "type": "object", "properties": { - "deviceProfile": { - "type": "string", - "enum": [ - "gpu" - ] + "featureGates": { + "type": "object", + "additionalProperties": { + "type": "boolean" + } } } } diff --git a/deployments/helm/dra-example-driver/values.yaml b/deployments/helm/dra-example-driver/values.yaml index e748842e..0ed701fd 100644 --- a/deployments/helm/dra-example-driver/values.yaml +++ b/deployments/helm/dra-example-driver/values.yaml @@ -9,13 +9,19 @@ selectorLabelsOverride: {} allowDefaultNamespace: false -# deviceProfile describes the overall shape of the devices managed by the -# driver. Available profiles are: -# - "gpu": Node-local devices configurable through opaque config -deviceProfile: "gpu" +# featureGates enables alpha/experimental driver behavior. Each entry is +# passed to components as FEATURE_GATES (comma-separated key=value pairs). +# +# VFIOGPUEnabled is recognized but not yet supported by this chart. Enabling it +# fails at install time until the vfio-gpu profile is added. +# +# Without feature gates, the default gpu profile is used (gpu.example.com). +featureGates: {} -# driverName uniquely identifies the driver within the cluster. When empty, its -# value is derived from the deviceProfile. +# driverName uniquely identifies the driver within the cluster. Leave empty for +# the driver components to derive it from the active device profile. The chart +# derives the same default for Kubernetes resources that need the name at render +# time. vfio-gpu.* driver names are not supported yet. driverName: "" # deviceClass configures the DeviceClass the driver creates for its devices. @@ -50,7 +56,7 @@ serviceAccount: kubeletPlugin: # numDevices describes how many GPUs to advertise on each node when the "gpu" - # deviceProfile is used. Not relevant for other profiles. + # profile is used. Not relevant for other profiles. numDevices: 8 # gpuPartitions sets the number of partitions per GPU. When set to a value # greater than 0, GPUs are exposed with shared counters allowing flexible diff --git a/pkg/featuregates/featuregates.go b/pkg/featuregates/featuregates.go new file mode 100644 index 00000000..4580e7a8 --- /dev/null +++ b/pkg/featuregates/featuregates.go @@ -0,0 +1,93 @@ +/* + * Copyright The Kubernetes Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package featuregates + +import ( + "sync" + + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/version" + "k8s.io/component-base/featuregate" + logsapi "k8s.io/component-base/logs/api/v1" +) + +// featureGateEmulationVersion must use Kubernetes-style versions (major.minor +// matching the vendored k8s.io/* release), not driver SemVer. Driver-local +// gates use 0.x Version fields and remain visible because 0.x < 1.y. +var featureGateEmulationVersion = version.MajorMinor(1, 36) + +const ( + // VFIOGPUEnabled enables PCI passthrough device discovery and preparation via + // the vfio-gpu profile. When enabled, the driver selects that profile + // instead of the default gpu profile. + VFIOGPUEnabled featuregate.Feature = "VFIOGPUEnabled" +) + +var defaultFeatureGates = map[featuregate.Feature]featuregate.VersionedSpecs{ + VFIOGPUEnabled: { + { + Default: false, + PreRelease: featuregate.Alpha, + Version: version.MajorMinor(0, 1), + }, + }, +} + +var ( + featureGatesOnce sync.Once + featureGates featuregate.MutableVersionedFeatureGate +) + +// FeatureGates returns the process-wide feature gate set, including logging +// gates from component-base and driver-specific gates. +func FeatureGates() featuregate.MutableVersionedFeatureGate { + if featureGates == nil { + featureGatesOnce.Do(func() { + featureGates = newFeatureGates(featureGateEmulationVersion) + }) + } + return featureGates +} + +func newFeatureGates(v *version.Version) featuregate.MutableVersionedFeatureGate { + fg := featuregate.NewVersionedFeatureGate(v) + utilruntime.Must(logsapi.AddFeatureGates(fg)) + utilruntime.Must(fg.AddVersioned(defaultFeatureGates)) + utilruntime.Must(fg.SetFromMap(map[string]bool{ + string(logsapi.ContextualLogging): true, + })) + return fg +} + +// Enabled reports whether feature is enabled on the process-wide gate set. +func Enabled(feature featuregate.Feature) bool { + return FeatureGates().Enabled(feature) +} + +const ( + profileGPU = "gpu" + profileVFIOGPU = "vfio-gpu" +) + +// DeviceProfile returns the active device profile. VFIOGPUEnabled selects vfio-gpu; +// otherwise the default gpu profile is used. +func DeviceProfile() string { + if Enabled(VFIOGPUEnabled) { + return profileVFIOGPU + } + return profileGPU +} diff --git a/pkg/featuregates/featuregates_test.go b/pkg/featuregates/featuregates_test.go new file mode 100644 index 00000000..0856d267 --- /dev/null +++ b/pkg/featuregates/featuregates_test.go @@ -0,0 +1,117 @@ +/* + * Copyright The Kubernetes Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package featuregates + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/assert" + + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/version" + "k8s.io/component-base/featuregate" +) + +// test-only feature gates used to exercise the feature gate machinery without +// coupling tests to production gate defaults. +const ( + testAlpha featuregate.Feature = "TestAlpha" + testBeta featuregate.Feature = "TestBeta" + testGA featuregate.Feature = "TestGA" +) + +var testFeatureGates = map[featuregate.Feature]featuregate.VersionedSpecs{ + testAlpha: { + { + Default: false, + PreRelease: featuregate.Alpha, + Version: version.MajorMinor(0, 1), + }, + }, + testBeta: { + { + Default: true, + PreRelease: featuregate.Beta, + Version: version.MajorMinor(0, 1), + }, + }, + testGA: { + { + Default: true, + PreRelease: featuregate.GA, + Version: version.MajorMinor(0, 2), + }, + }, +} + +func newTestFeatureGates(v *version.Version) featuregate.MutableVersionedFeatureGate { + fg := featuregate.NewVersionedFeatureGate(v) + utilruntime.Must(fg.AddVersioned(testFeatureGates)) + return fg +} + +func withFeatureGates(t *testing.T, enabled map[string]bool) { + t.Helper() + withCustomFeatureGates(t, newFeatureGates(version.MajorMinor(1, 36)), enabled) +} + +func withTestFeatureGates(t *testing.T, enabled map[string]bool) { + t.Helper() + withCustomFeatureGates(t, newTestFeatureGates(version.MajorMinor(1, 36)), enabled) +} + +func withCustomFeatureGates(t *testing.T, fg featuregate.MutableVersionedFeatureGate, enabled map[string]bool) { + t.Helper() + if len(enabled) > 0 { + utilruntime.Must(fg.SetFromMap(enabled)) + } + featureGates = fg + t.Cleanup(func() { + featureGates = nil + featureGatesOnce = sync.Once{} + }) +} + +func TestTestFeatureGateDefaults(t *testing.T) { + fg := newTestFeatureGates(version.MajorMinor(1, 36)) + + assert.False(t, fg.Enabled(testAlpha)) + assert.True(t, fg.Enabled(testBeta)) + assert.True(t, fg.Enabled(testGA)) +} + +func TestEnabledWithTestFeatures(t *testing.T) { + withTestFeatureGates(t, map[string]bool{ + string(testAlpha): true, + string(testBeta): false, + }) + + assert.True(t, Enabled(testAlpha)) + assert.False(t, Enabled(testBeta)) + assert.True(t, Enabled(testGA)) +} + +func TestDeviceProfile_DefaultGPU(t *testing.T) { + withFeatureGates(t, nil) + assert.Equal(t, "gpu", DeviceProfile()) +} + +func TestDeviceProfile_VFIOGPUEnabledGate(t *testing.T) { + withFeatureGates(t, map[string]bool{string(VFIOGPUEnabled): true}) + assert.Equal(t, "vfio-gpu", DeviceProfile()) +} diff --git a/pkg/flags/logging.go b/pkg/flags/logging.go index 15422069..8192a6db 100644 --- a/pkg/flags/logging.go +++ b/pkg/flags/logging.go @@ -22,10 +22,11 @@ import ( "github.com/spf13/pflag" "github.com/urfave/cli/v2" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/component-base/featuregate" logsapi "k8s.io/component-base/logs/api/v1" + "sigs.k8s.io/dra-example-driver/pkg/featuregates" + _ "k8s.io/component-base/logs/json/register" // for JSON log output support ) @@ -35,15 +36,10 @@ type LoggingConfig struct { } func NewLoggingConfig() *LoggingConfig { - fg := featuregate.NewFeatureGate() - var _ pflag.Value = fg // compile-time check for the type conversion below - l := &LoggingConfig{ - featureGate: fg, + return &LoggingConfig{ + featureGate: featuregates.FeatureGates(), config: logsapi.NewLoggingConfiguration(), } - utilruntime.Must(logsapi.AddFeatureGates(l.featureGate)) - utilruntime.Must(l.featureGate.SetFromMap(map[string]bool{string(logsapi.ContextualLogging): true})) - return l } // Apply should be called in a cli.App.Before directly after parsing command @@ -61,12 +57,7 @@ func (l *LoggingConfig) Flags() []cli.Flag { // with "logging" as category. In practice, the logging code is the // only code which uses the flag, therefore that seems like a good // place to report it. - fs.AddFlag(&pflag.Flag{ - Name: "feature-gates", - Usage: "A set of key=value pairs that describe feature gates for alpha/experimental features. " + - "Options are:\n " + strings.Join(l.featureGate.KnownFeatures(), "\n "), - Value: l.featureGate.(pflag.Value), //nolint:forcetypeassert // No need for type check: l.featureGate is a *featuregate.featureGate, which implements pflag.Value. - }) + l.featureGate.AddFlag(&fs) var flags []cli.Flag fs.VisitAll(func(flag *pflag.Flag) {